[{"_1":2,"_176":-5,"_177":-5},"loaderData",{"_3":4,"_93":94},"root",{"_5":6,"_36":37,"_66":67},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":-5,"_32":33,"_34":20,"_35":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesChatPanelState","notificationCount",0,"flagDenols","isTeamOrgMember","ENV",{"_38":39,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":57,"_58":59,"_60":61,"_62":63,"_64":65},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","c968fae119093346ef93ef058bc3be26c9b1ae2f","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","API_URL","https://api.val.town","clerkState",{"_68":69},"__internal_clerk_state",{"_70":71,"_72":47,"_73":74,"_75":74,"_76":20,"_77":74,"_78":74,"_79":74,"_80":74,"_81":82,"_91":20,"_92":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_83":20,"_19":20,"_84":74,"_85":86,"_87":74,"_88":47,"_89":20,"_90":74},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_95":96,"_97":98,"_99":100,"_101":102,"_154":155,"_173":174,"_175":-5},"handle","tylermyeo","name","index-builder-v1","path","main.ts","project",{"_103":104,"_97":98,"_105":106,"_107":-5,"_108":20,"_109":110,"_111":-5,"_112":-5,"_12":113,"_114":-5,"_115":20,"_116":-5,"_117":-5,"_118":33,"_14":119,"_120":-5,"_121":-5,"_122":-5,"_123":124,"_125":124,"_126":127,"_128":127,"_129":127,"_130":124,"_131":124,"_23":132,"_135":136,"_140":141,"_153":20},"id","ce6b98e6-f6a3-11f0-b62d-42dde27851f2","createdAt",["D",1768984385171],"deletedAt","legacyEnvVars","privacy","public","description","imageUrl","019bc224-5e41-7430-8a11-f9ef2a021060","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","019bc224-5e42-70fc-944f-5712b27516c9","tursoName","tursoHostname","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_95":96,"_133":134},"profileImageUrl","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18zOElZbGFhMzZWVEtwQUMwQXVVamtoWVp5V24ifQ","org",{"_95":96,"_137":134,"_12":113,"_138":139},"avatar","tier","free","branch",{"_142":143,"_97":45,"_103":144,"_145":104,"_146":-5,"_147":-5,"_148":-5,"_149":-5,"_150":-5,"_151":98,"_152":96},"version",8,"ce6bd63a-f6a3-11f0-b62d-42dde27851f2","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_103":156,"_97":100,"_157":158,"_159":-5,"_160":144,"_161":143,"_162":163,"_105":164,"_142":165,"_166":167,"_168":169,"_170":20,"_171":-5,"_99":172},"d8b60360-f6a6-11f0-8f22-42dde27851f2","fileId","ce6f8a1e-f6a3-11f0-b62d-42dde27851f2","parentId","branchId","branchVersion","latest",true,["D",1768985690918],7,"fileType","http","content","/**\n * Val.town Index Builder - Populates SQLite from Notion\n *\n * This script runs on Val.town after the local backfill completes.\n * It fetches all pages from Notion and populates the SQLite index.\n *\n * NO duplicate detection or tagging - just reads from Notion and writes to SQLite.\n * This makes it fast enough to complete within Val.town's timeout limits.\n *\n * Can be run as:\n * - Scheduled Val (runs every 15 minutes until complete)\n * - HTTP Val (click \"Run\" manually multiple times)\n */\n\nimport { sqlite } from \"https://esm.town/v/std/sqlite\";\n\n// Configuration\nconst BATCH_SIZE = 100; // Notion caps at 100 per call; request exactly that\nconst NOTION_API_BASE = \"https://api.notion.com/v1\";\nconst NOTION_VERSION = \"2022-06-28\";\nconst MAX_RUNTIME_MS = 55_000; // stop near Val.town timeout to stay safe\n\ninterface NotionPage {\n id: string;\n properties: {\n Name?: {\n title: Array<{ plain_text: string }>;\n };\n };\n}\n\ninterface NotionQueryResponse {\n results: NotionPage[];\n next_cursor: string | null;\n has_more: boolean;\n}\n\ninterface Progress {\n last_cursor: string | null;\n total_indexed: number;\n completed: boolean;\n}\n\n/**\n * Normalize name for comparison (trim and lowercase)\n */\nfunction normalizeName(name: string): string {\n return name.trim().toLowerCase();\n}\n\n/**\n * Get Notion API headers\n */\nfunction getNotionHeaders(): HeadersInit {\n const token = Deno.env.get(\"NOTION_TOKEN\");\n if (!token) {\n throw new Error(\"NOTION_TOKEN environment variable is not set\");\n }\n\n return {\n \"Authorization\": `Bearer ${token}`,\n \"Notion-Version\": NOTION_VERSION,\n \"Content-Type\": \"application/json\",\n };\n}\n\n/**\n * Retry a function with exponential backoff\n */\nasync function retryWithBackoff(\n fn: () => Promise,\n maxRetries: number = 3,\n initialDelay: number = 1000,\n): Promise {\n let lastError: any;\n\n for (let i = 0; i < maxRetries; i++) {\n try {\n return await fn();\n } catch (error: any) {\n lastError = error;\n\n // Check if it's a rate limit error\n if (\n error.message?.includes(\"429\") ||\n error.message?.includes(\"Rate limit\")\n ) {\n const delay = initialDelay * Math.pow(2, i);\n console.log(\n `Rate limit hit, retrying in ${delay}ms... (attempt ${\n i + 1\n }/${maxRetries})`,\n );\n await new Promise((resolve) => setTimeout(resolve, delay));\n continue;\n }\n\n // If it's not a rate limit error, throw immediately\n throw error;\n }\n }\n\n throw lastError;\n}\n\n/**\n * Initialize database tables\n */\nasync function initializeTables(): Promise {\n // Create name_index table (same schema as webhook)\n await sqlite.execute(`\n CREATE TABLE IF NOT EXISTS name_index (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n name TEXT NOT NULL COLLATE NOCASE,\n notion_page_id TEXT NOT NULL UNIQUE,\n created_at DATETIME DEFAULT CURRENT_TIMESTAMP\n )\n `);\n\n // Create index for fast name lookups\n await sqlite.execute(`\n CREATE INDEX IF NOT EXISTS idx_name ON name_index(name)\n `);\n\n // Create progress tracking table\n await sqlite.execute(`\n CREATE TABLE IF NOT EXISTS index_builder_progress (\n id INTEGER PRIMARY KEY CHECK (id = 1),\n last_cursor TEXT,\n total_indexed INTEGER DEFAULT 0,\n started_at DATETIME DEFAULT CURRENT_TIMESTAMP,\n last_run_at DATETIME,\n completed BOOLEAN DEFAULT FALSE\n )\n `);\n}\n\n/**\n * Get current progress from database\n */\nasync function getProgress(): Promise {\n const result = await retryWithBackoff(() =>\n sqlite.execute({\n sql:\n \"SELECT last_cursor, total_indexed, completed FROM index_builder_progress WHERE id = 1\",\n args: [],\n })\n );\n\n if (result.rows.length === 0) {\n // First run, initialize progress\n await retryWithBackoff(() =>\n sqlite.execute({\n sql:\n \"INSERT INTO index_builder_progress (id, last_cursor, total_indexed, completed) VALUES (1, NULL, 0, FALSE)\",\n args: [],\n })\n );\n return {\n last_cursor: null,\n total_indexed: 0,\n completed: false,\n };\n }\n\n const row = result.rows[0];\n return {\n last_cursor: row[0] as string | null,\n total_indexed: row[1] as number,\n completed: row[2] as boolean,\n };\n}\n\n/**\n * Update progress in database\n */\nasync function updateProgress(data: {\n last_cursor: string | null;\n total_indexed: number;\n completed: boolean;\n}): Promise {\n await retryWithBackoff(() =>\n sqlite.execute({\n sql: `\n UPDATE index_builder_progress \n SET last_cursor = ?, \n total_indexed = ?, \n last_run_at = datetime('now'), \n completed = ?\n WHERE id = 1\n `,\n args: [\n data.last_cursor,\n data.total_indexed,\n data.completed ? 1 : 0,\n ],\n })\n );\n}\n\n/**\n * Query Notion database with pagination\n */\nasync function queryNotionDatabase(\n databaseId: string,\n startCursor: string | null,\n pageSize: number,\n): Promise {\n const body: any = {\n page_size: pageSize,\n };\n\n if (startCursor) {\n body.start_cursor = startCursor;\n }\n\n const response = await fetch(\n `${NOTION_API_BASE}/databases/${databaseId}/query`,\n {\n method: \"POST\",\n headers: getNotionHeaders(),\n body: JSON.stringify(body),\n },\n );\n\n if (!response.ok) {\n const errorText = await response.text();\n throw new Error(\n `Failed to query Notion database: ${response.status} ${errorText}`,\n );\n }\n\n return await response.json();\n}\n\n/**\n * Insert a page into the index\n */\nasync function insertIntoIndex(name: string, pageId: string): Promise {\n const normalizedName = normalizeName(name);\n\n return retryWithBackoff(async () => {\n try {\n await sqlite.execute({\n sql: \"INSERT INTO name_index (name, notion_page_id) VALUES (?, ?)\",\n args: [normalizedName, pageId],\n });\n return true; // Successfully inserted\n } catch (error: any) {\n // If this specific page is already indexed, skip it\n if (error.message?.includes(\"UNIQUE constraint\")) {\n return false; // Already exists, skip\n } else {\n throw error;\n }\n }\n });\n}\n\n/**\n * Count existing rows in the name_index table\n */\nasync function getExistingIndexCount(): Promise {\n const result = await retryWithBackoff(() =>\n sqlite.execute({ sql: \"SELECT COUNT(*) FROM name_index\", args: [] })\n );\n return Number(result.rows[0][0] || 0);\n}\n\n/**\n * Process a batch of pages: index them in SQLite\n */\nasync function processBatch(pages: NotionPage[]): Promise {\n let indexed = 0;\n\n for (const page of pages) {\n const name = page.properties.Name?.title?.[0]?.plain_text;\n if (!name) {\n console.log(` Skipping page ${page.id} - no Name property`);\n continue;\n }\n\n const wasInserted = await insertIntoIndex(name, page.id);\n if (wasInserted) {\n indexed++;\n }\n }\n\n return indexed;\n}\n\n/**\n * Main scheduled/HTTP function\n */\nexport default async function indexBuilder(): Promise {\n const startTime = Date.now();\n const runId = new Date().toISOString();\n const responseHeaders = { \"Cache-Control\": \"no-store\" };\n console.log(\"=== Index Builder Run Started ===\");\n console.log(`Time: ${runId}`);\n console.log(\n `Batch size (requested): ${BATCH_SIZE} (Notion returns up to 100)`,\n );\n\n try {\n // 1. Initialize tables\n await initializeTables();\n\n // 2. Check progress and sync with any pre-existing index rows\n const progress = await getProgress();\n if (!progress.completed && progress.total_indexed === 0) {\n const existingCount = await getExistingIndexCount();\n if (existingCount > 0) {\n await updateProgress({\n last_cursor: progress.last_cursor,\n total_indexed: existingCount,\n completed: progress.completed,\n });\n progress.total_indexed = existingCount;\n }\n }\n\n // 3. Check if already completed\n if (progress.completed) {\n console.log(\"✅ Index building already completed!\");\n console.log(`Total indexed: ${progress.total_indexed}`);\n return new Response(\n `Index building already completed (run ${runId}). Total indexed: ${progress.total_indexed}`,\n { status: 200, headers: responseHeaders },\n );\n }\n\n console.log(`Previous progress: ${progress.total_indexed} pages indexed`);\n if (progress.last_cursor) {\n console.log(\n `Resuming from cursor: ${progress.last_cursor.substring(0, 20)}...`,\n );\n }\n\n // 4. Get database ID\n const databaseId = Deno.env.get(\"NOTION_DATABASE_ID\");\n if (!databaseId) {\n throw new Error(\"NOTION_DATABASE_ID environment variable is not set\");\n }\n\n // 5. Loop multiple batches until near timeout or no more pages\n let cursor = progress.last_cursor;\n let totalIndexed = progress.total_indexed;\n let completed = false;\n let loopCount = 0;\n\n while (Date.now() - startTime < MAX_RUNTIME_MS) {\n loopCount++;\n console.log(\n `\\n[Batch ${loopCount}] Fetching up to ${BATCH_SIZE} pages...`,\n );\n\n const response = await queryNotionDatabase(\n databaseId,\n cursor,\n BATCH_SIZE,\n );\n const pages = response.results;\n cursor = response.next_cursor;\n completed = !response.has_more;\n\n console.log(`[Batch ${loopCount}] Fetched ${pages.length} pages`);\n\n if (pages.length === 0) {\n console.log(\n `[Batch ${loopCount}] No pages to process, marking as complete`,\n );\n await updateProgress({\n last_cursor: null,\n total_indexed: totalIndexed,\n completed: true,\n });\n console.log(\"🎉 INDEX BUILDING COMPLETE!\");\n return new Response(\n `Index building complete (run ${runId}, empty batch). Total indexed: ${totalIndexed}`,\n { status: 200, headers: responseHeaders },\n );\n }\n\n console.log(`[Batch ${loopCount}] Indexing...`);\n const indexed = await processBatch(pages);\n totalIndexed += indexed;\n\n await updateProgress({\n last_cursor: cursor,\n total_indexed: totalIndexed,\n completed,\n });\n\n const duration = ((Date.now() - startTime) / 1000).toFixed(2);\n console.log(`[Batch ${loopCount}] Processed: ${pages.length} pages`);\n console.log(\n `[Batch ${loopCount}] Indexed: ${indexed} (${\n pages.length - indexed\n } already existed)`,\n );\n console.log(`[Batch ${loopCount}] Total indexed so far: ${totalIndexed}`);\n console.log(`[Batch ${loopCount}] Elapsed: ${duration}s`);\n\n if (completed) {\n console.log(\"\\n🎉 INDEX BUILDING COMPLETE! All pages indexed.\");\n console.log(\"You can now disable this Val if it's scheduled.\");\n return new Response(\n `Index building complete (run ${runId}). Total indexed: ${totalIndexed}`,\n { status: 200, headers: responseHeaders },\n );\n }\n\n // If we're close to the runtime budget, stop to avoid timeout\n if (Date.now() - startTime > MAX_RUNTIME_MS * 0.9) {\n console.log(\n \"\\n⏳ Approaching runtime limit, pausing until next run...\",\n );\n break;\n }\n }\n\n return new Response(\n `Index builder partial (run ${runId}). Total indexed so far: ${totalIndexed}`,\n { status: 200, headers: responseHeaders },\n );\n } catch (error: any) {\n console.error(\"\\n❌ Error during index building:\");\n console.error(error.message);\n console.error(error.stack);\n\n // Always return a Response for Val.town HTTP Vals\n return new Response(`Error during index building: ${error.message}`, {\n status: 200, // return 200 to avoid Notion/automation disabling on error\n headers: responseHeaders,\n });\n }\n}","deleted","label",[100],"permalink","https://www.val.town/x/tylermyeo/index-builder-v1/code/main.ts","extension","actionData","errors"]