[{"_1":2,"_170":-5,"_171":-5},"loaderData",{"_3":4,"_91":92},"root",{"_5":6,"_34":35,"_64":65},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":32,"_33":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","notificationCount",0,"flagDenols","ENV",{"_36":37,"_38":39,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":57,"_58":59,"_60":61,"_62":63},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","0e83a37603a7996d918363a2f608889037f26b4b","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","API_URL","https://api.val.town","clerkState",{"_66":67},"__internal_clerk_state",{"_68":69,"_70":45,"_71":72,"_73":72,"_74":20,"_75":72,"_76":72,"_77":72,"_78":72,"_79":80,"_89":20,"_90":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_81":20,"_19":20,"_82":72,"_83":84,"_85":72,"_86":45,"_87":20,"_88":72},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_93":94,"_95":96,"_97":98,"_99":100,"_148":149,"_167":168,"_169":-5},"handle","yawnxyz","name","groq-docs","path","main.tsx","project",{"_101":102,"_95":96,"_103":104,"_105":-5,"_106":107,"_108":109,"_110":-5,"_111":-5,"_12":112,"_113":114,"_115":107,"_116":-5,"_117":32,"_14":118,"_119":120,"_121":120,"_122":123,"_124":123,"_125":123,"_126":120,"_127":120,"_23":128,"_131":132,"_134":135,"_147":20},"id","632cc576-2793-11f0-9d52-569c3dd06744","createdAt",["D",1746217442601],"deletedAt","legacyEnvVars",true,"privacy","public","description","imageUrl","a8270e74-6479-475a-8025-9b69544ee0e7","pinnedFileId","632eed24-2793-11f0-9d52-569c3dd06744","pinPreview","screenshotVersion","likeCount","8319defa-6e1a-46d4-ba66-2114ffd3360b","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_93":94,"_129":130},"profileImageUrl","https://images.clerk.dev/oauth_github/img_2NnaHhpxNuH1xWRIRjQNoo16TVc.jpeg","org",{"_93":94,"_133":130},"avatar","branch",{"_136":137,"_95":43,"_101":138,"_139":102,"_140":-5,"_141":-5,"_142":-5,"_143":-5,"_144":-5,"_145":96,"_146":94},"version",105,"632d0edc-2793-11f0-9d52-569c3dd06744","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_101":150,"_95":98,"_151":114,"_152":-5,"_153":138,"_154":155,"_156":107,"_103":157,"_136":158,"_159":160,"_161":162,"_163":20,"_164":165,"_97":166},"807b7306-b9ca-11f0-83d0-0224a6c84d84","fileId","parentId","branchId","branchVersion",91,"latest",["D",1762293983847],15,"fileType","http","content","import { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport \"jsr:@std/dotenv/load\";\nimport {\n IS_VALTOWN,\n getFromCache,\n setCache,\n clearCache,\n getCacheStats,\n getTextFromUrl,\n parseFrontmatter,\n addUrlSourceToFrontmatter,\n calculateTokenCount,\n calculateContentHash,\n} from \"./utils.ts\";\nimport { generatePageMetadata } from \"./groq.ts\";\nimport { searchPages, getActiveEmbeddingsFunction, type Page } from \"./search/index.ts\";\nimport { answerQuestion, answerStrategy } from \"./answer/index.ts\";\nimport { urls } from \"./urls.ts\";\n\nconst app = new Hono();\n\n\n\n\n\n\nconst getPageContent = async () => {\n const promises = urls.map(async url => {\n const text = await getTextFromUrl(url);\n return addUrlSourceToFrontmatter(text, url);\n });\n const texts = await Promise.all(promises);\n return texts.join(\"\\n\\n\");\n};\n\nconst getPageData = async () => {\n const pageData = await Promise.all(\n urls.map(async (url) => {\n // Check if cached\n const cached = await getFromCache(url);\n let text, processedText, charCount, frontmatter, wasCached;\n \n let metadata = null;\n \n if (cached) {\n // Use cached data\n text = cached.content;\n processedText = cached.content;\n charCount = cached.charCount;\n frontmatter = cached.frontmatter;\n metadata = cached.metadata;\n wasCached = true;\n } else {\n // Fetch fresh\n text = await getTextFromUrl(url);\n processedText = addUrlSourceToFrontmatter(text, url);\n charCount = processedText.length;\n frontmatter = parseFrontmatter(processedText);\n wasCached = false;\n }\n \n // Extract urlSource and rename it to url, then flatten remaining frontmatter properties\n const urlSource = (frontmatter as any).urlSource;\n const restFrontmatter = { ...frontmatter };\n delete (restFrontmatter as any).urlSource;\n \n return {\n url: urlSource || url, // Use urlSource from frontmatter, fallback to url\n content: processedText,\n charCount,\n cached: wasCached,\n ...restFrontmatter, // Flatten frontmatter properties\n ...(metadata || {}), // Include AI-generated metadata if available\n };\n })\n );\n \n return {\n pages: pageData,\n totalPages: pageData.length,\n totalChars: pageData.reduce((sum, { charCount }) => sum + charCount, 0),\n cachedPages: pageData.filter(({ cached }) => cached).length,\n };\n};\n\napp.get(\"/\", async (c) => {\n const content = await getPageContent();\n return c.text(content);\n});\n\napp.get(\"/data\", async (c) => {\n const data = await getPageData();\n \n // Extract page paths/names (same logic as /list endpoint)\n const pagePaths = urls.map(url => {\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n if (match) {\n return match[1];\n }\n if (url.endsWith(\"/docs.md\")) {\n return \"docs\";\n }\n return null;\n }).filter(path => path !== null);\n \n // Return metadata first, then the large data object\n return c.json({\n metadata: {\n urls: pagePaths,\n },\n ...data,\n });\n});\n\napp.get(\"/list\", async (c) => {\n const paths = urls.map(url => {\n // Extract the path part after /docs/ and remove .md extension\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n if (match) {\n return match[1];\n }\n // Handle the case where URL is just /docs.md - return \"docs\"\n if (url.endsWith(\"/docs.md\")) {\n return \"docs\";\n }\n return null;\n }).filter(path => path !== null);\n \n return c.json(paths);\n});\n\n// Helper function to load data from latest.json (for local development)\nasync function loadLatestJsonForSearch(): Promise | null> {\n if (IS_VALTOWN) {\n return null; // Use SQLite when in Val Town\n }\n \n try {\n const latestData = await Deno.readTextFile(\"./data/latest.json\");\n \n // Use fast-json for faster parsing (only extracts what we need)\n const { parseLatestJson } = await import(\"./utils/json-parser.ts\");\n const data = await parseLatestJson(latestData, { enableTiming: false });\n \n const cacheMap: Record = {};\n \n if (data.results && Array.isArray(data.results)) {\n for (const result of data.results) {\n if (result.url) { // fast-json already filters skipped/error entries\n cacheMap[result.url] = {\n title: result.title,\n embeddings: result.embeddings || null,\n metadata: result.metadata || null,\n };\n }\n }\n }\n \n return cacheMap;\n } catch {\n return null;\n }\n}\n\n// Helper function to get all pages for searching\n// IMPORTANT: This ONLY uses cached data (JSON locally, SQLite on Val Town)\n// It NEVER fetches live data - that's what recalculate is for!\n// Run `deno task recalculate` to populate the cache before searching.\nexport const getAllPagesForSearch = async (enableTiming: boolean = false): Promise => {\n const timings: Record = {};\n const startTime = performance.now();\n \n // Load JSON cache when running locally (Val Town uses SQLite instead)\n const jsonCacheStart = performance.now();\n const jsonCache = await loadLatestJsonForSearch();\n if (enableTiming) {\n timings.jsonCacheLoad = performance.now() - jsonCacheStart;\n if (IS_VALTOWN) {\n // On Val Town, JSON cache is skipped (uses SQLite instead)\n timings.jsonCacheLoad = 0;\n }\n }\n \n const pageDataStart = performance.now();\n const uncachedUrls: string[] = [];\n \n const pageData: (Page | null)[] = await Promise.all(\n urls.map(async (url): Promise => {\n // Extract path from URL\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n const path = match ? match[1] : (url.endsWith(\"/docs.md\") ? \"docs\" : null);\n \n if (!path) {\n return null;\n }\n \n // Try to get from SQLite cache first (Val Town)\n const cached = await getFromCache(url);\n \n if (cached) {\n const frontmatter = cached.frontmatter;\n // On Val Town, SQLite cache already has everything we need\n // Content is available, embeddings are available - no need to fetch!\n return {\n path,\n url: cached.frontmatter?.urlSource || url,\n content: cached.content, // Already cached in SQLite, no fetch needed!\n title: frontmatter?.title || path,\n embeddings: cached.embeddings, // Already cached in SQLite\n metadata: cached.metadata,\n };\n }\n \n // If not in SQLite, try JSON cache (when running locally)\n if (jsonCache && jsonCache[url]) {\n const jsonData = jsonCache[url];\n // JSON cache has embeddings, title, metadata from recalculate\n // We skip fetching content to speed up search - content is only for snippets\n // NO NETWORK CALLS - everything is from the JSON cache!\n if (jsonData.embeddings && Array.isArray(jsonData.embeddings) && jsonData.embeddings.length > 0) {\n // Use cached data - NO network fetch! This is why it's fast (was 6.5s, now 0.16ms)\n return {\n path,\n url: url,\n content: '', // Skipped - only needed for snippets, can fetch lazily if needed\n title: jsonData.title || path,\n embeddings: jsonData.embeddings, // From JSON cache (from recalculate)\n metadata: jsonData.metadata || null,\n };\n } else {\n // Missing embeddings - this shouldn't happen if recalculate ran successfully\n uncachedUrls.push(url);\n }\n } else {\n // Not in cache at all!\n uncachedUrls.push(url);\n }\n \n // If not cached anywhere OR cached data has no embeddings, skip it\n // (search requires embeddings to work, and we NEVER fetch live data - that's the point!)\n return null;\n })\n );\n if (enableTiming) timings.pageDataLoad = performance.now() - pageDataStart;\n \n // Filter out nulls\n const filterStart = performance.now();\n const filtered = pageData.filter((p): p is Page => p !== null);\n if (enableTiming) timings.filter = performance.now() - filterStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(`⏱️ getAllPagesForSearch:`);\n if (IS_VALTOWN) {\n console.log(` SQLite cache load: ${timings.jsonCacheLoad?.toFixed(2)}ms (Val Town - using SQLite)`);\n } else if (timings.jsonCacheLoad) {\n console.log(` JSON cache load (local): ${timings.jsonCacheLoad.toFixed(2)}ms`);\n }\n console.log(` Page data load: ${timings.pageDataLoad?.toFixed(2)}ms`);\n console.log(` Filter: ${timings.filter?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total.toFixed(2)}ms`);\n \n if (uncachedUrls.length > 0) {\n console.warn(` ⚠️ ${uncachedUrls.length} pages not in cache (skipped): ${uncachedUrls.slice(0, 3).join(', ')}${uncachedUrls.length > 3 ? '...' : ''}`);\n console.warn(` 💡 Run 'deno task recalculate' to populate cache`);\n }\n }\n \n return filtered;\n};\n\n// Search endpoint - uses active strategy from search/index.ts\napp.get(\"/search\", async (c) => {\n const query = c.req.query(\"q\");\n const limit = parseInt(c.req.query(\"limit\") || \"10\");\n const minScore = parseFloat(c.req.query(\"minScore\") || \"0\");\n \n if (!query) {\n return c.json({ error: \"Query parameter 'q' is required\" }, 400);\n }\n \n const validPages = await getAllPagesForSearch();\n \n // Perform search using active strategy\n const results = await searchPages(query, validPages, {\n limit,\n minScore,\n });\n \n return c.json({\n query,\n results,\n totalResults: results.length,\n totalPages: validPages.length,\n });\n});\n\n// Test search endpoint - runs test queries against active search strategy\napp.get(\"/search/test\", async (c) => {\n try {\n // Import test utilities and get active strategy info\n const { runSearchTest } = await import(\"./testing/search-harness.ts\");\n const { questions } = await import(\"./testing/questions.ts\");\n const { searchStrategy } = await import(\"./search/index.ts\");\n \n const validPages = await getAllPagesForSearch();\n const testQueries = questions;\n \n const testResults: Array<{\n query: string;\n results: Array<{\n path: string;\n url: string;\n title?: string;\n score: number;\n snippet: string;\n metadata?: {\n strategy?: string;\n [key: string]: any;\n };\n }>;\n durationMs: number;\n resultCount: number;\n error?: string;\n }> = [];\n \n for (const query of testQueries) {\n const testResult = await runSearchTest(query, validPages, {\n limit: 10,\n minScore: 0,\n enableTiming: true,\n });\n \n testResults.push({\n query,\n results: testResult.results.map(r => ({\n path: r.path,\n url: r.url,\n title: r.title,\n score: r.score,\n snippet: r.snippet?.substring(0, 200) || '', // Limit snippet length\n metadata: r.metadata,\n })),\n durationMs: testResult.durationMs,\n resultCount: testResult.results.length,\n error: testResult.error || undefined,\n });\n }\n \n return c.json({\n strategy: {\n name: searchStrategy.name,\n description: searchStrategy.description,\n },\n totalQueries: testQueries.length,\n totalPages: validPages.length,\n tests: testResults,\n summary: {\n totalDurationMs: testResults.reduce((sum, t) => sum + t.durationMs, 0),\n avgDurationMs: testResults.reduce((sum, t) => sum + t.durationMs, 0) / testResults.length,\n totalResults: testResults.reduce((sum, t) => sum + t.resultCount, 0),\n errors: testResults.filter(t => t.error).length,\n },\n });\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n return c.json({ \n error: \"Test search failed\", \n message: errorMessage \n }, 500);\n }\n});\n\n// Answer endpoint - uses active answer strategy with RAG (search + LLM)\napp.get(\"/answer\", async (c) => {\n const query = c.req.query(\"q\");\n const limit = parseInt(c.req.query(\"limit\") || \"10\");\n const minScore = parseFloat(c.req.query(\"minScore\") || \"0\");\n const maxContextPages = parseInt(c.req.query(\"maxContextPages\") || \"5\");\n const temperature = parseFloat(c.req.query(\"temperature\") || \"0.3\");\n const model = c.req.query(\"model\");\n \n if (!query) {\n return c.json({ error: \"Query parameter 'q' is required\" }, 400);\n }\n \n try {\n // Generate answer using active strategy\n const result = await answerQuestion(query, {\n limit,\n minScore,\n maxContextPages,\n temperature,\n ...(model && { model }),\n enableTiming: true,\n });\n \n return c.json(result);\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n return c.json({ \n error: \"Answer generation failed\", \n message: errorMessage \n }, 500);\n }\n});\n\n// Answer info endpoint - returns information about the active answer strategy\napp.get(\"/answer/info\", async (c) => {\n return c.json({\n strategy: {\n name: answerStrategy.name,\n description: answerStrategy.description,\n },\n defaultOptions: {\n model: \"llama-3.3-70b-versatile\",\n temperature: 0.3,\n maxContextPages: 5,\n },\n availableParams: {\n q: \"Query string (required)\",\n limit: \"Max search results to consider (default: 10)\",\n minScore: \"Minimum search score threshold (default: 0)\",\n maxContextPages: \"Max pages to include in LLM context (default: 5)\",\n temperature: \"LLM temperature (default: 0.3)\",\n model: \"Override LLM model (optional)\",\n },\n });\n});\n\n// Test answer endpoint - runs test queries against active answer strategy\napp.get(\"/answer/test\", async (c) => {\n try {\n // Import test utilities\n const { runAnswerTest } = await import(\"./testing/answer-harness.ts\");\n const { questions } = await import(\"./testing/questions.ts\");\n \n const testQueries = questions;\n \n const testResults: Array<{\n query: string;\n answer: string;\n searchResults: Array<{\n path: string;\n url: string;\n title?: string;\n score: number;\n }>;\n contextUsed: number;\n totalTokens?: number;\n durationMs: number;\n timings?: Record;\n error?: string;\n }> = [];\n \n for (const query of testQueries) {\n const testResult = await runAnswerTest(query, {\n limit: 10,\n minScore: 0,\n maxContextPages: 5,\n temperature: 0.3,\n });\n \n testResults.push({\n query,\n ...testResult,\n });\n }\n \n // Calculate summary statistics\n const avgTimings = testResults.reduce((acc, t) => {\n if (t.timings) {\n acc.search += t.timings.search || 0;\n acc.contextPrep += t.timings.contextPrep || 0;\n acc.llm += t.timings.llm || 0;\n acc.total += t.timings.total || 0;\n acc.count++;\n }\n return acc;\n }, { search: 0, contextPrep: 0, llm: 0, total: 0, count: 0 });\n \n const avgCount = avgTimings.count || 1;\n \n return c.json({\n strategy: {\n name: answerStrategy.name,\n description: answerStrategy.description,\n },\n totalQueries: testQueries.length,\n tests: testResults,\n summary: {\n totalDurationMs: testResults.reduce((sum, t) => sum + t.durationMs, 0),\n avgDurationMs: testResults.reduce((sum, t) => sum + t.durationMs, 0) / testResults.length,\n avgSearchMs: avgTimings.search / avgCount,\n avgContextPrepMs: avgTimings.contextPrep / avgCount,\n avgLlmMs: avgTimings.llm / avgCount,\n avgTotalMs: avgTimings.total / avgCount,\n totalContextUsed: testResults.reduce((sum, t) => sum + t.contextUsed, 0),\n totalTokens: testResults.reduce((sum, t) => sum + (t.totalTokens || 0), 0),\n errors: testResults.filter(t => t.error).length,\n },\n });\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n return c.json({ \n error: \"Test answer failed\", \n message: errorMessage \n }, 500);\n }\n});\n\nconst getPageByPath = async (pathParam: string, useCache: boolean = true) => {\n // Find the matching URL\n const matchingUrl = urls.find(url => {\n // Extract the path part after /docs/ and remove .md extension\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n if (match) {\n return match[1] === pathParam;\n }\n // Handle the case where URL is just /docs.md and pathParam is \"docs\"\n if (url.endsWith(\"/docs.md\") && pathParam === \"docs\") {\n return true;\n }\n return false;\n });\n \n if (!matchingUrl) {\n return null;\n }\n \n // Try to get from cache first\n if (useCache) {\n const cached = await getFromCache(matchingUrl);\n if (cached) {\n // Extract urlSource and rename it to url, then flatten remaining frontmatter properties\n const urlSource = (cached.frontmatter as any).urlSource;\n const restFrontmatter = { ...cached.frontmatter };\n delete (restFrontmatter as any).urlSource;\n \n return {\n url: urlSource || matchingUrl,\n content: cached.content,\n charCount: cached.charCount,\n tokenCount: cached.tokenCount,\n cached: true,\n ...restFrontmatter,\n ...(cached.metadata || {}), // Include AI-generated metadata if available\n };\n }\n }\n \n // Fetch and process the single page\n const text = await getTextFromUrl(matchingUrl);\n const processedText = addUrlSourceToFrontmatter(text, matchingUrl);\n const charCount = processedText.length;\n const tokenCount = calculateTokenCount(processedText);\n const frontmatter = parseFrontmatter(processedText);\n \n // Cache the result (without metadata for now - generated during recalculate)\n if (useCache) {\n // Wrap in additional error handling to ensure it never throws\n try {\n await setCache(matchingUrl, {\n content: processedText,\n charCount,\n tokenCount,\n frontmatter,\n });\n } catch (error) {\n // Double safety - setCache should already catch, but just in case\n console.debug(\"Cache write error (ignored):\", error instanceof Error ? error.message.substring(0, 100) : String(error).substring(0, 100));\n }\n }\n \n // Extract urlSource and rename it to url, then flatten remaining frontmatter properties\n const urlSource = (frontmatter as any).urlSource;\n const restFrontmatter = { ...frontmatter };\n delete (restFrontmatter as any).urlSource;\n \n return {\n url: urlSource || matchingUrl, // Use urlSource from frontmatter, fallback to matchingUrl\n content: processedText,\n charCount,\n tokenCount,\n cached: false,\n ...restFrontmatter,\n };\n};\n\napp.get(\"/page/docs\", async (c) => {\n // Handle root docs case\n const pageData = await getPageByPath(\"docs\");\n if (!pageData) {\n return c.json({ error: \"Page not found\" }, 404);\n }\n return c.json(pageData);\n});\n\napp.get(\"/page/*\", async (c) => {\n // Get the path from the URL (everything after /page/)\n const urlPath = new URL(c.req.url).pathname;\n const pathParam = urlPath.replace(\"/page/\", \"\") || \"\";\n \n const pageData = await getPageByPath(pathParam);\n if (!pageData) {\n return c.json({ error: \"Page not found\" }, 404);\n }\n return c.json(pageData);\n});\n\n// Cache management endpoints (all GET for browser access)\napp.get(\"/cache/stats\", async (c) => {\n const stats = await getCacheStats();\n return c.json(stats);\n});\n\napp.get(\"/cache/clear\", async (c) => {\n try {\n await clearCache();\n return c.json({ message: \"Cache cleared\", success: true });\n } catch (error) {\n return c.json({ \n message: \"Cache unavailable (caching disabled)\", \n success: false,\n error: \"SQLite not available - caching disabled\" \n });\n }\n});\n\napp.get(\"/cache/clear/:path\", async (c) => {\n const pathParam = c.req.param(\"path\");\n \n // Find the matching URL\n const matchingUrl = urls.find(url => {\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n if (match) {\n return match[1] === pathParam;\n }\n if (url.endsWith(\"/docs.md\") && pathParam === \"docs\") {\n return true;\n }\n return false;\n });\n \n if (!matchingUrl) {\n return c.json({ error: \"Page not found\" }, 404);\n }\n \n try {\n await clearCache(matchingUrl);\n return c.json({ message: `Cache cleared for ${pathParam}`, success: true });\n } catch (error) {\n return c.json({ \n message: `Cache unavailable for ${pathParam} (caching disabled)`, \n success: false,\n error: \"SQLite not available - caching disabled\" \n });\n }\n});\n\napp.get(\"/cache/recalculate\", async (c) => {\n // Check for force mode (bypass hash checking)\n const force = c.req.query(\"force\") === \"true\";\n \n // Recalculate all pages (bypass cache) with AI metadata generation\n const results: Array<{ \n path: string; \n url: string; \n charCount: number; \n tokenCount: number | undefined;\n title?: string;\n metadata?: any;\n error?: string;\n cacheError?: string;\n skipped?: boolean;\n reason?: string;\n }> = [];\n \n let processedCount = 0;\n let skippedCount = 0;\n \n for (const url of urls) {\n // Extract path from URL\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n const path = match ? match[1] : (url.endsWith(\"/docs.md\") ? \"docs\" : null);\n \n if (path) {\n console.log(`[${processedCount + 1}/${urls.length}] Processing ${path}...`);\n \n try {\n // Check cached hash if not forcing\n let shouldSkip = false;\n if (!force) {\n const cached = await getFromCache(url);\n if (cached && cached.contentHash) {\n // Fetch fresh content to check hash\n try {\n const pageData = await getPageByPath(path, false);\n if (pageData) {\n const newHash = await calculateContentHash(pageData.content);\n if (newHash === cached.contentHash) {\n shouldSkip = true;\n skippedCount++;\n console.log(` Skipped ${path} (content unchanged, hash: ${newHash.substring(0, 8)}...)`);\n results.push({\n path,\n url,\n charCount: cached.charCount,\n tokenCount: cached.tokenCount || undefined,\n title: (cached.frontmatter as any).title || path,\n metadata: cached.metadata || undefined,\n skipped: true,\n reason: \"Content unchanged (hash matches)\",\n });\n }\n }\n } catch (hashCheckError) {\n // If hash check fails, continue with processing\n console.debug(` Hash check failed for ${path}, continuing...`);\n }\n }\n }\n \n if (shouldSkip) {\n processedCount++;\n continue;\n }\n \n // Fetch without cache (rate limiting handled by async-sema)\n let pageData;\n try {\n pageData = await getPageByPath(path, false);\n } catch (fetchError) {\n const fetchErrorMessage = fetchError instanceof Error ? fetchError.message : String(fetchError);\n console.error(` Failed to fetch ${path}:`, fetchErrorMessage);\n throw new Error(`Fetch failed: ${fetchErrorMessage}`);\n }\n \n if (pageData) {\n const frontmatter = parseFrontmatter(pageData.content);\n const title = (frontmatter as any).title || path;\n \n // Calculate content hash\n const contentHash = await calculateContentHash(pageData.content);\n \n // Generate AI metadata (failures are handled inside generatePageMetadata)\n const metadata = await generatePageMetadata(pageData.content, title);\n if (!metadata) {\n console.warn(` Metadata generation failed for ${path} (continuing without metadata)`);\n }\n \n // Generate embeddings using active strategy's embeddings function\n const generateEmbeddings = getActiveEmbeddingsFunction();\n const embeddings = await generateEmbeddings(pageData.content);\n \n // Cache it with metadata, hash, and embeddings - track errors\n const cacheError = await setCache(url, {\n content: pageData.content,\n charCount: pageData.charCount,\n tokenCount: pageData.tokenCount || null,\n frontmatter,\n metadata,\n contentHash,\n embeddings,\n });\n \n results.push({ \n path, \n url, \n charCount: pageData.charCount, \n tokenCount: pageData.tokenCount,\n title,\n metadata, // Include full metadata in results\n ...(cacheError && { cacheError }), // Include cache error if present\n });\n \n if (cacheError) {\n console.warn(`Completed ${path} (metadata: ${metadata ? 'yes' : 'no'}, embeddings: ${embeddings ? 'yes' : 'no'}, cache error: ${cacheError})`);\n } else {\n console.log(`Completed ${path} (metadata: ${metadata ? 'yes' : 'no'}, embeddings: ${embeddings ? 'yes' : 'no'}, cached: yes)`);\n }\n }\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(`Error processing ${path}:`, errorMessage);\n results.push({\n path,\n url,\n charCount: 0,\n tokenCount: 0,\n error: errorMessage,\n });\n }\n \n processedCount++;\n }\n }\n \n const cacheErrors = results.filter(r => r.cacheError);\n const processedResults = results.filter(r => !r.skipped);\n \n return c.json({\n message: force \n ? `Recalculated ${processedResults.length} pages with AI metadata (force mode)` \n : `Recalculated ${processedResults.length} pages, skipped ${skippedCount} unchanged pages`,\n results, // Now includes full metadata for each page\n totalPages: results.length,\n processed: processedResults.length,\n skipped: skippedCount,\n withMetadata: processedResults.filter(r => r.metadata).length,\n withoutMetadata: processedResults.filter(r => !r.metadata).length,\n cached: processedResults.length > 0 && cacheErrors.length === 0, // True if all pages cached successfully\n cacheErrors: cacheErrors.length > 0 ? cacheErrors.map(r => ({ path: r.path, error: r.cacheError })) : undefined,\n force,\n });\n});\n\n// Helper function to save test data to /data/mm-dd-yyyy.json and /data/latest.json\nexport const saveTestData = async (data: any, customDate?: Date) => {\n const date = customDate || new Date();\n const month = String(date.getMonth() + 1).padStart(2, '0');\n const day = String(date.getDate()).padStart(2, '0');\n const year = date.getFullYear();\n const filename = `${month}-${day}-${year}.json`;\n const datedFilepath = `./data/${filename}`;\n const latestFilepath = `./data/latest.json`;\n \n try {\n // Ensure data directory exists\n try {\n await Deno.mkdir(\"./data\", { recursive: true });\n } catch {\n // Directory already exists, ignore\n }\n \n const jsonData = JSON.stringify(data, null, 2);\n \n // Write both files\n await Deno.writeTextFile(datedFilepath, jsonData);\n await Deno.writeTextFile(latestFilepath, jsonData);\n \n console.log(`Data saved to ${datedFilepath} and ${latestFilepath}`);\n return { datedFilepath, latestFilepath };\n } catch (error) {\n console.error(`Failed to save test data:`, error);\n throw error;\n }\n};\n\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;\n","deleted","label","groqdocs",[98],"permalink","https://www.val.town/x/yawnxyz/groq-docs/code/main.tsx","extension","actionData","errors"]