[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_407":408,"_409":410},"routes/_app.x.$handle.$name",{"_95":94,"_31":44,"_103":85,"_26":104,"_13":14,"_105":-5,"_11":106,"_107":108,"_109":110,"_111":112,"_113":114,"_20":-5,"_115":-5,"_116":-5,"_117":24,"_118":119,"_120":48,"_121":-5,"_122":-5,"_123":124},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_42":43,"_31":44,"_37":45,"_46":-5,"_47":48,"_49":50,"_51":50,"_52":-5,"_53":-5,"_18":54,"_55":56,"_57":48,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":78,"_79":80,"_81":82,"_83":24},"version",41,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":-7,"_23":24,"_25":24,"_26":27,"_28":27},"file",{"_29":30,"_31":32,"_33":34,"_35":36,"_37":38,"_39":14,"_40":41,"_23":24},"previous","deleted",false,"deletedHere","path","testing/questions.ts","latestPath","fileId","31a38f30-b901-11f0-a3b9-0224a6c84d84","name","questions.ts","fileType","script","content","// Test questions for search testing\n// Full sentence questions about Groq\n\nexport const questions: string[] = [\n \"What is Compound and how does it work?\",\n // \"How do I authenticate with the Groq API?\",\n // \"What are the rate limits for Groq models?\",\n // \"How do I integrate Groq with my JavaScript application?\",\n // \"What is the difference between compound and compound-mini?\",\n // \"Can I use Groq for real-time web search?\",\n // \"How do I get started with Groq's API?\",\n // \"What models are available on Groq?\",\n // \"How does Groq's Compound AI system work?\",\n // \"Can I use custom tools with Groq's Compound AI?\",\n];\n\n","createdAt",["D",1762207522633],"branchVersion","branchId","632d0edc-2793-11f0-9d52-569c3dd06744","id","632cc576-2793-11f0-9d52-569c3dd06744","groq-docs",["D",1746217442601],"deletedAt","legacyEnvVars",true,"privacy","public","httpPrivacy","description","imageUrl","a8270e74-6479-475a-8025-9b69544ee0e7","pinnedFileId","632eed24-2793-11f0-9d52-569c3dd06744","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","8319defa-6e1a-46d4-ba66-2114ffd3360b","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_95":94,"_102":98},"org",{"_95":94,"_96":-5,"_97":98,"_18":54,"_99":100,"_101":24},"branch",{"_13":84,"_31":85,"_42":41,"_86":43,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":44,"_93":94},"isOwned",105,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","yawnxyz","handle","displayName","avatar","https://images.clerk.dev/oauth_github/img_2NnaHhpxNuH1xWRIRjQNoo16TVc.jpeg","tier","pro","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_42":43,"_31":44,"_37":403,"_46":-5,"_47":48,"_49":50,"_51":50,"_52":-5,"_53":-5,"_18":54,"_55":56,"_57":48,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":404,"_79":405,"_81":406,"_83":24},"tree",[252,253,262,266,267,268,269,270,271,272,273,274,275,276],"files",[252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276],"counts",{"_242":61,"_243":198,"_244":61,"_245":246,"_247":248},"contents",[125,126,127,128,129,130,131,132,133,134,135,136,137,138],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_42":236,"_31":237,"_29":238,"_142":-5,"_40":41,"_39":239,"_144":48,"_37":240,"_13":61,"_33":228,"_35":-5,"_23":24,"_147":-5,"_26":241},{"_42":230,"_31":231,"_29":232,"_142":-5,"_40":41,"_39":233,"_144":48,"_37":234,"_13":61,"_33":228,"_35":-5,"_23":24,"_147":-5,"_26":235},{"_42":223,"_31":224,"_29":225,"_142":-5,"_40":41,"_39":226,"_144":48,"_37":227,"_13":61,"_33":228,"_35":-5,"_23":24,"_147":-5,"_26":229},{"_42":216,"_31":217,"_29":218,"_142":-5,"_40":41,"_39":219,"_144":24,"_37":220,"_13":61,"_33":20,"_35":221,"_23":24,"_147":-5,"_26":222},{"_42":209,"_31":210,"_29":211,"_142":-5,"_40":41,"_39":212,"_144":48,"_37":213,"_13":61,"_33":20,"_35":214,"_23":24,"_147":-5,"_26":215},{"_42":201,"_31":202,"_29":203,"_142":-5,"_40":41,"_39":204,"_144":24,"_37":205,"_13":206,"_33":20,"_35":207,"_23":24,"_147":-5,"_26":208},{"_42":193,"_31":194,"_29":195,"_142":-5,"_40":41,"_39":196,"_144":24,"_37":197,"_13":198,"_33":20,"_35":199,"_23":24,"_147":-5,"_26":200},{"_42":186,"_31":187,"_29":188,"_142":-5,"_40":41,"_39":189,"_144":48,"_37":190,"_13":61,"_33":34,"_35":191,"_23":24,"_147":-5,"_26":192},{"_42":179,"_31":180,"_29":181,"_142":-5,"_40":41,"_39":182,"_144":24,"_37":183,"_13":61,"_33":34,"_35":184,"_23":24,"_147":-5,"_26":185},{"_42":170,"_31":171,"_29":56,"_142":-5,"_40":41,"_39":172,"_144":24,"_37":173,"_13":174,"_33":175,"_35":176,"_23":24,"_147":177,"_26":178},{"_42":163,"_31":164,"_29":165,"_142":-5,"_40":41,"_39":166,"_144":24,"_37":167,"_13":61,"_33":34,"_35":168,"_23":24,"_147":-5,"_26":169},{"_42":156,"_31":157,"_29":158,"_142":-5,"_40":41,"_39":159,"_144":24,"_37":160,"_13":61,"_33":20,"_35":161,"_23":24,"_147":-5,"_26":162},{"_42":149,"_31":150,"_29":151,"_142":-5,"_40":41,"_39":152,"_144":24,"_37":153,"_13":61,"_33":34,"_35":154,"_23":24,"_147":-5,"_26":155},{"_42":139,"_31":140,"_29":141,"_142":-5,"_40":41,"_39":143,"_144":48,"_37":145,"_13":61,"_33":34,"_35":146,"_23":24,"_147":-5,"_26":148},"93344376-b8f1-11f0-ab7b-0224a6c84d84","utils.ts","93344448-b8f1-11f0-ab7b-0224a6c84d84","parentId",24,"latest",["D",1762200814128],"import { sqlite } from \"https://esm.town/v/std/sqlite\";\nimport { encoding_for_model } from \"npm:tiktoken\";\nimport { RateLimit } from \"npm:async-sema@latest\";\n\n// Detect if we're running in Val Town (caching enabled) or locally (caching disabled)\n// valtown env var contains a token string when running in Val Town, undefined/null when local\nexport const IS_VALTOWN = typeof Deno !== \"undefined\" && !!Deno.env.get(\"valtown\");\n\n// Rate limiter: 0.33 requests per second (1 request per 3 seconds) to avoid WAF blocking\n// Slower rate to avoid console.groq.com WAF blocking\nexport const fetchRateLimiter = RateLimit(0.33);\n\n// Initialize cache table only in Val Town (bypass SQLite when running locally)\nconst CACHE_TABLE = \"groq_docs_cache_v3\"; // Updated table with hash and embeddings\nif (IS_VALTOWN) {\n try {\n await sqlite.execute(`CREATE TABLE IF NOT EXISTS ${CACHE_TABLE} (\n url TEXT PRIMARY KEY,\n content TEXT NOT NULL,\n charCount INTEGER NOT NULL,\n tokenCount INTEGER,\n frontmatter TEXT NOT NULL,\n metadata TEXT,\n contentHash TEXT,\n embeddings TEXT,\n cachedAt INTEGER NOT NULL\n )`);\n console.log(\"SQLite cache initialized (Val Town)\");\n } catch (error) {\n console.warn(\"Cache table creation failed:\", error instanceof Error ? error.message.substring(0, 100) : String(error).substring(0, 100));\n }\n} else {\n console.log(\"Running locally - SQLite caching disabled\");\n}\n\n// Calculate hash of content for change detection\nexport const calculateContentHash = async (content: string): Promise => {\n // Use Web Crypto API for SHA-256 hashing\n const encoder = new TextEncoder();\n const data = encoder.encode(content);\n const hashBuffer = await crypto.subtle.digest(\"SHA-256\", data);\n const hashArray = Array.from(new Uint8Array(hashBuffer));\n const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');\n return hashHex;\n};\n\n// Debug: Log environment detection\nif (typeof Deno !== \"undefined\") {\n const valtownToken = Deno.env.get(\"valtown\");\n console.log(`Environment detection: IS_VALTOWN=${IS_VALTOWN}, valtown token exists: ${!!valtownToken}`);\n}\n\n// Cache helper functions (bypass SQLite when running locally)\nexport const getFromCache = async (url: string): Promise<{\n content: string;\n charCount: number;\n tokenCount: number | null;\n frontmatter: any;\n metadata: any | null;\n contentHash: string | null;\n embeddings: number[] | null;\n} | null> => {\n if (!IS_VALTOWN) {\n return null; // Skip cache when running locally\n }\n \n try {\n const result = await sqlite.execute({\n sql: `SELECT content, charCount, tokenCount, frontmatter, metadata, contentHash, embeddings FROM ${CACHE_TABLE} WHERE url = ?`,\n args: [url]\n });\n if (result && result.rows && result.rows.length > 0) {\n // Convert row array to object using column names\n const row = result.rows[0];\n const rowObj: any = {};\n result.columns.forEach((col: string, index: number) => {\n rowObj[col] = row[index];\n });\n return {\n content: rowObj.content as string,\n charCount: rowObj.charCount as number,\n tokenCount: rowObj.tokenCount as number | null,\n frontmatter: JSON.parse(rowObj.frontmatter as string),\n metadata: rowObj.metadata ? JSON.parse(rowObj.metadata as string) : null,\n contentHash: rowObj.contentHash as string | null,\n embeddings: rowObj.embeddings ? JSON.parse(rowObj.embeddings as string) : null,\n };\n }\n } catch (error) {\n // Silently fail - caching unavailable\n const errorMessage = error instanceof Error ? error.message : String(error);\n const shortError = errorMessage.length > 200 ? errorMessage.substring(0, 200) + '...' : errorMessage;\n console.debug(\"Cache read failed:\", shortError);\n }\n return null;\n};\n\nexport const setCache = async (url: string, data: { content: string; charCount: number; tokenCount: number | null; frontmatter: any; metadata?: any; contentHash?: string; embeddings?: number[] | null }): Promise => {\n if (!IS_VALTOWN) {\n return \"Cache unavailable - running locally\";\n }\n \n try {\n await sqlite.execute({\n sql: `INSERT OR REPLACE INTO ${CACHE_TABLE} (url, content, charCount, tokenCount, frontmatter, metadata, contentHash, embeddings, cachedAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n url,\n data.content,\n data.charCount,\n data.tokenCount,\n JSON.stringify(data.frontmatter),\n data.metadata ? JSON.stringify(data.metadata) : null,\n data.contentHash || null,\n data.embeddings ? JSON.stringify(data.embeddings) : null,\n Date.now(),\n ]\n });\n return null; // Success - no error\n } catch (error) {\n // Return error message for tracking\n const errorMessage = error instanceof Error ? error.message : String(error);\n const shortError = errorMessage.length > 200 ? errorMessage.substring(0, 200) + '...' : errorMessage;\n console.debug(\"Cache write failed:\", shortError);\n return shortError;\n }\n};\n\nexport const clearCache = async (url?: string) => {\n if (!IS_VALTOWN) {\n throw new Error(\"Cache unavailable - running locally\");\n }\n \n try {\n if (url) {\n await sqlite.execute({\n sql: `DELETE FROM ${CACHE_TABLE} WHERE url = ?`,\n args: [url]\n });\n } else {\n await sqlite.execute(`DELETE FROM ${CACHE_TABLE}`);\n }\n } catch (error) {\n console.debug(\"Cache clear failed:\", error instanceof Error ? error.message.substring(0, 100) : String(error).substring(0, 100));\n throw error; // Re-throw for endpoints that need to know if it failed\n }\n};\n\nexport const getCacheStats = async () => {\n if (!IS_VALTOWN) {\n return {\n cachedPages: 0,\n totalTokens: 0,\n message: \"Cache unavailable - running locally\",\n };\n }\n \n try {\n const result = await sqlite.execute(`SELECT COUNT(*) as count, SUM(tokenCount) as totalTokens FROM ${CACHE_TABLE}`);\n if (result && result.rows && result.rows.length > 0) {\n const row = result.rows[0];\n const rowObj: any = {};\n result.columns.forEach((col: string, index: number) => {\n rowObj[col] = row[index];\n });\n return {\n cachedPages: rowObj.count || 0,\n totalTokens: rowObj.totalTokens || 0,\n };\n }\n return {\n cachedPages: 0,\n totalTokens: 0,\n };\n } catch (error) {\n console.debug(\"Cache stats failed:\", error instanceof Error ? error.message.substring(0, 100) : String(error).substring(0, 100));\n return {\n cachedPages: 0,\n totalTokens: 0,\n };\n }\n};\n\nexport const getTextFromUrl = async (url: string) => {\n // Use rate limiter to avoid WAF blocking\n await fetchRateLimiter();\n \n const response = await fetch(url, {\n headers: {\n \"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\n \"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",\n }\n });\n \n if (!response.ok) {\n throw new Error(`HTTP ${response.status}: ${response.statusText}`);\n }\n \n const text = await response.text();\n return text;\n};\n\nexport const parseFrontmatter = (text: string) => {\n const frontmatterRegex = /^---\\s*\\n([\\s\\S]*?)\\n---\\s*\\n?/;\n const match = text.match(frontmatterRegex);\n \n if (!match) {\n return {};\n }\n \n const frontmatterText = match[1];\n const metadata: any = {};\n \n // Parse YAML-like frontmatter (simple key: value parsing)\n const lines = frontmatterText.split('\\n');\n for (const line of lines) {\n const colonIndex = line.indexOf(':');\n if (colonIndex > 0) {\n const key = line.substring(0, colonIndex).trim();\n const value = line.substring(colonIndex + 1).trim();\n // Remove quotes if present\n metadata[key] = value.replace(/^[\"']|[\"']$/g, '');\n }\n }\n \n return metadata;\n};\n\nexport const addUrlSourceToFrontmatter = (text: string, url: string) => {\n // Check if frontmatter exists\n const frontmatterRegex = /^---\\s*\\n([\\s\\S]*?)\\n---\\s*\\n?/;\n const match = text.match(frontmatterRegex);\n \n if (match) {\n // Frontmatter exists, add urlSource to it\n const existingFrontmatter = match[1];\n \n // Check if urlSource already exists\n if (existingFrontmatter.includes('urlSource:')) {\n // Replace existing urlSource\n const updatedFrontmatter = existingFrontmatter.replace(\n /urlSource:\\s*.+$/m,\n `urlSource: ${url}`\n );\n return text.replace(frontmatterRegex, `---\\n${updatedFrontmatter}\\n---\\n`);\n } else {\n // Add urlSource at the end of frontmatter\n const updatedFrontmatter = existingFrontmatter.trim() + `\\nurlSource: ${url}`;\n return text.replace(frontmatterRegex, `---\\n${updatedFrontmatter}\\n---\\n\\n`);\n }\n } else {\n // No frontmatter exists, add it\n return `---\nurlSource: ${url}\n---\n\n${text}`;\n }\n};\n\nexport const calculateTokenCount = (text: string): number => {\n try {\n // Use cl100k_base encoding (used by GPT-4 and many models)\n const encoding = encoding_for_model(\"gpt-4\");\n return encoding.encode(text).length;\n } catch (error) {\n console.error(\"Error calculating token count:\", error);\n return 0;\n }\n};\n\n","label",[140],"3178e154-b901-11f0-87ce-0224a6c84d84","urls.ts","3178e244-b901-11f0-87ce-0224a6c84d84",36,["D",1762207522123],"export\n const urls = [\n \"https://console.groq.com/docs.md\",\n \"https://console.groq.com/docs/agentic-tooling.md\",\n \"https://console.groq.com/docs/agentic-tooling/compound-beta.md\",\n \"https://console.groq.com/docs/agentic-tooling/compound-beta-mini.md\",\n \"https://console.groq.com/docs/agno.md\",\n \"https://console.groq.com/docs/ai-sdk.md\",\n \"https://console.groq.com/docs/anchorbrowser.md\",\n // \"https://console.groq.com/docs/api-reference.md\",\n // \"https://console.groq.com/docs/arize.md\",\n // \"https://console.groq.com/docs/autogen.md\",\n // \"https://console.groq.com/docs/badge.md\",\n // \"https://console.groq.com/docs/batch.md\",\n // \"https://console.groq.com/docs/billing-faqs.md\",\n // \"https://console.groq.com/docs/browser-automation.md\",\n // \"https://console.groq.com/docs/browser-search.md\",\n // \"https://console.groq.com/docs/browserbase.md\",\n // \"https://console.groq.com/docs/browseruse.md\",\n // \"https://console.groq.com/docs/changelog.md\",\n // \"https://console.groq.com/docs/changelog/compound.md\",\n // \"https://console.groq.com/docs/code-execution.md\",\n // \"https://console.groq.com/docs/composio.md\",\n // \"https://console.groq.com/docs/compound.md\",\n // \"https://console.groq.com/docs/compound/built-in-tools.md\",\n // \"https://console.groq.com/docs/compound/systems.md\",\n // \"https://console.groq.com/docs/compound/systems/compound.md\",\n // \"https://console.groq.com/docs/compound/systems/compound-beta.md\",\n // \"https://console.groq.com/docs/compound/systems/compound-beta-mini.md\",\n // \"https://console.groq.com/docs/compound/systems/compound-mini.md\",\n // \"https://console.groq.com/docs/compound/use-cases.md\",\n // \"https://console.groq.com/docs/content-moderation.md\",\n // \"https://console.groq.com/docs/crewai.md\",\n // \"https://console.groq.com/docs/deprecations.md\",\n // \"https://console.groq.com/docs/e2b.md\",\n // \"https://console.groq.com/docs/errors.md\",\n // \"https://console.groq.com/docs/exa.md\",\n // \"https://console.groq.com/docs/examples.md\",\n // \"https://console.groq.com/docs/firecrawl.md\",\n // \"https://console.groq.com/docs/flex-processing.md\",\n // \"https://console.groq.com/docs/flutterflow.md\",\n // \"https://console.groq.com/docs/gradio.md\",\n // \"https://console.groq.com/docs/guides/speech-to-text/prompting.md\",\n // \"https://console.groq.com/docs/huggingface.md\",\n // \"https://console.groq.com/docs/integrations.md\",\n // \"https://console.groq.com/docs/jigsawstack.md\",\n // \"https://console.groq.com/docs/langchain.md\",\n // \"https://console.groq.com/docs/legacy-changelog.md\",\n // \"https://console.groq.com/docs/legal.md\",\n // \"https://console.groq.com/docs/legal/ai-policy.md\",\n // \"https://console.groq.com/docs/legal/services-agreement.md\",\n // \"https://console.groq.com/docs/libraries.md\",\n // \"https://console.groq.com/docs/litellm.md\",\n // \"https://console.groq.com/docs/livekit.md\",\n // \"https://console.groq.com/docs/llama-index.md\",\n // \"https://console.groq.com/docs/lora.md\",\n // \"https://console.groq.com/docs/mastra.md\",\n // \"https://console.groq.com/docs/mcp.md\",\n // \"https://console.groq.com/docs/mlflow.md\",\n // \"https://console.groq.com/docs/model-permissions.md\",\n // \"https://console.groq.com/docs/model/llama-3.1-8b-instant.md\",\n // \"https://console.groq.com/docs/model/llama-3.3-70b-versatile.md\",\n // \"https://console.groq.com/docs/model/llama-4-maverick-17b-128e-instruct.md\",\n // \"https://console.groq.com/docs/model/llama-4-scout-17b-16e-instruct.md\",\n // \"https://console.groq.com/docs/model/llama-guard-4-12b.md\",\n // \"https://console.groq.com/docs/model/llama-prompt-guard-2-22m.md\",\n // \"https://console.groq.com/docs/model/llama-prompt-guard-2-86m.md\",\n // \"https://console.groq.com/docs/model/meta-llama/llama-4-maverick-17b-128e-instruct.md\",\n // \"https://console.groq.com/docs/model/meta-llama/llama-4-scout-17b-16e-instruct.md\",\n // \"https://console.groq.com/docs/model/meta-llama/llama-guard-4-12b.md\",\n // \"https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-22m.md\",\n // \"https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m.md\",\n // \"https://console.groq.com/docs/model/moonshotai/kimi-k2-instruct.md\",\n // \"https://console.groq.com/docs/model/moonshotai/kimi-k2-instruct-0905.md\",\n // \"https://console.groq.com/docs/model/openai/gpt-oss-120b.md\",\n // \"https://console.groq.com/docs/model/openai/gpt-oss-20b.md\",\n // \"https://console.groq.com/docs/model/openai/gpt-oss-safeguard-20b.md\",\n // \"https://console.groq.com/docs/model/playai-tts.md\",\n // \"https://console.groq.com/docs/model/playai-tts-arabic.md\",\n // \"https://console.groq.com/docs/model/qwen/qwen3-32b.md\",\n // \"https://console.groq.com/docs/model/qwen3-32b.md\",\n // \"https://console.groq.com/docs/model/whisper-large-v3.md\",\n // \"https://console.groq.com/docs/model/whisper-large-v3-turbo.md\",\n // \"https://console.groq.com/docs/models.md\",\n // \"https://console.groq.com/docs/openai.md\",\n // \"https://console.groq.com/docs/overview.md\",\n // \"https://console.groq.com/docs/parallel.md\",\n // \"https://console.groq.com/docs/prefilling.md\",\n // \"https://console.groq.com/docs/production-readiness/optimizing-latency.md\",\n // \"https://console.groq.com/docs/production-readiness/production-ready-checklist.md\",\n // \"https://console.groq.com/docs/production-readiness/security-onboarding.md\",\n // \"https://console.groq.com/docs/projects.md\",\n // \"https://console.groq.com/docs/prompt-caching.md\",\n // \"https://console.groq.com/docs/prompting.md\",\n // \"https://console.groq.com/docs/prompting/model-migration.md\",\n // \"https://console.groq.com/docs/prompting/patterns.md\",\n // \"https://console.groq.com/docs/quickstart.md\",\n // \"https://console.groq.com/docs/rate-limits.md\",\n // \"https://console.groq.com/docs/reasoning.md\",\n // \"https://console.groq.com/docs/responses-api.md\",\n // \"https://console.groq.com/docs/speech-text.md\",\n // \"https://console.groq.com/docs/speech-to-text.md\",\n // \"https://console.groq.com/docs/spend-limits.md\",\n // \"https://console.groq.com/docs/structured-outputs.md\",\n // \"https://console.groq.com/docs/tavily.md\",\n // \"https://console.groq.com/docs/text-chat.md\",\n // \"https://console.groq.com/docs/text-to-speech.md\",\n // \"https://console.groq.com/docs/tool-use.md\",\n // \"https://console.groq.com/docs/toolhouse.md\",\n // \"https://console.groq.com/docs/vision.md\",\n // \"https://console.groq.com/docs/web-search.md\",\n // \"https://console.groq.com/docs/wolfram-alpha.md\",\n // \"https://console.groq.com/docs/xrx.md\",\n // \"https://console.groq.com/docs/your-data.md\",\n ];\n",[150],"7ebe7c70-b8ee-11f0-bdd9-0224a6c84d84","todo.md","7ebe7d74-b8ee-11f0-bdd9-0224a6c84d84",15,["D",1762199491650],"\n## todos\n- [ ] markdown endpoint\n- [ ] pre-calculate metadata around each markdown page\n- [ ] pre-calculate complex question query q&a pairs that map to various pages and categories",[157],"9300cdf2-b8f1-11f0-85fe-0224a6c84d84","search.ts","9300cec4-b8f1-11f0-85fe-0224a6c84d84",21,["D",1762200814117],"// Embeddings and Search utilities\n// TODO: Replace fake implementations with actual embeddings API and vector search\n\n// Generate embeddings for content (fake default implementation)\nexport const generateEmbeddings = async (content: string): Promise => {\n // TODO: Replace with actual embeddings API when available\n // For now, return a fake embedding vector\n console.debug(\"Generating fake embeddings (replace with actual API when available)\");\n \n // Generate a fake embedding of fixed size (e.g., 384 dimensions)\n // Using a simple hash-based approach to create deterministic fake embeddings\n const fakeEmbedding: number[] = [];\n const hash = content.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0);\n \n for (let i = 0; i < 384; i++) {\n // Create deterministic fake values based on content hash and position\n const value = Math.sin(hash + i) * 0.5 + 0.5; // Normalize to 0-1\n fakeEmbedding.push(value);\n }\n \n return fakeEmbedding;\n};\n\n// Calculate cosine similarity between two embedding vectors\nconst cosineSimilarity = (vecA: number[], vecB: number[]): number => {\n if (vecA.length !== vecB.length) {\n return 0;\n }\n \n let dotProduct = 0;\n let normA = 0;\n let normB = 0;\n \n for (let i = 0; i < vecA.length; i++) {\n dotProduct += vecA[i] * vecB[i];\n normA += vecA[i] * vecA[i];\n normB += vecB[i] * vecB[i];\n }\n \n if (normA === 0 || normB === 0) {\n return 0;\n }\n \n return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));\n};\n\n// Search pages by query (fake implementation using keyword matching)\n// TODO: Replace with actual vector similarity search using embeddings\nexport const searchPages = async (\n query: string,\n pages: Array<{\n path: string;\n url: string;\n content: string;\n title?: string;\n embeddings?: number[] | null;\n metadata?: any;\n }>,\n options: {\n limit?: number;\n minScore?: number;\n } = {}\n): Promise> => {\n const limit = options.limit || 10;\n const minScore = options.minScore || 0;\n \n // TODO: Use embeddings for semantic search\n // For now, use simple keyword matching\n \n const queryLower = query.toLowerCase();\n const queryWords = queryLower.split(/\\s+/).filter(w => w.length > 0);\n \n const results: Array<{\n path: string;\n url: string;\n title?: string;\n score: number;\n snippet?: string;\n }> = [];\n \n for (const page of pages) {\n let score = 0;\n const contentLower = page.content.toLowerCase();\n const titleLower = (page.title || page.path).toLowerCase();\n \n // Title matches are worth more\n for (const word of queryWords) {\n if (titleLower.includes(word)) {\n score += 10;\n }\n // Count occurrences in content\n const contentMatches = (contentLower.match(new RegExp(word, 'g')) || []).length;\n score += Math.min(contentMatches, 5); // Cap at 5 points per word\n }\n \n // Exact phrase match bonus\n if (contentLower.includes(queryLower)) {\n score += 20;\n }\n \n // Metadata boost (if query matches tags or categories)\n if (page.metadata) {\n const tags = (page.metadata.tags || []).join(' ').toLowerCase();\n const categories = (page.metadata.categories || []).join(' ').toLowerCase();\n const useCases = (page.metadata.useCases || []).join(' ').toLowerCase();\n \n for (const word of queryWords) {\n if (tags.includes(word)) score += 3;\n if (categories.includes(word)) score += 2;\n if (useCases.includes(word)) score += 2;\n }\n }\n \n if (score > minScore) {\n // Generate snippet (first 200 chars around first match)\n let snippet: string | undefined;\n const firstMatchIndex = contentLower.indexOf(queryWords[0]);\n if (firstMatchIndex >= 0) {\n const start = Math.max(0, firstMatchIndex - 50);\n const end = Math.min(page.content.length, firstMatchIndex + queryWords[0].length + 150);\n snippet = page.content.substring(start, end);\n if (start > 0) snippet = '...' + snippet;\n if (end < page.content.length) snippet = snippet + '...';\n } else {\n snippet = page.content.substring(0, 200) + '...';\n }\n \n results.push({\n path: page.path,\n url: page.url,\n title: page.title,\n score,\n snippet,\n });\n }\n }\n \n // Sort by score descending\n results.sort((a, b) => b.score - a.score);\n \n return results.slice(0, limit);\n};\n\n// Vector similarity search (for future use with embeddings)\nexport const vectorSearch = async (\n queryEmbedding: number[],\n pages: Array<{\n path: string;\n url: string;\n title?: string;\n embeddings: number[] | null;\n }>,\n options: {\n limit?: number;\n minSimilarity?: number;\n } = {}\n): Promise> => {\n const limit = options.limit || 10;\n const minSimilarity = options.minSimilarity || 0;\n \n const results: Array<{\n path: string;\n url: string;\n title?: string;\n similarity: number;\n }> = [];\n \n for (const page of pages) {\n if (!page.embeddings) {\n continue;\n }\n \n const similarity = cosineSimilarity(queryEmbedding, page.embeddings);\n \n if (similarity >= minSimilarity) {\n results.push({\n path: page.path,\n url: page.url,\n title: page.title,\n similarity,\n });\n }\n }\n \n // Sort by similarity descending\n results.sort((a, b) => b.similarity - a.similarity);\n \n return results.slice(0, limit);\n};\n\n",[164],"71afafe4-b8f4-11f0-a389-0224a6c84d84","main.tsx",26,["D",1762202046713],13,"http","import { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport \"jsr:@std/dotenv/load\";\nimport {\n IS_VALTOWN,\n getFromCache,\n setCache,\n clearCache,\n getCacheStats,\n getTextFromUrl,\n parseFrontmatter,\n addUrlSourceToFrontmatter,\n calculateTokenCount,\n calculateContentHash,\n} from \"./utils.ts\";\nimport { generatePageMetadata } from \"./groq.ts\";\nimport { generateEmbeddings, searchPages } from \"./search.ts\";\n\nconst app = new Hono();\n\n\n\nconst urls = [\n \"https://console.groq.com/docs.md\",\n \"https://console.groq.com/docs/agentic-tooling.md\",\n \"https://console.groq.com/docs/agentic-tooling/compound-beta.md\",\n \"https://console.groq.com/docs/agentic-tooling/compound-beta-mini.md\",\n \"https://console.groq.com/docs/agno.md\",\n \"https://console.groq.com/docs/ai-sdk.md\",\n \"https://console.groq.com/docs/anchorbrowser.md\",\n \"https://console.groq.com/docs/api-reference.md\",\n \"https://console.groq.com/docs/arize.md\",\n \"https://console.groq.com/docs/autogen.md\",\n \"https://console.groq.com/docs/badge.md\",\n \"https://console.groq.com/docs/batch.md\",\n \"https://console.groq.com/docs/billing-faqs.md\",\n \"https://console.groq.com/docs/browser-automation.md\",\n \"https://console.groq.com/docs/browser-search.md\",\n \"https://console.groq.com/docs/browserbase.md\",\n \"https://console.groq.com/docs/browseruse.md\",\n \"https://console.groq.com/docs/changelog.md\",\n \"https://console.groq.com/docs/changelog/compound.md\",\n \"https://console.groq.com/docs/code-execution.md\",\n \"https://console.groq.com/docs/composio.md\",\n \"https://console.groq.com/docs/compound.md\",\n \"https://console.groq.com/docs/compound/built-in-tools.md\",\n \"https://console.groq.com/docs/compound/systems.md\",\n \"https://console.groq.com/docs/compound/systems/compound.md\",\n \"https://console.groq.com/docs/compound/systems/compound-beta.md\",\n \"https://console.groq.com/docs/compound/systems/compound-beta-mini.md\",\n \"https://console.groq.com/docs/compound/systems/compound-mini.md\",\n \"https://console.groq.com/docs/compound/use-cases.md\",\n \"https://console.groq.com/docs/content-moderation.md\",\n \"https://console.groq.com/docs/crewai.md\",\n \"https://console.groq.com/docs/deprecations.md\",\n \"https://console.groq.com/docs/e2b.md\",\n \"https://console.groq.com/docs/errors.md\",\n \"https://console.groq.com/docs/exa.md\",\n \"https://console.groq.com/docs/examples.md\",\n \"https://console.groq.com/docs/firecrawl.md\",\n \"https://console.groq.com/docs/flex-processing.md\",\n \"https://console.groq.com/docs/flutterflow.md\",\n \"https://console.groq.com/docs/gradio.md\",\n \"https://console.groq.com/docs/guides/speech-to-text/prompting.md\",\n \"https://console.groq.com/docs/huggingface.md\",\n \"https://console.groq.com/docs/integrations.md\",\n \"https://console.groq.com/docs/jigsawstack.md\",\n \"https://console.groq.com/docs/langchain.md\",\n \"https://console.groq.com/docs/legacy-changelog.md\",\n \"https://console.groq.com/docs/legal.md\",\n \"https://console.groq.com/docs/legal/ai-policy.md\",\n \"https://console.groq.com/docs/legal/services-agreement.md\",\n \"https://console.groq.com/docs/libraries.md\",\n \"https://console.groq.com/docs/litellm.md\",\n \"https://console.groq.com/docs/livekit.md\",\n \"https://console.groq.com/docs/llama-index.md\",\n \"https://console.groq.com/docs/lora.md\",\n \"https://console.groq.com/docs/mastra.md\",\n \"https://console.groq.com/docs/mcp.md\",\n \"https://console.groq.com/docs/mlflow.md\",\n \"https://console.groq.com/docs/model-permissions.md\",\n \"https://console.groq.com/docs/model/llama-3.1-8b-instant.md\",\n \"https://console.groq.com/docs/model/llama-3.3-70b-versatile.md\",\n \"https://console.groq.com/docs/model/llama-4-maverick-17b-128e-instruct.md\",\n \"https://console.groq.com/docs/model/llama-4-scout-17b-16e-instruct.md\",\n \"https://console.groq.com/docs/model/llama-guard-4-12b.md\",\n \"https://console.groq.com/docs/model/llama-prompt-guard-2-22m.md\",\n \"https://console.groq.com/docs/model/llama-prompt-guard-2-86m.md\",\n \"https://console.groq.com/docs/model/meta-llama/llama-4-maverick-17b-128e-instruct.md\",\n \"https://console.groq.com/docs/model/meta-llama/llama-4-scout-17b-16e-instruct.md\",\n \"https://console.groq.com/docs/model/meta-llama/llama-guard-4-12b.md\",\n \"https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-22m.md\",\n \"https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m.md\",\n \"https://console.groq.com/docs/model/moonshotai/kimi-k2-instruct.md\",\n \"https://console.groq.com/docs/model/moonshotai/kimi-k2-instruct-0905.md\",\n \"https://console.groq.com/docs/model/openai/gpt-oss-120b.md\",\n \"https://console.groq.com/docs/model/openai/gpt-oss-20b.md\",\n \"https://console.groq.com/docs/model/openai/gpt-oss-safeguard-20b.md\",\n \"https://console.groq.com/docs/model/playai-tts.md\",\n \"https://console.groq.com/docs/model/playai-tts-arabic.md\",\n \"https://console.groq.com/docs/model/qwen/qwen3-32b.md\",\n \"https://console.groq.com/docs/model/qwen3-32b.md\",\n \"https://console.groq.com/docs/model/whisper-large-v3.md\",\n \"https://console.groq.com/docs/model/whisper-large-v3-turbo.md\",\n \"https://console.groq.com/docs/models.md\",\n \"https://console.groq.com/docs/openai.md\",\n \"https://console.groq.com/docs/overview.md\",\n \"https://console.groq.com/docs/parallel.md\",\n \"https://console.groq.com/docs/prefilling.md\",\n \"https://console.groq.com/docs/production-readiness/optimizing-latency.md\",\n \"https://console.groq.com/docs/production-readiness/production-ready-checklist.md\",\n \"https://console.groq.com/docs/production-readiness/security-onboarding.md\",\n \"https://console.groq.com/docs/projects.md\",\n \"https://console.groq.com/docs/prompt-caching.md\",\n \"https://console.groq.com/docs/prompting.md\",\n \"https://console.groq.com/docs/prompting/model-migration.md\",\n \"https://console.groq.com/docs/prompting/patterns.md\",\n \"https://console.groq.com/docs/quickstart.md\",\n \"https://console.groq.com/docs/rate-limits.md\",\n \"https://console.groq.com/docs/reasoning.md\",\n \"https://console.groq.com/docs/responses-api.md\",\n \"https://console.groq.com/docs/speech-text.md\",\n \"https://console.groq.com/docs/speech-to-text.md\",\n \"https://console.groq.com/docs/spend-limits.md\",\n \"https://console.groq.com/docs/structured-outputs.md\",\n \"https://console.groq.com/docs/tavily.md\",\n \"https://console.groq.com/docs/text-chat.md\",\n \"https://console.groq.com/docs/text-to-speech.md\",\n \"https://console.groq.com/docs/tool-use.md\",\n \"https://console.groq.com/docs/toolhouse.md\",\n \"https://console.groq.com/docs/vision.md\",\n \"https://console.groq.com/docs/web-search.md\",\n \"https://console.groq.com/docs/wolfram-alpha.md\",\n \"https://console.groq.com/docs/xrx.md\",\n \"https://console.groq.com/docs/your-data.md\",\n];\n\n\n\n\nconst getPageContent = async () => {\n const promises = urls.map(async url => {\n const text = await getTextFromUrl(url);\n return addUrlSourceToFrontmatter(text, url);\n });\n const texts = await Promise.all(promises);\n return texts.join(\"\\n\\n\");\n};\n\nconst getPageData = async () => {\n const pageData = await Promise.all(\n urls.map(async (url) => {\n // Check if cached\n const cached = await getFromCache(url);\n let text, processedText, charCount, frontmatter, wasCached;\n \n let metadata = null;\n \n if (cached) {\n // Use cached data\n text = cached.content;\n processedText = cached.content;\n charCount = cached.charCount;\n frontmatter = cached.frontmatter;\n metadata = cached.metadata;\n wasCached = true;\n } else {\n // Fetch fresh\n text = await getTextFromUrl(url);\n processedText = addUrlSourceToFrontmatter(text, url);\n charCount = processedText.length;\n frontmatter = parseFrontmatter(processedText);\n wasCached = false;\n }\n \n // Extract urlSource and rename it to url, then flatten remaining frontmatter properties\n const urlSource = (frontmatter as any).urlSource;\n const restFrontmatter = { ...frontmatter };\n delete (restFrontmatter as any).urlSource;\n \n return {\n url: urlSource || url, // Use urlSource from frontmatter, fallback to url\n content: processedText,\n charCount,\n cached: wasCached,\n ...restFrontmatter, // Flatten frontmatter properties\n ...(metadata || {}), // Include AI-generated metadata if available\n };\n })\n );\n \n return {\n pages: pageData,\n totalPages: pageData.length,\n totalChars: pageData.reduce((sum, { charCount }) => sum + charCount, 0),\n cachedPages: pageData.filter(({ cached }) => cached).length,\n };\n};\n\napp.get(\"/\", async (c) => {\n const content = await getPageContent();\n return c.text(content);\n});\n\napp.get(\"/data\", async (c) => {\n const data = await getPageData();\n \n // Extract page paths/names (same logic as /list endpoint)\n const pagePaths = urls.map(url => {\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n if (match) {\n return match[1];\n }\n if (url.endsWith(\"/docs.md\")) {\n return \"docs\";\n }\n return null;\n }).filter(path => path !== null);\n \n // Return metadata first, then the large data object\n return c.json({\n metadata: {\n urls: pagePaths,\n },\n ...data,\n });\n});\n\napp.get(\"/list\", async (c) => {\n const paths = urls.map(url => {\n // Extract the path part after /docs/ and remove .md extension\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n if (match) {\n return match[1];\n }\n // Handle the case where URL is just /docs.md - return \"docs\"\n if (url.endsWith(\"/docs.md\")) {\n return \"docs\";\n }\n return null;\n }).filter(path => path !== null);\n \n return c.json(paths);\n});\n\napp.get(\"/search\", async (c) => {\n const query = c.req.query(\"q\");\n const limit = parseInt(c.req.query(\"limit\") || \"10\");\n const minScore = parseFloat(c.req.query(\"minScore\") || \"0\");\n \n if (!query) {\n return c.json({ error: \"Query parameter 'q' is required\" }, 400);\n }\n \n // Get all pages for searching\n const pageData = await Promise.all(\n urls.map(async (url) => {\n // Extract path from URL\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n const path = match ? match[1] : (url.endsWith(\"/docs.md\") ? \"docs\" : null);\n \n if (!path) {\n return null;\n }\n \n // Try to get from cache first\n const cached = await getFromCache(url);\n \n if (cached) {\n const frontmatter = cached.frontmatter;\n return {\n path,\n url: cached.frontmatter?.urlSource || url,\n content: cached.content,\n title: frontmatter?.title || path,\n embeddings: cached.embeddings,\n metadata: cached.metadata,\n };\n }\n \n // If not cached, fetch it (but don't cache during search to avoid slowing down)\n try {\n const pageData = await getPageByPath(path, false);\n if (pageData) {\n const frontmatter = parseFrontmatter(pageData.content);\n return {\n path,\n url: pageData.url,\n content: pageData.content,\n title: frontmatter?.title || path,\n embeddings: null, // Not generated yet\n metadata: null,\n };\n }\n } catch (error) {\n // Skip pages that fail to fetch\n return null;\n }\n \n return null;\n })\n );\n \n // Filter out nulls\n const validPages = pageData.filter((p): p is NonNullable => p !== null);\n \n // Perform search\n const results = await searchPages(query, validPages, {\n limit,\n minScore,\n });\n \n return c.json({\n query,\n results,\n totalResults: results.length,\n totalPages: validPages.length,\n });\n});\n\n\nconst getPageByPath = async (pathParam: string, useCache: boolean = true) => {\n // Find the matching URL\n const matchingUrl = urls.find(url => {\n // Extract the path part after /docs/ and remove .md extension\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n if (match) {\n return match[1] === pathParam;\n }\n // Handle the case where URL is just /docs.md and pathParam is \"docs\"\n if (url.endsWith(\"/docs.md\") && pathParam === \"docs\") {\n return true;\n }\n return false;\n });\n \n if (!matchingUrl) {\n return null;\n }\n \n // Try to get from cache first\n if (useCache) {\n const cached = await getFromCache(matchingUrl);\n if (cached) {\n // Extract urlSource and rename it to url, then flatten remaining frontmatter properties\n const urlSource = (cached.frontmatter as any).urlSource;\n const restFrontmatter = { ...cached.frontmatter };\n delete (restFrontmatter as any).urlSource;\n \n return {\n url: urlSource || matchingUrl,\n content: cached.content,\n charCount: cached.charCount,\n tokenCount: cached.tokenCount,\n cached: true,\n ...restFrontmatter,\n ...(cached.metadata || {}), // Include AI-generated metadata if available\n };\n }\n }\n \n // Fetch and process the single page\n const text = await getTextFromUrl(matchingUrl);\n const processedText = addUrlSourceToFrontmatter(text, matchingUrl);\n const charCount = processedText.length;\n const tokenCount = calculateTokenCount(processedText);\n const frontmatter = parseFrontmatter(processedText);\n \n // Cache the result (without metadata for now - generated during recalculate)\n if (useCache) {\n // Wrap in additional error handling to ensure it never throws\n try {\n await setCache(matchingUrl, {\n content: processedText,\n charCount,\n tokenCount,\n frontmatter,\n });\n } catch (error) {\n // Double safety - setCache should already catch, but just in case\n console.debug(\"Cache write error (ignored):\", error instanceof Error ? error.message.substring(0, 100) : String(error).substring(0, 100));\n }\n }\n \n // Extract urlSource and rename it to url, then flatten remaining frontmatter properties\n const urlSource = (frontmatter as any).urlSource;\n const restFrontmatter = { ...frontmatter };\n delete (restFrontmatter as any).urlSource;\n \n return {\n url: urlSource || matchingUrl, // Use urlSource from frontmatter, fallback to matchingUrl\n content: processedText,\n charCount,\n tokenCount,\n cached: false,\n ...restFrontmatter,\n };\n};\n\napp.get(\"/page/docs\", async (c) => {\n // Handle root docs case\n const pageData = await getPageByPath(\"docs\");\n if (!pageData) {\n return c.json({ error: \"Page not found\" }, 404);\n }\n return c.json(pageData);\n});\n\napp.get(\"/page/*\", async (c) => {\n // Get the path from the URL (everything after /page/)\n const urlPath = new URL(c.req.url).pathname;\n const pathParam = urlPath.replace(\"/page/\", \"\") || \"\";\n \n const pageData = await getPageByPath(pathParam);\n if (!pageData) {\n return c.json({ error: \"Page not found\" }, 404);\n }\n return c.json(pageData);\n});\n\n// Cache management endpoints (all GET for browser access)\napp.get(\"/cache/stats\", async (c) => {\n const stats = await getCacheStats();\n return c.json(stats);\n});\n\napp.get(\"/cache/clear\", async (c) => {\n try {\n await clearCache();\n return c.json({ message: \"Cache cleared\", success: true });\n } catch (error) {\n return c.json({ \n message: \"Cache unavailable (caching disabled)\", \n success: false,\n error: \"SQLite not available - caching disabled\" \n });\n }\n});\n\napp.get(\"/cache/clear/:path\", async (c) => {\n const pathParam = c.req.param(\"path\");\n \n // Find the matching URL\n const matchingUrl = urls.find(url => {\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n if (match) {\n return match[1] === pathParam;\n }\n if (url.endsWith(\"/docs.md\") && pathParam === \"docs\") {\n return true;\n }\n return false;\n });\n \n if (!matchingUrl) {\n return c.json({ error: \"Page not found\" }, 404);\n }\n \n try {\n await clearCache(matchingUrl);\n return c.json({ message: `Cache cleared for ${pathParam}`, success: true });\n } catch (error) {\n return c.json({ \n message: `Cache unavailable for ${pathParam} (caching disabled)`, \n success: false,\n error: \"SQLite not available - caching disabled\" \n });\n }\n});\n\napp.get(\"/cache/recalculate\", async (c) => {\n // Check for force mode (bypass hash checking)\n const force = c.req.query(\"force\") === \"true\";\n \n // Recalculate all pages (bypass cache) with AI metadata generation\n const results: Array<{ \n path: string; \n url: string; \n charCount: number; \n tokenCount: number | undefined;\n title?: string;\n metadata?: any;\n error?: string;\n cacheError?: string;\n skipped?: boolean;\n reason?: string;\n }> = [];\n \n let processedCount = 0;\n let skippedCount = 0;\n \n for (const url of urls) {\n // Extract path from URL\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n const path = match ? match[1] : (url.endsWith(\"/docs.md\") ? \"docs\" : null);\n \n if (path) {\n console.log(`[${processedCount + 1}/${urls.length}] Processing ${path}...`);\n \n try {\n // Check cached hash if not forcing\n let shouldSkip = false;\n if (!force) {\n const cached = await getFromCache(url);\n if (cached && cached.contentHash) {\n // Fetch fresh content to check hash\n try {\n const pageData = await getPageByPath(path, false);\n if (pageData) {\n const newHash = await calculateContentHash(pageData.content);\n if (newHash === cached.contentHash) {\n shouldSkip = true;\n skippedCount++;\n console.log(` Skipped ${path} (content unchanged, hash: ${newHash.substring(0, 8)}...)`);\n results.push({\n path,\n url,\n charCount: cached.charCount,\n tokenCount: cached.tokenCount || undefined,\n title: (cached.frontmatter as any).title || path,\n metadata: cached.metadata || undefined,\n skipped: true,\n reason: \"Content unchanged (hash matches)\",\n });\n }\n }\n } catch (hashCheckError) {\n // If hash check fails, continue with processing\n console.debug(` Hash check failed for ${path}, continuing...`);\n }\n }\n }\n \n if (shouldSkip) {\n processedCount++;\n continue;\n }\n \n // Fetch without cache (rate limiting handled by async-sema)\n let pageData;\n try {\n pageData = await getPageByPath(path, false);\n } catch (fetchError) {\n const fetchErrorMessage = fetchError instanceof Error ? fetchError.message : String(fetchError);\n console.error(` Failed to fetch ${path}:`, fetchErrorMessage);\n throw new Error(`Fetch failed: ${fetchErrorMessage}`);\n }\n \n if (pageData) {\n const frontmatter = parseFrontmatter(pageData.content);\n const title = (frontmatter as any).title || path;\n \n // Calculate content hash\n const contentHash = await calculateContentHash(pageData.content);\n \n // Generate AI metadata (failures are handled inside generatePageMetadata)\n const metadata = await generatePageMetadata(pageData.content, title);\n if (!metadata) {\n console.warn(` Metadata generation failed for ${path} (continuing without metadata)`);\n }\n \n // Generate embeddings\n const embeddings = await generateEmbeddings(pageData.content);\n \n // Cache it with metadata, hash, and embeddings - track errors\n const cacheError = await setCache(url, {\n content: pageData.content,\n charCount: pageData.charCount,\n tokenCount: pageData.tokenCount || null,\n frontmatter,\n metadata,\n contentHash,\n embeddings,\n });\n \n results.push({ \n path, \n url, \n charCount: pageData.charCount, \n tokenCount: pageData.tokenCount,\n title,\n metadata, // Include full metadata in results\n ...(cacheError && { cacheError }), // Include cache error if present\n });\n \n if (cacheError) {\n console.warn(`Completed ${path} (metadata: ${metadata ? 'yes' : 'no'}, embeddings: ${embeddings ? 'yes' : 'no'}, cache error: ${cacheError})`);\n } else {\n console.log(`Completed ${path} (metadata: ${metadata ? 'yes' : 'no'}, embeddings: ${embeddings ? 'yes' : 'no'}, cached: yes)`);\n }\n }\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(`Error processing ${path}:`, errorMessage);\n results.push({\n path,\n url,\n charCount: 0,\n tokenCount: 0,\n error: errorMessage,\n });\n }\n \n processedCount++;\n }\n }\n \n const cacheErrors = results.filter(r => r.cacheError);\n const processedResults = results.filter(r => !r.skipped);\n \n return c.json({\n message: force \n ? `Recalculated ${processedResults.length} pages with AI metadata (force mode)` \n : `Recalculated ${processedResults.length} pages, skipped ${skippedCount} unchanged pages`,\n results, // Now includes full metadata for each page\n totalPages: results.length,\n processed: processedResults.length,\n skipped: skippedCount,\n withMetadata: processedResults.filter(r => r.metadata).length,\n withoutMetadata: processedResults.filter(r => !r.metadata).length,\n cached: processedResults.length > 0 && cacheErrors.length === 0, // True if all pages cached successfully\n cacheErrors: cacheErrors.length > 0 ? cacheErrors.map(r => ({ path: r.path, error: r.cacheError })) : undefined,\n force,\n });\n});\n\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;\n","groqdocs",[171],"657bf32a-b8e7-11f0-9be7-0224a6c84d84","main-old.tsx","657bf3c0-b8e7-11f0-9be7-0224a6c84d84",7,["D",1762196442795],"import { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport \"jsr:@std/dotenv/load\";\n\nconst app = new Hono();\n\n\nconst getTextFromUrl = async (url) => {\n const response = await fetch(\"https://r.jina.ai/\"+url);\n const text = await response.text();\n return text;\n};\n\nconst stripExistingFrontmatter = (text) => {\n // Remove any existing YAML frontmatter (between --- markers at start)\n const frontmatterRegex = /^---\\s*\\n[\\s\\S]*?\\n---\\s*\\n?/;\n return text.replace(frontmatterRegex, '').trim();\n};\n\nconst extractMetadata = (text, url) => {\n let title = '';\n let description = '';\n let image = '';\n \n // Jina includes \"Title: ...\" in the output\n const titleMatch = text.match(/^Title:\\s*(.+?)(?:\\n|$)/m);\n if (titleMatch) {\n title = titleMatch[1].trim();\n } else {\n // Fallback to first heading\n const headingMatch = text.match(/^#+\\s+(.+)$/m);\n if (headingMatch) {\n title = headingMatch[1].trim();\n }\n }\n \n // Try to find description (usually first paragraph after title)\n const descMatch = text.match(/Description:\\s*(.+?)(?:\\n|$)/i) || \n text.match(/^(?!#|Title:|URL Source:)(.+?)(?:\\n\\n|\\n#)/m);\n if (descMatch) {\n description = descMatch[1].trim();\n }\n \n // Try to find og:image or image metadata\n const imageMatch = text.match(/og:image[:\\s]+(.+?)(?:\\n|$)/i) ||\n text.match(/image[:\\s]+(.+?)(?:\\n|$)/i);\n if (imageMatch) {\n image = imageMatch[1].trim();\n }\n \n return { title, description, image };\n};\n\nconst cleanJinaContent = (text) => {\n // Remove Jina's metadata lines like \"Title:\", \"URL Source:\", \"Markdown Content:\"\n let cleaned = text\n .replace(/^Title:\\s*.+$/gm, '')\n .replace(/^URL Source:\\s*.+$/gm, '')\n .replace(/^Markdown Content:\\s*$/gm, '')\n .replace(/^={10,}$/gm, '') // Remove separator lines\n .trim();\n \n // Remove extra blank lines\n cleaned = cleaned.replace(/\\n{3,}/g, '\\n\\n');\n \n return cleaned;\n};\n\nconst formatWithYamlHeader = (url, text) => {\n // Strip any existing frontmatter first\n let cleanedText = stripExistingFrontmatter(text);\n \n // Extract metadata before cleaning content\n const metadata = extractMetadata(cleanedText, url);\n \n // Clean up the content\n cleanedText = cleanJinaContent(cleanedText);\n \n return `---\ntitle: ${metadata.title || url}\ndescription: ${metadata.description || ''}\nimage: ${metadata.image || ''}\nurlSource: ${url}\n---\n\n${cleanedText}\n`;\n};\n\n\nconst urls = [\n \"https://console.groq.com/docs/quickstart\",\n \"https://console.groq.com/docs/openai\",\n \"https://console.groq.com/docs/models\",\n \"https://console.groq.com/docs/rate-limits\",\n \"https://console.groq.com/docs/text-chat\",\n \"https://console.groq.com/docs/speech-to-text\",\n \"https://console.groq.com/docs/text-to-speech\",\n \"https://console.groq.com/docs/reasoning\",\n \"https://console.groq.com/docs/vision\",\n \"https://console.groq.com/docs/agentic-tooling\",\n \"https://console.groq.com/docs/batch\",\n \"https://console.groq.com/docs/flex-processing\",\n \"https://console.groq.com/docs/content-moderation\",\n \"https://console.groq.com/docs/prefilling\",\n \"https://console.groq.com/docs/tool-use\",\n \"https://console.groq.com/docs/libraries\",\n];\nconst getPageContent = async () => {\n const promises = urls.map(async url => {\n const text = await getTextFromUrl(url);\n return formatWithYamlHeader(url, text);\n });\n const texts = await Promise.all(promises);\n return texts.join(\"\\n\\n\");\n};\n\n\n\n\napp.get(\"/\", async (c) => {\n const content = await getPageContent();\n return c.text(content);\n});\n\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;\n",[180],"9315b67c-b8f1-11f0-85fe-0224a6c84d84","groq.ts","9315b744-b8f1-11f0-85fe-0224a6c84d84",23,["D",1762200814266],"import { RateLimit } from \"npm:async-sema@latest\";\n\n// Groq API Configuration\nconst GROQ_API_KEY = Deno.env.get(\"GROQ_API_KEY\");\nconst GROQ_API_URL = \"https://api.groq.com/openai/v1/chat/completions\";\n\n// Rate limiter for Groq API: 2 requests per second to avoid rate limits\nconst groqApiRateLimiter = RateLimit(2);\n\n// Groq Chat Completions Utility\nexport const groqChatCompletion = async (messages: Array<{ role: string; content: string }>, options: {\n model?: string;\n temperature?: number;\n max_tokens?: number;\n response_format?: { type: string };\n} = {}) => {\n if (!GROQ_API_KEY) {\n console.warn(\"GROQ_API_KEY not found - AI metadata generation disabled\");\n return null;\n }\n\n // Rate limit Groq API calls\n await groqApiRateLimiter();\n\n try {\n const response = await fetch(GROQ_API_URL, {\n method: \"POST\",\n headers: {\n \"Authorization\": `Bearer ${GROQ_API_KEY}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n model: options.model || \"llama-3.3-70b-versatile\",\n messages,\n temperature: options.temperature ?? 0.4,\n // max_tokens: options.max_tokens ?? 1024,\n ...(options.response_format && { response_format: options.response_format }),\n }),\n });\n\n if (!response.ok) {\n const errorText = await response.text().catch(() => response.statusText);\n throw new Error(`Groq API error: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);\n }\n\n const data = await response.json();\n return data.choices[0]?.message?.content || null;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"Groq API call failed:\", errorMessage);\n // Don't throw - return null so metadata generation doesn't break the whole process\n return null;\n }\n};\n\n// Generate AI metadata for a page\nexport const generatePageMetadata = async (content: string, title: string) => {\n const prompt = `Analyze this documentation page and provide metadata in JSON format.\n\nPage Title: ${title}\n\nContent:\n${content.substring(0, 4000)} ${content.length > 4000 ? '...(truncated)' : ''}\n\nGenerate the following metadata:\n1. categories: Array of 2-4 broad categories (e.g., \"API\", \"Authentication\", \"Models\", \"Integration\")\n2. tags: Array of 5-10 specific tags/keywords\n3. useCases: Array of 2-4 practical use cases or scenarios when this page would be helpful\n4. questions: Array of 5-10 questions users might ask that this page answers\n\nReturn ONLY valid JSON in this exact format:\n{\n \"categories\": [\"Category1\", \"Category2\"],\n \"tags\": [\"tag1\", \"tag2\", \"tag3\"],\n \"useCases\": [\"Use case 1\", \"Use case 2\"],\n \"questions\": [\"Question 1?\", \"Question 2?\"]\n}`;\n\n const response = await groqChatCompletion([\n { role: \"user\", content: prompt }\n ], {\n model: \"llama-3.3-70b-versatile\",\n temperature: 0.3,\n max_tokens: 1024,\n });\n\n if (!response) {\n return null;\n }\n\n try {\n // Extract JSON from response (in case it's wrapped in markdown)\n const jsonMatch = response.match(/\\{[\\s\\S]*\\}/);\n if (jsonMatch) {\n return JSON.parse(jsonMatch[0]);\n }\n return JSON.parse(response);\n } catch (error) {\n console.error(\"Failed to parse AI metadata:\", error);\n return null;\n }\n};\n\n",[187],"7ecd5b6e-b8ee-11f0-8d74-0224a6c84d84","deno.json","656e5404-b8e7-11f0-a061-0224a6c84d84",17,["D",1762199491670],1,"{\n \"tasks\": {\n \"serve\": \"deno serve --allow-net --allow-import --allow-read --allow-write --allow-env main.tsx\"\n },\n \"$schema\": \"https://raw.githubusercontent.com/denoland/deno/98f62cee78e85bfc47c62ed703777c6bc8794f1c/cli/schemas/config-file.v1.json\",\n \"lock\": false,\n \"compilerOptions\": {\n \"noImplicitAny\": false,\n \"strict\": true,\n \"types\": [\"https://www.val.town/types/valtown.d.ts\"],\n \"lib\": [\n \"dom\",\n \"dom.iterable\",\n \"dom.asynciterable\",\n \"deno.ns\",\n \"deno.unstable\"\n ]\n },\n \"lint\": {\n \"include\": [\"deno:/https/esm.town/**/*\"],\n \"rules\": {\n \"exclude\": [\"no-explicit-any\"]\n }\n },\n \"node_modules_dir\": false,\n \"experimental\": {\n \"unstable-node-globals\": true,\n \"unstable-temporal\": true,\n \"unstable-worker-options\": true,\n \"unstable-sloppy-imports\": true\n }\n}",[194],"936414fc-b8f1-11f0-92bd-0224a6c84d84","README.md","6566e58e-b8e7-11f0-9e62-0224a6c84d84",25,["D",1762200814124],2,"# Groq Docs API\n\nA Hono API server that fetches, caches, and processes Groq documentation pages with token counting and AI-generated metadata.\n\n## Features\n\n- Fetches documentation pages from Groq's console\n- Caches page content, metadata, token counts, and embeddings in SQLite\n- Token counting using tiktoken (GPT-4 encoding)\n- AI-generated metadata (categories, tags, use cases, sample questions)\n- Content embeddings generation (currently fake, ready for Groq API integration)\n- Hash-based change detection to skip unchanged pages during recalculation\n- Rate limiting with async-sema to avoid WAF blocking\n- RESTful API endpoints for accessing pages and managing cache\n- Modular code structure (utils.ts for utilities, groq.ts for Groq API functions)\n\n## First-Time Setup\n\n### 1. Initial Cache Population\n\nOn first run, the cache will be empty. You should populate it by running:\n\n```bash\nGET /cache/recalculate\n```\n\nThis will:\n- Fetch all pages from the URLs list\n- Calculate token counts for each page\n- Generate AI metadata (categories, tags, use cases, questions)\n- Generate embeddings for each page\n- Calculate content hashes for change detection\n- Store everything in the SQLite cache\n- Return a summary of what was cached\n\n**Important**: This will take some time as it processes all pages, generates metadata, and calculates tokens for each. Be patient!\n\n**Note**: On subsequent runs, unchanged pages (detected by content hash) will be automatically skipped unless you use force mode.\n\n### 2. Verify Cache\n\nCheck that the cache was populated:\n\n```bash\nGET /cache/stats\n```\n\nThis returns:\n```json\n{\n \"cachedPages\": 121,\n \"totalTokens\": 1234567\n}\n```\n\n## When to Recalculate\n\nYou should run `/cache/recalculate` in these scenarios:\n\n### ✅ Required Recalculations\n\n1. **First time setup** - Cache is empty\n2. **URL list changes** - You've added or removed URLs from the `urls` array\n3. **Content updates** - Documentation pages have been updated and you want fresh data\n4. **Token count needed** - You need accurate token counts for new content\n5. **Metadata refresh** - You want to regenerate AI metadata or embeddings\n\n### 🔄 Default Mode (Smart Recalculation)\n\nBy default, `/cache/recalculate` uses hash-based change detection:\n\n```bash\nGET /cache/recalculate\n```\n\n**Behavior:**\n- Fetches each page and calculates its content hash (SHA-256)\n- Compares hash with cached version\n- **Skips pages with unchanged content** (saves time and API calls)\n- Only processes pages that have changed\n- Still generates embeddings and metadata for changed pages\n\n**Response includes:**\n- `processed` - Number of pages actually processed\n- `skipped` - Number of pages skipped (unchanged)\n- `force` - Always `false` in default mode\n\n### ⚡ Force Mode (Recalculate Everything)\n\nTo force recalculation of all pages (ignoring hash checks):\n\n```bash\nGET /cache/recalculate?force=true\n```\n\n**Use cases:**\n- Regenerating all metadata/embeddings even if content unchanged\n- After updating metadata generation prompts\n- When you want to ensure everything is fresh\n\n### ⚠️ Partial Updates\n\nFor single page updates, you can use:\n\n```bash\nGET /cache/clear/:path\n```\n\nThis clears the cache for a specific page. The next time that page is requested via `/page/:path`, it will be fetched fresh and recached.\n\n### 🔄 Routine Maintenance\n\n- **Weekly**: Run recalculate (default mode) to catch any documentation updates efficiently\n- **After major docs changes**: Use force mode to regenerate everything\n- **When adding new pages**: Update the `urls` array, then run recalculate\n\n## API Endpoints\n\n### Page Endpoints\n\n#### `GET /page/docs`\nGet the root docs page (cached if available).\n\n#### `GET /page/:path`\nGet a specific page by path. Examples:\n- `/page/api-reference`\n- `/page/agentic-tooling/compound-beta`\n- `/page/model/llama-3.1-8b-instant`\n\n**Response includes:**\n- `url` - The source URL\n- `content` - Full page content with frontmatter\n- `charCount` - Character count\n- `tokenCount` - Token count (calculated with tiktoken)\n- All frontmatter fields flattened (title, description, image, etc.)\n\n**Caching**: Responses are cached. First request fetches and caches, subsequent requests are instant.\n\n#### `GET /list`\nGet a list of all available page paths.\n\n**Response:**\n```json\n[\n \"docs\",\n \"agentic-tooling\",\n \"api-reference\",\n ...\n]\n```\n\n#### `GET /search`\nSearch pages by query string.\n\n**Query Parameters:**\n- `q` (required) - Search query string\n- `limit` (optional) - Maximum number of results (default: 10)\n- `minScore` (optional) - Minimum score threshold (default: 0)\n\n**Example:**\n```bash\nGET /search?q=authentication&limit=5\n```\n\n**Response:**\n```json\n{\n \"query\": \"authentication\",\n \"results\": [\n {\n \"path\": \"api-reference\",\n \"url\": \"https://console.groq.com/docs/api-reference.md\",\n \"title\": \"API Reference\",\n \"score\": 45,\n \"snippet\": \"...authentication tokens are required for all API requests...\"\n },\n {\n \"path\": \"quickstart\",\n \"url\": \"https://console.groq.com/docs/quickstart.md\",\n \"title\": \"Quick Start\",\n \"score\": 32,\n \"snippet\": \"...get your API key for authentication...\"\n }\n ],\n \"totalResults\": 2,\n \"totalPages\": 121\n}\n```\n\n**Search Features:**\n- Keyword matching in titles and content\n- Metadata boost (tags, categories, use cases)\n- Score-based ranking\n- Content snippets around matches\n- Uses cached pages when available for faster results\n\n**Note**: Currently uses keyword-based search. Future versions will use embeddings for semantic search.\n\n#### `GET /data`\nGet metadata for all pages (does not use cache - fetches fresh).\n\n**Response:**\n```json\n{\n \"pages\": [\n {\n \"url\": \"...\",\n \"charCount\": 1234,\n \"frontmatter\": {...}\n }\n ],\n \"contents\": [...],\n \"totalPages\": 121,\n \"totalChars\": 1234567\n}\n```\n\n### Cache Management Endpoints\n\n#### `GET /cache/stats`\nGet cache statistics.\n\n**Response:**\n```json\n{\n \"cachedPages\": 121,\n \"totalTokens\": 1234567\n}\n```\n\n#### `GET /cache/clear`\nClear the entire cache.\n\n**Response:**\n```json\n{\n \"message\": \"Cache cleared\",\n \"success\": true\n}\n```\n\n#### `GET /cache/clear/:path`\nClear cache for a specific page.\n\n**Example:**\n```bash\nGET /cache/clear/api-reference\n```\n\n**Response:**\n```json\n{\n \"message\": \"Cache cleared for api-reference\",\n \"success\": true\n}\n```\n\n#### `GET /cache/recalculate`\nRecalculate pages with AI metadata and embeddings generation.\n\n**Query Parameters:**\n- `force` (optional): Set to `true` to force recalculation of all pages, ignoring hash checks\n\n**Default Mode** (no query params):\n```bash\nGET /cache/recalculate\n```\n\n**Force Mode**:\n```bash\nGET /cache/recalculate?force=true\n```\n\n**Response** (Default Mode):\n```json\n{\n \"message\": \"Recalculated 5 pages, skipped 116 unchanged pages\",\n \"results\": [\n {\n \"path\": \"api-reference\",\n \"url\": \"https://console.groq.com/docs/api-reference.md\",\n \"charCount\": 1234,\n \"tokenCount\": 567,\n \"title\": \"API Reference\",\n \"metadata\": {\n \"categories\": [\"API\", \"Reference\"],\n \"tags\": [\"api\", \"endpoints\", \"rest\"],\n \"useCases\": [\"Integrating with Groq API\"],\n \"questions\": [\"How do I authenticate?\", \"What endpoints are available?\"]\n }\n },\n {\n \"path\": \"docs\",\n \"skipped\": true,\n \"reason\": \"Content unchanged (hash matches)\"\n }\n ],\n \"totalPages\": 121,\n \"processed\": 5,\n \"skipped\": 116,\n \"withMetadata\": 5,\n \"withoutMetadata\": 0,\n \"cached\": true,\n \"force\": false\n}\n```\n\n**Response** (Force Mode):\n```json\n{\n \"message\": \"Recalculated 121 pages with AI metadata (force mode)\",\n \"results\": [...],\n \"totalPages\": 121,\n \"processed\": 121,\n \"skipped\": 0,\n \"force\": true\n}\n```\n\n**What it does:**\n- Fetches all pages (or skips unchanged ones in default mode)\n- Calculates token counts\n- Generates AI metadata (categories, tags, use cases, questions)\n- Generates embeddings (currently fake, ready for Groq API)\n- Calculates content hashes for change detection\n- Stores everything in cache\n\n**Important**: This can take several minutes depending on:\n- Number of pages to process (skipped pages are fast)\n- Network speed\n- Token calculation time\n- AI metadata generation time (uses Groq API)\n\n## Cache Behavior\n\n### How Caching Works\n\n1. **First Request**: \n - Check cache → Not found\n - Fetch from URL\n - Calculate tokens\n - Store in cache\n - Return data\n\n2. **Subsequent Requests**:\n - Check cache → Found\n - Return cached data immediately\n\n### Cache Storage\n\nCache is stored in SQLite with the following schema:\n\n```sql\nCREATE TABLE groq_docs_cache_v3 (\n url TEXT PRIMARY KEY,\n content TEXT NOT NULL,\n charCount INTEGER NOT NULL,\n tokenCount INTEGER,\n frontmatter TEXT NOT NULL,\n metadata TEXT,\n contentHash TEXT,\n embeddings TEXT,\n cachedAt INTEGER NOT NULL\n)\n```\n\n**Fields:**\n- `url` - Source URL (primary key)\n- `content` - Full page content with frontmatter\n- `charCount` - Character count\n- `tokenCount` - Token count (calculated with tiktoken)\n- `frontmatter` - Parsed frontmatter (JSON)\n- `metadata` - AI-generated metadata (categories, tags, use cases, questions)\n- `contentHash` - SHA-256 hash of content (for change detection)\n- `embeddings` - Content embeddings vector (JSON array)\n- `cachedAt` - Timestamp when cached\n\n### Cache Invalidation\n\nCache is invalidated when:\n- You manually clear it via `/cache/clear`\n- You recalculate via `/cache/recalculate`\n- Cache is cleared for a specific page via `/cache/clear/:path`\n\n**Note**: Cache does NOT automatically expire. If documentation changes, you must manually recalculate.\n\n## Adding New Pages\n\n1. **Add URL to the `urls` array** in `main.tsx`:\n ```typescript\n const urls = [\n // ... existing URLs\n \"https://console.groq.com/docs/new-page.md\",\n ];\n ```\n\n2. **Run recalculate**:\n ```bash\n POST /cache/recalculate\n ```\n\n3. **Verify**:\n ```bash\n GET /cache/stats\n GET /list # Should include your new page\n ```\n\n## Token Counting\n\nToken counts are calculated using [tiktoken](https://github.com/openai/tiktoken) with the `gpt-4` encoding (cl100k_base). This is the same encoding used by:\n- GPT-4\n- GPT-3.5-turbo\n- Many other OpenAI models\n\nToken counts are:\n- Calculated on first fetch\n- Stored in cache\n- Returned in API responses\n- Expensive to compute (which is why caching is important)\n\n## AI Metadata Generation\n\nEach page can have AI-generated metadata using Groq's chat completions API:\n\n- **Categories**: 2-4 broad categories (e.g., \"API\", \"Authentication\", \"Models\")\n- **Tags**: 5-10 specific tags/keywords\n- **Use Cases**: 2-4 practical use cases or scenarios\n- **Questions**: 5-10 questions users might ask\n\nMetadata is generated during `/cache/recalculate` and stored in the cache.\n\n## Search\n\nThe API includes a search endpoint (`/search`) that allows you to search across all documentation pages.\n\n### Current Implementation (Keyword-Based)\n\nCurrently uses keyword matching:\n- Searches in page titles and content\n- Boosts results matching metadata (tags, categories, use cases)\n- Returns ranked results with relevance scores\n- Includes content snippets around matches\n\n### Future Implementation (Semantic Search)\n\nThe search system is designed to support embeddings-based semantic search:\n- `generateEmbeddings()` - Generates embeddings (currently fake, ready for real API)\n- `vectorSearch()` - Vector similarity search function (ready to use when embeddings are real)\n- Will enable semantic understanding of queries (not just keyword matching)\n\n## Embeddings\n\nContent embeddings are generated for each page. Currently using a fake implementation (deterministic 384-dimensional vectors) that's ready to be replaced with actual embeddings API when available.\n\nEmbeddings are:\n- Generated during recalculation\n- Stored in cache\n- Will be used for semantic search and similarity matching (currently using keyword search)\n\n## Hash-Based Change Detection\n\nContent hashes (SHA-256) are calculated and stored for each page. This enables:\n\n- **Smart recalculation**: Skip unchanged pages automatically\n- **Efficient updates**: Only process pages that have actually changed\n- **Performance**: Significantly faster recalculation when most content is unchanged\n\nHashes are compared during `/cache/recalculate` (default mode) to determine if a page needs reprocessing.\n\n## Troubleshooting\n\n### Cache seems stale\nRun `/cache/recalculate` to refresh everything.\n\n### Page not found\n1. Check `/list` to see if the path exists\n2. Verify the URL is in the `urls` array\n3. Ensure the path matches the URL structure (e.g., `api-reference` for `/docs/api-reference.md`)\n\n### Token counts seem wrong\n1. Clear cache for that page: `POST /cache/clear/:path`\n2. Request the page again: `GET /page/:path`\n3. Or recalculate everything: `POST /cache/recalculate`\n\n### Performance issues\n- Use `/page/:path` endpoints (cached) instead of `/data` (uncached)\n- Check cache stats: `GET /cache/stats`\n- Ensure cache is populated before production use\n\n## Code Structure\n\nThe codebase is organized into modular files:\n\n- **`main.tsx`** - Main Hono app, routes, and URL definitions\n- **`utils.ts`** - Utility functions:\n - Cache management (getFromCache, setCache, clearCache, getCacheStats)\n - Content fetching (getTextFromUrl)\n - Frontmatter parsing (parseFrontmatter, addUrlSourceToFrontmatter)\n - Token counting (calculateTokenCount)\n - Hash calculation (calculateContentHash)\n - Rate limiting for fetches\n- **`groq.ts`** - Groq API functions:\n - Chat completions (groqChatCompletion)\n - Metadata generation (generatePageMetadata)\n- **`search.ts`** - Search and embeddings utilities:\n - Embeddings generation (generateEmbeddings) - fake implementation ready for real API\n - Search functions (searchPages) - keyword-based search (will use embeddings later)\n - Vector similarity search (vectorSearch) - ready for embeddings-based search\n\n## Development\n\n### Local Development\n\n```bash\ndeno run --allow-net --allow-env main.tsx\n```\n\n**Note**: SQLite caching is automatically disabled when running locally (detected via `valtown` environment variable). The app will work without caching, but cache-related endpoints will return appropriate messages.\n\n### Val Town\n\nThe app is configured to work with Val Town. Export uses:\n```typescript\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;\n```\n\nSQLite caching is automatically enabled when running in Val Town (detected via `valtown` environment variable).\n\n### Environment Variables\n\n- `GROQ_API_KEY` - Required for AI metadata generation (optional, will disable metadata if not set)\n- `valtown` - Automatically set by Val Town (detects environment)\n\n## Performance Tips\n\n1. **Use default recalculate mode** - Automatically skips unchanged pages\n2. **Cache is your friend** - Always populate cache before production use\n3. **Rate limiting** - Built-in rate limiting prevents WAF blocking (1 request per 3 seconds for docs, 2 requests per second for Groq API)\n4. **Hash checking** - Default recalculation mode is much faster when most content is unchanged\n\n",[202],"6575fc7c-b8e7-11f0-ba05-0224a6c84d84","AGENTS.md","6575fd58-b8e7-11f0-ba05-0224a6c84d84",6,["D",1762196442671],"You are an advanced assistant specialized in generating Val Town code.\n\n## Core Guidelines\n\n- Ask clarifying questions when requirements are ambiguous\n- Provide complete, functional solutions rather than skeleton implementations\n- Test your logic against edge cases before presenting the final solution\n- Ensure all code follows Val Town's specific platform requirements\n- If a section of code that you're working on is getting too complex, consider refactoring it into subcomponents\n\n## Code Standards\n\n- Generate code in TypeScript or TSX\n- Add appropriate TypeScript types and interfaces for all data structures\n- Prefer official SDKs or libraries than writing API calls directly\n- Ask the user to supply API or library documentation if you are at all unsure about it\n- **Never bake in secrets into the code** - always use environment variables\n- Include comments explaining complex logic (avoid commenting obvious operations)\n- Follow modern ES6+ conventions and functional programming practices if possible\n\n## Types of triggers\n\n### 1. HTTP Trigger\n\n- Create web APIs and endpoints\n- Handle HTTP requests and responses\n- Example structure:\n\n```ts\nexport default async function (req: Request) {\n return new Response(\"Hello World\");\n}\n```\n\nFiles that are HTTP triggers have http in their name like `foobar.http.tsx`\n\n### 2. Cron Triggers\n\n- Run on a schedule\n- Use cron expressions for timing\n- Example structure:\n\n```ts\nexport default async function () {\n // Scheduled task code\n}\n```\n\nFiles that are Cron triggers have cron in their name like `foobar.cron.tsx`\n\n### 3. Email Triggers\n\n- Process incoming emails\n- Handle email-based workflows\n- Example structure:\n\n```ts\nexport default async function (email: Email) {\n // Process email\n}\n```\n\nFiles that are Email triggers have email in their name like `foobar.email.tsx`\n\n\n## Val Town Standard Libraries\n\nVal Town provides several hosted services and utility functions.\n\n### Blob Storage\n\n```ts\nimport { blob } from \"https://esm.town/v/std/blob\";\nawait blob.setJSON(\"myKey\", { hello: \"world\" });\nlet blobDemo = await blob.getJSON(\"myKey\");\nlet appKeys = await blob.list(\"app_\");\nawait blob.delete(\"myKey\");\n```\n\n### SQLite\n\n```ts\nimport { sqlite } from \"https://esm.town/v/stevekrouse/sqlite\";\nconst TABLE_NAME = 'todo_app_users_2';\n// Create table - do this before usage and change table name when modifying schema\nawait sqlite.execute(`CREATE TABLE IF NOT EXISTS ${TABLE_NAME} (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n name TEXT NOT NULL\n)`);\n// Query data\nconst result = await sqlite.execute(`SELECT * FROM ${TABLE_NAME} WHERE id = ?`, [1]);\n```\n\nNote: When changing a SQLite table's schema, change the table's name (e.g., add _2 or _3) to create a fresh table.\n\n### OpenAI\n\n```ts\nimport { OpenAI } from \"https://esm.town/v/std/openai\";\nconst openai = new OpenAI();\nconst completion = await openai.chat.completions.create({\n messages: [\n { role: \"user\", content: \"Say hello in a creative way\" },\n ],\n model: \"gpt-4o-mini\",\n max_tokens: 30,\n});\n```\n\n### Email\n\n```ts\nimport { email } from \"https://esm.town/v/std/email\";\n// By default emails the owner of the val\nawait email({ \n subject: \"Hi\", \n text: \"Hi\", \n html: \"

Hi

\"\n});\n```\n\n## Val Town Utility Functions\n\nVal Town provides several utility functions to help with common project tasks.\n\n### Importing Utilities\n\nAlways import utilities with version pins to avoid breaking changes:\n\n```ts\nimport { parseProject, readFile, serveFile } from \"https://esm.town/v/std/utils@85-main/index.ts\";\n```\n\n### Available Utilities\n\n\n#### **serveFile** - Serve project files with proper content types\n\nFor example, in Hono:\n\n```ts\n// serve all files in frontend/ and shared/\napp.get(\"/frontend/*\", c => serveFile(c.req.path, import.meta.url));\napp.get(\"/shared/*\", c => serveFile(c.req.path, import.meta.url));\n```\n\n#### **readFile** - Read files from within the project:\n\n```ts\n// Read a file from the project\nconst fileContent = await readFile(\"/frontend/index.html\", import.meta.url);\n```\n\n#### **listFiles** - List all files in the project\n\n```ts\nconst files = await listFiles(import.meta.url);\n```\n\n#### **parseProject** - Extract information about the current project from import.meta.url\n\nThis is useful for including info for linking back to a val, ie in \"view source\" urls:\n\n```ts\nconst projectVal = parseProject(import.meta.url);\nconsole.log(projectVal.username); // Owner of the project\nconsole.log(projectVal.name); // Project name\nconsole.log(projectVal.version); // Version number\nconsole.log(projectVal.branch); // Branch name\nconsole.log(projectVal.links.self.project); // URL to the project page\n```\n\nHowever, it's *extremely importing* to note that `parseProject` and other Standard Library utilities ONLY RUN ON THE SERVER.\nIf you need access to this data on the client, run it in the server and pass it to the client by splicing it into the HTML page\nor by making an API request for it.\n\n## Val Town Platform Specifics\n\n- **Redirects:** Use `return new Response(null, { status: 302, headers: { Location: \"/place/to/redirect\" }})` instead of `Response.redirect` which is broken\n- **Images:** Avoid external images or base64 images. Use emojis, unicode symbols, or icon fonts/libraries instead\n- **AI Image:** To inline generate an AI image use: `

`\n- **Storage:** DO NOT use the Deno KV module for storage\n- **Browser APIs:** DO NOT use the `alert()`, `prompt()`, or `confirm()` methods\n- **Weather Data:** Use open-meteo for weather data (doesn't require API keys) unless otherwise specified\n- **View Source:** Add a view source link by importing & using `import.meta.url.replace(\"ems.sh\", \"val.town)\"` (or passing this data to the client) and include `target=\"_top\"` attribute\n- **Error Debugging:** Add `` to HTML to capture client-side errors\n- **Error Handling:** Only use try...catch when there's a clear local resolution; Avoid catches that merely log or return 500s. Let errors bubble up with full context\n- **Environment Variables:** Use `Deno.env.get('keyname')` when you need to, but generally prefer APIs that don't require keys\n- **Imports:** Use `https://esm.sh` for npm and Deno dependencies to ensure compatibility on server and browser\n- **Storage Strategy:** Only use backend storage if explicitly required; prefer simple static client-side sites\n- **React Configuration:** When using React libraries, pin versions with `?deps=react@18.2.0,react-dom@18.2.0` and start the file with `/** @jsxImportSource https://esm.sh/react@18.2.0 */`\n- Ensure all React dependencies and sub-dependencies are pinned to the same version\n- **Styling:** Default to using TailwindCSS via `` unless otherwise specified\n\n## Project Structure and Design Patterns\n\n### Recommended Directory Structure\n```\n├── backend/\n│ ├── database/\n│ │ ├── migrations.ts # Schema definitions\n│ │ ├── queries.ts # DB query functions\n│ │ └── README.md\n│ └── routes/ # Route modules\n│ ├── [route].ts\n│ └── static.ts # Static file serving\n│ ├── index.ts # Main entry point\n│ └── README.md\n├── frontend/\n│ ├── components/\n│ │ ├── App.tsx\n│ │ └── [Component].tsx\n│ ├── favicon.svg\n│ ├── index.html # Main HTML template\n│ ├── index.tsx # Frontend JS entry point\n│ ├── README.md\n│ └── style.css\n├── README.md\n└── shared/\n ├── README.md\n └── utils.ts # Shared types and functions\n```\n\n### Backend (Hono) Best Practices\n\n- Hono is the recommended API framework\n- Main entry point should be `backend/index.ts`\n- **Static asset serving:** Use the utility functions to read and serve project files:\n ```ts\n import { readFile, serveFile } from \"https://esm.town/v/std/utils@85-main/index.ts\";\n \n // serve all files in frontend/ and shared/\n app.get(\"/frontend/*\", c => serveFile(c.req.path, import.meta.url));\n app.get(\"/shared/*\", c => serveFile(c.req.path, import.meta.url));\n \n // For index.html, often you'll want to bootstrap with initial data\n app.get(\"/\", async c => {\n let html = await readFile(\"/frontend/index.html\", import.meta.url);\n \n // Inject data to avoid extra round-trips\n const initialData = await fetchInitialData();\n const dataScript = ``;\n \n html = html.replace(\"\", `${dataScript}`);\n return c.html(html);\n });\n ```\n- Create RESTful API routes for CRUD operations\n- Always include this snippet at the top-level Hono app to re-throwing errors to see full stack traces:\n ```ts\n // Unwrap Hono errors to see original error details\n app.onError((err, c) => {\n throw err;\n });\n ```\n\n### Database Patterns\n- Run migrations on startup or comment out for performance\n- Change table names when modifying schemas rather than altering\n- Export clear query functions with proper TypeScript typing\n\n## Common Gotchas and Solutions\n\n1. **Environment Limitations:** \n - Val Town runs on Deno in a serverless context, not Node.js\n - Code in `shared/` must work in both frontend and backend environments\n - Cannot use `Deno` keyword in shared code\n - Use `https://esm.sh` for imports that work in both environments\n\n2. **SQLite Peculiarities:**\n - Limited support for ALTER TABLE operations\n - Create new tables with updated schemas and copy data when needed\n - Always run table creation before querying\n\n3. **React Configuration:**\n - All React dependencies must be pinned to 18.2.0\n - Always include `@jsxImportSource https://esm.sh/react@18.2.0` at the top of React files\n - Rendering issues often come from mismatched React versions\n\n4. **File Handling:**\n - Val Town only supports text files, not binary\n - Use the provided utilities to read files across branches and forks\n - For files in the project, use `readFile` helpers\n\n5. **API Design:**\n - `fetch` handler is the entry point for HTTP vals\n - Run the Hono app with `export default app.fetch // This is the entry point for HTTP vals`\n\n",[210],"65892a86-b8e7-11f0-ba05-0224a6c84d84",".vtignore","65892b4e-b8e7-11f0-ba05-0224a6c84d84",9,["D",1762196442809],".git\n.vscode\n.cursorrules\n.DS_Store\nnode_modules\nvendor",[217],"30dc0596-b901-11f0-a3b9-0224a6c84d84","testing","30dc0668-b901-11f0-a3b9-0224a6c84d84",28,["D",1762207521420],"directory",[224],"30c2711c-b901-11f0-b5de-0224a6c84d84","search","30c2722a-b901-11f0-b5de-0224a6c84d84",27,["D",1762207521254],[231],"31189d8a-b901-11f0-87ce-0224a6c84d84","data","31189e3e-b901-11f0-87ce-0224a6c84d84",29,["D",1762207521817],[237],"pulls","branches","forks","envVars",{"_249":61,"_251":61},"connections",{"_249":61,"_250":61},"count","unfilledCount","emptyCount",{"_42":236,"_31":237,"_29":238,"_142":-5,"_40":41,"_39":239,"_144":48,"_37":400,"_13":61,"_33":228,"_35":-5,"_23":24,"_147":-5,"_278":401,"_26":402},{"_42":230,"_31":231,"_29":232,"_142":-5,"_40":41,"_39":233,"_144":48,"_37":397,"_13":61,"_33":228,"_35":-5,"_23":24,"_147":-5,"_278":398,"_26":399},{"_42":389,"_31":390,"_29":391,"_142":232,"_40":41,"_39":392,"_144":24,"_37":393,"_13":61,"_33":34,"_35":394,"_23":24,"_147":-5,"_278":395,"_26":396},{"_42":381,"_31":382,"_29":383,"_142":232,"_40":41,"_39":384,"_144":48,"_37":385,"_13":61,"_33":34,"_35":386,"_23":24,"_147":-5,"_278":387,"_26":388},{"_42":373,"_31":374,"_29":375,"_142":232,"_40":41,"_39":376,"_144":48,"_37":377,"_13":61,"_33":34,"_35":378,"_23":24,"_147":-5,"_278":379,"_26":380},{"_42":365,"_31":366,"_29":367,"_142":232,"_40":41,"_39":368,"_144":48,"_37":369,"_13":61,"_33":34,"_35":370,"_23":24,"_147":-5,"_278":371,"_26":372},{"_42":357,"_31":358,"_29":359,"_142":232,"_40":41,"_39":360,"_144":48,"_37":361,"_13":61,"_33":34,"_35":362,"_23":24,"_147":-5,"_278":363,"_26":364},{"_42":349,"_31":350,"_29":351,"_142":232,"_40":41,"_39":352,"_144":24,"_37":353,"_13":61,"_33":34,"_35":354,"_23":24,"_147":-5,"_278":355,"_26":356},{"_42":341,"_31":342,"_29":343,"_142":232,"_40":41,"_39":344,"_144":48,"_37":345,"_13":61,"_33":34,"_35":346,"_23":24,"_147":-5,"_278":347,"_26":348},{"_42":334,"_31":140,"_29":335,"_142":232,"_40":41,"_39":336,"_144":24,"_37":337,"_13":61,"_33":34,"_35":338,"_23":24,"_147":-5,"_278":339,"_26":340},{"_42":223,"_31":224,"_29":225,"_142":-5,"_40":41,"_39":226,"_144":48,"_37":331,"_13":61,"_33":228,"_35":-5,"_23":24,"_147":-5,"_278":332,"_26":333},{"_42":327,"_31":32,"_29":30,"_142":225,"_40":41,"_39":14,"_144":24,"_37":328,"_13":61,"_33":34,"_35":36,"_23":24,"_147":-5,"_278":329,"_26":330},{"_42":319,"_31":320,"_29":321,"_142":225,"_40":41,"_39":322,"_144":24,"_37":323,"_13":61,"_33":34,"_35":324,"_23":24,"_147":-5,"_278":325,"_26":326},{"_42":311,"_31":312,"_29":313,"_142":225,"_40":41,"_39":314,"_144":48,"_37":315,"_13":61,"_33":34,"_35":316,"_23":24,"_147":-5,"_278":317,"_26":318},{"_42":216,"_31":217,"_29":218,"_142":-5,"_40":41,"_39":219,"_144":24,"_37":308,"_13":61,"_33":20,"_35":221,"_23":24,"_147":-5,"_278":309,"_26":310},{"_42":209,"_31":210,"_29":211,"_142":-5,"_40":41,"_39":212,"_144":48,"_37":305,"_13":61,"_33":20,"_35":214,"_23":24,"_147":-5,"_278":306,"_26":307},{"_42":201,"_31":202,"_29":203,"_142":-5,"_40":41,"_39":204,"_144":24,"_37":302,"_13":206,"_33":20,"_35":207,"_23":24,"_147":-5,"_278":303,"_26":304},{"_42":193,"_31":194,"_29":195,"_142":-5,"_40":41,"_39":196,"_144":24,"_37":299,"_13":198,"_33":20,"_35":199,"_23":24,"_147":-5,"_278":300,"_26":301},{"_42":186,"_31":187,"_29":188,"_142":-5,"_40":41,"_39":189,"_144":48,"_37":296,"_13":61,"_33":34,"_35":191,"_23":24,"_147":-5,"_278":297,"_26":298},{"_42":179,"_31":180,"_29":181,"_142":-5,"_40":41,"_39":182,"_144":24,"_37":293,"_13":61,"_33":34,"_35":184,"_23":24,"_147":-5,"_278":294,"_26":295},{"_42":170,"_31":171,"_29":56,"_142":-5,"_40":41,"_39":172,"_144":24,"_37":290,"_13":174,"_33":175,"_35":176,"_23":24,"_147":177,"_278":291,"_26":292},{"_42":163,"_31":164,"_29":165,"_142":-5,"_40":41,"_39":166,"_144":24,"_37":287,"_13":61,"_33":34,"_35":168,"_23":24,"_147":-5,"_278":288,"_26":289},{"_42":156,"_31":157,"_29":158,"_142":-5,"_40":41,"_39":159,"_144":24,"_37":284,"_13":61,"_33":20,"_35":161,"_23":24,"_147":-5,"_278":285,"_26":286},{"_42":149,"_31":150,"_29":151,"_142":-5,"_40":41,"_39":152,"_144":24,"_37":281,"_13":61,"_33":34,"_35":154,"_23":24,"_147":-5,"_278":282,"_26":283},{"_42":139,"_31":140,"_29":141,"_142":-5,"_40":41,"_39":143,"_144":48,"_37":277,"_13":61,"_33":34,"_35":146,"_23":24,"_147":-5,"_278":279,"_26":280},["D",1762200814128],"children",[],[140],["D",1762207522123],[],[150],["D",1762199491650],[],[157],["D",1762200814117],[],[164],["D",1762202046713],[],[171],["D",1762196442795],[],[180],["D",1762200814266],[],[187],["D",1762199491670],[],[194],["D",1762200814124],[],[202],["D",1762196442671],[],[210],["D",1762196442809],[],[217],"3193ad5e-b901-11f0-a764-0224a6c84d84","search-harness.ts","3193ae26-b901-11f0-a764-0224a6c84d84",39,["D",1762207522627],"// Test harness for search functionality\n// Reusable function for testing search queries\n\nimport { searchPages } from \"../search/index.ts\";\nimport type { Page, SearchResult, SearchOptions } from \"../search/types.ts\";\n\nexport interface SearchTestResult {\n results: SearchResult[];\n durationMs: number;\n error?: string;\n}\n\n/**\n * Test harness function for running a single search query\n * @param query - The search query string\n * @param pages - Array of pages to search through\n * @param options - Search options (limit, minScore, etc.)\n * @returns Promise with results and timing information\n */\nexport async function runSearchTest(\n query: string,\n pages: Page[],\n options: SearchOptions = {}\n): Promise {\n const startTime = performance.now();\n \n try {\n const results = await searchPages(query, pages, options);\n const endTime = performance.now();\n const durationMs = endTime - startTime;\n \n return {\n results,\n durationMs,\n };\n } catch (error) {\n const endTime = performance.now();\n const durationMs = endTime - startTime;\n \n return {\n results: [],\n durationMs,\n error: error instanceof Error ? error.message : String(error),\n };\n }\n}\n\n",[],[224,312],"319c6fde-b901-11f0-a8ca-0224a6c84d84","recalculate.ts","319c70b0-b901-11f0-a8ca-0224a6c84d84",40,["D",1762207522645],"// Recalculate script - runs cache recalculation and saves to /data/latest.json and /data/mm-dd-yyyy.json\n// Run with: deno task recalculate\n\nimport \"jsr:@std/dotenv/load\";\nimport { urls } from \"../urls.ts\";\nimport {\n IS_VALTOWN,\n getFromCache,\n setCache,\n calculateContentHash,\n getTextFromUrl,\n parseFrontmatter,\n addUrlSourceToFrontmatter,\n calculateTokenCount,\n} from \"../utils.ts\";\nimport { generatePageMetadata } from \"../groq.ts\";\nimport { getActiveEmbeddingsFunction } from \"../search/index.ts\";\nimport { saveTestData } from \"../main.tsx\";\n\n// Load cached data from latest.json (for local hash checking)\nasync function loadLatestJson(): Promise | null> {\n if (IS_VALTOWN) {\n return null; // Use SQLite when in Val Town\n }\n \n try {\n const latestData = await Deno.readTextFile(\"./data/latest.json\");\n const data = JSON.parse(latestData);\n \n // Create a map of url -> cached data\n const cacheMap: Record = {};\n \n if (data.results && Array.isArray(data.results)) {\n for (const result of data.results) {\n if (result.url) {\n cacheMap[result.url] = {\n contentHash: result.contentHash,\n charCount: result.charCount,\n tokenCount: result.tokenCount,\n title: result.title,\n metadata: result.metadata,\n embeddings: result.embeddings || null,\n };\n }\n }\n }\n \n return cacheMap;\n } catch {\n // latest.json doesn't exist or is invalid - that's okay, just return null\n return null;\n }\n}\n\nasync function recalculate(force: boolean = false) {\n console.log(`Starting recalculation (force: ${force}, local: ${!IS_VALTOWN})...`);\n \n // Load latest.json for hash checking when running locally\n const latestJsonData = await loadLatestJson();\n if (latestJsonData && !force) {\n console.log(`Loaded ${Object.keys(latestJsonData).length} cached hashes from latest.json`);\n }\n \n const results: Array<{ \n path: string; \n url: string; \n charCount: number; \n tokenCount: number | undefined;\n title?: string;\n metadata?: any;\n contentHash?: string;\n embeddings?: number[] | null;\n error?: string;\n cacheError?: string;\n skipped?: boolean;\n reason?: string;\n }> = [];\n \n let processedCount = 0;\n let skippedCount = 0;\n \n const getPageByPath = async (pathParam: string, useCache: boolean = true) => {\n // Find the matching URL\n const matchingUrl = urls.find(url => {\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n if (match) {\n return match[1] === pathParam;\n }\n if (url.endsWith(\"/docs.md\") && pathParam === \"docs\") {\n return true;\n }\n return false;\n });\n \n if (!matchingUrl) {\n return null;\n }\n \n // Try to get from cache first\n if (useCache) {\n const cached = await getFromCache(matchingUrl);\n if (cached) {\n return {\n url: cached.frontmatter?.urlSource || matchingUrl,\n content: cached.content,\n charCount: cached.charCount,\n tokenCount: cached.tokenCount,\n };\n }\n }\n \n // Fetch and process the single page\n const text = await getTextFromUrl(matchingUrl);\n const processedText = addUrlSourceToFrontmatter(text, matchingUrl);\n const charCount = processedText.length;\n const tokenCount = calculateTokenCount(processedText);\n \n return {\n url: matchingUrl,\n content: processedText,\n charCount,\n tokenCount,\n };\n };\n \n for (const url of urls) {\n // Extract path from URL\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n const path = match ? match[1] : (url.endsWith(\"/docs.md\") ? \"docs\" : null);\n \n if (path) {\n console.log(`[${processedCount + 1}/${urls.length}] Processing ${path}...`);\n \n try {\n // Check cached hash if not forcing\n let shouldSkip = false;\n if (!force) {\n // Try SQLite cache first (if in Val Town)\n let cached = await getFromCache(url);\n \n // If no SQLite cache, try latest.json (for local development)\n let cachedHash: string | null = null;\n let cachedFromJson: any = null;\n \n if (cached) {\n cachedHash = cached.contentHash;\n } else if (latestJsonData && latestJsonData[url]) {\n cachedFromJson = latestJsonData[url];\n cachedHash = cachedFromJson.contentHash || null;\n }\n \n if (cachedHash) {\n // Fetch fresh content to check hash\n try {\n const pageData = await getPageByPath(path, false);\n if (pageData) {\n const newHash = await calculateContentHash(pageData.content);\n if (newHash === cachedHash) {\n shouldSkip = true;\n skippedCount++;\n console.log(` Skipped ${path} (content unchanged, hash: ${newHash.substring(0, 8)}...)`);\n \n // Use cached data from SQLite or latest.json\n if (cached) {\n results.push({\n path,\n url,\n charCount: cached.charCount,\n tokenCount: cached.tokenCount || undefined,\n title: (cached.frontmatter as any).title || path,\n metadata: cached.metadata || undefined,\n contentHash: cachedHash,\n embeddings: cached.embeddings || null,\n skipped: true,\n reason: \"Content unchanged (hash matches)\",\n });\n } else if (cachedFromJson) {\n results.push({\n path,\n url,\n charCount: cachedFromJson.charCount || 0,\n tokenCount: cachedFromJson.tokenCount || undefined,\n title: cachedFromJson.title || path,\n metadata: cachedFromJson.metadata || undefined,\n contentHash: cachedHash,\n embeddings: cachedFromJson.embeddings || null,\n skipped: true,\n reason: \"Content unchanged (hash matches)\",\n });\n }\n }\n }\n } catch (hashCheckError) {\n // If hash check fails, continue with processing\n console.debug(` Hash check failed for ${path}, continuing...`);\n }\n }\n }\n \n if (shouldSkip) {\n processedCount++;\n continue;\n }\n \n // Fetch without cache\n let pageData;\n try {\n pageData = await getPageByPath(path, false);\n } catch (fetchError) {\n const fetchErrorMessage = fetchError instanceof Error ? fetchError.message : String(fetchError);\n console.error(` Failed to fetch ${path}:`, fetchErrorMessage);\n throw new Error(`Fetch failed: ${fetchErrorMessage}`);\n }\n \n if (pageData) {\n const frontmatter = parseFrontmatter(pageData.content);\n const title = (frontmatter as any).title || path;\n \n // Calculate content hash\n const contentHash = await calculateContentHash(pageData.content);\n \n // Generate AI metadata\n const metadata = await generatePageMetadata(pageData.content, title);\n if (!metadata) {\n console.warn(` Metadata generation failed for ${path} (continuing without metadata)`);\n }\n \n // Generate embeddings using active strategy's embeddings function\n const generateEmbeddings = getActiveEmbeddingsFunction();\n let embeddings: number[] | null = null;\n let embeddingsError: string | undefined = undefined;\n \n try {\n embeddings = await generateEmbeddings(pageData.content);\n if (!embeddings) {\n embeddingsError = \"Embeddings generation returned null\";\n console.error(` ❌ Embeddings generation failed for ${path}: ${embeddingsError}`);\n }\n } catch (embedError) {\n const embedErrorMessage = embedError instanceof Error ? embedError.message : String(embedError);\n embeddingsError = `Embeddings generation threw error: ${embedErrorMessage}`;\n console.error(` ❌ Embeddings generation failed for ${path}: ${embeddingsError}`);\n // Don't throw - continue with null embeddings, but log the error\n }\n \n // Cache it with metadata, hash, and embeddings - track errors\n const cacheError = await setCache(url, {\n content: pageData.content,\n charCount: pageData.charCount,\n tokenCount: pageData.tokenCount || null,\n frontmatter,\n metadata,\n contentHash,\n embeddings,\n });\n \n results.push({ \n path, \n url, \n charCount: pageData.charCount, \n tokenCount: pageData.tokenCount,\n title,\n metadata,\n contentHash,\n embeddings: embeddings || null,\n ...(cacheError && { cacheError }),\n ...(embeddingsError && { embeddingsError }),\n });\n \n const statusParts = [\n `metadata: ${metadata ? 'yes' : 'no'}`,\n `embeddings: ${embeddings ? 'yes' : 'no'}`,\n ];\n \n if (embeddingsError) {\n statusParts.push(`embeddings error: ${embeddingsError.substring(0, 50)}...`);\n }\n if (cacheError) {\n statusParts.push(`cache error: ${cacheError}`);\n }\n \n const status = statusParts.join(', ');\n if (embeddingsError || cacheError) {\n console.warn(`Completed ${path} (${status})`);\n } else {\n console.log(`Completed ${path} (${status}, cached: yes)`);\n }\n }\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(`Error processing ${path}:`, errorMessage);\n results.push({\n path,\n url,\n charCount: 0,\n tokenCount: 0,\n error: errorMessage,\n });\n }\n \n processedCount++;\n }\n }\n \n const cacheErrors = results.filter(r => r.cacheError);\n const processedResults = results.filter(r => !r.skipped);\n \n const summary = {\n message: force \n ? `Recalculated ${processedResults.length} pages with AI metadata (force mode)` \n : `Recalculated ${processedResults.length} pages, skipped ${skippedCount} unchanged pages`,\n timestamp: new Date().toISOString(),\n results,\n totalPages: results.length,\n processed: processedResults.length,\n skipped: skippedCount,\n withMetadata: processedResults.filter(r => r.metadata).length,\n withoutMetadata: processedResults.filter(r => !r.metadata).length,\n cached: processedResults.length > 0 && cacheErrors.length === 0,\n cacheErrors: cacheErrors.length > 0 ? cacheErrors.map(r => ({ path: r.path, error: r.cacheError })) : undefined,\n force,\n };\n \n // Save to both latest.json and dated file\n await saveTestData(summary);\n console.log(\"\\nRecalculation completed!\");\n \n return summary;\n}\n\n// Run if executed directly\nif (import.meta.main) {\n const force = Deno.args.includes(\"--force\") || Deno.args.includes(\"-f\");\n recalculate(force).catch(console.error);\n}\n\n",[],[224,320],"31a38e68-b901-11f0-a3b9-0224a6c84d84",["D",1762207522633],[],[224,32],["D",1762207521420],[263,264,265],[224],"31528cd4-b901-11f0-a3b9-0224a6c84d84","31528d88-b901-11f0-a3b9-0224a6c84d84",32,["D",1762207522155],"// Shared utilities for search strategies\n\n// Calculate cosine similarity between two embedding vectors\nexport const cosineSimilarity = (vecA: number[], vecB: number[]): number => {\n if (vecA.length !== vecB.length) {\n return 0;\n }\n \n let dotProduct = 0;\n let normA = 0;\n let normB = 0;\n \n for (let i = 0; i < vecA.length; i++) {\n dotProduct += vecA[i] * vecB[i];\n normA += vecA[i] * vecA[i];\n normB += vecB[i] * vecB[i];\n }\n \n if (normA === 0 || normB === 0) {\n return 0;\n }\n \n return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));\n};\n\n// Helper function to generate snippets\nexport const generateSnippet = (content: string, queryWords: string[], queryLower: string): string => {\n const contentLower = content.toLowerCase();\n const firstMatchIndex = contentLower.indexOf(queryWords[0]);\n \n if (firstMatchIndex >= 0) {\n const start = Math.max(0, firstMatchIndex - 50);\n const end = Math.min(content.length, firstMatchIndex + queryWords[0].length + 150);\n let snippet = content.substring(start, end);\n if (start > 0) snippet = '...' + snippet;\n if (end < content.length) snippet = snippet + '...';\n return snippet;\n }\n \n return content.substring(0, 200) + '...';\n};\n\n",[],[231,140],"316a968a-b901-11f0-9996-0224a6c84d84","types.ts","316a973e-b901-11f0-9996-0224a6c84d84",34,["D",1762207522301],"// Search strategy types and interfaces\n\nexport type SearchResult = {\n path: string;\n url: string;\n title?: string;\n score: number;\n snippet?: string;\n metadata?: {\n strategy?: string;\n [key: string]: any;\n };\n};\n\nexport type Page = {\n path: string;\n url: string;\n content: string;\n title?: string;\n embeddings?: number[] | null;\n metadata?: any;\n};\n\nexport type SearchOptions = {\n limit?: number;\n minScore?: number;\n [key: string]: any;\n};\n\nexport type SearchStrategy = {\n name: string;\n description: string;\n search: (query: string, pages: Page[], options: SearchOptions) => Promise;\n};\n\n",[],[231,342],"31884234-b901-11f0-b784-0224a6c84d84","transformers-cosine.ts","31884324-b901-11f0-b784-0224a6c84d84",38,["D",1762207522387],"// Transformers.js + Cosine Similarity Strategy: Local embeddings in browser/Deno\n// Fastest option - no API calls, runs entirely client-side\n// Model: all-MiniLM-L6-v2 (80MB, ~384 dimensions, very fast)\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { generateSnippet, cosineSimilarity } from \"./utils.ts\";\n\n// Lazy-loaded pipeline (cached after first load)\nlet embeddingPipeline: any = null;\nlet pipelineLoading: Promise | null = null;\n\nasync function getEmbeddingPipeline() {\n // Return cached pipeline if already loaded\n if (embeddingPipeline) {\n return embeddingPipeline;\n }\n \n // If loading in progress, wait for it\n if (pipelineLoading) {\n return pipelineLoading;\n }\n \n // Start loading pipeline\n pipelineLoading = (async () => {\n try {\n // Import transformers.js - use npm: prefix for Deno compatibility\n // NOTE: transformers.js is primarily designed for browsers. In Deno, it may require --allow-ffi\n // For browser usage, this will work seamlessly. For Deno, you may need additional permissions.\n const { pipeline } = await import(\"npm:@huggingface/transformers@latest\");\n \n // Load feature extraction pipeline with all-MiniLM-L6-v2 model\n // This is a small, fast model optimized for embeddings (80MB, ~384 dims)\n embeddingPipeline = await pipeline(\n 'feature-extraction',\n 'Xenova/all-MiniLM-L6-v2',\n {\n // Use CPU - WebGPU might not be available in all environments\n device: 'cpu',\n dtype: 'fp32',\n }\n );\n \n return embeddingPipeline;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n const fullError = `Failed to load transformers.js pipeline: ${errorMessage}\\n` +\n `NOTE: transformers.js is primarily designed for browser environments. ` +\n `If running in Deno, you may need --allow-ffi permissions, or use a different embedding strategy.`;\n console.error(fullError);\n throw new Error(fullError);\n } finally {\n pipelineLoading = null;\n }\n })();\n \n return pipelineLoading;\n}\n\n// Generate embeddings using transformers.js (runs locally, no API calls)\nexport const generateEmbeddings = async (content: string): Promise => {\n try {\n const pipeline = await getEmbeddingPipeline();\n \n // Generate embeddings - returns a tensor, we need to extract the array\n const output = await pipeline(content, {\n pooling: 'mean',\n normalize: true,\n });\n \n // Extract the embedding vector from the tensor\n // Transformers.js returns a Tensor object - handle different formats\n if (output && typeof output === 'object') {\n // Check for .data property (common tensor format)\n if (output.data && Array.isArray(output.data)) {\n return output.data;\n }\n \n // Check if it's a nested array (tensor shape)\n if (Array.isArray(output)) {\n // Flatten if nested\n if (Array.isArray(output[0])) {\n return output[0]; // Return first row if 2D\n }\n return output;\n }\n \n // Try accessing tensor properties\n if (output.size && output.data) {\n return Array.from(output.data);\n }\n \n // Try tolist() method if available\n if (typeof output.tolist === 'function') {\n const list = output.tolist();\n return Array.isArray(list[0]) ? list[0] : list;\n }\n }\n \n // Fallback: try to convert to array\n return Array.from(output as any);\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"Transformers.js embeddings failed:\", errorMessage);\n return null;\n }\n};\n\nexport const searchStrategy: SearchStrategy = {\n name: \"transformers-cosine\",\n description: \"Semantic search using Transformers.js local embeddings with cosine similarity (fastest, no API calls)\",\n search: async (query: string, pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const minSimilarity = (options.minScore || 0) / 100; // Convert to 0-1 scale\n const enableTiming = options.enableTiming || false;\n \n const timings: Record = {};\n const startTime = performance.now();\n \n // Generate query embedding (local, no API call)\n const queryEmbedStart = performance.now();\n const queryEmbedding = await generateEmbeddings(query);\n if (enableTiming) {\n timings.queryEmbedding = performance.now() - queryEmbedStart;\n }\n \n if (!queryEmbedding) {\n console.warn(\"Query embedding generation failed, returning empty results\");\n return [];\n }\n \n // Calculate cosine similarity for each page\n const similarityStart = performance.now();\n const scoredPages: Array<{ page: Page; similarity: number }> = [];\n \n for (const page of pages) {\n const pageEmbedding = page.embeddings;\n if (!pageEmbedding || pageEmbedding.length !== queryEmbedding.length) {\n continue; // Skip pages without embeddings or wrong dimension\n }\n \n const similarity = cosineSimilarity(queryEmbedding, pageEmbedding);\n if (similarity >= minSimilarity) {\n scoredPages.push({ page, similarity });\n }\n }\n \n if (enableTiming) timings.similarity = performance.now() - similarityStart;\n \n // Sort by similarity (descending) and limit\n const sortStart = performance.now();\n scoredPages.sort((a, b) => b.similarity - a.similarity);\n const topPages = scoredPages.slice(0, limit);\n if (enableTiming) timings.sort = performance.now() - sortStart;\n \n // Transform to SearchResult format\n const transformStart = performance.now();\n const queryWords = query.toLowerCase().split(/\\s+/).filter(w => w.length > 0);\n const results: SearchResult[] = topPages.map(({ page, similarity }) => ({\n path: page.path,\n url: page.url,\n title: page.title || page.path,\n score: similarity * 100, // Convert to 0-100 scale\n snippet: generateSnippet(page.content, queryWords, query.toLowerCase()),\n metadata: {\n strategy: \"transformers-cosine\",\n similarity,\n ...(enableTiming && { timings }),\n },\n }));\n if (enableTiming) timings.transform = performance.now() - transformStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Timing breakdown:`);\n console.log(` Query embedding: ${timings.queryEmbedding?.toFixed(2)}ms (local model - no API)`);\n console.log(` Cosine similarity: ${timings.similarity?.toFixed(2)}ms`);\n console.log(` Sort: ${timings.sort?.toFixed(2)}ms`);\n console.log(` Transform: ${timings.transform?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n return results;\n },\n};\n\n",[],[231,350],"3181e164-b901-11f0-a3b9-0224a6c84d84","placeholder.ts","3181e22c-b901-11f0-a3b9-0224a6c84d84",37,["D",1762207522458],"// Placeholder Strategy: Keyword-based search (current default)\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { generateSnippet } from \"./utils.ts\";\n\n// Fake embeddings function for placeholder (not actually used, but kept for interface consistency)\nexport const generateEmbeddings = async (content: string): Promise => {\n // Not used in placeholder strategy, but kept for consistency\n // Returns fake embeddings for caching purposes\n console.debug(\"Generating fake embeddings for placeholder strategy\");\n \n const fakeEmbedding: number[] = [];\n const hash = content.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0);\n \n for (let i = 0; i < 384; i++) {\n const value = Math.sin(hash + i) * 0.5 + 0.5;\n fakeEmbedding.push(value);\n }\n \n return fakeEmbedding;\n};\n\nexport const searchStrategy: SearchStrategy = {\n name: \"placeholder\",\n description: \"Keyword-based search using title, content, and metadata matching\",\n search: async (query: string, pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const minScore = options.minScore || 0;\n \n const queryLower = query.toLowerCase();\n const queryWords = queryLower.split(/\\s+/).filter(w => w.length > 0);\n \n const results: SearchResult[] = [];\n \n for (const page of pages) {\n let score = 0;\n const contentLower = page.content.toLowerCase();\n const titleLower = (page.title || page.path).toLowerCase();\n \n // Title matches are worth more\n for (const word of queryWords) {\n if (titleLower.includes(word)) {\n score += 10;\n }\n // Count occurrences in content\n const contentMatches = (contentLower.match(new RegExp(word, 'g')) || []).length;\n score += Math.min(contentMatches, 5); // Cap at 5 points per word\n }\n \n // Exact phrase match bonus\n if (contentLower.includes(queryLower)) {\n score += 20;\n }\n \n // Metadata boost (if query matches tags or categories)\n if (page.metadata) {\n const tags = (page.metadata.tags || []).join(' ').toLowerCase();\n const categories = (page.metadata.categories || []).join(' ').toLowerCase();\n const useCases = (page.metadata.useCases || []).join(' ').toLowerCase();\n \n for (const word of queryWords) {\n if (tags.includes(word)) score += 3;\n if (categories.includes(word)) score += 2;\n if (useCases.includes(word)) score += 2;\n }\n }\n \n if (score > minScore) {\n results.push({\n path: page.path,\n url: page.url,\n title: page.title,\n score,\n snippet: generateSnippet(page.content, queryWords, queryLower),\n metadata: {\n strategy: \"placeholder\",\n },\n });\n }\n }\n \n // Sort by score descending\n results.sort((a, b) => b.score - a.score);\n \n return results.slice(0, limit);\n },\n};\n\n",[],[231,358],"314db04c-b901-11f0-a764-0224a6c84d84","openai-orama.ts","314db0ec-b901-11f0-a764-0224a6c84d84",31,["D",1762207522111],"// OpenAI + Orama Strategy: Semantic search using OpenAI embeddings and Orama\n// Faster than JigsawStack (~100-200ms vs ~550ms for query embeddings)\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { generateSnippet } from \"./utils.ts\";\nimport { create, insertMultiple, search } from \"npm:@orama/orama@latest\";\n\n// OpenAI embeddings function\nexport const generateEmbeddings = async (content: string): Promise => {\n const OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\n const OPENAI_API_URL = \"https://api.openai.com/v1/embeddings\";\n \n if (!OPENAI_API_KEY) {\n console.warn(\"OPENAI_API_KEY not found - embeddings disabled\");\n return null;\n }\n \n try {\n const response = await fetch(OPENAI_API_URL, {\n method: \"POST\",\n headers: {\n \"Authorization\": `Bearer ${OPENAI_API_KEY}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n model: \"text-embedding-3-small\", // Fast and efficient model\n input: content,\n }),\n });\n \n if (!response.ok) {\n const errorText = await response.text().catch(() => response.statusText);\n throw new Error(`OpenAI API error: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);\n }\n \n const data = await response.json();\n \n if (data.data && Array.isArray(data.data) && data.data.length > 0) {\n return data.data[0].embedding; // Return the embedding vector\n }\n \n return null;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"OpenAI embeddings failed:\", errorMessage);\n return null;\n }\n};\n\n// No caching for query embeddings - we want to benchmark actual API performance\n\nexport const searchStrategy: SearchStrategy = {\n name: \"openai-orama\",\n description: \"Semantic search using OpenAI embeddings with Orama vector search (faster than JigsawStack)\",\n search: async (query: string, pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const minSimilarity = (options.minScore || 0) / 100; // Convert to 0-1 scale\n const enableTiming = options.enableTiming || false;\n \n const timings: Record = {};\n const startTime = performance.now();\n \n // Generate query embedding (no caching - benchmarking actual API performance)\n const queryEmbedStart = performance.now();\n const queryEmbedding = await generateEmbeddings(query);\n if (enableTiming) {\n timings.queryEmbedding = performance.now() - queryEmbedStart;\n }\n \n if (!queryEmbedding) {\n console.warn(\"Query embedding generation failed, returning empty results\");\n return [];\n }\n \n // Determine embedding dimension from query (or first page with embeddings)\n const embeddingDimension = queryEmbedding.length;\n if (embeddingDimension === 0) {\n return [];\n }\n \n // Create Orama database with vector field\n const dbCreateStart = performance.now();\n const db = await create({\n schema: {\n path: \"string\",\n url: \"string\",\n title: \"string\",\n content: \"string\",\n embedding: `vector[${embeddingDimension}]`,\n },\n });\n if (enableTiming) timings.dbCreate = performance.now() - dbCreateStart;\n \n // Prepare documents with embeddings for insertion\n const documents: Array<{\n path: string;\n url: string;\n title: string;\n content: string;\n embedding: number[];\n }> = [];\n \n // Collect pages with embeddings\n const docPrepStart = performance.now();\n for (const page of pages) {\n // Use cached embeddings if available, otherwise generate\n let pageEmbedding = page.embeddings;\n if (!pageEmbedding) {\n // Generate embedding if not cached (should be cached from recalculate, but handle missing)\n pageEmbedding = await generateEmbeddings(page.content);\n if (!pageEmbedding || pageEmbedding.length !== embeddingDimension) {\n continue; // Skip pages without embeddings or wrong dimension\n }\n }\n \n // Ensure embedding dimension matches\n if (pageEmbedding.length !== embeddingDimension) {\n continue;\n }\n \n documents.push({\n path: page.path,\n url: page.url,\n title: page.title || page.path,\n content: page.content,\n embedding: pageEmbedding,\n });\n }\n if (enableTiming) timings.docPrep = performance.now() - docPrepStart;\n \n // Insert all documents into Orama\n const insertStart = performance.now();\n if (documents.length > 0) {\n await insertMultiple(db, documents);\n } else {\n return []; // No documents to search\n }\n if (enableTiming) timings.insert = performance.now() - insertStart;\n \n // Perform vector similarity search using Orama\n const searchStart = performance.now();\n const searchResults = await search(db, {\n mode: \"vector\",\n vector: {\n value: queryEmbedding,\n property: \"embedding\",\n },\n similarity: minSimilarity,\n limit,\n includeVectors: false, // Don't need vectors in response\n });\n if (enableTiming) timings.search = performance.now() - searchStart;\n \n // Transform Orama results to SearchResult format\n const transformStart = performance.now();\n const results: SearchResult[] = searchResults.hits.map((hit: any) => {\n const page = pages.find(p => p.path === hit.document.path);\n const queryWords = query.toLowerCase().split(/\\s+/).filter(w => w.length > 0);\n \n return {\n path: hit.document.path,\n url: hit.document.url,\n title: hit.document.title,\n score: (hit.score || 0) * 100, // Convert similarity to score (0-100)\n snippet: page ? generateSnippet(page.content, queryWords, query.toLowerCase()) : hit.document.content.substring(0, 200) + '...',\n metadata: {\n strategy: \"openai-orama\",\n similarity: hit.score,\n ...(enableTiming && { timings }),\n },\n };\n });\n if (enableTiming) timings.transform = performance.now() - transformStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Timing breakdown:`);\n console.log(` Query embedding: ${timings.queryEmbedding?.toFixed(2)}ms (API call)`);\n console.log(` DB create: ${timings.dbCreate?.toFixed(2)}ms`);\n console.log(` Doc prep: ${timings.docPrep?.toFixed(2)}ms`);\n console.log(` Insert: ${timings.insert?.toFixed(2)}ms`);\n console.log(` Search: ${timings.search?.toFixed(2)}ms`);\n console.log(` Transform: ${timings.transform?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n return results;\n },\n};\n\n",[],[231,366],"31459b96-b901-11f0-9996-0224a6c84d84","openai-cosine.ts","31459c22-b901-11f0-9996-0224a6c84d84",30,["D",1762207522106],"// OpenAI + Cosine Similarity Strategy: Direct cosine similarity calculation\n// Fastest for small datasets (<100 pages) - no DB overhead\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { generateSnippet, cosineSimilarity } from \"./utils.ts\";\n\n// OpenAI embeddings function\nexport const generateEmbeddings = async (content: string): Promise => {\n const OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\n const OPENAI_API_URL = \"https://api.openai.com/v1/embeddings\";\n \n if (!OPENAI_API_KEY) {\n console.warn(\"OPENAI_API_KEY not found - embeddings disabled\");\n return null;\n }\n \n try {\n const response = await fetch(OPENAI_API_URL, {\n method: \"POST\",\n headers: {\n \"Authorization\": `Bearer ${OPENAI_API_KEY}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n model: \"text-embedding-3-small\", // Fast and efficient model\n input: content,\n }),\n });\n \n if (!response.ok) {\n const errorText = await response.text().catch(() => response.statusText);\n throw new Error(`OpenAI API error: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);\n }\n \n const data = await response.json();\n \n if (data.data && Array.isArray(data.data) && data.data.length > 0) {\n return data.data[0].embedding; // Return the embedding vector\n }\n \n return null;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"OpenAI embeddings failed:\", errorMessage);\n return null;\n }\n};\n\n// No caching for query embeddings - we want to benchmark actual API performance\n\n\nexport const searchStrategy: SearchStrategy = {\n name: \"openai-cosine\",\n description: \"Semantic search using OpenAI embeddings with direct cosine similarity (fastest for small datasets)\",\n search: async (query: string, pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const minSimilarity = (options.minScore || 0) / 100; // Convert to 0-1 scale\n const enableTiming = options.enableTiming || false;\n \n const timings: Record = {};\n const startTime = performance.now();\n \n // Generate query embedding (no caching - benchmarking actual API performance)\n const queryEmbedStart = performance.now();\n const queryEmbedding = await generateEmbeddings(query);\n if (enableTiming) {\n timings.queryEmbedding = performance.now() - queryEmbedStart;\n }\n \n if (!queryEmbedding) {\n console.warn(\"Query embedding generation failed, returning empty results\");\n return [];\n }\n \n // Calculate cosine similarity for each page\n const similarityStart = performance.now();\n const scoredPages: Array<{ page: Page; similarity: number }> = [];\n \n for (const page of pages) {\n const pageEmbedding = page.embeddings;\n if (!pageEmbedding || pageEmbedding.length !== queryEmbedding.length) {\n continue; // Skip pages without embeddings or wrong dimension\n }\n \n const similarity = cosineSimilarity(queryEmbedding, pageEmbedding);\n if (similarity >= minSimilarity) {\n scoredPages.push({ page, similarity });\n }\n }\n \n if (enableTiming) timings.similarity = performance.now() - similarityStart;\n \n // Sort by similarity (descending) and limit\n const sortStart = performance.now();\n scoredPages.sort((a, b) => b.similarity - a.similarity);\n const topPages = scoredPages.slice(0, limit);\n if (enableTiming) timings.sort = performance.now() - sortStart;\n \n // Transform to SearchResult format\n const transformStart = performance.now();\n const queryWords = query.toLowerCase().split(/\\s+/).filter(w => w.length > 0);\n const results: SearchResult[] = topPages.map(({ page, similarity }) => ({\n path: page.path,\n url: page.url,\n title: page.title || page.path,\n score: similarity * 100, // Convert to 0-100 scale\n snippet: generateSnippet(page.content, queryWords, query.toLowerCase()),\n metadata: {\n strategy: \"openai-cosine\",\n similarity,\n ...(enableTiming && { timings }),\n },\n }));\n if (enableTiming) timings.transform = performance.now() - transformStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Timing breakdown:`);\n console.log(` Query embedding: ${timings.queryEmbedding?.toFixed(2)}ms (API call)`);\n console.log(` Cosine similarity: ${timings.similarity?.toFixed(2)}ms`);\n console.log(` Sort: ${timings.sort?.toFixed(2)}ms`);\n console.log(` Transform: ${timings.transform?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n return results;\n },\n};\n\n",[],[231,374],"315f55ae-b901-11f0-8f9e-0224a6c84d84","jigsawstack-orama.ts","315f56bc-b901-11f0-8f9e-0224a6c84d84",33,["D",1762207522117],"// JigsawStack + Orama Strategy: Semantic search using JigsawStack embeddings and Orama\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { generateSnippet } from \"./utils.ts\";\nimport { create, insertMultiple, search } from \"npm:@orama/orama@latest\";\n\n// JigsawStack embeddings function (v2 API)\nexport const generateEmbeddings = async (content: string): Promise => {\n const JIGSAWSTACK_API_KEY = Deno.env.get(\"JIGSAWSTACK_API_KEY\");\n const JIGSAWSTACK_API_URL = \"https://api.jigsawstack.com/v2/embedding\";\n \n if (!JIGSAWSTACK_API_KEY) {\n console.warn(\"JIGSAWSTACK_API_KEY not found - embeddings disabled\");\n return null;\n }\n \n try {\n const response = await fetch(JIGSAWSTACK_API_URL, {\n method: \"POST\",\n headers: {\n \"X-Api-Key\": JIGSAWSTACK_API_KEY,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n text: content,\n type: \"text\",\n }),\n });\n \n if (!response.ok) {\n const errorText = await response.text().catch(() => response.statusText);\n throw new Error(`JigsawStack API error: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);\n }\n \n const data = await response.json();\n \n // JigsawStack v2 returns: { success: true, embeddings: [[...]], chunks: [...], _usage: {...} }\n // embeddings is an array of arrays, we want the first embedding vector\n if (data.success && data.embeddings && Array.isArray(data.embeddings) && data.embeddings.length > 0) {\n return data.embeddings[0]; // Return the first embedding vector\n }\n \n return null;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"JigsawStack embeddings failed:\", errorMessage);\n return null;\n }\n};\n\nexport const searchStrategy: SearchStrategy = {\n name: \"jigsawstack-orama\",\n description: \"Semantic search using JigsawStack embeddings with Orama vector search\",\n search: async (query: string, pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const minSimilarity = (options.minScore || 0) / 100; // Convert to 0-1 scale\n const enableTiming = options.enableTiming || false;\n \n const timings: Record = {};\n const startTime = performance.now();\n \n // Generate query embedding (this is the main bottleneck - external API call)\n const queryEmbedStart = performance.now();\n const queryEmbedding = await generateEmbeddings(query);\n if (enableTiming) timings.queryEmbedding = performance.now() - queryEmbedStart;\n \n if (!queryEmbedding) {\n console.warn(\"Query embedding generation failed, returning empty results\");\n return [];\n }\n \n // Determine embedding dimension from query (or first page with embeddings)\n const embeddingDimension = queryEmbedding.length;\n if (embeddingDimension === 0) {\n return [];\n }\n \n // Create Orama database with vector field\n const dbCreateStart = performance.now();\n const db = await create({\n schema: {\n path: \"string\",\n url: \"string\",\n title: \"string\",\n content: \"string\",\n embedding: `vector[${embeddingDimension}]`,\n },\n });\n if (enableTiming) timings.dbCreate = performance.now() - dbCreateStart;\n \n // Prepare documents with embeddings for insertion\n const documents: Array<{\n path: string;\n url: string;\n title: string;\n content: string;\n embedding: number[];\n }> = [];\n \n // Collect pages with embeddings\n const docPrepStart = performance.now();\n for (const page of pages) {\n // Use cached embeddings if available, otherwise generate\n let pageEmbedding = page.embeddings;\n if (!pageEmbedding) {\n // Generate embedding if not cached (should be cached from recalculate, but handle missing)\n pageEmbedding = await generateEmbeddings(page.content);\n if (!pageEmbedding || pageEmbedding.length !== embeddingDimension) {\n continue; // Skip pages without embeddings or wrong dimension\n }\n }\n \n // Ensure embedding dimension matches\n if (pageEmbedding.length !== embeddingDimension) {\n continue;\n }\n \n documents.push({\n path: page.path,\n url: page.url,\n title: page.title || page.path,\n content: page.content,\n embedding: pageEmbedding,\n });\n }\n if (enableTiming) timings.docPrep = performance.now() - docPrepStart;\n \n // Insert all documents into Orama\n const insertStart = performance.now();\n if (documents.length > 0) {\n await insertMultiple(db, documents);\n } else {\n return []; // No documents to search\n }\n if (enableTiming) timings.insert = performance.now() - insertStart;\n \n // Perform vector similarity search using Orama\n const searchStart = performance.now();\n const searchResults = await search(db, {\n mode: \"vector\",\n vector: {\n value: queryEmbedding,\n property: \"embedding\",\n },\n similarity: minSimilarity,\n limit,\n includeVectors: false, // Don't need vectors in response\n });\n if (enableTiming) timings.search = performance.now() - searchStart;\n \n // Transform Orama results to SearchResult format\n const transformStart = performance.now();\n const results: SearchResult[] = searchResults.hits.map((hit: any) => {\n const page = pages.find(p => p.path === hit.document.path);\n const queryWords = query.toLowerCase().split(/\\s+/).filter(w => w.length > 0);\n \n return {\n path: hit.document.path,\n url: hit.document.url,\n title: hit.document.title,\n score: (hit.score || 0) * 100, // Convert similarity to score (0-100)\n snippet: page ? generateSnippet(page.content, queryWords, query.toLowerCase()) : hit.document.content.substring(0, 200) + '...',\n metadata: {\n strategy: \"jigsawstack-orama\",\n similarity: hit.score,\n ...(enableTiming && { timings }),\n },\n };\n });\n if (enableTiming) timings.transform = performance.now() - transformStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Timing breakdown:`);\n console.log(` Query embedding: ${timings.queryEmbedding?.toFixed(2)}ms (API call - main bottleneck)`);\n console.log(` DB create: ${timings.dbCreate?.toFixed(2)}ms`);\n console.log(` Doc prep: ${timings.docPrep?.toFixed(2)}ms`);\n console.log(` Insert: ${timings.insert?.toFixed(2)}ms`);\n console.log(` Search: ${timings.search?.toFixed(2)}ms`);\n console.log(` Transform: ${timings.transform?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n return results;\n },\n};\n\n",[],[231,382],"31712298-b901-11f0-878c-0224a6c84d84","index.ts","3171232e-b901-11f0-878c-0224a6c84d84",35,["D",1762207522385],"// Search - Main entry point\n// To switch strategies, comment/uncomment the imports and function calls below\n\nexport * from \"./types.ts\";\nexport * from \"./utils.ts\";\n\nimport type { Page, SearchOptions, SearchResult } from \"./types.ts\";\n// import { searchStrategy, generateEmbeddings } from \"./placeholder.ts\";\n// import { searchStrategy, generateEmbeddings } from \"./jigsawstack-orama.ts\"; // ~550ms query embeddings (API)\n// import { searchStrategy, generateEmbeddings } from \"./openai-orama.ts\"; // ~100-200ms query embeddings (API), ~29ms DB overhead\n// import { searchStrategy, generateEmbeddings } from \"./transformers-cosine.ts\"; // ~10-30ms (browser only), requires --allow-ffi in Deno\nimport { searchStrategy, generateEmbeddings } from \"./openai-cosine.ts\"; // ~100-200ms query embeddings (API), <1ms cosine calc\n\n// Main search function - comment/uncomment imports above to switch strategies\nexport const searchPages = async (\n query: string,\n pages: Page[],\n options: SearchOptions = {}\n): Promise => {\n return searchStrategy.search(query, pages, options);\n};\n\n// Export search strategy for introspection (e.g., /search/test endpoint)\nexport { searchStrategy };\n\n// Embeddings function for recalculate - uses same import as search strategy\nexport const getActiveEmbeddingsFunction = (): ((content: string) => Promise) => {\n return generateEmbeddings;\n};\n\n\n",[],[231,390],["D",1762207521254],[254,255,256,257,258,259,260,261],[231],["D",1762207521817],[],[237],["D",1746217442601],{"_95":94,"_102":98},{"_95":94,"_96":-5,"_97":98,"_18":54,"_99":100,"_101":24},{"_13":84,"_31":85,"_42":41,"_86":43,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":44,"_93":94},"ctx",{"_77":-5,"_122":440,"_441":442,"_443":444,"_445":24,"_446":24,"_447":24,"_448":48,"_449":-5,"_450":-7,"_451":-7,"_452":61,"_453":24,"_454":24,"_455":-5,"_456":24,"_457":24,"_458":24,"_459":-5},"ENV",{"_411":412,"_413":414,"_415":85,"_416":417,"_418":419,"_420":421,"_422":423,"_424":425,"_426":427,"_428":429,"_430":431,"_432":433,"_434":435,"_436":437,"_438":439},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","fdb67791413f4386b4dbf18f354029a617f6160f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v41