[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_775":776,"_777":778},"routes/_app.x.$handle.$name",{"_97":96,"_32":46,"_105":87,"_27":106,"_13":14,"_107":-5,"_11":108,"_109":110,"_111":112,"_113":114,"_115":116,"_20":-5,"_117":-5,"_118":-5,"_119":25,"_120":121,"_122":50,"_123":-5,"_124":-5,"_125":126},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_44":45,"_32":46,"_39":47,"_48":-5,"_49":50,"_51":52,"_53":52,"_54":-5,"_55":-5,"_18":56,"_57":58,"_59":50,"_60":-5,"_61":-5,"_62":63,"_64":65,"_66":-5,"_67":-5,"_68":-5,"_69":-5,"_70":71,"_72":71,"_73":74,"_75":74,"_76":74,"_77":71,"_78":71,"_79":80,"_81":82,"_83":84,"_85":25},"version",96,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":33,"_34":35,"_36":38,"_39":40,"_41":14,"_42":43,"_24":25},"previous",{"_30":31,"_32":33,"_34":35,"_36":37},"deleted",false,"deletedHere","path","search/transformers-cosine.ts","latestPath","fileId","31884324-b901-11f0-b784-0224a6c84d84","name","transformers-cosine.ts","fileType","script","content","// Transformers.js + Cosine Similarity Strategy: Local embeddings in browser/Deno\n// Fastest option - no API calls, runs entirely client-side\n// Model: all-MiniLM-L6-v2 (80MB, ~384 dimensions, very fast)\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { generateSnippet, cosineSimilarity } from \"./utils.ts\";\n\n// Lazy-loaded pipeline (cached after first load)\nlet embeddingPipeline: any = null;\nlet pipelineLoading: Promise | null = null;\n\nasync function getEmbeddingPipeline() {\n // Return cached pipeline if already loaded\n if (embeddingPipeline) {\n return embeddingPipeline;\n }\n \n // If loading in progress, wait for it\n if (pipelineLoading) {\n return pipelineLoading;\n }\n \n // Start loading pipeline\n pipelineLoading = (async () => {\n try {\n // Import transformers.js - use npm: prefix for Deno compatibility\n // NOTE: transformers.js is primarily designed for browsers. In Deno, it may require --allow-ffi\n // For browser usage, this will work seamlessly. For Deno, you may need additional permissions.\n const { pipeline } = await import(\"npm:@huggingface/transformers@latest\");\n \n // Load feature extraction pipeline with all-MiniLM-L6-v2 model\n // This is a small, fast model optimized for embeddings (80MB, ~384 dims)\n embeddingPipeline = await pipeline(\n 'feature-extraction',\n 'Xenova/all-MiniLM-L6-v2',\n {\n // Use CPU - WebGPU might not be available in all environments\n device: 'cpu',\n dtype: 'fp32',\n }\n );\n \n return embeddingPipeline;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n const fullError = `Failed to load transformers.js pipeline: ${errorMessage}\\n` +\n `NOTE: transformers.js is primarily designed for browser environments. ` +\n `If running in Deno, you may need --allow-ffi permissions, or use a different embedding strategy.`;\n console.error(fullError);\n throw new Error(fullError);\n } finally {\n pipelineLoading = null;\n }\n })();\n \n return pipelineLoading;\n}\n\n// Generate embeddings using transformers.js (runs locally, no API calls)\nexport const generateEmbeddings = async (content: string): Promise => {\n try {\n const pipeline = await getEmbeddingPipeline();\n \n // Generate embeddings - returns a tensor, we need to extract the array\n const output = await pipeline(content, {\n pooling: 'mean',\n normalize: true,\n });\n \n // Extract the embedding vector from the tensor\n // Transformers.js returns a Tensor object - handle different formats\n if (output && typeof output === 'object') {\n // Check for .data property (common tensor format)\n if (output.data && Array.isArray(output.data)) {\n return output.data;\n }\n \n // Check if it's a nested array (tensor shape)\n if (Array.isArray(output)) {\n // Flatten if nested\n if (Array.isArray(output[0])) {\n return output[0]; // Return first row if 2D\n }\n return output;\n }\n \n // Try accessing tensor properties\n if (output.size && output.data) {\n return Array.from(output.data);\n }\n \n // Try tolist() method if available\n if (typeof output.tolist === 'function') {\n const list = output.tolist();\n return Array.isArray(list[0]) ? list[0] : list;\n }\n }\n \n // Fallback: try to convert to array\n return Array.from(output as any);\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"Transformers.js embeddings failed:\", errorMessage);\n return null;\n }\n};\n\nexport const searchStrategy: SearchStrategy = {\n name: \"transformers-cosine\",\n description: \"Semantic search using Transformers.js local embeddings with cosine similarity (fastest, no API calls)\",\n search: async (query: string, pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const minSimilarity = (options.minScore || 0) / 100; // Convert to 0-1 scale\n const enableTiming = options.enableTiming || false;\n \n const timings: Record = {};\n const startTime = performance.now();\n \n // Generate query embedding (local, no API call)\n const queryEmbedStart = performance.now();\n const queryEmbedding = await generateEmbeddings(query);\n if (enableTiming) {\n timings.queryEmbedding = performance.now() - queryEmbedStart;\n }\n \n if (!queryEmbedding) {\n console.warn(\"Query embedding generation failed, returning empty results\");\n return [];\n }\n \n // Calculate cosine similarity for each page\n const similarityStart = performance.now();\n const scoredPages: Array<{ page: Page; similarity: number }> = [];\n \n for (const page of pages) {\n const pageEmbedding = page.embeddings;\n if (!pageEmbedding || pageEmbedding.length !== queryEmbedding.length) {\n continue; // Skip pages without embeddings or wrong dimension\n }\n \n const similarity = cosineSimilarity(queryEmbedding, pageEmbedding);\n if (similarity >= minSimilarity) {\n scoredPages.push({ page, similarity });\n }\n }\n \n if (enableTiming) timings.similarity = performance.now() - similarityStart;\n \n // Sort by similarity (descending) and limit\n const sortStart = performance.now();\n scoredPages.sort((a, b) => b.similarity - a.similarity);\n const topPages = scoredPages.slice(0, limit);\n if (enableTiming) timings.sort = performance.now() - sortStart;\n \n // Transform to SearchResult format\n const transformStart = performance.now();\n const queryWords = query.toLowerCase().split(/\\s+/).filter(w => w.length > 0);\n const results: SearchResult[] = topPages.map(({ page, similarity }) => ({\n path: page.path,\n url: page.url,\n title: page.title || page.path,\n score: similarity * 100, // Convert to 0-100 scale\n snippet: generateSnippet(page.content, queryWords, query.toLowerCase()),\n metadata: {\n strategy: \"transformers-cosine\",\n similarity,\n ...(enableTiming && { timings }),\n },\n }));\n if (enableTiming) timings.transform = performance.now() - transformStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Timing breakdown:`);\n console.log(` Query embedding: ${timings.queryEmbedding?.toFixed(2)}ms (local model - no API)`);\n console.log(` Cosine similarity: ${timings.similarity?.toFixed(2)}ms`);\n console.log(` Sort: ${timings.sort?.toFixed(2)}ms`);\n console.log(` Transform: ${timings.transform?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n return results;\n },\n};\n\n","// Transformers.js + Cosine Similarity Strategy: Local embeddings in browser/Deno\n// Fastest option - no API calls, runs entirely client-side\n// Model: all-MiniLM-L6-v2 (~86MB, ~384 dimensions, cached locally after first load)\n// Downloads from Hugging Face on first run, then uses cache\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { generateSnippet, cosineSimilarity } from \"./utils.ts\";\n\n// Lazy-loaded pipeline (cached after first load)\nlet embeddingPipeline: any = null;\nlet pipelineLoading: Promise | null = null;\n\n// Cleanup function to dispose of pipeline resources\nexport const disposePipeline = async (): Promise => {\n if (embeddingPipeline) {\n try {\n // Try to dispose/disconnect if the pipeline has cleanup methods\n if (typeof embeddingPipeline.dispose === 'function') {\n await embeddingPipeline.dispose();\n } else if (typeof embeddingPipeline.disconnect === 'function') {\n await embeddingPipeline.disconnect();\n }\n embeddingPipeline = null;\n } catch (error) {\n // Ignore cleanup errors - pipeline may already be disposed\n console.debug(\"Pipeline cleanup error (ignored):\", error);\n }\n }\n};\n\n// Prevent ONNX runtime cleanup crash on exit\n// The crash is harmless but annoying - this suppresses it\nif (typeof Deno !== \"undefined\") {\n Deno.addSignalListener(\"SIGINT\", () => {\n // Force immediate exit without cleanup\n Deno.exit(0);\n });\n \n // Also handle unload event\n globalThis.addEventListener(\"unload\", () => {\n // Prevent any cleanup that might cause mutex errors\n embeddingPipeline = null;\n });\n}\n\nasync function getEmbeddingPipeline() {\n // Return cached pipeline if already loaded\n if (embeddingPipeline) {\n return embeddingPipeline;\n }\n \n // If loading in progress, wait for it\n if (pipelineLoading) {\n return pipelineLoading;\n }\n \n // Start loading pipeline\n pipelineLoading = (async () => {\n const modelLoadTimings: Record = {};\n const modelLoadStart = performance.now();\n \n try {\n // Import transformers.js - use npm: prefix for Deno compatibility\n // NOTE: transformers.js is primarily designed for browsers. In Deno, it may require --allow-ffi\n // For browser usage, this will work seamlessly. For Deno, you may need additional permissions.\n const importStart = performance.now();\n const { pipeline } = await import(\"npm:@huggingface/transformers@latest\");\n modelLoadTimings.npmImport = performance.now() - importStart;\n \n // Load feature extraction pipeline with all-MiniLM-L6-v2 model\n // This is a small, fast model optimized for embeddings (~86MB, ~384 dims)\n // Downloads from Hugging Face on first run, then uses cache (~150ms vs several seconds)\n const pipelineStart = performance.now();\n embeddingPipeline = await pipeline(\n 'feature-extraction',\n 'Xenova/all-MiniLM-L6-v2',\n {\n // Use CPU - WebGPU might not be available in all environments\n device: 'cpu',\n dtype: 'fp32',\n }\n );\n modelLoadTimings.pipelineLoad = performance.now() - pipelineStart;\n modelLoadTimings.totalModelLoad = performance.now() - modelLoadStart;\n \n // Check if this was a cache hit (fast load) or download (slow load)\n const wasCached = modelLoadTimings.pipelineLoad < 1000; // <1s = likely cached\n \n console.log(`⏱️ Transformers.js model loading:`);\n console.log(` npm import: ${modelLoadTimings.npmImport?.toFixed(2)}ms`);\n console.log(` pipeline load: ${modelLoadTimings.pipelineLoad?.toFixed(2)}ms${wasCached ? ' (cached!)' : ' (downloaded ~86MB)'}`);\n console.log(` total model load: ${modelLoadTimings.totalModelLoad?.toFixed(2)}ms`);\n \n return embeddingPipeline;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n const fullError = `Failed to load transformers.js pipeline: ${errorMessage}\\n` +\n `NOTE: transformers.js is primarily designed for browser environments. ` +\n `If running in Deno, you may need --allow-ffi permissions, or use a different embedding strategy.`;\n console.error(fullError);\n if (modelLoadTimings.npmImport) {\n console.error(` Failed after npm import (${modelLoadTimings.npmImport.toFixed(2)}ms)`);\n }\n throw new Error(fullError);\n } finally {\n pipelineLoading = null;\n }\n })();\n \n return pipelineLoading;\n}\n\n// Generate embeddings using transformers.js (runs locally, no API calls)\nexport const generateEmbeddings = async (content: string): Promise => {\n try {\n const pipeline = await getEmbeddingPipeline();\n \n // Generate embeddings - returns a tensor, we need to extract the array\n const output = await pipeline(content, {\n pooling: 'mean',\n normalize: true,\n });\n \n // Extract the embedding vector from the tensor\n // Transformers.js returns a Tensor object - handle different formats\n if (output && typeof output === 'object') {\n // Check for .data property (common tensor format)\n if (output.data && Array.isArray(output.data)) {\n return output.data;\n }\n \n // Check if it's a nested array (tensor shape)\n if (Array.isArray(output)) {\n // Flatten if nested\n if (Array.isArray(output[0])) {\n return output[0]; // Return first row if 2D\n }\n return output;\n }\n \n // Try accessing tensor properties\n if (output.size && output.data) {\n return Array.from(output.data);\n }\n \n // Try tolist() method if available\n if (typeof output.tolist === 'function') {\n const list = output.tolist();\n return Array.isArray(list[0]) ? list[0] : list;\n }\n }\n \n // Fallback: try to convert to array\n return Array.from(output as any);\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"Transformers.js embeddings failed:\", errorMessage);\n return null;\n }\n};\n\nexport const searchStrategy: SearchStrategy = {\n name: \"transformers-cosine\",\n description: \"Semantic search using Transformers.js local embeddings with cosine similarity (fastest, no API calls)\",\n search: async (query: string, pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const minSimilarity = (options.minScore || 0) / 100; // Convert to 0-1 scale\n const enableTiming = options.enableTiming || false;\n \n const timings: Record = {};\n const startTime = performance.now();\n \n // Generate query embedding (local, no API call)\n const queryEmbedStart = performance.now();\n const queryEmbedding = await generateEmbeddings(query);\n if (enableTiming) {\n timings.queryEmbedding = performance.now() - queryEmbedStart;\n }\n \n if (!queryEmbedding) {\n console.warn(\"Query embedding generation failed, returning empty results\");\n return [];\n }\n \n // Calculate cosine similarity for each page\n const similarityStart = performance.now();\n const scoredPages: Array<{ page: Page; similarity: number }> = [];\n \n for (const page of pages) {\n const pageEmbedding = page.embeddings;\n if (!pageEmbedding || pageEmbedding.length !== queryEmbedding.length) {\n continue; // Skip pages without embeddings or wrong dimension\n }\n \n const similarity = cosineSimilarity(queryEmbedding, pageEmbedding);\n if (similarity >= minSimilarity) {\n scoredPages.push({ page, similarity });\n }\n }\n \n if (enableTiming) timings.similarity = performance.now() - similarityStart;\n \n // Sort by similarity (descending) and limit\n const sortStart = performance.now();\n scoredPages.sort((a, b) => b.similarity - a.similarity);\n const topPages = scoredPages.slice(0, limit);\n if (enableTiming) timings.sort = performance.now() - sortStart;\n \n // Transform to SearchResult format\n const transformStart = performance.now();\n const queryWords = query.toLowerCase().split(/\\s+/).filter(w => w.length > 0);\n const results: SearchResult[] = topPages.map(({ page, similarity }) => ({\n path: page.path,\n url: page.url,\n title: page.title || page.path,\n score: similarity * 100, // Convert to 0-100 scale\n snippet: generateSnippet(page.content, queryWords, query.toLowerCase()),\n metadata: {\n strategy: \"transformers-cosine\",\n similarity,\n ...(enableTiming && { timings }),\n },\n }));\n if (enableTiming) timings.transform = performance.now() - transformStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Timing breakdown:`);\n console.log(` Query embedding: ${timings.queryEmbedding?.toFixed(2)}ms (local model - no API)`);\n console.log(` Cosine similarity: ${timings.similarity?.toFixed(2)}ms`);\n console.log(` Sort: ${timings.sort?.toFixed(2)}ms`);\n console.log(` Transform: ${timings.transform?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n return results;\n },\n};\n\n","createdAt",["D",1762293984186],"branchVersion","branchId","632d0edc-2793-11f0-9d52-569c3dd06744","id","632cc576-2793-11f0-9d52-569c3dd06744","groq-docs",["D",1746217442601],"deletedAt","legacyEnvVars",true,"privacy","public","httpPrivacy","description","imageUrl","a8270e74-6479-475a-8025-9b69544ee0e7","pinnedFileId","632eed24-2793-11f0-9d52-569c3dd06744","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","8319defa-6e1a-46d4-ba66-2114ffd3360b","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_97":96,"_104":100},"org",{"_97":96,"_98":-5,"_99":100,"_18":56,"_101":102,"_103":25},"branch",{"_13":86,"_32":87,"_44":43,"_88":45,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":-5,"_94":46,"_95":96},"isOwned",105,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","yawnxyz","handle","displayName","avatar","https://images.clerk.dev/oauth_github/img_2NnaHhpxNuH1xWRIRjQNoo16TVc.jpeg","tier","pro","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_44":45,"_32":46,"_39":771,"_48":-5,"_49":50,"_51":52,"_53":52,"_54":-5,"_55":-5,"_18":56,"_57":58,"_59":50,"_60":-5,"_61":-5,"_62":63,"_64":65,"_66":-5,"_67":-5,"_68":-5,"_69":-5,"_70":71,"_72":71,"_73":74,"_75":74,"_76":74,"_77":71,"_78":71,"_79":772,"_81":773,"_83":774,"_85":25},"tree",[253,263,264,301,309,311,312,313,314,315,316,317,318,319],"files",[253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319],"counts",{"_243":63,"_244":156,"_245":63,"_246":247,"_248":249},"contents",[127,128,129,130,131,132,133,134,135,136,137,138,139,140],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_44":237,"_32":238,"_30":239,"_144":-5,"_42":43,"_41":240,"_146":50,"_39":241,"_13":63,"_34":217,"_36":-5,"_24":25,"_149":-5,"_27":242},{"_44":231,"_32":232,"_30":233,"_144":-5,"_42":43,"_41":234,"_146":50,"_39":235,"_13":63,"_34":217,"_36":-5,"_24":25,"_149":-5,"_27":236},{"_44":225,"_32":226,"_30":227,"_144":-5,"_42":43,"_41":228,"_146":50,"_39":229,"_13":63,"_34":217,"_36":-5,"_24":25,"_149":-5,"_27":230},{"_44":219,"_32":220,"_30":221,"_144":-5,"_42":43,"_41":222,"_146":50,"_39":223,"_13":63,"_34":217,"_36":-5,"_24":25,"_149":-5,"_27":224},{"_44":212,"_32":213,"_30":214,"_144":-5,"_42":43,"_41":215,"_146":50,"_39":216,"_13":63,"_34":217,"_36":-5,"_24":25,"_149":-5,"_27":218},{"_44":205,"_32":206,"_30":207,"_144":-5,"_42":43,"_41":208,"_146":25,"_39":209,"_13":63,"_34":20,"_36":210,"_24":25,"_149":-5,"_27":211},{"_44":198,"_32":199,"_30":200,"_144":-5,"_42":43,"_41":201,"_146":50,"_39":202,"_13":63,"_34":20,"_36":203,"_24":25,"_149":-5,"_27":204},{"_44":191,"_32":192,"_30":193,"_144":-5,"_42":43,"_41":194,"_146":25,"_39":195,"_13":164,"_34":20,"_36":196,"_24":25,"_149":-5,"_27":197},{"_44":183,"_32":184,"_30":185,"_144":-5,"_42":43,"_41":186,"_146":50,"_39":187,"_13":188,"_34":20,"_36":189,"_24":25,"_149":-5,"_27":190},{"_44":176,"_32":177,"_30":178,"_144":-5,"_42":43,"_41":179,"_146":50,"_39":180,"_13":63,"_34":35,"_36":181,"_24":25,"_149":-5,"_27":182},{"_44":167,"_32":168,"_30":58,"_144":-5,"_42":43,"_41":169,"_146":50,"_39":170,"_13":171,"_34":172,"_36":173,"_24":25,"_149":174,"_27":175},{"_44":159,"_32":160,"_30":161,"_144":-5,"_42":43,"_41":162,"_146":50,"_39":163,"_13":164,"_34":20,"_36":165,"_24":25,"_149":-5,"_27":166},{"_44":151,"_32":152,"_30":153,"_144":-5,"_42":43,"_41":154,"_146":25,"_39":155,"_13":156,"_34":35,"_36":157,"_24":25,"_149":-5,"_27":158},{"_44":141,"_32":142,"_30":143,"_144":-5,"_42":43,"_41":145,"_146":50,"_39":147,"_13":63,"_34":35,"_36":148,"_24":25,"_149":-5,"_27":150},"93344376-b8f1-11f0-ab7b-0224a6c84d84","utils.ts","93344448-b8f1-11f0-ab7b-0224a6c84d84","parentId",24,"latest",["D",1762200814128],"import { sqlite } from \"https://esm.town/v/std/sqlite\";\nimport { encoding_for_model } from \"npm:tiktoken\";\nimport { RateLimit } from \"npm:async-sema@latest\";\n\n// Detect if we're running in Val Town (caching enabled) or locally (caching disabled)\n// valtown env var contains a token string when running in Val Town, undefined/null when local\nexport const IS_VALTOWN = typeof Deno !== \"undefined\" && !!Deno.env.get(\"valtown\");\n\n// Rate limiter: 0.33 requests per second (1 request per 3 seconds) to avoid WAF blocking\n// Slower rate to avoid console.groq.com WAF blocking\nexport const fetchRateLimiter = RateLimit(0.33);\n\n// Initialize cache table only in Val Town (bypass SQLite when running locally)\nconst CACHE_TABLE = \"groq_docs_cache_v3\"; // Updated table with hash and embeddings\nif (IS_VALTOWN) {\n try {\n await sqlite.execute(`CREATE TABLE IF NOT EXISTS ${CACHE_TABLE} (\n url TEXT PRIMARY KEY,\n content TEXT NOT NULL,\n charCount INTEGER NOT NULL,\n tokenCount INTEGER,\n frontmatter TEXT NOT NULL,\n metadata TEXT,\n contentHash TEXT,\n embeddings TEXT,\n cachedAt INTEGER NOT NULL\n )`);\n console.log(\"SQLite cache initialized (Val Town)\");\n } catch (error) {\n console.warn(\"Cache table creation failed:\", error instanceof Error ? error.message.substring(0, 100) : String(error).substring(0, 100));\n }\n} else {\n console.log(\"Running locally - SQLite caching disabled\");\n}\n\n// Calculate hash of content for change detection\nexport const calculateContentHash = async (content: string): Promise => {\n // Use Web Crypto API for SHA-256 hashing\n const encoder = new TextEncoder();\n const data = encoder.encode(content);\n const hashBuffer = await crypto.subtle.digest(\"SHA-256\", data);\n const hashArray = Array.from(new Uint8Array(hashBuffer));\n const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');\n return hashHex;\n};\n\n// Debug: Log environment detection\nif (typeof Deno !== \"undefined\") {\n const valtownToken = Deno.env.get(\"valtown\");\n console.log(`Environment detection: IS_VALTOWN=${IS_VALTOWN}, valtown token exists: ${!!valtownToken}`);\n}\n\n// Cache helper functions (bypass SQLite when running locally)\nexport const getFromCache = async (url: string): Promise<{\n content: string;\n charCount: number;\n tokenCount: number | null;\n frontmatter: any;\n metadata: any | null;\n contentHash: string | null;\n embeddings: number[] | null;\n} | null> => {\n if (!IS_VALTOWN) {\n return null; // Skip cache when running locally\n }\n \n try {\n const result = await sqlite.execute({\n sql: `SELECT content, charCount, tokenCount, frontmatter, metadata, contentHash, embeddings FROM ${CACHE_TABLE} WHERE url = ?`,\n args: [url]\n });\n if (result && result.rows && result.rows.length > 0) {\n // Convert row array to object using column names\n const row = result.rows[0];\n const rowObj: any = {};\n result.columns.forEach((col: string, index: number) => {\n rowObj[col] = row[index];\n });\n return {\n content: rowObj.content as string,\n charCount: rowObj.charCount as number,\n tokenCount: rowObj.tokenCount as number | null,\n frontmatter: JSON.parse(rowObj.frontmatter as string),\n metadata: rowObj.metadata ? JSON.parse(rowObj.metadata as string) : null,\n contentHash: rowObj.contentHash as string | null,\n embeddings: rowObj.embeddings ? JSON.parse(rowObj.embeddings as string) : null,\n };\n }\n } catch (error) {\n // Silently fail - caching unavailable\n const errorMessage = error instanceof Error ? error.message : String(error);\n const shortError = errorMessage.length > 200 ? errorMessage.substring(0, 200) + '...' : errorMessage;\n console.debug(\"Cache read failed:\", shortError);\n }\n return null;\n};\n\nexport const setCache = async (url: string, data: { content: string; charCount: number; tokenCount: number | null; frontmatter: any; metadata?: any; contentHash?: string; embeddings?: number[] | null }): Promise => {\n if (!IS_VALTOWN) {\n return \"Cache unavailable - running locally\";\n }\n \n try {\n await sqlite.execute({\n sql: `INSERT OR REPLACE INTO ${CACHE_TABLE} (url, content, charCount, tokenCount, frontmatter, metadata, contentHash, embeddings, cachedAt) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n url,\n data.content,\n data.charCount,\n data.tokenCount,\n JSON.stringify(data.frontmatter),\n data.metadata ? JSON.stringify(data.metadata) : null,\n data.contentHash || null,\n data.embeddings ? JSON.stringify(data.embeddings) : null,\n Date.now(),\n ]\n });\n return null; // Success - no error\n } catch (error) {\n // Return error message for tracking\n const errorMessage = error instanceof Error ? error.message : String(error);\n const shortError = errorMessage.length > 200 ? errorMessage.substring(0, 200) + '...' : errorMessage;\n console.debug(\"Cache write failed:\", shortError);\n return shortError;\n }\n};\n\nexport const clearCache = async (url?: string) => {\n if (!IS_VALTOWN) {\n throw new Error(\"Cache unavailable - running locally\");\n }\n \n try {\n if (url) {\n await sqlite.execute({\n sql: `DELETE FROM ${CACHE_TABLE} WHERE url = ?`,\n args: [url]\n });\n } else {\n await sqlite.execute(`DELETE FROM ${CACHE_TABLE}`);\n }\n } catch (error) {\n console.debug(\"Cache clear failed:\", error instanceof Error ? error.message.substring(0, 100) : String(error).substring(0, 100));\n throw error; // Re-throw for endpoints that need to know if it failed\n }\n};\n\nexport const getCacheStats = async () => {\n if (!IS_VALTOWN) {\n return {\n cachedPages: 0,\n totalTokens: 0,\n message: \"Cache unavailable - running locally\",\n };\n }\n \n try {\n const result = await sqlite.execute(`SELECT COUNT(*) as count, SUM(tokenCount) as totalTokens FROM ${CACHE_TABLE}`);\n if (result && result.rows && result.rows.length > 0) {\n const row = result.rows[0];\n const rowObj: any = {};\n result.columns.forEach((col: string, index: number) => {\n rowObj[col] = row[index];\n });\n return {\n cachedPages: rowObj.count || 0,\n totalTokens: rowObj.totalTokens || 0,\n };\n }\n return {\n cachedPages: 0,\n totalTokens: 0,\n };\n } catch (error) {\n console.debug(\"Cache stats failed:\", error instanceof Error ? error.message.substring(0, 100) : String(error).substring(0, 100));\n return {\n cachedPages: 0,\n totalTokens: 0,\n };\n }\n};\n\nexport const getTextFromUrl = async (url: string) => {\n // Use rate limiter to avoid WAF blocking\n await fetchRateLimiter();\n \n const response = await fetch(url, {\n headers: {\n \"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\",\n \"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",\n }\n });\n \n if (!response.ok) {\n throw new Error(`HTTP ${response.status}: ${response.statusText}`);\n }\n \n const text = await response.text();\n return text;\n};\n\nexport const parseFrontmatter = (text: string) => {\n const frontmatterRegex = /^---\\s*\\n([\\s\\S]*?)\\n---\\s*\\n?/;\n const match = text.match(frontmatterRegex);\n \n if (!match) {\n return {};\n }\n \n const frontmatterText = match[1];\n const metadata: any = {};\n \n // Parse YAML-like frontmatter (simple key: value parsing)\n const lines = frontmatterText.split('\\n');\n for (const line of lines) {\n const colonIndex = line.indexOf(':');\n if (colonIndex > 0) {\n const key = line.substring(0, colonIndex).trim();\n const value = line.substring(colonIndex + 1).trim();\n // Remove quotes if present\n metadata[key] = value.replace(/^[\"']|[\"']$/g, '');\n }\n }\n \n return metadata;\n};\n\nexport const addUrlSourceToFrontmatter = (text: string, url: string) => {\n // Check if frontmatter exists\n const frontmatterRegex = /^---\\s*\\n([\\s\\S]*?)\\n---\\s*\\n?/;\n const match = text.match(frontmatterRegex);\n \n if (match) {\n // Frontmatter exists, add urlSource to it\n const existingFrontmatter = match[1];\n \n // Check if urlSource already exists\n if (existingFrontmatter.includes('urlSource:')) {\n // Replace existing urlSource\n const updatedFrontmatter = existingFrontmatter.replace(\n /urlSource:\\s*.+$/m,\n `urlSource: ${url}`\n );\n return text.replace(frontmatterRegex, `---\\n${updatedFrontmatter}\\n---\\n`);\n } else {\n // Add urlSource at the end of frontmatter\n const updatedFrontmatter = existingFrontmatter.trim() + `\\nurlSource: ${url}`;\n return text.replace(frontmatterRegex, `---\\n${updatedFrontmatter}\\n---\\n\\n`);\n }\n } else {\n // No frontmatter exists, add it\n return `---\nurlSource: ${url}\n---\n\n${text}`;\n }\n};\n\nexport const calculateTokenCount = (text: string): number => {\n try {\n // Use cl100k_base encoding (used by GPT-4 and many models)\n const encoding = encoding_for_model(\"gpt-4\");\n return encoding.encode(text).length;\n } catch (error) {\n console.error(\"Error calculating token count:\", error);\n return 0;\n }\n};\n\n","label",[142],"8085cebe-b9ca-11f0-98dd-0224a6c84d84","urls.ts","3178e244-b901-11f0-87ce-0224a6c84d84",92,["D",1762293983912],1,"export\n const urls = [\n // partners // kind of poisons the answers\n // \"https://console.groq.com/docs/agno.md\",\n // \"https://console.groq.com/docs/ai-sdk.md\",\n // \"https://console.groq.com/docs/anchorbrowser.md\",\n // \"https://console.groq.com/docs/arize.md\",\n // \"https://console.groq.com/docs/autogen.md\",\n // \"https://console.groq.com/docs/browserbase.md\",\n // \"https://console.groq.com/docs/composio.md\",\n // \"https://console.groq.com/docs/crewai.md\",\n // \"https://console.groq.com/docs/e2b.md\",\n // \"https://console.groq.com/docs/firecrawl.md\",\n // \"https://console.groq.com/docs/flutterflow.md\",\n // \"https://console.groq.com/docs/gradio.md\",\n // \"https://console.groq.com/docs/huggingface.md\",\n // \"https://console.groq.com/docs/jigsawstack.md\",\n \"https://console.groq.com/docs/langchain.md\",\n \"https://console.groq.com/docs/litellm.md\",\n \"https://console.groq.com/docs/livekit.md\",\n \"https://console.groq.com/docs/openai.md\",\n \"https://console.groq.com/docs/tavily.md\",\n // \"https://console.groq.com/docs/toolhouse.md\",\n // \"https://console.groq.com/docs/wolfram-alpha.md\",\n // \"https://console.groq.com/docs/xrx.md\",\n // \"https://console.groq.com/docs/parallel.md\",\n // \"https://console.groq.com/docs/exa.md\",\n\n // other\n // \"https://console.groq.com/docs/changelog.md\",\n // \"https://console.groq.com/docs/changelog/compound.md\",\n // \"https://console.groq.com/docs/legacy-changelog.md\",\n\n // main\n \"https://console.groq.com/docs.md\",\n \"https://console.groq.com/docs/agentic-tooling.md\",\n \"https://console.groq.com/docs/agentic-tooling/compound-beta.md\",\n \"https://console.groq.com/docs/agentic-tooling/compound-beta-mini.md\",\n \"https://console.groq.com/docs/api-reference.md\",\n \"https://console.groq.com/docs/badge.md\",\n \"https://console.groq.com/docs/batch.md\",\n \"https://console.groq.com/docs/billing-faqs.md\",\n \"https://console.groq.com/docs/browser-automation.md\",\n \"https://console.groq.com/docs/browser-search.md\",\n \"https://console.groq.com/docs/browseruse.md\",\n \"https://console.groq.com/docs/code-execution.md\",\n \"https://console.groq.com/docs/compound.md\",\n \"https://console.groq.com/docs/compound/built-in-tools.md\",\n \"https://console.groq.com/docs/compound/systems.md\",\n \"https://console.groq.com/docs/compound/systems/compound.md\",\n \"https://console.groq.com/docs/compound/systems/compound-mini.md\",\n \"https://console.groq.com/docs/compound/systems/compound-beta.md\",\n \"https://console.groq.com/docs/compound/systems/compound-beta-mini.md\",\n \"https://console.groq.com/docs/compound/use-cases.md\",\n \"https://console.groq.com/docs/content-moderation.md\",\n \"https://console.groq.com/docs/deprecations.md\",\n \"https://console.groq.com/docs/errors.md\",\n \"https://console.groq.com/docs/examples.md\",\n \"https://console.groq.com/docs/flex-processing.md\",\n \"https://console.groq.com/docs/guides/speech-to-text/prompting.md\",\n \"https://console.groq.com/docs/integrations.md\",\n \"https://console.groq.com/docs/legal.md\",\n \"https://console.groq.com/docs/legal/ai-policy.md\",\n \"https://console.groq.com/docs/legal/services-agreement.md\",\n \"https://console.groq.com/docs/libraries.md\",\n \"https://console.groq.com/docs/llama-index.md\",\n \"https://console.groq.com/docs/lora.md\",\n \"https://console.groq.com/docs/mastra.md\",\n \"https://console.groq.com/docs/mcp.md\",\n \"https://console.groq.com/docs/mlflow.md\",\n \"https://console.groq.com/docs/model-permissions.md\",\n \"https://console.groq.com/docs/model/llama-3.1-8b-instant.md\",\n \"https://console.groq.com/docs/model/llama-3.3-70b-versatile.md\",\n \"https://console.groq.com/docs/model/llama-4-maverick-17b-128e-instruct.md\",\n \"https://console.groq.com/docs/model/llama-4-scout-17b-16e-instruct.md\",\n \"https://console.groq.com/docs/model/llama-guard-4-12b.md\",\n \"https://console.groq.com/docs/model/llama-prompt-guard-2-22m.md\",\n \"https://console.groq.com/docs/model/llama-prompt-guard-2-86m.md\",\n \"https://console.groq.com/docs/model/meta-llama/llama-4-maverick-17b-128e-instruct.md\",\n \"https://console.groq.com/docs/model/meta-llama/llama-4-scout-17b-16e-instruct.md\",\n \"https://console.groq.com/docs/model/meta-llama/llama-guard-4-12b.md\",\n \"https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-22m.md\",\n \"https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m.md\",\n \"https://console.groq.com/docs/model/moonshotai/kimi-k2-instruct.md\",\n \"https://console.groq.com/docs/model/moonshotai/kimi-k2-instruct-0905.md\",\n \"https://console.groq.com/docs/model/openai/gpt-oss-120b.md\",\n \"https://console.groq.com/docs/model/openai/gpt-oss-20b.md\",\n \"https://console.groq.com/docs/model/openai/gpt-oss-safeguard-20b.md\",\n \"https://console.groq.com/docs/model/playai-tts.md\",\n \"https://console.groq.com/docs/model/playai-tts-arabic.md\",\n \"https://console.groq.com/docs/model/qwen/qwen3-32b.md\",\n \"https://console.groq.com/docs/model/qwen3-32b.md\",\n \"https://console.groq.com/docs/model/whisper-large-v3.md\",\n \"https://console.groq.com/docs/model/whisper-large-v3-turbo.md\",\n \"https://console.groq.com/docs/models.md\",\n \"https://console.groq.com/docs/overview.md\",\n \"https://console.groq.com/docs/prefilling.md\",\n \"https://console.groq.com/docs/production-readiness/optimizing-latency.md\",\n \"https://console.groq.com/docs/production-readiness/production-ready-checklist.md\",\n \"https://console.groq.com/docs/production-readiness/security-onboarding.md\",\n \"https://console.groq.com/docs/projects.md\",\n \"https://console.groq.com/docs/prompt-caching.md\",\n \"https://console.groq.com/docs/prompting.md\",\n \"https://console.groq.com/docs/prompting/model-migration.md\",\n \"https://console.groq.com/docs/prompting/patterns.md\",\n \"https://console.groq.com/docs/quickstart.md\",\n \"https://console.groq.com/docs/rate-limits.md\",\n \"https://console.groq.com/docs/reasoning.md\",\n \"https://console.groq.com/docs/responses-api.md\",\n \"https://console.groq.com/docs/speech-text.md\",\n \"https://console.groq.com/docs/speech-to-text.md\",\n \"https://console.groq.com/docs/spend-limits.md\",\n \"https://console.groq.com/docs/structured-outputs.md\",\n \"https://console.groq.com/docs/text-chat.md\",\n \"https://console.groq.com/docs/text-to-speech.md\",\n \"https://console.groq.com/docs/tool-use.md\",\n \"https://console.groq.com/docs/vision.md\",\n \"https://console.groq.com/docs/web-search.md\",\n \"https://console.groq.com/docs/your-data.md\",\n ];\n",[152],"80956108-b9ca-11f0-83d0-0224a6c84d84","todo.md","7ebe7d74-b8ee-11f0-bdd9-0224a6c84d84",94,["D",1762293984010],2,"\n## todos\n- [x] try search strategies\n - [ ] add kapa.ai as baseline\n - [ ] create a test suite -> doc with all the comparisons\n - [ ] actually make it flexible / selectable again instead of comment out for easier benchmarking\n- [x] markdown endpoint\n- [x] pre-calculate metadata around each markdown page\n- [x] pre-calculate complex question query q&a pairs that map to various pages and categories\n\n\n## future\n- [ ] (Ben) can we have a blind eval ranker - aka we show the two results side by side and you pick A or B and do this like 20 times, then it shows you which one ranked better\n\n\n\n## thoughts\n- for TINY DATA SETS you do very different things than massive data sets; vectorize, mixedbread, turbopuffer are all set up for MASSIVE data sets\n- doing 2sec responses is EASY; sub 1-sec prob requires vps\n- network hops + isolate warmups kill latency - cosine dist calc and embeddings calc don't matter as much\n- small/cheap embeddings seem to not be a problem (?? quality??)\n- generating embeddings locally w/ a small model requires downloading a model (size+computationally expensive) while generating embeddings from an API costs around ~600ms\n- loading an 80mb xenova--all-miniLM is really good/fast but you it sucks for serverless and for mobile users to download that\n - possibly best way is to host this somewhere w/ the onnx and everything saved locally and you can just run it\n- cloudflare ai embeddings are very fast, but worker needs to be warmed up; if not warm then expect ~800ms-2000ms\n - if you ping the cf ai embeddings with a fake request for warmup then it's fast (e.g. while a user is typing)\n- for testing at least, loading in a massive json file into memory takes a very long time 10+ seconds\n-",[160],"807b7306-b9ca-11f0-83d0-0224a6c84d84","main.tsx",91,["D",1762293983847],15,"http","import { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport \"jsr:@std/dotenv/load\";\nimport {\n IS_VALTOWN,\n getFromCache,\n setCache,\n clearCache,\n getCacheStats,\n getTextFromUrl,\n parseFrontmatter,\n addUrlSourceToFrontmatter,\n calculateTokenCount,\n calculateContentHash,\n} from \"./utils.ts\";\nimport { generatePageMetadata } from \"./groq.ts\";\nimport { searchPages, getActiveEmbeddingsFunction, type Page } from \"./search/index.ts\";\nimport { answerQuestion, answerStrategy } from \"./answer/index.ts\";\nimport { urls } from \"./urls.ts\";\n\nconst app = new Hono();\n\n\n\n\n\n\nconst getPageContent = async () => {\n const promises = urls.map(async url => {\n const text = await getTextFromUrl(url);\n return addUrlSourceToFrontmatter(text, url);\n });\n const texts = await Promise.all(promises);\n return texts.join(\"\\n\\n\");\n};\n\nconst getPageData = async () => {\n const pageData = await Promise.all(\n urls.map(async (url) => {\n // Check if cached\n const cached = await getFromCache(url);\n let text, processedText, charCount, frontmatter, wasCached;\n \n let metadata = null;\n \n if (cached) {\n // Use cached data\n text = cached.content;\n processedText = cached.content;\n charCount = cached.charCount;\n frontmatter = cached.frontmatter;\n metadata = cached.metadata;\n wasCached = true;\n } else {\n // Fetch fresh\n text = await getTextFromUrl(url);\n processedText = addUrlSourceToFrontmatter(text, url);\n charCount = processedText.length;\n frontmatter = parseFrontmatter(processedText);\n wasCached = false;\n }\n \n // Extract urlSource and rename it to url, then flatten remaining frontmatter properties\n const urlSource = (frontmatter as any).urlSource;\n const restFrontmatter = { ...frontmatter };\n delete (restFrontmatter as any).urlSource;\n \n return {\n url: urlSource || url, // Use urlSource from frontmatter, fallback to url\n content: processedText,\n charCount,\n cached: wasCached,\n ...restFrontmatter, // Flatten frontmatter properties\n ...(metadata || {}), // Include AI-generated metadata if available\n };\n })\n );\n \n return {\n pages: pageData,\n totalPages: pageData.length,\n totalChars: pageData.reduce((sum, { charCount }) => sum + charCount, 0),\n cachedPages: pageData.filter(({ cached }) => cached).length,\n };\n};\n\napp.get(\"/\", async (c) => {\n const content = await getPageContent();\n return c.text(content);\n});\n\napp.get(\"/data\", async (c) => {\n const data = await getPageData();\n \n // Extract page paths/names (same logic as /list endpoint)\n const pagePaths = urls.map(url => {\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n if (match) {\n return match[1];\n }\n if (url.endsWith(\"/docs.md\")) {\n return \"docs\";\n }\n return null;\n }).filter(path => path !== null);\n \n // Return metadata first, then the large data object\n return c.json({\n metadata: {\n urls: pagePaths,\n },\n ...data,\n });\n});\n\napp.get(\"/list\", async (c) => {\n const paths = urls.map(url => {\n // Extract the path part after /docs/ and remove .md extension\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n if (match) {\n return match[1];\n }\n // Handle the case where URL is just /docs.md - return \"docs\"\n if (url.endsWith(\"/docs.md\")) {\n return \"docs\";\n }\n return null;\n }).filter(path => path !== null);\n \n return c.json(paths);\n});\n\n// Helper function to load data from latest.json (for local development)\nasync function loadLatestJsonForSearch(): Promise | null> {\n if (IS_VALTOWN) {\n return null; // Use SQLite when in Val Town\n }\n \n try {\n const latestData = await Deno.readTextFile(\"./data/latest.json\");\n \n // Use fast-json for faster parsing (only extracts what we need)\n const { parseLatestJson } = await import(\"./utils/json-parser.ts\");\n const data = await parseLatestJson(latestData, { enableTiming: false });\n \n const cacheMap: Record = {};\n \n if (data.results && Array.isArray(data.results)) {\n for (const result of data.results) {\n if (result.url) { // fast-json already filters skipped/error entries\n cacheMap[result.url] = {\n title: result.title,\n embeddings: result.embeddings || null,\n metadata: result.metadata || null,\n };\n }\n }\n }\n \n return cacheMap;\n } catch {\n return null;\n }\n}\n\n// Helper function to get all pages for searching\n// IMPORTANT: This ONLY uses cached data (JSON locally, SQLite on Val Town)\n// It NEVER fetches live data - that's what recalculate is for!\n// Run `deno task recalculate` to populate the cache before searching.\nexport const getAllPagesForSearch = async (enableTiming: boolean = false): Promise => {\n const timings: Record = {};\n const startTime = performance.now();\n \n // Load JSON cache when running locally (Val Town uses SQLite instead)\n const jsonCacheStart = performance.now();\n const jsonCache = await loadLatestJsonForSearch();\n if (enableTiming) {\n timings.jsonCacheLoad = performance.now() - jsonCacheStart;\n if (IS_VALTOWN) {\n // On Val Town, JSON cache is skipped (uses SQLite instead)\n timings.jsonCacheLoad = 0;\n }\n }\n \n const pageDataStart = performance.now();\n const uncachedUrls: string[] = [];\n \n const pageData: (Page | null)[] = await Promise.all(\n urls.map(async (url): Promise => {\n // Extract path from URL\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n const path = match ? match[1] : (url.endsWith(\"/docs.md\") ? \"docs\" : null);\n \n if (!path) {\n return null;\n }\n \n // Try to get from SQLite cache first (Val Town)\n const cached = await getFromCache(url);\n \n if (cached) {\n const frontmatter = cached.frontmatter;\n // On Val Town, SQLite cache already has everything we need\n // Content is available, embeddings are available - no need to fetch!\n return {\n path,\n url: cached.frontmatter?.urlSource || url,\n content: cached.content, // Already cached in SQLite, no fetch needed!\n title: frontmatter?.title || path,\n embeddings: cached.embeddings, // Already cached in SQLite\n metadata: cached.metadata,\n };\n }\n \n // If not in SQLite, try JSON cache (when running locally)\n if (jsonCache && jsonCache[url]) {\n const jsonData = jsonCache[url];\n // JSON cache has embeddings, title, metadata from recalculate\n // We skip fetching content to speed up search - content is only for snippets\n // NO NETWORK CALLS - everything is from the JSON cache!\n if (jsonData.embeddings && Array.isArray(jsonData.embeddings) && jsonData.embeddings.length > 0) {\n // Use cached data - NO network fetch! This is why it's fast (was 6.5s, now 0.16ms)\n return {\n path,\n url: url,\n content: '', // Skipped - only needed for snippets, can fetch lazily if needed\n title: jsonData.title || path,\n embeddings: jsonData.embeddings, // From JSON cache (from recalculate)\n metadata: jsonData.metadata || null,\n };\n } else {\n // Missing embeddings - this shouldn't happen if recalculate ran successfully\n uncachedUrls.push(url);\n }\n } else {\n // Not in cache at all!\n uncachedUrls.push(url);\n }\n \n // If not cached anywhere OR cached data has no embeddings, skip it\n // (search requires embeddings to work, and we NEVER fetch live data - that's the point!)\n return null;\n })\n );\n if (enableTiming) timings.pageDataLoad = performance.now() - pageDataStart;\n \n // Filter out nulls\n const filterStart = performance.now();\n const filtered = pageData.filter((p): p is Page => p !== null);\n if (enableTiming) timings.filter = performance.now() - filterStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(`⏱️ getAllPagesForSearch:`);\n if (IS_VALTOWN) {\n console.log(` SQLite cache load: ${timings.jsonCacheLoad?.toFixed(2)}ms (Val Town - using SQLite)`);\n } else if (timings.jsonCacheLoad) {\n console.log(` JSON cache load (local): ${timings.jsonCacheLoad.toFixed(2)}ms`);\n }\n console.log(` Page data load: ${timings.pageDataLoad?.toFixed(2)}ms`);\n console.log(` Filter: ${timings.filter?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total.toFixed(2)}ms`);\n \n if (uncachedUrls.length > 0) {\n console.warn(` ⚠️ ${uncachedUrls.length} pages not in cache (skipped): ${uncachedUrls.slice(0, 3).join(', ')}${uncachedUrls.length > 3 ? '...' : ''}`);\n console.warn(` 💡 Run 'deno task recalculate' to populate cache`);\n }\n }\n \n return filtered;\n};\n\n// Search endpoint - uses active strategy from search/index.ts\napp.get(\"/search\", async (c) => {\n const query = c.req.query(\"q\");\n const limit = parseInt(c.req.query(\"limit\") || \"10\");\n const minScore = parseFloat(c.req.query(\"minScore\") || \"0\");\n \n if (!query) {\n return c.json({ error: \"Query parameter 'q' is required\" }, 400);\n }\n \n const validPages = await getAllPagesForSearch();\n \n // Perform search using active strategy\n const results = await searchPages(query, validPages, {\n limit,\n minScore,\n });\n \n return c.json({\n query,\n results,\n totalResults: results.length,\n totalPages: validPages.length,\n });\n});\n\n// Test search endpoint - runs test queries against active search strategy\napp.get(\"/search/test\", async (c) => {\n try {\n // Import test utilities and get active strategy info\n const { runSearchTest } = await import(\"./testing/search-harness.ts\");\n const { questions } = await import(\"./testing/questions.ts\");\n const { searchStrategy } = await import(\"./search/index.ts\");\n \n const validPages = await getAllPagesForSearch();\n const testQueries = questions;\n \n const testResults: Array<{\n query: string;\n results: Array<{\n path: string;\n url: string;\n title?: string;\n score: number;\n snippet: string;\n metadata?: {\n strategy?: string;\n [key: string]: any;\n };\n }>;\n durationMs: number;\n resultCount: number;\n error?: string;\n }> = [];\n \n for (const query of testQueries) {\n const testResult = await runSearchTest(query, validPages, {\n limit: 10,\n minScore: 0,\n enableTiming: true,\n });\n \n testResults.push({\n query,\n results: testResult.results.map(r => ({\n path: r.path,\n url: r.url,\n title: r.title,\n score: r.score,\n snippet: r.snippet?.substring(0, 200) || '', // Limit snippet length\n metadata: r.metadata,\n })),\n durationMs: testResult.durationMs,\n resultCount: testResult.results.length,\n error: testResult.error || undefined,\n });\n }\n \n return c.json({\n strategy: {\n name: searchStrategy.name,\n description: searchStrategy.description,\n },\n totalQueries: testQueries.length,\n totalPages: validPages.length,\n tests: testResults,\n summary: {\n totalDurationMs: testResults.reduce((sum, t) => sum + t.durationMs, 0),\n avgDurationMs: testResults.reduce((sum, t) => sum + t.durationMs, 0) / testResults.length,\n totalResults: testResults.reduce((sum, t) => sum + t.resultCount, 0),\n errors: testResults.filter(t => t.error).length,\n },\n });\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n return c.json({ \n error: \"Test search failed\", \n message: errorMessage \n }, 500);\n }\n});\n\n// Answer endpoint - uses active answer strategy with RAG (search + LLM)\napp.get(\"/answer\", async (c) => {\n const query = c.req.query(\"q\");\n const limit = parseInt(c.req.query(\"limit\") || \"10\");\n const minScore = parseFloat(c.req.query(\"minScore\") || \"0\");\n const maxContextPages = parseInt(c.req.query(\"maxContextPages\") || \"5\");\n const temperature = parseFloat(c.req.query(\"temperature\") || \"0.3\");\n const model = c.req.query(\"model\");\n \n if (!query) {\n return c.json({ error: \"Query parameter 'q' is required\" }, 400);\n }\n \n try {\n // Generate answer using active strategy\n const result = await answerQuestion(query, {\n limit,\n minScore,\n maxContextPages,\n temperature,\n ...(model && { model }),\n enableTiming: true,\n });\n \n return c.json(result);\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n return c.json({ \n error: \"Answer generation failed\", \n message: errorMessage \n }, 500);\n }\n});\n\n// Answer info endpoint - returns information about the active answer strategy\napp.get(\"/answer/info\", async (c) => {\n return c.json({\n strategy: {\n name: answerStrategy.name,\n description: answerStrategy.description,\n },\n defaultOptions: {\n model: \"llama-3.3-70b-versatile\",\n temperature: 0.3,\n maxContextPages: 5,\n },\n availableParams: {\n q: \"Query string (required)\",\n limit: \"Max search results to consider (default: 10)\",\n minScore: \"Minimum search score threshold (default: 0)\",\n maxContextPages: \"Max pages to include in LLM context (default: 5)\",\n temperature: \"LLM temperature (default: 0.3)\",\n model: \"Override LLM model (optional)\",\n },\n });\n});\n\n// Test answer endpoint - runs test queries against active answer strategy\napp.get(\"/answer/test\", async (c) => {\n try {\n // Import test utilities\n const { runAnswerTest } = await import(\"./testing/answer-harness.ts\");\n const { questions } = await import(\"./testing/questions.ts\");\n \n const testQueries = questions;\n \n const testResults: Array<{\n query: string;\n answer: string;\n searchResults: Array<{\n path: string;\n url: string;\n title?: string;\n score: number;\n }>;\n contextUsed: number;\n totalTokens?: number;\n durationMs: number;\n timings?: Record;\n error?: string;\n }> = [];\n \n for (const query of testQueries) {\n const testResult = await runAnswerTest(query, {\n limit: 10,\n minScore: 0,\n maxContextPages: 5,\n temperature: 0.3,\n });\n \n testResults.push({\n query,\n ...testResult,\n });\n }\n \n // Calculate summary statistics\n const avgTimings = testResults.reduce((acc, t) => {\n if (t.timings) {\n acc.search += t.timings.search || 0;\n acc.contextPrep += t.timings.contextPrep || 0;\n acc.llm += t.timings.llm || 0;\n acc.total += t.timings.total || 0;\n acc.count++;\n }\n return acc;\n }, { search: 0, contextPrep: 0, llm: 0, total: 0, count: 0 });\n \n const avgCount = avgTimings.count || 1;\n \n return c.json({\n strategy: {\n name: answerStrategy.name,\n description: answerStrategy.description,\n },\n totalQueries: testQueries.length,\n tests: testResults,\n summary: {\n totalDurationMs: testResults.reduce((sum, t) => sum + t.durationMs, 0),\n avgDurationMs: testResults.reduce((sum, t) => sum + t.durationMs, 0) / testResults.length,\n avgSearchMs: avgTimings.search / avgCount,\n avgContextPrepMs: avgTimings.contextPrep / avgCount,\n avgLlmMs: avgTimings.llm / avgCount,\n avgTotalMs: avgTimings.total / avgCount,\n totalContextUsed: testResults.reduce((sum, t) => sum + t.contextUsed, 0),\n totalTokens: testResults.reduce((sum, t) => sum + (t.totalTokens || 0), 0),\n errors: testResults.filter(t => t.error).length,\n },\n });\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n return c.json({ \n error: \"Test answer failed\", \n message: errorMessage \n }, 500);\n }\n});\n\nconst getPageByPath = async (pathParam: string, useCache: boolean = true) => {\n // Find the matching URL\n const matchingUrl = urls.find(url => {\n // Extract the path part after /docs/ and remove .md extension\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n if (match) {\n return match[1] === pathParam;\n }\n // Handle the case where URL is just /docs.md and pathParam is \"docs\"\n if (url.endsWith(\"/docs.md\") && pathParam === \"docs\") {\n return true;\n }\n return false;\n });\n \n if (!matchingUrl) {\n return null;\n }\n \n // Try to get from cache first\n if (useCache) {\n const cached = await getFromCache(matchingUrl);\n if (cached) {\n // Extract urlSource and rename it to url, then flatten remaining frontmatter properties\n const urlSource = (cached.frontmatter as any).urlSource;\n const restFrontmatter = { ...cached.frontmatter };\n delete (restFrontmatter as any).urlSource;\n \n return {\n url: urlSource || matchingUrl,\n content: cached.content,\n charCount: cached.charCount,\n tokenCount: cached.tokenCount,\n cached: true,\n ...restFrontmatter,\n ...(cached.metadata || {}), // Include AI-generated metadata if available\n };\n }\n }\n \n // Fetch and process the single page\n const text = await getTextFromUrl(matchingUrl);\n const processedText = addUrlSourceToFrontmatter(text, matchingUrl);\n const charCount = processedText.length;\n const tokenCount = calculateTokenCount(processedText);\n const frontmatter = parseFrontmatter(processedText);\n \n // Cache the result (without metadata for now - generated during recalculate)\n if (useCache) {\n // Wrap in additional error handling to ensure it never throws\n try {\n await setCache(matchingUrl, {\n content: processedText,\n charCount,\n tokenCount,\n frontmatter,\n });\n } catch (error) {\n // Double safety - setCache should already catch, but just in case\n console.debug(\"Cache write error (ignored):\", error instanceof Error ? error.message.substring(0, 100) : String(error).substring(0, 100));\n }\n }\n \n // Extract urlSource and rename it to url, then flatten remaining frontmatter properties\n const urlSource = (frontmatter as any).urlSource;\n const restFrontmatter = { ...frontmatter };\n delete (restFrontmatter as any).urlSource;\n \n return {\n url: urlSource || matchingUrl, // Use urlSource from frontmatter, fallback to matchingUrl\n content: processedText,\n charCount,\n tokenCount,\n cached: false,\n ...restFrontmatter,\n };\n};\n\napp.get(\"/page/docs\", async (c) => {\n // Handle root docs case\n const pageData = await getPageByPath(\"docs\");\n if (!pageData) {\n return c.json({ error: \"Page not found\" }, 404);\n }\n return c.json(pageData);\n});\n\napp.get(\"/page/*\", async (c) => {\n // Get the path from the URL (everything after /page/)\n const urlPath = new URL(c.req.url).pathname;\n const pathParam = urlPath.replace(\"/page/\", \"\") || \"\";\n \n const pageData = await getPageByPath(pathParam);\n if (!pageData) {\n return c.json({ error: \"Page not found\" }, 404);\n }\n return c.json(pageData);\n});\n\n// Cache management endpoints (all GET for browser access)\napp.get(\"/cache/stats\", async (c) => {\n const stats = await getCacheStats();\n return c.json(stats);\n});\n\napp.get(\"/cache/clear\", async (c) => {\n try {\n await clearCache();\n return c.json({ message: \"Cache cleared\", success: true });\n } catch (error) {\n return c.json({ \n message: \"Cache unavailable (caching disabled)\", \n success: false,\n error: \"SQLite not available - caching disabled\" \n });\n }\n});\n\napp.get(\"/cache/clear/:path\", async (c) => {\n const pathParam = c.req.param(\"path\");\n \n // Find the matching URL\n const matchingUrl = urls.find(url => {\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n if (match) {\n return match[1] === pathParam;\n }\n if (url.endsWith(\"/docs.md\") && pathParam === \"docs\") {\n return true;\n }\n return false;\n });\n \n if (!matchingUrl) {\n return c.json({ error: \"Page not found\" }, 404);\n }\n \n try {\n await clearCache(matchingUrl);\n return c.json({ message: `Cache cleared for ${pathParam}`, success: true });\n } catch (error) {\n return c.json({ \n message: `Cache unavailable for ${pathParam} (caching disabled)`, \n success: false,\n error: \"SQLite not available - caching disabled\" \n });\n }\n});\n\napp.get(\"/cache/recalculate\", async (c) => {\n // Check for force mode (bypass hash checking)\n const force = c.req.query(\"force\") === \"true\";\n \n // Recalculate all pages (bypass cache) with AI metadata generation\n const results: Array<{ \n path: string; \n url: string; \n charCount: number; \n tokenCount: number | undefined;\n title?: string;\n metadata?: any;\n error?: string;\n cacheError?: string;\n skipped?: boolean;\n reason?: string;\n }> = [];\n \n let processedCount = 0;\n let skippedCount = 0;\n \n for (const url of urls) {\n // Extract path from URL\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n const path = match ? match[1] : (url.endsWith(\"/docs.md\") ? \"docs\" : null);\n \n if (path) {\n console.log(`[${processedCount + 1}/${urls.length}] Processing ${path}...`);\n \n try {\n // Check cached hash if not forcing\n let shouldSkip = false;\n if (!force) {\n const cached = await getFromCache(url);\n if (cached && cached.contentHash) {\n // Fetch fresh content to check hash\n try {\n const pageData = await getPageByPath(path, false);\n if (pageData) {\n const newHash = await calculateContentHash(pageData.content);\n if (newHash === cached.contentHash) {\n shouldSkip = true;\n skippedCount++;\n console.log(` Skipped ${path} (content unchanged, hash: ${newHash.substring(0, 8)}...)`);\n results.push({\n path,\n url,\n charCount: cached.charCount,\n tokenCount: cached.tokenCount || undefined,\n title: (cached.frontmatter as any).title || path,\n metadata: cached.metadata || undefined,\n skipped: true,\n reason: \"Content unchanged (hash matches)\",\n });\n }\n }\n } catch (hashCheckError) {\n // If hash check fails, continue with processing\n console.debug(` Hash check failed for ${path}, continuing...`);\n }\n }\n }\n \n if (shouldSkip) {\n processedCount++;\n continue;\n }\n \n // Fetch without cache (rate limiting handled by async-sema)\n let pageData;\n try {\n pageData = await getPageByPath(path, false);\n } catch (fetchError) {\n const fetchErrorMessage = fetchError instanceof Error ? fetchError.message : String(fetchError);\n console.error(` Failed to fetch ${path}:`, fetchErrorMessage);\n throw new Error(`Fetch failed: ${fetchErrorMessage}`);\n }\n \n if (pageData) {\n const frontmatter = parseFrontmatter(pageData.content);\n const title = (frontmatter as any).title || path;\n \n // Calculate content hash\n const contentHash = await calculateContentHash(pageData.content);\n \n // Generate AI metadata (failures are handled inside generatePageMetadata)\n const metadata = await generatePageMetadata(pageData.content, title);\n if (!metadata) {\n console.warn(` Metadata generation failed for ${path} (continuing without metadata)`);\n }\n \n // Generate embeddings using active strategy's embeddings function\n const generateEmbeddings = getActiveEmbeddingsFunction();\n const embeddings = await generateEmbeddings(pageData.content);\n \n // Cache it with metadata, hash, and embeddings - track errors\n const cacheError = await setCache(url, {\n content: pageData.content,\n charCount: pageData.charCount,\n tokenCount: pageData.tokenCount || null,\n frontmatter,\n metadata,\n contentHash,\n embeddings,\n });\n \n results.push({ \n path, \n url, \n charCount: pageData.charCount, \n tokenCount: pageData.tokenCount,\n title,\n metadata, // Include full metadata in results\n ...(cacheError && { cacheError }), // Include cache error if present\n });\n \n if (cacheError) {\n console.warn(`Completed ${path} (metadata: ${metadata ? 'yes' : 'no'}, embeddings: ${embeddings ? 'yes' : 'no'}, cache error: ${cacheError})`);\n } else {\n console.log(`Completed ${path} (metadata: ${metadata ? 'yes' : 'no'}, embeddings: ${embeddings ? 'yes' : 'no'}, cached: yes)`);\n }\n }\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(`Error processing ${path}:`, errorMessage);\n results.push({\n path,\n url,\n charCount: 0,\n tokenCount: 0,\n error: errorMessage,\n });\n }\n \n processedCount++;\n }\n }\n \n const cacheErrors = results.filter(r => r.cacheError);\n const processedResults = results.filter(r => !r.skipped);\n \n return c.json({\n message: force \n ? `Recalculated ${processedResults.length} pages with AI metadata (force mode)` \n : `Recalculated ${processedResults.length} pages, skipped ${skippedCount} unchanged pages`,\n results, // Now includes full metadata for each page\n totalPages: results.length,\n processed: processedResults.length,\n skipped: skippedCount,\n withMetadata: processedResults.filter(r => r.metadata).length,\n withoutMetadata: processedResults.filter(r => !r.metadata).length,\n cached: processedResults.length > 0 && cacheErrors.length === 0, // True if all pages cached successfully\n cacheErrors: cacheErrors.length > 0 ? cacheErrors.map(r => ({ path: r.path, error: r.cacheError })) : undefined,\n force,\n });\n});\n\n// Helper function to save test data to /data/mm-dd-yyyy.json and /data/latest.json\nexport const saveTestData = async (data: any, customDate?: Date) => {\n const date = customDate || new Date();\n const month = String(date.getMonth() + 1).padStart(2, '0');\n const day = String(date.getDate()).padStart(2, '0');\n const year = date.getFullYear();\n const filename = `${month}-${day}-${year}.json`;\n const datedFilepath = `./data/${filename}`;\n const latestFilepath = `./data/latest.json`;\n \n try {\n // Ensure data directory exists\n try {\n await Deno.mkdir(\"./data\", { recursive: true });\n } catch {\n // Directory already exists, ignore\n }\n \n const jsonData = JSON.stringify(data, null, 2);\n \n // Write both files\n await Deno.writeTextFile(datedFilepath, jsonData);\n await Deno.writeTextFile(latestFilepath, jsonData);\n \n console.log(`Data saved to ${datedFilepath} and ${latestFilepath}`);\n return { datedFilepath, latestFilepath };\n } catch (error) {\n console.error(`Failed to save test data:`, error);\n throw error;\n }\n};\n\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;\n","groqdocs",[168],"9315b67c-b8f1-11f0-85fe-0224a6c84d84","groq.ts","9315b744-b8f1-11f0-85fe-0224a6c84d84",23,["D",1762200814266],"import { RateLimit } from \"npm:async-sema@latest\";\n\n// Groq API Configuration\nconst GROQ_API_KEY = Deno.env.get(\"GROQ_API_KEY\");\nconst GROQ_API_URL = \"https://api.groq.com/openai/v1/chat/completions\";\n\n// Rate limiter for Groq API: 2 requests per second to avoid rate limits\nconst groqApiRateLimiter = RateLimit(2);\n\n// Groq Chat Completions Utility\nexport const groqChatCompletion = async (messages: Array<{ role: string; content: string }>, options: {\n model?: string;\n temperature?: number;\n max_tokens?: number;\n response_format?: { type: string };\n} = {}) => {\n if (!GROQ_API_KEY) {\n console.warn(\"GROQ_API_KEY not found - AI metadata generation disabled\");\n return null;\n }\n\n // Rate limit Groq API calls\n await groqApiRateLimiter();\n\n try {\n const response = await fetch(GROQ_API_URL, {\n method: \"POST\",\n headers: {\n \"Authorization\": `Bearer ${GROQ_API_KEY}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n model: options.model || \"llama-3.3-70b-versatile\",\n messages,\n temperature: options.temperature ?? 0.4,\n // max_tokens: options.max_tokens ?? 1024,\n ...(options.response_format && { response_format: options.response_format }),\n }),\n });\n\n if (!response.ok) {\n const errorText = await response.text().catch(() => response.statusText);\n throw new Error(`Groq API error: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);\n }\n\n const data = await response.json();\n return data.choices[0]?.message?.content || null;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"Groq API call failed:\", errorMessage);\n // Don't throw - return null so metadata generation doesn't break the whole process\n return null;\n }\n};\n\n// Generate AI metadata for a page\nexport const generatePageMetadata = async (content: string, title: string) => {\n const prompt = `Analyze this documentation page and provide metadata in JSON format.\n\nPage Title: ${title}\n\nContent:\n${content.substring(0, 4000)} ${content.length > 4000 ? '...(truncated)' : ''}\n\nGenerate the following metadata:\n1. categories: Array of 2-4 broad categories (e.g., \"API\", \"Authentication\", \"Models\", \"Integration\")\n2. tags: Array of 5-10 specific tags/keywords\n3. useCases: Array of 2-4 practical use cases or scenarios when this page would be helpful\n4. questions: Array of 5-10 questions users might ask that this page answers\n\nReturn ONLY valid JSON in this exact format:\n{\n \"categories\": [\"Category1\", \"Category2\"],\n \"tags\": [\"tag1\", \"tag2\", \"tag3\"],\n \"useCases\": [\"Use case 1\", \"Use case 2\"],\n \"questions\": [\"Question 1?\", \"Question 2?\"]\n}`;\n\n const response = await groqChatCompletion([\n { role: \"user\", content: prompt }\n ], {\n model: \"llama-3.3-70b-versatile\",\n temperature: 0.3,\n max_tokens: 1024,\n });\n\n if (!response) {\n return null;\n }\n\n try {\n // Extract JSON from response (in case it's wrapped in markdown)\n const jsonMatch = response.match(/\\{[\\s\\S]*\\}/);\n if (jsonMatch) {\n return JSON.parse(jsonMatch[0]);\n }\n return JSON.parse(response);\n } catch (error) {\n console.error(\"Failed to parse AI metadata:\", error);\n return null;\n }\n};\n\n",[177],"808c02b6-b9ca-11f0-9b64-0224a6c84d84","deno.json","656e5404-b8e7-11f0-a061-0224a6c84d84",93,["D",1762293983950],3,"{\n \"tasks\": {\n \"serve\": \"deno serve --allow-net --allow-import --allow-read --allow-write --allow-env main.tsx\",\n \"recalc\": \"deno run --allow-ffi --allow-net --allow-import --allow-read --allow-write --allow-env testing/recalculate.ts\",\n \"recalc-force\": \"deno run --allow-ffi --allow-net --allow-import --allow-read --allow-write --allow-env testing/recalculate.ts --force\",\n \"recalc-mxbai\": \"deno run --allow-net --allow-import --allow-read --allow-write --allow-env testing/recalculate-mixedbread.ts\",\n \"recalc-mxbai-force\": \"deno run --allow-net --allow-import --allow-read --allow-write --allow-env testing/recalculate-mixedbread.ts --force\",\n \"search\": \"deno run --allow-ffi --allow-net --allow-import --allow-read --allow-write --allow-env testing/test-search.ts\",\n \"answer\": \"deno run --allow-ffi --allow-net --allow-import --allow-read --allow-write --allow-env testing/test-answer.ts\"\n },\n \"$schema\": \"https://raw.githubusercontent.com/denoland/deno/98f62cee78e85bfc47c62ed703777c6bc8794f1c/cli/schemas/config-file.v1.json\",\n \"lock\": false,\n \"compilerOptions\": {\n \"noImplicitAny\": false,\n \"strict\": true,\n \"types\": [\"https://www.val.town/types/valtown.d.ts\"],\n \"lib\": [\n \"dom\",\n \"dom.iterable\",\n \"dom.asynciterable\",\n \"deno.ns\",\n \"deno.unstable\"\n ]\n },\n \"lint\": {\n \"include\": [\"deno:/https/esm.town/**/*\"],\n \"rules\": {\n \"exclude\": [\"no-explicit-any\"]\n }\n },\n \"node_modules_dir\": false,\n \"experimental\": {\n \"unstable-node-globals\": true,\n \"unstable-temporal\": true,\n \"unstable-worker-options\": true,\n \"unstable-sloppy-imports\": true\n }\n}",[184],"936414fc-b8f1-11f0-92bd-0224a6c84d84","README.md","6566e58e-b8e7-11f0-9e62-0224a6c84d84",25,["D",1762200814124],"# Groq Docs API\n\nA Hono API server that fetches, caches, and processes Groq documentation pages with token counting and AI-generated metadata.\n\n## Features\n\n- Fetches documentation pages from Groq's console\n- Caches page content, metadata, token counts, and embeddings in SQLite\n- Token counting using tiktoken (GPT-4 encoding)\n- AI-generated metadata (categories, tags, use cases, sample questions)\n- Content embeddings generation (currently fake, ready for Groq API integration)\n- Hash-based change detection to skip unchanged pages during recalculation\n- Rate limiting with async-sema to avoid WAF blocking\n- RESTful API endpoints for accessing pages and managing cache\n- Modular code structure (utils.ts for utilities, groq.ts for Groq API functions)\n\n## First-Time Setup\n\n### 1. Initial Cache Population\n\nOn first run, the cache will be empty. You should populate it by running:\n\n```bash\nGET /cache/recalculate\n```\n\nThis will:\n- Fetch all pages from the URLs list\n- Calculate token counts for each page\n- Generate AI metadata (categories, tags, use cases, questions)\n- Generate embeddings for each page\n- Calculate content hashes for change detection\n- Store everything in the SQLite cache\n- Return a summary of what was cached\n\n**Important**: This will take some time as it processes all pages, generates metadata, and calculates tokens for each. Be patient!\n\n**Note**: On subsequent runs, unchanged pages (detected by content hash) will be automatically skipped unless you use force mode.\n\n### 2. Verify Cache\n\nCheck that the cache was populated:\n\n```bash\nGET /cache/stats\n```\n\nThis returns:\n```json\n{\n \"cachedPages\": 121,\n \"totalTokens\": 1234567\n}\n```\n\n## When to Recalculate\n\nYou should run `/cache/recalculate` in these scenarios:\n\n### ✅ Required Recalculations\n\n1. **First time setup** - Cache is empty\n2. **URL list changes** - You've added or removed URLs from the `urls` array\n3. **Content updates** - Documentation pages have been updated and you want fresh data\n4. **Token count needed** - You need accurate token counts for new content\n5. **Metadata refresh** - You want to regenerate AI metadata or embeddings\n\n### 🔄 Default Mode (Smart Recalculation)\n\nBy default, `/cache/recalculate` uses hash-based change detection:\n\n```bash\nGET /cache/recalculate\n```\n\n**Behavior:**\n- Fetches each page and calculates its content hash (SHA-256)\n- Compares hash with cached version\n- **Skips pages with unchanged content** (saves time and API calls)\n- Only processes pages that have changed\n- Still generates embeddings and metadata for changed pages\n\n**Response includes:**\n- `processed` - Number of pages actually processed\n- `skipped` - Number of pages skipped (unchanged)\n- `force` - Always `false` in default mode\n\n### ⚡ Force Mode (Recalculate Everything)\n\nTo force recalculation of all pages (ignoring hash checks):\n\n```bash\nGET /cache/recalculate?force=true\n```\n\n**Use cases:**\n- Regenerating all metadata/embeddings even if content unchanged\n- After updating metadata generation prompts\n- When you want to ensure everything is fresh\n\n### ⚠️ Partial Updates\n\nFor single page updates, you can use:\n\n```bash\nGET /cache/clear/:path\n```\n\nThis clears the cache for a specific page. The next time that page is requested via `/page/:path`, it will be fetched fresh and recached.\n\n### 🔄 Routine Maintenance\n\n- **Weekly**: Run recalculate (default mode) to catch any documentation updates efficiently\n- **After major docs changes**: Use force mode to regenerate everything\n- **When adding new pages**: Update the `urls` array, then run recalculate\n\n## API Endpoints\n\n### Page Endpoints\n\n#### `GET /page/docs`\nGet the root docs page (cached if available).\n\n#### `GET /page/:path`\nGet a specific page by path. Examples:\n- `/page/api-reference`\n- `/page/agentic-tooling/compound-beta`\n- `/page/model/llama-3.1-8b-instant`\n\n**Response includes:**\n- `url` - The source URL\n- `content` - Full page content with frontmatter\n- `charCount` - Character count\n- `tokenCount` - Token count (calculated with tiktoken)\n- All frontmatter fields flattened (title, description, image, etc.)\n\n**Caching**: Responses are cached. First request fetches and caches, subsequent requests are instant.\n\n#### `GET /list`\nGet a list of all available page paths.\n\n**Response:**\n```json\n[\n \"docs\",\n \"agentic-tooling\",\n \"api-reference\",\n ...\n]\n```\n\n#### `GET /search`\nSearch pages by query string.\n\n**Query Parameters:**\n- `q` (required) - Search query string\n- `limit` (optional) - Maximum number of results (default: 10)\n- `minScore` (optional) - Minimum score threshold (default: 0)\n\n**Example:**\n```bash\nGET /search?q=authentication&limit=5\n```\n\n**Response:**\n```json\n{\n \"query\": \"authentication\",\n \"results\": [\n {\n \"path\": \"api-reference\",\n \"url\": \"https://console.groq.com/docs/api-reference.md\",\n \"title\": \"API Reference\",\n \"score\": 45,\n \"snippet\": \"...authentication tokens are required for all API requests...\"\n },\n {\n \"path\": \"quickstart\",\n \"url\": \"https://console.groq.com/docs/quickstart.md\",\n \"title\": \"Quick Start\",\n \"score\": 32,\n \"snippet\": \"...get your API key for authentication...\"\n }\n ],\n \"totalResults\": 2,\n \"totalPages\": 121\n}\n```\n\n**Search Features:**\n- Keyword matching in titles and content\n- Metadata boost (tags, categories, use cases)\n- Score-based ranking\n- Content snippets around matches\n- Uses cached pages when available for faster results\n\n**Note**: Currently uses keyword-based search. Future versions will use embeddings for semantic search.\n\n#### `GET /data`\nGet metadata for all pages (does not use cache - fetches fresh).\n\n**Response:**\n```json\n{\n \"pages\": [\n {\n \"url\": \"...\",\n \"charCount\": 1234,\n \"frontmatter\": {...}\n }\n ],\n \"contents\": [...],\n \"totalPages\": 121,\n \"totalChars\": 1234567\n}\n```\n\n### Cache Management Endpoints\n\n#### `GET /cache/stats`\nGet cache statistics.\n\n**Response:**\n```json\n{\n \"cachedPages\": 121,\n \"totalTokens\": 1234567\n}\n```\n\n#### `GET /cache/clear`\nClear the entire cache.\n\n**Response:**\n```json\n{\n \"message\": \"Cache cleared\",\n \"success\": true\n}\n```\n\n#### `GET /cache/clear/:path`\nClear cache for a specific page.\n\n**Example:**\n```bash\nGET /cache/clear/api-reference\n```\n\n**Response:**\n```json\n{\n \"message\": \"Cache cleared for api-reference\",\n \"success\": true\n}\n```\n\n#### `GET /cache/recalculate`\nRecalculate pages with AI metadata and embeddings generation.\n\n**Query Parameters:**\n- `force` (optional): Set to `true` to force recalculation of all pages, ignoring hash checks\n\n**Default Mode** (no query params):\n```bash\nGET /cache/recalculate\n```\n\n**Force Mode**:\n```bash\nGET /cache/recalculate?force=true\n```\n\n**Response** (Default Mode):\n```json\n{\n \"message\": \"Recalculated 5 pages, skipped 116 unchanged pages\",\n \"results\": [\n {\n \"path\": \"api-reference\",\n \"url\": \"https://console.groq.com/docs/api-reference.md\",\n \"charCount\": 1234,\n \"tokenCount\": 567,\n \"title\": \"API Reference\",\n \"metadata\": {\n \"categories\": [\"API\", \"Reference\"],\n \"tags\": [\"api\", \"endpoints\", \"rest\"],\n \"useCases\": [\"Integrating with Groq API\"],\n \"questions\": [\"How do I authenticate?\", \"What endpoints are available?\"]\n }\n },\n {\n \"path\": \"docs\",\n \"skipped\": true,\n \"reason\": \"Content unchanged (hash matches)\"\n }\n ],\n \"totalPages\": 121,\n \"processed\": 5,\n \"skipped\": 116,\n \"withMetadata\": 5,\n \"withoutMetadata\": 0,\n \"cached\": true,\n \"force\": false\n}\n```\n\n**Response** (Force Mode):\n```json\n{\n \"message\": \"Recalculated 121 pages with AI metadata (force mode)\",\n \"results\": [...],\n \"totalPages\": 121,\n \"processed\": 121,\n \"skipped\": 0,\n \"force\": true\n}\n```\n\n**What it does:**\n- Fetches all pages (or skips unchanged ones in default mode)\n- Calculates token counts\n- Generates AI metadata (categories, tags, use cases, questions)\n- Generates embeddings (currently fake, ready for Groq API)\n- Calculates content hashes for change detection\n- Stores everything in cache\n\n**Important**: This can take several minutes depending on:\n- Number of pages to process (skipped pages are fast)\n- Network speed\n- Token calculation time\n- AI metadata generation time (uses Groq API)\n\n## Cache Behavior\n\n### How Caching Works\n\n1. **First Request**: \n - Check cache → Not found\n - Fetch from URL\n - Calculate tokens\n - Store in cache\n - Return data\n\n2. **Subsequent Requests**:\n - Check cache → Found\n - Return cached data immediately\n\n### Cache Storage\n\nCache is stored in SQLite with the following schema:\n\n```sql\nCREATE TABLE groq_docs_cache_v3 (\n url TEXT PRIMARY KEY,\n content TEXT NOT NULL,\n charCount INTEGER NOT NULL,\n tokenCount INTEGER,\n frontmatter TEXT NOT NULL,\n metadata TEXT,\n contentHash TEXT,\n embeddings TEXT,\n cachedAt INTEGER NOT NULL\n)\n```\n\n**Fields:**\n- `url` - Source URL (primary key)\n- `content` - Full page content with frontmatter\n- `charCount` - Character count\n- `tokenCount` - Token count (calculated with tiktoken)\n- `frontmatter` - Parsed frontmatter (JSON)\n- `metadata` - AI-generated metadata (categories, tags, use cases, questions)\n- `contentHash` - SHA-256 hash of content (for change detection)\n- `embeddings` - Content embeddings vector (JSON array)\n- `cachedAt` - Timestamp when cached\n\n### Cache Invalidation\n\nCache is invalidated when:\n- You manually clear it via `/cache/clear`\n- You recalculate via `/cache/recalculate`\n- Cache is cleared for a specific page via `/cache/clear/:path`\n\n**Note**: Cache does NOT automatically expire. If documentation changes, you must manually recalculate.\n\n## Adding New Pages\n\n1. **Add URL to the `urls` array** in `main.tsx`:\n ```typescript\n const urls = [\n // ... existing URLs\n \"https://console.groq.com/docs/new-page.md\",\n ];\n ```\n\n2. **Run recalculate**:\n ```bash\n POST /cache/recalculate\n ```\n\n3. **Verify**:\n ```bash\n GET /cache/stats\n GET /list # Should include your new page\n ```\n\n## Token Counting\n\nToken counts are calculated using [tiktoken](https://github.com/openai/tiktoken) with the `gpt-4` encoding (cl100k_base). This is the same encoding used by:\n- GPT-4\n- GPT-3.5-turbo\n- Many other OpenAI models\n\nToken counts are:\n- Calculated on first fetch\n- Stored in cache\n- Returned in API responses\n- Expensive to compute (which is why caching is important)\n\n## AI Metadata Generation\n\nEach page can have AI-generated metadata using Groq's chat completions API:\n\n- **Categories**: 2-4 broad categories (e.g., \"API\", \"Authentication\", \"Models\")\n- **Tags**: 5-10 specific tags/keywords\n- **Use Cases**: 2-4 practical use cases or scenarios\n- **Questions**: 5-10 questions users might ask\n\nMetadata is generated during `/cache/recalculate` and stored in the cache.\n\n## Search\n\nThe API includes a search endpoint (`/search`) that allows you to search across all documentation pages.\n\n### Current Implementation (Keyword-Based)\n\nCurrently uses keyword matching:\n- Searches in page titles and content\n- Boosts results matching metadata (tags, categories, use cases)\n- Returns ranked results with relevance scores\n- Includes content snippets around matches\n\n### Future Implementation (Semantic Search)\n\nThe search system is designed to support embeddings-based semantic search:\n- `generateEmbeddings()` - Generates embeddings (currently fake, ready for real API)\n- `vectorSearch()` - Vector similarity search function (ready to use when embeddings are real)\n- Will enable semantic understanding of queries (not just keyword matching)\n\n## Embeddings\n\nContent embeddings are generated for each page. Currently using a fake implementation (deterministic 384-dimensional vectors) that's ready to be replaced with actual embeddings API when available.\n\nEmbeddings are:\n- Generated during recalculation\n- Stored in cache\n- Will be used for semantic search and similarity matching (currently using keyword search)\n\n## Hash-Based Change Detection\n\nContent hashes (SHA-256) are calculated and stored for each page. This enables:\n\n- **Smart recalculation**: Skip unchanged pages automatically\n- **Efficient updates**: Only process pages that have actually changed\n- **Performance**: Significantly faster recalculation when most content is unchanged\n\nHashes are compared during `/cache/recalculate` (default mode) to determine if a page needs reprocessing.\n\n## Troubleshooting\n\n### Cache seems stale\nRun `/cache/recalculate` to refresh everything.\n\n### Page not found\n1. Check `/list` to see if the path exists\n2. Verify the URL is in the `urls` array\n3. Ensure the path matches the URL structure (e.g., `api-reference` for `/docs/api-reference.md`)\n\n### Token counts seem wrong\n1. Clear cache for that page: `POST /cache/clear/:path`\n2. Request the page again: `GET /page/:path`\n3. Or recalculate everything: `POST /cache/recalculate`\n\n### Performance issues\n- Use `/page/:path` endpoints (cached) instead of `/data` (uncached)\n- Check cache stats: `GET /cache/stats`\n- Ensure cache is populated before production use\n\n## Code Structure\n\nThe codebase is organized into modular files:\n\n- **`main.tsx`** - Main Hono app, routes, and URL definitions\n- **`utils.ts`** - Utility functions:\n - Cache management (getFromCache, setCache, clearCache, getCacheStats)\n - Content fetching (getTextFromUrl)\n - Frontmatter parsing (parseFrontmatter, addUrlSourceToFrontmatter)\n - Token counting (calculateTokenCount)\n - Hash calculation (calculateContentHash)\n - Rate limiting for fetches\n- **`groq.ts`** - Groq API functions:\n - Chat completions (groqChatCompletion)\n - Metadata generation (generatePageMetadata)\n- **`search.ts`** - Search and embeddings utilities:\n - Embeddings generation (generateEmbeddings) - fake implementation ready for real API\n - Search functions (searchPages) - keyword-based search (will use embeddings later)\n - Vector similarity search (vectorSearch) - ready for embeddings-based search\n\n## Development\n\n### Local Development\n\n```bash\ndeno run --allow-net --allow-env main.tsx\n```\n\n**Note**: SQLite caching is automatically disabled when running locally (detected via `valtown` environment variable). The app will work without caching, but cache-related endpoints will return appropriate messages.\n\n### Val Town\n\nThe app is configured to work with Val Town. Export uses:\n```typescript\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;\n```\n\nSQLite caching is automatically enabled when running in Val Town (detected via `valtown` environment variable).\n\n### Environment Variables\n\n- `GROQ_API_KEY` - Required for AI metadata generation (optional, will disable metadata if not set)\n- `valtown` - Automatically set by Val Town (detects environment)\n\n## Performance Tips\n\n1. **Use default recalculate mode** - Automatically skips unchanged pages\n2. **Cache is your friend** - Always populate cache before production use\n3. **Rate limiting** - Built-in rate limiting prevents WAF blocking (1 request per 3 seconds for docs, 2 requests per second for Groq API)\n4. **Hash checking** - Default recalculation mode is much faster when most content is unchanged\n\n",[192],"6575fc7c-b8e7-11f0-ba05-0224a6c84d84","AGENTS.md","6575fd58-b8e7-11f0-ba05-0224a6c84d84",6,["D",1762196442671],"You are an advanced assistant specialized in generating Val Town code.\n\n## Core Guidelines\n\n- Ask clarifying questions when requirements are ambiguous\n- Provide complete, functional solutions rather than skeleton implementations\n- Test your logic against edge cases before presenting the final solution\n- Ensure all code follows Val Town's specific platform requirements\n- If a section of code that you're working on is getting too complex, consider refactoring it into subcomponents\n\n## Code Standards\n\n- Generate code in TypeScript or TSX\n- Add appropriate TypeScript types and interfaces for all data structures\n- Prefer official SDKs or libraries than writing API calls directly\n- Ask the user to supply API or library documentation if you are at all unsure about it\n- **Never bake in secrets into the code** - always use environment variables\n- Include comments explaining complex logic (avoid commenting obvious operations)\n- Follow modern ES6+ conventions and functional programming practices if possible\n\n## Types of triggers\n\n### 1. HTTP Trigger\n\n- Create web APIs and endpoints\n- Handle HTTP requests and responses\n- Example structure:\n\n```ts\nexport default async function (req: Request) {\n return new Response(\"Hello World\");\n}\n```\n\nFiles that are HTTP triggers have http in their name like `foobar.http.tsx`\n\n### 2. Cron Triggers\n\n- Run on a schedule\n- Use cron expressions for timing\n- Example structure:\n\n```ts\nexport default async function () {\n // Scheduled task code\n}\n```\n\nFiles that are Cron triggers have cron in their name like `foobar.cron.tsx`\n\n### 3. Email Triggers\n\n- Process incoming emails\n- Handle email-based workflows\n- Example structure:\n\n```ts\nexport default async function (email: Email) {\n // Process email\n}\n```\n\nFiles that are Email triggers have email in their name like `foobar.email.tsx`\n\n\n## Val Town Standard Libraries\n\nVal Town provides several hosted services and utility functions.\n\n### Blob Storage\n\n```ts\nimport { blob } from \"https://esm.town/v/std/blob\";\nawait blob.setJSON(\"myKey\", { hello: \"world\" });\nlet blobDemo = await blob.getJSON(\"myKey\");\nlet appKeys = await blob.list(\"app_\");\nawait blob.delete(\"myKey\");\n```\n\n### SQLite\n\n```ts\nimport { sqlite } from \"https://esm.town/v/stevekrouse/sqlite\";\nconst TABLE_NAME = 'todo_app_users_2';\n// Create table - do this before usage and change table name when modifying schema\nawait sqlite.execute(`CREATE TABLE IF NOT EXISTS ${TABLE_NAME} (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n name TEXT NOT NULL\n)`);\n// Query data\nconst result = await sqlite.execute(`SELECT * FROM ${TABLE_NAME} WHERE id = ?`, [1]);\n```\n\nNote: When changing a SQLite table's schema, change the table's name (e.g., add _2 or _3) to create a fresh table.\n\n### OpenAI\n\n```ts\nimport { OpenAI } from \"https://esm.town/v/std/openai\";\nconst openai = new OpenAI();\nconst completion = await openai.chat.completions.create({\n messages: [\n { role: \"user\", content: \"Say hello in a creative way\" },\n ],\n model: \"gpt-4o-mini\",\n max_tokens: 30,\n});\n```\n\n### Email\n\n```ts\nimport { email } from \"https://esm.town/v/std/email\";\n// By default emails the owner of the val\nawait email({ \n subject: \"Hi\", \n text: \"Hi\", \n html: \"

Hi

\"\n});\n```\n\n## Val Town Utility Functions\n\nVal Town provides several utility functions to help with common project tasks.\n\n### Importing Utilities\n\nAlways import utilities with version pins to avoid breaking changes:\n\n```ts\nimport { parseProject, readFile, serveFile } from \"https://esm.town/v/std/utils@85-main/index.ts\";\n```\n\n### Available Utilities\n\n\n#### **serveFile** - Serve project files with proper content types\n\nFor example, in Hono:\n\n```ts\n// serve all files in frontend/ and shared/\napp.get(\"/frontend/*\", c => serveFile(c.req.path, import.meta.url));\napp.get(\"/shared/*\", c => serveFile(c.req.path, import.meta.url));\n```\n\n#### **readFile** - Read files from within the project:\n\n```ts\n// Read a file from the project\nconst fileContent = await readFile(\"/frontend/index.html\", import.meta.url);\n```\n\n#### **listFiles** - List all files in the project\n\n```ts\nconst files = await listFiles(import.meta.url);\n```\n\n#### **parseProject** - Extract information about the current project from import.meta.url\n\nThis is useful for including info for linking back to a val, ie in \"view source\" urls:\n\n```ts\nconst projectVal = parseProject(import.meta.url);\nconsole.log(projectVal.username); // Owner of the project\nconsole.log(projectVal.name); // Project name\nconsole.log(projectVal.version); // Version number\nconsole.log(projectVal.branch); // Branch name\nconsole.log(projectVal.links.self.project); // URL to the project page\n```\n\nHowever, it's *extremely importing* to note that `parseProject` and other Standard Library utilities ONLY RUN ON THE SERVER.\nIf you need access to this data on the client, run it in the server and pass it to the client by splicing it into the HTML page\nor by making an API request for it.\n\n## Val Town Platform Specifics\n\n- **Redirects:** Use `return new Response(null, { status: 302, headers: { Location: \"/place/to/redirect\" }})` instead of `Response.redirect` which is broken\n- **Images:** Avoid external images or base64 images. Use emojis, unicode symbols, or icon fonts/libraries instead\n- **AI Image:** To inline generate an AI image use: `

`\n- **Storage:** DO NOT use the Deno KV module for storage\n- **Browser APIs:** DO NOT use the `alert()`, `prompt()`, or `confirm()` methods\n- **Weather Data:** Use open-meteo for weather data (doesn't require API keys) unless otherwise specified\n- **View Source:** Add a view source link by importing & using `import.meta.url.replace(\"ems.sh\", \"val.town)\"` (or passing this data to the client) and include `target=\"_top\"` attribute\n- **Error Debugging:** Add `` to HTML to capture client-side errors\n- **Error Handling:** Only use try...catch when there's a clear local resolution; Avoid catches that merely log or return 500s. Let errors bubble up with full context\n- **Environment Variables:** Use `Deno.env.get('keyname')` when you need to, but generally prefer APIs that don't require keys\n- **Imports:** Use `https://esm.sh` for npm and Deno dependencies to ensure compatibility on server and browser\n- **Storage Strategy:** Only use backend storage if explicitly required; prefer simple static client-side sites\n- **React Configuration:** When using React libraries, pin versions with `?deps=react@18.2.0,react-dom@18.2.0` and start the file with `/** @jsxImportSource https://esm.sh/react@18.2.0 */`\n- Ensure all React dependencies and sub-dependencies are pinned to the same version\n- **Styling:** Default to using TailwindCSS via `` unless otherwise specified\n\n## Project Structure and Design Patterns\n\n### Recommended Directory Structure\n```\n├── backend/\n│ ├── database/\n│ │ ├── migrations.ts # Schema definitions\n│ │ ├── queries.ts # DB query functions\n│ │ └── README.md\n│ └── routes/ # Route modules\n│ ├── [route].ts\n│ └── static.ts # Static file serving\n│ ├── index.ts # Main entry point\n│ └── README.md\n├── frontend/\n│ ├── components/\n│ │ ├── App.tsx\n│ │ └── [Component].tsx\n│ ├── favicon.svg\n│ ├── index.html # Main HTML template\n│ ├── index.tsx # Frontend JS entry point\n│ ├── README.md\n│ └── style.css\n├── README.md\n└── shared/\n ├── README.md\n └── utils.ts # Shared types and functions\n```\n\n### Backend (Hono) Best Practices\n\n- Hono is the recommended API framework\n- Main entry point should be `backend/index.ts`\n- **Static asset serving:** Use the utility functions to read and serve project files:\n ```ts\n import { readFile, serveFile } from \"https://esm.town/v/std/utils@85-main/index.ts\";\n \n // serve all files in frontend/ and shared/\n app.get(\"/frontend/*\", c => serveFile(c.req.path, import.meta.url));\n app.get(\"/shared/*\", c => serveFile(c.req.path, import.meta.url));\n \n // For index.html, often you'll want to bootstrap with initial data\n app.get(\"/\", async c => {\n let html = await readFile(\"/frontend/index.html\", import.meta.url);\n \n // Inject data to avoid extra round-trips\n const initialData = await fetchInitialData();\n const dataScript = ``;\n \n html = html.replace(\"\", `${dataScript}`);\n return c.html(html);\n });\n ```\n- Create RESTful API routes for CRUD operations\n- Always include this snippet at the top-level Hono app to re-throwing errors to see full stack traces:\n ```ts\n // Unwrap Hono errors to see original error details\n app.onError((err, c) => {\n throw err;\n });\n ```\n\n### Database Patterns\n- Run migrations on startup or comment out for performance\n- Change table names when modifying schemas rather than altering\n- Export clear query functions with proper TypeScript typing\n\n## Common Gotchas and Solutions\n\n1. **Environment Limitations:** \n - Val Town runs on Deno in a serverless context, not Node.js\n - Code in `shared/` must work in both frontend and backend environments\n - Cannot use `Deno` keyword in shared code\n - Use `https://esm.sh` for imports that work in both environments\n\n2. **SQLite Peculiarities:**\n - Limited support for ALTER TABLE operations\n - Create new tables with updated schemas and copy data when needed\n - Always run table creation before querying\n\n3. **React Configuration:**\n - All React dependencies must be pinned to 18.2.0\n - Always include `@jsxImportSource https://esm.sh/react@18.2.0` at the top of React files\n - Rendering issues often come from mismatched React versions\n\n4. **File Handling:**\n - Val Town only supports text files, not binary\n - Use the provided utilities to read files across branches and forks\n - For files in the project, use `readFile` helpers\n\n5. **API Design:**\n - `fetch` handler is the entry point for HTTP vals\n - Run the Hono app with `export default app.fetch // This is the entry point for HTTP vals`\n\n",[199],"65892a86-b8e7-11f0-ba05-0224a6c84d84",".vtignore","65892b4e-b8e7-11f0-ba05-0224a6c84d84",9,["D",1762196442809],".git\n.vscode\n.cursorrules\n.DS_Store\nnode_modules\nvendor",[206],"7ecff464-b9ca-11f0-8b68-0224a6c84d84","utils","7ecff54a-b9ca-11f0-8b68-0224a6c84d84",49,["D",1762293981048],"directory",[213],"30dc0596-b901-11f0-a3b9-0224a6c84d84","testing","30dc0668-b901-11f0-a3b9-0224a6c84d84",28,["D",1762207521420],[220],"30c2711c-b901-11f0-b5de-0224a6c84d84","search","30c2722a-b901-11f0-b5de-0224a6c84d84",27,["D",1762207521254],[226],"31189d8a-b901-11f0-87ce-0224a6c84d84","data","31189e3e-b901-11f0-87ce-0224a6c84d84",29,["D",1762207521817],[232],"7eb3aa8e-b9ca-11f0-98dd-0224a6c84d84","answer","7eb3ab56-b9ca-11f0-98dd-0224a6c84d84",48,["D",1762293980863],[238],"pulls","branches","forks","envVars",{"_250":63,"_252":63},"connections",{"_250":63,"_251":63},"count","unfilledCount","emptyCount",{"_44":237,"_32":238,"_30":239,"_144":-5,"_42":43,"_41":240,"_146":50,"_39":768,"_13":63,"_34":217,"_36":-5,"_24":25,"_149":-5,"_321":769,"_27":770},{"_44":760,"_32":761,"_30":762,"_144":239,"_42":43,"_41":763,"_146":50,"_39":764,"_13":63,"_34":20,"_36":765,"_24":25,"_149":-5,"_321":766,"_27":767},{"_44":752,"_32":753,"_30":754,"_144":239,"_42":43,"_41":755,"_146":50,"_39":756,"_13":63,"_34":20,"_36":757,"_24":25,"_149":-5,"_321":758,"_27":759},{"_44":745,"_32":192,"_30":746,"_144":239,"_42":43,"_41":747,"_146":50,"_39":748,"_13":63,"_34":20,"_36":749,"_24":25,"_149":-5,"_321":750,"_27":751},{"_44":738,"_32":518,"_30":739,"_144":239,"_42":43,"_41":740,"_146":50,"_39":741,"_13":63,"_34":20,"_36":742,"_24":25,"_149":-5,"_321":743,"_27":744},{"_44":730,"_32":731,"_30":732,"_144":239,"_42":43,"_41":733,"_146":50,"_39":734,"_13":63,"_34":35,"_36":735,"_24":25,"_149":-5,"_321":736,"_27":737},{"_44":723,"_32":494,"_30":724,"_144":239,"_42":43,"_41":725,"_146":50,"_39":726,"_13":63,"_34":35,"_36":727,"_24":25,"_149":-5,"_321":728,"_27":729},{"_44":715,"_32":716,"_30":717,"_144":239,"_42":43,"_41":718,"_146":50,"_39":719,"_13":63,"_34":35,"_36":720,"_24":25,"_149":-5,"_321":721,"_27":722},{"_44":708,"_32":426,"_30":709,"_144":239,"_42":43,"_41":710,"_146":50,"_39":711,"_13":63,"_34":35,"_36":712,"_24":25,"_149":-5,"_321":713,"_27":714},{"_44":701,"_32":142,"_30":702,"_144":239,"_42":43,"_41":703,"_146":50,"_39":704,"_13":63,"_34":35,"_36":705,"_24":25,"_149":-5,"_321":706,"_27":707},{"_44":231,"_32":232,"_30":233,"_144":-5,"_42":43,"_41":234,"_146":50,"_39":698,"_13":63,"_34":217,"_36":-5,"_24":25,"_149":-5,"_321":699,"_27":700},{"_44":225,"_32":226,"_30":227,"_144":-5,"_42":43,"_41":228,"_146":50,"_39":695,"_13":63,"_34":217,"_36":-5,"_24":25,"_149":-5,"_321":696,"_27":697},{"_44":690,"_32":541,"_30":535,"_144":227,"_42":43,"_41":691,"_146":50,"_39":692,"_13":63,"_34":217,"_36":-5,"_24":25,"_149":-5,"_321":693,"_27":694},{"_44":685,"_32":598,"_30":592,"_144":535,"_42":43,"_41":686,"_146":50,"_39":687,"_13":63,"_34":217,"_36":-5,"_24":25,"_149":-5,"_321":688,"_27":689},{"_44":680,"_32":639,"_30":633,"_144":592,"_42":43,"_41":681,"_146":50,"_39":682,"_13":63,"_34":217,"_36":-5,"_24":25,"_149":-5,"_321":683,"_27":684},{"_44":672,"_32":673,"_30":674,"_144":633,"_42":43,"_41":675,"_146":50,"_39":676,"_13":63,"_34":20,"_36":677,"_24":25,"_149":-5,"_321":678,"_27":679},{"_44":664,"_32":665,"_30":666,"_144":633,"_42":43,"_41":667,"_146":50,"_39":668,"_13":63,"_34":20,"_36":669,"_24":25,"_149":-5,"_321":670,"_27":671},{"_44":656,"_32":657,"_30":658,"_144":633,"_42":43,"_41":659,"_146":50,"_39":660,"_13":63,"_34":20,"_36":661,"_24":25,"_149":-5,"_321":662,"_27":663},{"_44":648,"_32":649,"_30":650,"_144":633,"_42":43,"_41":651,"_146":50,"_39":652,"_13":63,"_34":20,"_36":653,"_24":25,"_149":-5,"_321":654,"_27":655},{"_44":640,"_32":641,"_30":642,"_144":633,"_42":43,"_41":643,"_146":50,"_39":644,"_13":63,"_34":20,"_36":645,"_24":25,"_149":-5,"_321":646,"_27":647},{"_44":630,"_32":631,"_30":632,"_144":633,"_42":43,"_41":634,"_146":50,"_39":635,"_13":63,"_34":20,"_36":636,"_24":25,"_149":-5,"_321":637,"_27":638},{"_44":622,"_32":623,"_30":624,"_144":592,"_42":43,"_41":625,"_146":50,"_39":626,"_13":63,"_34":20,"_36":627,"_24":25,"_149":-5,"_321":628,"_27":629},{"_44":615,"_32":192,"_30":616,"_144":592,"_42":43,"_41":617,"_146":50,"_39":618,"_13":63,"_34":20,"_36":619,"_24":25,"_149":-5,"_321":620,"_27":621},{"_44":607,"_32":608,"_30":609,"_144":592,"_42":43,"_41":610,"_146":50,"_39":611,"_13":63,"_34":20,"_36":612,"_24":25,"_149":-5,"_321":613,"_27":614},{"_44":599,"_32":600,"_30":601,"_144":592,"_42":43,"_41":602,"_146":50,"_39":603,"_13":63,"_34":20,"_36":604,"_24":25,"_149":-5,"_321":605,"_27":606},{"_44":589,"_32":590,"_30":591,"_144":592,"_42":43,"_41":593,"_146":50,"_39":594,"_13":63,"_34":20,"_36":595,"_24":25,"_149":-5,"_321":596,"_27":597},{"_44":581,"_32":582,"_30":583,"_144":535,"_42":43,"_41":584,"_146":50,"_39":585,"_13":63,"_34":20,"_36":586,"_24":25,"_149":-5,"_321":587,"_27":588},{"_44":574,"_32":192,"_30":575,"_144":535,"_42":43,"_41":576,"_146":50,"_39":577,"_13":63,"_34":20,"_36":578,"_24":25,"_149":-5,"_321":579,"_27":580},{"_44":566,"_32":567,"_30":568,"_144":535,"_42":43,"_41":569,"_146":50,"_39":570,"_13":63,"_34":20,"_36":571,"_24":25,"_149":-5,"_321":572,"_27":573},{"_44":558,"_32":559,"_30":560,"_144":535,"_42":43,"_41":561,"_146":50,"_39":562,"_13":63,"_34":20,"_36":563,"_24":25,"_149":-5,"_321":564,"_27":565},{"_44":550,"_32":551,"_30":552,"_144":535,"_42":43,"_41":553,"_146":50,"_39":554,"_13":63,"_34":35,"_36":555,"_24":25,"_149":-5,"_321":556,"_27":557},{"_44":542,"_32":543,"_30":544,"_144":535,"_42":43,"_41":545,"_146":50,"_39":546,"_13":63,"_34":35,"_36":547,"_24":25,"_149":-5,"_321":548,"_27":549},{"_44":532,"_32":533,"_30":534,"_144":535,"_42":43,"_41":536,"_146":50,"_39":537,"_13":63,"_34":35,"_36":538,"_24":25,"_149":-5,"_321":539,"_27":540},{"_44":525,"_32":192,"_30":526,"_144":227,"_42":43,"_41":527,"_146":50,"_39":528,"_13":63,"_34":20,"_36":529,"_24":25,"_149":-5,"_321":530,"_27":531},{"_44":517,"_32":518,"_30":519,"_144":227,"_42":43,"_41":520,"_146":50,"_39":521,"_13":63,"_34":20,"_36":522,"_24":25,"_149":-5,"_321":523,"_27":524},{"_44":509,"_32":510,"_30":511,"_144":227,"_42":43,"_41":512,"_146":50,"_39":513,"_13":63,"_34":35,"_36":514,"_24":25,"_149":-5,"_321":515,"_27":516},{"_44":501,"_32":502,"_30":503,"_144":227,"_42":43,"_41":504,"_146":50,"_39":505,"_13":63,"_34":35,"_36":506,"_24":25,"_149":-5,"_321":507,"_27":508},{"_44":493,"_32":494,"_30":495,"_144":227,"_42":43,"_41":496,"_146":25,"_39":497,"_13":63,"_34":35,"_36":498,"_24":25,"_149":-5,"_321":499,"_27":500},{"_44":485,"_32":486,"_30":487,"_144":227,"_42":43,"_41":488,"_146":50,"_39":489,"_13":63,"_34":35,"_36":490,"_24":25,"_149":-5,"_321":491,"_27":492},{"_44":477,"_32":478,"_30":479,"_144":227,"_42":43,"_41":480,"_146":50,"_39":481,"_13":63,"_34":35,"_36":482,"_24":25,"_149":-5,"_321":483,"_27":484},{"_44":469,"_32":470,"_30":471,"_144":227,"_42":43,"_41":472,"_146":50,"_39":473,"_13":63,"_34":35,"_36":474,"_24":25,"_149":-5,"_321":475,"_27":476},{"_44":461,"_32":462,"_30":463,"_144":227,"_42":43,"_41":464,"_146":50,"_39":465,"_13":63,"_34":35,"_36":466,"_24":25,"_149":-5,"_321":467,"_27":468},{"_44":453,"_32":454,"_30":455,"_144":227,"_42":43,"_41":456,"_146":50,"_39":457,"_13":63,"_34":35,"_36":458,"_24":25,"_149":-5,"_321":459,"_27":460},{"_44":445,"_32":446,"_30":447,"_144":227,"_42":43,"_41":448,"_146":50,"_39":449,"_13":63,"_34":35,"_36":450,"_24":25,"_149":-5,"_321":451,"_27":452},{"_44":441,"_32":33,"_30":31,"_144":227,"_42":43,"_41":14,"_146":50,"_39":442,"_13":156,"_34":35,"_36":38,"_24":25,"_149":-5,"_321":443,"_27":444},{"_44":433,"_32":434,"_30":435,"_144":227,"_42":43,"_41":436,"_146":50,"_39":437,"_13":63,"_34":35,"_36":438,"_24":25,"_149":-5,"_321":439,"_27":440},{"_44":425,"_32":426,"_30":427,"_144":227,"_42":43,"_41":428,"_146":50,"_39":429,"_13":63,"_34":35,"_36":430,"_24":25,"_149":-5,"_321":431,"_27":432},{"_44":418,"_32":142,"_30":419,"_144":227,"_42":43,"_41":420,"_146":50,"_39":421,"_13":156,"_34":35,"_36":422,"_24":25,"_149":-5,"_321":423,"_27":424},{"_44":219,"_32":220,"_30":221,"_144":-5,"_42":43,"_41":222,"_146":50,"_39":415,"_13":63,"_34":217,"_36":-5,"_24":25,"_149":-5,"_321":416,"_27":417},{"_44":407,"_32":408,"_30":409,"_144":221,"_42":43,"_41":410,"_146":50,"_39":411,"_13":63,"_34":35,"_36":412,"_24":25,"_149":-5,"_321":413,"_27":414},{"_44":399,"_32":400,"_30":401,"_144":221,"_42":43,"_41":402,"_146":25,"_39":403,"_13":63,"_34":35,"_36":404,"_24":25,"_149":-5,"_321":405,"_27":406},{"_44":391,"_32":392,"_30":393,"_144":221,"_42":43,"_41":394,"_146":50,"_39":395,"_13":63,"_34":35,"_36":396,"_24":25,"_149":-5,"_321":397,"_27":398},{"_44":383,"_32":384,"_30":385,"_144":221,"_42":43,"_41":386,"_146":25,"_39":387,"_13":63,"_34":35,"_36":388,"_24":25,"_149":-5,"_321":389,"_27":390},{"_44":375,"_32":376,"_30":377,"_144":221,"_42":43,"_41":378,"_146":50,"_39":379,"_13":63,"_34":35,"_36":380,"_24":25,"_149":-5,"_321":381,"_27":382},{"_44":367,"_32":368,"_30":369,"_144":221,"_42":43,"_41":370,"_146":50,"_39":371,"_13":63,"_34":35,"_36":372,"_24":25,"_149":-5,"_321":373,"_27":374},{"_44":359,"_32":360,"_30":361,"_144":221,"_42":43,"_41":362,"_146":25,"_39":363,"_13":63,"_34":35,"_36":364,"_24":25,"_149":-5,"_321":365,"_27":366},{"_44":212,"_32":213,"_30":214,"_144":-5,"_42":43,"_41":215,"_146":50,"_39":356,"_13":63,"_34":217,"_36":-5,"_24":25,"_149":-5,"_321":357,"_27":358},{"_44":348,"_32":349,"_30":350,"_144":214,"_42":43,"_41":351,"_146":50,"_39":352,"_13":63,"_34":35,"_36":353,"_24":25,"_149":-5,"_321":354,"_27":355},{"_44":205,"_32":206,"_30":207,"_144":-5,"_42":43,"_41":208,"_146":25,"_39":345,"_13":63,"_34":20,"_36":210,"_24":25,"_149":-5,"_321":346,"_27":347},{"_44":198,"_32":199,"_30":200,"_144":-5,"_42":43,"_41":201,"_146":50,"_39":342,"_13":63,"_34":20,"_36":203,"_24":25,"_149":-5,"_321":343,"_27":344},{"_44":191,"_32":192,"_30":193,"_144":-5,"_42":43,"_41":194,"_146":25,"_39":339,"_13":164,"_34":20,"_36":196,"_24":25,"_149":-5,"_321":340,"_27":341},{"_44":183,"_32":184,"_30":185,"_144":-5,"_42":43,"_41":186,"_146":50,"_39":336,"_13":188,"_34":20,"_36":189,"_24":25,"_149":-5,"_321":337,"_27":338},{"_44":176,"_32":177,"_30":178,"_144":-5,"_42":43,"_41":179,"_146":50,"_39":333,"_13":63,"_34":35,"_36":181,"_24":25,"_149":-5,"_321":334,"_27":335},{"_44":167,"_32":168,"_30":58,"_144":-5,"_42":43,"_41":169,"_146":50,"_39":330,"_13":171,"_34":172,"_36":173,"_24":25,"_149":174,"_321":331,"_27":332},{"_44":159,"_32":160,"_30":161,"_144":-5,"_42":43,"_41":162,"_146":50,"_39":327,"_13":164,"_34":20,"_36":165,"_24":25,"_149":-5,"_321":328,"_27":329},{"_44":151,"_32":152,"_30":153,"_144":-5,"_42":43,"_41":154,"_146":25,"_39":324,"_13":156,"_34":35,"_36":157,"_24":25,"_149":-5,"_321":325,"_27":326},{"_44":141,"_32":142,"_30":143,"_144":-5,"_42":43,"_41":145,"_146":50,"_39":320,"_13":63,"_34":35,"_36":148,"_24":25,"_149":-5,"_321":322,"_27":323},["D",1762200814128],"children",[],[142],["D",1762293983912],[],[152],["D",1762293984010],[],[160],["D",1762293983847],[],[168],["D",1762200814266],[],[177],["D",1762293983950],[],[184],["D",1762200814124],[],[192],["D",1762196442671],[],[199],["D",1762196442809],[],[206],"7f7fe9aa-b9ca-11f0-9789-0224a6c84d84","json-parser.ts","7f7fea72-b9ca-11f0-9789-0224a6c84d84",60,["D",1762293982187],"// Fast JSON parser utility using fast-json for selective parsing\n// This is much faster than JSON.parse() for large JSON files when we only need specific fields\n\nimport { FastJson } from \"npm:fast-json@latest\";\n\ninterface ParseLatestJsonOptions {\n enableTiming?: boolean;\n}\n\n/**\n * Parse latest.json using fast-json, extracting only the fields we need\n * Much faster than JSON.parse() for large files\n */\nexport async function parseLatestJson(\n jsonString: string,\n options: ParseLatestJsonOptions = {}\n): Promise<{\n totalPages?: number;\n results?: Array<{\n url: string;\n title?: string;\n embeddings?: number[] | null;\n metadata?: any;\n contentHash?: string;\n charCount?: number;\n tokenCount?: number;\n skipped?: boolean;\n error?: string;\n }>;\n}> {\n const { enableTiming = false } = options;\n const timings: Record = {};\n const startTime = performance.now();\n \n return new Promise((resolve, reject) => {\n const fastJson = new FastJson();\n const resultsMap = new Map();\n let totalPages: number | undefined;\n let currentIndex = -1;\n \n // Track array index as we enter arrays\n fastJson.on('results', () => {\n currentIndex = -1;\n });\n \n // Extract totalPages\n fastJson.on('totalPages', (value) => {\n totalPages = value as unknown as number;\n });\n \n // Extract results array items - track index for each result\n fastJson.on('results[*]', () => {\n currentIndex++;\n if (!resultsMap.has(currentIndex)) {\n resultsMap.set(currentIndex, {});\n }\n });\n \n fastJson.on('results[*].url', (value) => {\n if (!resultsMap.has(currentIndex)) resultsMap.set(currentIndex, {});\n resultsMap.get(currentIndex)!.url = value as unknown as string;\n });\n \n fastJson.on('results[*].title', (value) => {\n if (!resultsMap.has(currentIndex)) resultsMap.set(currentIndex, {});\n resultsMap.get(currentIndex)!.title = value as unknown as string;\n });\n \n // Embeddings are arrays - need to handle them specially\n // fast-json emits arrays element by element, so we need to collect them\n const embeddingsBuffer = new Map();\n \n fastJson.on('results[*].embeddings[*]', (value) => {\n if (!resultsMap.has(currentIndex)) resultsMap.set(currentIndex, {});\n if (!embeddingsBuffer.has(currentIndex)) embeddingsBuffer.set(currentIndex, []);\n embeddingsBuffer.get(currentIndex)!.push(value as unknown as number);\n });\n \n // When embeddings array is done, store it\n fastJson.on('results[*].embeddings', (value) => {\n if (!resultsMap.has(currentIndex)) resultsMap.set(currentIndex, {});\n // If value is already an array, use it directly (fallback)\n if (Array.isArray(value)) {\n resultsMap.get(currentIndex)!.embeddings = value;\n } else if (embeddingsBuffer.has(currentIndex)) {\n // Use collected array elements\n resultsMap.get(currentIndex)!.embeddings = embeddingsBuffer.get(currentIndex)!;\n embeddingsBuffer.delete(currentIndex);\n } else {\n resultsMap.get(currentIndex)!.embeddings = value as unknown as number[] | null;\n }\n });\n \n fastJson.on('results[*].metadata', (value) => {\n if (!resultsMap.has(currentIndex)) resultsMap.set(currentIndex, {});\n resultsMap.get(currentIndex)!.metadata = value;\n });\n \n fastJson.on('results[*].contentHash', (value) => {\n if (!resultsMap.has(currentIndex)) resultsMap.set(currentIndex, {});\n resultsMap.get(currentIndex)!.contentHash = value as unknown as string;\n });\n \n fastJson.on('results[*].charCount', (value) => {\n if (!resultsMap.has(currentIndex)) resultsMap.set(currentIndex, {});\n resultsMap.get(currentIndex)!.charCount = value as unknown as number;\n });\n \n fastJson.on('results[*].tokenCount', (value) => {\n if (!resultsMap.has(currentIndex)) resultsMap.set(currentIndex, {});\n resultsMap.get(currentIndex)!.tokenCount = value as unknown as number;\n });\n \n fastJson.on('results[*].skipped', (value) => {\n if (!resultsMap.has(currentIndex)) resultsMap.set(currentIndex, {});\n resultsMap.get(currentIndex)!.skipped = value as unknown as boolean;\n });\n \n fastJson.on('results[*].error', (value) => {\n if (!resultsMap.has(currentIndex)) resultsMap.set(currentIndex, {});\n resultsMap.get(currentIndex)!.error = value as unknown as string;\n });\n \n // Parse the JSON string\n const parseStart = performance.now();\n try {\n fastJson.write(jsonString);\n \n const results = Array.from(resultsMap.values()).filter(r => r.url);\n \n if (enableTiming) {\n timings.parse = performance.now() - parseStart;\n timings.total = performance.now() - startTime;\n console.log(`⏱️ fast-json parsing:`);\n console.log(` Parse: ${timings.parse?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total.toFixed(2)}ms`);\n console.log(` Results extracted: ${results.length}`);\n }\n \n resolve({\n totalPages,\n results,\n });\n } catch (error) {\n // Fall back to JSON.parse if fast-json fails\n console.warn('fast-json failed, falling back to JSON.parse:', error);\n const fallbackStart = performance.now();\n try {\n const data = JSON.parse(jsonString);\n if (enableTiming) {\n timings.fallbackParse = performance.now() - fallbackStart;\n console.log(` Fallback JSON.parse: ${timings.fallbackParse?.toFixed(2)}ms`);\n }\n resolve(data);\n } catch (parseError) {\n reject(parseError);\n }\n }\n });\n}\n\n",[],[213,349],["D",1762293981048],[310],[213],"31b6d658-b901-11f0-bfbc-0224a6c84d84","test-search.ts","31b6d70c-b901-11f0-bfbc-0224a6c84d84",43,["D",1762207522693],"// Test script for search strategies\n// Run with: deno task test-search\n// Requires: Run `deno task recalculate` first to generate /data/latest.json\n\nimport \"jsr:@std/dotenv/load\";\nimport { getAllPagesForSearch } from \"../main.tsx\";\nimport { runSearchTest } from \"./search-harness.ts\";\nimport { questions } from \"./questions.ts\";\n\nasync function testSearchStrategies() {\n // Load cached data from latest.json\n let cachedData: any = null;\n try {\n const latestData = await Deno.readTextFile(\"./data/latest.json\");\n cachedData = JSON.parse(latestData);\n console.log(`Loaded cached data from /data/latest.json (${cachedData.totalPages} pages)`);\n } catch (error) {\n console.error(\"Failed to load /data/latest.json. Please run 'deno task recalculate' first.\");\n console.error(error);\n Deno.exit(1);\n }\n \n // Load actual pages with embeddings for search\n const pagesWithEmbeddings = await getAllPagesForSearch();\n console.log(`Loaded ${pagesWithEmbeddings.length} pages with embeddings for search`);\n \n // Load test questions\n const testQueries = questions;\n console.log(`Loaded ${testQueries.length} test questions\\n`);\n \n for (const query of testQueries) {\n console.log(`\\n${\"=\".repeat(60)}`);\n console.log(`Testing query: \"${query}\"`);\n console.log(`${\"=\".repeat(60)}`);\n \n const testResult = await runSearchTest(query, pagesWithEmbeddings, {\n limit: 10,\n minScore: 0,\n enableTiming: true, // Enable timing breakdown\n });\n \n if (testResult.error) {\n console.error(` ❌ Error: ${testResult.error}`);\n console.log(` ⏱️ Duration: ${testResult.durationMs.toFixed(2)}ms`);\n } else {\n console.log(` ✅ Found ${testResult.results.length} results in ${testResult.durationMs.toFixed(2)}ms\\n`);\n \n if (testResult.results.length === 0) {\n console.log(\" No results found.\");\n } else {\n testResult.results.forEach((result, index) => {\n console.log(` ${index + 1}. ${result.title || result.path}`);\n console.log(` Path: ${result.path}`);\n console.log(` Score: ${result.score.toFixed(2)}${result.metadata?.similarity ? ` (similarity: ${(result.metadata.similarity * 100).toFixed(2)}%)` : ''}`);\n if (result.snippet) {\n const snippet = result.snippet.length > 150 \n ? result.snippet.substring(0, 150) + '...' \n : result.snippet;\n console.log(` Snippet: ${snippet}`);\n }\n console.log('');\n });\n }\n }\n }\n \n console.log(`\\n${\"=\".repeat(60)}`);\n console.log(\"Test completed!\");\n console.log(`${\"=\".repeat(60)}\\n`);\n}\n\n// Run if executed directly\nif (import.meta.main) {\n testSearchStrategies().catch(console.error);\n}\n\n",[],[220,360],"806c6d48-b9ca-11f0-b384-0224a6c84d84","test-answer.ts","806c6de8-b9ca-11f0-b384-0224a6c84d84",89,["D",1762293983677],"// Test script for answer strategies\n// Run with: deno task answer\n// Requires: Run `deno task recalculate` first to generate /data/latest.json\n\nimport \"jsr:@std/dotenv/load\";\nimport { runAnswerTest, type AnswerTestResult } from \"./answer-harness.ts\";\nimport { questions } from \"./questions.ts\";\n\nasync function testAnswerStrategies() {\n const overallStart = performance.now();\n const timings: Record = {};\n const timePoints: Array<{ label: string; time: number }> = [];\n \n // Track time point helper\n const trackTime = (label: string) => {\n const time = performance.now() - overallStart;\n timePoints.push({ label, time });\n };\n \n trackTime(\"Test start\");\n \n // Load cached data from latest.json\n let cachedData: any = null;\n trackTime(\"JSON load start\");\n const loadJsonStart = performance.now();\n try {\n const latestData = await Deno.readTextFile(\"./data/latest.json\");\n timings.jsonReadFile = performance.now() - loadJsonStart;\n trackTime(\"JSON file read complete\");\n \n const parseStart = performance.now();\n const { parseLatestJson } = await import(\"../utils/json-parser.ts\");\n cachedData = await parseLatestJson(latestData, { enableTiming: true });\n timings.jsonParse = performance.now() - parseStart;\n trackTime(\"JSON parse complete\");\n \n console.log(`⏱️ Loading latest.json:`);\n console.log(` File read: ${timings.jsonReadFile?.toFixed(2)}ms`);\n console.log(` JSON parse: ${timings.jsonParse?.toFixed(2)}ms`);\n console.log(` Total: ${(timings.jsonReadFile + timings.jsonParse).toFixed(2)}ms`);\n console.log(` Pages in cache: ${cachedData.totalPages}`);\n } catch (error) {\n console.error(\"Failed to load /data/latest.json. Please run 'deno task recalculate' first.\");\n console.error(error);\n Deno.exit(1);\n }\n \n // Content is now included in latest.json from recalculate\n console.log(`\\n✅ Content loaded from JSON (${cachedData.results?.length || 0} pages with embeddings and content)\\n`);\n \n // Load test questions\n trackTime(\"Questions load start\");\n const loadQuestionsStart = performance.now();\n const testQueries = questions;\n timings.loadQuestions = performance.now() - loadQuestionsStart;\n trackTime(\"Questions load complete\");\n console.log(`⏱️ Loading test questions: ${timings.loadQuestions?.toFixed(2)}ms`);\n console.log(` Loaded ${testQueries.length} test questions\\n`);\n \n // Get active strategy info\n const { answerStrategy } = await import(\"../answer/index.ts\");\n const { searchStrategy } = await import(\"../search/index.ts\");\n console.log(`📋 Active Answer Strategy: ${answerStrategy.name}`);\n console.log(` Description: ${answerStrategy.description}`);\n console.log(`📋 Active Search Strategy: ${searchStrategy.name}`);\n console.log(` Description: ${searchStrategy.description}\\n`);\n \n // Test configuration\n const answerOptions = {\n limit: 20, // Search results to consider\n minScore: 0, // Minimum search score\n maxContextPages: 7, // Pages in context\n temperature: 0.3, // LLM temperature\n };\n \n console.log(`⚙️ Test configuration:`);\n console.log(` Search limit: ${answerOptions.limit}`);\n console.log(` Min score: ${answerOptions.minScore}`);\n console.log(` Max context pages: ${answerOptions.maxContextPages}`);\n console.log(` Temperature: ${answerOptions.temperature}\\n`);\n \n // Track total timings across all queries\n const allQueryTimings: Record = {\n search: [],\n queryEmbedding: [], // Track query embedding separately\n contextPrep: [],\n llm: [],\n total: [],\n };\n \n // Create trackTime callback that captures absolute time\n const createTrackTimeCallback = () => {\n return (label: string) => {\n const time = performance.now() - overallStart;\n timePoints.push({ label, time });\n };\n };\n \n for (const query of testQueries) {\n console.log(`\\n${\"=\".repeat(80)}`);\n console.log(`Testing query: \"${query}\"`);\n console.log(`${\"=\".repeat(80)}`);\n \n trackTime(`Query start: \"${query.substring(0, 40)}...\"`);\n const queryStart = performance.now();\n \n // Pass trackTime callback to answer options\n const queryOptions = {\n ...answerOptions,\n trackTime: createTrackTimeCallback(),\n };\n \n const testResult: AnswerTestResult = await runAnswerTest(query, queryOptions);\n const queryDuration = performance.now() - queryStart;\n trackTime(`Query complete: \"${query.substring(0, 40)}...\"`); \n \n if (testResult.error) {\n console.error(` ❌ Error: ${testResult.error}`);\n console.log(` ⏱️ Total query duration: ${queryDuration.toFixed(2)}ms`);\n console.log(` ⏱️ Answer harness duration: ${testResult.durationMs.toFixed(2)}ms`);\n } else {\n console.log(` ✅ Answer generated successfully`);\n console.log(` 📊 Context used: ${testResult.contextUsed} pages (${testResult.totalTokens?.toLocaleString() || 'N/A'} tokens)`);\n console.log(` 🔍 Search results: ${testResult.searchResults.length} pages found\\n`);\n \n // Show timing breakdown\n if (testResult.timings) {\n console.log(` ⏱️ Timing breakdown:`);\n \n // Show detailed search timings if available\n if (testResult.metadata?.searchTimings) {\n const st = testResult.metadata.searchTimings;\n console.log(` Search (${testResult.timings.search?.toFixed(2)}ms total):`);\n if (st.queryEmbedding !== undefined) {\n const strategy = testResult.metadata.searchStrategy || 'unknown';\n console.log(` Query embedding: ${st.queryEmbedding.toFixed(2)}ms (${strategy})`);\n }\n if (st.similarity !== undefined) {\n console.log(` Cosine similarity: ${st.similarity.toFixed(2)}ms`);\n }\n if (st.sort !== undefined) {\n console.log(` Sort: ${st.sort.toFixed(2)}ms`);\n }\n if (st.transform !== undefined) {\n console.log(` Transform: ${st.transform.toFixed(2)}ms`);\n }\n } else {\n console.log(` Search: ${testResult.timings.search?.toFixed(2)}ms`);\n }\n \n console.log(` Context prep: ${testResult.timings.contextPrep?.toFixed(2)}ms`);\n console.log(` LLM call: ${testResult.timings.llm?.toFixed(2)}ms`);\n console.log(` Total: ${testResult.timings.total?.toFixed(2)}ms`);\n console.log(` ⏱️ Test harness duration: ${testResult.durationMs.toFixed(2)}ms\\n`);\n \n // Track for summary\n allQueryTimings.search.push(testResult.timings.search || 0);\n if (testResult.metadata?.searchTimings?.queryEmbedding !== undefined) {\n allQueryTimings.queryEmbedding.push(testResult.metadata.searchTimings.queryEmbedding);\n }\n allQueryTimings.contextPrep.push(testResult.timings.contextPrep || 0);\n allQueryTimings.llm.push(testResult.timings.llm || 0);\n allQueryTimings.total.push(testResult.timings.total || 0);\n }\n \n // Show search results used\n if (testResult.searchResults.length > 0) {\n console.log(` 📚 Search results used (top ${testResult.contextUsed}):`);\n testResult.searchResults.forEach((result, index) => {\n const wasUsed = index < testResult.contextUsed ? \"✓\" : \" \";\n console.log(` ${wasUsed} ${index + 1}. ${result.title || result.path}`);\n console.log(` Path: ${result.path}`);\n console.log(` Score: ${result.score.toFixed(2)}`);\n console.log('');\n });\n }\n \n // Show full answer\n console.log(` 💬 Generated Answer:`);\n console.log(` ${\"─\".repeat(76)}`);\n testResult.answer.split('\\n').forEach(line => {\n console.log(` ${line}`);\n });\n console.log(` ${\"─\".repeat(76)}\\n`);\n }\n }\n \n const overallDuration = performance.now() - overallStart;\n \n // Calculate averages\n const avgSearch = allQueryTimings.search.length > 0 \n ? allQueryTimings.search.reduce((a, b) => a + b, 0) / allQueryTimings.search.length \n : 0;\n const avgQueryEmbedding = allQueryTimings.queryEmbedding.length > 0 \n ? allQueryTimings.queryEmbedding.reduce((a, b) => a + b, 0) / allQueryTimings.queryEmbedding.length \n : 0;\n const avgContextPrep = allQueryTimings.contextPrep.length > 0 \n ? allQueryTimings.contextPrep.reduce((a, b) => a + b, 0) / allQueryTimings.contextPrep.length \n : 0;\n const avgLlm = allQueryTimings.llm.length > 0 \n ? allQueryTimings.llm.reduce((a, b) => a + b, 0) / allQueryTimings.llm.length \n : 0;\n const avgTotal = allQueryTimings.total.length > 0 \n ? allQueryTimings.total.reduce((a, b) => a + b, 0) / allQueryTimings.total.length \n : 0;\n \n trackTime(\"Test complete\");\n \n console.log(`\\n${\"=\".repeat(80)}`);\n console.log(\"Test completed!\");\n console.log(`${\"=\".repeat(80)}`);\n console.log(`⏱️ Overall timing summary:`);\n console.log(` JSON file load: ${(timings.jsonReadFile + timings.jsonParse).toFixed(2)}ms`);\n console.log(` Questions load: ${timings.loadQuestions?.toFixed(2)}ms`);\n console.log(` Total test duration: ${overallDuration.toFixed(2)}ms\\n`);\n \n if (allQueryTimings.total.length > 0) {\n console.log(`⏱️ Average timings per query (${allQueryTimings.total.length} queries):`);\n console.log(` Search (total): ${avgSearch.toFixed(2)}ms`);\n if (avgQueryEmbedding > 0) {\n console.log(` Query embedding: ${avgQueryEmbedding.toFixed(2)}ms`);\n const avgSearchOverhead = avgSearch - avgQueryEmbedding;\n console.log(` Search overhead: ${avgSearchOverhead.toFixed(2)}ms (similarity calc, sort, transform)`);\n }\n console.log(` Context prep: ${avgContextPrep.toFixed(2)}ms`);\n console.log(` LLM call: ${avgLlm.toFixed(2)}ms`);\n console.log(` Total: ${avgTotal.toFixed(2)}ms\\n`);\n \n console.log(`⏱️ Total time spent:`);\n console.log(` Search: ${allQueryTimings.search.reduce((a, b) => a + b, 0).toFixed(2)}ms`);\n if (allQueryTimings.queryEmbedding.length > 0) {\n console.log(` Query embedding: ${allQueryTimings.queryEmbedding.reduce((a, b) => a + b, 0).toFixed(2)}ms`);\n }\n console.log(` Context prep: ${allQueryTimings.contextPrep.reduce((a, b) => a + b, 0).toFixed(2)}ms`);\n console.log(` LLM calls: ${allQueryTimings.llm.reduce((a, b) => a + b, 0).toFixed(2)}ms`);\n console.log(` All queries total: ${allQueryTimings.total.reduce((a, b) => a + b, 0).toFixed(2)}ms`);\n }\n \n console.log(`${\"=\".repeat(80)}\\n`);\n \n // Display timeline of events\n console.log(`⏱️ Timeline of events (start/end times):\\n`);\n console.log(` Time (ms) | Event`);\n console.log(` ${\"-\".repeat(10)} | ${\"-\".repeat(60)}`);\n \n for (const point of timePoints) {\n const timeStr = point.time.toFixed(2).padStart(10);\n console.log(` ${timeStr} | ${point.label}`);\n }\n \n console.log(`\\n${\"=\".repeat(80)}\\n`);\n}\n\n// Run if executed directly\nif (import.meta.main) {\n testAnswerStrategies().catch(console.error);\n}\n\n",[],[220,368],"3193ad5e-b901-11f0-a764-0224a6c84d84","search-harness.ts","3193ae26-b901-11f0-a764-0224a6c84d84",39,["D",1762207522627],"// Test harness for search functionality\n// Reusable function for testing search queries\n\nimport { searchPages } from \"../search/index.ts\";\nimport type { Page, SearchResult, SearchOptions } from \"../search/types.ts\";\n\nexport interface SearchTestResult {\n results: SearchResult[];\n durationMs: number;\n error?: string;\n}\n\n/**\n * Test harness function for running a single search query\n * @param query - The search query string\n * @param pages - Array of pages to search through\n * @param options - Search options (limit, minScore, etc.)\n * @returns Promise with results and timing information\n */\nexport async function runSearchTest(\n query: string,\n pages: Page[],\n options: SearchOptions = {}\n): Promise {\n const startTime = performance.now();\n \n try {\n const results = await searchPages(query, pages, options);\n const endTime = performance.now();\n const durationMs = endTime - startTime;\n \n return {\n results,\n durationMs,\n };\n } catch (error) {\n const endTime = performance.now();\n const durationMs = endTime - startTime;\n \n return {\n results: [],\n durationMs,\n error: error instanceof Error ? error.message : String(error),\n };\n }\n}\n\n",[],[220,376],"319c6fde-b901-11f0-a8ca-0224a6c84d84","recalculate.ts","319c70b0-b901-11f0-a8ca-0224a6c84d84",40,["D",1762207522645],"// Recalculate script - runs cache recalculation and saves to /data/latest.json and /data/mm-dd-yyyy.json\n// Run with: deno task recalculate\n\nimport \"jsr:@std/dotenv/load\";\nimport { urls } from \"../urls.ts\";\nimport {\n IS_VALTOWN,\n getFromCache,\n setCache,\n calculateContentHash,\n getTextFromUrl,\n parseFrontmatter,\n addUrlSourceToFrontmatter,\n calculateTokenCount,\n} from \"../utils.ts\";\nimport { generatePageMetadata } from \"../groq.ts\";\nimport { getActiveEmbeddingsFunction } from \"../search/index.ts\";\nimport { saveTestData } from \"../main.tsx\";\n\n// Load cached data from latest.json (for local hash checking)\nasync function loadLatestJson(): Promise | null> {\n if (IS_VALTOWN) {\n return null; // Use SQLite when in Val Town\n }\n \n try {\n const latestData = await Deno.readTextFile(\"./data/latest.json\");\n const data = JSON.parse(latestData);\n \n // Create a map of url -> cached data\n const cacheMap: Record = {};\n \n if (data.results && Array.isArray(data.results)) {\n for (const result of data.results) {\n if (result.url) {\n cacheMap[result.url] = {\n contentHash: result.contentHash,\n charCount: result.charCount,\n tokenCount: result.tokenCount,\n title: result.title,\n metadata: result.metadata,\n embeddings: result.embeddings || null,\n };\n }\n }\n }\n \n return cacheMap;\n } catch {\n // latest.json doesn't exist or is invalid - that's okay, just return null\n return null;\n }\n}\n\nasync function recalculate(force: boolean = false) {\n console.log(`Starting recalculation (force: ${force}, local: ${!IS_VALTOWN})...`);\n \n // Load latest.json for hash checking when running locally\n const latestJsonData = await loadLatestJson();\n if (latestJsonData && !force) {\n console.log(`Loaded ${Object.keys(latestJsonData).length} cached hashes from latest.json`);\n }\n \n const results: Array<{ \n path: string; \n url: string; \n charCount: number; \n tokenCount: number | undefined;\n title?: string;\n metadata?: any;\n contentHash?: string;\n embeddings?: number[] | null;\n error?: string;\n cacheError?: string;\n skipped?: boolean;\n reason?: string;\n }> = [];\n \n let processedCount = 0;\n let skippedCount = 0;\n \n const getPageByPath = async (pathParam: string, useCache: boolean = true) => {\n // Find the matching URL\n const matchingUrl = urls.find(url => {\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n if (match) {\n return match[1] === pathParam;\n }\n if (url.endsWith(\"/docs.md\") && pathParam === \"docs\") {\n return true;\n }\n return false;\n });\n \n if (!matchingUrl) {\n return null;\n }\n \n // Try to get from cache first\n if (useCache) {\n const cached = await getFromCache(matchingUrl);\n if (cached) {\n return {\n url: cached.frontmatter?.urlSource || matchingUrl,\n content: cached.content,\n charCount: cached.charCount,\n tokenCount: cached.tokenCount,\n };\n }\n }\n \n // Fetch and process the single page\n const text = await getTextFromUrl(matchingUrl);\n const processedText = addUrlSourceToFrontmatter(text, matchingUrl);\n const charCount = processedText.length;\n const tokenCount = calculateTokenCount(processedText);\n \n return {\n url: matchingUrl,\n content: processedText,\n charCount,\n tokenCount,\n };\n };\n \n for (const url of urls) {\n // Extract path from URL\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n const path = match ? match[1] : (url.endsWith(\"/docs.md\") ? \"docs\" : null);\n \n if (path) {\n console.log(`[${processedCount + 1}/${urls.length}] Processing ${path}...`);\n \n try {\n // Check cached hash if not forcing\n let shouldSkip = false;\n if (!force) {\n // Try SQLite cache first (if in Val Town)\n let cached = await getFromCache(url);\n \n // If no SQLite cache, try latest.json (for local development)\n let cachedHash: string | null = null;\n let cachedFromJson: any = null;\n \n if (cached) {\n cachedHash = cached.contentHash;\n } else if (latestJsonData && latestJsonData[url]) {\n cachedFromJson = latestJsonData[url];\n cachedHash = cachedFromJson.contentHash || null;\n }\n \n if (cachedHash) {\n // Fetch fresh content to check hash\n try {\n const pageData = await getPageByPath(path, false);\n if (pageData) {\n const newHash = await calculateContentHash(pageData.content);\n if (newHash === cachedHash) {\n shouldSkip = true;\n skippedCount++;\n console.log(` Skipped ${path} (content unchanged, hash: ${newHash.substring(0, 8)}...)`);\n \n // Use cached data from SQLite or latest.json\n if (cached) {\n results.push({\n path,\n url,\n charCount: cached.charCount,\n tokenCount: cached.tokenCount || undefined,\n title: (cached.frontmatter as any).title || path,\n metadata: cached.metadata || undefined,\n contentHash: cachedHash,\n embeddings: cached.embeddings || null,\n skipped: true,\n reason: \"Content unchanged (hash matches)\",\n });\n } else if (cachedFromJson) {\n results.push({\n path,\n url,\n charCount: cachedFromJson.charCount || 0,\n tokenCount: cachedFromJson.tokenCount || undefined,\n title: cachedFromJson.title || path,\n metadata: cachedFromJson.metadata || undefined,\n contentHash: cachedHash,\n embeddings: cachedFromJson.embeddings || null,\n skipped: true,\n reason: \"Content unchanged (hash matches)\",\n });\n }\n }\n }\n } catch (hashCheckError) {\n // If hash check fails, continue with processing\n console.debug(` Hash check failed for ${path}, continuing...`);\n }\n }\n }\n \n if (shouldSkip) {\n processedCount++;\n continue;\n }\n \n // Fetch without cache\n let pageData;\n try {\n pageData = await getPageByPath(path, false);\n } catch (fetchError) {\n const fetchErrorMessage = fetchError instanceof Error ? fetchError.message : String(fetchError);\n console.error(` Failed to fetch ${path}:`, fetchErrorMessage);\n throw new Error(`Fetch failed: ${fetchErrorMessage}`);\n }\n \n if (pageData) {\n const frontmatter = parseFrontmatter(pageData.content);\n const title = (frontmatter as any).title || path;\n \n // Calculate content hash\n const contentHash = await calculateContentHash(pageData.content);\n \n // Generate AI metadata\n const metadata = await generatePageMetadata(pageData.content, title);\n if (!metadata) {\n console.warn(` Metadata generation failed for ${path} (continuing without metadata)`);\n }\n \n // Generate embeddings using active strategy's embeddings function\n const generateEmbeddings = getActiveEmbeddingsFunction();\n let embeddings: number[] | null = null;\n let embeddingsError: string | undefined = undefined;\n \n try {\n embeddings = await generateEmbeddings(pageData.content);\n if (!embeddings) {\n embeddingsError = \"Embeddings generation returned null\";\n console.error(` ❌ Embeddings generation failed for ${path}: ${embeddingsError}`);\n }\n } catch (embedError) {\n const embedErrorMessage = embedError instanceof Error ? embedError.message : String(embedError);\n embeddingsError = `Embeddings generation threw error: ${embedErrorMessage}`;\n console.error(` ❌ Embeddings generation failed for ${path}: ${embeddingsError}`);\n // Don't throw - continue with null embeddings, but log the error\n }\n \n // Cache it with metadata, hash, and embeddings - track errors\n const cacheError = await setCache(url, {\n content: pageData.content,\n charCount: pageData.charCount,\n tokenCount: pageData.tokenCount || null,\n frontmatter,\n metadata,\n contentHash,\n embeddings,\n });\n \n results.push({ \n path, \n url, \n charCount: pageData.charCount, \n tokenCount: pageData.tokenCount,\n title,\n metadata,\n contentHash,\n embeddings: embeddings || null,\n ...(cacheError && { cacheError }),\n ...(embeddingsError && { embeddingsError }),\n });\n \n const statusParts = [\n `metadata: ${metadata ? 'yes' : 'no'}`,\n `embeddings: ${embeddings ? 'yes' : 'no'}`,\n ];\n \n if (embeddingsError) {\n statusParts.push(`embeddings error: ${embeddingsError.substring(0, 50)}...`);\n }\n if (cacheError) {\n statusParts.push(`cache error: ${cacheError}`);\n }\n \n const status = statusParts.join(', ');\n if (embeddingsError || cacheError) {\n console.warn(`Completed ${path} (${status})`);\n } else {\n console.log(`Completed ${path} (${status}, cached: yes)`);\n }\n }\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(`Error processing ${path}:`, errorMessage);\n results.push({\n path,\n url,\n charCount: 0,\n tokenCount: 0,\n error: errorMessage,\n });\n }\n \n processedCount++;\n }\n }\n \n const cacheErrors = results.filter(r => r.cacheError);\n const processedResults = results.filter(r => !r.skipped);\n \n const summary = {\n message: force \n ? `Recalculated ${processedResults.length} pages with AI metadata (force mode)` \n : `Recalculated ${processedResults.length} pages, skipped ${skippedCount} unchanged pages`,\n timestamp: new Date().toISOString(),\n results,\n totalPages: results.length,\n processed: processedResults.length,\n skipped: skippedCount,\n withMetadata: processedResults.filter(r => r.metadata).length,\n withoutMetadata: processedResults.filter(r => !r.metadata).length,\n cached: processedResults.length > 0 && cacheErrors.length === 0,\n cacheErrors: cacheErrors.length > 0 ? cacheErrors.map(r => ({ path: r.path, error: r.cacheError })) : undefined,\n force,\n };\n \n // Save to both latest.json and dated file\n await saveTestData(summary);\n console.log(\"\\nRecalculation completed!\");\n \n return summary;\n}\n\n// Run if executed directly\nif (import.meta.main) {\n const force = Deno.args.includes(\"--force\") || Deno.args.includes(\"-f\");\n recalculate(force).catch(console.error);\n}\n\n",[],[220,384],"807481c2-b9ca-11f0-83d0-0224a6c84d84","recalculate-mixedbread.ts","80748280-b9ca-11f0-83d0-0224a6c84d84",90,["D",1762293983725],"// Recalculate for Mixedbread - uploads documents to Mixedbread Store\n// Run with: deno task recalc-mxbai\n\nimport \"jsr:@std/dotenv/load\";\nimport { urls } from \"../urls.ts\";\nimport {\n getTextFromUrl,\n addUrlSourceToFrontmatter,\n parseFrontmatter,\n} from \"../utils.ts\";\nimport { ensureStore, uploadDocument } from \"../search/mixedbread.ts\";\n\nasync function recalculateMixedbread() {\n console.log(`Starting Mixedbread Store upload...`);\n \n // Check for --force flag to delete and recreate\n const forceRecreate = Deno.args.includes(\"--force\") || Deno.args.includes(\"-f\");\n \n // Ensure store exists (creates or uses existing)\n await ensureStore(forceRecreate);\n \n // Process each URL\n let successCount = 0;\n let failCount = 0;\n \n for (let i = 0; i < urls.length; i++) {\n const url = urls[i];\n \n // Extract path from URL\n const match = url.match(/\\/docs\\/(.+?)\\.md$/);\n const path = match ? match[1] : (url.endsWith(\"/docs.md\") ? \"docs\" : null);\n \n if (!path) {\n console.log(`[${i + 1}/${urls.length}] Skipping ${url} (no path)`);\n continue;\n }\n \n console.log(`[${i + 1}/${urls.length}] Processing ${path}...`);\n \n try {\n // Fetch and process content\n const text = await getTextFromUrl(url);\n const processedText = addUrlSourceToFrontmatter(text, url);\n const frontmatter = parseFrontmatter(processedText);\n const title = (frontmatter as any).title || path;\n \n // Upload document using the helper function\n const result = await uploadDocument(path, processedText, {\n url,\n title,\n charCount: processedText.length,\n });\n \n if (result.success) {\n console.log(` ✅ Uploaded ${path} (${processedText.length} chars)`);\n successCount++;\n } else {\n throw new Error(result.error || \"Upload failed\");\n }\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(` ❌ Failed to upload ${path}:`, errorMessage);\n failCount++;\n }\n }\n \n console.log(`\\n📊 Upload Summary:`);\n console.log(` Success: ${successCount}`);\n console.log(` Failed: ${failCount}`);\n console.log(` Total: ${urls.length}`);\n console.log(`\\n✅ Mixedbread Store upload completed!`);\n \n return {\n success: successCount,\n failed: failCount,\n total: urls.length,\n };\n}\n\n// Run if executed directly\nif (import.meta.main) {\n recalculateMixedbread().catch(console.error);\n}\n\n",[],[220,392],"31a38e68-b901-11f0-a3b9-0224a6c84d84","questions.ts","31a38f30-b901-11f0-a3b9-0224a6c84d84",41,["D",1762207522633],"// Test questions for search testing\n// Full sentence questions about Groq\n\nexport const questions: string[] = [\n \"What is Compound and how does it work?\",\n // \"How do I authenticate with the Groq API?\",\n // \"What are the rate limits for Groq models?\",\n // \"How do I integrate Groq with my JavaScript application?\",\n // \"What is the difference between compound and compound-mini?\",\n // \"Can I use Groq for real-time web search?\",\n // \"How do I get started with Groq's API?\",\n // \"What models are available on Groq?\",\n // \"How does Groq's Compound AI system work?\",\n // \"Can I use custom tools with Groq's Compound AI?\",\n];\n\n",[],[220,400],"80603528-b9ca-11f0-a963-0224a6c84d84","answer-harness.ts","806035f0-b9ca-11f0-a963-0224a6c84d84",87,["D",1762293983672],"// Test harness for answer functionality\n// Reusable function for testing answer queries\n\nimport { answerQuestion } from \"../answer/index.ts\";\nimport type { AnswerResult, AnswerOptions } from \"../answer/types.ts\";\n\nexport interface AnswerTestResult {\n answer: string;\n searchResults: Array<{\n path: string;\n url: string;\n title?: string;\n score: number;\n }>;\n contextUsed: number;\n totalTokens?: number;\n durationMs: number;\n timings?: Record;\n metadata?: {\n searchStrategy?: string;\n searchTimings?: Record;\n [key: string]: any;\n };\n error?: string;\n}\n\n/**\n * Test harness function for running a single answer query\n * @param query - The question to answer\n * @param options - Answer options (limit, minScore, maxContextPages, etc.)\n * @returns Promise with answer results and timing information\n */\nexport async function runAnswerTest(\n query: string,\n options: AnswerOptions = {}\n): Promise {\n const startTime = performance.now();\n \n try {\n const result: AnswerResult = await answerQuestion(query, {\n ...options,\n enableTiming: true, // Always enable timing for tests\n });\n const endTime = performance.now();\n const durationMs = endTime - startTime;\n \n return {\n answer: result.answer,\n searchResults: result.searchResults,\n contextUsed: result.contextUsed,\n totalTokens: result.totalTokens,\n durationMs,\n timings: result.metadata?.timings,\n metadata: result.metadata, // Pass through full metadata for search details\n };\n } catch (error) {\n const endTime = performance.now();\n const durationMs = endTime - startTime;\n \n return {\n answer: \"\",\n searchResults: [],\n contextUsed: 0,\n durationMs,\n error: error instanceof Error ? error.message : String(error),\n };\n }\n}\n\n",[],[220,408],["D",1762207521420],[302,303,304,305,306,307,308],[220],"809b99ec-b9ca-11f0-b384-0224a6c84d84","31528d88-b901-11f0-a3b9-0224a6c84d84",95,["D",1762293984044],"// Shared utilities for search strategies\n\n// Calculate cosine similarity between two embedding vectors\nexport const cosineSimilarity = (vecA: number[], vecB: number[]): number => {\n if (vecA.length !== vecB.length) {\n return 0;\n }\n \n let dotProduct = 0;\n let normA = 0;\n let normB = 0;\n \n for (let i = 0; i < vecA.length; i++) {\n dotProduct += vecA[i] * vecB[i];\n normA += vecA[i] * vecA[i];\n normB += vecB[i] * vecB[i];\n }\n \n if (normA === 0 || normB === 0) {\n return 0;\n }\n \n return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));\n};\n\n// Helper function to generate snippets\nexport const generateSnippet = (content: string, queryWords: string[], queryLower: string): string => {\n // Handle empty content (when content wasn't fetched during search initialization)\n if (!content || content.length === 0) {\n return `[Content available at page] - Query: ${queryLower}`;\n }\n \n const contentLower = content.toLowerCase();\n const firstMatchIndex = contentLower.indexOf(queryWords[0]);\n \n if (firstMatchIndex >= 0) {\n const start = Math.max(0, firstMatchIndex - 50);\n const end = Math.min(content.length, firstMatchIndex + queryWords[0].length + 150);\n let snippet = content.substring(start, end);\n if (start > 0) snippet = '...' + snippet;\n if (end < content.length) snippet = snippet + '...';\n return snippet;\n }\n \n return content.substring(0, 200) + '...';\n};\n\n",[],[226,142],"316a968a-b901-11f0-9996-0224a6c84d84","types.ts","316a973e-b901-11f0-9996-0224a6c84d84",34,["D",1762207522301],"// Search strategy types and interfaces\n\nexport type SearchResult = {\n path: string;\n url: string;\n title?: string;\n score: number;\n snippet?: string;\n metadata?: {\n strategy?: string;\n [key: string]: any;\n };\n};\n\nexport type Page = {\n path: string;\n url: string;\n content: string;\n title?: string;\n embeddings?: number[] | null;\n metadata?: any;\n};\n\nexport type SearchOptions = {\n limit?: number;\n minScore?: number;\n [key: string]: any;\n};\n\nexport type SearchStrategy = {\n name: string;\n description: string;\n search: (query: string, pages: Page[], options: SearchOptions) => Promise;\n};\n\n",[],[226,426],"7f8a6948-b9ca-11f0-ac30-0224a6c84d84","transformers-local-onnx.ts","7f8a6d08-b9ca-11f0-ac30-0224a6c84d84",61,["D",1762293982236],"// Transformers.js + Local ONNX Models + Cosine Similarity Strategy\n// Uses pre-downloaded ONNX models from the models/ directory\n// Fastest option - no downloads, no API calls, runs entirely locally\n// Model: all-MiniLM-L6-v2 (~23MB or ~6MB quantized, ~384 dimensions)\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { generateSnippet, cosineSimilarity } from \"./utils.ts\";\n\n// Lazy-loaded pipeline (cached after first load)\nlet embeddingPipeline: any = null;\nlet pipelineLoading: Promise | null = null;\n\n// Configuration\nconst MODEL_PATH = new URL(\"./models/all-MiniLM-L6-v2\", import.meta.url).pathname;\nconst USE_QUANTIZED = false; // Set to true to use model_quantized.onnx (faster, slightly less accurate)\n\n// Cleanup function to dispose of pipeline resources\nexport const disposePipeline = async (): Promise => {\n if (embeddingPipeline) {\n try {\n if (typeof embeddingPipeline.dispose === 'function') {\n await embeddingPipeline.dispose();\n } else if (typeof embeddingPipeline.disconnect === 'function') {\n await embeddingPipeline.disconnect();\n }\n embeddingPipeline = null;\n } catch (error) {\n console.debug(\"Pipeline cleanup error (ignored):\", error);\n }\n }\n};\n\n// Prevent ONNX runtime cleanup crash on exit\n// The crash is harmless but annoying - this suppresses it\nif (typeof Deno !== \"undefined\") {\n Deno.addSignalListener(\"SIGINT\", () => {\n // Force immediate exit without cleanup\n Deno.exit(0);\n });\n \n // Also handle unload event\n globalThis.addEventListener(\"unload\", () => {\n // Prevent any cleanup that might cause mutex errors\n embeddingPipeline = null;\n });\n}\n\nasync function getEmbeddingPipeline() {\n // Return cached pipeline if already loaded\n if (embeddingPipeline) {\n return embeddingPipeline;\n }\n \n // If loading in progress, wait for it\n if (pipelineLoading) {\n return pipelineLoading;\n }\n \n // Start loading pipeline\n pipelineLoading = (async () => {\n const modelLoadTimings: Record = {};\n const modelLoadStart = performance.now();\n \n try {\n // Import transformers.js - use npm: prefix for Deno compatibility\n const importStart = performance.now();\n const { pipeline, env } = await import(\"npm:@huggingface/transformers@latest\");\n modelLoadTimings.npmImport = performance.now() - importStart;\n \n // Configure to use local models\n env.allowRemoteModels = false; // Prevent downloading from HuggingFace\n env.allowLocalModels = true;\n \n // Load feature extraction pipeline from local ONNX model\n const pipelineStart = performance.now();\n const modelType = USE_QUANTIZED ? 'quantized' : 'full';\n \n console.log(`📂 Loading local ONNX model from: ${MODEL_PATH}`);\n console.log(` Using ${modelType} model`);\n \n embeddingPipeline = await pipeline(\n 'feature-extraction',\n MODEL_PATH,\n {\n device: 'cpu',\n dtype: 'fp32',\n // Point to local model files - don't specify model_file_name as it causes double .onnx extension\n local_files_only: true,\n ...(USE_QUANTIZED ? { quantized: true } : {}),\n }\n );\n \n modelLoadTimings.pipelineLoad = performance.now() - pipelineStart;\n modelLoadTimings.totalModelLoad = performance.now() - modelLoadStart;\n \n console.log(`✅ Local ONNX model loaded successfully:`);\n console.log(` npm import: ${modelLoadTimings.npmImport?.toFixed(2)}ms`);\n console.log(` pipeline load: ${modelLoadTimings.pipelineLoad?.toFixed(2)}ms (from local files)`);\n console.log(` total model load: ${modelLoadTimings.totalModelLoad?.toFixed(2)}ms`);\n \n return embeddingPipeline;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n const fullError = `Failed to load local ONNX model from ${MODEL_PATH}: ${errorMessage}\\n` +\n `\\nPlease ensure the model files are downloaded. See search/models/README.md for setup instructions.\\n` +\n `\\nTo download the model:\\n` +\n ` cd search/models\\n` +\n ` git clone https://huggingface.co/Xenova/all-MiniLM-L6-v2\\n` +\n ` rm -rf all-MiniLM-L6-v2/.git\\n`;\n console.error(fullError);\n if (modelLoadTimings.npmImport) {\n console.error(` Failed after npm import (${modelLoadTimings.npmImport.toFixed(2)}ms)`);\n }\n throw new Error(fullError);\n } finally {\n pipelineLoading = null;\n }\n })();\n \n return pipelineLoading;\n}\n\n// Generate embeddings using local ONNX model (runs locally, no network calls)\nexport const generateEmbeddings = async (content: string): Promise => {\n try {\n const pipeline = await getEmbeddingPipeline();\n \n // Generate embeddings - returns a tensor, we need to extract the array\n const output = await pipeline(content, {\n pooling: 'mean',\n normalize: true,\n });\n \n // Extract the embedding vector from the tensor\n if (output && typeof output === 'object') {\n // Check for .data property (common tensor format)\n if (output.data && Array.isArray(output.data)) {\n return output.data;\n }\n \n // Check if it's a nested array (tensor shape)\n if (Array.isArray(output)) {\n if (Array.isArray(output[0])) {\n return output[0]; // Return first row if 2D\n }\n return output;\n }\n \n // Try accessing tensor properties\n if (output.size && output.data) {\n return Array.from(output.data);\n }\n \n // Try tolist() method if available\n if (typeof output.tolist === 'function') {\n const list = output.tolist();\n return Array.isArray(list[0]) ? list[0] : list;\n }\n }\n \n // Fallback: try to convert to array\n return Array.from(output as any);\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"Local ONNX embeddings failed:\", errorMessage);\n return null;\n }\n};\n\nexport const searchStrategy: SearchStrategy = {\n name: \"transformers-local-onnx\",\n description: \"Semantic search using local pre-downloaded ONNX models with cosine similarity (fastest, no downloads, no API calls)\",\n search: async (query: string, pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const minSimilarity = (options.minScore || 0) / 100; // Convert to 0-1 scale\n const enableTiming = options.enableTiming || false;\n \n const timings: Record = {};\n const startTime = performance.now();\n \n // Generate query embedding (local, no network calls)\n const queryEmbedStart = performance.now();\n const queryEmbedding = await generateEmbeddings(query);\n if (enableTiming) {\n timings.queryEmbedding = performance.now() - queryEmbedStart;\n }\n \n if (!queryEmbedding) {\n console.warn(\"Query embedding generation failed, returning empty results\");\n return [];\n }\n \n // Calculate cosine similarity for each page\n const similarityStart = performance.now();\n const scoredPages: Array<{ page: Page; similarity: number }> = [];\n \n for (const page of pages) {\n const pageEmbedding = page.embeddings;\n if (!pageEmbedding || pageEmbedding.length !== queryEmbedding.length) {\n continue; // Skip pages without embeddings or wrong dimension\n }\n \n const similarity = cosineSimilarity(queryEmbedding, pageEmbedding);\n if (similarity >= minSimilarity) {\n scoredPages.push({ page, similarity });\n }\n }\n \n if (enableTiming) timings.similarity = performance.now() - similarityStart;\n \n // Sort by similarity (descending) and limit\n const sortStart = performance.now();\n scoredPages.sort((a, b) => b.similarity - a.similarity);\n const topPages = scoredPages.slice(0, limit);\n if (enableTiming) timings.sort = performance.now() - sortStart;\n \n // Transform to SearchResult format\n const transformStart = performance.now();\n const queryWords = query.toLowerCase().split(/\\s+/).filter(w => w.length > 0);\n const results: SearchResult[] = topPages.map(({ page, similarity }) => ({\n path: page.path,\n url: page.url,\n title: page.title || page.path,\n score: similarity * 100, // Convert to 0-100 scale\n snippet: generateSnippet(page.content, queryWords, query.toLowerCase()),\n metadata: {\n strategy: \"transformers-local-onnx\",\n similarity,\n localModel: true,\n modelPath: MODEL_PATH,\n quantized: USE_QUANTIZED,\n ...(enableTiming && { timings }),\n },\n }));\n if (enableTiming) timings.transform = performance.now() - transformStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Timing breakdown (Local ONNX):`);\n console.log(` Query embedding: ${timings.queryEmbedding?.toFixed(2)}ms (local model - no network)`);\n console.log(` Cosine similarity: ${timings.similarity?.toFixed(2)}ms`);\n console.log(` Sort: ${timings.sort?.toFixed(2)}ms`);\n console.log(` Transform: ${timings.transform?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n return results;\n },\n};\n\n",[],[226,434],"80af3f56-b9ca-11f0-b384-0224a6c84d84",["D",1762293984186],[],[226,33],"3181e164-b901-11f0-a3b9-0224a6c84d84","placeholder.ts","3181e22c-b901-11f0-a3b9-0224a6c84d84",37,["D",1762207522458],"// Placeholder Strategy: Keyword-based search (current default)\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { generateSnippet } from \"./utils.ts\";\n\n// Fake embeddings function for placeholder (not actually used, but kept for interface consistency)\nexport const generateEmbeddings = async (content: string): Promise => {\n // Not used in placeholder strategy, but kept for consistency\n // Returns fake embeddings for caching purposes\n console.debug(\"Generating fake embeddings for placeholder strategy\");\n \n const fakeEmbedding: number[] = [];\n const hash = content.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0);\n \n for (let i = 0; i < 384; i++) {\n const value = Math.sin(hash + i) * 0.5 + 0.5;\n fakeEmbedding.push(value);\n }\n \n return fakeEmbedding;\n};\n\nexport const searchStrategy: SearchStrategy = {\n name: \"placeholder\",\n description: \"Keyword-based search using title, content, and metadata matching\",\n search: async (query: string, pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const minScore = options.minScore || 0;\n \n const queryLower = query.toLowerCase();\n const queryWords = queryLower.split(/\\s+/).filter(w => w.length > 0);\n \n const results: SearchResult[] = [];\n \n for (const page of pages) {\n let score = 0;\n const contentLower = page.content.toLowerCase();\n const titleLower = (page.title || page.path).toLowerCase();\n \n // Title matches are worth more\n for (const word of queryWords) {\n if (titleLower.includes(word)) {\n score += 10;\n }\n // Count occurrences in content\n const contentMatches = (contentLower.match(new RegExp(word, 'g')) || []).length;\n score += Math.min(contentMatches, 5); // Cap at 5 points per word\n }\n \n // Exact phrase match bonus\n if (contentLower.includes(queryLower)) {\n score += 20;\n }\n \n // Metadata boost (if query matches tags or categories)\n if (page.metadata) {\n const tags = (page.metadata.tags || []).join(' ').toLowerCase();\n const categories = (page.metadata.categories || []).join(' ').toLowerCase();\n const useCases = (page.metadata.useCases || []).join(' ').toLowerCase();\n \n for (const word of queryWords) {\n if (tags.includes(word)) score += 3;\n if (categories.includes(word)) score += 2;\n if (useCases.includes(word)) score += 2;\n }\n }\n \n if (score > minScore) {\n results.push({\n path: page.path,\n url: page.url,\n title: page.title,\n score,\n snippet: generateSnippet(page.content, queryWords, queryLower),\n metadata: {\n strategy: \"placeholder\",\n },\n });\n }\n }\n \n // Sort by score descending\n results.sort((a, b) => b.score - a.score);\n \n return results.slice(0, limit);\n },\n};\n\n",[],[226,446],"314db04c-b901-11f0-a764-0224a6c84d84","openai-orama.ts","314db0ec-b901-11f0-a764-0224a6c84d84",31,["D",1762207522111],"// OpenAI + Orama Strategy: Semantic search using OpenAI embeddings and Orama\n// Faster than JigsawStack (~100-200ms vs ~550ms for query embeddings)\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { generateSnippet } from \"./utils.ts\";\nimport { create, insertMultiple, search } from \"npm:@orama/orama@latest\";\n\n// OpenAI embeddings function\nexport const generateEmbeddings = async (content: string): Promise => {\n const OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\n const OPENAI_API_URL = \"https://api.openai.com/v1/embeddings\";\n \n if (!OPENAI_API_KEY) {\n console.warn(\"OPENAI_API_KEY not found - embeddings disabled\");\n return null;\n }\n \n try {\n const response = await fetch(OPENAI_API_URL, {\n method: \"POST\",\n headers: {\n \"Authorization\": `Bearer ${OPENAI_API_KEY}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n model: \"text-embedding-3-small\", // Fast and efficient model\n input: content,\n }),\n });\n \n if (!response.ok) {\n const errorText = await response.text().catch(() => response.statusText);\n throw new Error(`OpenAI API error: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);\n }\n \n const data = await response.json();\n \n if (data.data && Array.isArray(data.data) && data.data.length > 0) {\n return data.data[0].embedding; // Return the embedding vector\n }\n \n return null;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"OpenAI embeddings failed:\", errorMessage);\n return null;\n }\n};\n\n// No caching for query embeddings - we want to benchmark actual API performance\n\nexport const searchStrategy: SearchStrategy = {\n name: \"openai-orama\",\n description: \"Semantic search using OpenAI embeddings with Orama vector search (faster than JigsawStack)\",\n search: async (query: string, pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const minSimilarity = (options.minScore || 0) / 100; // Convert to 0-1 scale\n const enableTiming = options.enableTiming || false;\n \n const timings: Record = {};\n const startTime = performance.now();\n \n // Generate query embedding (no caching - benchmarking actual API performance)\n const queryEmbedStart = performance.now();\n const queryEmbedding = await generateEmbeddings(query);\n if (enableTiming) {\n timings.queryEmbedding = performance.now() - queryEmbedStart;\n }\n \n if (!queryEmbedding) {\n console.warn(\"Query embedding generation failed, returning empty results\");\n return [];\n }\n \n // Determine embedding dimension from query (or first page with embeddings)\n const embeddingDimension = queryEmbedding.length;\n if (embeddingDimension === 0) {\n return [];\n }\n \n // Create Orama database with vector field\n const dbCreateStart = performance.now();\n const db = await create({\n schema: {\n path: \"string\",\n url: \"string\",\n title: \"string\",\n content: \"string\",\n embedding: `vector[${embeddingDimension}]`,\n },\n });\n if (enableTiming) timings.dbCreate = performance.now() - dbCreateStart;\n \n // Prepare documents with embeddings for insertion\n const documents: Array<{\n path: string;\n url: string;\n title: string;\n content: string;\n embedding: number[];\n }> = [];\n \n // Collect pages with embeddings\n const docPrepStart = performance.now();\n for (const page of pages) {\n // Use cached embeddings if available, otherwise generate\n let pageEmbedding = page.embeddings;\n if (!pageEmbedding) {\n // Generate embedding if not cached (should be cached from recalculate, but handle missing)\n pageEmbedding = await generateEmbeddings(page.content);\n if (!pageEmbedding || pageEmbedding.length !== embeddingDimension) {\n continue; // Skip pages without embeddings or wrong dimension\n }\n }\n \n // Ensure embedding dimension matches\n if (pageEmbedding.length !== embeddingDimension) {\n continue;\n }\n \n documents.push({\n path: page.path,\n url: page.url,\n title: page.title || page.path,\n content: page.content,\n embedding: pageEmbedding,\n });\n }\n if (enableTiming) timings.docPrep = performance.now() - docPrepStart;\n \n // Insert all documents into Orama\n const insertStart = performance.now();\n if (documents.length > 0) {\n await insertMultiple(db, documents);\n } else {\n return []; // No documents to search\n }\n if (enableTiming) timings.insert = performance.now() - insertStart;\n \n // Perform vector similarity search using Orama\n const searchStart = performance.now();\n const searchResults = await search(db, {\n mode: \"vector\",\n vector: {\n value: queryEmbedding,\n property: \"embedding\",\n },\n similarity: minSimilarity,\n limit,\n includeVectors: false, // Don't need vectors in response\n });\n if (enableTiming) timings.search = performance.now() - searchStart;\n \n // Transform Orama results to SearchResult format\n const transformStart = performance.now();\n const results: SearchResult[] = searchResults.hits.map((hit: any) => {\n const page = pages.find(p => p.path === hit.document.path);\n const queryWords = query.toLowerCase().split(/\\s+/).filter(w => w.length > 0);\n \n return {\n path: hit.document.path,\n url: hit.document.url,\n title: hit.document.title,\n score: (hit.score || 0) * 100, // Convert similarity to score (0-100)\n snippet: page ? generateSnippet(page.content, queryWords, query.toLowerCase()) : hit.document.content.substring(0, 200) + '...',\n metadata: {\n strategy: \"openai-orama\",\n similarity: hit.score,\n ...(enableTiming && { timings }),\n },\n };\n });\n if (enableTiming) timings.transform = performance.now() - transformStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Timing breakdown:`);\n console.log(` Query embedding: ${timings.queryEmbedding?.toFixed(2)}ms (API call)`);\n console.log(` DB create: ${timings.dbCreate?.toFixed(2)}ms`);\n console.log(` Doc prep: ${timings.docPrep?.toFixed(2)}ms`);\n console.log(` Insert: ${timings.insert?.toFixed(2)}ms`);\n console.log(` Search: ${timings.search?.toFixed(2)}ms`);\n console.log(` Transform: ${timings.transform?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n return results;\n },\n};\n\n",[],[226,454],"31459b96-b901-11f0-9996-0224a6c84d84","openai-cosine.ts","31459c22-b901-11f0-9996-0224a6c84d84",30,["D",1762207522106],"// OpenAI + Cosine Similarity Strategy: Direct cosine similarity calculation\n// Fastest for small datasets (<100 pages) - no DB overhead\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { generateSnippet, cosineSimilarity } from \"./utils.ts\";\n\n// OpenAI embeddings function\nexport const generateEmbeddings = async (content: string): Promise => {\n const OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\n const OPENAI_API_URL = \"https://api.openai.com/v1/embeddings\";\n \n if (!OPENAI_API_KEY) {\n console.warn(\"OPENAI_API_KEY not found - embeddings disabled\");\n return null;\n }\n \n try {\n const response = await fetch(OPENAI_API_URL, {\n method: \"POST\",\n headers: {\n \"Authorization\": `Bearer ${OPENAI_API_KEY}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n model: \"text-embedding-3-small\", // Fast and efficient model\n input: content,\n }),\n });\n \n if (!response.ok) {\n const errorText = await response.text().catch(() => response.statusText);\n throw new Error(`OpenAI API error: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);\n }\n \n const data = await response.json();\n \n if (data.data && Array.isArray(data.data) && data.data.length > 0) {\n return data.data[0].embedding; // Return the embedding vector\n }\n \n return null;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"OpenAI embeddings failed:\", errorMessage);\n return null;\n }\n};\n\n// No caching for query embeddings - we want to benchmark actual API performance\n\n\nexport const searchStrategy: SearchStrategy = {\n name: \"openai-cosine\",\n description: \"Semantic search using OpenAI embeddings with direct cosine similarity (fastest for small datasets)\",\n search: async (query: string, pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const minSimilarity = (options.minScore || 0) / 100; // Convert to 0-1 scale\n const enableTiming = options.enableTiming || false;\n \n const timings: Record = {};\n const startTime = performance.now();\n \n // Generate query embedding (no caching - benchmarking actual API performance)\n const queryEmbedStart = performance.now();\n const queryEmbedding = await generateEmbeddings(query);\n if (enableTiming) {\n timings.queryEmbedding = performance.now() - queryEmbedStart;\n }\n \n if (!queryEmbedding) {\n console.warn(\"Query embedding generation failed, returning empty results\");\n return [];\n }\n \n // Calculate cosine similarity for each page\n const similarityStart = performance.now();\n const scoredPages: Array<{ page: Page; similarity: number }> = [];\n \n for (const page of pages) {\n const pageEmbedding = page.embeddings;\n if (!pageEmbedding || pageEmbedding.length !== queryEmbedding.length) {\n continue; // Skip pages without embeddings or wrong dimension\n }\n \n const similarity = cosineSimilarity(queryEmbedding, pageEmbedding);\n if (similarity >= minSimilarity) {\n scoredPages.push({ page, similarity });\n }\n }\n \n if (enableTiming) timings.similarity = performance.now() - similarityStart;\n \n // Sort by similarity (descending) and limit\n const sortStart = performance.now();\n scoredPages.sort((a, b) => b.similarity - a.similarity);\n const topPages = scoredPages.slice(0, limit);\n if (enableTiming) timings.sort = performance.now() - sortStart;\n \n // Transform to SearchResult format\n const transformStart = performance.now();\n const queryWords = query.toLowerCase().split(/\\s+/).filter(w => w.length > 0);\n const results: SearchResult[] = topPages.map(({ page, similarity }) => ({\n path: page.path,\n url: page.url,\n title: page.title || page.path,\n score: similarity * 100, // Convert to 0-100 scale\n snippet: generateSnippet(page.content, queryWords, query.toLowerCase()),\n metadata: {\n strategy: \"openai-cosine\",\n similarity,\n ...(enableTiming && { timings }),\n },\n }));\n if (enableTiming) timings.transform = performance.now() - transformStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Timing breakdown:`);\n console.log(` Query embedding: ${timings.queryEmbedding?.toFixed(2)}ms (API call)`);\n console.log(` Cosine similarity: ${timings.similarity?.toFixed(2)}ms`);\n console.log(` Sort: ${timings.sort?.toFixed(2)}ms`);\n console.log(` Transform: ${timings.transform?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n return results;\n },\n};\n\n",[],[226,462],"804933a0-b9ca-11f0-9362-0224a6c84d84","mixedbread.ts","8049344a-b9ca-11f0-9362-0224a6c84d84",85,["D",1762293983485],"// Mixedbread Stores Strategy: Managed AI Search Service\n// Uses Mixedbread's Store API for document storage and semantic search\n// No local embeddings needed - Mixedbread handles everything\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { generateSnippet } from \"./utils.ts\";\n\n// Lazy-loaded Mixedbread client\nlet mxbaiClient: any = null;\nlet clientLoading: Promise | null = null;\n\nasync function getMixedbreadClient() {\n // Return cached client if already loaded\n if (mxbaiClient) {\n return mxbaiClient;\n }\n \n // If loading in progress, wait for it\n if (clientLoading) {\n return clientLoading;\n }\n \n // Start loading client\n clientLoading = (async () => {\n try {\n // Import Mixedbread SDK\n const { Mixedbread } = await import(\"npm:@mixedbread/sdk@latest\");\n \n const MIXEDBREAD_API_KEY = Deno.env.get(\"MIXEDBREAD_API_KEY\") || Deno.env.get(\"MXBAI_API_KEY\");\n \n if (!MIXEDBREAD_API_KEY) {\n throw new Error(\"MIXEDBREAD_API_KEY or MXBAI_API_KEY not found - search disabled\");\n }\n \n mxbaiClient = new Mixedbread({ apiKey: MIXEDBREAD_API_KEY });\n console.log(\"✅ Mixedbread client initialized\");\n return mxbaiClient;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n throw new Error(`Failed to load Mixedbread client: ${errorMessage}`);\n } finally {\n clientLoading = null;\n }\n })();\n \n return clientLoading;\n}\n\n// Store configuration\nconst STORE_NAME = \"groq-docs\";\n\n// Helper: Ensure store exists\nexport const ensureStore = async (forceRecreate: boolean = false): Promise => {\n const client = await getMixedbreadClient();\n \n if (forceRecreate) {\n // Force mode: Delete existing store and create fresh\n console.log(`🔄 Force mode: Deleting and recreating store...`);\n try {\n await client.stores.retrieve(STORE_NAME);\n console.log(`🗑️ Deleting existing store: ${STORE_NAME}`);\n await client.stores.delete(STORE_NAME);\n console.log(`✅ Existing store deleted`);\n } catch (error: any) {\n if (error?.status === 404) {\n console.log(`📦 No existing store found`);\n }\n }\n \n console.log(`📦 Creating fresh store: ${STORE_NAME}`);\n await client.stores.create({ name: STORE_NAME });\n console.log(`✅ Store created successfully`);\n } else {\n // Normal mode: Use existing store or create if doesn't exist\n try {\n const store = await client.stores.retrieve(STORE_NAME);\n console.log(`📦 Using existing store: ${STORE_NAME}`);\n console.log(` Files: ${store.file_counts?.total || 0} total`);\n } catch (error: any) {\n if (error?.status === 404) {\n console.log(`📦 Creating new store: ${STORE_NAME}`);\n await client.stores.create({ name: STORE_NAME });\n console.log(`✅ Store created successfully`);\n } else {\n throw error;\n }\n }\n }\n};\n\n// Helper: Upload a single document to the store\nexport const uploadDocument = async (\n path: string,\n content: string,\n metadata: { url: string; title: string; charCount: number }\n): Promise<{ success: boolean; error?: string }> => {\n const client = await getMixedbreadClient();\n \n // Two-step process:\n // 1. Upload file to Files API to get file_id\n // 2. Add that file_id to the Store with metadata\n \n try {\n // Step 1: Create a File object from the content\n // Create a blob from the text content\n const blob = new Blob([content], { type: 'text/plain' });\n // Create a File object (which is like a blob with a name)\n const file = new File([blob], `${path.replace(/\\//g, '_')}.txt`, { type: 'text/plain' });\n \n // Upload file to Files API to get a file_id\n const uploadedFile = await client.files.create({\n file: file,\n });\n \n // Step 2: Add the file_id to the Store with metadata\n const storeFile = await client.stores.files.create(\n STORE_NAME,\n {\n file_id: uploadedFile.id,\n metadata: {\n path,\n url: metadata.url,\n title: metadata.title,\n charCount: metadata.charCount,\n },\n }\n );\n \n return { success: true };\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n return { success: false, error: errorMessage };\n }\n};\n\n// This function is required for compatibility with the recalculation system\n// For Mixedbread managed Store, embeddings are handled by Mixedbread internally\n// Return a dummy array to satisfy the recalculation script\n// The actual embeddings are generated and stored by Mixedbread when documents are uploaded\nexport const generateEmbeddings = async (_content: string): Promise => {\n // Return a dummy embedding array to satisfy recalculation\n // The recalculation script requires this, but for Mixedbread Store,\n // you should use `deno task recalc-mxbai` instead, which uploads docs to Mixedbread\n return [0]; // Dummy value - actual embeddings are handled by Mixedbread Store\n};\n\nexport const searchStrategy: SearchStrategy = {\n name: \"mixedbread\",\n description: \"Managed AI search using Mixedbread Stores (handles storage, embeddings, and search)\",\n search: async (query: string, _pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const enableTiming = options.enableTiming || false;\n \n const timings: Record = {};\n const startTime = performance.now();\n \n try {\n const client = await getMixedbreadClient();\n \n // Search the Mixedbread Store\n const searchStart = performance.now();\n const response = await client.stores.search({\n query,\n store_identifiers: [STORE_NAME],\n top_k: limit,\n // Optional: enable reranking for better results\n // search_options: {\n // rerank: true,\n // },\n });\n if (enableTiming) {\n timings.mixedbreadSearch = performance.now() - searchStart;\n }\n \n // Transform Mixedbread results to our SearchResult format\n const transformStart = performance.now();\n const results: SearchResult[] = response.data.map((chunk: any) => {\n const metadata = chunk.metadata || {};\n const path = metadata.path || chunk.filename || 'unknown';\n const url = metadata.url || '';\n const title = metadata.title || path;\n \n // Extract content (text or OCR text from images)\n const content = chunk.text || chunk.ocr_text || '';\n \n // Generate snippet from the chunk content\n const queryWords = query.toLowerCase().split(/\\s+/).filter((w: string) => w.length > 0);\n const snippet = generateSnippet(content, queryWords, query.toLowerCase());\n \n return {\n path,\n url,\n title,\n score: chunk.score * 100, // Convert to 0-100 scale\n snippet,\n metadata: {\n strategy: \"mixedbread\",\n chunk_index: chunk.chunk_index,\n file_id: chunk.file_id,\n filename: chunk.filename,\n mixedbread_score: chunk.score,\n has_image: !!chunk.image_url,\n ...(enableTiming && { timings }),\n },\n };\n });\n if (enableTiming) timings.transform = performance.now() - transformStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Timing breakdown:`);\n console.log(` Mixedbread search: ${timings.mixedbreadSearch?.toFixed(2)}ms (managed API)`);\n console.log(` Transform: ${timings.transform?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n return results;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"Mixedbread search failed:\", errorMessage);\n return [];\n }\n },\n};\n\n",[],[226,470],"7f93bcdc-b9ca-11f0-8edf-0224a6c84d84","mixedbread-embeddings-cosine.ts","7f93bde0-b9ca-11f0-8edf-0224a6c84d84",63,["D",1762293982319],"// Mixedbread Embeddings + Cosine Similarity Strategy\n// Uses Mixedbread's Embeddings API for vector generation with local cosine similarity\n// Model: mxbai-embed-large-v1 (state-of-the-art English embeddings)\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { generateSnippet, cosineSimilarity } from \"./utils.ts\";\n\n// Lazy-loaded Mixedbread client\nlet mxbaiClient: any = null;\nlet clientLoading: Promise | null = null;\n\nasync function getMixedbreadClient() {\n // Return cached client if already loaded\n if (mxbaiClient) {\n return mxbaiClient;\n }\n \n // If loading in progress, wait for it\n if (clientLoading) {\n return clientLoading;\n }\n \n // Start loading client\n clientLoading = (async () => {\n try {\n // Import Mixedbread SDK\n const { Mixedbread } = await import(\"npm:@mixedbread/sdk@latest\");\n \n const MIXEDBREAD_API_KEY = Deno.env.get(\"MIXEDBREAD_API_KEY\") || Deno.env.get(\"MXBAI_API_KEY\");\n \n if (!MIXEDBREAD_API_KEY) {\n throw new Error(\"MIXEDBREAD_API_KEY or MXBAI_API_KEY not found - embeddings disabled\");\n }\n \n mxbaiClient = new Mixedbread({ apiKey: MIXEDBREAD_API_KEY });\n return mxbaiClient;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n throw new Error(`Failed to load Mixedbread client: ${errorMessage}`);\n } finally {\n clientLoading = null;\n }\n })();\n \n return clientLoading;\n}\n\n// Generate embeddings using Mixedbread Embeddings API\nexport const generateEmbeddings = async (content: string): Promise => {\n try {\n const client = await getMixedbreadClient();\n \n // Generate embeddings using the SDK's embeddings.create() method\n const response = await client.embeddings.create({\n model: \"mixedbread-ai/mxbai-embed-large-v1\",\n input: content,\n normalized: true, // Normalize for cosine similarity\n encoding_format: \"float\", // Use float for compatibility\n truncation_strategy: \"end\", // Truncate long content at the end\n });\n \n // Extract the embedding vector from the response\n // Response format: { data: [{ embedding: [...] }] }\n if (response.data && Array.isArray(response.data) && response.data.length > 0) {\n const embedding = response.data[0].embedding;\n if (Array.isArray(embedding)) {\n return embedding;\n }\n }\n \n console.warn(\"Unexpected Mixedbread Embeddings API response format:\", JSON.stringify(response, null, 2));\n return null;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"Mixedbread embeddings failed:\", errorMessage);\n if (error instanceof Error && error.stack) {\n console.error(\"Stack trace:\", error.stack);\n }\n return null;\n }\n};\n\nexport const searchStrategy: SearchStrategy = {\n name: \"mixedbread-embeddings-cosine\",\n description: \"Semantic search using Mixedbread Embeddings API (mxbai-embed-large-v1) with local cosine similarity\",\n search: async (query: string, pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const minSimilarity = (options.minScore || 0) / 100; // Convert to 0-1 scale\n const enableTiming = options.enableTiming || false;\n \n const timings: Record = {};\n const startTime = performance.now();\n \n // Generate query embedding (Mixedbread API call)\n const queryEmbedStart = performance.now();\n const queryEmbedding = await generateEmbeddings(query);\n if (enableTiming) {\n timings.queryEmbedding = performance.now() - queryEmbedStart;\n }\n \n if (!queryEmbedding) {\n console.warn(\"Query embedding generation failed, returning empty results\");\n return [];\n }\n \n // Calculate cosine similarity for each page\n const similarityStart = performance.now();\n const scoredPages: Array<{ page: Page; similarity: number }> = [];\n \n for (const page of pages) {\n const pageEmbedding = page.embeddings;\n if (!pageEmbedding || pageEmbedding.length !== queryEmbedding.length) {\n continue; // Skip pages without embeddings or wrong dimension\n }\n \n const similarity = cosineSimilarity(queryEmbedding, pageEmbedding);\n if (similarity >= minSimilarity) {\n scoredPages.push({ page, similarity });\n }\n }\n \n if (enableTiming) timings.similarity = performance.now() - similarityStart;\n \n // Sort by similarity (descending) and limit\n const sortStart = performance.now();\n scoredPages.sort((a, b) => b.similarity - a.similarity);\n const topPages = scoredPages.slice(0, limit);\n if (enableTiming) timings.sort = performance.now() - sortStart;\n \n // Transform to SearchResult format\n const transformStart = performance.now();\n const queryWords = query.toLowerCase().split(/\\s+/).filter(w => w.length > 0);\n const results: SearchResult[] = topPages.map(({ page, similarity }) => ({\n path: page.path,\n url: page.url,\n title: page.title || page.path,\n score: similarity * 100, // Convert to 0-100 scale\n snippet: generateSnippet(page.content, queryWords, query.toLowerCase()),\n metadata: {\n strategy: \"mixedbread-embeddings-cosine\",\n similarity,\n embeddingDimension: queryEmbedding.length,\n model: \"mxbai-embed-large-v1\",\n ...(enableTiming && { timings }),\n },\n }));\n if (enableTiming) timings.transform = performance.now() - transformStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Timing breakdown:`);\n console.log(` Query embedding: ${timings.queryEmbedding?.toFixed(2)}ms (Mixedbread API)`);\n console.log(` Cosine similarity: ${timings.similarity?.toFixed(2)}ms`);\n console.log(` Sort: ${timings.sort?.toFixed(2)}ms`);\n console.log(` Transform: ${timings.transform?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n return results;\n },\n};\n\n",[],[226,478],"315f55ae-b901-11f0-8f9e-0224a6c84d84","jigsawstack-orama.ts","315f56bc-b901-11f0-8f9e-0224a6c84d84",33,["D",1762207522117],"// JigsawStack + Orama Strategy: Semantic search using JigsawStack embeddings and Orama\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { generateSnippet } from \"./utils.ts\";\nimport { create, insertMultiple, search } from \"npm:@orama/orama@latest\";\n\n// JigsawStack embeddings function (v2 API)\nexport const generateEmbeddings = async (content: string): Promise => {\n const JIGSAWSTACK_API_KEY = Deno.env.get(\"JIGSAWSTACK_API_KEY\");\n const JIGSAWSTACK_API_URL = \"https://api.jigsawstack.com/v2/embedding\";\n \n if (!JIGSAWSTACK_API_KEY) {\n console.warn(\"JIGSAWSTACK_API_KEY not found - embeddings disabled\");\n return null;\n }\n \n try {\n const response = await fetch(JIGSAWSTACK_API_URL, {\n method: \"POST\",\n headers: {\n \"X-Api-Key\": JIGSAWSTACK_API_KEY,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n text: content,\n type: \"text\",\n }),\n });\n \n if (!response.ok) {\n const errorText = await response.text().catch(() => response.statusText);\n throw new Error(`JigsawStack API error: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);\n }\n \n const data = await response.json();\n \n // JigsawStack v2 returns: { success: true, embeddings: [[...]], chunks: [...], _usage: {...} }\n // embeddings is an array of arrays, we want the first embedding vector\n if (data.success && data.embeddings && Array.isArray(data.embeddings) && data.embeddings.length > 0) {\n return data.embeddings[0]; // Return the first embedding vector\n }\n \n return null;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"JigsawStack embeddings failed:\", errorMessage);\n return null;\n }\n};\n\nexport const searchStrategy: SearchStrategy = {\n name: \"jigsawstack-orama\",\n description: \"Semantic search using JigsawStack embeddings with Orama vector search\",\n search: async (query: string, pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const minSimilarity = (options.minScore || 0) / 100; // Convert to 0-1 scale\n const enableTiming = options.enableTiming || false;\n \n const timings: Record = {};\n const startTime = performance.now();\n \n // Generate query embedding (this is the main bottleneck - external API call)\n const queryEmbedStart = performance.now();\n const queryEmbedding = await generateEmbeddings(query);\n if (enableTiming) timings.queryEmbedding = performance.now() - queryEmbedStart;\n \n if (!queryEmbedding) {\n console.warn(\"Query embedding generation failed, returning empty results\");\n return [];\n }\n \n // Determine embedding dimension from query (or first page with embeddings)\n const embeddingDimension = queryEmbedding.length;\n if (embeddingDimension === 0) {\n return [];\n }\n \n // Create Orama database with vector field\n const dbCreateStart = performance.now();\n const db = await create({\n schema: {\n path: \"string\",\n url: \"string\",\n title: \"string\",\n content: \"string\",\n embedding: `vector[${embeddingDimension}]`,\n },\n });\n if (enableTiming) timings.dbCreate = performance.now() - dbCreateStart;\n \n // Prepare documents with embeddings for insertion\n const documents: Array<{\n path: string;\n url: string;\n title: string;\n content: string;\n embedding: number[];\n }> = [];\n \n // Collect pages with embeddings\n const docPrepStart = performance.now();\n for (const page of pages) {\n // Use cached embeddings if available, otherwise generate\n let pageEmbedding = page.embeddings;\n if (!pageEmbedding) {\n // Generate embedding if not cached (should be cached from recalculate, but handle missing)\n pageEmbedding = await generateEmbeddings(page.content);\n if (!pageEmbedding || pageEmbedding.length !== embeddingDimension) {\n continue; // Skip pages without embeddings or wrong dimension\n }\n }\n \n // Ensure embedding dimension matches\n if (pageEmbedding.length !== embeddingDimension) {\n continue;\n }\n \n documents.push({\n path: page.path,\n url: page.url,\n title: page.title || page.path,\n content: page.content,\n embedding: pageEmbedding,\n });\n }\n if (enableTiming) timings.docPrep = performance.now() - docPrepStart;\n \n // Insert all documents into Orama\n const insertStart = performance.now();\n if (documents.length > 0) {\n await insertMultiple(db, documents);\n } else {\n return []; // No documents to search\n }\n if (enableTiming) timings.insert = performance.now() - insertStart;\n \n // Perform vector similarity search using Orama\n const searchStart = performance.now();\n const searchResults = await search(db, {\n mode: \"vector\",\n vector: {\n value: queryEmbedding,\n property: \"embedding\",\n },\n similarity: minSimilarity,\n limit,\n includeVectors: false, // Don't need vectors in response\n });\n if (enableTiming) timings.search = performance.now() - searchStart;\n \n // Transform Orama results to SearchResult format\n const transformStart = performance.now();\n const results: SearchResult[] = searchResults.hits.map((hit: any) => {\n const page = pages.find(p => p.path === hit.document.path);\n const queryWords = query.toLowerCase().split(/\\s+/).filter(w => w.length > 0);\n \n return {\n path: hit.document.path,\n url: hit.document.url,\n title: hit.document.title,\n score: (hit.score || 0) * 100, // Convert similarity to score (0-100)\n snippet: page ? generateSnippet(page.content, queryWords, query.toLowerCase()) : hit.document.content.substring(0, 200) + '...',\n metadata: {\n strategy: \"jigsawstack-orama\",\n similarity: hit.score,\n ...(enableTiming && { timings }),\n },\n };\n });\n if (enableTiming) timings.transform = performance.now() - transformStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Timing breakdown:`);\n console.log(` Query embedding: ${timings.queryEmbedding?.toFixed(2)}ms (API call - main bottleneck)`);\n console.log(` DB create: ${timings.dbCreate?.toFixed(2)}ms`);\n console.log(` Doc prep: ${timings.docPrep?.toFixed(2)}ms`);\n console.log(` Insert: ${timings.insert?.toFixed(2)}ms`);\n console.log(` Search: ${timings.search?.toFixed(2)}ms`);\n console.log(` Transform: ${timings.transform?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n return results;\n },\n};\n\n",[],[226,486],"31712298-b901-11f0-878c-0224a6c84d84","index.ts","3171232e-b901-11f0-878c-0224a6c84d84",35,["D",1762207522385],"// Search - Main entry point\n// To switch strategies, comment/uncomment the imports and function calls below\n\nexport * from \"./types.ts\";\nexport * from \"./utils.ts\";\n\nimport type { Page, SearchOptions, SearchResult } from \"./types.ts\";\n// import { searchStrategy, generateEmbeddings } from \"./placeholder.ts\";\n// import { searchStrategy, generateEmbeddings } from \"./jigsawstack-orama.ts\"; // ~550ms query embeddings (API)\n// import { searchStrategy, generateEmbeddings } from \"./openai-orama.ts\"; // ~100-200ms query embeddings (API), ~29ms DB overhead\n// import { searchStrategy, generateEmbeddings } from \"./transformers-cosine.ts\"; // ~10-30ms (browser only), requires --allow-ffi in Deno\nimport { searchStrategy, generateEmbeddings } from \"./openai-cosine.ts\"; // ~100-200ms query embeddings (API), <1ms cosine calc\n\n// Main search function - comment/uncomment imports above to switch strategies\nexport const searchPages = async (\n query: string,\n pages: Page[],\n options: SearchOptions = {}\n): Promise => {\n return searchStrategy.search(query, pages, options);\n};\n\n// Export search strategy for introspection (e.g., /search/test endpoint)\nexport { searchStrategy };\n\n// Embeddings function for recalculate - uses same import as search strategy\nexport const getActiveEmbeddingsFunction = (): ((content: string) => Promise) => {\n return generateEmbeddings;\n};\n\n\n",[],[226,494],"8065d488-b9ca-11f0-b636-0224a6c84d84","hf-inference-qwen3-cosine.ts","8065d672-b9ca-11f0-b636-0224a6c84d84",88,["D",1762293983675],"// Hugging Face Inference Client + Qwen3-Embedding-8B + Cosine Similarity Strategy\n// Uses @huggingface/inference SDK for embeddings via API\n// Model: Qwen/Qwen3-Embedding-8B (8B params, up to 4096 dimensions, 32k context)\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { generateSnippet, cosineSimilarity } from \"./utils.ts\";\n\n// Lazy-loaded InferenceClient (cached after first load)\nlet inferenceClient: any = null;\nlet clientLoading: Promise | null = null;\n\nasync function getInferenceClient() {\n // Return cached client if already loaded\n if (inferenceClient) {\n return inferenceClient;\n }\n \n // If loading in progress, wait for it\n if (clientLoading) {\n return clientLoading;\n }\n \n // Start loading client\n clientLoading = (async () => {\n try {\n // Import @huggingface/inference - use npm: prefix for Deno compatibility\n const { InferenceClient } = await import(\"npm:@huggingface/inference@latest\");\n \n const HF_TOKEN = Deno.env.get(\"HF_TOKEN\") || Deno.env.get(\"HF_API_KEY\") || Deno.env.get(\"HUGGINGFACE_API_KEY\");\n \n if (!HF_TOKEN) {\n throw new Error(\"HF_TOKEN, HF_API_KEY, or HUGGINGFACE_API_KEY not found - embeddings disabled\");\n }\n \n inferenceClient = new InferenceClient(HF_TOKEN);\n return inferenceClient;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n throw new Error(`Failed to load Hugging Face Inference client: ${errorMessage}`);\n } finally {\n clientLoading = null;\n }\n })();\n \n return clientLoading;\n}\n\n// Generate embeddings using Hugging Face Inference Client\nexport const generateEmbeddings = async (content: string): Promise => {\n try {\n const client = await getInferenceClient();\n \n // Use featureExtraction endpoint with Qwen3-Embedding-8B model\n const output = await client.featureExtraction({\n model: \"Qwen/Qwen3-Embedding-8B\",\n inputs: content,\n provider: \"auto\", // Let HF choose the best provider\n });\n \n // Extract the embedding vector from the response\n // The API typically returns a nested array or flat array\n if (Array.isArray(output)) {\n // Handle nested array format [[...embeddings...]]\n if (Array.isArray(output[0])) {\n return output[0]; // Return first (and typically only) embedding vector\n }\n // Handle flat array format [...embeddings...]\n return output;\n }\n \n // Handle object format with embeddings property\n if (output && typeof output === 'object') {\n if (output.embeddings && Array.isArray(output.embeddings)) {\n return Array.isArray(output.embeddings[0]) ? output.embeddings[0] : output.embeddings;\n }\n \n if (output.data && Array.isArray(output.data)) {\n return Array.isArray(output.data[0]) ? output.data[0] : output.data;\n }\n \n // Try to convert to array if it's a typed array or similar\n if (output.length !== undefined) {\n return Array.from(output as any);\n }\n }\n \n console.warn(\"Unexpected HF Inference API response format:\", typeof output);\n return null;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"HF Inference Client embeddings failed:\", errorMessage);\n return null;\n }\n};\n\nexport const searchStrategy: SearchStrategy = {\n name: \"hf-inference-qwen3-cosine\",\n description: \"Semantic search using Hugging Face Inference Client (Qwen3-Embedding-8B) with cosine similarity\",\n search: async (query: string, pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const minSimilarity = (options.minScore || 0) / 100; // Convert to 0-1 scale\n const enableTiming = options.enableTiming || false;\n \n const timings: Record = {};\n const startTime = performance.now();\n \n // Generate query embedding (HF Inference API call)\n const queryEmbedStart = performance.now();\n const queryEmbedding = await generateEmbeddings(query);\n if (enableTiming) {\n timings.queryEmbedding = performance.now() - queryEmbedStart;\n }\n \n if (!queryEmbedding) {\n console.warn(\"Query embedding generation failed, returning empty results\");\n return [];\n }\n \n // Calculate cosine similarity for each page\n const similarityStart = performance.now();\n const scoredPages: Array<{ page: Page; similarity: number }> = [];\n \n for (const page of pages) {\n const pageEmbedding = page.embeddings;\n if (!pageEmbedding || pageEmbedding.length !== queryEmbedding.length) {\n continue; // Skip pages without embeddings or wrong dimension\n }\n \n const similarity = cosineSimilarity(queryEmbedding, pageEmbedding);\n if (similarity >= minSimilarity) {\n scoredPages.push({ page, similarity });\n }\n }\n \n if (enableTiming) timings.similarity = performance.now() - similarityStart;\n \n // Sort by similarity (descending) and limit\n const sortStart = performance.now();\n scoredPages.sort((a, b) => b.similarity - a.similarity);\n const topPages = scoredPages.slice(0, limit);\n if (enableTiming) timings.sort = performance.now() - sortStart;\n \n // Transform to SearchResult format\n const transformStart = performance.now();\n const queryWords = query.toLowerCase().split(/\\s+/).filter(w => w.length > 0);\n const results: SearchResult[] = topPages.map(({ page, similarity }) => ({\n path: page.path,\n url: page.url,\n title: page.title || page.path,\n score: similarity * 100, // Convert to 0-100 scale\n snippet: generateSnippet(page.content, queryWords, query.toLowerCase()),\n metadata: {\n strategy: \"hf-inference-qwen3-cosine\",\n similarity,\n embeddingDimension: queryEmbedding.length,\n ...(enableTiming && { timings }),\n },\n }));\n if (enableTiming) timings.transform = performance.now() - transformStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Timing breakdown:`);\n console.log(` Query embedding: ${timings.queryEmbedding?.toFixed(2)}ms (HF Inference API)`);\n console.log(` Cosine similarity: ${timings.similarity?.toFixed(2)}ms`);\n console.log(` Sort: ${timings.sort?.toFixed(2)}ms`);\n console.log(` Transform: ${timings.transform?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n return results;\n },\n};\n",[],[226,502],"8050b04e-b9ca-11f0-b636-0224a6c84d84","cloudflare-bge-cosine.ts","8050b1ca-b9ca-11f0-b636-0224a6c84d84",86,["D",1762293983492],"// Cloudflare Workers AI BGE-Large Strategy\n// Uses Cloudflare's @cf/baai/bge-large-en-v1.5 model for embeddings with local cosine similarity\n\nimport type { SearchStrategy, SearchResult, Page, SearchOptions } from \"./types.ts\";\nimport { cosineSimilarity, generateSnippet } from \"./utils.ts\";\n\nconst CLOUDFLARE_MODEL = \"@cf/baai/bge-large-en-v1.5\";\n\n// Cloudflare API configuration\nlet CLOUDFLARE_ACCOUNT_ID: string | undefined;\nlet CLOUDFLARE_API_TOKEN: string | undefined;\n\nfunction getCloudflareConfig() {\n if (!CLOUDFLARE_ACCOUNT_ID) {\n CLOUDFLARE_ACCOUNT_ID = Deno.env.get(\"CLOUDFLARE_ACCOUNT_ID\");\n }\n if (!CLOUDFLARE_API_TOKEN) {\n CLOUDFLARE_API_TOKEN = Deno.env.get(\"CLOUDFLARE_API_TOKEN\");\n }\n \n if (!CLOUDFLARE_ACCOUNT_ID || !CLOUDFLARE_API_TOKEN) {\n throw new Error(\"CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN environment variables are required\");\n }\n \n return { accountId: CLOUDFLARE_ACCOUNT_ID, apiToken: CLOUDFLARE_API_TOKEN };\n}\n\nexport const generateEmbeddings = async (content: string): Promise => {\n try {\n const { accountId, apiToken } = getCloudflareConfig();\n \n const url = `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/run/${CLOUDFLARE_MODEL}`;\n \n const response = await fetch(url, {\n method: \"POST\",\n headers: {\n \"Authorization\": `Bearer ${apiToken}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n text: [content], // Cloudflare expects an array\n }),\n });\n \n if (!response.ok) {\n const errorText = await response.text();\n console.error(`Cloudflare API error: ${response.status} ${response.statusText} - ${errorText}`);\n return null;\n }\n \n const responseData = await response.json();\n \n // Cloudflare wraps the response in: { result: { shape: [1, 1024], data: [[...embeddings...]] }, success: true, errors: [], messages: [] }\n if (responseData.result && responseData.result.data && Array.isArray(responseData.result.data) && responseData.result.data.length > 0) {\n const embedding = responseData.result.data[0];\n if (Array.isArray(embedding) && embedding.length > 0) {\n return embedding;\n }\n }\n \n console.warn(\"Unexpected Cloudflare API response format:\", JSON.stringify(responseData, null, 2));\n return null;\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n console.error(\"Cloudflare embeddings failed:\", errorMessage);\n if (error instanceof Error && error.stack) {\n console.error(\"Stack trace:\", error.stack);\n }\n return null;\n }\n};\n\nexport const searchStrategy: SearchStrategy = {\n name: \"cloudflare-bge-cosine\",\n description: \"Semantic search using Cloudflare Workers AI (@cf/baai/bge-large-en-v1.5) with local cosine similarity\",\n search: async (query: string, pages: Page[], options: SearchOptions = {}): Promise => {\n const limit = options.limit || 10;\n const minSimilarity = (options.minScore || 0) / 100;\n const enableTiming = options.enableTiming || false;\n \n const timings: Record = {};\n const startTime = performance.now();\n \n // Generate query embedding\n const queryEmbedStart = performance.now();\n const queryEmbedding = await generateEmbeddings(query);\n if (enableTiming) {\n timings.queryEmbedding = performance.now() - queryEmbedStart;\n }\n \n if (!queryEmbedding) {\n console.warn(\"Query embedding generation failed, returning empty results\");\n return [];\n }\n \n // Calculate similarity scores\n const similarityStart = performance.now();\n const scoredPages: Array<{ page: Page; similarity: number }> = [];\n \n for (const page of pages) {\n const pageEmbedding = page.embeddings;\n if (!pageEmbedding || pageEmbedding.length !== queryEmbedding.length) {\n continue;\n }\n \n const similarity = cosineSimilarity(queryEmbedding, pageEmbedding);\n if (similarity >= minSimilarity) {\n scoredPages.push({ page, similarity });\n }\n }\n \n if (enableTiming) timings.similarity = performance.now() - similarityStart;\n \n // Sort and limit results\n const sortStart = performance.now();\n scoredPages.sort((a, b) => b.similarity - a.similarity);\n const topPages = scoredPages.slice(0, limit);\n if (enableTiming) timings.sort = performance.now() - sortStart;\n \n // Transform to SearchResult format\n const transformStart = performance.now();\n const queryWords = query.toLowerCase().split(/\\s+/).filter(w => w.length > 0);\n const results: SearchResult[] = topPages.map(({ page, similarity }) => ({\n path: page.path,\n url: page.url,\n title: page.title || page.path,\n score: similarity * 100,\n snippet: generateSnippet(page.content, queryWords, query.toLowerCase()),\n metadata: {\n strategy: \"cloudflare-bge-cosine\",\n similarity,\n embeddingDimension: queryEmbedding.length,\n model: CLOUDFLARE_MODEL,\n ...(enableTiming && { timings }),\n },\n }));\n if (enableTiming) timings.transform = performance.now() - transformStart;\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Timing breakdown:`);\n console.log(` Query embedding: ${timings.queryEmbedding?.toFixed(2)}ms (Cloudflare Workers AI)`);\n console.log(` Cosine similarity: ${timings.similarity?.toFixed(2)}ms`);\n console.log(` Sort: ${timings.sort?.toFixed(2)}ms`);\n console.log(` Transform: ${timings.transform?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n return results;\n },\n};\n\n",[],[226,510],"7f996d58-b9ca-11f0-83d0-0224a6c84d84","STRATEGY-COMPARISON.md","7f996e0c-b9ca-11f0-83d0-0224a6c84d84",64,["D",1762293982370],"# Search Strategy Comparison\n\nThis document helps you choose the best search strategy for your use case.\n\n## Quick Decision Tree\n\n```\n┌─ Need to run on Cloudflare Workers?\n│ └─ YES → Use cloudflare-bge-cosine.ts\n│\n├─ Need 100% offline, no network calls?\n│ └─ YES → Use transformers-local-onnx.ts (after downloading model)\n│\n├─ Want fastest setup with good performance?\n│ └─ YES → Use transformers-cosine.ts (auto-downloads on first run)\n│\n├─ Need the best accuracy?\n│ └─ YES → Use mixedbread-embeddings-cosine.ts or openai-cosine.ts\n│\n└─ Want managed search (no embeddings management)?\n └─ YES → Use jigsawstack-orama.ts or mixedbread.ts\n```\n\n## Detailed Comparison\n\n### Performance Metrics\n\n| Strategy | First Load | Cached Load | Query Time | Total (cached) | Network Required |\n|----------|-----------|-------------|------------|----------------|------------------|\n| **transformers-local-onnx** | ~150ms | ~50ms | ~10-30ms | **~60-80ms** | ❌ No |\n| **transformers-cosine** | ~3-5s | ~150ms | ~10-30ms | ~160-180ms | ✅ First run only |\n| **mixedbread-embeddings** | N/A | N/A | ~50-100ms | ~50-100ms | ✅ Every query |\n| **openai-cosine** | N/A | N/A | ~100-200ms | ~100-200ms | ✅ Every query |\n| **hf-inference-qwen3** | N/A | N/A | ~150-300ms | ~150-300ms | ✅ Every query |\n| **cloudflare-bge** | N/A | N/A | ~50-150ms | ~50-150ms | ✅ Every query |\n| **jigsawstack-orama** | N/A | N/A | ~550ms | ~550ms | ✅ Every query |\n\n### Cost Comparison\n\n| Strategy | Cost | Free Tier | Notes |\n|----------|------|-----------|-------|\n| **transformers-local-onnx** | $0 | ∞ | 100% free, runs locally |\n| **transformers-cosine** | $0 | ∞ | 100% free, runs locally |\n| **mixedbread-embeddings** | $0-$ | Generous | Free tier: 150 req/min, 100M tokens/mo |\n| **openai-cosine** | $$ | Limited | $0.0001/1K tokens (text-embedding-3-small) |\n| **hf-inference-qwen3** | $0 | Generous | Free tier: 1000 req/day |\n| **cloudflare-bge** | $0 | Generous | Free tier: 10,000 req/day |\n| **jigsawstack-orama** | $0-$ | Limited | Free tier: limited requests |\n\n### Accuracy Comparison\n\nBased on MTEB (Massive Text Embedding Benchmark) scores:\n\n| Strategy | Model | Dimensions | MTEB Score | Notes |\n|----------|-------|------------|------------|-------|\n| **transformers-local-onnx** | all-MiniLM-L6-v2 | 384 | ~58 | Fast, good quality |\n| **transformers-cosine** | all-MiniLM-L6-v2 | 384 | ~58 | Same as local |\n| **mixedbread-embeddings** | mxbai-embed-large-v1 | 1024 | ~64 | Higher quality |\n| **openai-cosine** | text-embedding-3-small | 1536 | ~62 | Reliable, tested |\n| **hf-inference-qwen3** | Qwen3-Embedding-8B | 768 | ~65 | Very high quality |\n| **cloudflare-bge** | bge-large-en-v1.5 | 1024 | ~64 | Good quality |\n\n## Use Case Recommendations\n\n### 🏆 Production Deployment (Recommended)\n\n**Use**: `transformers-local-onnx.ts`\n\n**Why**:\n- Predictable performance (no network variance)\n- No API costs\n- No rate limits\n- Works offline\n- Fast after initial load\n\n**Best for**:\n- Deno Deploy\n- Render / Railway / Fly.io\n- Docker containers\n- Any environment with file system access\n\n### 🚀 Quick Start / Prototyping\n\n**Use**: `transformers-cosine.ts`\n\n**Why**:\n- Zero setup (auto-downloads model)\n- No API keys needed\n- Good performance after first run\n- Easy to switch to local later\n\n**Best for**:\n- Development\n- Testing\n- Quick demos\n- When you don't want to download models manually\n\n### ☁️ Cloudflare Workers\n\n**Use**: `cloudflare-bge-cosine.ts`\n\n**Why**:\n- Workers have 1MB code size limit (can't fit local models)\n- Cloudflare AI is optimized for Workers\n- Free tier is generous\n- Low latency\n\n**Best for**:\n- Cloudflare Workers/Pages\n- Edge deployment\n- Global low-latency requirements\n\n### 🎯 Maximum Accuracy\n\n**Use**: `hf-inference-qwen3-cosine.ts` or `mixedbread-embeddings-cosine.ts`\n\n**Why**:\n- Higher MTEB scores\n- Better semantic understanding\n- More dimensions (768-1024 vs 384)\n\n**Best for**:\n- When accuracy matters more than speed\n- Complex semantic queries\n- Production with budget for API calls\n\n### 📦 Managed Solution\n\n**Use**: `mixedbread.ts` or `jigsawstack-orama.ts`\n\n**Why**:\n- No embedding management needed\n- Handles storage, search, and embeddings\n- Less code to maintain\n\n**Best for**:\n- When you want a managed solution\n- Don't want to store embeddings yourself\n- Prefer APIs over local computation\n\n## Migration Path\n\n### From API-based to Local\n\n1. Download the model:\n ```bash\n cd search/models\n ./download-model.sh\n ```\n\n2. Update `search/index.ts`:\n ```typescript\n // Before\n // import { searchStrategy, generateEmbeddings } from \"./openai-cosine.ts\";\n \n // After\n import { searchStrategy, generateEmbeddings } from \"./transformers-local-onnx.ts\";\n ```\n\n3. Recalculate embeddings (if dimensions differ):\n ```bash\n GET /cache/recalculate?force=true\n ```\n\n### From transformers-cosine to transformers-local-onnx\n\n1. Download the model (same as above)\n2. Update the import\n3. No recalculation needed (same model, same dimensions)\n\n## Environment-Specific Notes\n\n### Val.town\n- **Recommended**: `transformers-cosine.ts` (isolate has caching)\n- **Alternative**: Any API-based strategy\n- **Avoid**: `transformers-local-onnx.ts` (no persistent file system)\n\n### Deno Deploy\n- **Recommended**: `transformers-local-onnx.ts` (with deployment package)\n- **Alternative**: `transformers-cosine.ts` or API-based\n- **Note**: Include model files in deployment\n\n### Docker\n- **Recommended**: `transformers-local-onnx.ts`\n- **Note**: Include model files in image\n- **Example**:\n ```dockerfile\n COPY search/models/all-MiniLM-L6-v2 /app/search/models/all-MiniLM-L6-v2\n ```\n\n### Serverless (AWS Lambda, Google Cloud Functions)\n- **Recommended**: API-based strategies (less cold start time)\n- **Alternative**: `transformers-local-onnx.ts` (if you can handle cold starts)\n- **Note**: Local models increase deployment package size\n\n### Local Development\n- **Recommended**: `transformers-cosine.ts` (auto-download)\n- **Alternative**: `transformers-local-onnx.ts` (if downloaded)\n- **Note**: Both work great for development\n\n## Benchmarking Your Setup\n\nRun the test harness to compare strategies on your infrastructure:\n\n```bash\ncd testing\ndeno run --allow-read --allow-env --allow-net test-search.ts\n```\n\nThis will show actual performance numbers for your specific setup.\n\n## Summary Table\n\n| Criteria | Best Choice |\n|----------|-------------|\n| **Fastest** | transformers-local-onnx |\n| **Easiest Setup** | transformers-cosine |\n| **Most Accurate** | hf-inference-qwen3 |\n| **Cheapest** | transformers-local-onnx (free) |\n| **Best for Production** | transformers-local-onnx |\n| **Best for Cloudflare** | cloudflare-bge-cosine |\n| **Best for Val.town** | transformers-cosine |\n| **Most Reliable** | openai-cosine |\n| **Fully Managed** | mixedbread or jigsawstack |\n\n## Still Unsure?\n\nDefault recommendation: **transformers-local-onnx.ts**\n\nIt offers the best combination of speed, cost (free), and reliability for most production use cases. The only downside is the initial setup (downloading model files), which takes a few minutes.\n\nIf you can't download models or need to deploy immediately, use **transformers-cosine.ts** as it auto-downloads on first run.\n\n",[],[226,518],"8039999a-b9ca-11f0-9362-0224a6c84d84","80399a58-b9ca-11f0-9362-0224a6c84d84",83,["D",1762293983419],"# Search Module\n\nPluggable semantic search system with multiple embedding strategies.\n\n## Quick Start\n\n### 1. Choose Your Strategy\n\nThe fastest option for production is **local ONNX models**:\n\n```bash\n# Download the model (one-time setup, ~90MB)\ncd models\n./download-model.sh\n```\n\n### 2. Activate the Strategy\n\nEdit `search/index.ts` and uncomment the desired strategy:\n\n```typescript\nimport { searchStrategy, generateEmbeddings } from \"./transformers-local-onnx.ts\";\n```\n\n### 3. Use the Search\n\n```typescript\nimport { searchPages } from \"./search/index.ts\";\n\nconst results = await searchPages(\"How to use Groq API?\", pages, {\n limit: 10,\n minScore: 50,\n enableTiming: true,\n});\n```\n\n## Available Strategies\n\n| Strategy | Speed | Cost | Setup | Best For |\n|----------|-------|------|-------|----------|\n| **transformers-local-onnx** ⭐ | ~60-80ms | Free | Download model | Production |\n| **transformers-cosine** | ~160-180ms | Free | None (auto-download) | Development |\n| **mixedbread-embeddings** | ~50-100ms | Free tier | API key | High accuracy |\n| **openai-cosine** | ~100-200ms | Paid | API key | Reliability |\n| **hf-inference-qwen3** | ~150-300ms | Free tier | API key | Best accuracy |\n| **cloudflare-bge** | ~50-150ms | Free tier | API key | Cloudflare Workers |\n| **jigsawstack-orama** | ~550ms | Free tier | API key | Managed solution |\n\n⭐ = Recommended for production\n\n## Files\n\n- **`index.ts`** - Main entry point, switch strategies here\n- **`types.ts`** - TypeScript interfaces for search system\n- **`utils.ts`** - Shared utilities (cosine similarity, snippet generation)\n- **`transformers-local-onnx.ts`** - Local ONNX models (fastest, recommended)\n- **`transformers-cosine.ts`** - Auto-download ONNX models\n- **`mixedbread-embeddings-cosine.ts`** - Mixedbread API + local cosine\n- **`openai-cosine.ts`** - OpenAI embeddings + local cosine\n- **`hf-inference-qwen3-cosine.ts`** - HuggingFace Qwen3-8B embeddings\n- **`cloudflare-bge-cosine.ts`** - Cloudflare Workers AI\n- **`jigsawstack-orama.ts`** - JigsawStack managed search\n- **`mixedbread.ts`** - Mixedbread Stores (managed)\n- **`placeholder.ts`** - Fake embeddings for testing\n\n## Documentation\n\n- **`models/README.md`** - Model setup instructions\n- **`models/SETUP.md`** - Detailed setup guide\n- **`STRATEGY-COMPARISON.md`** - Detailed comparison of all strategies\n\n## Testing\n\nTest the local ONNX model:\n\n```bash\ncd models\ndeno run --allow-read --allow-env --allow-net test-local-model.ts\n```\n\nRun the full search harness:\n\n```bash\ncd ../testing\ndeno run --allow-read --allow-env --allow-net test-search.ts\n```\n\n## API\n\n### Search Function\n\n```typescript\nasync function searchPages(\n query: string,\n pages: Page[],\n options?: SearchOptions\n): Promise\n```\n\n**Options**:\n- `limit`: Maximum results to return (default: 10)\n- `minScore`: Minimum similarity score 0-100 (default: 0)\n- `enableTiming`: Log timing breakdown (default: false)\n\n**Returns**: Array of search results sorted by relevance\n\n### Generate Embeddings\n\n```typescript\nasync function generateEmbeddings(\n content: string\n): Promise\n```\n\n**Returns**: 384-dimensional embedding vector (or configured dimensions)\n\n## Architecture\n\n```\nQuery\n ↓\nGenerate Query Embedding (10-30ms)\n ↓\nCompare with Page Embeddings (cosine similarity, <1ms per page)\n ↓\nSort by Similarity\n ↓\nGenerate Snippets\n ↓\nReturn Results\n```\n\n## Performance Tips\n\n1. **Use local ONNX models** for production (fastest, most reliable)\n2. **Pre-calculate embeddings** during recalculation (don't generate at query time)\n3. **Cache the pipeline** (automatically done, but worth noting)\n4. **Use quantized models** if memory is constrained (set `USE_QUANTIZED = true`)\n5. **Adjust `minScore`** to filter low-quality results\n\n## Deployment\n\n### Deno Deploy / Render / Railway\n\n✅ Use `transformers-local-onnx.ts`\n- Include model files in deployment\n- Fast, reliable, no network calls\n\n### Cloudflare Workers\n\n✅ Use `cloudflare-bge-cosine.ts`\n- Workers have size limits (can't fit local models)\n- Cloudflare AI is optimized for Workers\n\n### Val.town\n\n✅ Use `transformers-cosine.ts`\n- Isolate has caching for downloaded models\n- No persistent file system for pre-downloaded models\n\n### Docker\n\n✅ Use `transformers-local-onnx.ts`\n- Include models in image:\n ```dockerfile\n COPY search/models/all-MiniLM-L6-v2 /app/search/models/all-MiniLM-L6-v2\n ```\n\n## Troubleshooting\n\n### \"Failed to load local ONNX model\"\n\nDownload the model:\n```bash\ncd search/models\n./download-model.sh\n```\n\n### \"Module not found\"\n\nCheck the import in `search/index.ts`:\n```typescript\nimport { searchStrategy, generateEmbeddings } from \"./transformers-local-onnx.ts\";\n```\n\n### Slow performance\n\n1. Check which strategy is active\n2. Ensure model is cached (first run is slower)\n3. Try quantized model (`USE_QUANTIZED = true`)\n4. Check network latency (for API-based strategies)\n\n### Different embedding dimensions\n\nIf switching strategies with different dimensions:\n```bash\nGET /cache/recalculate?force=true\n```\n\nThis regenerates all embeddings with the new strategy.\n\n## Contributing\n\nTo add a new search strategy:\n\n1. Create `search/my-strategy.ts`\n2. Implement `SearchStrategy` interface:\n ```typescript\n export const searchStrategy: SearchStrategy = {\n name: \"my-strategy\",\n description: \"Description...\",\n search: async (query, pages, options) => {\n // Implementation\n },\n };\n \n export const generateEmbeddings = async (content: string) => {\n // Generate embeddings\n };\n ```\n3. Document in `STRATEGY-COMPARISON.md`\n4. Add to `index.ts` as an option\n\n## License\n\nPart of the groq-docs project.\n\n",[],[226,192],"7fbe24b8-b9ca-11f0-83d0-0224a6c84d84","test-server-mode.ts","7fbe2580-b9ca-11f0-83d0-0224a6c84d84","7eec2a9e-b9ca-11f0-8b68-0224a6c84d84",68,["D",1762293982610],"// Test that verifies the ONNX crash doesn't happen in server mode (long-running process)\n// Run with: deno run --allow-read --allow-env --allow-net --allow-ffi test-server-mode.ts\n\nimport { searchStrategy, generateEmbeddings } from \"../transformers-local-onnx.ts\";\n\nconsole.log(\"🧪 Testing ONNX in Server Mode (long-running process)\\n\");\n\n// Simulate a server that stays running\nconsole.log(\"Starting simulated server...\");\n\nlet requestCount = 0;\n\n// Simulate incoming requests every second\nconst interval = setInterval(async () => {\n requestCount++;\n console.log(`\\n📨 Request #${requestCount}`);\n \n const query = requestCount % 2 === 0 ? \"How to use Groq API?\" : \"What are the pricing options?\";\n console.log(` Query: \"${query}\"`);\n \n const start = performance.now();\n const embedding = await generateEmbeddings(query);\n const elapsed = performance.now() - start;\n \n if (embedding) {\n console.log(` ✅ Generated embedding (${embedding.length} dims) in ${elapsed.toFixed(2)}ms`);\n } else {\n console.log(` ❌ Failed to generate embedding`);\n }\n \n // Stop after 5 requests\n if (requestCount >= 5) {\n console.log(\"\\n✅ Completed 5 requests successfully!\");\n console.log(\" No crashes! Server mode works perfectly.\");\n console.log(\"\\n💡 The mutex crash only affects standalone scripts that exit immediately.\");\n console.log(\" Your main.tsx server will run without any issues.\");\n clearInterval(interval);\n \n // Exit gracefully\n console.log(\"\\nStopping server...\");\n setTimeout(() => {\n console.log(\"Server stopped cleanly!\");\n Deno.exit(0);\n }, 100);\n }\n}, 1000);\n\nconsole.log(\"Server running... (will process 5 requests then stop)\\n\");\n\n",[],[226,541,533],"models","7fb0abf8-b9ca-11f0-a8b3-0224a6c84d84","test-local-model.ts","7fb0acc0-b9ca-11f0-a8b3-0224a6c84d84",67,["D",1762293982506],"// Test script for local ONNX model\n// Run with: deno run --allow-read --allow-env --allow-net test-local-model.ts\n\nimport { searchStrategy, generateEmbeddings } from \"../transformers-local-onnx.ts\";\n\nconsole.log(\"🧪 Testing Local ONNX Model Strategy\\n\");\n\n// Test 1: Generate embeddings for a simple query\nconsole.log(\"Test 1: Generate embeddings for a query\");\nconsole.log(\"Query: 'What is Groq?'\\n\");\n\nconst start = performance.now();\nconst embeddings = await generateEmbeddings(\"What is Groq?\");\nconst elapsed = performance.now() - start;\n\nif (embeddings) {\n console.log(`✅ Generated embeddings successfully!`);\n console.log(` Dimensions: ${embeddings.length}`);\n console.log(` First 5 values: [${embeddings.slice(0, 5).map(v => v.toFixed(4)).join(\", \")}...]`);\n console.log(` Time: ${elapsed.toFixed(2)}ms`);\n} else {\n console.log(`❌ Failed to generate embeddings`);\n}\n\nconsole.log(\"\\n\" + \"=\".repeat(60) + \"\\n\");\n\n// Test 2: Generate embeddings for multiple queries (to test caching)\nconsole.log(\"Test 2: Generate embeddings for multiple queries (testing cache)\");\n\nconst queries = [\n \"How to use Groq API?\",\n \"What models are available?\",\n \"Pricing information\"\n];\n\nfor (const query of queries) {\n const queryStart = performance.now();\n const queryEmbedding = await generateEmbeddings(query);\n const queryElapsed = performance.now() - queryStart;\n \n if (queryEmbedding) {\n console.log(`✅ \"${query}\"`);\n console.log(` Time: ${queryElapsed.toFixed(2)}ms (cached pipeline)`);\n } else {\n console.log(`❌ \"${query}\" - Failed`);\n }\n}\n\nconsole.log(\"\\n\" + \"=\".repeat(60) + \"\\n\");\n\n// Test 3: Test search strategy with mock data\nconsole.log(\"Test 3: Test search strategy with mock data\");\n\nconst mockPages = [\n {\n path: \"/docs/introduction\",\n url: \"https://console.groq.com/docs/introduction\",\n title: \"Introduction to Groq\",\n content: \"Groq is a fast AI inference platform that provides APIs for various language models.\",\n embeddings: await generateEmbeddings(\"Groq is a fast AI inference platform that provides APIs for various language models.\"),\n },\n {\n path: \"/docs/api-keys\",\n url: \"https://console.groq.com/docs/api-keys\",\n title: \"API Keys\",\n content: \"Learn how to create and manage your Groq API keys for authentication.\",\n embeddings: await generateEmbeddings(\"Learn how to create and manage your Groq API keys for authentication.\"),\n },\n {\n path: \"/docs/models\",\n url: \"https://console.groq.com/docs/models\",\n title: \"Available Models\",\n content: \"Groq supports various language models including Llama, Mixtral, and Gemma.\",\n embeddings: await generateEmbeddings(\"Groq supports various language models including Llama, Mixtral, and Gemma.\"),\n },\n];\n\nconst searchQuery = \"How do I get API keys?\";\nconsole.log(`Query: \"${searchQuery}\"\\n`);\n\nconst searchStart = performance.now();\nconst results = await searchStrategy.search(searchQuery, mockPages, {\n limit: 3,\n enableTiming: true,\n});\nconst searchElapsed = performance.now() - searchStart;\n\nconsole.log(`\\nSearch Results (${results.length}):`);\nresults.forEach((result, idx) => {\n console.log(`\\n${idx + 1}. ${result.title}`);\n console.log(` Path: ${result.path}`);\n console.log(` Score: ${result.score.toFixed(2)}`);\n console.log(` Snippet: ${result.snippet}`);\n});\n\nconsole.log(`\\n⏱️ Total search time: ${searchElapsed.toFixed(2)}ms`);\n\nconsole.log(\"\\n\" + \"=\".repeat(60) + \"\\n\");\n\nconsole.log(\"✅ All tests completed!\");\nconsole.log(\"\\n💡 Tips:\");\nconsole.log(\" - First run downloads the model (~90MB)\");\nconsole.log(\" - Subsequent runs use cached model (~50-150ms load)\");\nconsole.log(\" - Local ONNX = No network calls, completely offline\");\nconsole.log(\" - Perfect for serverless deployments (Deno Deploy, Render, Railway)\");\n\n// Note: You may see a \"mutex lock failed\" error on exit\n// This is a known issue with ONNX runtime cleanup in Deno when scripts exit immediately\n// The error is harmless and only affects standalone scripts - long-running servers (like your main app) won't have this issue\n// All functionality works correctly despite the error message\n\n",[],[226,541,543],"802f1326-b9ca-11f0-b384-0224a6c84d84","test-local-model-safe.ts","802f13ee-b9ca-11f0-b384-0224a6c84d84",82,["D",1762293983351],"// Safe wrapper for test-local-model.ts that prevents crash on exit\n// Run with: deno run --allow-read --allow-env --allow-net --allow-ffi --allow-run test-local-model-safe.ts\n\nconst command = new Deno.Command(\"deno\", {\n args: [\n \"run\",\n \"--allow-read\",\n \"--allow-env\", \n \"--allow-net\",\n \"--allow-ffi\",\n \"test-local-model.ts\"\n ],\n stdout: \"inherit\",\n stderr: \"inherit\",\n});\n\nconst { code } = await command.output();\n\n// Exit code 134 is SIGABRT from the mutex cleanup error - treat as success\nif (code === 134 || code === 0) {\n console.log(\"\\n✅ Test completed successfully (exit code normalized)\");\n Deno.exit(0);\n} else {\n console.error(`\\n❌ Test failed with exit code: ${code}`);\n Deno.exit(code);\n}\n\n",[],[226,541,551],"7fc2b366-b9ca-11f0-8b68-0224a6c84d84","download-model.sh","7fc2b3fc-b9ca-11f0-8b68-0224a6c84d84",69,["D",1762293982640],"#!/bin/bash\n# Download all-MiniLM-L6-v2 ONNX model from Hugging Face\n\nset -e # Exit on error\n\nMODEL_NAME=\"all-MiniLM-L6-v2\"\nHF_REPO=\"Xenova/all-MiniLM-L6-v2\"\n\necho \"📦 Downloading ${MODEL_NAME} from Hugging Face...\"\necho \" Repository: https://huggingface.co/${HF_REPO}\"\necho \"\"\n\n# Check if git-lfs is installed\nif ! command -v git-lfs &> /dev/null; then\n echo \"⚠️ Warning: git-lfs is not installed. Large files may not download correctly.\"\n echo \" Install git-lfs first: https://git-lfs.github.com/\"\n echo \"\"\n read -p \"Continue anyway? (y/N) \" -n 1 -r\n echo\n if [[ ! $REPLY =~ ^[Yy]$ ]]; then\n exit 1\n fi\nfi\n\n# Clone the repository\necho \"🔽 Cloning repository (this may take a few minutes)...\"\ngit clone https://huggingface.co/${HF_REPO} ${MODEL_NAME}\n\n# Clean up .git directory to save space\necho \"🧹 Cleaning up .git directory...\"\nrm -rf ${MODEL_NAME}/.git\n\n# Display downloaded files\necho \"\"\necho \"✅ Model downloaded successfully!\"\necho \"\"\necho \"📂 Files in ${MODEL_NAME}:\"\nls -lh ${MODEL_NAME}\necho \"\"\necho \"📊 Total size:\"\ndu -sh ${MODEL_NAME}\necho \"\"\n\n# Check for required files\necho \"🔍 Verifying required files...\"\nREQUIRED_FILES=(\n \"${MODEL_NAME}/onnx/model.onnx\"\n \"${MODEL_NAME}/tokenizer.json\"\n \"${MODEL_NAME}/tokenizer_config.json\"\n \"${MODEL_NAME}/config.json\"\n)\n\nALL_FOUND=true\nfor file in \"${REQUIRED_FILES[@]}\"; do\n if [ -f \"$file\" ]; then\n echo \" ✅ Found: $file\"\n else\n echo \" ❌ Missing: $file\"\n ALL_FOUND=false\n fi\ndone\n\necho \"\"\nif [ \"$ALL_FOUND\" = true ]; then\n echo \"🎉 All required files are present!\"\n echo \"\"\n echo \"Next steps:\"\n echo \"1. Update search/index.ts to use the local ONNX strategy:\"\n echo \" import { searchStrategy, generateEmbeddings } from \\\"./transformers-local-onnx.ts\\\";\"\n echo \"\"\n echo \"2. Run your application - the model will load from local files!\"\nelse\n echo \"⚠️ Some required files are missing. The model may not work correctly.\"\n echo \" Try running this script again or download manually from:\"\n echo \" https://huggingface.co/${HF_REPO}/tree/main\"\nfi\n\n",[],[226,541,559],"7fab0310-b9ca-11f0-8b68-0224a6c84d84","SETUP.md","7fab03b0-b9ca-11f0-8b68-0224a6c84d84",66,["D",1762293982468],"# Local ONNX Model Setup Guide\n\nThis guide will help you set up pre-downloaded ONNX models for the fastest embedding generation with zero network overhead.\n\n## Quick Start\n\n```bash\n# Navigate to the models directory\ncd search/models\n\n# Run the download script\n./download-model.sh\n\n# Wait for download to complete (may take a few minutes depending on connection)\n# The model is ~90MB total\n```\n\n## Manual Download (Alternative)\n\nIf the script doesn't work, you can download manually:\n\n```bash\ncd search/models\n\n# Clone the Hugging Face repository\ngit clone https://huggingface.co/Xenova/all-MiniLM-L6-v2\n\n# Remove .git directory to save space\nrm -rf all-MiniLM-L6-v2/.git\n```\n\nOr download individual files from: https://huggingface.co/Xenova/all-MiniLM-L6-v2/tree/main\n\nRequired files:\n- `onnx/model.onnx` (~23MB) - Main ONNX model\n- `onnx/model_quantized.onnx` (~6MB) - Quantized version (optional, faster)\n- `tokenizer.json` - Tokenizer vocabulary\n- `tokenizer_config.json` - Tokenizer configuration\n- `config.json` - Model configuration\n\n## Verify Installation\n\nCheck that the model files exist:\n\n```bash\nls -lh all-MiniLM-L6-v2/\nls -lh all-MiniLM-L6-v2/onnx/\n```\n\nYou should see:\n```\nall-MiniLM-L6-v2/\n├── onnx/\n│ ├── model.onnx (~23MB)\n│ └── model_quantized.onnx (~6MB)\n├── config.json\n├── tokenizer.json\n├── tokenizer_config.json\n└── ...\n```\n\n## Switch to Local Model Strategy\n\nIn `search/index.ts`, make sure this line is active:\n\n```typescript\nimport { searchStrategy, generateEmbeddings } from \"./transformers-local-onnx.ts\";\n```\n\nComment out other strategies:\n\n```typescript\n// import { searchStrategy, generateEmbeddings } from \"./transformers-cosine.ts\";\n```\n\n## Configuration Options\n\nEdit `search/transformers-local-onnx.ts` to customize:\n\n```typescript\n// Use quantized model (faster, slightly less accurate)\nconst USE_QUANTIZED = true; // Default: false\n\n// Model path (change if you want to use a different model)\nconst MODEL_PATH = new URL(\"./models/all-MiniLM-L6-v2\", import.meta.url).pathname;\n```\n\n## Performance Comparison\n\n| Strategy | First Run | Cached | Network | Model Size |\n|----------|-----------|--------|---------|------------|\n| `transformers-cosine.ts` | ~3-5s | ~150ms | Required (downloads from HF) | ~23MB cached |\n| `transformers-local-onnx.ts` | ~150ms | ~50ms | **None** | ~23MB local |\n\nThe local ONNX version:\n- ✅ No network calls - works completely offline\n- ✅ No downloads on first run - instant startup\n- ✅ No isolate loading delays\n- ✅ Same accuracy as the cached version\n- ✅ Perfect for serverless/edge deployments\n\n## Deployment Notes\n\n### For Val.town\n- Upload the model files to Val.town's blob storage or use a CDN\n- Update `MODEL_PATH` to point to the remote location\n- Or use the regular `transformers-cosine.ts` which downloads to isolate cache\n\n### For Deno Deploy\n- Include the models in your deployment package\n- Models will be read from the file system\n- Works perfectly with the local strategy\n\n### For Cloudflare Workers\n- Workers have a 1MB size limit, so the local strategy won't work\n- Use `cloudflare-bge-cosine.ts` instead (uses Cloudflare AI)\n\n### For Render / Railway / Fly.io\n- Include the models directory in your Docker image\n- Works perfectly with local file system access\n- Fastest option for these platforms\n\n## Troubleshooting\n\n### Error: \"Failed to load local ONNX model\"\n- Ensure model files are downloaded: `ls -lh search/models/all-MiniLM-L6-v2/`\n- Run the download script: `./search/models/download-model.sh`\n- Check file permissions: `chmod -R 755 search/models/all-MiniLM-L6-v2/`\n\n### Error: \"Cannot find module\"\n- Make sure you're using the correct import in `search/index.ts`\n- Check that the file path is correct\n\n### Slow performance\n- Try the quantized model: set `USE_QUANTIZED = true` in `transformers-local-onnx.ts`\n- Check if model files are on a fast disk (SSD recommended)\n\n### Out of memory\n- The model requires ~100-200MB RAM when loaded\n- Use the quantized version to reduce memory usage\n\n## Alternative: Use Different Models\n\nYou can use other ONNX models from Hugging Face. Popular options:\n\n- **all-MiniLM-L6-v2** (default) - 384 dims, 23MB, fast\n- **all-mpnet-base-v2** - 768 dims, 420MB, more accurate\n- **paraphrase-multilingual-MiniLM-L12-v2** - 384 dims, 470MB, multilingual\n\nTo use a different model:\n1. Download it to `search/models/[model-name]`\n2. Update `MODEL_PATH` in `transformers-local-onnx.ts`\n3. Update embedding dimension in your data pipeline if needed\n\n## Clean Up\n\nTo remove the model and reclaim disk space:\n\n```bash\ncd search/models\nrm -rf all-MiniLM-L6-v2\n```\n\nThen switch back to a different strategy in `search/index.ts`.\n\n",[],[226,541,567],"7fdcccf6-b9ca-11f0-9362-0224a6c84d84","7fdccdbe-b9ca-11f0-9362-0224a6c84d84",72,["D",1762293982616],"# Local ONNX Models\n\nThis directory contains pre-downloaded ONNX models for local embedding generation.\n\n## Setup Instructions\n\n### 1. Download the all-MiniLM-L6-v2 Model\n\nDownload the ONNX model files from Hugging Face:\n\n```bash\n# Create the model directory\nmkdir -p all-MiniLM-L6-v2\n\n# Download the ONNX model files\n# You can download these manually from: https://huggingface.co/Xenova/all-MiniLM-L6-v2/tree/main/onnx\n# Required files:\n# - model.onnx\n# - tokenizer.json\n# - tokenizer_config.json\n# - config.json\n# - special_tokens_map.json (optional)\n```\n\nOr use this script to download automatically:\n\n```bash\ncd models\ngit clone https://huggingface.co/Xenova/all-MiniLM-L6-v2\n# Clean up .git directory to save space\nrm -rf all-MiniLM-L6-v2/.git\n```\n\n### 2. Verify the Files\n\nYour directory structure should look like:\n\n```\nmodels/\n all-MiniLM-L6-v2/\n onnx/\n model.onnx # ~23MB - the main ONNX model\n model_quantized.onnx # ~6MB - quantized version (optional, faster)\n tokenizer.json # Tokenizer configuration\n tokenizer_config.json # Additional tokenizer settings\n config.json # Model configuration\n```\n\n### 3. Switch to Local Model Strategy\n\nIn `search/index.ts`, uncomment the local model strategy:\n\n```typescript\nimport { searchStrategy, generateEmbeddings } from \"./transformers-local-onnx.ts\";\n```\n\n## Model Details\n\n- **Model**: all-MiniLM-L6-v2 (Sentence Transformers)\n- **Dimensions**: 384\n- **Size**: ~23MB (full) or ~6MB (quantized)\n- **Speed**: ~10-30ms per embedding (after initial load)\n- **Advantage**: No network calls, works offline, completely local\n\n## Notes\n\n- The model files are binary and should be kept in this directory\n- The quantized model (`model_quantized.onnx`) is smaller and faster but slightly less accurate\n- This strategy requires the model files to be present before running\n- For deployment, ensure these files are included in your build/deployment package\n\n### About the \"mutex lock failed\" Error\n\nYou may see this error when running test scripts:\n```\nlibc++abi: terminating due to uncaught exception of type std::__1::system_error: mutex lock failed: Invalid argument\n```\n\n**This is harmless and expected!** Here's why:\n- The error happens in ONNX runtime's C++ cleanup code when the process exits\n- It occurs **after** all work is completed successfully\n- It **does not** affect functionality, data, or results\n- It only appears when scripts exit immediately\n- Long-running servers (like your main app) typically don't exit, so you won't see this in production\n\n**What to do**: Just ignore the error message. All tests pass successfully before it appears.\n\n",[],[226,541,192],"7fc802e4-b9ca-11f0-a8b3-0224a6c84d84",".gitignore","7fc80370-b9ca-11f0-a8b3-0224a6c84d84",70,["D",1762293982674],"# Ignore all downloaded model files (they're large!)\nall-MiniLM-L6-v2/\n\n# But keep the README and download script\n!README.md\n!download-model.sh\n!.gitignore\n\n",[],[226,541,582],"7fd41336-b9ca-11f0-8b68-0224a6c84d84","tokenizer_config.json","7fd413ea-b9ca-11f0-8b68-0224a6c84d84","7f049ff2-b9ca-11f0-98dd-0224a6c84d84",71,["D",1762293982754],"{\n \"clean_up_tokenization_spaces\": true,\n \"cls_token\": \"[CLS]\",\n \"do_basic_tokenize\": true,\n \"do_lower_case\": true,\n \"mask_token\": \"[MASK]\",\n \"model_max_length\": 512,\n \"never_split\": null,\n \"pad_token\": \"[PAD]\",\n \"sep_token\": \"[SEP]\",\n \"strip_accents\": null,\n \"tokenize_chinese_chars\": true,\n \"tokenizer_class\": \"BertTokenizer\",\n \"unk_token\": \"[UNK]\"\n}\n",[],[226,541,598,590],"all-MiniLM-L6-v2","7ff2d50a-b9ca-11f0-83d0-0224a6c84d84","special_tokens_map.json","7ff2d5a0-b9ca-11f0-83d0-0224a6c84d84",75,["D",1762293982796],"{\n \"cls_token\": \"[CLS]\",\n \"mask_token\": \"[MASK]\",\n \"pad_token\": \"[PAD]\",\n \"sep_token\": \"[SEP]\",\n \"unk_token\": \"[UNK]\"\n}\n",[],[226,541,598,600],"8013cf8a-b9ca-11f0-8051-0224a6c84d84","config.json","8013d052-b9ca-11f0-8051-0224a6c84d84",79,["D",1762293983167],"{\n \"_name_or_path\": \"sentence-transformers/all-MiniLM-L6-v2\",\n \"architectures\": [\n \"BertModel\"\n ],\n \"attention_probs_dropout_prob\": 0.1,\n \"classifier_dropout\": null,\n \"gradient_checkpointing\": false,\n \"hidden_act\": \"gelu\",\n \"hidden_dropout_prob\": 0.1,\n \"hidden_size\": 384,\n \"initializer_range\": 0.02,\n \"intermediate_size\": 1536,\n \"layer_norm_eps\": 1e-12,\n \"max_position_embeddings\": 512,\n \"model_type\": \"bert\",\n \"num_attention_heads\": 12,\n \"num_hidden_layers\": 6,\n \"pad_token_id\": 0,\n \"position_embedding_type\": \"absolute\",\n \"transformers_version\": \"4.29.2\",\n \"type_vocab_size\": 2,\n \"use_cache\": true,\n \"vocab_size\": 30522\n}\n",[],[226,541,598,608],"801cceb4-b9ca-11f0-8b68-0224a6c84d84","801ccf7c-b9ca-11f0-8b68-0224a6c84d84",80,["D",1762293983230],"---\nbase_model: sentence-transformers/all-MiniLM-L6-v2\nlibrary_name: transformers.js\nlicense: apache-2.0\n---\n\nhttps://huggingface.co/sentence-transformers/all-MiniLM-L6-v2 with ONNX weights to be compatible with Transformers.js.\n\n## Usage (Transformers.js)\n\nIf you haven't already, you can install the [Transformers.js](https://huggingface.co/docs/transformers.js) JavaScript library from [NPM](https://www.npmjs.com/package/@huggingface/transformers) using:\n```bash\nnpm i @huggingface/transformers\n```\n\nYou can then use the model to compute embeddings like this:\n\n```js\nimport { pipeline } from '@huggingface/transformers';\n\n// Create a feature-extraction pipeline\nconst extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');\n\n// Compute sentence embeddings\nconst sentences = ['This is an example sentence', 'Each sentence is converted'];\nconst output = await extractor(sentences, { pooling: 'mean', normalize: true });\nconsole.log(output);\n// Tensor {\n// dims: [ 2, 384 ],\n// type: 'float32',\n// data: Float32Array(768) [ 0.04592696577310562, 0.07328180968761444, ... ],\n// size: 768\n// }\n```\n\nYou can convert this Tensor to a nested JavaScript array using `.tolist()`:\n```js\nconsole.log(output.tolist());\n// [\n// [ 0.04592696577310562, 0.07328180968761444, 0.05400655046105385, ... ],\n// [ 0.08188057690858841, 0.10760223120450974, -0.013241755776107311, ... ]\n// ]\n```\n\nNote: Having a separate repo for ONNX weights is intended to be a temporary solution until WebML gains more traction. If you would like to make your models web-ready, we recommend converting to ONNX using [🤗 Optimum](https://huggingface.co/docs/optimum/index) and structuring your repo like this one (with ONNX weights located in a subfolder named `onnx`).",[],[226,541,598,192],"8027e81c-b9ca-11f0-8b68-0224a6c84d84",".gitattributes","8027e8d0-b9ca-11f0-8b68-0224a6c84d84",81,["D",1762293983303],"*.7z filter=lfs diff=lfs merge=lfs -text\n*.arrow filter=lfs diff=lfs merge=lfs -text\n*.bin filter=lfs diff=lfs merge=lfs -text\n*.bz2 filter=lfs diff=lfs merge=lfs -text\n*.ckpt filter=lfs diff=lfs merge=lfs -text\n*.ftz filter=lfs diff=lfs merge=lfs -text\n*.gz filter=lfs diff=lfs merge=lfs -text\n*.h5 filter=lfs diff=lfs merge=lfs -text\n*.joblib filter=lfs diff=lfs merge=lfs -text\n*.lfs.* filter=lfs diff=lfs merge=lfs -text\n*.mlmodel filter=lfs diff=lfs merge=lfs -text\n*.model filter=lfs diff=lfs merge=lfs -text\n*.msgpack filter=lfs diff=lfs merge=lfs -text\n*.npy filter=lfs diff=lfs merge=lfs -text\n*.npz filter=lfs diff=lfs merge=lfs -text\n*.onnx filter=lfs diff=lfs merge=lfs -text\n*.ot filter=lfs diff=lfs merge=lfs -text\n*.parquet filter=lfs diff=lfs merge=lfs -text\n*.pb filter=lfs diff=lfs merge=lfs -text\n*.pickle filter=lfs diff=lfs merge=lfs -text\n*.pkl filter=lfs diff=lfs merge=lfs -text\n*.pt filter=lfs diff=lfs merge=lfs -text\n*.pth filter=lfs diff=lfs merge=lfs -text\n*.rar filter=lfs diff=lfs merge=lfs -text\n*.safetensors filter=lfs diff=lfs merge=lfs -text\nsaved_model/**/* filter=lfs diff=lfs merge=lfs -text\n*.tar.* filter=lfs diff=lfs merge=lfs -text\n*.tflite filter=lfs diff=lfs merge=lfs -text\n*.tgz filter=lfs diff=lfs merge=lfs -text\n*.wasm filter=lfs diff=lfs merge=lfs -text\n*.xz filter=lfs diff=lfs merge=lfs -text\n*.zip filter=lfs diff=lfs merge=lfs -text\n*.zst filter=lfs diff=lfs merge=lfs -text\n*tfevents* filter=lfs diff=lfs merge=lfs -text\n",[],[226,541,598,623],"800ecdbe-b9ca-11f0-8bad-0224a6c84d84","model_uint8.onnx","800ece68-b9ca-11f0-8bad-0224a6c84d84","7f271ab4-b9ca-11f0-8b68-0224a6c84d84",78,["D",1762293983138],"version https://git-lfs.github.com/spec/v1\noid sha256:57097fb267dcffb2a9f12fba945201c76ae99b7640ed1c6389cc9e0341c71ee6\nsize 22845806\n",[],[226,541,598,639,631],"onnx","7fed5bac-b9ca-11f0-b636-0224a6c84d84","model_q4f16.onnx","7fed5c9c-b9ca-11f0-b636-0224a6c84d84",74,["D",1762293982920],"version https://git-lfs.github.com/spec/v1\noid sha256:eb08a666c46109637e0b6cb04f6052a68efd59bb0252d4e0438d28fb6b2d853d\nsize 30018257\n",[],[226,541,598,639,641],"7fe5757c-b9ca-11f0-b636-0224a6c84d84","model_q4.onnx","7fe5766c-b9ca-11f0-b636-0224a6c84d84",73,["D",1762293982831],"version https://git-lfs.github.com/spec/v1\noid sha256:ac0f989345605769651a18bb796fda6235d921a37bd4ac065cb51781da44d079\nsize 54578772\n",[],[226,541,598,639,649],"80018eb0-b9ca-11f0-83d0-0224a6c84d84","model_int8.onnx","80018f6e-b9ca-11f0-83d0-0224a6c84d84",77,["D",1762293983038],"version https://git-lfs.github.com/spec/v1\noid sha256:afdb6f1a0e45b715d0bb9b11772f032c399babd23bfc31fed1c170afc848bdb1\nsize 22972370\n",[],[226,541,598,639,657],"8043967a-b9ca-11f0-a963-0224a6c84d84","model_fp16.onnx","8043972e-b9ca-11f0-a963-0224a6c84d84",84,["D",1762293983161],"version https://git-lfs.github.com/spec/v1\noid sha256:2cdb5e58291813b6d6e248ed69010100246821a367fa17b1b81ae9483744533d\nsize 45297825\n",[],[226,541,598,639,665],"7ff7bc28-b9ca-11f0-b636-0224a6c84d84","model_bnb4.onnx","7ff7bce6-b9ca-11f0-b636-0224a6c84d84",76,["D",1762293982987],"version https://git-lfs.github.com/spec/v1\noid sha256:106c4807d85ceaf3590d9da9c07b3a22ded29bcc0d47333da0b1b741410eb128\nsize 53915448\n",[],[226,541,598,639,673],"7f271a00-b9ca-11f0-8b68-0224a6c84d84",52,["D",1762293981620],[268,269,270,271,272,273],[226,541,598,639],"7f049f52-b9ca-11f0-98dd-0224a6c84d84",51,["D",1762293981385],[267,274,275,276,277,278],[226,541,598],"7eec29cc-b9ca-11f0-8b68-0224a6c84d84",50,["D",1762293981234],[266,279,280,281,282,283,284,285],[226,541],["D",1762207521254],[265,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300],[226],["D",1762207521817],[],[232],"7f537686-b9ca-11f0-8b68-0224a6c84d84","7f537758-b9ca-11f0-8b68-0224a6c84d84",55,["D",1762293981908],"// Answer strategy utilities\n\nimport type { Page } from \"../search/types.ts\";\n\n// Calculate approximate token count (rough estimate: 1 token ≈ 4 characters)\nexport const estimateTokens = (text: string): number => {\n return Math.ceil(text.length / 4);\n};\n\n// Truncate content to fit within token limit\nexport const truncateToTokenLimit = (content: string, maxTokens: number): string => {\n const maxChars = maxTokens * 4;\n if (content.length <= maxChars) {\n return content;\n }\n return content.substring(0, maxChars) + \"\\n...(truncated)\";\n};\n\n// Format pages into context for LLM\nexport const formatPagesAsContext = (\n pages: Page[],\n maxPages?: number,\n maxTokensPerPage?: number\n): { context: string; pagesUsed: number; totalTokens: number } => {\n const pagesToUse = maxPages ? pages.slice(0, maxPages) : pages;\n \n let context = \"\";\n let totalTokens = 0;\n \n for (const page of pagesToUse) {\n let pageContent = page.content;\n \n // Truncate individual page content if needed\n if (maxTokensPerPage) {\n pageContent = truncateToTokenLimit(pageContent, maxTokensPerPage);\n }\n \n const pageTokens = estimateTokens(pageContent);\n totalTokens += pageTokens;\n \n context += `\\n\\n---\\n`;\n context += `Source: ${page.title || page.path}\\n`;\n context += `URL: ${page.url}\\n`;\n context += `---\\n\\n`;\n context += pageContent;\n }\n \n return {\n context,\n pagesUsed: pagesToUse.length,\n totalTokens,\n };\n};\n\n// Default system prompt for documentation Q&A\nexport const DEFAULT_SYSTEM_PROMPT = `You are a helpful AI assistant that answers questions about Groq's documentation.\n\nYou will be provided with relevant documentation pages as context. Use this context to provide accurate, helpful answers to user questions.\n\nGuidelines:\n- Base your answers on the provided documentation context\n- If the answer isn't in the context, say so politely\n- Be concise but thorough\n- Include relevant code examples when available in the context\n- Reference specific documentation sections when helpful\n- Format your response in markdown\n- If multiple approaches are mentioned in the docs, present them clearly\n- When linking to documentation, remove the .md extension from URLs (e.g., use /docs/api-reference instead of /docs/api-reference.md)\n- Use the exact URLs provided in the context, but without .md extensions\n\nRemember: You are answering questions specifically about Groq's API, models, and services based on their official documentation.`;\n\n// Post-process answer to clean up any .md extensions in URLs\nexport const cleanupMarkdownLinks = (answer: string): string => {\n // Remove .md from markdown links: [text](url.md) -> [text](url)\n // Handles various cases:\n // - [text](url.md) -> [text](url)\n // - [text](url.md#anchor) -> [text](url#anchor)\n // - [text](url.md?query) -> [text](url?query)\n return answer.replace(/(\\[[^\\]]+\\]\$[^)]+)\\.md(\$|#|\\?)/g, '$1$2');\n};\n\n",[],[238,142],"7f600aae-b9ca-11f0-8b68-0224a6c84d84","7f600b58-b9ca-11f0-8b68-0224a6c84d84",57,["D",1762293981956],"// Answer strategy types and interfaces\n\nexport type AnswerResult = {\n answer: string;\n query: string;\n searchResults: Array<{\n path: string;\n url: string;\n title?: string;\n score: number;\n }>;\n contextUsed: number; // Number of pages used in context\n totalTokens?: number;\n metadata?: {\n strategy?: string;\n model?: string;\n timings?: Record;\n [key: string]: any;\n };\n};\n\nexport type AnswerOptions = {\n limit?: number; // Max number of search results to use\n minScore?: number; // Minimum search score threshold\n maxContextPages?: number; // Max pages to include in context\n temperature?: number; // LLM temperature\n model?: string; // Override the default model\n systemPrompt?: string; // Override the default system prompt\n enableTiming?: boolean; // Enable timing tracking\n trackTime?: (label: string) => void; // Callback for tracking time points\n [key: string]: any;\n};\n\nexport type AnswerStrategy = {\n name: string;\n description: string;\n answer: (query: string, options?: AnswerOptions) => Promise;\n};\n\n",[],[238,426],"7f8f7a6e-b9ca-11f0-8f8b-0224a6c84d84","llama-3.3-70b-default.ts","7f8f7b36-b9ca-11f0-8f8b-0224a6c84d84",62,["D",1762293982145],"// Default Answer Strategy using Llama 3.3 70B\n// Uses the active search strategy to find relevant docs, then feeds them to Groq's Llama 3.3 70B model\n\nimport \"jsr:@std/dotenv/load\"; // needed for deno run; not req for smallweb or valtown\nimport type { AnswerStrategy, AnswerResult, AnswerOptions } from \"./types.ts\";\nimport type { Page } from \"../search/types.ts\";\nimport { formatPagesAsContext, DEFAULT_SYSTEM_PROMPT, cleanupMarkdownLinks } from \"./utils.ts\";\nimport { groqChatCompletion } from \"../groq.ts\";\nimport { searchPages } from \"../search/index.ts\";\n\n// Model configuration\nconst DEFAULT_MODEL = \"llama-3.3-70b-versatile\";\nconst DEFAULT_MAX_CONTEXT_PAGES = 5; // Use top 5 search results\nconst DEFAULT_TEMPERATURE = 0.3;\nconst MAX_TOKENS_PER_PAGE = 10000; // Limit per page to manage context window\n\n// Get all pages for answering with content\n// Loads from latest.json which now includes full content\nconst getAllPagesForAnswering = async (): Promise => {\n try {\n const jsonPath = new URL(\"../data/latest.json\", import.meta.url).pathname;\n const jsonText = await Deno.readTextFile(jsonPath);\n const jsonData = JSON.parse(jsonText);\n \n if (jsonData.results && Array.isArray(jsonData.results)) {\n return jsonData.results.map((result: any) => ({\n path: result.path,\n url: result.url,\n content: result.content || \"\", // Content now included in JSON\n title: result.title,\n embeddings: result.embeddings,\n metadata: result.metadata,\n }));\n }\n \n return [];\n } catch (error) {\n console.error(\"Failed to load pages for answering:\", error);\n return [];\n }\n};\n\nexport const answerStrategy: AnswerStrategy = {\n name: \"llama-3.3-70b-default\",\n description: \"RAG using active search strategy + Llama 3.3 70B with up to 5 doc pages in context\",\n \n answer: async (query: string, options: AnswerOptions = {}): Promise => {\n const enableTiming = options.enableTiming || false;\n const timings: Record = {};\n const startTime = performance.now();\n \n // Step 1: Search for relevant pages\n const searchStart = performance.now();\n const pages = await getAllPagesForAnswering();\n const searchResults = await searchPages(query, pages, {\n limit: options.limit || 10, // Get more results, but we'll only use top N for context\n minScore: options.minScore || 0,\n enableTiming: true, // Enable detailed search timing\n });\n if (enableTiming) timings.search = performance.now() - searchStart;\n \n // Capture detailed search timings from first result's metadata\n const searchMetadata = searchResults[0]?.metadata || {};\n \n if (searchResults.length === 0) {\n return {\n answer: \"I couldn't find any relevant documentation to answer your question. Please try rephrasing or ask about a different topic.\",\n query,\n searchResults: [],\n contextUsed: 0,\n metadata: {\n strategy: \"llama-3.3-70b-default\",\n model: DEFAULT_MODEL,\n error: \"No search results found\",\n ...(enableTiming && { timings }),\n },\n };\n }\n \n // Step 2: Get full content for top search results\n const contextStart = performance.now();\n const maxContextPages = options.maxContextPages || DEFAULT_MAX_CONTEXT_PAGES;\n const topResults = searchResults.slice(0, maxContextPages);\n \n // Get pages with content (now included in JSON from recalculate)\n const pagesWithContent: Page[] = [];\n for (const result of topResults) {\n const page = pages.find(p => p.path === result.path);\n if (page && page.content) {\n pagesWithContent.push(page);\n }\n }\n \n // Format pages as context\n const { context, pagesUsed, totalTokens } = formatPagesAsContext(\n pagesWithContent,\n maxContextPages,\n MAX_TOKENS_PER_PAGE\n );\n if (enableTiming) timings.contextPrep = performance.now() - contextStart;\n \n // Step 3: Call Groq LLM with context\n const llmStart = performance.now();\n const systemPrompt = options.systemPrompt || DEFAULT_SYSTEM_PROMPT;\n const userPrompt = `Context from documentation:\\n${context}\\n\\n---\\n\\nUser Question: ${query}\\n\\nPlease provide a helpful answer based on the documentation context above.`;\n\n \n const llmResponse = await groqChatCompletion([\n { role: \"system\", content: systemPrompt },\n { role: \"user\", content: userPrompt },\n ], {\n model: options.model || DEFAULT_MODEL,\n temperature: options.temperature ?? DEFAULT_TEMPERATURE,\n });\n if (enableTiming) timings.llm = performance.now() - llmStart;\n \n if (!llmResponse) {\n return {\n answer: \"Sorry, I encountered an error while generating the answer. Please try again.\",\n query,\n searchResults: topResults.map(r => ({\n path: r.path,\n url: r.url,\n title: r.title,\n score: r.score,\n })),\n contextUsed: pagesUsed,\n metadata: {\n strategy: \"llama-3.3-70b-default\",\n model: DEFAULT_MODEL,\n error: \"LLM call failed\",\n ...(enableTiming && { timings }),\n },\n };\n }\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Answer timing breakdown:`);\n console.log(` Search: ${timings.search?.toFixed(2)}ms`);\n console.log(` Context prep: ${timings.contextPrep?.toFixed(2)}ms`);\n console.log(` LLM call: ${timings.llm?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n // Clean up any .md extensions in markdown links\n const cleanedAnswer = cleanupMarkdownLinks(llmResponse);\n \n // Return result\n return {\n answer: cleanedAnswer,\n query,\n searchResults: topResults.map(r => ({\n path: r.path,\n url: r.url,\n title: r.title,\n score: r.score,\n })),\n contextUsed: pagesUsed,\n totalTokens,\n metadata: {\n strategy: \"llama-3.3-70b-default\",\n model: options.model || DEFAULT_MODEL,\n temperature: options.temperature ?? DEFAULT_TEMPERATURE,\n searchResultsCount: searchResults.length,\n searchStrategy: searchMetadata.strategy,\n searchTimings: searchMetadata.timings, // Detailed search timings\n ...(enableTiming && { timings }),\n },\n };\n },\n};\n\n",[],[238,716],"7fa5ca80-b9ca-11f0-9362-0224a6c84d84","7fa5cb2a-b9ca-11f0-9362-0224a6c84d84",65,["D",1762293982125],"// Answer - Main entry point\n// To switch strategies, comment/uncomment the imports and function calls below\n\nexport * from \"./types.ts\";\nexport * from \"./utils.ts\"; // Exports cleanupMarkdownLinks and other utilities\n\nimport type { AnswerOptions, AnswerResult } from \"./types.ts\";\n\n// Answer strategies - uncomment to switch\n// import { answerStrategy } from \"./llama-3.3-70b-default.ts\"; // Llama 3.3 70B with up to 5 doc pages\nimport { answerStrategy } from \"./gpt-oss-20b-default.ts\"; // Default: GPT-OSS 20B with up to 5 doc pages\n// Add more strategies here as needed:\n// import { answerStrategy } from \"./llama-3.1-8b-fast.ts\"; // Fast strategy with smaller model\n// import { answerStrategy } from \"./mixtral-8x7b-extended.ts\"; // More context with Mixtral\n\n// Main answer function - comment/uncomment imports above to switch strategies\nexport const answerQuestion = async (\n query: string,\n options: AnswerOptions = {}\n): Promise => {\n return answerStrategy.answer(query, options);\n};\n\n// Export answer strategy for introspection (e.g., /answer/info endpoint)\nexport { answerStrategy };\n\n",[],[238,494],"7f745b80-b9ca-11f0-83d0-0224a6c84d84","gpt-oss-20b-default.ts","7f745c16-b9ca-11f0-83d0-0224a6c84d84",59,["D",1762293982099],"// Answer Strategy using GPT-OSS 20B\n// Uses the active search strategy to find relevant docs, then feeds them to Groq's GPT-OSS 20B model\n\nimport \"jsr:@std/dotenv/load\"; // needed for deno run; not req for smallweb or valtown\nimport type { AnswerStrategy, AnswerResult, AnswerOptions } from \"./types.ts\";\nimport type { Page } from \"../search/types.ts\";\nimport { formatPagesAsContext, DEFAULT_SYSTEM_PROMPT, cleanupMarkdownLinks } from \"./utils.ts\";\nimport { groqChatCompletion } from \"../groq.ts\";\nimport { searchPages } from \"../search/index.ts\";\n\n// Model configuration\nconst DEFAULT_MODEL = \"openai/gpt-oss-20b\";\nconst DEFAULT_MAX_CONTEXT_PAGES = 7; // Use top 5 search results\nconst DEFAULT_TEMPERATURE = 0.3;\nconst MAX_TOKENS_PER_PAGE = 10000; // Limit per page to manage context window\n\n// Get all pages for answering with content\n// Loads from latest.json which now includes full content\nconst getAllPagesForAnswering = async (): Promise => {\n try {\n const jsonPath = new URL(\"../data/latest.json\", import.meta.url).pathname;\n const jsonText = await Deno.readTextFile(jsonPath);\n const jsonData = JSON.parse(jsonText);\n \n if (jsonData.results && Array.isArray(jsonData.results)) {\n return jsonData.results.map((result: any) => ({\n path: result.path,\n url: result.url,\n content: result.content || \"\", // Content now included in JSON\n title: result.title,\n embeddings: result.embeddings,\n metadata: result.metadata,\n }));\n }\n \n return [];\n } catch (error) {\n console.error(\"Failed to load pages for answering:\", error);\n return [];\n }\n};\n\nexport const answerStrategy: AnswerStrategy = {\n name: \"gpt-oss-20b-default\",\n description: \"RAG using active search strategy + GPT-OSS 20B with up to 5 doc pages in context\",\n \n answer: async (query: string, options: AnswerOptions = {}): Promise => {\n const enableTiming = options.enableTiming || false;\n const timings: Record = {};\n const startTime = performance.now();\n const trackTime = options.trackTime || (() => {});\n \n // Step 1: Load pages and search\n trackTime(\"Pre: Load pages for search\");\n const pages = await getAllPagesForAnswering();\n trackTime(\"Post: Pages loaded\");\n \n trackTime(\"Pre: Generate query embedding\");\n const searchStart = performance.now();\n const searchResults = await searchPages(query, pages, {\n limit: options.limit || 10, // Get more results, but we'll only use top N for context\n minScore: options.minScore || 0,\n enableTiming: true, // Enable detailed search timing\n });\n if (enableTiming) timings.search = performance.now() - searchStart;\n trackTime(\"Post: Search complete (embedding + similarity)\");\n \n // Capture detailed search timings from first result's metadata\n const searchMetadata = searchResults[0]?.metadata || {};\n \n if (searchResults.length === 0) {\n return {\n answer: \"I couldn't find any relevant documentation to answer your question. Please try rephrasing or ask about a different topic.\",\n query,\n searchResults: [],\n contextUsed: 0,\n metadata: {\n strategy: \"gpt-oss-20b-default\",\n model: DEFAULT_MODEL,\n error: \"No search results found\",\n ...(enableTiming && { timings }),\n },\n };\n }\n \n // Step 2: Prepare context from search results\n trackTime(\"Pre: Prepare context from search results\");\n const contextStart = performance.now();\n const maxContextPages = options.maxContextPages || DEFAULT_MAX_CONTEXT_PAGES;\n const topResults = searchResults.slice(0, maxContextPages);\n \n // Get pages with content (now included in JSON from recalculate)\n const pagesWithContent: Page[] = [];\n for (const result of topResults) {\n const page = pages.find(p => p.path === result.path);\n if (page && page.content) {\n pagesWithContent.push(page);\n }\n }\n \n // Format pages as context\n const { context, pagesUsed, totalTokens } = formatPagesAsContext(\n pagesWithContent,\n maxContextPages,\n MAX_TOKENS_PER_PAGE\n );\n if (enableTiming) timings.contextPrep = performance.now() - contextStart;\n trackTime(\"Post: Context prepared\");\n \n // Step 3: Call Groq LLM with context\n trackTime(\"Pre: Call LLM\");\n const llmStart = performance.now();\n const systemPrompt = options.systemPrompt || DEFAULT_SYSTEM_PROMPT;\n const userPrompt = `Context from documentation:\\n${context}\\n\\n---\\n\\nUser Question: ${query}\\n\\nPlease provide a helpful answer based on the documentation context above.`;\n\n \n const llmResponse = await groqChatCompletion([\n { role: \"system\", content: systemPrompt },\n { role: \"user\", content: userPrompt },\n ], {\n model: options.model || DEFAULT_MODEL,\n temperature: options.temperature ?? DEFAULT_TEMPERATURE,\n });\n if (enableTiming) timings.llm = performance.now() - llmStart;\n trackTime(\"Post: LLM response received\");\n \n if (!llmResponse) {\n return {\n answer: \"Sorry, I encountered an error while generating the answer. Please try again.\",\n query,\n searchResults: topResults.map(r => ({\n path: r.path,\n url: r.url,\n title: r.title,\n score: r.score,\n })),\n contextUsed: pagesUsed,\n metadata: {\n strategy: \"gpt-oss-20b-default\",\n model: DEFAULT_MODEL,\n error: \"LLM call failed\",\n ...(enableTiming && { timings }),\n },\n };\n }\n \n if (enableTiming) {\n timings.total = performance.now() - startTime;\n console.log(` ⏱️ Answer timing breakdown:`);\n console.log(` Search: ${timings.search?.toFixed(2)}ms`);\n console.log(` Context prep: ${timings.contextPrep?.toFixed(2)}ms`);\n console.log(` LLM call: ${timings.llm?.toFixed(2)}ms`);\n console.log(` Total: ${timings.total?.toFixed(2)}ms`);\n }\n \n // Clean up any .md extensions in markdown links\n const cleanedAnswer = cleanupMarkdownLinks(llmResponse);\n \n // Return result\n return {\n answer: cleanedAnswer,\n query,\n searchResults: topResults.map(r => ({\n path: r.path,\n url: r.url,\n title: r.title,\n score: r.score,\n })),\n contextUsed: pagesUsed,\n totalTokens,\n metadata: {\n strategy: \"gpt-oss-20b-default\",\n model: options.model || DEFAULT_MODEL,\n temperature: options.temperature ?? DEFAULT_TEMPERATURE,\n searchResultsCount: searchResults.length,\n searchStrategy: searchMetadata.strategy,\n searchTimings: searchMetadata.timings, // Detailed search timings\n ...(enableTiming && { timings }),\n },\n };\n },\n};\n\n\n",[],[238,731],"7f483adc-b9ca-11f0-b7be-0224a6c84d84","7f483b86-b9ca-11f0-b7be-0224a6c84d84",54,["D",1762293981801],"# Answer Strategy Comparison\n\nThis document helps you choose the right answer strategy for your use case.\n\n## Current Strategies\n\n### 1. llama-3.3-70b-default ✅ (Active)\n\n**Best for**: General-purpose Q&A, complex questions, high-quality answers\n\n**Configuration**:\n- Model: `llama-3.3-70b-versatile`\n- Max context pages: 5\n- Temperature: 0.3\n- Tokens per page: ~2000\n\n**Performance**:\n- Search: 50-500ms (depends on search strategy)\n- Context prep: <50ms\n- LLM call: 500-3000ms\n- **Total: ~1-3.5s**\n\n**Quality**: ⭐⭐⭐⭐⭐ (5/5)\n**Speed**: ⭐⭐⭐ (3/5)\n**Cost**: ⭐⭐⭐ (3/5)\n\n**Pros**:\n- ✅ Excellent reasoning and comprehension\n- ✅ Handles complex multi-part questions\n- ✅ Good at code examples and technical details\n- ✅ Consistent, reliable answers\n\n**Cons**:\n- ❌ Slower than 8B models\n- ❌ More expensive per token\n- ❌ May be overkill for simple questions\n\n**Use cases**:\n- Complex how-to questions\n- Multi-step implementations\n- Comparing features/models\n- Debugging and troubleshooting\n- Code generation with explanations\n\n---\n\n## Future Strategies (Templates)\n\n### 2. llama-3.1-8b-fast (Not yet implemented)\n\n**Best for**: Simple factual questions, quick lookups, high-volume traffic\n\n**Configuration**:\n- Model: `llama-3.1-8b-instant`\n- Max context pages: 2-3\n- Temperature: 0.3\n- Tokens per page: ~1000\n\n**Expected Performance**:\n- Search: 50-500ms\n- Context prep: <30ms\n- LLM call: 200-800ms\n- **Total: ~0.3-1.3s**\n\n**Quality**: ⭐⭐⭐⭐ (4/5)\n**Speed**: ⭐⭐⭐⭐⭐ (5/5)\n**Cost**: ⭐⭐⭐⭐⭐ (5/5)\n\n**Best for**:\n- \"What models are available?\"\n- \"What is X?\"\n- \"How much does Y cost?\"\n- Simple API lookups\n\n---\n\n### 3. mixtral-8x7b-extended (Not yet implemented)\n\n**Best for**: Very complex questions requiring lots of context\n\n**Configuration**:\n- Model: `mixtral-8x7b-32768`\n- Max context pages: 10\n- Temperature: 0.3\n- Tokens per page: ~3000\n\n**Expected Performance**:\n- Search: 50-500ms\n- Context prep: ~100ms\n- LLM call: 1000-4000ms\n- **Total: ~1.5-5s**\n\n**Quality**: ⭐⭐⭐⭐⭐ (5/5)\n**Speed**: ⭐⭐ (2/5)\n**Cost**: ⭐⭐ (2/5)\n\n**Best for**:\n- \"Compare all available models\"\n- \"How do I implement a complete chat system with X, Y, Z features?\"\n- Questions requiring synthesis across many docs\n\n---\n\n### 4. llama-3.3-70b-code (Not yet implemented)\n\n**Best for**: Code generation and technical implementation questions\n\n**Configuration**:\n- Model: `llama-3.3-70b-versatile`\n- Max context pages: 5\n- Temperature: 0.2 (more deterministic for code)\n- Custom system prompt optimized for code\n\n**Expected Performance**:\n- Similar to llama-3.3-70b-default\n- **Total: ~1-3.5s**\n\n**Quality**: ⭐⭐⭐⭐⭐ (5/5) for code\n**Speed**: ⭐⭐⭐ (3/5)\n**Cost**: ⭐⭐⭐ (3/5)\n\n**Best for**:\n- \"Show me example code for X\"\n- \"How do I implement Y in Python/JavaScript?\"\n- API usage examples\n- Debugging code issues\n\n---\n\n### 5. citation-mode (Not yet implemented)\n\n**Best for**: Academic/research use, when you need to cite sources\n\n**Configuration**:\n- Model: `llama-3.3-70b-versatile`\n- Max context pages: 5\n- Temperature: 0.3\n- Custom system prompt for citations\n- Post-processing to add inline citations\n\n**Expected Performance**:\n- Similar to llama-3.3-70b-default\n- +50-100ms for citation processing\n- **Total: ~1.5-4s**\n\n**Quality**: ⭐⭐⭐⭐⭐ (5/5)\n**Speed**: ⭐⭐⭐ (3/5)\n**Cost**: ⭐⭐⭐ (3/5)\n\n**Best for**:\n- Documentation generation\n- Training materials\n- When you need exact source references\n- Verifying information\n\n---\n\n## Comparison Matrix\n\n| Strategy | Speed | Quality | Cost | Context | Best For |\n|----------|-------|---------|------|---------|----------|\n| **llama-3.3-70b-default** ✅ | Medium | Excellent | Medium | 5 pages | General purpose, complex Q&A |\n| llama-3.1-8b-fast | Fast | Good | Low | 2-3 pages | Simple lookups, high volume |\n| mixtral-8x7b-extended | Slow | Excellent | High | 10 pages | Very complex questions |\n| llama-3.3-70b-code | Medium | Excellent | Medium | 5 pages | Code generation |\n| citation-mode | Medium | Excellent | Medium | 5 pages | Research, documentation |\n\n## Choosing a Strategy\n\n### Decision Tree\n\n1. **Is the question simple and factual?**\n - Yes → Use `llama-3.1-8b-fast` (when implemented)\n - No → Continue\n\n2. **Does it require lots of documentation context?**\n - Yes → Use `mixtral-8x7b-extended` (when implemented)\n - No → Continue\n\n3. **Is it primarily about code/implementation?**\n - Yes → Use `llama-3.3-70b-code` (when implemented)\n - No → Continue\n\n4. **Do you need citations?**\n - Yes → Use `citation-mode` (when implemented)\n - No → Use `llama-3.3-70b-default` ✅\n\n### By Question Type\n\n**Factual Questions** (\"What is X?\")\n- 🥇 llama-3.1-8b-fast (fast + good enough)\n- 🥈 llama-3.3-70b-default (better quality)\n\n**How-To Questions** (\"How do I X?\")\n- 🥇 llama-3.3-70b-default (balanced)\n- 🥈 llama-3.3-70b-code (if code-heavy)\n\n**Complex Questions** (\"How do I build X with Y and Z?\")\n- 🥇 mixtral-8x7b-extended (more context)\n- 🥈 llama-3.3-70b-default (usually sufficient)\n\n**Code Examples** (\"Show me code for X\")\n- 🥇 llama-3.3-70b-code (optimized)\n- 🥈 llama-3.3-70b-default (good enough)\n\n**Comparison Questions** (\"What's the difference between X and Y?\")\n- 🥇 llama-3.3-70b-default (good reasoning)\n- 🥈 mixtral-8x7b-extended (if comparing many things)\n\n### By Performance Requirements\n\n**Need fastest response** (<1s goal)\n- Use `llama-3.1-8b-fast` (when implemented)\n- Reduce `maxContextPages` to 2\n- Use fastest search strategy\n\n**Need best quality** (no time constraint)\n- Use `mixtral-8x7b-extended` (when implemented)\n- Increase `maxContextPages` to 10\n- Raise `minScore` to 60+\n\n**Balanced** (1-3s acceptable)\n- Use `llama-3.3-70b-default` ✅ (current default)\n- Default settings work well\n\n### By Traffic Patterns\n\n**High volume, many simple questions**\n- Implement and use `llama-3.1-8b-fast`\n- Consider caching common answers\n- Use rate limiting\n\n**Low volume, complex questions**\n- Use `llama-3.3-70b-default` or `mixtral-8x7b-extended`\n- Quality over speed\n\n**Mixed traffic**\n- Route simple questions to fast strategy\n- Route complex questions to powerful strategy\n- Consider implementing multi-strategy endpoint\n\n## Cost Considerations\n\nApproximate token costs per answer (based on Groq pricing):\n\n### llama-3.3-70b-default\n- Context: ~10k tokens (5 pages × 2k each)\n- Answer: ~500 tokens\n- **Total: ~10.5k tokens per answer**\n\n### llama-3.1-8b-fast (estimated)\n- Context: ~3k tokens (3 pages × 1k each)\n- Answer: ~300 tokens\n- **Total: ~3.3k tokens per answer**\n- **~3x cheaper than 70B**\n\n### mixtral-8x7b-extended (estimated)\n- Context: ~30k tokens (10 pages × 3k each)\n- Answer: ~700 tokens\n- **Total: ~30.7k tokens per answer**\n- **~3x more expensive than default**\n\n### Cost-saving strategies:\n1. Use smaller models for simple questions\n2. Reduce `maxContextPages`\n3. Increase `minScore` (fewer pages in context)\n4. Cache common questions/answers\n5. Implement tiered routing\n\n## Implementation Guide\n\n### Adding a New Strategy\n\n1. **Copy template**:\n ```bash\n cp answer/llama-3.3-70b-default.ts answer/my-strategy.ts\n ```\n\n2. **Modify configuration**:\n ```typescript\n const DEFAULT_MODEL = \"your-model\";\n const DEFAULT_MAX_CONTEXT_PAGES = 3;\n const DEFAULT_TEMPERATURE = 0.5;\n ```\n\n3. **Update strategy metadata**:\n ```typescript\n export const answerStrategy: AnswerStrategy = {\n name: \"my-strategy\",\n description: \"What makes it special\",\n answer: async (query, options) => { /* ... */ }\n };\n ```\n\n4. **Activate**:\n ```typescript\n // answer/index.ts\n import { answerStrategy } from \"./my-strategy.ts\";\n ```\n\n5. **Test**:\n ```bash\n curl \"http://localhost:8000/answer/info\"\n curl \"http://localhost:8000/answer?q=test\"\n ```\n\n### Testing Multiple Strategies\n\nCreate a comparison test script:\n\n```typescript\n// testing/answer-comparison.ts\nimport { questions } from \"./questions.ts\";\n\nconst strategies = [\n \"llama-3.3-70b-default\",\n \"llama-3.1-8b-fast\",\n \"mixtral-8x7b-extended\"\n];\n\nfor (const strategy of strategies) {\n for (const question of questions) {\n // Switch strategy\n // Run question\n // Collect metrics\n }\n}\n// Compare results\n```\n\n## Monitoring and Optimization\n\n### Key Metrics to Track\n\n1. **Response time breakdown**:\n - Search duration\n - Context prep duration\n - LLM duration\n - Total duration\n\n2. **Quality metrics**:\n - User feedback/ratings\n - Answer correctness\n - Relevance of sources\n\n3. **Cost metrics**:\n - Tokens per answer\n - Cost per answer\n - Monthly spend\n\n4. **Usage patterns**:\n - Question types\n - Model distribution\n - Cache hit rates\n\n### Optimization Tips\n\n1. **Speed**: \n - Profile with `enableTiming: true`\n - Optimize slowest component\n - Consider caching\n\n2. **Quality**:\n - Test with real questions\n - Tune system prompts\n - Adjust context size\n\n3. **Cost**:\n - Route by question complexity\n - Cache common answers\n - Use smaller models when possible\n\n## Future Enhancements\n\n### Multi-Strategy Routing\n\nAutomatically select strategy based on question analysis:\n\n```typescript\nasync function smartRoute(question: string) {\n const complexity = analyzeComplexity(question);\n const type = classifyQuestion(question);\n \n if (complexity === \"simple\" && type === \"factual\") {\n return \"llama-3.1-8b-fast\";\n } else if (type === \"code\") {\n return \"llama-3.3-70b-code\";\n } else if (complexity === \"complex\") {\n return \"mixtral-8x7b-extended\";\n }\n return \"llama-3.3-70b-default\";\n}\n```\n\n### Answer Caching\n\nCache answers for common questions:\n\n```typescript\nconst cachedAnswer = await cache.get(questionHash);\nif (cachedAnswer) return cachedAnswer;\n\nconst answer = await generateAnswer(question);\nawait cache.set(questionHash, answer, ttl);\n```\n\n### Follow-up Questions\n\nSupport conversation context:\n\n```typescript\nawait answerQuestion(question, {\n conversationHistory: previousQA,\n maxContextPages: 3 // Reduced to fit history\n});\n```\n\n## Resources\n\n- **Groq Models**: https://console.groq.com/docs/models\n- **Groq Pricing**: https://groq.com/pricing/\n- **Model Benchmarks**: https://console.groq.com/docs/benchmarks\n\n",[],[238,518],"7f6d8314-b9ca-11f0-8f8b-0224a6c84d84","7f6d83c8-b9ca-11f0-8f8b-0224a6c84d84",58,["D",1762293981918],"# Answer Strategies\n\nAnswer strategies implement Retrieval-Augmented Generation (RAG) to answer user questions using documentation search and LLM inference.\n\n## How It Works\n\nEach answer strategy follows this general pattern:\n\n1. **Search**: Use the active search strategy to find relevant documentation pages\n2. **Retrieve**: Get full content from the top N search results\n3. **Format**: Format the documentation as context for the LLM\n4. **Generate**: Call the Groq LLM with the context and user question to generate an answer\n\n## Active Strategy\n\nThe active strategy is selected in `answer/index.ts` by uncommenting the desired import.\n\n### Current Active Strategy\n\n**llama-3.3-70b-default** - Default strategy using Llama 3.3 70B\n- Uses up to 5 documentation pages in context\n- Truncates each page to ~2000 tokens to manage context window\n- Temperature: 0.3 (relatively focused/deterministic)\n- Model: `llama-3.3-70b-versatile`\n\n## Available Strategies\n\n### 1. llama-3.3-70b-default.ts\n\n**Best for**: General purpose Q&A with good balance of quality and speed\n\n- **Model**: `llama-3.3-70b-versatile`\n- **Context**: Up to 5 pages, 2000 tokens each\n- **Temperature**: 0.3 (default)\n- **Performance**: ~1-3s depending on search and LLM response time\n\n**Configuration options**:\n```typescript\n{\n limit: 10, // Search results to consider\n minScore: 0, // Minimum search score\n maxContextPages: 5, // Pages to include in context\n temperature: 0.3, // LLM temperature\n model: \"llama-3.3-70b-versatile\" // Override model\n}\n```\n\n## Creating New Strategies\n\nTo create a new answer strategy:\n\n1. Create a new file in `/answer/` (e.g., `llama-3.1-8b-fast.ts`)\n2. Implement the `AnswerStrategy` interface:\n\n```typescript\nimport type { AnswerStrategy, AnswerResult, AnswerOptions } from \"./types.ts\";\n\nexport const answerStrategy: AnswerStrategy = {\n name: \"your-strategy-name\",\n description: \"Brief description of your strategy\",\n \n answer: async (query: string, options: AnswerOptions = {}): Promise => {\n // Your implementation here\n // 1. Search for relevant pages\n // 2. Get full content\n // 3. Format as context\n // 4. Call LLM\n // 5. Return AnswerResult\n },\n};\n```\n\n3. Add the import to `answer/index.ts` and comment out other strategies\n4. Test your strategy\n\n### Strategy Ideas\n\nHere are some ideas for additional strategies you might want to implement:\n\n- **llama-3.1-8b-fast**: Faster responses with smaller model, fewer pages\n- **mixtral-8x7b-extended**: More context pages with Mixtral's larger context window\n- **llama-3.3-70b-code-focused**: Specialized prompts for code examples and API usage\n- **multi-step-reasoning**: Break down complex questions into sub-questions\n- **citation-mode**: Include specific citations/references in the answer\n- **conversational**: Support follow-up questions with conversation history\n\n## API Usage\n\n### Basic Answer Request\n\n```bash\nGET /answer?q=How+do+I+use+the+Groq+API\n```\n\n### With Options\n\n```bash\nGET /answer?q=What+models+are+available&maxContextPages=3&temperature=0.5\n```\n\n### Get Strategy Info\n\n```bash\nGET /answer/info\n```\n\n## Response Format\n\n```json\n{\n \"answer\": \"markdown formatted answer\",\n \"query\": \"user's original question\",\n \"searchResults\": [\n {\n \"path\": \"api/reference\",\n \"url\": \"https://...\",\n \"title\": \"API Reference\",\n \"score\": 95.2\n }\n ],\n \"contextUsed\": 5,\n \"totalTokens\": 8500,\n \"metadata\": {\n \"strategy\": \"llama-3.3-70b-default\",\n \"model\": \"llama-3.3-70b-versatile\",\n \"temperature\": 0.3,\n \"searchResultsCount\": 10,\n \"timings\": {\n \"search\": 45.2,\n \"contextPrep\": 5.1,\n \"llm\": 1250.3,\n \"total\": 1300.6\n }\n }\n}\n```\n\n## Tuning Your Strategy\n\n### Context Size\n\n- **More pages** = more comprehensive but slower and more expensive\n- **Fewer pages** = faster but might miss relevant info\n- Recommended: 3-5 pages for most questions\n\n### Token Limits\n\n- **Per page**: 1000-3000 tokens is a good range\n- **Total context**: Stay under 32k tokens for most models\n- Llama 3.3 70B supports up to 128k context, but quality may degrade\n\n### Temperature\n\n- **0.0-0.3**: Focused, deterministic answers (good for facts)\n- **0.4-0.7**: Balanced creativity and accuracy\n- **0.8-1.0**: More creative but less consistent (avoid for docs Q&A)\n\n### Search Configuration\n\n- **Higher limit**: More pages to choose from (10-20 recommended)\n- **Higher minScore**: Only use highly relevant pages (50-70 threshold)\n- Balance between precision (fewer, better pages) and recall (more pages, might miss relevant info)\n\n## Best Practices\n\n1. **Match model to task**:\n - Simple factual questions: Smaller/faster models\n - Complex reasoning: Larger models (70B+)\n - Code generation: Models fine-tuned for code\n\n2. **System prompts matter**:\n - Be specific about the documentation domain (Groq API)\n - Set expectations for answer format (markdown, citations, etc.)\n - Provide guidelines for handling missing information\n - Include URL formatting instructions (e.g., \"remove .md extensions\")\n\n3. **Manage context window**:\n - Don't just dump all pages - be selective\n - Truncate long pages intelligently\n - Consider summarizing very long pages first\n\n4. **Error handling**:\n - Gracefully handle search failures\n - Provide fallback responses when LLM fails\n - Include relevant metadata for debugging\n\n5. **Post-processing**:\n - Use `cleanupMarkdownLinks()` to remove .md extensions from generated links\n - Add custom post-processing for your specific needs\n - Validate links and citations if needed\n\n## URL Cleanup\n\nAnswers automatically clean up markdown links to remove `.md` extensions:\n\n```typescript\n// Before cleanup\n[Compound](https://console.groq.com/docs/agentic-tooling/compound-beta.md)\n\n// After cleanup \n[Compound](https://console.groq.com/docs/agentic-tooling/compound-beta)\n```\n\nThis happens through:\n1. **System prompt**: Instructs the LLM to avoid .md extensions\n2. **Post-processing**: `cleanupMarkdownLinks()` function removes any remaining .md extensions\n\nTo disable cleanup (not recommended):\n```typescript\n// In your custom strategy, skip the cleanup:\nreturn {\n answer: llmResponse, // Don't call cleanupMarkdownLinks()\n // ... rest of result\n};\n```\n\n## Testing\n\nTest your strategy with various question types:\n\n- **Factual**: \"What models does Groq support?\"\n- **How-to**: \"How do I authenticate with the API?\"\n- **Comparison**: \"What's the difference between model X and Y?\"\n- **Code examples**: \"Show me an example of streaming responses\"\n- **Complex**: \"How do I implement rate limiting with retries?\"\n\n## Performance Benchmarks\n\nTarget performance for answer strategies:\n\n- **Search**: 50-500ms (depends on active search strategy)\n- **Context prep**: <50ms\n- **LLM call**: 500-3000ms (depends on model and response length)\n- **Total**: 1-4s for most queries\n\nStrategies taking >5s should be optimized or marked as \"detailed\" mode.\n\n",[],[238,192],"7f58d5d6-b9ca-11f0-afaf-0224a6c84d84","QUICK-START.md","7f58d680-b9ca-11f0-afaf-0224a6c84d84",56,["D",1762293981932],"# Answer Strategies - Quick Start Guide\n\nThis guide will help you quickly get started with answer strategies for RAG-based question answering.\n\n## What Are Answer Strategies?\n\nAnswer strategies implement **Retrieval-Augmented Generation (RAG)** - they search documentation, retrieve relevant pages, and use an LLM to generate answers to user questions.\n\n## Quick Test\n\n### 1. Run Tests Locally (Recommended)\n\n```bash\n# Run comprehensive test suite with timing breakdown\ndeno task answer\n```\n\nThis will:\n- Load test questions from `testing/questions.ts`\n- Run each question through the active answer strategy\n- Show detailed timing breakdown (search, context prep, LLM call)\n- Display search results used for context\n- Show generated answers\n- Provide summary statistics\n\n### 2. Test via API Endpoints\n\n#### Start the Server\n\n```bash\ndeno task serve\n```\n\n#### Test Single Questions\n\n```bash\n# Basic question\ncurl \"http://localhost:8000/answer?q=How+do+I+use+the+Groq+API\"\n\n# With options\ncurl \"http://localhost:8000/answer?q=What+models+are+available&maxContextPages=3\"\n\n# Get strategy info\ncurl \"http://localhost:8000/answer/info\"\n\n# Run test suite via API\ncurl \"http://localhost:8000/answer/test\"\n```\n\n## How It Works\n\n### Step-by-Step Process\n\n1. **User asks a question**: \"How do I authenticate with the Groq API?\"\n\n2. **Search** (50-500ms): The active search strategy finds relevant docs\n ```\n → Search query: \"authenticate groq api\"\n → Returns: Top 10 matching pages ranked by relevance\n ```\n\n3. **Retrieve** (fast, from cache): Get full content for top 5 pages\n ```\n → Loads content from latest.json (local cache)\n → Each page limited to ~2000 tokens\n ```\n\n4. **Format** (<50ms): Package docs as LLM context\n ```\n → Formats pages with titles, URLs, and content\n → Adds system prompt with instructions\n → Creates user prompt with question\n ```\n\n5. **Generate** (500-3000ms): LLM generates answer\n ```\n → Calls Groq API with Llama 3.3 70B\n → Returns markdown-formatted answer\n → Includes citations and examples from docs\n ```\n\n## Understanding the Response\n\n```json\n{\n \"answer\": \"To authenticate with the Groq API...\",\n \"query\": \"How do I authenticate with the Groq API?\",\n \"searchResults\": [\n {\n \"path\": \"api/authentication\",\n \"url\": \"https://console.groq.com/docs/api/authentication\",\n \"title\": \"Authentication\",\n \"score\": 92.5\n }\n ],\n \"contextUsed\": 5,\n \"totalTokens\": 8500,\n \"metadata\": {\n \"strategy\": \"llama-3.3-70b-default\",\n \"model\": \"llama-3.3-70b-versatile\",\n \"temperature\": 0.3,\n \"searchResultsCount\": 10,\n \"timings\": {\n \"search\": 45.2,\n \"contextPrep\": 5.1,\n \"llm\": 1250.3,\n \"total\": 1300.6\n }\n }\n}\n```\n\n### Key Fields\n\n- **answer**: The LLM-generated answer (markdown formatted)\n- **query**: Your original question\n- **searchResults**: Top pages used as context (with scores)\n- **contextUsed**: Number of pages actually sent to the LLM\n- **totalTokens**: Approximate tokens in the context\n- **metadata.timings**: Performance breakdown\n\n## Switching Strategies\n\n### Current Strategy\n\nThe active strategy is set in `answer/index.ts`:\n\n```typescript\n// Default: Llama 3.3 70B with up to 5 doc pages\nimport { answerStrategy } from \"./llama-3.3-70b-default.ts\";\n```\n\n### Available Strategies\n\nCurrently available:\n- ✅ **llama-3.3-70b-default**: Balanced quality and speed\n\nComing soon (you can implement these!):\n- ⚡ **llama-3.1-8b-fast**: Faster responses with smaller model\n- 📚 **mixtral-8x7b-extended**: More context with larger model\n- 💻 **llama-3.3-70b-code**: Optimized for code examples\n- 🔗 **citation-mode**: Includes specific citations\n\n### Switching\n\n1. Comment out current strategy in `answer/index.ts`\n2. Uncomment the strategy you want to use\n3. Restart the server\n\n```typescript\n// answer/index.ts\n\n// Comment out default\n// import { answerStrategy } from \"./llama-3.3-70b-default.ts\";\n\n// Use fast strategy instead\nimport { answerStrategy } from \"./llama-3.1-8b-fast.ts\";\n```\n\n## Query Parameters\n\n### Required\n\n- **q**: The question to answer\n ```bash\n ?q=How+do+I+use+streaming\n ```\n\n### Optional\n\n- **limit**: Max search results (default: 10)\n ```bash\n ?limit=20\n ```\n\n- **minScore**: Minimum search score 0-100 (default: 0)\n ```bash\n ?minScore=50\n ```\n\n- **maxContextPages**: Pages to send to LLM (default: 5)\n ```bash\n ?maxContextPages=3\n ```\n\n- **temperature**: LLM temperature 0-1 (default: 0.3)\n ```bash\n ?temperature=0.5\n ```\n\n- **model**: Override the model (optional)\n ```bash\n ?model=llama-3.1-8b-instant\n ```\n\n### Example Combinations\n\n```bash\n# Fast mode: fewer pages, instant model\ncurl \"http://localhost:8000/answer?q=What+is+Groq&maxContextPages=2&model=llama-3.1-8b-instant\"\n\n# High quality: more pages, higher threshold\ncurl \"http://localhost:8000/answer?q=How+to+implement+retry+logic&limit=20&minScore=60&maxContextPages=7\"\n\n# Creative mode: higher temperature\ncurl \"http://localhost:8000/answer?q=Ideas+for+using+Groq+API&temperature=0.8\"\n```\n\n## Common Question Types\n\n### Factual Questions\n\n**Good for**: Quick facts, definitions, model names\n\n```bash\ncurl \"http://localhost:8000/answer?q=What+models+does+Groq+support\"\n```\n\n**Best settings**: Default (5 pages, temp 0.3)\n\n### How-To Questions\n\n**Good for**: Implementation guides, step-by-step instructions\n\n```bash\ncurl \"http://localhost:8000/answer?q=How+do+I+implement+streaming\"\n```\n\n**Best settings**: More pages (6-8), low temp (0.2-0.3)\n\n### Code Examples\n\n**Good for**: Requesting code snippets\n\n```bash\ncurl \"http://localhost:8000/answer?q=Show+me+a+Python+example+of+calling+the+API\"\n```\n\n**Best settings**: Code-focused strategy (when available), temp 0.3\n\n### Comparison Questions\n\n**Good for**: Comparing models, features, approaches\n\n```bash\ncurl \"http://localhost:8000/answer?q=Difference+between+Llama+3.1+and+3.3\"\n```\n\n**Best settings**: More pages (7-10), temp 0.3\n\n### Complex Questions\n\n**Good for**: Multi-part questions requiring reasoning\n\n```bash\ncurl \"http://localhost:8000/answer?q=How+to+build+a+chatbot+with+streaming+and+conversation+history\"\n```\n\n**Best settings**: More pages (8-10), 70B+ model, temp 0.4\n\n## Troubleshooting\n\n### No Answer Generated\n\n**Problem**: Returns error or empty answer\n\n**Possible causes**:\n- No search results found\n- GROQ_API_KEY not set\n- Rate limit exceeded\n\n**Solution**:\n```bash\n# Check if search works\ncurl \"http://localhost:8000/search?q=your+question\"\n\n# Verify API key\necho $GROQ_API_KEY\n\n# Check strategy info\ncurl \"http://localhost:8000/answer/info\"\n```\n\n### Poor Answer Quality\n\n**Problem**: Answer is vague, incorrect, or unhelpful\n\n**Possible causes**:\n- Search results not relevant\n- Not enough context (too few pages)\n- Temperature too high\n\n**Solutions**:\n```bash\n# Check search results first\ncurl \"http://localhost:8000/search?q=your+question&limit=10\"\n\n# Increase context\ncurl \"http://localhost:8000/answer?q=your+question&maxContextPages=8\"\n\n# Lower temperature\ncurl \"http://localhost:8000/answer?q=your+question&temperature=0.2\"\n\n# Increase search quality threshold\ncurl \"http://localhost:8000/answer?q=your+question&minScore=60\"\n```\n\n### Slow Responses\n\n**Problem**: Takes >5 seconds to respond\n\n**Possible causes**:\n- Slow search strategy\n- Too many pages in context\n- Large model\n\n**Solutions**:\n```bash\n# Use fewer pages\ncurl \"http://localhost:8000/answer?q=your+question&maxContextPages=3\"\n\n# Switch to faster model\ncurl \"http://localhost:8000/answer?q=your+question&model=llama-3.1-8b-instant\"\n\n# Check timing breakdown in response metadata\n```\n\n## Creating Your Own Strategy\n\n### 1. Copy the Template\n\n```bash\ncp answer/llama-3.3-70b-default.ts answer/my-custom-strategy.ts\n```\n\n### 2. Modify Configuration\n\n```typescript\n// Change these constants\nconst DEFAULT_MODEL = \"your-preferred-model\";\nconst DEFAULT_MAX_CONTEXT_PAGES = 3; // Adjust as needed\nconst DEFAULT_TEMPERATURE = 0.5;\nconst MAX_TOKENS_PER_PAGE = 1000; // Adjust as needed\n```\n\n### 3. Customize System Prompt (Optional)\n\n```typescript\nimport { DEFAULT_SYSTEM_PROMPT } from \"./utils.ts\";\n\n// Or create your own:\nconst CUSTOM_SYSTEM_PROMPT = `You are a helpful assistant that...`;\n```\n\n### 4. Update Strategy Info\n\n```typescript\nexport const answerStrategy: AnswerStrategy = {\n name: \"my-custom-strategy\",\n description: \"What makes your strategy special\",\n answer: async (query: string, options: AnswerOptions = {}) => {\n // Your implementation\n },\n};\n```\n\n### 5. Activate Your Strategy\n\nEdit `answer/index.ts`:\n\n```typescript\nimport { answerStrategy } from \"./my-custom-strategy.ts\";\n```\n\n### 6. Test It\n\n```bash\ncurl \"http://localhost:8000/answer/info\" # Verify active strategy\ncurl \"http://localhost:8000/answer?q=test+question\"\n```\n\n## Performance Tips\n\n### Optimize for Speed\n\n1. Use faster models (8B instead of 70B)\n2. Reduce context pages (2-3 instead of 5)\n3. Use faster search strategy\n4. Lower token limits per page\n\n### Optimize for Quality\n\n1. Use larger models (70B+)\n2. Increase context pages (7-10)\n3. Use higher search score thresholds\n4. Include more search results to choose from\n\n### Balance Both\n\n1. Start with default (5 pages, 70B, temp 0.3)\n2. Monitor timing breakdown in responses\n3. Adjust based on question complexity\n4. Cache common questions (future enhancement)\n\n## Next Steps\n\n1. ✅ Test the default strategy\n2. ✅ Try different query parameters\n3. ✅ Check the README.md for more details\n4. ⚡ Create a fast strategy for simple questions\n5. 📚 Create an extended strategy for complex questions\n6. 🔧 Customize system prompts for your use case\n7. 📊 Build a testing harness (like `/search/test`)\n\n## Resources\n\n- **Search Strategies**: See `/search/` for how document retrieval works\n- **Groq Models**: https://console.groq.com/docs/models\n- **API Reference**: https://console.groq.com/docs/api-reference\n\n## Questions?\n\nCheck out:\n- `answer/README.md` - Detailed documentation\n- `answer/types.ts` - Type definitions\n- `answer/utils.ts` - Helper functions\n- `main.tsx` - How endpoints are implemented\n\n",[],[238,753],"7f40d292-b9ca-11f0-8b68-0224a6c84d84",".gitkeep","7f40d350-b9ca-11f0-8b68-0224a6c84d84",53,["D",1762293981787],"# Answer Strategies Directory\n\nThis folder contains RAG (Retrieval-Augmented Generation) strategies for answering questions about Groq documentation.\n\nSee README.md and QUICK-START.md for documentation.\n\n",[],[238,761],["D",1762293980863],[254,255,256,257,258,259,260,261,262],[238],["D",1746217442601],{"_97":96,"_104":100},{"_97":96,"_98":-5,"_99":100,"_18":56,"_101":102,"_103":25},{"_13":86,"_32":87,"_44":43,"_88":45,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":-5,"_94":46,"_95":96},"ctx",{"_79":-5,"_124":808,"_809":810,"_811":812,"_813":25,"_814":25,"_815":25,"_816":50,"_817":-5,"_818":-7,"_819":-7,"_820":63,"_821":25,"_822":25,"_823":-5,"_824":25,"_825":25,"_826":25,"_827":-5},"ENV",{"_779":780,"_781":782,"_783":87,"_784":785,"_786":787,"_788":789,"_790":791,"_792":793,"_794":795,"_796":797,"_798":799,"_800":801,"_802":803,"_804":805,"_806":807},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v96