[{"_1":2,"_180":-5,"_181":-5},"loaderData",{"_3":4,"_93":94},"root",{"_5":6,"_36":37,"_66":67},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":-5,"_32":33,"_34":20,"_35":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesChatPanelState","notificationCount",0,"flagDenols","isTeamOrgMember","ENV",{"_38":39,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":57,"_58":59,"_60":61,"_62":63,"_64":65},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","e9e636d7b466d0cb012ac9b89f74068c1d9c5525","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","API_URL","https://api.val.town","clerkState",{"_68":69},"__internal_clerk_state",{"_70":71,"_72":47,"_73":74,"_75":74,"_76":20,"_77":74,"_78":74,"_79":74,"_80":74,"_81":82,"_91":20,"_92":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_83":20,"_19":20,"_84":74,"_85":86,"_87":74,"_88":47,"_89":20,"_90":74},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_95":96,"_97":98,"_99":100,"_101":102,"_157":158,"_177":178,"_179":-5},"handle","kamenxrider","name","slimarmor","path","vectordb.ts","project",{"_103":104,"_97":98,"_105":106,"_107":-5,"_108":20,"_109":110,"_111":112,"_113":-5,"_12":114,"_115":116,"_117":20,"_118":-5,"_119":-5,"_120":33,"_14":121,"_122":123,"_124":125,"_126":127,"_128":127,"_129":130,"_131":130,"_132":130,"_133":127,"_134":127,"_23":135,"_138":139,"_143":144,"_156":20},"id","bb3f3862-ffd7-11f0-977c-42dde27851f2","createdAt",["D",1769996247298],"deletedAt","legacyEnvVars","privacy","public","description","Semantic vector DB on Val Town SQLite — DiskANN, hybrid search","imageUrl","d4de366f-8281-484e-8856-3824b6bdfb0b","pinnedFileId","95fbe492-ffe1-11f0-bee9-42dde27851f2","pinPreview","pinMigrated","screenshotVersion","likeCount","9bb2e2b6-8aa1-4b6a-b4ef-ad7adab2b868","tursoName","bb3f3862ffd711f0977c42dde27851f2","tursoHostname","bb3f3862ffd711f0977c42dde27851f2-val-town.aws-us-east-2.turso.io","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_95":96,"_136":137},"profileImageUrl","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18ybFJkNm8zSzFzbGE5RzhsTUt3OGJMZFdGZFkifQ","org",{"_95":96,"_140":137,"_12":114,"_141":142},"avatar","tier","pro","branch",{"_145":146,"_97":45,"_103":147,"_148":104,"_149":-5,"_150":-5,"_151":-5,"_152":-5,"_153":-5,"_154":98,"_155":96},"version",96,"bb3f8592-ffd7-11f0-977c-42dde27851f2","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_103":159,"_97":100,"_160":161,"_162":-5,"_163":147,"_164":165,"_166":167,"_105":168,"_145":169,"_170":171,"_172":173,"_174":20,"_175":-5,"_99":176},"440ce362-15d4-11f1-8b72-42dde27851f2","fileId","85b83e64-ffe1-11f0-a01c-42dde27851f2","parentId","branchId","branchVersion",78,"latest",true,["D",1772413684396],6,"fileType","script","content","/**\n * Mini Vector DB for Val Town (v3 - Multi-Provider)\n * \n * Supports any OpenAI-compatible embedding provider:\n * - Nebius (default)\n * - OpenAI\n * - OpenRouter\n * - Azure OpenAI\n * - Local (Ollama, LM Studio)\n * - Any OpenAI-compatible API\n * \n * Configure via environment variables:\n * - EMBEDDING_API_URL (default: Nebius)\n * - EMBEDDING_API_KEY (required)\n * - EMBEDDING_MODEL (default: Qwen/Qwen3-Embedding-8B)\n * - EMBEDDING_DIM (default: 4096)\n */\n\nimport sqlite from \"https://esm.town/v/std/sqlite/main.ts\";\n\n// ============================================================\n// CONFIGURATION (via environment variables)\n// ============================================================\n\n// Provider presets\nconst PROVIDERS = {\n nebius: {\n url: \"https://api.tokenfactory.nebius.com/v1/embeddings\",\n keyEnv: \"NEBIUS_API_KEY\",\n defaultModel: \"Qwen/Qwen3-Embedding-8B\",\n defaultDim: 4096,\n },\n openai: {\n url: \"https://api.openai.com/v1/embeddings\",\n keyEnv: \"OPENAI_API_KEY\",\n defaultModel: \"text-embedding-3-small\",\n defaultDim: 1536,\n },\n openrouter: {\n url: \"https://openrouter.ai/api/v1/embeddings\",\n keyEnv: \"OPENROUTER_API_KEY\",\n defaultModel: \"openai/text-embedding-3-small\",\n defaultDim: 1536,\n },\n} as const;\n\ntype ProviderName = keyof typeof PROVIDERS;\n\n// Table/index names\nconst TABLE_NAME = \"vectordb\";\nconst INDEX_NAME = \"vectordb_embedding_idx\";\nconst FTS_TABLE = \"vectordb_fts\";\nconst META_TABLE = \"vectordb_meta\";\n\nlet RESOLVED_DIM: number | null = null;\nlet FTS_AVAILABLE: boolean | null = null;\n\n// Fix #2: Module-level setup guard — skips redundant SQLite round-trips on warm requests\nlet _setupDone = false;\n\n// Get configuration from environment\nfunction getConfig() {\n // Check for explicit provider setting\n const providerName = (Deno.env.get(\"EMBEDDING_PROVIDER\") || \"nebius\") as ProviderName;\n const provider = PROVIDERS[providerName] || PROVIDERS.nebius;\n \n // Allow overrides via env vars\n const url = Deno.env.get(\"EMBEDDING_API_URL\") || provider.url;\n const model = Deno.env.get(\"EMBEDDING_MODEL\") || provider.defaultModel;\n const rawDim = Deno.env.get(\"EMBEDDING_DIM\");\n const dimAuto = (rawDim || \"\").toLowerCase() === \"auto\";\n const parsedDim = parseInt(rawDim || String(provider.defaultDim), 10);\n const dim = RESOLVED_DIM ?? (Number.isFinite(parsedDim) && parsedDim > 0 ? parsedDim : provider.defaultDim);\n \n // Get API key (check provider-specific first, then generic)\n const apiKey = Deno.env.get(provider.keyEnv) || Deno.env.get(\"EMBEDDING_API_KEY\");\n \n return { url, model, dim, dimAuto, apiKey, providerName };\n}\n\nfunction getIndexConfig() {\n const metricRaw = (Deno.env.get(\"INDEX_METRIC\") || \"\").toLowerCase();\n const metric = metricRaw === \"l2\" ? \"l2\" : \"cosine\";\n const maxNeighborsRaw = parseInt(Deno.env.get(\"INDEX_MAX_NEIGHBORS\") || \"64\", 10);\n const maxNeighbors = Number.isFinite(maxNeighborsRaw) && maxNeighborsRaw >= 8 && maxNeighborsRaw <= 256\n ? maxNeighborsRaw\n : 64;\n const compressRaw = (Deno.env.get(\"INDEX_COMPRESS_NEIGHBORS\") || \"\").toLowerCase();\n const compressNeighbors = (\n compressRaw === \"none\" ||\n compressRaw === \"float1bit\" ||\n compressRaw === \"float8\" ||\n compressRaw === \"float16\" ||\n compressRaw === \"floatb16\" ||\n compressRaw === \"float32\"\n )\n ? compressRaw\n : \"float8\";\n\n const alphaRaw = Deno.env.get(\"INDEX_ALPHA\");\n const alphaParsed = alphaRaw ? parseFloat(alphaRaw) : 1.2;\n const alpha = Number.isFinite(alphaParsed) && alphaParsed >= 1 ? alphaParsed : 1.2;\n\n const searchLRaw = Deno.env.get(\"INDEX_SEARCH_L\");\n const searchLParsed = searchLRaw ? parseInt(searchLRaw, 10) : 200;\n const searchL = Number.isFinite(searchLParsed) && searchLParsed > 0 ? searchLParsed : 200;\n\n const insertLRaw = Deno.env.get(\"INDEX_INSERT_L\");\n const insertLParsed = insertLRaw ? parseInt(insertLRaw, 10) : 70;\n const insertL = Number.isFinite(insertLParsed) && insertLParsed > 0 ? insertLParsed : 70;\n\n return {\n metric,\n maxNeighbors,\n compressNeighbors,\n alpha,\n searchL,\n insertL,\n hasAlpha: !!alphaRaw,\n hasSearchL: !!searchLRaw,\n hasInsertL: !!insertLRaw,\n };\n}\n\n// ============================================================\n// TYPES\n// ============================================================\n\nexport interface VectorRecord {\n id: string;\n text: string;\n meta?: Record;\n updated_at: number;\n}\n\nexport interface SearchResult {\n id: string;\n text: string;\n meta?: Record;\n distance: number;\n score?: number;\n keyword_score?: number;\n}\n\nexport interface Stats {\n count: number;\n estimated_storage_mb: number;\n}\n\nexport interface ProviderInfo {\n provider: string;\n url: string;\n model: string;\n dimensions: number;\n dim_auto: boolean;\n}\n\nexport interface IndexInfo {\n metric: string;\n max_neighbors: number;\n compress_neighbors: string;\n alpha: number;\n search_l: number;\n insert_l: number;\n}\n\nexport interface EmbeddingAuthInfo {\n required_env: string;\n has_key: boolean;\n}\n\nexport type FilterValue = string | number | boolean | null | Array;\n\nexport interface SearchOptions {\n filters?: Record;\n offset?: number;\n hybrid?: {\n enabled?: boolean;\n alpha?: number; // 0..1, keyword weight\n keyword_k?: number;\n };\n}\n\nexport interface UpsertInput {\n id: string;\n text: string;\n meta?: Record;\n}\n\nexport interface ChunkOptions {\n size?: number; // approx chars per chunk\n overlap?: number; // approx chars overlap\n}\n\n// ============================================================\n// GUARDRAILS\n// ============================================================\n\nfunction assertDim(vec: number[], expected: number): void {\n if (vec.length !== expected) {\n throw new Error(`Embedding dim mismatch: got ${vec.length}, expected ${expected}`);\n }\n}\n\nasync function hashText(text: string): Promise {\n const encoder = new TextEncoder();\n const data = encoder.encode(text);\n const hashBuffer = await crypto.subtle.digest(\"SHA-256\", data);\n const hashArray = Array.from(new Uint8Array(hashBuffer));\n return hashArray.map(b => b.toString(16).padStart(2, \"0\")).join(\"\");\n}\n\n// ============================================================\n// EMBEDDING CLIENT (OpenAI-Compatible)\n// ============================================================\n\ninterface EmbeddingResponse {\n data: Array<{ embedding: number[]; index: number }>;\n model: string;\n usage?: { prompt_tokens: number; total_tokens: number };\n}\n\nfunction getExpectedDim(config: ReturnType): number | null {\n if (RESOLVED_DIM) return RESOLVED_DIM;\n if (config.dimAuto) return null;\n return config.dim;\n}\n\nasync function embedManyRaw(inputs: string[], config = getConfig()): Promise {\n if (!config.apiKey) {\n throw new Error(`Missing API key. Set ${PROVIDERS[config.providerName]?.keyEnv || \"EMBEDDING_API_KEY\"} env var`);\n }\n\n const controller = new AbortController();\n const timeoutId = setTimeout(() => controller.abort(), 30000);\n\n try {\n const res = await fetch(config.url, {\n method: \"POST\",\n headers: {\n \"Authorization\": `Bearer ${config.apiKey}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ \n model: config.model, \n input: inputs,\n }),\n signal: controller.signal,\n });\n\n if (!res.ok) {\n const errText = await res.text();\n throw new Error(`Embedding API error ${res.status}: ${errText}`);\n }\n\n const json = (await res.json()) as EmbeddingResponse;\n const embeddings: Array = new Array(inputs.length);\n for (const item of json.data) {\n embeddings[item.index] = item.embedding;\n }\n\n if (embeddings.some((e) => !e)) {\n throw new Error(\"Embedding API returned incomplete data\");\n }\n\n return embeddings as number[][];\n } finally {\n clearTimeout(timeoutId);\n }\n}\n\nasync function embedMany(inputs: string[]): Promise {\n const config = getConfig();\n const expectedDim = getExpectedDim(config);\n const batchSize = 96;\n const result: number[][] = [];\n\n for (let i = 0; i < inputs.length; i += batchSize) {\n const batch = inputs.slice(i, i + batchSize);\n const embeddings = await embedManyRaw(batch, config);\n if (expectedDim !== null) {\n for (const emb of embeddings) assertDim(emb, expectedDim);\n }\n result.push(...embeddings);\n }\n\n return result;\n}\n\nasync function embed(input: string): Promise {\n const [embedding] = await embedMany([input]);\n return embedding;\n}\n\n// ============================================================\n// VECTOR DB API\n// ============================================================\n\nasync function ensureFtsTable(): Promise {\n if (FTS_AVAILABLE !== null) return;\n try {\n await sqlite.execute(`CREATE VIRTUAL TABLE IF NOT EXISTS ${FTS_TABLE} USING fts5(id, text)`);\n FTS_AVAILABLE = true;\n } catch {\n FTS_AVAILABLE = false;\n }\n}\n\nasync function resolveDimForSetup(): Promise {\n const config = getConfig();\n if (!config.dimAuto) return config.dim;\n if (RESOLVED_DIM) return RESOLVED_DIM;\n\n await sqlite.execute(`\n CREATE TABLE IF NOT EXISTS ${META_TABLE} (\n key TEXT PRIMARY KEY,\n value TEXT\n )\n `);\n\n const existing = await sqlite.execute({\n sql: `SELECT value FROM ${META_TABLE} WHERE key = ?`,\n args: [\"dim\"],\n });\n\n const stored = (existing.rows?.[0] as { value?: string } | undefined)?.value;\n const parsed = stored ? parseInt(stored, 10) : NaN;\n if (Number.isFinite(parsed) && parsed > 0) {\n RESOLVED_DIM = parsed;\n return parsed;\n }\n\n // Try to read dimension from existing table schema (if present)\n const tableInfo = await sqlite.execute(`PRAGMA table_info(${TABLE_NAME})`);\n const embeddingCol = (tableInfo.rows as Array<{ name: string; type: string }>).find(r => r.name === \"embedding\");\n if (embeddingCol && typeof embeddingCol.type === \"string\") {\n const match = embeddingCol.type.match(/F32_BLOB\\((\\d+)\\)/i);\n if (match) {\n const detected = parseInt(match[1], 10);\n if (Number.isFinite(detected) && detected > 0) {\n RESOLVED_DIM = detected;\n await sqlite.execute({\n sql: `INSERT OR REPLACE INTO ${META_TABLE} (key, value) VALUES (?, ?)`,\n args: [\"dim\", String(RESOLVED_DIM)],\n });\n return RESOLVED_DIM;\n }\n }\n }\n\n const [embedding] = await embedManyRaw([\"dimension probe\"], config);\n RESOLVED_DIM = embedding.length;\n\n await sqlite.execute({\n sql: `INSERT OR REPLACE INTO ${META_TABLE} (key, value) VALUES (?, ?)`,\n args: [\"dim\", String(RESOLVED_DIM)],\n });\n\n return RESOLVED_DIM;\n}\n\n// Fix #5: syncFtsUpsert uses a single batch instead of 2 sequential queries\nasync function syncFtsUpsert(id: string, text: string): Promise {\n if (FTS_AVAILABLE === null) await ensureFtsTable();\n if (!FTS_AVAILABLE) return;\n try {\n await sqlite.batch([\n { sql: `DELETE FROM ${FTS_TABLE} WHERE id = ?`, args: [id] },\n { sql: `INSERT INTO ${FTS_TABLE} (id, text) VALUES (?, ?)`, args: [id, text] },\n ]);\n } catch {\n // ignore FTS failures to keep core functionality working\n }\n}\n\nasync function syncFtsDelete(id: string): Promise {\n if (FTS_AVAILABLE === null) await ensureFtsTable();\n if (!FTS_AVAILABLE) return;\n try {\n await sqlite.execute({\n sql: `DELETE FROM ${FTS_TABLE} WHERE id = ?`,\n args: [id],\n });\n } catch {\n // ignore FTS failures to keep core functionality working\n }\n}\n\n/**\n * Get current provider configuration\n */\nexport function getProviderInfo(): ProviderInfo {\n const config = getConfig();\n return {\n provider: config.providerName,\n url: config.url,\n model: config.model,\n dimensions: config.dim,\n dim_auto: config.dimAuto,\n };\n}\n\nexport function getIndexInfo(): IndexInfo {\n const cfg = getIndexConfig();\n return {\n metric: cfg.metric,\n max_neighbors: cfg.maxNeighbors,\n compress_neighbors: cfg.compressNeighbors,\n alpha: cfg.alpha,\n search_l: cfg.searchL,\n insert_l: cfg.insertL,\n };\n}\n\nexport function getEmbeddingAuthInfo(): EmbeddingAuthInfo {\n const config = getConfig();\n const requiredEnv = PROVIDERS[config.providerName]?.keyEnv || \"EMBEDDING_API_KEY\";\n return {\n required_env: requiredEnv,\n has_key: !!config.apiKey,\n };\n}\n\n/**\n * Setup: Create table and optimized index\n * Automatically uses the configured embedding dimension.\n * Fix #2: Skips all SQLite work on subsequent warm calls via _setupDone guard.\n */\nexport async function setup(): Promise {\n if (_setupDone) return;\n\n const indexConfig = getIndexConfig();\n const dim = await resolveDimForSetup();\n\n await sqlite.execute(`\n CREATE TABLE IF NOT EXISTS ${META_TABLE} (\n key TEXT PRIMARY KEY,\n value TEXT\n )\n `);\n \n await sqlite.execute(`\n CREATE TABLE IF NOT EXISTS ${TABLE_NAME} (\n id TEXT PRIMARY KEY,\n text TEXT NOT NULL,\n text_hash TEXT NOT NULL,\n embedding F32_BLOB(${dim}),\n meta_json TEXT,\n updated_at INTEGER NOT NULL\n )\n `);\n\n await ensureFtsTable();\n\n try {\n const idxArgs = [\n `metric=${indexConfig.metric}`,\n `max_neighbors=${indexConfig.maxNeighbors}`,\n ];\n if (indexConfig.compressNeighbors !== \"none\") {\n idxArgs.push(`compress_neighbors=${indexConfig.compressNeighbors}`);\n }\n if (indexConfig.hasAlpha) idxArgs.push(`alpha=${indexConfig.alpha}`);\n if (indexConfig.hasSearchL) idxArgs.push(`search_l=${indexConfig.searchL}`);\n if (indexConfig.hasInsertL) idxArgs.push(`insert_l=${indexConfig.insertL}`);\n const idxArgsSql = idxArgs.map((s) => `'${s}'`).join(\", \");\n\n await sqlite.execute(`\n CREATE INDEX ${INDEX_NAME} \n ON ${TABLE_NAME} (libsql_vector_idx(embedding, ${idxArgsSql}))\n `);\n } catch (e) {\n const msg = e instanceof Error ? e.message : String(e);\n if (!msg.includes(\"already exists\")) {\n throw e;\n }\n }\n\n _setupDone = true;\n}\n\n/**\n * Recreate index with optimized settings\n */\nexport async function reindex(): Promise<{ success: boolean; message: string }> {\n try {\n const indexConfig = getIndexConfig();\n await sqlite.execute(`DROP INDEX IF EXISTS ${INDEX_NAME}`);\n\n const idxArgs = [\n `metric=${indexConfig.metric}`,\n `max_neighbors=${indexConfig.maxNeighbors}`,\n ];\n if (indexConfig.compressNeighbors !== \"none\") {\n idxArgs.push(`compress_neighbors=${indexConfig.compressNeighbors}`);\n }\n if (indexConfig.hasAlpha) idxArgs.push(`alpha=${indexConfig.alpha}`);\n if (indexConfig.hasSearchL) idxArgs.push(`search_l=${indexConfig.searchL}`);\n if (indexConfig.hasInsertL) idxArgs.push(`insert_l=${indexConfig.insertL}`);\n const idxArgsSql = idxArgs.map((s) => `'${s}'`).join(\", \");\n\n await sqlite.execute(`\n CREATE INDEX ${INDEX_NAME} \n ON ${TABLE_NAME} (libsql_vector_idx(embedding, ${idxArgsSql}))\n `);\n return { success: true, message: \"Index recreated with optimized settings\" };\n } catch (e) {\n const msg = e instanceof Error ? e.message : String(e);\n return { success: false, message: msg };\n }\n}\n\n/**\n * Upsert: Insert or update a record\n */\nexport async function upsert(\n id: string, \n text: string, \n meta?: Record\n): Promise<{ embedded: boolean }> {\n const newHash = await hashText(text);\n const metaJson = meta ? JSON.stringify(meta) : null;\n const now = Date.now();\n\n const existing = await sqlite.execute({\n sql: `SELECT text_hash FROM ${TABLE_NAME} WHERE id = ?`,\n args: [id],\n });\n\n const existingHash = (existing.rows?.[0] as { text_hash: string } | undefined)?.text_hash;\n\n if (existingHash === newHash) {\n await sqlite.execute({\n sql: `UPDATE ${TABLE_NAME} SET meta_json = ?, updated_at = ? WHERE id = ?`,\n args: [metaJson, now, id],\n });\n return { embedded: false };\n }\n\n const embedding = await embed(text);\n\n await sqlite.execute({\n sql: `\n INSERT INTO ${TABLE_NAME} (id, text, text_hash, embedding, meta_json, updated_at)\n VALUES (?, ?, ?, vector32(?), ?, ?)\n ON CONFLICT(id) DO UPDATE SET\n text = excluded.text,\n text_hash = excluded.text_hash,\n embedding = excluded.embedding,\n meta_json = excluded.meta_json,\n updated_at = excluded.updated_at\n `,\n args: [id, text, newHash, JSON.stringify(embedding), metaJson, now],\n });\n\n await syncFtsUpsert(id, text);\n\n return { embedded: true };\n}\n\n/**\n * Batch upsert: Insert or update many records\n * Fix #1: Uses sqlite.batch() instead of per-record loops for both embed and skip paths.\n * Fix #5: FTS syncs are batched alongside main upserts in one round-trip.\n */\nexport async function upsertMany(\n records: UpsertInput[]\n): Promise<{ embedded: number; skipped: number; results: Array<{ id: string; embedded: boolean }> }> {\n if (records.length === 0) return { embedded: 0, skipped: 0, results: [] };\n\n const ids = records.map((r) => r.id);\n const placeholders = ids.map(() => \"?\").join(\",\");\n const existing = await sqlite.execute({\n sql: `SELECT id, text_hash FROM ${TABLE_NAME} WHERE id IN (${placeholders})`,\n args: ids,\n });\n\n const existingMap = new Map();\n for (const row of existing.rows as Array<{ id: string; text_hash: string }>) {\n existingMap.set(row.id, row.text_hash);\n }\n\n const hashes = await Promise.all(records.map((r) => hashText(r.text)));\n const toEmbed: Array<{ record: UpsertInput; hash: string }> = [];\n const toSkip: Array<{ record: UpsertInput; hash: string }> = [];\n\n records.forEach((record, idx) => {\n const newHash = hashes[idx];\n const existingHash = existingMap.get(record.id);\n if (existingHash === newHash) {\n toSkip.push({ record, hash: newHash });\n } else {\n toEmbed.push({ record, hash: newHash });\n }\n });\n\n const results: Array<{ id: string; embedded: boolean }> = [];\n const now = Date.now();\n\n // Fix #1a: Batch all meta-only updates in one round-trip\n if (toSkip.length > 0) {\n const skipStatements = toSkip.map(({ record }) => ({\n sql: `UPDATE ${TABLE_NAME} SET meta_json = ?, updated_at = ? WHERE id = ?`,\n args: [record.meta ? JSON.stringify(record.meta) : null, now, record.id],\n }));\n await sqlite.batch(skipStatements);\n for (const { record } of toSkip) {\n results.push({ id: record.id, embedded: false });\n }\n }\n\n // Fix #1b: Batch all embed upserts + FTS syncs in one round-trip\n if (toEmbed.length > 0) {\n const texts = toEmbed.map((item) => item.record.text);\n const embeddings = await embedMany(texts);\n\n const upsertStatements = toEmbed.map(({ record, hash }, i) => ({\n sql: `\n INSERT INTO ${TABLE_NAME} (id, text, text_hash, embedding, meta_json, updated_at)\n VALUES (?, ?, ?, vector32(?), ?, ?)\n ON CONFLICT(id) DO UPDATE SET\n text = excluded.text,\n text_hash = excluded.text_hash,\n embedding = excluded.embedding,\n meta_json = excluded.meta_json,\n updated_at = excluded.updated_at\n `,\n args: [\n record.id,\n record.text,\n hash,\n JSON.stringify(embeddings[i]),\n record.meta ? JSON.stringify(record.meta) : null,\n now,\n ],\n }));\n\n // FTS sync statements batched alongside upserts (Fix #5)\n if (FTS_AVAILABLE === null) await ensureFtsTable();\n const ftsStatements = FTS_AVAILABLE\n ? toEmbed.flatMap(({ record }) => [\n { sql: `DELETE FROM ${FTS_TABLE} WHERE id = ?`, args: [record.id] },\n { sql: `INSERT INTO ${FTS_TABLE} (id, text) VALUES (?, ?)`, args: [record.id, record.text] },\n ])\n : [];\n\n await sqlite.batch([...upsertStatements, ...ftsStatements]);\n\n for (const { record } of toEmbed) {\n results.push({ id: record.id, embedded: true });\n }\n }\n\n return {\n embedded: toEmbed.length,\n skipped: toSkip.length,\n results,\n };\n}\n\n/**\n * Simple chunking helper (approx chars)\n */\nexport function chunkText(text: string, options: ChunkOptions = {}): string[] {\n const size = Math.max(200, options.size ?? 800);\n const overlap = Math.max(0, options.overlap ?? 100);\n if (text.length <= size) return [text];\n\n const chunks: string[] = [];\n let start = 0;\n while (start < text.length) {\n const end = Math.min(text.length, start + size);\n let chunk = text.slice(start, end);\n\n // Try to break on whitespace for nicer chunks\n const lastSpace = chunk.lastIndexOf(\" \");\n if (lastSpace > size * 0.6) {\n chunk = chunk.slice(0, lastSpace);\n }\n\n chunks.push(chunk.trim());\n start += Math.max(chunk.length - overlap, 1);\n }\n\n return chunks.filter((c) => c.length > 0);\n}\n\n/**\n * Chunked upsert: splits text and upserts as id::chunkN\n */\nexport async function upsertChunked(\n id: string,\n text: string,\n meta?: Record,\n options: ChunkOptions = {}\n): Promise<{ chunks: number; embedded: number; skipped: number }> {\n const chunks = chunkText(text, options);\n const records: UpsertInput[] = chunks.map((chunk, i) => ({\n id: `${id}::chunk${i + 1}`,\n text: chunk,\n meta: {\n ...(meta || {}),\n parent_id: id,\n chunk_index: i + 1,\n chunk_total: chunks.length,\n },\n }));\n\n const result = await upsertMany(records);\n return { chunks: chunks.length, embedded: result.embedded, skipped: result.skipped };\n}\n\n/**\n * Search: Semantic search using vector similarity\n */\nfunction buildFilterClause(filters?: Record): { clause: string; args: Array } {\n if (!filters || Object.keys(filters).length === 0) return { clause: \"\", args: [] };\n\n const clauses: string[] = [];\n const args: Array = [];\n\n for (const [key, value] of Object.entries(filters)) {\n const safeKey = key.replace(/[^a-zA-Z0-9_]/g, \"\");\n if (!safeKey || safeKey !== key) {\n throw new Error(`Invalid filter key: ${key}`);\n }\n\n const path = `$.${safeKey}`;\n\n if (Array.isArray(value)) {\n if (value.length === 0) continue;\n const placeholders = value.map(() => \"?\").join(\",\");\n clauses.push(`json_extract(t.meta_json, ?) IN (${placeholders})`);\n args.push(path, ...value);\n } else if (value === null) {\n clauses.push(`json_extract(t.meta_json, ?) IS NULL`);\n args.push(path);\n } else {\n clauses.push(`json_extract(t.meta_json, ?) = ?`);\n args.push(path, value);\n }\n }\n\n if (clauses.length === 0) return { clause: \"\", args: [] };\n return { clause: `WHERE ${clauses.join(\" AND \")}`, args };\n}\n\nasync function getKeywordScores(query: string, limit: number): Promise> {\n if (FTS_AVAILABLE === null) await ensureFtsTable();\n if (!FTS_AVAILABLE) return new Map();\n\n try {\n const res = await sqlite.execute({\n sql: `SELECT id, bm25(${FTS_TABLE}) as score FROM ${FTS_TABLE} WHERE ${FTS_TABLE} MATCH ? ORDER BY score LIMIT ?`,\n args: [query, limit],\n });\n\n const map = new Map();\n for (const row of res.rows as Array<{ id: string; score: number }>) {\n const raw = typeof row.score === \"number\" ? row.score : Number(row.score);\n const clamped = Math.max(0, raw);\n const score = 1 / (1 + clamped);\n map.set(row.id, score);\n }\n return map;\n } catch {\n return new Map();\n }\n}\n\nexport async function search(\n query: string, \n k: number = 10,\n maxDistance?: number,\n options: SearchOptions = {}\n): Promise {\n const offset = Math.max(0, options.offset ?? 0);\n const filters = options.filters;\n const { clause, args: filterArgs } = buildFilterClause(filters);\n\n const queryEmbedding = await embed(query);\n const queryVecJson = JSON.stringify(queryEmbedding);\n const internalK = Math.min((k + offset) * (filters ? 3 : 1), 1000);\n\n const result = await sqlite.execute({\n sql: `\n SELECT \n t.id, \n t.text, \n t.meta_json,\n vector_distance_cos(t.embedding, vector32(?)) as distance\n FROM vector_top_k('${INDEX_NAME}', vector32(?), ?) v\n JOIN ${TABLE_NAME} t ON t.rowid = v.id\n ${clause}\n ORDER BY distance ASC\n `,\n args: [queryVecJson, queryVecJson, internalK, ...filterArgs],\n });\n\n let results = (result.rows as Array<{ \n id: string; \n text: string; \n meta_json: string | null;\n distance: number;\n }>).map(row => ({\n id: row.id,\n text: row.text,\n meta: row.meta_json ? JSON.parse(row.meta_json) : undefined,\n distance: row.distance,\n }));\n\n if (maxDistance !== undefined) {\n results = results.filter(r => r.distance <= maxDistance);\n }\n\n const hybridEnabled = !!options.hybrid?.enabled || typeof options.hybrid?.alpha === \"number\";\n if (hybridEnabled) {\n const alpha = Math.min(1, Math.max(0, options.hybrid?.alpha ?? 0.25));\n const keywordK = Math.max(20, options.hybrid?.keyword_k ?? (k + offset) * 2);\n const keywordScores = await getKeywordScores(query, keywordK);\n\n results = results.map((r) => {\n const vecScore = Math.max(0, 1 - r.distance);\n const kwScore = keywordScores.get(r.id) ?? 0;\n const combined = (1 - alpha) * vecScore + alpha * kwScore;\n return { ...r, score: combined, keyword_score: kwScore };\n });\n\n results.sort((a, b) => (b.score ?? 0) - (a.score ?? 0));\n }\n\n return results.slice(offset, offset + k);\n}\n\n/**\n * Delete: Remove a record by ID\n */\nexport async function remove(id: string): Promise<{ deleted: boolean }> {\n const result = await sqlite.execute({\n sql: `DELETE FROM ${TABLE_NAME} WHERE id = ?`,\n args: [id],\n });\n if ((result.rowsAffected ?? 0) > 0) {\n await syncFtsDelete(id);\n }\n return { deleted: (result.rowsAffected ?? 0) > 0 };\n}\n\n/**\n * Stats: Get record count and storage estimate\n * Fix #6: Uses RESOLVED_DIM when available so auto-dim mode reports correctly.\n */\nexport async function stats(): Promise {\n const config = getConfig();\n const result = await sqlite.execute(`SELECT COUNT(*) as count FROM ${TABLE_NAME}`);\n const count = Number((result.rows[0] as { count: number }).count);\n \n // Fix #6: prefer RESOLVED_DIM over config.dim for accurate estimate in auto-dim mode\n const dim = RESOLVED_DIM ?? config.dim;\n const bytesPerRecord = (dim * 4) + 6000;\n const estimated_storage_mb = Math.round((count * bytesPerRecord) / 1024 / 1024 * 100) / 100;\n\n return { count, estimated_storage_mb };\n}\n\n/**\n * Get a single record by ID\n */\nexport async function get(id: string): Promise {\n const result = await sqlite.execute({\n sql: `SELECT id, text, meta_json, updated_at FROM ${TABLE_NAME} WHERE id = ?`,\n args: [id],\n });\n\n if (!result.rows || result.rows.length === 0) return null;\n\n const row = result.rows[0] as { id: string; text: string; meta_json: string | null; updated_at: number };\n return {\n id: row.id,\n text: row.text,\n meta: row.meta_json ? JSON.parse(row.meta_json) : undefined,\n updated_at: row.updated_at,\n };\n}\n\n/**\n * List all record IDs\n */\nexport async function listIds(limit: number = 100, offset: number = 0, prefix?: string): Promise {\n const args: Array = [];\n let whereClause = \"\";\n if (prefix) {\n whereClause = \"WHERE id LIKE ?\";\n args.push(`${prefix}%`);\n }\n\n args.push(limit, offset);\n\n const result = await sqlite.execute({\n sql: `SELECT id FROM ${TABLE_NAME} ${whereClause} ORDER BY updated_at DESC LIMIT ? OFFSET ?`,\n args,\n });\n return (result.rows as Array<{ id: string }>).map(r => r.id);\n}\n","deleted","label",[100],"permalink","https://www.val.town/x/kamenxrider/slimarmor/code/vectordb.ts","extension","actionData","errors"]