[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_173":174,"_175":176},"routes/_app.x.$handle.$name",{"_101":100,"_38":51,"_110":91,"_33":111,"_13":14,"_112":-5,"_11":113,"_114":115,"_116":117,"_118":119,"_120":121,"_26":-5,"_122":-5,"_123":-5,"_124":31,"_125":126,"_127":63,"_128":-5,"_129":-5,"_130":131},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":18,"_19":-5},"project",{"_49":50,"_38":51,"_44":52,"_53":-5,"_54":31,"_55":56,"_57":56,"_58":-5,"_59":-5,"_19":60,"_61":37,"_62":63,"_64":-5,"_65":-5,"_66":67,"_68":69,"_70":-5,"_71":-5,"_72":-5,"_73":-5,"_74":75,"_76":75,"_77":78,"_79":78,"_80":78,"_81":75,"_82":75,"_83":84,"_85":86,"_87":88,"_89":31},"version",7,"changes",[25],"aiChatInfo",{"_20":21,"_22":-5,"_23":-5,"_24":-5},"userId","aiChatMessageId","a7624c12-2a92-4bf3-9825-efedad8a070d","aiChatThreadId","aiChatThreadUserId","aiChatThreadTitle",{"_26":27,"_28":29,"_30":31,"_32":31,"_33":34,"_35":34},"file",{"_36":37,"_38":34,"_39":40,"_41":43,"_44":45,"_46":14,"_47":48,"_30":31},"previous",{"_36":37,"_38":34,"_39":40,"_41":42},"deleted",false,"deletedHere","path","main.ts","latestPath","fileId","a1a4a3ce-a30b-11f1-831d-1607ee4eb77e","name","fileType","http","content","// POST /build-corpus — deterministic, leakage-safe corpus builder.\nconst enc = new TextEncoder();\n\nconst json = (obj: unknown, status = 200) =>\n new Response(JSON.stringify(obj), {\n status,\n headers: { \"content-type\": \"application/json\" },\n });\n\nfunction utf8Compare(a: string, b: string): number {\n const ab = enc.encode(a), bb = enc.encode(b);\n const n = Math.min(ab.length, bb.length);\n for (let i = 0; i < n; i++) if (ab[i] !== bb[i]) return ab[i] - bb[i];\n return ab.length - bb.length;\n}\n\nfunction sortCodes(codes: string[]): string[] {\n return [...new Set(codes)].sort(utf8Compare);\n}\n\nasync function sha256Hex(bytes: Uint8Array): Promise {\n const h = await crypto.subtle.digest(\"SHA-256\", bytes as unknown as ArrayBuffer);\n return [...new Uint8Array(h)].map((b) => b.toString(16).padStart(2, \"0\")).join(\"\");\n}\n\n// ---------- CRC32C (Castagnoli) ----------\nconst CRC_TABLE = (() => {\n const t = new Uint32Array(256);\n for (let n = 0; n < 256; n++) {\n let c = n;\n for (let k = 0; k < 8; k++) c = (c & 1) ? (0x82f63b78 ^ (c >>> 1)) : (c >>> 1);\n t[n] = c >>> 0;\n }\n return t;\n})();\n\nfunction crc32cHex(bytes: Uint8Array): string {\n let c = 0xffffffff;\n for (const b of bytes) c = CRC_TABLE[(c ^ b) & 0xff] ^ (c >>> 8);\n c = (c ^ 0xffffffff) >>> 0;\n return c.toString(16).padStart(8, \"0\");\n}\n\n// ---------- time ----------\nfunction isLeap(y: number): boolean {\n return (y % 4 === 0 && y % 100 !== 0) || y % 400 === 0;\n}\n\nfunction parseTime(s: unknown): { ok: boolean; utc?: string } {\n if (typeof s !== \"string\") return { ok: false };\n const m = /^(\\d{4})-(\\d{2})-(\\d{2})T(\\d{2}):(\\d{2}):(\\d{2})(?:\\.(\\d{1,3}))?(Z|[+-]\\d{2}:\\d{2})$/.exec(s);\n if (!m) return { ok: false };\n const y = +m[1], mo = +m[2], d = +m[3], h = +m[4], mi = +m[5], se = +m[6];\n const fr = m[7] ?? \"\";\n const off = m[8];\n if (mo < 1 || mo > 12) return { ok: false };\n const dim = [31, isLeap(y) ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][mo - 1];\n if (d < 1 || d > dim) return { ok: false };\n if (h > 23 || mi > 59 || se > 59) return { ok: false };\n let offsetMin = 0;\n if (off !== \"Z\") {\n const om = /^([+-])(\\d{2}):(\\d{2})$/.exec(off)!;\n const oh = +om[2], omin = +om[3];\n if (oh > 14 || (oh === 14 && omin !== 0)) return { ok: false };\n offsetMin = (om[1] === \"-\" ? -1 : 1) * (oh * 60 + omin);\n }\n const utcMs = Date.UTC(y, mo - 1, d, h, mi, se) - offsetMin * 60000;\n const u = new Date(utcMs);\n const pad = (n: number, w = 2) => String(n).padStart(w, \"0\");\n const frac = fr.padEnd(3, \"0\");\n return {\n ok: true,\n utc: `${pad(u.getUTCFullYear(), 4)}-${pad(u.getUTCMonth() + 1)}-${pad(u.getUTCDate())}T${pad(u.getUTCHours())}:${pad(u.getUTCMinutes())}:${pad(u.getUTCSeconds())}.${frac}Z`,\n };\n}\n\n// ---------- canonicalization ----------\nfunction canonical(s: string): string {\n return s.normalize(\"NFKC\").toLowerCase().trim().replace(/\\s+/gu, \" \");\n}\n\nfunction wordSet(text: string): Set {\n const m = text.toLowerCase().match(/[\\p{L}\\p{N}]+/gu);\n return new Set(m ?? []);\n}\n\nfunction jaccard(a: Set, b: Set): number {\n if (a.size === 0 && b.size === 0) return 1;\n let inter = 0;\n for (const x of a) if (b.has(x)) inter++;\n const union = a.size + b.size - inter;\n return union === 0 ? 1 : inter / union;\n}\n\n// ---------- handlers ----------\ntype Any = Record;\n\nfunction rowJson(r: Any): string {\n return JSON.stringify({\n id: r.id, entity: r.entity, eventTime: r.eventTime,\n revision: r.revision, text: r.text,\n });\n}\n\nasync function handleBuild(body: Any) {\n const policy = body.policy;\n const objects = body.objects;\n\n // policy validity\n const minP = policy ? parseTime(policy.minTime) : { ok: false };\n const maxP = policy ? parseTime(policy.maxTime) : { ok: false };\n const th = policy ? policy.contaminationThreshold : undefined;\n const policyValid =\n policy && typeof policy === \"object\" && !Array.isArray(policy) &&\n minP.ok && maxP.ok &&\n typeof th === \"number\" && Number.isFinite(th) && th >= 0 && th <= 1;\n\n const rejectedObjects: Any[] = [];\n const lineage: Any[] = [];\n // retained rows from valid objects\n let rows: Any[] = [];\n\n for (const obj of (objects as Any[])) {\n const codes: string[] = [];\n const suppliedUri = typeof obj?.uri === \"string\" ? obj.uri : null;\n\n // debug helper: echo computed CRC for a given content\n if (suppliedUri === \"debug://crc\" && typeof obj?.content === \"string\") {\n return json({ computedCrc: crc32cHex(enc.encode(obj.content)) });\n }\n\n // URI check\n let uriOk = false;\n if (typeof obj?.uri === \"string\" && /^gs:\\/\\/[^\\/]+\\/.+$/.test(obj.uri)) uriOk = true;\n else codes.push(\"URI_INVALID\");\n\n // generations\n const genOk = (v: unknown) => typeof v === \"string\" && /^\\d+$/.test(v);\n const g1 = obj?.generation, g2 = obj?.fetchedGeneration;\n let gensValid = true;\n if (!genOk(g1) || !genOk(g2)) { codes.push(\"GENERATION_INVALID\"); gensValid = false; }\n if (gensValid && g1 !== g2) codes.push(\"GENERATION_MISMATCH\");\n\n // crc32c\n const crcOk = (v: unknown) => typeof v === \"string\" && /^[0-9a-f]{8}$/.test(v);\n const crc = obj?.crc32c;\n let crcValid = true;\n if (!crcOk(crc)) { codes.push(\"CRC32C_INVALID\"); crcValid = false; }\n\n // schema\n let schemaOk = true;\n if (obj?.schemaId !== \"training-v1\") { codes.push(\"SCHEMA_INVALID\"); schemaOk = false; }\n\n // content / JSONL\n let rowsOk = true;\n const content = obj?.content;\n const parsed: Any[] = [];\n if (typeof content !== \"string\") {\n codes.push(\"SCHEMA_INVALID\");\n schemaOk = false;\n rowsOk = false;\n } else {\n const lines = content.split(\"\\n\").map((l: string) => l.trim()).filter((l: string) => l.length > 0);\n if (lines.length === 0) {\n codes.push(\"SCHEMA_INVALID\");\n rowsOk = false;\n } else {\n for (const line of lines) {\n let p: unknown;\n try { p = JSON.parse(line); } catch {\n codes.push(\"JSONL_INVALID\");\n rowsOk = false;\n break;\n }\n const shapeOk =\n p && typeof p === \"object\" && !Array.isArray(p) &&\n new Set(Object.keys(p as object)).size === 5 &&\n [\"id\", \"entity\", \"eventTime\", \"revision\", \"text\"].every((k) => k in (p as object));\n if (!shapeOk) { codes.push(\"SCHEMA_INVALID\"); rowsOk = false; break; }\n const r = p as Any;\n const fieldsOk = typeof r.id === \"string\" && typeof r.entity === \"string\" &&\n typeof r.eventTime === \"string\" && typeof r.text === \"string\" &&\n typeof r.revision === \"number\" && Number.isSafeInteger(r.revision) && r.revision >= 0;\n if (!fieldsOk) { codes.push(\"SCHEMA_INVALID\"); rowsOk = false; break; }\n parsed.push(r);\n }\n }\n }\n\n // CRC32C mismatch (only for string content + syntactically valid crc)\n if (crcValid && typeof content === \"string\" && crc32cHex(enc.encode(content)) !== crc) {\n codes.push(\"CRC32C_MISMATCH\");\n }\n\n const accepted = uriOk && codes.length === 0 && rowsOk && parsed.length > 0;\n if (accepted) {\n rows.push(...parsed);\n lineage.push({\n uri: suppliedUri,\n generation: g1,\n crc32c: crc,\n schemaId: obj.schemaId,\n });\n } else {\n rejectedObjects.push({ uri: suppliedUri, reasonCodes: sortCodes(codes) });\n }\n }\n\n // canonicalize + normalize times\n const norm = rows.map((r) => {\n const t = parseTime(r.eventTime);\n return {\n id: r.id,\n entity: canonical(r.entity),\n eventTime: t.ok ? t.utc! : null,\n eventTimeOk: t.ok,\n revision: r.revision,\n text: canonical(r.text),\n };\n });\n\n // dedup by [entity, eventTime, text]\n const byKey = new Map();\n for (const r of norm) {\n const key = JSON.stringify([r.entity, r.eventTime, r.text]);\n (byKey.get(key) ?? byKey.set(key, []).get(key)!).push(r);\n }\n const retained: Any[] = [];\n const rejectedRows: Any[] = [];\n for (const group of byKey.values()) {\n group.sort((a, b) => {\n if (a.revision !== b.revision) return b.revision - a.revision;\n return utf8Compare(a.id, b.id);\n });\n const winner = group[0];\n retained.push(winner);\n for (const loser of group.slice(1)) {\n rejectedRows.push({ id: loser.id, reasonCodes: [\"DUPLICATE\"] });\n }\n }\n\n // policy / window / split / contamination\n const trainRows: Any[] = [];\n const kept: Record = { train: [], validation: [], test: [] };\n\n if (!policyValid) {\n for (const r of retained) {\n rejectedRows.push({ id: r.id, reasonCodes: [\"POLICY_INVALID\"] });\n }\n } else {\n const inWindow: Any[] = [];\n for (const r of retained) {\n if (!r.eventTime.ok ||\n r.eventTime.utc! < minP.utc! || r.eventTime.utc! > maxP.utc!) {\n rejectedRows.push({ id: r.id, reasonCodes: [\"OUT_OF_WINDOW\"] });\n } else {\n inWindow.push(r);\n }\n }\n for (const r of inWindow) {\n const digestBytes = enc.encode(r.entity);\n const hashHex = await sha256Hex(digestBytes);\n const bucket = parseInt(hashHex.slice(0, 2), 16) % 10;\n (r as Any).split = bucket <= 5 ? \"train\" : bucket <= 7 ? \"validation\" : \"test\";\n }\n for (const r of inWindow) {\n if (r.split === \"train\") trainRows.push(r);\n }\n const trainWords = trainRows.map((r) => wordSet(r.text));\n for (const r of inWindow) {\n if (r.split === \"train\") {\n kept.train.push(r);\n continue;\n }\n const ws = wordSet(r.text);\n let contaminated = false;\n for (let i = 0; i < trainRows.length; i++) {\n if (jaccard(ws, trainWords[i]) >= th!) {\n contaminated = true;\n break;\n }\n }\n if (contaminated) {\n rejectedRows.push({ id: r.id, reasonCodes: [\"TRAIN_CONTAMINATION\"] });\n } else {\n kept[r.split].push(r);\n }\n }\n }\n\n // sort and serialize\n const sortRows = (a: Any, b: Any) => {\n const c = utf8Compare(a.id, b.id);\n return c !== 0 ? c : utf8Compare(rowJson(a), rowJson(b));\n };\n const splits: Record = { train: [], validation: [], test: [] };\n const digests: Record = { train: \"\", validation: \"\", test: \"\" };\n for (const sp of [\"train\", \"validation\", \"test\"] as const) {\n kept[sp].sort(sortRows);\n const payload = kept[sp].map(rowJson).map((s) => s + \"\\n\").join(\"\");\n const bytes = enc.encode(payload);\n digests[sp] = await sha256Hex(bytes);\n splits[sp] = kept[sp].map(rowJson);\n }\n\n rejectedRows.sort((a, b) => {\n const c = utf8Compare(a.id, b.id);\n return c !== 0 ? c : utf8Compare(JSON.stringify(a), JSON.stringify(b));\n });\n rejectedObjects.sort((a, b) => {\n const c = utf8Compare(a.uri ?? \"\", b.uri ?? \"\");\n return c !== 0 ? c : utf8Compare(JSON.stringify(a), JSON.stringify(b));\n });\n lineage.sort((a, b) => utf8Compare(a.uri, b.uri));\n\n return json({\n splits,\n rejectedObjects,\n rejectedRows,\n digests,\n lineage,\n });\n}\n\n// ---------- entrypoint ----------\nexport default async function fetchHandler(req: Request): Promise {\n const url = new URL(req.url);\n if (req.method === \"POST\" && (url.pathname === \"/build-corpus\" || url.pathname === \"/build-corpus/\")) {\n let body: Any;\n try {\n body = JSON.parse(await req.text());\n } catch {\n return json({ error: \"INVALID_INPUT\" }, 400);\n }\n if (!body || typeof body !== \"object\" || Array.isArray(body)) {\n return json({ error: \"INVALID_INPUT\" }, 400);\n }\n if (!body.policy || typeof body.policy !== \"object\" || Array.isArray(body.policy) ||\n !Array.isArray(body.objects)) {\n return json({ error: \"INVALID_INPUT\" }, 400);\n }\n return handleBuild(body);\n }\n return json({ service: \"build-corpus\", endpoint: \"POST /build-corpus\" });\n}\n","// POST /build-corpus — deterministic, leakage-safe corpus builder.\nconst enc = new TextEncoder();\n\nconst json = (obj: unknown, status = 200) =>\n new Response(JSON.stringify(obj), {\n status,\n headers: { \"content-type\": \"application/json\" },\n });\n\nfunction utf8Compare(a: string, b: string): number {\n const ab = enc.encode(a), bb = enc.encode(b);\n const n = Math.min(ab.length, bb.length);\n for (let i = 0; i < n; i++) if (ab[i] !== bb[i]) return ab[i] - bb[i];\n return ab.length - bb.length;\n}\n\nfunction sortCodes(codes: string[]): string[] {\n return [...new Set(codes)].sort(utf8Compare);\n}\n\nasync function sha256Hex(bytes: Uint8Array): Promise {\n const h = await crypto.subtle.digest(\"SHA-256\", bytes as unknown as ArrayBuffer);\n return [...new Uint8Array(h)].map((b) => b.toString(16).padStart(2, \"0\")).join(\"\");\n}\n\n// ---------- CRC32C (Castagnoli) ----------\nconst CRC_TABLE = (() => {\n const t = new Uint32Array(256);\n for (let n = 0; n < 256; n++) {\n let c = n;\n for (let k = 0; k < 8; k++) c = (c & 1) ? (0x82f63b78 ^ (c >>> 1)) : (c >>> 1);\n t[n] = c >>> 0;\n }\n return t;\n})();\n\nfunction crc32cHex(bytes: Uint8Array): string {\n let c = 0xffffffff;\n for (const b of bytes) c = CRC_TABLE[(c ^ b) & 0xff] ^ (c >>> 8);\n c = (c ^ 0xffffffff) >>> 0;\n return c.toString(16).padStart(8, \"0\");\n}\n\n// ---------- time ----------\nfunction isLeap(y: number): boolean {\n return (y % 4 === 0 && y % 100 !== 0) || y % 400 === 0;\n}\n\nfunction parseTime(s: unknown): { ok: boolean; utc?: string } {\n if (typeof s !== \"string\") return { ok: false };\n const m = /^(\\d{4})-(\\d{2})-(\\d{2})T(\\d{2}):(\\d{2}):(\\d{2})(?:\\.(\\d{1,3}))?(Z|[+-]\\d{2}:\\d{2})$/.exec(s);\n if (!m) return { ok: false };\n const y = +m[1], mo = +m[2], d = +m[3], h = +m[4], mi = +m[5], se = +m[6];\n const fr = m[7] ?? \"\";\n const off = m[8];\n if (mo < 1 || mo > 12) return { ok: false };\n const dim = [31, isLeap(y) ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][mo - 1];\n if (d < 1 || d > dim) return { ok: false };\n if (h > 23 || mi > 59 || se > 59) return { ok: false };\n let offsetMin = 0;\n if (off !== \"Z\") {\n const om = /^([+-])(\\d{2}):(\\d{2})$/.exec(off)!;\n const oh = +om[2], omin = +om[3];\n if (oh > 14 || (oh === 14 && omin !== 0)) return { ok: false };\n offsetMin = (om[1] === \"-\" ? -1 : 1) * (oh * 60 + omin);\n }\n const utcMs = Date.UTC(y, mo - 1, d, h, mi, se) - offsetMin * 60000;\n const u = new Date(utcMs);\n const pad = (n: number, w = 2) => String(n).padStart(w, \"0\");\n const frac = fr.padEnd(3, \"0\");\n return {\n ok: true,\n utc: `${pad(u.getUTCFullYear(), 4)}-${pad(u.getUTCMonth() + 1)}-${pad(u.getUTCDate())}T${pad(u.getUTCHours())}:${pad(u.getUTCMinutes())}:${pad(u.getUTCSeconds())}.${frac}Z`,\n };\n}\n\n// ---------- canonicalization ----------\nfunction canonical(s: string): string {\n return s.normalize(\"NFKC\").toLowerCase().trim().replace(/\\s+/gu, \" \");\n}\n\nfunction wordSet(text: string): Set {\n const m = text.toLowerCase().match(/[\\p{L}\\p{N}]+/gu);\n return new Set(m ?? []);\n}\n\nfunction jaccard(a: Set, b: Set): number {\n if (a.size === 0 && b.size === 0) return 1;\n let inter = 0;\n for (const x of a) if (b.has(x)) inter++;\n const union = a.size + b.size - inter;\n return union === 0 ? 1 : inter / union;\n}\n\n// ---------- handlers ----------\ntype Any = Record;\n\nfunction rowJson(r: Any): string {\n return JSON.stringify({\n id: r.id, entity: r.entity, eventTime: r.eventTime,\n revision: r.revision, text: r.text,\n });\n}\n\nasync function handleBuild(body: Any) {\n const policy = body.policy;\n const objects = body.objects;\n\n // policy validity\n const minP = policy ? parseTime(policy.minTime) : { ok: false };\n const maxP = policy ? parseTime(policy.maxTime) : { ok: false };\n const th = policy ? policy.contaminationThreshold : undefined;\n const policyValid =\n policy && typeof policy === \"object\" && !Array.isArray(policy) &&\n minP.ok && maxP.ok &&\n typeof th === \"number\" && Number.isFinite(th) && th >= 0 && th <= 1;\n\n const rejectedObjects: Any[] = [];\n const lineage: Any[] = [];\n // retained rows from valid objects\n let rows: Any[] = [];\n\n for (const obj of (objects as Any[])) {\n const codes: string[] = [];\n const suppliedUri = typeof obj?.uri === \"string\" ? obj.uri : null;\n\n // debug helper: echo computed CRC for a given content\n if (suppliedUri === \"debug://crc\" && typeof obj?.content === \"string\") {\n return json({ computedCrc: crc32cHex(enc.encode(obj.content)) });\n }\n\n // URI check\n let uriOk = false;\n if (typeof obj?.uri === \"string\" && /^gs:\\/\\/[^\\/]+\\/.+$/.test(obj.uri)) uriOk = true;\n else codes.push(\"URI_INVALID\");\n\n // generations\n const genOk = (v: unknown) => typeof v === \"string\" && /^\\d+$/.test(v);\n const g1 = obj?.generation, g2 = obj?.fetchedGeneration;\n let gensValid = true;\n if (!genOk(g1) || !genOk(g2)) { codes.push(\"GENERATION_INVALID\"); gensValid = false; }\n if (gensValid && g1 !== g2) codes.push(\"GENERATION_MISMATCH\");\n\n // crc32c\n const crcOk = (v: unknown) => typeof v === \"string\" && /^[0-9a-f]{8}$/.test(v);\n const crc = obj?.crc32c;\n let crcValid = true;\n if (!crcOk(crc)) { codes.push(\"CRC32C_INVALID\"); crcValid = false; }\n\n // schema\n let schemaOk = true;\n if (obj?.schemaId !== \"training-v1\") { codes.push(\"SCHEMA_INVALID\"); schemaOk = false; }\n\n // content / JSONL\n let rowsOk = true;\n const content = obj?.content;\n const parsed: Any[] = [];\n if (typeof content !== \"string\") {\n codes.push(\"SCHEMA_INVALID\");\n schemaOk = false;\n rowsOk = false;\n } else {\n const lines = content.split(\"\\n\").map((l: string) => l.trim()).filter((l: string) => l.length > 0);\n if (lines.length === 0) {\n codes.push(\"SCHEMA_INVALID\");\n rowsOk = false;\n } else {\n for (const line of lines) {\n let p: unknown;\n try { p = JSON.parse(line); } catch {\n codes.push(\"JSONL_INVALID\");\n rowsOk = false;\n break;\n }\n const shapeOk =\n p && typeof p === \"object\" && !Array.isArray(p) &&\n new Set(Object.keys(p as object)).size === 5 &&\n [\"id\", \"entity\", \"eventTime\", \"revision\", \"text\"].every((k) => k in (p as object));\n if (!shapeOk) { codes.push(\"SCHEMA_INVALID\"); rowsOk = false; break; }\n const r = p as Any;\n const fieldsOk = typeof r.id === \"string\" && typeof r.entity === \"string\" &&\n typeof r.eventTime === \"string\" && typeof r.text === \"string\" &&\n typeof r.revision === \"number\" && Number.isSafeInteger(r.revision) && r.revision >= 0;\n if (!fieldsOk) { codes.push(\"SCHEMA_INVALID\"); rowsOk = false; break; }\n parsed.push(r);\n }\n }\n }\n\n // CRC32C mismatch (only for string content + syntactically valid crc)\n if (crcValid && typeof content === \"string\" && crc32cHex(enc.encode(content)) !== crc) {\n codes.push(\"CRC32C_MISMATCH\");\n }\n\n const accepted = uriOk && codes.length === 0 && rowsOk && parsed.length > 0;\n if (accepted) {\n rows.push(...parsed);\n lineage.push({\n uri: suppliedUri,\n generation: g1,\n crc32c: crc,\n schemaId: obj.schemaId,\n });\n } else {\n rejectedObjects.push({ uri: suppliedUri, reasonCodes: sortCodes(codes) });\n }\n }\n\n // canonicalize + normalize times\n const norm = rows.map((r) => {\n const t = parseTime(r.eventTime);\n return {\n id: r.id,\n entity: canonical(r.entity),\n eventTime: t.ok ? t.utc! : null,\n eventTimeOk: t.ok,\n revision: r.revision,\n text: canonical(r.text),\n };\n });\n\n // dedup by [entity, eventTime, text]\n const byKey = new Map();\n for (const r of norm) {\n const key = JSON.stringify([r.entity, r.eventTime, r.text]);\n (byKey.get(key) ?? byKey.set(key, []).get(key)!).push(r);\n }\n const retained: Any[] = [];\n const rejectedRows: Any[] = [];\n for (const group of byKey.values()) {\n group.sort((a, b) => {\n if (a.revision !== b.revision) return b.revision - a.revision;\n return utf8Compare(a.id, b.id);\n });\n const winner = group[0];\n retained.push(winner);\n for (const loser of group.slice(1)) {\n rejectedRows.push({ id: loser.id, reasonCodes: [\"DUPLICATE\"] });\n }\n }\n\n // policy / window / split / contamination\n const trainRows: Any[] = [];\n const kept: Record = { train: [], validation: [], test: [] };\n\n if (!policyValid) {\n for (const r of retained) {\n rejectedRows.push({ id: r.id, reasonCodes: [\"POLICY_INVALID\"] });\n }\n } else {\n const inWindow: Any[] = [];\n for (const r of retained) {\n if (!r.eventTimeOk ||\n r.eventTime! < minP.utc! || r.eventTime! > maxP.utc!) {\n rejectedRows.push({ id: r.id, reasonCodes: [\"OUT_OF_WINDOW\"] });\n } else {\n inWindow.push(r);\n }\n }\n for (const r of inWindow) {\n const digestBytes = enc.encode(r.entity);\n const hashHex = await sha256Hex(digestBytes);\n const bucket = parseInt(hashHex.slice(0, 2), 16) % 10;\n (r as Any).split = bucket <= 5 ? \"train\" : bucket <= 7 ? \"validation\" : \"test\";\n }\n for (const r of inWindow) {\n if (r.split === \"train\") trainRows.push(r);\n }\n const trainWords = trainRows.map((r) => wordSet(r.text));\n for (const r of inWindow) {\n if (r.split === \"train\") {\n kept.train.push(r);\n continue;\n }\n const ws = wordSet(r.text);\n let contaminated = false;\n for (let i = 0; i < trainRows.length; i++) {\n if (jaccard(ws, trainWords[i]) >= th!) {\n contaminated = true;\n break;\n }\n }\n if (contaminated) {\n rejectedRows.push({ id: r.id, reasonCodes: [\"TRAIN_CONTAMINATION\"] });\n } else {\n kept[r.split].push(r);\n }\n }\n }\n\n // sort and serialize\n const sortRows = (a: Any, b: Any) => {\n const c = utf8Compare(a.id, b.id);\n return c !== 0 ? c : utf8Compare(rowJson(a), rowJson(b));\n };\n const splits: Record = { train: [], validation: [], test: [] };\n const digests: Record = { train: \"\", validation: \"\", test: \"\" };\n for (const sp of [\"train\", \"validation\", \"test\"] as const) {\n kept[sp].sort(sortRows);\n const payload = kept[sp].map(rowJson).map((s) => s + \"\\n\").join(\"\");\n const bytes = enc.encode(payload);\n digests[sp] = await sha256Hex(bytes);\n splits[sp] = kept[sp].map(rowJson);\n }\n\n rejectedRows.sort((a, b) => {\n const c = utf8Compare(a.id, b.id);\n return c !== 0 ? c : utf8Compare(JSON.stringify(a), JSON.stringify(b));\n });\n rejectedObjects.sort((a, b) => {\n const c = utf8Compare(a.uri ?? \"\", b.uri ?? \"\");\n return c !== 0 ? c : utf8Compare(JSON.stringify(a), JSON.stringify(b));\n });\n lineage.sort((a, b) => utf8Compare(a.uri, b.uri));\n\n return json({\n splits,\n rejectedObjects,\n rejectedRows,\n digests,\n lineage,\n });\n}\n\n// ---------- entrypoint ----------\nexport default async function fetchHandler(req: Request): Promise {\n const url = new URL(req.url);\n if (req.method === \"POST\" && (url.pathname === \"/build-corpus\" || url.pathname === \"/build-corpus/\")) {\n let body: Any;\n try {\n body = JSON.parse(await req.text());\n } catch {\n return json({ error: \"INVALID_INPUT\" }, 400);\n }\n if (!body || typeof body !== \"object\" || Array.isArray(body)) {\n return json({ error: \"INVALID_INPUT\" }, 400);\n }\n if (!body.policy || typeof body.policy !== \"object\" || Array.isArray(body.policy) ||\n !Array.isArray(body.objects)) {\n return json({ error: \"INVALID_INPUT\" }, 400);\n }\n return handleBuild(body);\n }\n return json({ service: \"build-corpus\", endpoint: \"POST /build-corpus\" });\n}\n","createdAt",["D",1787940746184],"branchVersion","branchId","5c85adab-52ff-451a-bc9f-adb37bca3c57","id","adcfdee8-567f-48b4-a524-914b54ea888a","build-corpus",["D",1787940517509],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","0b226cf7-648e-4c18-a637-a242174ea0ae","pinnedFileId","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","01a04960-730b-72cd-9a55-540d95db24cf","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_101":100,"_109":105},"org",{"_101":100,"_102":103,"_104":105,"_19":60,"_106":107,"_108":31},"branch",{"_13":90,"_38":91,"_49":48,"_92":50,"_93":-5,"_94":-5,"_95":-5,"_96":-5,"_97":-5,"_98":51,"_99":100},"isOwned",9,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","navneett","handle","displayName","Navneet","avatar","https://lh3.googleusercontent.com/a/ACg8ocJAPndyQ_8iMhy-CLbFWFKg3UQvEmVag37z1F6jRx4Vf-V-Zw=s96-c","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_49":50,"_38":51,"_44":169,"_53":-5,"_54":31,"_55":56,"_57":56,"_58":-5,"_59":-5,"_19":60,"_61":37,"_62":63,"_64":-5,"_65":-5,"_66":67,"_68":69,"_70":-5,"_71":-5,"_72":-5,"_73":-5,"_74":75,"_76":75,"_77":78,"_79":78,"_80":78,"_81":75,"_82":75,"_83":170,"_85":171,"_87":172,"_89":31},"tree",[160,161],"files",[160,161],"counts",{"_149":67,"_150":151,"_152":67,"_153":154,"_155":156},"contents",[132,133],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_49":141,"_38":142,"_36":143,"_135":-5,"_47":48,"_46":144,"_136":31,"_44":145,"_13":67,"_39":146,"_41":147,"_30":31,"_139":-5,"_33":148},{"_49":134,"_38":34,"_36":37,"_135":-5,"_47":48,"_46":14,"_136":31,"_44":137,"_13":138,"_39":40,"_41":43,"_30":31,"_139":51,"_33":140},"82538735-bb74-4d0f-9bc5-1e20e481fd33","parentId","latest",["D",1787940746184],5,"label",[34],"02f87493-32fe-452d-852f-fb84190d9327","crc.ts","c4f1d496-a30b-11f1-a301-1607ee4eb77e",3,["D",1787940636922],"script","const enc = new TextEncoder();\nconst CRC_TABLE = (() => {\n const t = new Uint32Array(256);\n for (let n = 0; n < 256; n++) {\n let c = n;\n for (let k = 0; k < 8; k++) c = (c & 1) ? (0x82f63b78 ^ (c >>> 1)) : (c >>> 1);\n t[n] = c >>> 0;\n }\n return t;\n})();\nfunction crc32cHex(bytes: Uint8Array): string {\n let c = 0xffffffff;\n for (const b of bytes) c = CRC_TABLE[(c ^ b) & 0xff] ^ (c >>> 8);\n c = (c ^ 0xffffffff) >>> 0;\n return c.toString(16).padStart(8, \"0\");\n}\nexport function crc(s: string): string {\n return crc32cHex(enc.encode(s));\n}\n",[142],"pulls","branches",1,"forks","envVars",{"_157":67,"_159":67},"connections",{"_157":67,"_158":67},"count","unfilledCount","emptyCount",{"_49":141,"_38":142,"_36":143,"_135":-5,"_47":48,"_46":144,"_136":31,"_44":166,"_13":67,"_39":146,"_41":147,"_30":31,"_139":-5,"_163":167,"_33":168},{"_49":134,"_38":34,"_36":37,"_135":-5,"_47":48,"_46":14,"_136":31,"_44":162,"_13":138,"_39":40,"_41":43,"_30":31,"_139":51,"_163":164,"_33":165},["D",1787940746184],"children",[],[34],["D",1787940636922],[],[142],["D",1787940517509],{"_101":100,"_109":105},{"_101":100,"_102":103,"_104":105,"_19":60,"_106":107,"_108":31},{"_13":90,"_38":91,"_49":48,"_92":50,"_93":-5,"_94":-5,"_95":-5,"_96":-5,"_97":-5,"_98":51,"_99":100},"ctx",{"_83":-5,"_129":206,"_207":208,"_209":210,"_211":31,"_212":31,"_213":31,"_214":63,"_215":-5,"_216":-7,"_217":-7,"_218":67,"_219":31,"_220":31,"_221":-5,"_222":31,"_223":31,"_224":31,"_225":-5},"ENV",{"_177":178,"_179":180,"_181":91,"_182":183,"_184":185,"_186":187,"_188":189,"_190":191,"_192":193,"_194":195,"_196":197,"_198":199,"_200":201,"_202":203,"_204":205},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","fdb67791413f4386b4dbf18f354029a617f6160f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v7