[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8},"actionData","errors","root",{"_113":114,"_115":116},"routes/embed.x.$handle.$name.$",{"_9":10,"_11":12,"_13":14,"_15":16,"_17":18,"_19":20,"_21":-5},"handle","stevekrouse","name","deepseek-v41-flash-benchmark","path","db/results.ts","project",{"_22":48,"_11":12,"_35":49,"_50":-5,"_51":44,"_52":53,"_54":53,"_55":56,"_57":-5,"_58":59,"_60":61,"_62":34,"_63":-5,"_64":-5,"_65":66,"_67":68,"_69":70,"_71":72,"_73":74,"_75":-5,"_76":77,"_78":77,"_79":80,"_81":80,"_82":80,"_83":77,"_84":77,"_85":86,"_87":88,"_89":90,"_91":44},"file",{"_22":23,"_11":24,"_25":26,"_27":28,"_29":30,"_31":32,"_33":34,"_35":36,"_37":38,"_39":40,"_41":42,"_43":44,"_45":-5,"_13":46},"permalink","https://www.val.town/x/stevekrouse/deepseek-v41-flash-benchmark/code/db/results.ts","extension","id","268034b0-6d04-45c8-8159-5a75d79ccafc","results.ts","fileId","01a08da5-899f-73ab-baf1-a1ceda355235","parentId","01a08da5-899f-73ab-baf1-a881a20f339e","branchId","0cb5a767-9630-4119-a265-8c9f5f633c9a","branchVersion",54,"latest",true,"createdAt",["D",1789486867485],"version",7,"fileType","script","content","import type {\n BenchmarkResults,\n GatewayEndpoint,\n ProviderMetricPoint,\n ProviderResult,\n} from \"../shared/types.ts\";\nimport { BENCHMARK_VERSION } from \"../benchmark/prompt.ts\";\nimport { ensureSchema, sqlite } from \"./schema.ts\";\n\ntype Row = Record;\n\ntype AggregateMeasurementRow = {\n run_id: unknown;\n provider: unknown;\n attempt: unknown;\n status: unknown;\n ttft_ms: unknown;\n duration_ms: unknown;\n tokens_per_second: unknown;\n input_tokens: unknown;\n output_tokens: unknown;\n cache_read_tokens: unknown;\n cost_usd: unknown;\n final_provider: unknown;\n provider_attempts: unknown;\n};\n\ntype RoutingStats = {\n routedProviders: Record;\n recoveredAttempts: number;\n};\n\ntype EndpointInfo = Pick<\n ProviderResult,\n \"kind\" | \"hasZdr\" | \"hasNoTraining\" | \"gatewayOptions\"\n>;\n\n/**\n * Only completed runs of the current BENCHMARK_VERSION are pooled. Older\n * versions stay in SQLite as raw measurements but changed the prompt or metric\n * definitions, so mixing them would corrupt the comparison.\n */\nconst COMPLETED_CURRENT_RUNS = `r.status = 'completed'\n AND r.model_id = ?\n AND json_extract(r.config_json, '$.benchmarkVersion') = ?`;\n\nfunction asNullableBoolean(value: unknown): boolean | null {\n return typeof value === \"boolean\" ? value : null;\n}\n\n/**\n * Endpoint metadata (ZDR, no-training, strategy options) from the most recent\n * run that recorded each provider, so freshly discovered flags win.\n */\nasync function loadEndpointInfo(\n modelId: string,\n): Promise> {\n const result = await sqlite.execute({\n sql: `SELECT s.provider, s.endpoint_json\n FROM benchmark_provider_snapshots s\n INNER JOIN benchmark_runs r ON r.id = s.run_id\n WHERE ${COMPLETED_CURRENT_RUNS}\n ORDER BY r.started_at ASC`,\n args: [modelId, BENCHMARK_VERSION],\n });\n const info = new Map();\n for (const row of result.rows as Row[]) {\n let endpoint: GatewayEndpoint;\n try {\n endpoint = JSON.parse(String(row.endpoint_json));\n } catch {\n continue;\n }\n info.set(String(row.provider), {\n kind: endpoint.kind === \"strategy\" ? \"strategy\" : \"provider\",\n hasZdr: asNullableBoolean(endpoint.has_zdr),\n hasNoTraining: asNullableBoolean(endpoint.has_no_training),\n gatewayOptions: endpoint.kind === \"strategy\"\n ? endpoint.gateway_options ?? {}\n : null,\n });\n }\n return info;\n}\n\nfunction asNullableNumber(value: unknown): number | null {\n if (value === null || value === undefined) return null;\n const parsed = typeof value === \"number\" ? value : Number(value);\n return Number.isFinite(parsed) ? parsed : null;\n}\n\nfunction percentile(values: number[], fraction: number): number | null {\n if (values.length === 0) return null;\n const sorted = [...values].sort((a, b) => a - b);\n const index = Math.max(0, Math.ceil(fraction * sorted.length) - 1);\n return sorted[index];\n}\n\nfunction median(values: number[]): number | null {\n if (values.length === 0) return null;\n const sorted = [...values].sort((a, b) => a - b);\n const middle = Math.floor(sorted.length / 2);\n return sorted.length % 2 === 0\n ? (sorted[middle - 1] + sorted[middle]) / 2\n : sorted[middle];\n}\n\nfunction numericValues(\n rows: AggregateMeasurementRow[],\n key:\n | \"ttft_ms\"\n | \"duration_ms\"\n | \"tokens_per_second\",\n): number[] {\n return rows\n .map((row) => asNullableNumber(row[key]))\n .filter((value): value is number => value !== null);\n}\n\ntype MetricFields = Pick<\n ProviderResult,\n | \"attempts\"\n | \"successes\"\n | \"failures\"\n | \"successRate\"\n | \"p50TtftMs\"\n | \"p95TtftMs\"\n | \"p50TokensPerSecond\"\n | \"p05TokensPerSecond\"\n | \"cacheEligibleAttempts\"\n | \"cacheHits\"\n | \"cacheHitRate\"\n | \"effectiveCostPerMillionTokensUsd\"\n>;\n\nfunction computeMetrics(\n rows: AggregateMeasurementRow[],\n stats: { eligible: number; hits: number },\n): MetricFields {\n const successful = rows.filter((row) => String(row.status) === \"ok\");\n const ttftValues = numericValues(successful, \"ttft_ms\");\n const throughputValues = numericValues(successful, \"tokens_per_second\");\n let observedCostUsd = 0;\n let observedTokens = 0;\n for (const row of successful) {\n const costUsd = asNullableNumber(row.cost_usd);\n const inputTokens = asNullableNumber(row.input_tokens);\n const outputTokens = asNullableNumber(row.output_tokens);\n if (costUsd === null || inputTokens === null || outputTokens === null) {\n continue;\n }\n const totalTokens = inputTokens + outputTokens;\n if (totalTokens <= 0) continue;\n observedCostUsd += costUsd;\n observedTokens += totalTokens;\n }\n const attempts = rows.length;\n const successes = successful.length;\n\n return {\n attempts,\n successes,\n failures: attempts - successes,\n successRate: attempts > 0 ? successes / attempts : 0,\n p50TtftMs: median(ttftValues),\n p95TtftMs: percentile(ttftValues, 0.95),\n p50TokensPerSecond: median(throughputValues),\n p05TokensPerSecond: percentile(throughputValues, 0.05),\n cacheEligibleAttempts: stats.eligible,\n cacheHits: stats.hits,\n cacheHitRate: stats.eligible > 0 ? stats.hits / stats.eligible : null,\n effectiveCostPerMillionTokensUsd: observedTokens > 0\n ? observedCostUsd / observedTokens * 1_000_000\n : null,\n };\n}\n\n/** Which provider the gateway actually served from, and how often it retried. */\nfunction routingStats(rows: AggregateMeasurementRow[]): RoutingStats {\n const routedProviders: Record = {};\n let recoveredAttempts = 0;\n for (const row of rows) {\n if (String(row.status) !== \"ok\") continue;\n const finalProvider = row.final_provider === null ||\n row.final_provider === undefined\n ? null\n : String(row.final_provider);\n if (finalProvider) {\n routedProviders[finalProvider] = (routedProviders[finalProvider] ?? 0) +\n 1;\n }\n if ((asNullableNumber(row.provider_attempts) ?? 1) > 1) {\n recoveredAttempts += 1;\n }\n }\n return { routedProviders, recoveredAttempts };\n}\n\nfunction metricPoint(\n row: AggregateMeasurementRow,\n fields: MetricFields,\n): ProviderMetricPoint {\n return {\n runId: String(row.run_id),\n runStartedAt: String(row.run_started_at),\n successRate: fields.successRate,\n attempts: fields.attempts,\n successes: fields.successes,\n p50TtftMs: fields.p50TtftMs,\n p95TtftMs: fields.p95TtftMs,\n p50TokensPerSecond: fields.p50TokensPerSecond,\n p05TokensPerSecond: fields.p05TokensPerSecond,\n cacheHitRate: fields.cacheHitRate,\n effectiveCostPerMillionTokensUsd: fields.effectiveCostPerMillionTokensUsd,\n };\n}\n\nexport async function getBenchmarkResults(\n modelId: string,\n): Promise {\n await ensureSchema();\n\n const runResult = await sqlite.execute({\n sql: `SELECT id, started_at, completed_at\n FROM benchmark_runs r\n WHERE ${COMPLETED_CURRENT_RUNS}\n ORDER BY started_at ASC`,\n args: [modelId, BENCHMARK_VERSION],\n });\n const runs = runResult.rows as Row[];\n\n const measurementResult = await sqlite.execute({\n sql: `SELECT\n m.run_id, m.provider, m.attempt, m.status, m.ttft_ms,\n m.duration_ms, m.tokens_per_second, m.input_tokens, m.output_tokens,\n m.cache_read_tokens, m.cost_usd, r.started_at AS run_started_at,\n json_extract(m.provider_metadata_json, '$.gateway.routing.finalProvider')\n AS final_provider,\n json_extract(m.provider_metadata_json, '$.gateway.routing.totalProviderAttemptCount')\n AS provider_attempts\n FROM benchmark_measurements m\n INNER JOIN benchmark_runs r ON r.id = m.run_id\n WHERE ${COMPLETED_CURRENT_RUNS}\n ORDER BY r.started_at ASC, m.provider ASC, m.attempt ASC`,\n args: [modelId, BENCHMARK_VERSION],\n });\n const rows = measurementResult.rows as AggregateMeasurementRow[];\n const endpointInfo = await loadEndpointInfo(modelId);\n\n // Group rows per provider, and within each provider per run (rows are\n // already ordered by run start time, so history stays chronological).\n const byProvider = new Map<\n string,\n Map\n >();\n const cacheStats = new Map();\n const priorSuccessful = new Set();\n\n for (const row of rows) {\n const provider = String(row.provider);\n const runId = String(row.run_id);\n let byRun = byProvider.get(provider);\n if (!byRun) {\n byRun = new Map();\n byProvider.set(provider, byRun);\n }\n const runRows = byRun.get(runId) ?? [];\n runRows.push(row);\n byRun.set(runId, runRows);\n\n const sequenceKey = runId + \":\" + provider;\n const isSuccessful = String(row.status) === \"ok\";\n const cacheReadTokens = asNullableNumber(row.cache_read_tokens);\n if (\n isSuccessful &&\n cacheReadTokens !== null &&\n priorSuccessful.has(sequenceKey)\n ) {\n const stats = cacheStats.get(sequenceKey) ?? { eligible: 0, hits: 0 };\n stats.eligible += 1;\n if (cacheReadTokens > 0) stats.hits += 1;\n cacheStats.set(sequenceKey, stats);\n }\n if (isSuccessful) priorSuccessful.add(sequenceKey);\n }\n\n const providers: ProviderResult[] = [...byProvider.entries()]\n .map(([provider, byRun]) => {\n const allRows = [...byRun.values()].flat();\n // Sum only this provider's per-run cache stats (the map holds every\n // provider's stats keyed by runId:provider, so filter first).\n const overallStats = { eligible: 0, hits: 0 };\n for (const [key, stats] of cacheStats) {\n if (!key.endsWith(\":\" + provider)) continue;\n overallStats.eligible += stats.eligible;\n overallStats.hits += stats.hits;\n }\n\n const history: ProviderMetricPoint[] = [];\n for (const [runId, runRows] of byRun) {\n const stats = cacheStats.get(runId + \":\" + provider) ??\n { eligible: 0, hits: 0 };\n history.push(metricPoint(runRows[0], computeMetrics(runRows, stats)));\n }\n\n const info = endpointInfo.get(provider) ?? {\n kind: provider.startsWith(\"gateway:\") ? \"strategy\" : \"provider\",\n hasZdr: null,\n hasNoTraining: null,\n gatewayOptions: null,\n };\n\n return {\n provider,\n ...info,\n ...routingStats(allRows),\n runCount: byRun.size,\n ...computeMetrics(allRows, overallStats),\n p50DurationMs: median(numericValues(allRows, \"duration_ms\")),\n p95DurationMs: percentile(numericValues(allRows, \"duration_ms\"), 0.95),\n history,\n };\n })\n .sort((a, b) => a.provider.localeCompare(b.provider));\n\n const completedDates = runs\n .map((run) => run.completed_at ? String(run.completed_at) : null)\n .filter((value): value is string => value !== null);\n\n return {\n modelId,\n benchmarkVersion: BENCHMARK_VERSION,\n runCount: runs.length,\n attemptCount: rows.length,\n successfulAttempts: rows.filter((row) => String(row.status) === \"ok\").length,\n failedAttempts: rows.filter((row) => String(row.status) === \"error\").length,\n firstRunAt: runs[0] ? String(runs[0].started_at) : null,\n lastCompletedAt: completedDates.length\n ? completedDates.sort().at(-1) ?? null\n : null,\n providers,\n };\n}\n","deleted",false,"label",[47,24],"db","1ea4658b-1977-4a2c-8356-9d6ab6609397",["D",1789082831202],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Run and compare AI Gateway providers for deepseek-v4.1-flash","imageUrl","userId","a0bf3b31-15a5-4d5c-880e-4b1e22c9bc18","pinnedFileId","01a08da5-899f-73ab-baf1-f0227ea1ff9b","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","e81e7335-6d4c-4764-a902-d0962bbfa9f7","tursoName","1ea4658b19774a2c83569d6ab6609397","tursoHostname","1ea4658b19774a2c83569d6ab6609397-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789518676100],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_9":10,"_112":108},"org",{"_9":10,"_106":-5,"_107":108,"_58":59,"_109":110,"_111":44},"branch",{"_37":92,"_11":93,"_22":30,"_94":48,"_95":96,"_97":98,"_99":53,"_100":101,"_102":103,"_104":12,"_105":10},"isOwned",81,"main","projectId","forkedBranchId","24cb14b4-1851-4548-adc6-43a574860a3d","forkedBranchVersion",322,"forkedProjectPrivacy","forkedProjectId","d055324c-28f9-4529-80dd-e9bbfbdbeb65","forkedProjectOrgId","ba7bb5fb-1abb-4ffb-8f3d-7c244782e5f7","projectName","userHandle","displayName","avatar","https://images.clerk.dev/uploaded/img_2PqHa2Gsy93xQrjh2w78Xu0cChW.jpeg","tier","pro","canRestrictVals","profileImageUrl","ctx",{"_85":-5,"_146":147,"_148":149,"_150":151,"_152":44,"_153":44,"_154":44,"_155":34,"_156":-5,"_157":-7,"_158":-7,"_159":66,"_160":44,"_161":44,"_162":-5,"_163":44,"_164":44,"_165":44,"_166":-5},"ENV",{"_117":118,"_119":120,"_121":93,"_122":123,"_124":125,"_126":127,"_128":129,"_130":131,"_132":133,"_134":135,"_136":137,"_138":139,"_140":141,"_142":143,"_144":145},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town","orgs",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]