Measurements

\n One SQLite row per provider attempt, with timing, token usage,\n cache telemetry, cost, and captured failures.\n

\n\n \n

\n \n Provider\n changeFilter(\"provider\", event.currentTarget.value)}\n className=\"mt-1.5 block h-10 w-full rounded border border-gray-300 bg-white px-3 text-sm font-normal text-black\"\n >\n \n {(data?.providers ?? []).map((value) => (\n \n ))}\n \n \n\n \n Status\n changeFilter(\"status\", event.currentTarget.value)}\n className=\"mt-1.5 block h-10 w-full rounded border border-gray-300 bg-white px-3 text-sm font-normal text-black\"\n >\n \n \n \n \n \n\n \n Attempt\n changeFilter(\"attempt\", event.currentTarget.value)}\n className=\"mt-1.5 block h-10 w-full rounded border border-gray-300 bg-white px-3 text-sm font-normal text-black\"\n >\n \n {(data?.attempts ?? []).map((value) => (\n \n ))}\n \n \n\n \n Run\n changeFilter(\"run\", event.currentTarget.value)}\n className=\"mt-1.5 block h-10 w-full rounded border border-gray-300 bg-white px-3 text-sm font-normal text-black\"\n >\n \n {(data?.runs ?? []).map((run) => (\n \n ))}\n \n \n

\n\n

\n {loading\n ? \"Loading measurements…\"\n : data\n ? \"Showing \" + first + \"–\" + last + \" of \" + data.total + \" measurements\"\n : \"No measurements loaded\"}\n

\n {hasActiveQuery && (\n updateParams({\n provider: null,\n status: null,\n attempt: null,\n run: null,\n sort: null,\n direction: null,\n page: null,\n })}\n className=\"text-gray-600 hover:text-black hover:underline\"\n >\n Clear filters\n \n )}\n

\n\n {error && (\n

\n {error}\n

\n )}\n\n {loading\n ? (\n

\n Loading measurements…\n

\n )\n : data\n ? (\n \n )\n : null}\n\n {data && data.total > PAGE_SIZE && (\n

\n updateParams({\n page: page > 2 ? String(page - 1) : null,\n })}\n className=\"rounded border border-gray-300 bg-white px-3 py-2 text-black hover:bg-gray-50 disabled:cursor-not-allowed disabled:opacity-40\"\n >\n Previous\n \n \n Page {Math.min(page, totalPages)} of {totalPages}\n \n = totalPages || loading}\n onClick={() => updateParams({ page: String(page + 1) })}\n className=\"rounded border border-gray-300 bg-white px-3 py-2 text-black hover:bg-gray-50 disabled:cursor-not-allowed disabled:opacity-40\"\n >\n Next\n \n

\n )}\n \n

\n );\n}\n","label",[173,174,68],"frontend","components","pulls","branches","forks","envVars",{"_182":39,"_184":26},"connections",{"_182":26,"_183":26},"count","unfilledCount","emptyCount",{"_32":474,"_13":410,"_66":403,"_42":-5,"_34":35,"_166":39,"_167":88,"_36":475,"_38":26,"_69":259,"_169":-5,"_71":24,"_171":-5,"_226":476,"_17":477},{"_32":466,"_13":467,"_66":468,"_42":403,"_34":35,"_166":469,"_167":88,"_36":470,"_38":378,"_69":70,"_169":471,"_71":24,"_171":-5,"_226":472,"_17":473},{"_32":458,"_13":459,"_66":460,"_42":403,"_34":35,"_166":461,"_167":88,"_36":462,"_38":406,"_69":70,"_169":463,"_71":24,"_171":-5,"_226":464,"_17":465},{"_32":451,"_13":452,"_66":453,"_42":403,"_34":35,"_166":39,"_167":88,"_36":454,"_38":26,"_69":70,"_169":455,"_71":24,"_171":-5,"_226":456,"_17":457},{"_32":443,"_13":444,"_66":445,"_42":403,"_34":35,"_166":446,"_167":88,"_36":447,"_38":406,"_69":70,"_169":448,"_71":24,"_171":-5,"_226":449,"_17":450},{"_32":436,"_13":437,"_66":438,"_42":403,"_34":35,"_166":39,"_167":88,"_36":439,"_38":26,"_69":70,"_169":440,"_71":24,"_171":-5,"_226":441,"_17":442},{"_32":428,"_13":429,"_66":430,"_42":403,"_34":35,"_166":431,"_167":88,"_36":432,"_38":266,"_69":70,"_169":433,"_71":24,"_171":-5,"_226":434,"_17":435},{"_32":419,"_13":420,"_66":421,"_42":403,"_34":35,"_166":422,"_167":88,"_36":423,"_38":424,"_69":70,"_169":425,"_71":24,"_171":-5,"_226":426,"_17":427},{"_32":411,"_13":412,"_66":413,"_42":403,"_34":35,"_166":39,"_167":88,"_36":414,"_38":26,"_69":415,"_169":416,"_71":24,"_171":-5,"_226":417,"_17":418},{"_32":400,"_13":401,"_66":402,"_42":403,"_34":35,"_166":404,"_167":88,"_36":405,"_38":406,"_69":70,"_169":407,"_71":24,"_171":-5,"_226":408,"_17":409},{"_32":396,"_13":372,"_66":367,"_42":-5,"_34":35,"_166":39,"_167":88,"_36":397,"_38":26,"_69":259,"_169":-5,"_71":24,"_171":-5,"_226":398,"_17":399},{"_32":389,"_13":390,"_66":391,"_42":367,"_34":35,"_166":39,"_167":88,"_36":392,"_38":26,"_69":70,"_169":393,"_71":24,"_171":-5,"_226":394,"_17":395},{"_32":382,"_13":383,"_66":384,"_42":367,"_34":35,"_166":39,"_167":88,"_36":385,"_38":26,"_69":70,"_169":386,"_71":24,"_171":-5,"_226":387,"_17":388},{"_32":373,"_13":374,"_66":375,"_42":367,"_34":35,"_166":376,"_167":88,"_36":377,"_38":378,"_69":70,"_169":379,"_71":24,"_171":-5,"_226":380,"_17":381},{"_32":364,"_13":365,"_66":366,"_42":367,"_34":35,"_166":39,"_167":88,"_36":368,"_38":26,"_69":70,"_169":369,"_71":24,"_171":-5,"_226":370,"_17":371},{"_32":360,"_13":173,"_66":265,"_42":-5,"_34":35,"_166":39,"_167":88,"_36":361,"_38":26,"_69":259,"_169":-5,"_71":24,"_171":-5,"_226":362,"_17":363},{"_32":356,"_13":174,"_66":165,"_42":265,"_34":35,"_166":39,"_167":88,"_36":357,"_38":26,"_69":259,"_169":-5,"_71":24,"_171":-5,"_226":358,"_17":359},{"_32":348,"_13":349,"_66":350,"_42":165,"_34":35,"_166":351,"_167":88,"_36":352,"_38":266,"_69":70,"_169":353,"_71":24,"_171":-5,"_226":354,"_17":355},{"_32":341,"_13":342,"_66":343,"_42":165,"_34":35,"_166":39,"_167":88,"_36":344,"_38":26,"_69":70,"_169":345,"_71":24,"_171":-5,"_226":346,"_17":347},{"_32":164,"_13":68,"_66":67,"_42":165,"_34":35,"_166":39,"_167":88,"_36":338,"_38":26,"_69":70,"_169":170,"_71":24,"_171":-5,"_226":339,"_17":340},{"_32":331,"_13":332,"_66":333,"_42":165,"_34":35,"_166":39,"_167":88,"_36":334,"_38":26,"_69":70,"_169":335,"_71":24,"_171":-5,"_226":336,"_17":337},{"_32":322,"_13":323,"_66":324,"_42":165,"_34":35,"_166":325,"_167":88,"_36":326,"_38":327,"_69":70,"_169":328,"_71":24,"_171":-5,"_226":329,"_17":330},{"_32":315,"_13":316,"_66":317,"_42":165,"_34":35,"_166":39,"_167":88,"_36":318,"_38":26,"_69":70,"_169":319,"_71":24,"_171":-5,"_226":320,"_17":321},{"_32":307,"_13":308,"_66":309,"_42":165,"_34":35,"_166":310,"_167":88,"_36":311,"_38":26,"_69":70,"_169":312,"_71":24,"_171":-5,"_226":313,"_17":314},{"_32":300,"_13":301,"_66":302,"_42":265,"_34":35,"_166":39,"_167":88,"_36":303,"_38":26,"_69":21,"_169":304,"_71":24,"_171":-5,"_226":305,"_17":306},{"_32":293,"_13":294,"_66":295,"_42":265,"_34":35,"_166":39,"_167":88,"_36":296,"_38":26,"_69":70,"_169":297,"_71":24,"_171":-5,"_226":298,"_17":299},{"_32":286,"_13":287,"_66":288,"_42":265,"_34":35,"_166":39,"_167":88,"_36":289,"_38":26,"_69":70,"_169":290,"_71":24,"_171":-5,"_226":291,"_17":292},{"_32":279,"_13":280,"_66":281,"_42":265,"_34":35,"_166":39,"_167":88,"_36":282,"_38":26,"_69":70,"_169":283,"_71":24,"_171":-5,"_226":284,"_17":285},{"_32":271,"_13":272,"_66":273,"_42":265,"_34":35,"_166":274,"_167":88,"_36":275,"_38":39,"_69":70,"_169":276,"_71":24,"_171":-5,"_226":277,"_17":278},{"_32":262,"_13":263,"_66":264,"_42":265,"_34":35,"_166":266,"_167":88,"_36":267,"_38":39,"_69":70,"_169":268,"_71":24,"_171":-5,"_226":269,"_17":270},{"_32":257,"_13":256,"_66":249,"_42":-5,"_34":35,"_166":39,"_167":88,"_36":258,"_38":26,"_69":259,"_169":-5,"_71":24,"_171":-5,"_226":260,"_17":261},{"_32":246,"_13":247,"_66":248,"_42":249,"_34":35,"_166":250,"_167":88,"_36":251,"_38":252,"_69":70,"_169":253,"_71":24,"_171":-5,"_226":254,"_17":255},{"_32":237,"_13":238,"_66":239,"_42":-5,"_34":35,"_166":240,"_167":88,"_36":241,"_38":242,"_69":21,"_169":243,"_71":24,"_171":-5,"_226":244,"_17":245},{"_32":229,"_13":230,"_66":231,"_42":-5,"_34":35,"_166":114,"_167":88,"_36":232,"_38":233,"_69":21,"_169":234,"_71":24,"_171":-5,"_226":235,"_17":236},{"_32":220,"_13":221,"_66":86,"_42":-5,"_34":35,"_166":39,"_167":88,"_36":222,"_38":26,"_69":223,"_169":224,"_71":24,"_171":225,"_226":227,"_17":228},"8ad4c023-63e7-4774-ae26-ef9aa9990c1f","index.ts",["D",1789082831202],"http","import { parseVal, serveImmutableFile } from \"https://esm.town/v/std/utils/index.ts\";\nimport { Hono } from \"npm:hono@4\";\nimport { FULL_BENCHMARK_CONFIG } from \"./benchmark/config.ts\";\nimport { listMeasurements } from \"./db/measurements.ts\";\nimport { getBenchmarkResults } from \"./db/results.ts\";\nimport { Root } from \"./frontend/root.tsx\";\nimport type {\n MeasurementPublicStatus,\n MeasurementSortDirection,\n MeasurementSortKey,\n} from \"./shared/types.ts\";\n\nconst MEASUREMENT_SORT_KEYS = new Set([\n \"runStartedAt\",\n \"provider\",\n \"attempt\",\n \"status\",\n \"startedAt\",\n \"ttftMs\",\n \"durationMs\",\n \"tokensPerSecond\",\n \"inputTokens\",\n \"outputTokens\",\n \"cacheReadTokens\",\n \"cacheWriteTokens\",\n \"costUsd\",\n \"message\",\n]);\nconst app = new Hono();\n\nfunction measurementSortKey(value: string | undefined): MeasurementSortKey {\n const key = value as MeasurementSortKey | undefined;\n return key && MEASUREMENT_SORT_KEYS.has(key) ? key : \"startedAt\";\n}\n\nfunction measurementStatus(\n value: string | undefined,\n): MeasurementPublicStatus | undefined {\n return value === \"success\" || value === \"error\" ? value : undefined;\n}\n\nfunction positiveInteger(\n value: string | undefined,\n fallback: number,\n maximum: number,\n): number {\n const parsed = Number(value);\n return Number.isInteger(parsed) && parsed > 0\n ? Math.min(parsed, maximum)\n : fallback;\n}\n\nfunction nonNegativeInteger(\n value: string | undefined,\n fallback: number,\n maximum: number,\n): number {\n const parsed = Number(value);\n return Number.isSafeInteger(parsed) && parsed >= 0\n ? Math.min(parsed, maximum)\n : fallback;\n}\n\napp.get(\"/\", (c) => c.html(Root()));\napp.get(\"/measurements\", (c) => c.html(Root()));\n\napp.get(\"/__immutable/*\", (c) => serveImmutableFile(c.req.path));\n\napp.get(\"/api/results\", async (c) => {\n const results = await getBenchmarkResults(FULL_BENCHMARK_CONFIG.modelId);\n c.header(\"Cache-Control\", \"no-store\");\n c.header(\"X-Content-Type-Options\", \"nosniff\");\n return c.json({ results });\n});\n\napp.get(\"/api/measurements\", async (c) => {\n const attemptValue = c.req.query(\"attempt\");\n const attempt = positiveInteger(attemptValue, 0, 100);\n const direction: MeasurementSortDirection =\n c.req.query(\"direction\") === \"asc\" ? \"asc\" : \"desc\";\n\n const measurements = await listMeasurements({\n modelId: FULL_BENCHMARK_CONFIG.modelId,\n provider: c.req.query(\"provider\") || undefined,\n status: measurementStatus(c.req.query(\"status\")),\n attempt: attempt > 0 ? attempt : undefined,\n runId: c.req.query(\"run\") || undefined,\n sortKey: measurementSortKey(c.req.query(\"sort\")),\n direction,\n limit: positiveInteger(c.req.query(\"limit\"), 50, 100),\n offset: nonNegativeInteger(c.req.query(\"offset\"), 0, 10_000_000),\n });\n\n c.header(\"Cache-Control\", \"no-store\");\n c.header(\"X-Content-Type-Options\", \"nosniff\");\n return c.json({ measurements });\n});\n\napp.get(\"/source\", (c) => c.redirect(parseVal().links.self.val));\n\napp.onError((err, c) => {\n console.error(\n \"Benchmark request failed:\",\n err instanceof Error ? err.message : String(err),\n );\n c.header(\"Cache-Control\", \"no-store\");\n c.header(\"X-Content-Type-Options\", \"nosniff\");\n return c.req.path.startsWith(\"/api/\")\n ? c.json({ error: \"The request could not be completed.\" }, 500)\n : c.text(\"The page could not be loaded.\", 500);\n});\n\nexport default app.fetch;\n","deepseek-v41-flash","children",[],[221],"1942649d-0d91-4e32-9922-61ab69102068","README.md","01a08da5-899f-73ab-baf1-ef75ca3ea3d7",["D",1789499550928],11,"# DeepSeek V4.1 Flash Gateway Benchmark\n\nA public, read-only Val Town page that compares providers serving\n`deepseek/deepseek-v4.1-flash` through Vercel AI Gateway.\n\n## How results work\n\nEvery provider attempt is one row in the val-scoped\n`benchmark_measurements` SQLite table. The default provider comparison pools\nmeasurements from every completed run for the model. Each run keeps its\nbenchmark version in SQLite so methodology changes remain auditable without\nremoving older completed measurements from the aggregate.\n\nBefore each run, the worker fetches Vercel's live endpoint catalog. It\nbenchmarks every returned provider and stores a normalized provider snapshot.\nProviders newly added by Vercel are included automatically. An interrupted run\nresumes against the exact provider set it started with.\n\nEach provider receives the same fixed multi-turn message array four times.\nAttempts for one provider run sequentially to preserve the cache warmup signal,\nwhile different providers run concurrently. Gateway automatic caching is\nenabled and each request is restricted to the provider being measured.\n\n## Routing strategies\n\nAlongside the solo providers (`only: [provider]`), the run also benchmarks\nthree synthetic \"strategy\" endpoints defined in `benchmark/strategies.ts`. All\nof them pass `only: US_PROVIDERS` and differ only in ordering:\n\n| Row | `providerOptions.gateway` |\n|-----|---------------------------|\n| US ordered | `only: US_PROVIDERS`, `order: US_PROVIDERS` — the config to ship |\n| US · sort by TTFT | `only: US_PROVIDERS`, `sort: \"ttft\"` |\n| US · sort by throughput | `only: US_PROVIDERS`, `sort: \"tps\"` |\n\nEvery strategy also sends `zeroDataRetention: true` and\n`disallowPromptTraining: true`, so the gateway only routes to providers with\nno-store / no-train agreements. Those flags are shown per provider in the\n**No store** / **No train** columns, read from `has_zdr` / `has_no_training`\nin Vercel's endpoint catalogue.\n\nFor strategy rows the table also shows which provider the gateway actually\nserved from (`providerMetadata.gateway.routing.finalProvider`) and how many\nsuccessful requests needed a failover (`totalProviderAttemptCount > 1`). TTFT\nfor a strategy row therefore includes any retry time — the true cost of\nautorouting.\n\n## Reasoning and the 300-token cap\n\n`deepseek-v4.1-flash` is a reasoning model and this benchmark measures it with\nreasoning **on** — the product default. Since v4 the request explicitly sends\n`providerOptions.deepseek.thinking = enabled` for every attempt: probing showed\nmodal and wafer serve the model with thinking **off by default** at the gateway\n(46-token answers, no reasoning), while particle and novita default it on\n(~130 reasoning + ~50 text tokens). Without the explicit flag the benchmark was\ncomparing providers doing different tasks.\n\nOutput is capped at 300 tokens to keep the benchmark cheap, so the final\nquestion is deliberately pure recall from the conversation: measured on\nparticle/novita/deepseek it costs 80–160 reasoning tokens plus ~50 tokens of\nvisible text, leaving comfortable headroom.\n\nHistory: `static-multiturn-v2` asked an open-ended \"most important invariant\"\nquestion that burned 100–370 reasoning tokens; every `no_text` error in that\nversion had exactly 300 output tokens (budget spent thinking, no text emitted),\nwhich made reasoning-on providers look ~25 points less reliable. v3 switched to\nthe recall question and subtracted usage-reported reasoning tokens from the\ntok/s numerator. v4 forces thinking on everywhere and, because modal streams\nreasoning but reports `reasoningTokens: 0` (folded into `outputTokens`),\nderives a floor estimate from streamed reasoning characters for the tok/s\nsubtraction. v5 pins `reasoningEffort: \"low\"` (the gateway's default\n`\"medium\"` is rejected by togetherai) and replaces the tok/s formula — v4's\npost-TTFT window was ~0.4 ms for providers that deliver all text in one chunk,\nproducing 80k tok/s. Older version rows remain in SQLite but are not pooled.\n\nThe sortable provider table reports:\n\n- Reliability across all attempts.\n- TTFT p50 and p95 across successful responses.\n- p50 and p5 observed output tokens per second after first text.\n- Cache hits among eligible repeated successful responses.\n- Observed cost per million input plus output tokens.\n\nTokens/sec is all output tokens (reasoning included) divided by the window\nfrom the first streamed reasoning/text delta to the end of the stream.\nResponses delivered in fewer than 3 chunks or a window under 100 ms were\nbuffered rather than streamed and are excluded (null) instead of reported as\nabsurdly fast. p50 is the median, TTFT p95 is the slow latency tail, and\ntokens/sec p5 is the slow throughput tail.\n\n$/MTok is total observed cost divided by total input plus output tokens,\nmultiplied by one million. Cached tokens remain part of input tokens, while\ntheir discount is reflected in observed cost.\n\nThe measurements page exposes Success and Error. Success is based on the\ncomplete consumed text stream, independent of the bounded private response\npreview. Error rows show a short, redacted message; timeouts include the\nconfigured 75 second limit. Raw response text and provider metadata remain\nprivate.\n\n## Scope\n\nSince v5 the benchmark only covers US-based providers with ZDR + no-train\nagreements, in the fallback order we'd ship in an app (see `US_PROVIDERS` in\n`benchmark/strategies.ts`). The three `gateway:us-*` strategies all restrict\nrouting to that set and differ only in ordering.\n\nCosts roughly $0.017 per run → ~$70/month at a 10-minute cadence.\n\n## Schedule and recovery\n\n`benchmark/scheduled.ts` runs every 10 minutes. Requests have a\n75 second timeout and automatic SDK retries are disabled.\n\nA renewable SQLite lease prevents scheduled and manual starts from\noverlapping. The lease can be reclaimed after five minutes without a renewal,\nso an interrupted invocation does not block several later intervals.\nIndividual attempts are checkpointed, and a later invocation resumes the same\nbenchmark version without duplicating stored measurements.\n\nTo run manually, edit `benchmark/config.ts` if needed, open\n`benchmark/scheduled.ts` in Val Town, and click **Run**. Public visitors cannot\nstart model requests from the HTTP app.\n\n## Access\n\nThe Gateway credential is stored as the encrypted\n`VERCEL_AI_GATEWAY_API_KEY` environment variable. The HTTP app exposes only\nread-only routes.\n\n## Structure\n\n- `benchmark/prompt.ts` — fixed messages and benchmark version.\n- `benchmark/attempt.ts` — one streamed AI SDK request.\n- `benchmark/measurement.ts` — pure observation-to-row conversion.\n- `benchmark/endpoints.ts` — live Gateway provider discovery.\n- `benchmark/strategies.ts` — synthetic routing-strategy endpoints.\n- `benchmark/lease.ts` — renewable execution lease.\n- `benchmark/execute.ts` — benchmark orchestration.\n- `db/` — val-scoped SQLite schema, writes, provider snapshots, and aggregation.\n- `frontend/` — React interface styled with Twind.\n- `shared/` — shared benchmark and result types.\n",[],[230],"d68f608f-1f96-40e5-9764-b4ab53c2a5d1","AGENTS.md","81dba77c-ad6f-11f1-bc49-1607ee4eb77e",76,["D",1789499529648],8,"# deepseek-v41-flash-benchmark\n\nGateway provider benchmark for `deepseek/deepseek-v4.1-flash` through Vercel AI\nGateway. Remixed from `valdottown/glm53-flash-benchmark` with the model id and\nUI copy switched to DeepSeek. Diverges from upstream by also benchmarking gateway\nrouting strategies (`benchmark/strategies.ts`) and showing ZDR / no-train flags.\n\nScope (v5+): only the US-based ZDR + no-train providers in `US_PROVIDERS`\n(`benchmark/strategies.ts`) are benchmarked, both solo and via three\n`gateway:us-*` strategies that all pass `only: US_PROVIDERS`. Steve's app\nconfig is the `gateway:us-ordered` options object. Cron: every 10 min → 144\nruns/day × 48 attempts; measured ~$0.017/run ≈ **$70/month** at current\nsuccess rates (failures cost $0, so it tracks below that).\n\nGotchas:\n- Reasoning is ON and forced via `providerOptions.deepseek.thinking = enabled` (v4+): modal/wafer default thinking OFF at the gateway while particle/novita default it ON, so without the flag the comparison mixes different tasks. Keep the final prompt question pure recall so reasoning stays ~80–160 tokens under the 300 cap; in v2 an open-ended question produced `no_text` errors (all 300 tokens spent thinking). Any prompt/metric change → bump `BENCHMARK_VERSION`.\n- `reasoningEffort: \"low\"` is set explicitly (v5+). The gateway default `\"medium\"` is rejected by togetherai's V4.1 chat template (`reasoning_effort must be low, high, xhigh, max`), which made togetherai 0 % in v4.\n- tok/s (v5+) = all output tokens (reasoning included) ÷ (stream end − first reasoning/text delta); null when < 3 streamed chunks or < 100 ms window. v4's post-TTFT formula produced 80k tok/s because many providers (alibaba 93 %, wafer 33 %) deliver all text in a single chunk ~0.4 ms after the \"first\" text delta.\n- `inferenceRegion` is NOT used: the gateway catalogue only declares `inference_regions` for baseten, so pinning `geoRegion: \"us\"` would route exclusively to baseten (16 % success). \"US-based\" here means company HQ, not gateway-enforced residency.\n- Routing metadata lives in `provider_metadata_json` → `$.gateway.routing.{finalProvider,totalProviderAttemptCount,planningReasoning,sort}`. Only present on successful attempts.\n- Request-level `zeroDataRetention` requires a Vercel Pro/Enterprise plan — confirmed working on this key.\n\n## File map\n\n| Path | Type | Purpose |\n|------|------|---------|\n| `index.ts` | http | Hono read-only app (`/`, `/measurements`, `/api/results`, `/api/measurements`) |\n| `benchmark/config.ts` | script | **Model + run config** — change `modelId` here |\n| `benchmark/prompt.ts` | script | Fixed multi-turn messages, `BENCHMARK_VERSION` |\n| `benchmark/attempt.ts` | script | One streamed AI SDK request |\n| `benchmark/measurement.ts` | script | Pure observation-to-row conversion |\n| `benchmark/endpoints.ts` | script | Live Gateway provider discovery |\n| `benchmark/strategies.ts` | script | **`US_PROVIDERS` list** + three `gateway:us-*` strategies (`only` + ZDR + no-train) |\n| `benchmark/lease.ts` | script | Renewable execution lease |\n| `benchmark/execute.ts` | script | Benchmark orchestration |\n| `benchmark/scheduled.ts` | interval | Cron entrypoint — runs every 10 min |\n| `db/schema.ts` | script | SQLite schema + `sqlite` export |\n| `db/queries.ts` | script | Run/measurement writes and lookups |\n| `db/results.ts` | script | Aggregate provider comparison query |\n| `db/measurements.ts` | script | Paginated measurements listing |\n| `frontend/root.tsx` | script | HTML shell + title/meta |\n| `frontend/index.tsx` | script | Client entrypoint, page switch |\n| `frontend/components/App.tsx` | script | Model header + provider table view |\n| `frontend/components/ProviderTable.tsx` | script | Sortable provider table (used twice: strategies + providers) |\n| `frontend/components/StrategyCells.tsx` | script | No store / No train ✓ cells, routing summary under strategy names |\n| `frontend/components/MeasurementsPage.tsx` | script | Measurements page |\n| `frontend/components/Sparkline.tsx` | script | History sparklines |\n| `frontend/components/InfoPopover.tsx` | script | Column explainers |\n| `frontend/{providers,history,format}.ts` | script | Provider names, hourly series, formatters |\n| `frontend/favicon.svg` | file | Favicon (still the GLM-era glyph) |\n| `shared/types.ts` | script | Shared benchmark/result types |\n| `README.md` | file | Overview + schedule docs |\n\n## Credential\n\n`VERCEL_AI_GATEWAY_API_KEY` (scoped env var). HTTP app is read-only.\n",[],[238],"a74fcb35-b29c-496f-9cb2-fd41b7daec04","types.ts","01a08da5-899f-73ab-baf1-e4b182203ecc","01a08da5-899f-73ab-baf1-e8049921b5fd",55,["D",1789486876335],3,"export type BenchmarkStatus = \"running\" | \"completed\" | \"failed\";\nexport type MeasurementStatus = \"ok\" | \"error\";\n\nexport interface BenchmarkConfig {\n benchmarkVersion: string;\n label: string;\n modelId: string;\n runsPerProvider: number;\n maxOutputTokens: number;\n providerConcurrency: number;\n requestTimeoutMs: number;\n providers?: string[];\n}\n\n/**\n * \"provider\" rows pin one provider with `only: [name]`. \"strategy\" rows are\n * synthetic endpoints that exercise a way of calling the gateway (default\n * autorouting, `sort`, `order`, handpicked `only` sets…).\n */\nexport type EndpointKind = \"provider\" | \"strategy\";\n\nexport interface GatewayEndpoint {\n provider_name: string;\n kind?: EndpointKind;\n description?: string;\n gateway_options?: Record;\n name?: string;\n has_zdr?: boolean;\n has_no_training?: boolean;\n supports_implicit_caching?: boolean;\n pricing?: Record;\n latency_last_1h?: { p50?: number; p95?: number };\n throughput_last_1h?: { p50?: number; p95?: number };\n tags?: string[];\n [key: string]: unknown;\n}\n\nexport interface BenchmarkRunSummary {\n id: string;\n label: string;\n modelId: string;\n status: BenchmarkStatus;\n startedAt: string;\n completedAt: string | null;\n runsPerProvider: number;\n maxOutputTokens: number;\n providerCount: number;\n successfulMeasurements: number;\n failedMeasurements: number;\n error: string | null;\n}\n\nexport type StoredMeasurementOutcome = \"text\" | \"no_text\" | \"error\";\nexport type MeasurementErrorCode =\n | \"no_text\"\n | \"timeout\"\n | \"provider_error\"\n | null;\n\nexport interface BenchmarkMeasurement {\n runId: string;\n provider: string;\n attempt: number;\n status: MeasurementStatus;\n outcome: StoredMeasurementOutcome;\n errorCode: MeasurementErrorCode;\n startedAt: string;\n completedAt: string;\n ttftMs: number | null;\n durationMs: number | null;\n inputTokens: number | null;\n outputTokens: number | null;\n cacheReadTokens: number | null;\n cacheWriteTokens: number | null;\n costUsd: number | null;\n tokensPerSecond: number | null;\n responsePreview: string | null;\n error: string | null;\n providerMetadata: Record | null;\n providerInfo: GatewayEndpoint;\n}\n\nexport type MeasurementPublicStatus = \"success\" | \"error\";\n\nexport type MeasurementSortKey =\n | \"runStartedAt\"\n | \"provider\"\n | \"attempt\"\n | \"status\"\n | \"startedAt\"\n | \"ttftMs\"\n | \"durationMs\"\n | \"tokensPerSecond\"\n | \"inputTokens\"\n | \"outputTokens\"\n | \"cacheReadTokens\"\n | \"cacheWriteTokens\"\n | \"costUsd\"\n | \"message\";\n\nexport type MeasurementSortDirection = \"asc\" | \"desc\";\n\nexport interface MeasurementListItem {\n runId: string;\n runStartedAt: string;\n runStatus: BenchmarkStatus;\n provider: string;\n attempt: number;\n status: MeasurementPublicStatus;\n startedAt: string;\n completedAt: string;\n ttftMs: number | null;\n durationMs: number | null;\n inputTokens: number | null;\n outputTokens: number | null;\n cacheReadTokens: number | null;\n cacheWriteTokens: number | null;\n costUsd: number | null;\n tokensPerSecond: number | null;\n error: string | null;\n}\n\nexport interface MeasurementRunOption {\n id: string;\n startedAt: string;\n status: BenchmarkStatus;\n measurementCount: number;\n}\n\nexport interface MeasurementListResponse {\n items: MeasurementListItem[];\n total: number;\n limit: number;\n offset: number;\n providers: string[];\n attempts: number[];\n runs: MeasurementRunOption[];\n}\n\nexport interface BenchmarkRunDetail extends BenchmarkRunSummary {\n config: BenchmarkConfig;\n endpointSnapshot: GatewayEndpoint[];\n measurements: BenchmarkMeasurement[];\n}\n\nexport interface ProviderMetricPoint {\n runId: string;\n runStartedAt: string;\n successRate: number;\n attempts: number;\n successes: number;\n p50TtftMs: number | null;\n p95TtftMs: number | null;\n p50TokensPerSecond: number | null;\n p05TokensPerSecond: number | null;\n cacheHitRate: number | null;\n effectiveCostPerMillionTokensUsd: number | null;\n}\n\nexport interface ProviderResult {\n provider: string;\n kind: EndpointKind;\n /** Zero data retention per the Vercel endpoint catalogue (null = unknown). */\n hasZdr: boolean | null;\n /** Provider does not train on prompts per the catalogue (null = unknown). */\n hasNoTraining: boolean | null;\n /** providerOptions.gateway used by a strategy row; null for provider rows. */\n gatewayOptions: Record | null;\n /** Successful attempts by the provider the gateway finally routed to. */\n routedProviders: Record;\n /** Successful attempts where the gateway retried after a provider failure. */\n recoveredAttempts: number;\n runCount: number;\n attempts: number;\n successes: number;\n failures: number;\n successRate: number;\n p50TtftMs: number | null;\n p95TtftMs: number | null;\n p50DurationMs: number | null;\n p95DurationMs: number | null;\n p50TokensPerSecond: number | null;\n p05TokensPerSecond: number | null;\n cacheEligibleAttempts: number;\n cacheHits: number;\n cacheHitRate: number | null;\n effectiveCostPerMillionTokensUsd: number | null;\n history: ProviderMetricPoint[];\n}\n\nexport interface BenchmarkResults {\n modelId: string;\n /** Only runs of this BENCHMARK_VERSION are pooled. */\n benchmarkVersion: string;\n runCount: number;\n attemptCount: number;\n successfulAttempts: number;\n failedAttempts: number;\n firstRunAt: string | null;\n lastCompletedAt: string | null;\n providers: ProviderResult[];\n}\n",[],[256,247],"shared","46eb97b6-37cb-403b-8d01-53b8d6528ad3",["D",1789082831202],"directory",[216],[256],"70e5d894-3ea5-4a94-9391-a86dad9e083f","root.tsx","01a08da5-899f-73ab-baf1-df2fd8ed278d","01a08da5-899f-73ab-baf1-e025e2c5cf8b",6,["D",1789082872323],"/** @jsxImportSource npm:hono@4/jsx */\nimport { raw } from \"npm:hono@4/html\";\nimport { immutableFileUrl } from \"https://esm.town/v/std/utils/index.ts\";\n\nexport function Root() {\n return (\n <>\n {raw(\"\")}\n \n \n \n \n \n \n DeepSeek V4.1 Flash Provider Benchmark\n

History · MeasurementsPage.tsx