GLM-5.3-Flash Provider Benchmark

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_488":489,"_490":491},"routes/_app.x.$handle.$name",{"_107":106,"_38":52,"_116":97,"_33":117,"_13":14,"_118":-5,"_11":119,"_120":121,"_122":123,"_124":125,"_126":127,"_26":-5,"_128":-5,"_129":-5,"_130":31,"_131":132,"_133":66,"_134":-5,"_135":-5,"_136":137},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":18,"_19":-5},"project",{"_50":51,"_38":52,"_45":53,"_54":-5,"_55":31,"_56":57,"_58":57,"_59":60,"_61":-5,"_19":62,"_63":64,"_65":66,"_67":-5,"_68":-5,"_69":70,"_71":72,"_73":74,"_75":76,"_77":78,"_79":-5,"_80":81,"_82":81,"_83":84,"_85":84,"_86":84,"_87":81,"_88":81,"_89":90,"_91":92,"_93":94,"_95":31},"version",328,"changes",[25],"aiChatInfo",{"_20":21,"_22":-5,"_23":-5,"_24":-5},"userId","aiChatMessageId","8c3e7fa8-eaf4-4be7-9a3a-6bb1e4b5a6ce","aiChatThreadId","aiChatThreadUserId","aiChatThreadTitle",{"_26":27,"_28":29,"_30":31,"_32":31,"_33":34,"_35":34},"file",{"_36":37,"_38":39,"_40":41,"_42":44,"_45":46,"_47":14,"_48":49,"_30":31},"previous",{"_36":37,"_38":39,"_40":41,"_42":43},"deleted",false,"deletedHere","path","db/results.ts","latestPath","fileId","3a3bf3bc-a244-11f1-b162-1607ee4eb77e","name","results.ts","fileType","script","content","import type {\n BenchmarkResults,\n ProviderMetricPoint,\n ProviderResult,\n} from \"../shared/types.ts\";\nimport { ensureSchema, sqlite } from \"./schema.ts\";\n\ntype Row = Record;\n\ntype AggregateMeasurementRow = {\n run_id: unknown;\n provider: unknown;\n attempt: unknown;\n status: unknown;\n ttft_ms: unknown;\n duration_ms: unknown;\n tokens_per_second: unknown;\n input_tokens: unknown;\n output_tokens: unknown;\n cache_read_tokens: unknown;\n cost_usd: unknown;\n};\n\nfunction asNullableNumber(value: unknown): number | null {\n if (value === null || value === undefined) return null;\n const parsed = typeof value === \"number\" ? value : Number(value);\n return Number.isFinite(parsed) ? parsed : null;\n}\n\nfunction percentile(values: number[], fraction: number): number | null {\n if (values.length === 0) return null;\n const sorted = [...values].sort((a, b) => a - b);\n const index = Math.max(0, Math.ceil(fraction * sorted.length) - 1);\n return sorted[index];\n}\n\nfunction median(values: number[]): number | null {\n if (values.length === 0) return null;\n const sorted = [...values].sort((a, b) => a - b);\n const middle = Math.floor(sorted.length / 2);\n return sorted.length % 2 === 0\n ? (sorted[middle - 1] + sorted[middle]) / 2\n : sorted[middle];\n}\n\nfunction numericValues(\n rows: AggregateMeasurementRow[],\n key:\n | \"ttft_ms\"\n | \"duration_ms\"\n | \"tokens_per_second\",\n): number[] {\n return rows\n .map((row) => asNullableNumber(row[key]))\n .filter((value): value is number => value !== null);\n}\n\ntype MetricFields = Pick<\n ProviderResult,\n | \"attempts\"\n | \"successes\"\n | \"failures\"\n | \"successRate\"\n | \"p50TtftMs\"\n | \"p95TtftMs\"\n | \"p50TokensPerSecond\"\n | \"p05TokensPerSecond\"\n | \"cacheEligibleAttempts\"\n | \"cacheHits\"\n | \"cacheHitRate\"\n | \"effectiveCostPerMillionTokensUsd\"\n>;\n\nfunction computeMetrics(\n rows: AggregateMeasurementRow[],\n stats: { eligible: number; hits: number },\n): MetricFields {\n const successful = rows.filter((row) => String(row.status) === \"ok\");\n const ttftValues = numericValues(successful, \"ttft_ms\");\n const throughputValues = numericValues(successful, \"tokens_per_second\");\n let observedCostUsd = 0;\n let observedTokens = 0;\n for (const row of successful) {\n const costUsd = asNullableNumber(row.cost_usd);\n const inputTokens = asNullableNumber(row.input_tokens);\n const outputTokens = asNullableNumber(row.output_tokens);\n if (costUsd === null || inputTokens === null || outputTokens === null) {\n continue;\n }\n const totalTokens = inputTokens + outputTokens;\n if (totalTokens <= 0) continue;\n observedCostUsd += costUsd;\n observedTokens += totalTokens;\n }\n const attempts = rows.length;\n const successes = successful.length;\n\n return {\n attempts,\n successes,\n failures: attempts - successes,\n successRate: attempts > 0 ? successes / attempts : 0,\n p50TtftMs: median(ttftValues),\n p95TtftMs: percentile(ttftValues, 0.95),\n p50TokensPerSecond: median(throughputValues),\n p05TokensPerSecond: percentile(throughputValues, 0.05),\n cacheEligibleAttempts: stats.eligible,\n cacheHits: stats.hits,\n cacheHitRate: stats.eligible > 0 ? stats.hits / stats.eligible : null,\n effectiveCostPerMillionTokensUsd: observedTokens > 0\n ? observedCostUsd / observedTokens * 1_000_000\n : null,\n };\n}\n\nfunction metricPoint(\n row: AggregateMeasurementRow,\n fields: MetricFields,\n): ProviderMetricPoint {\n return {\n runId: String(row.run_id),\n runStartedAt: String(row.run_started_at),\n successRate: fields.successRate,\n attempts: fields.attempts,\n successes: fields.successes,\n p50TtftMs: fields.p50TtftMs,\n p95TtftMs: fields.p95TtftMs,\n p50TokensPerSecond: fields.p50TokensPerSecond,\n p05TokensPerSecond: fields.p05TokensPerSecond,\n cacheHitRate: fields.cacheHitRate,\n effectiveCostPerMillionTokensUsd: fields.effectiveCostPerMillionTokensUsd,\n };\n}\n\nasync function computeResults(\n modelId: string,\n): Promise {\n await ensureSchema();\n\n const runResult = await sqlite.execute({\n sql: `SELECT id, started_at, completed_at\n FROM benchmark_runs\n WHERE status = 'completed'\n AND model_id = ?\n ORDER BY started_at ASC`,\n args: [modelId],\n });\n const runs = runResult.rows as Row[];\n\n const measurementResult = await sqlite.execute({\n sql: `SELECT\n m.run_id, m.provider, m.attempt, m.status, m.ttft_ms,\n m.duration_ms, m.tokens_per_second, m.input_tokens, m.output_tokens,\n m.cache_read_tokens, m.cost_usd, r.started_at AS run_started_at\n FROM benchmark_measurements m\n INNER JOIN benchmark_runs r ON r.id = m.run_id\n WHERE r.status = 'completed'\n AND r.model_id = ?\n ORDER BY r.started_at ASC`,\n args: [modelId],\n });\n const rows = measurementResult.rows as AggregateMeasurementRow[];\n\n // Group rows per provider, and within each provider per run (rows are\n // already ordered by run start time, so history stays chronological).\n const byProvider = new Map<\n string,\n Map\n >();\n const cacheStats = new Map();\n const priorSuccessful = new Set();\n\n for (const row of rows) {\n const provider = String(row.provider);\n const runId = String(row.run_id);\n let byRun = byProvider.get(provider);\n if (!byRun) {\n byRun = new Map();\n byProvider.set(provider, byRun);\n }\n const runRows = byRun.get(runId) ?? [];\n runRows.push(row);\n byRun.set(runId, runRows);\n\n const sequenceKey = runId + \":\" + provider;\n const isSuccessful = String(row.status) === \"ok\";\n const cacheReadTokens = asNullableNumber(row.cache_read_tokens);\n if (\n isSuccessful &&\n cacheReadTokens !== null &&\n priorSuccessful.has(sequenceKey)\n ) {\n const stats = cacheStats.get(sequenceKey) ?? { eligible: 0, hits: 0 };\n stats.eligible += 1;\n if (cacheReadTokens > 0) stats.hits += 1;\n cacheStats.set(sequenceKey, stats);\n }\n if (isSuccessful) priorSuccessful.add(sequenceKey);\n }\n\n const providers: ProviderResult[] = [...byProvider.entries()]\n .map(([provider, byRun]) => {\n const allRows = [...byRun.values()].flat();\n // Sum only this provider's per-run cache stats (the map holds every\n // provider's stats keyed by runId:provider, so filter first).\n const overallStats = { eligible: 0, hits: 0 };\n for (const [key, stats] of cacheStats) {\n if (!key.endsWith(\":\" + provider)) continue;\n overallStats.eligible += stats.eligible;\n overallStats.hits += stats.hits;\n }\n\n const history: ProviderMetricPoint[] = [];\n for (const [runId, runRows] of byRun) {\n const stats = cacheStats.get(runId + \":\" + provider) ??\n { eligible: 0, hits: 0 };\n history.push(metricPoint(runRows[0], computeMetrics(runRows, stats)));\n }\n\n return {\n provider,\n runCount: byRun.size,\n ...computeMetrics(allRows, overallStats),\n p50DurationMs: median(numericValues(allRows, \"duration_ms\")),\n p95DurationMs: percentile(numericValues(allRows, \"duration_ms\"), 0.95),\n history,\n };\n })\n .sort((a, b) => a.provider.localeCompare(b.provider));\n\n const completedDates = runs\n .map((run) => run.completed_at ? String(run.completed_at) : null)\n .filter((value): value is string => value !== null);\n\n return {\n modelId,\n runCount: runs.length,\n attemptCount: rows.length,\n successfulAttempts: rows.filter((row) => String(row.status) === \"ok\").length,\n failedAttempts: rows.filter((row) => String(row.status) === \"error\").length,\n firstRunAt: runs[0] ? String(runs[0].started_at) : null,\n lastCompletedAt: completedDates.length\n ? completedDates.sort().at(-1) ?? null\n : null,\n providers,\n };\n}\n","import type {\n BenchmarkResults,\n ProviderMetricPoint,\n ProviderResult,\n} from \"../shared/types.ts\";\nimport { ensureSchema, sqlite } from \"./schema.ts\";\n\ntype Row = Record;\n\ntype AggregateMeasurementRow = {\n run_id: unknown;\n provider: unknown;\n attempt: unknown;\n status: unknown;\n ttft_ms: unknown;\n duration_ms: unknown;\n tokens_per_second: unknown;\n input_tokens: unknown;\n output_tokens: unknown;\n cache_read_tokens: unknown;\n cost_usd: unknown;\n};\n\nfunction asNullableNumber(value: unknown): number | null {\n if (value === null || value === undefined) return null;\n const parsed = typeof value === \"number\" ? value : Number(value);\n return Number.isFinite(parsed) ? parsed : null;\n}\n\nfunction percentile(values: number[], fraction: number): number | null {\n if (values.length === 0) return null;\n const sorted = [...values].sort((a, b) => a - b);\n const index = Math.max(0, Math.ceil(fraction * sorted.length) - 1);\n return sorted[index];\n}\n\nfunction median(values: number[]): number | null {\n if (values.length === 0) return null;\n const sorted = [...values].sort((a, b) => a - b);\n const middle = Math.floor(sorted.length / 2);\n return sorted.length % 2 === 0\n ? (sorted[middle - 1] + sorted[middle]) / 2\n : sorted[middle];\n}\n\nfunction numericValues(\n rows: AggregateMeasurementRow[],\n key:\n | \"ttft_ms\"\n | \"duration_ms\"\n | \"tokens_per_second\",\n): number[] {\n return rows\n .map((row) => asNullableNumber(row[key]))\n .filter((value): value is number => value !== null);\n}\n\ntype MetricFields = Pick<\n ProviderResult,\n | \"attempts\"\n | \"successes\"\n | \"failures\"\n | \"successRate\"\n | \"p50TtftMs\"\n | \"p95TtftMs\"\n | \"p50TokensPerSecond\"\n | \"p05TokensPerSecond\"\n | \"cacheEligibleAttempts\"\n | \"cacheHits\"\n | \"cacheHitRate\"\n | \"effectiveCostPerMillionTokensUsd\"\n>;\n\nfunction computeMetrics(\n rows: AggregateMeasurementRow[],\n stats: { eligible: number; hits: number },\n): MetricFields {\n const successful = rows.filter((row) => String(row.status) === \"ok\");\n const ttftValues = numericValues(successful, \"ttft_ms\");\n const throughputValues = numericValues(successful, \"tokens_per_second\");\n let observedCostUsd = 0;\n let observedTokens = 0;\n for (const row of successful) {\n const costUsd = asNullableNumber(row.cost_usd);\n const inputTokens = asNullableNumber(row.input_tokens);\n const outputTokens = asNullableNumber(row.output_tokens);\n if (costUsd === null || inputTokens === null || outputTokens === null) {\n continue;\n }\n const totalTokens = inputTokens + outputTokens;\n if (totalTokens <= 0) continue;\n observedCostUsd += costUsd;\n observedTokens += totalTokens;\n }\n const attempts = rows.length;\n const successes = successful.length;\n\n return {\n attempts,\n successes,\n failures: attempts - successes,\n successRate: attempts > 0 ? successes / attempts : 0,\n p50TtftMs: median(ttftValues),\n p95TtftMs: percentile(ttftValues, 0.95),\n p50TokensPerSecond: median(throughputValues),\n p05TokensPerSecond: percentile(throughputValues, 0.05),\n cacheEligibleAttempts: stats.eligible,\n cacheHits: stats.hits,\n cacheHitRate: stats.eligible > 0 ? stats.hits / stats.eligible : null,\n effectiveCostPerMillionTokensUsd: observedTokens > 0\n ? observedCostUsd / observedTokens * 1_000_000\n : null,\n };\n}\n\nfunction metricPoint(\n row: AggregateMeasurementRow,\n fields: MetricFields,\n): ProviderMetricPoint {\n return {\n runId: String(row.run_id),\n runStartedAt: String(row.run_started_at),\n successRate: fields.successRate,\n attempts: fields.attempts,\n successes: fields.successes,\n p50TtftMs: fields.p50TtftMs,\n p95TtftMs: fields.p95TtftMs,\n p50TokensPerSecond: fields.p50TokensPerSecond,\n p05TokensPerSecond: fields.p05TokensPerSecond,\n cacheHitRate: fields.cacheHitRate,\n effectiveCostPerMillionTokensUsd: fields.effectiveCostPerMillionTokensUsd,\n };\n}\n\nasync function computeResults(\n modelId: string,\n): Promise {\n await ensureSchema();\n\n const runResult = await sqlite.execute({\n sql: `SELECT id, started_at, completed_at\n FROM benchmark_runs\n WHERE status = 'completed'\n AND model_id = ?\n ORDER BY started_at ASC`,\n args: [modelId],\n });\n const runs = runResult.rows as Row[];\n\n const measurementResult = await sqlite.execute({\n sql: `SELECT\n m.run_id, m.provider, m.attempt, m.status, m.ttft_ms,\n m.duration_ms, m.tokens_per_second, m.input_tokens, m.output_tokens,\n m.cache_read_tokens, m.cost_usd, r.started_at AS run_started_at\n FROM benchmark_measurements m\n INNER JOIN benchmark_runs r ON r.id = m.run_id\n WHERE r.status = 'completed'\n AND r.model_id = ?\n ORDER BY r.started_at ASC`,\n args: [modelId],\n });\n const rows = measurementResult.rows as AggregateMeasurementRow[];\n\n // Group rows per provider, and within each provider per run (rows are\n // already ordered by run start time, so history stays chronological).\n const byProvider = new Map<\n string,\n Map\n >();\n const cacheStats = new Map();\n const priorSuccessful = new Set();\n\n for (const row of rows) {\n const provider = String(row.provider);\n const runId = String(row.run_id);\n let byRun = byProvider.get(provider);\n if (!byRun) {\n byRun = new Map();\n byProvider.set(provider, byRun);\n }\n const runRows = byRun.get(runId) ?? [];\n runRows.push(row);\n byRun.set(runId, runRows);\n\n const sequenceKey = runId + \":\" + provider;\n const isSuccessful = String(row.status) === \"ok\";\n const cacheReadTokens = asNullableNumber(row.cache_read_tokens);\n if (\n isSuccessful &&\n cacheReadTokens !== null &&\n priorSuccessful.has(sequenceKey)\n ) {\n const stats = cacheStats.get(sequenceKey) ?? { eligible: 0, hits: 0 };\n stats.eligible += 1;\n if (cacheReadTokens > 0) stats.hits += 1;\n cacheStats.set(sequenceKey, stats);\n }\n if (isSuccessful) priorSuccessful.add(sequenceKey);\n }\n\n const providers: ProviderResult[] = [...byProvider.entries()]\n .map(([provider, byRun]) => {\n const allRows = [...byRun.values()].flat();\n // Sum only this provider's per-run cache stats (the map holds every\n // provider's stats keyed by runId:provider, so filter first).\n const overallStats = { eligible: 0, hits: 0 };\n for (const [key, stats] of cacheStats) {\n if (!key.endsWith(\":\" + provider)) continue;\n overallStats.eligible += stats.eligible;\n overallStats.hits += stats.hits;\n }\n\n const history: ProviderMetricPoint[] = [];\n for (const [runId, runRows] of byRun) {\n const stats = cacheStats.get(runId + \":\" + provider) ??\n { eligible: 0, hits: 0 };\n history.push(metricPoint(runRows[0], computeMetrics(runRows, stats)));\n }\n\n return {\n provider,\n runCount: byRun.size,\n ...computeMetrics(allRows, overallStats),\n p50DurationMs: median(numericValues(allRows, \"duration_ms\")),\n p95DurationMs: percentile(numericValues(allRows, \"duration_ms\"), 0.95),\n history,\n };\n })\n .sort((a, b) => a.provider.localeCompare(b.provider));\n\n const completedDates = runs\n .map((run) => run.completed_at ? String(run.completed_at) : null)\n .filter((value): value is string => value !== null);\n\n return {\n modelId,\n runCount: runs.length,\n attemptCount: rows.length,\n successfulAttempts: rows.filter((row) => String(row.status) === \"ok\").length,\n failedAttempts: rows.filter((row) => String(row.status) === \"error\").length,\n firstRunAt: runs[0] ? String(runs[0].started_at) : null,\n lastCompletedAt: completedDates.length\n ? completedDates.sort().at(-1) ?? null\n : null,\n providers,\n };\n}\nconst CACHE_HOURS = 24;\nconst CACHE_PRUNE_HOURS = 24 * 7;\n\n/**\n * Return the cached results for the model's latest completed run, or compute\n * and store them on a cache miss. Measurements only change when a run\n * completes (every ~15 minutes), so this avoids rescanning all 84k rows on\n * every page load.\n */\nexport async function getBenchmarkResults(\n modelId: string,\n): Promise {\n await ensureSchema();\n\n const latestResult = await sqlite.execute({\n sql: `SELECT id, completed_at FROM benchmark_runs\n WHERE status = 'completed' AND model_id = ?\n ORDER BY started_at DESC LIMIT 1`,\n args: [modelId],\n });\n const latest = (latestResult.rows as Row[])[0];\n const cacheKey = latest\n ? `${modelId}:${String(latest.id)}:${String(latest.completed_at)}`\n : `${modelId}:none`;\n\n const cachedResult = await sqlite.execute({\n sql: `SELECT results_json FROM benchmark_results_cache\n WHERE cache_key = ?`,\n args: [cacheKey],\n });\n const cachedRow = (cachedResult.rows as Row[])[0];\n if (cachedRow) {\n return JSON.parse(String(cachedRow.results_json)) as BenchmarkResults;\n }\n\n const results = await computeResults(modelId);\n const now = new Date().toISOString();\n await sqlite.batch([\n {\n sql: `INSERT INTO benchmark_results_cache (cache_key, results_json, created_at)\n VALUES (?, ?, ?)\n ON CONFLICT(cache_key) DO UPDATE SET\n results_json = excluded.results_json,\n created_at = excluded.created_at`,\n args: [cacheKey, JSON.stringify(results), now],\n },\n {\n // Bound the table: drop stale keys for runs we no longer render.\n sql: `DELETE FROM benchmark_results_cache\n WHERE created_at < datetime('now', ?)`,\n args: [`-${CACHE_PRUNE_HOURS} hours`],\n },\n ]);\n return results;\n}\n","createdAt",["D",1789083135992],"branchVersion","branchId","24cb14b4-1851-4548-adc6-43a574860a3d","id","d055324c-28f9-4529-80dd-e9bbfbdbeb65","glm53-flash-benchmark",["D",1787848294548],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Run and compare AI Gateway providers","imageUrl","a0bf3b31-15a5-4d5c-880e-4b1e22c9bc18","pinnedFileId","01a04410-0100-7214-ba2d-a172e514af90","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","ba7bb5fb-1abb-4ffb-8f3d-7c244782e5f7","tursoName","d055324c28f9452980dde9bbfbdbeb65","tursoHostname","d055324c28f9452980dde9bbfbdbeb65-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789500119686],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_107":106,"_115":111},"org",{"_107":106,"_108":109,"_110":111,"_19":-5,"_112":113,"_114":66},"branch",{"_13":96,"_38":97,"_50":49,"_98":51,"_99":-5,"_100":-5,"_101":-5,"_102":-5,"_103":-5,"_104":52,"_105":106},"isOwned",338,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","valdottown","handle","displayName","Val Town ","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvdXBsb2FkZWQvaW1nXzJyWnQ5cFE0c3pleVRCSEVTWmxzaXJsTUlGciJ9","tier","business","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_50":51,"_38":52,"_45":483,"_54":-5,"_55":31,"_56":57,"_58":57,"_59":60,"_61":-5,"_19":62,"_63":64,"_65":66,"_67":-5,"_68":-5,"_69":70,"_71":72,"_73":74,"_75":76,"_77":484,"_79":-5,"_80":81,"_82":81,"_83":84,"_85":84,"_86":84,"_87":81,"_88":81,"_89":485,"_91":486,"_93":487,"_95":31},"tree",[207,217,222,236,238,239,240],"files",[207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240],"counts",{"_197":70,"_198":194,"_199":182,"_200":201,"_202":203},"contents",[138,139,140,141,142,143,144],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_50":191,"_38":192,"_36":193,"_147":-5,"_48":49,"_47":194,"_149":66,"_45":195,"_13":70,"_40":177,"_42":-5,"_30":31,"_154":-5,"_33":196},{"_50":185,"_38":186,"_36":187,"_147":-5,"_48":49,"_47":188,"_149":66,"_45":189,"_13":70,"_40":177,"_42":-5,"_30":31,"_154":-5,"_33":190},{"_50":179,"_38":180,"_36":181,"_147":-5,"_48":49,"_47":182,"_149":66,"_45":183,"_13":70,"_40":177,"_42":-5,"_30":31,"_154":-5,"_33":184},{"_50":172,"_38":173,"_36":174,"_147":-5,"_48":49,"_47":175,"_149":66,"_45":176,"_13":70,"_40":177,"_42":-5,"_30":31,"_154":-5,"_33":178},{"_50":165,"_38":166,"_36":167,"_147":-5,"_48":49,"_47":168,"_149":31,"_45":169,"_13":70,"_40":26,"_42":170,"_30":31,"_154":-5,"_33":171},{"_50":157,"_38":158,"_36":159,"_147":-5,"_48":49,"_47":160,"_149":66,"_45":161,"_13":162,"_40":26,"_42":163,"_30":31,"_154":-5,"_33":164},{"_50":145,"_38":146,"_36":64,"_147":-5,"_48":49,"_47":148,"_149":66,"_45":150,"_13":151,"_40":152,"_42":153,"_30":31,"_154":155,"_33":156},"723d94e7-9d1d-4977-9a00-ef16aded8d63","index.ts","parentId",266,"latest",["D",1787865246169],13,"http","import { parseVal, serveImmutableFile } from \"https://esm.town/v/std/utils/index.ts\";\nimport { Hono } from \"npm:hono@4\";\nimport { FULL_BENCHMARK_CONFIG } from \"./benchmark/config.ts\";\nimport { listMeasurements } from \"./db/measurements.ts\";\nimport { getBenchmarkResults } from \"./db/results.ts\";\nimport { Root } from \"./frontend/root.tsx\";\nimport type {\n MeasurementPublicStatus,\n MeasurementSortDirection,\n MeasurementSortKey,\n} from \"./shared/types.ts\";\n\nconst MEASUREMENT_SORT_KEYS = new Set([\n \"runStartedAt\",\n \"provider\",\n \"attempt\",\n \"status\",\n \"startedAt\",\n \"ttftMs\",\n \"durationMs\",\n \"tokensPerSecond\",\n \"inputTokens\",\n \"outputTokens\",\n \"cacheReadTokens\",\n \"cacheWriteTokens\",\n \"costUsd\",\n \"message\",\n]);\nconst app = new Hono();\n\nfunction measurementSortKey(value: string | undefined): MeasurementSortKey {\n const key = value as MeasurementSortKey | undefined;\n return key && MEASUREMENT_SORT_KEYS.has(key) ? key : \"startedAt\";\n}\n\nfunction measurementStatus(\n value: string | undefined,\n): MeasurementPublicStatus | undefined {\n return value === \"success\" || value === \"error\" ? value : undefined;\n}\n\nfunction positiveInteger(\n value: string | undefined,\n fallback: number,\n maximum: number,\n): number {\n const parsed = Number(value);\n return Number.isInteger(parsed) && parsed > 0\n ? Math.min(parsed, maximum)\n : fallback;\n}\n\nfunction nonNegativeInteger(\n value: string | undefined,\n fallback: number,\n maximum: number,\n): number {\n const parsed = Number(value);\n return Number.isSafeInteger(parsed) && parsed >= 0\n ? Math.min(parsed, maximum)\n : fallback;\n}\n\napp.get(\"/\", (c) => c.html(Root()));\napp.get(\"/measurements\", (c) => c.html(Root()));\n\napp.get(\"/__immutable/*\", (c) => serveImmutableFile(c.req.path));\n\napp.get(\"/api/results\", async (c) => {\n const results = await getBenchmarkResults(FULL_BENCHMARK_CONFIG.modelId);\n c.header(\"Cache-Control\", \"no-store\");\n c.header(\"X-Content-Type-Options\", \"nosniff\");\n return c.json({ results });\n});\n\napp.get(\"/api/measurements\", async (c) => {\n const attemptValue = c.req.query(\"attempt\");\n const attempt = positiveInteger(attemptValue, 0, 100);\n const direction: MeasurementSortDirection =\n c.req.query(\"direction\") === \"asc\" ? \"asc\" : \"desc\";\n\n const measurements = await listMeasurements({\n modelId: FULL_BENCHMARK_CONFIG.modelId,\n provider: c.req.query(\"provider\") || undefined,\n status: measurementStatus(c.req.query(\"status\")),\n attempt: attempt > 0 ? attempt : undefined,\n runId: c.req.query(\"run\") || undefined,\n sortKey: measurementSortKey(c.req.query(\"sort\")),\n direction,\n limit: positiveInteger(c.req.query(\"limit\"), 50, 100),\n offset: nonNegativeInteger(c.req.query(\"offset\"), 0, 10_000_000),\n });\n\n c.header(\"Cache-Control\", \"no-store\");\n c.header(\"X-Content-Type-Options\", \"nosniff\");\n return c.json({ measurements });\n});\n\napp.get(\"/source\", (c) => c.redirect(parseVal().links.self.val));\n\napp.onError((err, c) => {\n console.error(\n \"Benchmark request failed:\",\n err instanceof Error ? err.message : String(err),\n );\n c.header(\"Cache-Control\", \"no-store\");\n c.header(\"X-Content-Type-Options\", \"nosniff\");\n return c.req.path.startsWith(\"/api/\")\n ? c.json({ error: \"The request could not be completed.\" }, 500)\n : c.text(\"The page could not be loaded.\", 500);\n});\n\nexport default app.fetch;\n","label","glm53flash",[146],"3d770df2-a2f0-4900-89a3-b9a19285526e","README.md","01a04410-0100-7214-ba2d-9ca990fb87d3",267,["D",1787865252841],24,"# GLM-5.3-Flash Gateway Benchmark\n\nA public, read-only Val Town page that compares providers serving\n`zai/glm-5.3-flash` through Vercel AI Gateway.\n\n## How results work\n\nEvery provider attempt is one row in the val-scoped\n`benchmark_measurements` SQLite table. The default provider comparison pools\nmeasurements from every completed run for the model. Each run keeps its\nbenchmark version in SQLite so methodology changes remain auditable without\nremoving older completed measurements from the aggregate.\n\nBefore each run, the worker fetches Vercel's live endpoint catalog. It\nbenchmarks every returned provider and stores a normalized provider snapshot.\nProviders newly added by Vercel are included automatically. An interrupted run\nresumes against the exact provider set it started with.\n\nEach provider receives the same fixed multi-turn message array four times.\nAttempts for one provider run sequentially to preserve the cache warmup signal,\nwhile different providers run concurrently. Gateway automatic caching is\nenabled and each request is restricted to the provider being measured.\n\nThe sortable provider table reports:\n\n- Reliability across all attempts.\n- TTFT p50 and p95 across successful responses.\n- p50 and p5 observed output tokens per second after first text.\n- Cache hits among eligible repeated successful responses.\n- Observed cost per million input plus output tokens.\n\nTokens/sec is output tokens divided by `duration - TTFT`. It measures how\nquickly text reaches this client after the first text delta. Providers that\nbuffer output into a few large chunks can therefore appear unusually fast.\np50 is the median, TTFT p95 is the slow latency tail, and tokens/sec p5 is the\nslow throughput tail.\n\n$/MTok is total observed cost divided by total input plus output tokens,\nmultiplied by one million. Cached tokens remain part of input tokens, while\ntheir discount is reflected in observed cost.\n\nThe measurements page exposes Success and Error. Success is based on the\ncomplete consumed text stream, independent of the bounded private response\npreview. Error rows show a short, redacted message; timeouts include the\nconfigured 75 second limit. Raw response text and provider metadata remain\nprivate.\n\n## Schedule and recovery\n\n`benchmark/scheduled.ts` runs every 15 minutes in UTC. Requests have a\n75 second timeout and automatic SDK retries are disabled.\n\nA renewable SQLite lease prevents scheduled and manual starts from\noverlapping. The lease can be reclaimed after five minutes without a renewal,\nso an interrupted invocation does not block several later intervals.\nIndividual attempts are checkpointed, and a later invocation resumes the same\nbenchmark version without duplicating stored measurements.\n\nTo run manually, edit `benchmark/config.ts` if needed, open\n`benchmark/scheduled.ts` in Val Town, and click **Run**. Public visitors cannot\nstart model requests from the HTTP app.\n\n## Access\n\nThe Gateway credential is stored as the encrypted\n`VERCEL_AI_GATEWAY_API_KEY` environment variable. The HTTP app exposes only\nread-only routes.\n\n## Structure\n\n- `benchmark/prompt.ts` — fixed messages and benchmark version.\n- `benchmark/attempt.ts` — one streamed AI SDK request.\n- `benchmark/measurement.ts` — pure observation-to-row conversion.\n- `benchmark/endpoints.ts` — live Gateway provider discovery.\n- `benchmark/lease.ts` — renewable execution lease.\n- `benchmark/execute.ts` — benchmark orchestration.\n- `db/` — val-scoped SQLite schema, writes, provider snapshots, and aggregation.\n- `frontend/` — React interface styled with Twind.\n- `shared/` — shared benchmark and result types.\n",[158],"2f31d864-814f-4d05-bcef-38b6c5558fee","AGENTS.md","d0414890-ad6f-11f1-b770-1607ee4eb77e",324,["D",1789083117196],"# GLM-5.3-Flash Benchmark — AGENTS.md\n\n## File map\n\n- `index.ts` — Hono HTTP app. Routes: `/` (results page), `/measurements`,\n `/api/results`, `/api/measurements`, `/__immutable/*`, `/source`.\n- `benchmark/` — scheduled benchmark orchestration (interval `scheduled.ts`),\n prompt, attempt, measurement, endpoints, lease, execute, config.\n- `db/schema.ts` — `ensureSchema()` (tables + indexes) and the `sqlite` handle.\n- `db/results.ts` — `getBenchmarkResults()`: aggregates measurements into\n per-provider results. **Cached** in `benchmark_results_cache` keyed by the\n latest completed run, so page loads don't rescan all rows.\n- `db/measurements.ts` — `listMeasurements()`: paginated measurements list.\n- `db/queries.ts` — run/measurement write + read helpers used by the scheduler.\n- `frontend/` — React (Twind) client: `root.tsx`, `index.tsx`, `components/`,\n `history.ts`, `providers.ts`, `format.ts`.\n- `shared/types.ts` — shared benchmark/result types.\n",[166],"57180eb9-f846-47ed-9f7f-5ec6aa1d02be","shared","f91bd0d2-a234-11f1-8cec-1607ee4eb77e",4,["D",1787848382634],"directory",[173],"6f24ec24-8f0c-4d0d-b0d3-fbc1bac995ce","frontend","01a04410-0100-7214-ba2d-95062f41a4ba",1,["D",1787848294548],[180],"5e355af5-f49e-4221-87df-a392a9184c4b","db","f73349d0-a234-11f1-922e-1607ee4eb77e",3,["D",1787848379437],[186],"1d67c685-0a4f-421f-a3c8-63f7187dfcee","benchmark","f52e35aa-a234-11f1-971d-1607ee4eb77e",2,["D",1787848376051],[192],"pulls","branches","forks","envVars",{"_204":182,"_206":70},"connections",{"_204":70,"_205":70},"count","unfilledCount","emptyCount",{"_50":191,"_38":192,"_36":193,"_147":-5,"_48":49,"_47":194,"_149":66,"_45":480,"_13":70,"_40":177,"_42":-5,"_30":31,"_154":-5,"_242":481,"_33":482},{"_50":472,"_38":473,"_36":474,"_147":193,"_48":49,"_47":475,"_149":66,"_45":476,"_13":70,"_40":41,"_42":477,"_30":31,"_154":-5,"_242":478,"_33":479},{"_50":464,"_38":465,"_36":466,"_147":193,"_48":49,"_47":467,"_149":66,"_45":468,"_13":194,"_40":41,"_42":469,"_30":31,"_154":-5,"_242":470,"_33":471},{"_50":456,"_38":457,"_36":458,"_147":193,"_48":49,"_47":459,"_149":66,"_45":460,"_13":70,"_40":41,"_42":461,"_30":31,"_154":-5,"_242":462,"_33":463},{"_50":448,"_38":449,"_36":450,"_147":193,"_48":49,"_47":451,"_149":66,"_45":452,"_13":362,"_40":41,"_42":453,"_30":31,"_154":-5,"_242":454,"_33":455},{"_50":440,"_38":441,"_36":442,"_147":193,"_48":49,"_47":443,"_149":66,"_45":444,"_13":70,"_40":41,"_42":445,"_30":31,"_154":-5,"_242":446,"_33":447},{"_50":432,"_38":433,"_36":434,"_147":193,"_48":49,"_47":435,"_149":66,"_45":436,"_13":70,"_40":41,"_42":437,"_30":31,"_154":-5,"_242":438,"_33":439},{"_50":424,"_38":425,"_36":426,"_147":193,"_48":49,"_47":427,"_149":66,"_45":428,"_13":194,"_40":41,"_42":429,"_30":31,"_154":-5,"_242":430,"_33":431},{"_50":415,"_38":416,"_36":417,"_147":193,"_48":49,"_47":418,"_149":66,"_45":419,"_13":175,"_40":420,"_42":421,"_30":31,"_154":-5,"_242":422,"_33":423},{"_50":407,"_38":408,"_36":409,"_147":193,"_48":49,"_47":410,"_149":31,"_45":411,"_13":70,"_40":41,"_42":412,"_30":31,"_154":-5,"_242":413,"_33":414},{"_50":185,"_38":186,"_36":187,"_147":-5,"_48":49,"_47":188,"_149":66,"_45":404,"_13":70,"_40":177,"_42":-5,"_30":31,"_154":-5,"_242":405,"_33":406},{"_50":395,"_38":396,"_36":397,"_147":187,"_48":49,"_47":398,"_149":66,"_45":399,"_13":400,"_40":41,"_42":401,"_30":31,"_154":-5,"_242":402,"_33":403},{"_50":386,"_38":387,"_36":388,"_147":187,"_48":49,"_47":389,"_149":66,"_45":390,"_13":391,"_40":41,"_42":392,"_30":31,"_154":-5,"_242":393,"_33":394},{"_50":381,"_38":39,"_36":37,"_147":187,"_48":49,"_47":14,"_149":31,"_45":382,"_13":383,"_40":41,"_42":44,"_30":31,"_154":-5,"_242":384,"_33":385},{"_50":373,"_38":374,"_36":375,"_147":187,"_48":49,"_47":376,"_149":31,"_45":377,"_13":300,"_40":41,"_42":378,"_30":31,"_154":-5,"_242":379,"_33":380},{"_50":179,"_38":180,"_36":181,"_147":-5,"_48":49,"_47":182,"_149":66,"_45":370,"_13":70,"_40":177,"_42":-5,"_30":31,"_154":-5,"_242":371,"_33":372},{"_50":366,"_38":321,"_36":314,"_147":181,"_48":49,"_47":182,"_149":66,"_45":367,"_13":70,"_40":177,"_42":-5,"_30":31,"_154":-5,"_242":368,"_33":369},{"_50":357,"_38":358,"_36":359,"_147":314,"_48":49,"_47":360,"_149":66,"_45":361,"_13":362,"_40":41,"_42":363,"_30":31,"_154":-5,"_242":364,"_33":365},{"_50":348,"_38":349,"_36":350,"_147":314,"_48":49,"_47":351,"_149":66,"_45":352,"_13":353,"_40":41,"_42":354,"_30":31,"_154":-5,"_242":355,"_33":356},{"_50":340,"_38":341,"_36":342,"_147":314,"_48":49,"_47":343,"_149":66,"_45":344,"_13":317,"_40":41,"_42":345,"_30":31,"_154":-5,"_242":346,"_33":347},{"_50":331,"_38":332,"_36":333,"_147":314,"_48":49,"_47":334,"_149":66,"_45":335,"_13":336,"_40":41,"_42":337,"_30":31,"_154":-5,"_242":338,"_33":339},{"_50":322,"_38":323,"_36":324,"_147":314,"_48":49,"_47":325,"_149":66,"_45":326,"_13":327,"_40":41,"_42":328,"_30":31,"_154":-5,"_242":329,"_33":330},{"_50":311,"_38":312,"_36":313,"_147":314,"_48":49,"_47":315,"_149":66,"_45":316,"_13":317,"_40":41,"_42":318,"_30":31,"_154":-5,"_242":319,"_33":320},{"_50":304,"_38":305,"_36":306,"_147":181,"_48":49,"_47":182,"_149":66,"_45":307,"_13":70,"_40":26,"_42":308,"_30":31,"_154":-5,"_242":309,"_33":310},{"_50":295,"_38":296,"_36":297,"_147":181,"_48":49,"_47":298,"_149":66,"_45":299,"_13":300,"_40":41,"_42":301,"_30":31,"_154":-5,"_242":302,"_33":303},{"_50":287,"_38":288,"_36":289,"_147":181,"_48":49,"_47":290,"_149":66,"_45":291,"_13":182,"_40":41,"_42":292,"_30":31,"_154":-5,"_242":293,"_33":294},{"_50":279,"_38":280,"_36":281,"_147":181,"_48":49,"_47":282,"_149":66,"_45":283,"_13":182,"_40":41,"_42":284,"_30":31,"_154":-5,"_242":285,"_33":286},{"_50":271,"_38":272,"_36":273,"_147":181,"_48":49,"_47":274,"_149":66,"_45":275,"_13":70,"_40":41,"_42":276,"_30":31,"_154":-5,"_242":277,"_33":278},{"_50":263,"_38":264,"_36":265,"_147":181,"_48":49,"_47":266,"_149":66,"_45":267,"_13":188,"_40":41,"_42":268,"_30":31,"_154":-5,"_242":269,"_33":270},{"_50":172,"_38":173,"_36":174,"_147":-5,"_48":49,"_47":175,"_149":66,"_45":260,"_13":70,"_40":177,"_42":-5,"_30":31,"_154":-5,"_242":261,"_33":262},{"_50":251,"_38":252,"_36":253,"_147":174,"_48":49,"_47":254,"_149":66,"_45":255,"_13":256,"_40":41,"_42":257,"_30":31,"_154":-5,"_242":258,"_33":259},{"_50":165,"_38":166,"_36":167,"_147":-5,"_48":49,"_47":168,"_149":31,"_45":248,"_13":70,"_40":26,"_42":170,"_30":31,"_154":-5,"_242":249,"_33":250},{"_50":157,"_38":158,"_36":159,"_147":-5,"_48":49,"_47":160,"_149":66,"_45":245,"_13":162,"_40":26,"_42":163,"_30":31,"_154":-5,"_242":246,"_33":247},{"_50":145,"_38":146,"_36":64,"_147":-5,"_48":49,"_47":148,"_149":66,"_45":241,"_13":151,"_40":152,"_42":153,"_30":31,"_154":155,"_242":243,"_33":244},["D",1787865246169],"children",[],[146],["D",1787865252841],[],[158],["D",1789083117196],[],[166],"e3844660-98df-4332-ae1a-2a5a02b0a8c6","types.ts","0553d0c0-a235-11f1-94c6-1607ee4eb77e",280,["D",1787939464092],14,"export type BenchmarkStatus = \"running\" | \"completed\" | \"failed\";\nexport type MeasurementStatus = \"ok\" | \"error\";\n\nexport interface BenchmarkConfig {\n benchmarkVersion: string;\n label: string;\n modelId: string;\n runsPerProvider: number;\n maxOutputTokens: number;\n providerConcurrency: number;\n requestTimeoutMs: number;\n providers?: string[];\n}\n\nexport interface GatewayEndpoint {\n provider_name: string;\n name?: string;\n has_zdr?: boolean;\n has_no_training?: boolean;\n supports_implicit_caching?: boolean;\n pricing?: Record;\n latency_last_1h?: { p50?: number; p95?: number };\n throughput_last_1h?: { p50?: number; p95?: number };\n tags?: string[];\n [key: string]: unknown;\n}\n\nexport interface BenchmarkRunSummary {\n id: string;\n label: string;\n modelId: string;\n status: BenchmarkStatus;\n startedAt: string;\n completedAt: string | null;\n runsPerProvider: number;\n maxOutputTokens: number;\n providerCount: number;\n successfulMeasurements: number;\n failedMeasurements: number;\n error: string | null;\n}\n\nexport type StoredMeasurementOutcome = \"text\" | \"no_text\" | \"error\";\nexport type MeasurementErrorCode =\n | \"no_text\"\n | \"timeout\"\n | \"provider_error\"\n | null;\n\nexport interface BenchmarkMeasurement {\n runId: string;\n provider: string;\n attempt: number;\n status: MeasurementStatus;\n outcome: StoredMeasurementOutcome;\n errorCode: MeasurementErrorCode;\n startedAt: string;\n completedAt: string;\n ttftMs: number | null;\n durationMs: number | null;\n inputTokens: number | null;\n outputTokens: number | null;\n cacheReadTokens: number | null;\n cacheWriteTokens: number | null;\n costUsd: number | null;\n tokensPerSecond: number | null;\n responsePreview: string | null;\n error: string | null;\n providerMetadata: Record | null;\n providerInfo: GatewayEndpoint;\n}\n\nexport type MeasurementPublicStatus = \"success\" | \"error\";\n\nexport type MeasurementSortKey =\n | \"runStartedAt\"\n | \"provider\"\n | \"attempt\"\n | \"status\"\n | \"startedAt\"\n | \"ttftMs\"\n | \"durationMs\"\n | \"tokensPerSecond\"\n | \"inputTokens\"\n | \"outputTokens\"\n | \"cacheReadTokens\"\n | \"cacheWriteTokens\"\n | \"costUsd\"\n | \"message\";\n\nexport type MeasurementSortDirection = \"asc\" | \"desc\";\n\nexport interface MeasurementListItem {\n runId: string;\n runStartedAt: string;\n runStatus: BenchmarkStatus;\n provider: string;\n attempt: number;\n status: MeasurementPublicStatus;\n startedAt: string;\n completedAt: string;\n ttftMs: number | null;\n durationMs: number | null;\n inputTokens: number | null;\n outputTokens: number | null;\n cacheReadTokens: number | null;\n cacheWriteTokens: number | null;\n costUsd: number | null;\n tokensPerSecond: number | null;\n error: string | null;\n}\n\nexport interface MeasurementRunOption {\n id: string;\n startedAt: string;\n status: BenchmarkStatus;\n measurementCount: number;\n}\n\nexport interface MeasurementListResponse {\n items: MeasurementListItem[];\n total: number;\n limit: number;\n offset: number;\n providers: string[];\n attempts: number[];\n runs: MeasurementRunOption[];\n}\n\nexport interface BenchmarkRunDetail extends BenchmarkRunSummary {\n config: BenchmarkConfig;\n endpointSnapshot: GatewayEndpoint[];\n measurements: BenchmarkMeasurement[];\n}\n\nexport interface ProviderMetricPoint {\n runId: string;\n runStartedAt: string;\n successRate: number;\n attempts: number;\n successes: number;\n p50TtftMs: number | null;\n p95TtftMs: number | null;\n p50TokensPerSecond: number | null;\n p05TokensPerSecond: number | null;\n cacheHitRate: number | null;\n effectiveCostPerMillionTokensUsd: number | null;\n}\n\nexport interface ProviderResult {\n provider: string;\n runCount: number;\n attempts: number;\n successes: number;\n failures: number;\n successRate: number;\n p50TtftMs: number | null;\n p95TtftMs: number | null;\n p50DurationMs: number | null;\n p95DurationMs: number | null;\n p50TokensPerSecond: number | null;\n p05TokensPerSecond: number | null;\n cacheEligibleAttempts: number;\n cacheHits: number;\n cacheHitRate: number | null;\n effectiveCostPerMillionTokensUsd: number | null;\n history: ProviderMetricPoint[];\n}\n\nexport interface BenchmarkResults {\n modelId: string;\n runCount: number;\n attemptCount: number;\n successfulAttempts: number;\n failedAttempts: number;\n firstRunAt: string | null;\n lastCompletedAt: string | null;\n providers: ProviderResult[];\n}\n",[],[173,252],["D",1787848382634],[237],[173],"0921c7f1-3a63-4c80-b035-7a8887bae310","root.tsx","01a04410-0100-7214-ba2d-918a7f307571",44,["D",1787852742071],"/** @jsxImportSource npm:hono@4/jsx */\nimport { raw } from \"npm:hono@4/html\";\nimport { immutableFileUrl } from \"https://esm.town/v/std/utils/index.ts\";\n\nexport function Root() {\n return (\n <>\n {raw(\"\")}\n \n \n \n \n \n \n GLM-5.3-Flash Provider Benchmark\n

Diff · v328