GLM-5.3-Flash Provider Benchmark

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_340":341,"_342":343},"routes/_app.x.$handle.$name",{"_101":100,"_32":46,"_110":91,"_27":111,"_13":14,"_112":-5,"_11":113,"_114":115,"_116":117,"_118":119,"_120":121,"_20":-5,"_122":-5,"_123":-5,"_124":25,"_125":126,"_127":60,"_128":-5,"_129":-5,"_130":131},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_44":45,"_32":46,"_39":47,"_48":-5,"_49":25,"_50":51,"_52":51,"_53":54,"_55":-5,"_18":56,"_57":58,"_59":60,"_61":-5,"_62":-5,"_63":64,"_65":66,"_67":68,"_69":70,"_71":72,"_73":-5,"_74":75,"_76":75,"_77":78,"_79":78,"_80":78,"_81":75,"_82":75,"_83":84,"_85":86,"_87":88,"_89":25},"version",47,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":33,"_34":35,"_36":38,"_39":40,"_41":14,"_42":43,"_24":25},"previous",{"_30":31,"_32":33,"_34":35,"_36":37},"deleted",false,"deletedHere","path","db/schema.ts","latestPath","fileId","0bf3433e-a235-11f1-922e-1607ee4eb77e","name","schema.ts","fileType","script","content","import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\n\nexport { sqlite };\n\nexport async function ensureSchema(): Promise {\n await sqlite.batch([\n {\n sql: `CREATE TABLE IF NOT EXISTS benchmark_runs (\n id TEXT PRIMARY KEY,\n label TEXT NOT NULL,\n model_id TEXT NOT NULL,\n status TEXT NOT NULL CHECK (status IN ('running', 'completed', 'failed')),\n started_at TEXT NOT NULL,\n completed_at TEXT,\n runs_per_provider INTEGER NOT NULL,\n max_output_tokens INTEGER NOT NULL,\n provider_count INTEGER NOT NULL DEFAULT 0,\n successful_measurements INTEGER NOT NULL DEFAULT 0,\n failed_measurements INTEGER NOT NULL DEFAULT 0,\n config_json TEXT NOT NULL,\n endpoint_snapshot_json TEXT NOT NULL,\n error TEXT\n )`,\n },\n {\n sql: `CREATE TABLE IF NOT EXISTS benchmark_measurements (\n run_id TEXT NOT NULL REFERENCES benchmark_runs(id) ON DELETE CASCADE,\n provider TEXT NOT NULL,\n attempt INTEGER NOT NULL,\n status TEXT NOT NULL CHECK (status IN ('ok', 'error')),\n started_at TEXT NOT NULL,\n completed_at TEXT NOT NULL,\n ttft_ms REAL,\n duration_ms REAL,\n input_tokens INTEGER,\n output_tokens INTEGER,\n cache_read_tokens INTEGER,\n cache_write_tokens INTEGER,\n cost_usd REAL,\n tokens_per_second REAL,\n response_preview TEXT,\n error TEXT,\n provider_metadata_json TEXT,\n provider_info_json TEXT NOT NULL,\n PRIMARY KEY (run_id, provider, attempt)\n )`,\n },\n {\n sql: \"CREATE INDEX IF NOT EXISTS benchmark_runs_started_at ON benchmark_runs(started_at DESC)\",\n },\n {\n sql: \"CREATE INDEX IF NOT EXISTS benchmark_measurements_run_id ON benchmark_measurements(run_id)\",\n },\n ]);\n}\n","import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\n\nexport { sqlite };\n\nexport async function ensureSchema(): Promise {\n await sqlite.batch([\n {\n sql: `CREATE TABLE IF NOT EXISTS benchmark_runs (\n id TEXT PRIMARY KEY,\n label TEXT NOT NULL,\n model_id TEXT NOT NULL,\n status TEXT NOT NULL CHECK (status IN ('running', 'completed', 'failed')),\n started_at TEXT NOT NULL,\n completed_at TEXT,\n runs_per_provider INTEGER NOT NULL,\n max_output_tokens INTEGER NOT NULL,\n provider_count INTEGER NOT NULL DEFAULT 0,\n successful_measurements INTEGER NOT NULL DEFAULT 0,\n failed_measurements INTEGER NOT NULL DEFAULT 0,\n config_json TEXT NOT NULL,\n endpoint_snapshot_json TEXT NOT NULL,\n error TEXT\n )`,\n },\n {\n sql: `CREATE TABLE IF NOT EXISTS benchmark_provider_snapshots (\n run_id TEXT NOT NULL REFERENCES benchmark_runs(id) ON DELETE CASCADE,\n provider TEXT NOT NULL,\n model_id TEXT NOT NULL,\n discovered_at TEXT NOT NULL,\n supports_implicit_caching INTEGER,\n context_length INTEGER,\n max_completion_tokens INTEGER,\n input_price_per_token TEXT,\n output_price_per_token TEXT,\n cache_read_price_per_token TEXT,\n cache_write_price_per_token TEXT,\n endpoint_json TEXT NOT NULL,\n PRIMARY KEY (run_id, provider)\n )`,\n },\n {\n sql: `CREATE TABLE IF NOT EXISTS benchmark_measurements (\n run_id TEXT NOT NULL REFERENCES benchmark_runs(id) ON DELETE CASCADE,\n provider TEXT NOT NULL,\n attempt INTEGER NOT NULL,\n status TEXT NOT NULL CHECK (status IN ('ok', 'error')),\n started_at TEXT NOT NULL,\n completed_at TEXT NOT NULL,\n ttft_ms REAL,\n duration_ms REAL,\n input_tokens INTEGER,\n output_tokens INTEGER,\n cache_read_tokens INTEGER,\n cache_write_tokens INTEGER,\n cost_usd REAL,\n tokens_per_second REAL,\n response_preview TEXT,\n error TEXT,\n provider_metadata_json TEXT,\n provider_info_json TEXT NOT NULL,\n PRIMARY KEY (run_id, provider, attempt)\n )`,\n },\n {\n sql: \"CREATE INDEX IF NOT EXISTS benchmark_runs_started_at ON benchmark_runs(started_at DESC)\",\n },\n {\n sql: \"CREATE INDEX IF NOT EXISTS benchmark_provider_snapshots_provider ON benchmark_provider_snapshots(provider, discovered_at DESC)\",\n },\n {\n sql: \"CREATE INDEX IF NOT EXISTS benchmark_measurements_run_id ON benchmark_measurements(run_id)\",\n },\n ]);\n}\n","createdAt",["D",1787854871402],"branchVersion","branchId","24cb14b4-1851-4548-adc6-43a574860a3d","id","d055324c-28f9-4529-80dd-e9bbfbdbeb65","glm53-flash-benchmark",["D",1787848294548],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Run and compare AI Gateway providers","imageUrl","a0bf3b31-15a5-4d5c-880e-4b1e22c9bc18","pinnedFileId","01a04410-0100-7214-ba2d-a172e514af90","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","ba7bb5fb-1abb-4ffb-8f3d-7c244782e5f7","tursoName","d055324c28f9452980dde9bbfbdbeb65","tursoHostname","d055324c28f9452980dde9bbfbdbeb65-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789500119686],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_101":100,"_109":105},"org",{"_101":100,"_102":103,"_104":105,"_18":-5,"_106":107,"_108":60},"branch",{"_13":90,"_32":91,"_44":43,"_92":45,"_93":-5,"_94":-5,"_95":-5,"_96":-5,"_97":-5,"_98":46,"_99":100},"isOwned",338,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","valdottown","handle","displayName","Val Town ","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvdXBsb2FkZWQvaW1nXzJyWnQ5cFE0c3pleVRCSEVTWmxzaXJsTUlGciJ9","tier","business","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_44":45,"_32":46,"_39":335,"_48":-5,"_49":25,"_50":51,"_52":51,"_53":54,"_55":-5,"_18":56,"_57":58,"_59":60,"_61":-5,"_62":-5,"_63":64,"_65":66,"_67":68,"_69":70,"_71":336,"_73":-5,"_74":75,"_76":75,"_77":78,"_79":78,"_80":78,"_81":75,"_82":75,"_83":337,"_85":338,"_87":339,"_89":25},"tree",[191,196,199,207,209,210],"files",[191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210],"counts",{"_181":64,"_182":144,"_183":167,"_184":185,"_186":187},"contents",[132,133,134,135,136,137],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_44":176,"_32":177,"_30":178,"_140":-5,"_42":43,"_41":144,"_142":60,"_39":179,"_13":64,"_34":162,"_36":-5,"_24":25,"_147":-5,"_27":180},{"_44":170,"_32":171,"_30":172,"_140":-5,"_42":43,"_41":173,"_142":60,"_39":174,"_13":64,"_34":162,"_36":-5,"_24":25,"_147":-5,"_27":175},{"_44":164,"_32":165,"_30":166,"_140":-5,"_42":43,"_41":167,"_142":60,"_39":168,"_13":64,"_34":162,"_36":-5,"_24":25,"_147":-5,"_27":169},{"_44":158,"_32":159,"_30":160,"_140":-5,"_42":43,"_41":155,"_142":60,"_39":161,"_13":64,"_34":162,"_36":-5,"_24":25,"_147":-5,"_27":163},{"_44":150,"_32":151,"_30":152,"_140":-5,"_42":43,"_41":153,"_142":25,"_39":154,"_13":155,"_34":20,"_36":156,"_24":25,"_147":-5,"_27":157},{"_44":138,"_32":139,"_30":58,"_140":-5,"_42":43,"_41":141,"_142":25,"_39":143,"_13":144,"_34":145,"_36":146,"_24":25,"_147":148,"_27":149},"022125a0-5e3b-4a22-a73d-17a06bdd9b86","index.ts","parentId",28,"latest",["D",1787852248745],2,"http","import { parseVal, serveImmutableFile } from \"https://esm.town/v/std/utils/index.ts\";\nimport { Hono } from \"npm:hono@4\";\nimport { getBenchmarkResults } from \"./db/queries.ts\";\nimport { Root } from \"./frontend/root.tsx\";\n\nconst MODEL_ID = \"zai/glm-5.3-flash\";\nconst app = new Hono();\n\napp.get(\"/\", (c) => c.html(Root()));\n\napp.get(\"/__immutable/*\", (c) => serveImmutableFile(c.req.path));\n\napp.get(\"/api/results\", async (c) => {\n const results = await getBenchmarkResults(MODEL_ID);\n c.header(\"Cache-Control\", \"no-store\");\n c.header(\"X-Content-Type-Options\", \"nosniff\");\n return c.json({ results });\n});\n\napp.get(\"/source\", (c) => c.redirect(parseVal().links.self.val));\n\napp.onError((err) => Promise.reject(err));\n\nexport default app.fetch;\n","label","glm53flash",[139],"426f4855-6aea-459b-91a4-ff44220371d7","README.md","01a04410-0100-7214-ba2d-9ca990fb87d3",45,["D",1787852756447],4,"# GLM-5.3-Flash Gateway Benchmark\n\nA public, read-only Val Town page that compares providers serving\n`zai/glm-5.3-flash` through Vercel AI Gateway.\n\n## How results work\n\nEvery provider attempt is stored as an individual row in the val-scoped\n`benchmark_measurements` SQLite table. The page pools measurements from every\ncompleted benchmark run, so a newly completed run is included automatically.\n\nThe provider table reports:\n\n- Reliability: successful requests divided by attempted requests.\n- Median TTFT across successful requests.\n- Median output tokens per second over the full request.\n- Cache hits among eligible repeated successful requests.\n- Median observed Gateway cost per successful request.\n\n## Run a benchmark\n\n1. Edit [benchmark/config.ts](./benchmark/config.ts) if needed.\n2. Open [benchmark/run.ts](./benchmark/run.ts) in Val Town and click **Run**.\n3. Refresh the public page after the benchmark completes.\n\nThe runner checkpoints each provider attempt and resumes an interrupted run with\nthe same configuration.\n\n## Access\n\nThe Gateway credential is stored as the encrypted\n`VERCEL_AI_GATEWAY_API_KEY` environment variable. The HTTP app exposes only\nread-only routes, so public visitors cannot start paid model requests.\n\n## Structure\n\n- `benchmark/` — configuration, fixed conversation, and manual runner.\n- `db/` — val-scoped SQLite schema, writes, and cross-run aggregation.\n- `frontend/` — React interface styled with Twind.\n- `shared/` — shared benchmark and result types.\n",[151],"57180eb9-f846-47ed-9f7f-5ec6aa1d02be","shared","f91bd0d2-a234-11f1-8cec-1607ee4eb77e",["D",1787848382634],"directory",[159],"6f24ec24-8f0c-4d0d-b0d3-fbc1bac995ce","frontend","01a04410-0100-7214-ba2d-95062f41a4ba",1,["D",1787848294548],[165],"5e355af5-f49e-4221-87df-a392a9184c4b","db","f73349d0-a234-11f1-922e-1607ee4eb77e",3,["D",1787848379437],[171],"1d67c685-0a4f-421f-a3c8-63f7187dfcee","benchmark","f52e35aa-a234-11f1-971d-1607ee4eb77e",["D",1787848376051],[177],"pulls","branches","forks","envVars",{"_188":167,"_190":64},"connections",{"_188":64,"_189":64},"count","unfilledCount","emptyCount",{"_44":176,"_32":177,"_30":178,"_140":-5,"_42":43,"_41":144,"_142":60,"_39":332,"_13":64,"_34":162,"_36":-5,"_24":25,"_147":-5,"_212":333,"_27":334},{"_44":324,"_32":325,"_30":326,"_140":178,"_42":43,"_41":327,"_142":25,"_39":328,"_13":167,"_34":35,"_36":329,"_24":25,"_147":-5,"_212":330,"_27":331},{"_44":316,"_32":317,"_30":318,"_140":178,"_42":43,"_41":319,"_142":25,"_39":320,"_13":167,"_34":35,"_36":321,"_24":25,"_147":-5,"_212":322,"_27":323},{"_44":308,"_32":309,"_30":310,"_140":178,"_42":43,"_41":311,"_142":25,"_39":312,"_13":167,"_34":35,"_36":313,"_24":25,"_147":-5,"_212":314,"_27":315},{"_44":300,"_32":301,"_30":302,"_140":178,"_42":43,"_41":303,"_142":25,"_39":304,"_13":64,"_34":35,"_36":305,"_24":25,"_147":-5,"_212":306,"_27":307},{"_44":170,"_32":171,"_30":172,"_140":-5,"_42":43,"_41":173,"_142":60,"_39":297,"_13":64,"_34":162,"_36":-5,"_24":25,"_147":-5,"_212":298,"_27":299},{"_44":288,"_32":289,"_30":290,"_140":172,"_42":43,"_41":291,"_142":25,"_39":292,"_13":293,"_34":35,"_36":294,"_24":25,"_147":-5,"_212":295,"_27":296},{"_44":284,"_32":33,"_30":31,"_140":172,"_42":43,"_41":14,"_142":25,"_39":285,"_13":167,"_34":35,"_36":38,"_24":25,"_147":-5,"_212":286,"_27":287},{"_44":164,"_32":165,"_30":166,"_140":-5,"_42":43,"_41":167,"_142":60,"_39":281,"_13":64,"_34":162,"_36":-5,"_24":25,"_147":-5,"_212":282,"_27":283},{"_44":277,"_32":268,"_30":262,"_140":166,"_42":43,"_41":167,"_142":60,"_39":278,"_13":64,"_34":162,"_36":-5,"_24":25,"_147":-5,"_212":279,"_27":280},{"_44":269,"_32":270,"_30":271,"_140":262,"_42":43,"_41":272,"_142":25,"_39":273,"_13":173,"_34":35,"_36":274,"_24":25,"_147":-5,"_212":275,"_27":276},{"_44":259,"_32":260,"_30":261,"_140":262,"_42":43,"_41":263,"_142":25,"_39":264,"_13":144,"_34":35,"_36":265,"_24":25,"_147":-5,"_212":266,"_27":267},{"_44":252,"_32":253,"_30":254,"_140":166,"_42":43,"_41":167,"_142":60,"_39":255,"_13":64,"_34":20,"_36":256,"_24":25,"_147":-5,"_212":257,"_27":258},{"_44":244,"_32":245,"_30":246,"_140":166,"_42":43,"_41":247,"_142":25,"_39":248,"_13":144,"_34":35,"_36":249,"_24":25,"_147":-5,"_212":250,"_27":251},{"_44":237,"_32":238,"_30":239,"_140":166,"_42":43,"_41":167,"_142":25,"_39":240,"_13":64,"_34":35,"_36":241,"_24":25,"_147":-5,"_212":242,"_27":243},{"_44":229,"_32":230,"_30":231,"_140":166,"_42":43,"_41":232,"_142":60,"_39":233,"_13":173,"_34":35,"_36":234,"_24":25,"_147":-5,"_212":235,"_27":236},{"_44":158,"_32":159,"_30":160,"_140":-5,"_42":43,"_41":155,"_142":60,"_39":226,"_13":64,"_34":162,"_36":-5,"_24":25,"_147":-5,"_212":227,"_27":228},{"_44":218,"_32":219,"_30":220,"_140":160,"_42":43,"_41":221,"_142":25,"_39":222,"_13":167,"_34":35,"_36":223,"_24":25,"_147":-5,"_212":224,"_27":225},{"_44":150,"_32":151,"_30":152,"_140":-5,"_42":43,"_41":153,"_142":25,"_39":215,"_13":155,"_34":20,"_36":156,"_24":25,"_147":-5,"_212":216,"_27":217},{"_44":138,"_32":139,"_30":58,"_140":-5,"_42":43,"_41":141,"_142":25,"_39":211,"_13":144,"_34":145,"_36":146,"_24":25,"_147":148,"_212":213,"_27":214},["D",1787852248745],"children",[],[139],["D",1787852756447],[],[151],"b4cbfd8a-1be8-41a2-a897-01dfb796bbac","types.ts","0553d0c0-a235-11f1-94c6-1607ee4eb77e",25,["D",1787852207538],"export type BenchmarkStatus = \"running\" | \"completed\" | \"failed\";\nexport type MeasurementStatus = \"ok\" | \"error\";\n\nexport interface BenchmarkConfig {\n label: string;\n modelId: string;\n runsPerProvider: number;\n maxOutputTokens: number;\n providerConcurrency: number;\n requestTimeoutMs: number;\n providers?: string[];\n}\n\nexport interface GatewayEndpoint {\n provider_name: string;\n name?: string;\n has_zdr?: boolean;\n has_no_training?: boolean;\n supports_implicit_caching?: boolean;\n pricing?: Record;\n latency_last_1h?: { p50?: number; p95?: number };\n throughput_last_1h?: { p50?: number; p95?: number };\n tags?: string[];\n [key: string]: unknown;\n}\n\nexport interface BenchmarkRunSummary {\n id: string;\n label: string;\n modelId: string;\n status: BenchmarkStatus;\n startedAt: string;\n completedAt: string | null;\n runsPerProvider: number;\n maxOutputTokens: number;\n providerCount: number;\n successfulMeasurements: number;\n failedMeasurements: number;\n error: string | null;\n}\n\nexport interface BenchmarkMeasurement {\n runId: string;\n provider: string;\n attempt: number;\n status: MeasurementStatus;\n startedAt: string;\n completedAt: string;\n ttftMs: number | null;\n durationMs: number | null;\n inputTokens: number | null;\n outputTokens: number | null;\n cacheReadTokens: number | null;\n cacheWriteTokens: number | null;\n costUsd: number | null;\n tokensPerSecond: number | null;\n responsePreview: string | null;\n error: string | null;\n providerMetadata: Record | null;\n providerInfo: GatewayEndpoint;\n}\n\nexport interface BenchmarkRunDetail extends BenchmarkRunSummary {\n config: BenchmarkConfig;\n endpointSnapshot: GatewayEndpoint[];\n measurements: BenchmarkMeasurement[];\n}\n\nexport interface ProviderResult {\n provider: string;\n runCount: number;\n attempts: number;\n successes: number;\n failures: number;\n successRate: number;\n medianTtftMs: number | null;\n medianTokensPerSecond: number | null;\n cacheEligibleAttempts: number;\n cacheHits: number;\n cacheHitRate: number | null;\n medianCostPerSuccessUsd: number | null;\n}\n\nexport interface BenchmarkResults {\n modelId: string;\n runCount: number;\n attemptCount: number;\n successfulAttempts: number;\n failedAttempts: number;\n firstRunAt: string | null;\n lastCompletedAt: string | null;\n providers: ProviderResult[];\n}\n",[],[159,219],["D",1787848382634],[208],[159],"0921c7f1-3a63-4c80-b035-7a8887bae310","root.tsx","01a04410-0100-7214-ba2d-918a7f307571",44,["D",1787852742071],"/** @jsxImportSource npm:hono@4/jsx */\nimport { raw } from \"npm:hono@4/html\";\nimport { immutableFileUrl } from \"https://esm.town/v/std/utils/index.ts\";\n\nexport function Root() {\n return (\n <>\n {raw(\"\")}\n \n \n \n \n \n \n GLM-5.3-Flash Provider Benchmark\n

Diff · v47