GLM-5.3-Flash Provider Benchmark

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_399":400,"_401":402},"routes/_app.x.$handle.$name",{"_101":100,"_32":46,"_110":91,"_27":111,"_13":14,"_112":-5,"_11":113,"_114":115,"_116":117,"_118":119,"_120":121,"_20":-5,"_122":-5,"_123":-5,"_124":25,"_125":126,"_127":60,"_128":-5,"_129":-5,"_130":131},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_44":45,"_32":46,"_39":47,"_48":-5,"_49":25,"_50":51,"_52":51,"_53":54,"_55":-5,"_18":56,"_57":58,"_59":60,"_61":-5,"_62":-5,"_63":64,"_65":66,"_67":68,"_69":70,"_71":72,"_73":-5,"_74":75,"_76":75,"_77":78,"_79":78,"_80":78,"_81":75,"_82":75,"_83":84,"_85":86,"_87":88,"_89":25},"version",174,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":33,"_34":35,"_36":38,"_39":40,"_41":14,"_42":43,"_24":25},"previous",{"_30":31,"_32":33,"_34":35,"_36":37},"deleted",false,"deletedHere","path","db/measurements.ts","latestPath","fileId","f1bcaed4-a247-11f1-b3b5-1607ee4eb77e","name","measurements.ts","fileType","script","content","import type {\n BenchmarkStatus,\n MeasurementListItem,\n MeasurementListResponse,\n MeasurementOutcome,\n MeasurementSortDirection,\n MeasurementSortKey,\n} from \"../shared/types.ts\";\nimport { ensureSchema, sqlite } from \"./schema.ts\";\n\ntype Row = Record;\n\nconst NO_TEXT_ERROR = \"Stream completed without text output\";\nconst OUTCOME_SQL = `CASE\n WHEN m.status = 'ok' THEN 'text'\n WHEN m.error = '${NO_TEXT_ERROR}' THEN 'no_text'\n ELSE 'error'\nEND`;\n\nconst SORT_COLUMNS: Record = {\n runStartedAt: \"r.started_at\",\n provider: \"m.provider\",\n attempt: \"m.attempt\",\n outcome: OUTCOME_SQL,\n startedAt: \"m.started_at\",\n ttftMs: \"m.ttft_ms\",\n durationMs: \"m.duration_ms\",\n tokensPerSecond: \"m.tokens_per_second\",\n inputTokens: \"m.input_tokens\",\n outputTokens: \"m.output_tokens\",\n cacheReadTokens: \"m.cache_read_tokens\",\n cacheWriteTokens: \"m.cache_write_tokens\",\n costUsd: \"m.cost_usd\",\n message: \"COALESCE(m.error, m.response_preview, '')\",\n};\n\nconst NULLABLE_SORT_KEYS = new Set([\n \"ttftMs\",\n \"durationMs\",\n \"tokensPerSecond\",\n \"inputTokens\",\n \"outputTokens\",\n \"cacheReadTokens\",\n \"cacheWriteTokens\",\n \"costUsd\",\n]);\n\nfunction asNumber(value: unknown): number {\n return typeof value === \"number\" ? value : Number(value ?? 0);\n}\n\nfunction asNullableNumber(value: unknown): number | null {\n if (value === null || value === undefined) return null;\n const parsed = asNumber(value);\n return Number.isFinite(parsed) ? parsed : null;\n}\n\nfunction asPublicText(value: unknown, maxLength: number): string | null {\n if (value === null || value === undefined) return null;\n const sanitized = String(value)\n .replace(/[\\u0000-\\u0008\\u000b\\u000c\\u000e-\\u001f\\u007f]/g, \"\")\n .slice(0, maxLength);\n return sanitized || null;\n}\n\nfunction asOutcome(value: unknown): MeasurementOutcome {\n if (value === \"text\" || value === \"no_text\") return value;\n return \"error\";\n}\n\nfunction mapItem(row: Row): MeasurementListItem {\n return {\n runId: String(row.run_id),\n runStartedAt: String(row.run_started_at),\n runStatus: String(row.run_status) as BenchmarkStatus,\n provider: String(row.provider),\n attempt: asNumber(row.attempt),\n outcome: asOutcome(row.outcome),\n startedAt: String(row.started_at),\n completedAt: String(row.completed_at),\n ttftMs: asNullableNumber(row.ttft_ms),\n durationMs: asNullableNumber(row.duration_ms),\n inputTokens: asNullableNumber(row.input_tokens),\n outputTokens: asNullableNumber(row.output_tokens),\n cacheReadTokens: asNullableNumber(row.cache_read_tokens),\n cacheWriteTokens: asNullableNumber(row.cache_write_tokens),\n costUsd: asNullableNumber(row.cost_usd),\n tokensPerSecond: asNullableNumber(row.tokens_per_second),\n responsePreview: asPublicText(row.response_preview, 1_000),\n error: asPublicText(row.error, 1_000),\n };\n}\n\nexport interface ListMeasurementsInput {\n modelId: string;\n provider?: string;\n outcome?: MeasurementOutcome;\n attempt?: number;\n runId?: string;\n sortKey: MeasurementSortKey;\n direction: MeasurementSortDirection;\n limit: number;\n offset: number;\n}\n\nexport async function listMeasurements(\n input: ListMeasurementsInput,\n): Promise {\n await ensureSchema();\n\n const where = [\"r.model_id = ?\"];\n const args: Array = [input.modelId];\n\n if (input.provider) {\n where.push(\"m.provider = ?\");\n args.push(input.provider);\n }\n if (input.outcome === \"text\") {\n where.push(\"m.status = 'ok'\");\n } else if (input.outcome === \"no_text\") {\n where.push(\"m.status = 'error' AND m.error = ?\");\n args.push(NO_TEXT_ERROR);\n } else if (input.outcome === \"error\") {\n where.push(\"m.status = 'error' AND COALESCE(m.error, '') <> ?\");\n args.push(NO_TEXT_ERROR);\n }\n if (input.attempt !== undefined) {\n where.push(\"m.attempt = ?\");\n args.push(input.attempt);\n }\n if (input.runId) {\n where.push(\"m.run_id = ?\");\n args.push(input.runId);\n }\n\n const whereSql = where.join(\" AND \");\n const sortColumn = SORT_COLUMNS[input.sortKey];\n const direction = input.direction === \"asc\" ? \"ASC\" : \"DESC\";\n const nullsLast = NULLABLE_SORT_KEYS.has(input.sortKey)\n ? `(${sortColumn}) IS NULL ASC, `\n : \"\";\n const limit = Math.max(1, Math.min(input.limit, 200));\n const offset = Math.max(0, input.offset);\n\n const results = await sqlite.batch([\n {\n sql: `SELECT COUNT(*) AS total\n FROM benchmark_measurements m\n INNER JOIN benchmark_runs r ON r.id = m.run_id\n WHERE ${whereSql}`,\n args,\n },\n {\n sql: `SELECT\n m.run_id,\n r.started_at AS run_started_at,\n r.status AS run_status,\n m.provider,\n m.attempt,\n ${OUTCOME_SQL} AS outcome,\n m.started_at,\n m.completed_at,\n m.ttft_ms,\n m.duration_ms,\n m.input_tokens,\n m.output_tokens,\n m.cache_read_tokens,\n m.cache_write_tokens,\n m.cost_usd,\n m.tokens_per_second,\n m.response_preview,\n m.error\n FROM benchmark_measurements m\n INNER JOIN benchmark_runs r ON r.id = m.run_id\n WHERE ${whereSql}\n ORDER BY ${nullsLast}${sortColumn} ${direction},\n r.started_at DESC, m.run_id ASC, m.provider ASC, m.attempt ASC\n LIMIT ? OFFSET ?`,\n args: [...args, limit, offset],\n },\n {\n sql: `SELECT DISTINCT m.provider\n FROM benchmark_measurements m\n INNER JOIN benchmark_runs r ON r.id = m.run_id\n WHERE r.model_id = ?\n ORDER BY m.provider ASC`,\n args: [input.modelId],\n },\n {\n sql: `SELECT DISTINCT m.attempt\n FROM benchmark_measurements m\n INNER JOIN benchmark_runs r ON r.id = m.run_id\n WHERE r.model_id = ?\n ORDER BY m.attempt ASC`,\n args: [input.modelId],\n },\n {\n sql: `SELECT\n r.id,\n r.started_at,\n r.status,\n COUNT(m.attempt) AS measurement_count\n FROM benchmark_runs r\n LEFT JOIN benchmark_measurements m ON m.run_id = r.id\n WHERE r.model_id = ?\n GROUP BY r.id, r.started_at, r.status\n ORDER BY r.started_at DESC\n LIMIT 200`,\n args: [input.modelId],\n },\n ]);\n\n const countRow = (results[0].rows as Row[])[0];\n const itemRows = results[1].rows as Row[];\n const providerRows = results[2].rows as Row[];\n const attemptRows = results[3].rows as Row[];\n const runRows = results[4].rows as Row[];\n\n return {\n items: itemRows.map(mapItem),\n total: countRow ? asNumber(countRow.total) : 0,\n limit,\n offset,\n providers: providerRows.map((row) => String(row.provider)),\n attempts: attemptRows.map((row) => asNumber(row.attempt)),\n runs: runRows.map((row) => ({\n id: String(row.id),\n startedAt: String(row.started_at),\n status: String(row.status) as BenchmarkStatus,\n measurementCount: asNumber(row.measurement_count),\n })),\n };\n}\n","import type {\n BenchmarkStatus,\n MeasurementListItem,\n MeasurementListResponse,\n MeasurementOutcome,\n MeasurementSortDirection,\n MeasurementSortKey,\n} from \"../shared/types.ts\";\nimport { ensureSchema, sqlite } from \"./schema.ts\";\n\ntype Row = Record;\n\nconst OUTCOME_SQL = \"m.outcome_code\";\n\nconst SORT_COLUMNS: Record = {\n runStartedAt: \"r.started_at\",\n provider: \"m.provider\",\n attempt: \"m.attempt\",\n outcome: OUTCOME_SQL,\n startedAt: \"m.started_at\",\n ttftMs: \"m.ttft_ms\",\n durationMs: \"m.duration_ms\",\n tokensPerSecond: \"m.tokens_per_second\",\n inputTokens: \"m.input_tokens\",\n outputTokens: \"m.output_tokens\",\n cacheReadTokens: \"m.cache_read_tokens\",\n cacheWriteTokens: \"m.cache_write_tokens\",\n costUsd: \"m.cost_usd\",\n message: \"COALESCE(m.error, m.response_preview, '')\",\n};\n\nconst NULLABLE_SORT_KEYS = new Set([\n \"ttftMs\",\n \"durationMs\",\n \"tokensPerSecond\",\n \"inputTokens\",\n \"outputTokens\",\n \"cacheReadTokens\",\n \"cacheWriteTokens\",\n \"costUsd\",\n]);\n\nfunction asNumber(value: unknown): number {\n return typeof value === \"number\" ? value : Number(value ?? 0);\n}\n\nfunction asNullableNumber(value: unknown): number | null {\n if (value === null || value === undefined) return null;\n const parsed = asNumber(value);\n return Number.isFinite(parsed) ? parsed : null;\n}\n\nfunction asPublicText(value: unknown, maxLength: number): string | null {\n if (value === null || value === undefined) return null;\n const sanitized = String(value)\n .replace(/[\\u0000-\\u0008\\u000b\\u000c\\u000e-\\u001f\\u007f]/g, \"\")\n .slice(0, maxLength);\n return sanitized || null;\n}\n\nfunction asOutcome(value: unknown): MeasurementOutcome {\n if (value === \"text\" || value === \"no_text\") return value;\n return \"error\";\n}\n\nfunction mapItem(row: Row): MeasurementListItem {\n return {\n runId: String(row.run_id),\n runStartedAt: String(row.run_started_at),\n runStatus: String(row.run_status) as BenchmarkStatus,\n provider: String(row.provider),\n attempt: asNumber(row.attempt),\n outcome: asOutcome(row.outcome),\n startedAt: String(row.started_at),\n completedAt: String(row.completed_at),\n ttftMs: asNullableNumber(row.ttft_ms),\n durationMs: asNullableNumber(row.duration_ms),\n inputTokens: asNullableNumber(row.input_tokens),\n outputTokens: asNullableNumber(row.output_tokens),\n cacheReadTokens: asNullableNumber(row.cache_read_tokens),\n cacheWriteTokens: asNullableNumber(row.cache_write_tokens),\n costUsd: asNullableNumber(row.cost_usd),\n tokensPerSecond: asNullableNumber(row.tokens_per_second),\n responsePreview: asPublicText(row.response_preview, 1_000),\n error: asPublicText(row.error, 1_000),\n };\n}\n\nexport interface ListMeasurementsInput {\n modelId: string;\n provider?: string;\n outcome?: MeasurementOutcome;\n attempt?: number;\n runId?: string;\n sortKey: MeasurementSortKey;\n direction: MeasurementSortDirection;\n limit: number;\n offset: number;\n}\n\nexport async function listMeasurements(\n input: ListMeasurementsInput,\n): Promise {\n await ensureSchema();\n\n const where = [\"r.model_id = ?\"];\n const args: Array = [input.modelId];\n\n if (input.provider) {\n where.push(\"m.provider = ?\");\n args.push(input.provider);\n }\n if (input.outcome === \"text\") {\n where.push(\"m.status = 'ok'\");\n } else if (input.outcome === \"no_text\") {\n where.push(\"m.status = 'error' AND m.error = ?\");\n args.push(NO_TEXT_ERROR);\n } else if (input.outcome === \"error\") {\n where.push(\"m.status = 'error' AND COALESCE(m.error, '') <> ?\");\n args.push(NO_TEXT_ERROR);\n }\n if (input.attempt !== undefined) {\n where.push(\"m.attempt = ?\");\n args.push(input.attempt);\n }\n if (input.runId) {\n where.push(\"m.run_id = ?\");\n args.push(input.runId);\n }\n\n const whereSql = where.join(\" AND \");\n const sortColumn = SORT_COLUMNS[input.sortKey];\n const direction = input.direction === \"asc\" ? \"ASC\" : \"DESC\";\n const nullsLast = NULLABLE_SORT_KEYS.has(input.sortKey)\n ? `(${sortColumn}) IS NULL ASC, `\n : \"\";\n const limit = Math.max(1, Math.min(input.limit, 200));\n const offset = Math.max(0, input.offset);\n\n const results = await sqlite.batch([\n {\n sql: `SELECT COUNT(*) AS total\n FROM benchmark_measurements m\n INNER JOIN benchmark_runs r ON r.id = m.run_id\n WHERE ${whereSql}`,\n args,\n },\n {\n sql: `SELECT\n m.run_id,\n r.started_at AS run_started_at,\n r.status AS run_status,\n m.provider,\n m.attempt,\n ${OUTCOME_SQL} AS outcome,\n m.started_at,\n m.completed_at,\n m.ttft_ms,\n m.duration_ms,\n m.input_tokens,\n m.output_tokens,\n m.cache_read_tokens,\n m.cache_write_tokens,\n m.cost_usd,\n m.tokens_per_second,\n m.response_preview,\n m.error\n FROM benchmark_measurements m\n INNER JOIN benchmark_runs r ON r.id = m.run_id\n WHERE ${whereSql}\n ORDER BY ${nullsLast}${sortColumn} ${direction},\n r.started_at DESC, m.run_id ASC, m.provider ASC, m.attempt ASC\n LIMIT ? OFFSET ?`,\n args: [...args, limit, offset],\n },\n {\n sql: `SELECT DISTINCT m.provider\n FROM benchmark_measurements m\n INNER JOIN benchmark_runs r ON r.id = m.run_id\n WHERE r.model_id = ?\n ORDER BY m.provider ASC`,\n args: [input.modelId],\n },\n {\n sql: `SELECT DISTINCT m.attempt\n FROM benchmark_measurements m\n INNER JOIN benchmark_runs r ON r.id = m.run_id\n WHERE r.model_id = ?\n ORDER BY m.attempt ASC`,\n args: [input.modelId],\n },\n {\n sql: `SELECT\n r.id,\n r.started_at,\n r.status,\n COUNT(m.attempt) AS measurement_count\n FROM benchmark_runs r\n LEFT JOIN benchmark_measurements m ON m.run_id = r.id\n WHERE r.model_id = ?\n GROUP BY r.id, r.started_at, r.status\n ORDER BY r.started_at DESC\n LIMIT 200`,\n args: [input.modelId],\n },\n ]);\n\n const countRow = (results[0].rows as Row[])[0];\n const itemRows = results[1].rows as Row[];\n const providerRows = results[2].rows as Row[];\n const attemptRows = results[3].rows as Row[];\n const runRows = results[4].rows as Row[];\n\n return {\n items: itemRows.map(mapItem),\n total: countRow ? asNumber(countRow.total) : 0,\n limit,\n offset,\n providers: providerRows.map((row) => String(row.provider)),\n attempts: attemptRows.map((row) => asNumber(row.attempt)),\n runs: runRows.map((row) => ({\n id: String(row.id),\n startedAt: String(row.started_at),\n status: String(row.status) as BenchmarkStatus,\n measurementCount: asNumber(row.measurement_count),\n })),\n };\n}\n","createdAt",["D",1787857500530],"branchVersion","branchId","24cb14b4-1851-4548-adc6-43a574860a3d","id","d055324c-28f9-4529-80dd-e9bbfbdbeb65","glm53-flash-benchmark",["D",1787848294548],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Run and compare AI Gateway providers","imageUrl","a0bf3b31-15a5-4d5c-880e-4b1e22c9bc18","pinnedFileId","01a04410-0100-7214-ba2d-a172e514af90","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","ba7bb5fb-1abb-4ffb-8f3d-7c244782e5f7","tursoName","d055324c28f9452980dde9bbfbdbeb65","tursoHostname","d055324c28f9452980dde9bbfbdbeb65-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789500119686],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_101":100,"_109":105},"org",{"_101":100,"_102":103,"_104":105,"_18":-5,"_106":107,"_108":60},"branch",{"_13":90,"_32":91,"_44":43,"_92":45,"_93":-5,"_94":-5,"_95":-5,"_96":-5,"_97":-5,"_98":46,"_99":100},"isOwned",338,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","valdottown","handle","displayName","Val Town ","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvdXBsb2FkZWQvaW1nXzJyWnQ5cFE0c3pleVRCSEVTWmxzaXJsTUlGciJ9","tier","business","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_44":45,"_32":46,"_39":394,"_48":-5,"_49":25,"_50":51,"_52":51,"_53":54,"_55":-5,"_18":56,"_57":58,"_59":60,"_61":-5,"_62":-5,"_63":64,"_65":66,"_67":68,"_69":70,"_71":395,"_73":-5,"_74":75,"_76":75,"_77":78,"_79":78,"_80":78,"_81":75,"_82":75,"_83":396,"_85":397,"_87":398,"_89":25},"tree",[193,199,204,215,217,218],"files",[193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218],"counts",{"_183":64,"_184":180,"_185":168,"_186":187,"_188":189},"contents",[132,133,134,135,136,137],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_44":177,"_32":178,"_30":179,"_140":-5,"_42":43,"_41":180,"_142":60,"_39":181,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_27":182},{"_44":171,"_32":172,"_30":173,"_140":-5,"_42":43,"_41":174,"_142":60,"_39":175,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_27":176},{"_44":165,"_32":166,"_30":167,"_140":-5,"_42":43,"_41":168,"_142":60,"_39":169,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_27":170},{"_44":158,"_32":159,"_30":160,"_140":-5,"_42":43,"_41":161,"_142":60,"_39":162,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_27":164},{"_44":150,"_32":151,"_30":152,"_140":-5,"_42":43,"_41":153,"_142":25,"_39":154,"_13":155,"_34":20,"_36":156,"_24":25,"_147":-5,"_27":157},{"_44":138,"_32":139,"_30":58,"_140":-5,"_42":43,"_41":141,"_142":25,"_39":143,"_13":144,"_34":145,"_36":146,"_24":25,"_147":148,"_27":149},"9d64168c-3091-4cd0-8fc6-bcaee882427c","index.ts","parentId",151,"latest",["D",1787857208217],7,"http","import { parseVal, serveImmutableFile } from \"https://esm.town/v/std/utils/index.ts\";\nimport { Hono } from \"npm:hono@4\";\nimport { listMeasurements } from \"./db/measurements.ts\";\nimport { getBenchmarkResults } from \"./db/results.ts\";\nimport { Root } from \"./frontend/root.tsx\";\nimport type {\n MeasurementOutcome,\n MeasurementSortDirection,\n MeasurementSortKey,\n} from \"./shared/types.ts\";\n\nconst MODEL_ID = \"zai/glm-5.3-flash\";\nconst MEASUREMENT_SORT_KEYS = new Set([\n \"runStartedAt\",\n \"provider\",\n \"attempt\",\n \"outcome\",\n \"startedAt\",\n \"ttftMs\",\n \"durationMs\",\n \"tokensPerSecond\",\n \"inputTokens\",\n \"outputTokens\",\n \"cacheReadTokens\",\n \"cacheWriteTokens\",\n \"costUsd\",\n \"message\",\n]);\nconst app = new Hono();\n\nfunction measurementSortKey(value: string | undefined): MeasurementSortKey {\n const key = value as MeasurementSortKey | undefined;\n return key && MEASUREMENT_SORT_KEYS.has(key) ? key : \"startedAt\";\n}\n\nfunction measurementOutcome(\n value: string | undefined,\n): MeasurementOutcome | undefined {\n return value === \"text\" || value === \"no_text\" || value === \"error\"\n ? value\n : undefined;\n}\n\nfunction positiveInteger(\n value: string | undefined,\n fallback: number,\n maximum: number,\n): number {\n const parsed = Number(value);\n return Number.isInteger(parsed) && parsed > 0\n ? Math.min(parsed, maximum)\n : fallback;\n}\n\nfunction nonNegativeInteger(\n value: string | undefined,\n fallback: number,\n maximum: number,\n): number {\n const parsed = Number(value);\n return Number.isSafeInteger(parsed) && parsed >= 0\n ? Math.min(parsed, maximum)\n : fallback;\n}\n\napp.get(\"/\", (c) => c.html(Root()));\napp.get(\"/measurements\", (c) => c.html(Root()));\n\napp.get(\"/__immutable/*\", (c) => serveImmutableFile(c.req.path));\n\napp.get(\"/api/results\", async (c) => {\n const results = await getBenchmarkResults(MODEL_ID);\n c.header(\"Cache-Control\", \"no-store\");\n c.header(\"X-Content-Type-Options\", \"nosniff\");\n return c.json({ results });\n});\n\napp.get(\"/api/measurements\", async (c) => {\n const attemptValue = c.req.query(\"attempt\");\n const attempt = positiveInteger(attemptValue, 0, 100);\n const direction: MeasurementSortDirection =\n c.req.query(\"direction\") === \"asc\" ? \"asc\" : \"desc\";\n\n const measurements = await listMeasurements({\n modelId: MODEL_ID,\n provider: c.req.query(\"provider\") || undefined,\n outcome: measurementOutcome(c.req.query(\"outcome\")),\n attempt: attempt > 0 ? attempt : undefined,\n runId: c.req.query(\"run\") || undefined,\n sortKey: measurementSortKey(c.req.query(\"sort\")),\n direction,\n limit: positiveInteger(c.req.query(\"limit\"), 50, 100),\n offset: nonNegativeInteger(c.req.query(\"offset\"), 0, 10_000_000),\n });\n\n c.header(\"Cache-Control\", \"no-store\");\n c.header(\"X-Content-Type-Options\", \"nosniff\");\n return c.json({ measurements });\n});\n\napp.get(\"/source\", (c) => c.redirect(parseVal().links.self.val));\n\napp.onError((err, c) => {\n console.error(\n \"Benchmark request failed:\",\n err instanceof Error ? err.message : String(err),\n );\n c.header(\"Cache-Control\", \"no-store\");\n c.header(\"X-Content-Type-Options\", \"nosniff\");\n return c.req.path.startsWith(\"/api/\")\n ? c.json({ error: \"The request could not be completed.\" }, 500)\n : c.text(\"The page could not be loaded.\", 500);\n});\n\nexport default app.fetch;\n","label","glm53flash",[139],"5eaad40e-4914-4351-96c9-4e22b5bdca4b","README.md","01a04410-0100-7214-ba2d-9ca990fb87d3",130,["D",1787856319670],20,"# GLM-5.3-Flash Gateway Benchmark\n\nA public, read-only Val Town page that compares providers serving\n`zai/glm-5.3-flash` through Vercel AI Gateway.\n\n## How results work\n\nEvery provider attempt is one row in the val-scoped\n`benchmark_measurements` SQLite table. Results pool measurements from every\ncompleted run, so hourly and manual runs update the same comparison.\n\nBefore each new run, the worker fetches Vercel's live endpoint catalog. It\nbenchmarks every returned provider and stores one normalized row per provider\nin `benchmark_provider_snapshots`. A provider newly added by Vercel is\ntherefore included automatically on the next run. The full catalog JSON also\nstays on the run record so an interrupted run can resume against the exact\nprovider set it started with.\n\nThe sortable provider table reports:\n\n- Reliability: responses with visible text divided by attempted requests.\n- TTFT p50 and p95 across usable text responses.\n- p50 and p5 output tokens per second over usable text responses.\n- Cache hits among eligible repeated usable responses.\n- Effective observed cost per million input plus output tokens across usable responses.\n\nEffective $/MTok is total observed cost divided by total input plus output\ntokens, multiplied by one million. Cached tokens remain part of input tokens,\nwhile their discounted price is reflected in observed cost.\n\np50 is the median. TTFT p95 is the slow latency tail, while tokens/sec p5 is\nthe slow throughput tail. With fewer than 20 observations, nearest-rank p95 and\np5 are the observed maximum and minimum.\n\n## Schedule\n\n`benchmark/hourly.ts` runs at minute 17 of every hour (UTC). It refreshes the\nprovider catalog, then executes the full benchmark. Requests use a 75 second\ntimeout and disable automatic SDK retries. A recent running benchmark causes\nthe interval to skip; an older interrupted run remains resumable.\n\n## Run manually\n\n1. Edit [benchmark/config.ts](./benchmark/config.ts) if needed.\n2. Open [benchmark/run.ts](./benchmark/run.ts) in Val Town and click **Run**.\n3. Refresh the public page after the benchmark completes.\n\nThe runner checkpoints each attempt. Running the same configuration resumes an\ninterrupted run and skips persisted attempts.\n\n## Access\n\nThe Gateway credential is stored as the encrypted\n`VERCEL_AI_GATEWAY_API_KEY` environment variable. The HTTP app exposes only\nread-only routes, so public visitors cannot start paid model requests.\n\n## Structure\n\n- `benchmark/` — configuration, fixed conversation, manual runner, and hourly interval.\n- `db/` — val-scoped SQLite schema, writes, provider snapshots, and cross-run aggregation.\n- `frontend/` — React interface styled with Twind.\n- `shared/` — shared benchmark and result types.\n",[151],"57180eb9-f846-47ed-9f7f-5ec6aa1d02be","shared","f91bd0d2-a234-11f1-8cec-1607ee4eb77e",4,["D",1787848382634],"directory",[159],"6f24ec24-8f0c-4d0d-b0d3-fbc1bac995ce","frontend","01a04410-0100-7214-ba2d-95062f41a4ba",1,["D",1787848294548],[166],"5e355af5-f49e-4221-87df-a392a9184c4b","db","f73349d0-a234-11f1-922e-1607ee4eb77e",3,["D",1787848379437],[172],"1d67c685-0a4f-421f-a3c8-63f7187dfcee","benchmark","f52e35aa-a234-11f1-971d-1607ee4eb77e",2,["D",1787848376051],[178],"pulls","branches","forks","envVars",{"_190":168,"_192":64},"connections",{"_190":64,"_191":64},"count","unfilledCount","emptyCount",{"_44":177,"_32":178,"_30":179,"_140":-5,"_42":43,"_41":180,"_142":60,"_39":391,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_220":392,"_27":393},{"_44":383,"_32":384,"_30":385,"_140":179,"_42":43,"_41":386,"_142":25,"_39":387,"_13":168,"_34":35,"_36":388,"_24":25,"_147":-5,"_220":389,"_27":390},{"_44":375,"_32":376,"_30":377,"_140":179,"_42":43,"_41":378,"_142":25,"_39":379,"_13":231,"_34":35,"_36":380,"_24":25,"_147":-5,"_220":381,"_27":382},{"_44":366,"_32":367,"_30":368,"_140":179,"_42":43,"_41":369,"_142":25,"_39":370,"_13":168,"_34":371,"_36":372,"_24":25,"_147":-5,"_220":373,"_27":374},{"_44":358,"_32":359,"_30":360,"_140":179,"_42":43,"_41":361,"_142":25,"_39":362,"_13":168,"_34":35,"_36":363,"_24":25,"_147":-5,"_220":364,"_27":365},{"_44":351,"_32":352,"_30":353,"_140":179,"_42":43,"_41":340,"_142":25,"_39":354,"_13":64,"_34":35,"_36":355,"_24":25,"_147":-5,"_220":356,"_27":357},{"_44":171,"_32":172,"_30":173,"_140":-5,"_42":43,"_41":174,"_142":60,"_39":348,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_220":349,"_27":350},{"_44":344,"_32":33,"_30":31,"_140":173,"_42":43,"_41":14,"_142":25,"_39":345,"_13":144,"_34":35,"_36":38,"_24":25,"_147":-5,"_220":346,"_27":347},{"_44":335,"_32":336,"_30":337,"_140":173,"_42":43,"_41":338,"_142":25,"_39":339,"_13":340,"_34":35,"_36":341,"_24":25,"_147":-5,"_220":342,"_27":343},{"_44":327,"_32":328,"_30":329,"_140":173,"_42":43,"_41":330,"_142":25,"_39":331,"_13":144,"_34":35,"_36":332,"_24":25,"_147":-5,"_220":333,"_27":334},{"_44":319,"_32":320,"_30":321,"_140":173,"_42":43,"_41":322,"_142":25,"_39":323,"_13":161,"_34":35,"_36":324,"_24":25,"_147":-5,"_220":325,"_27":326},{"_44":165,"_32":166,"_30":167,"_140":-5,"_42":43,"_41":168,"_142":60,"_39":316,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_220":317,"_27":318},{"_44":312,"_32":287,"_30":280,"_140":167,"_42":43,"_41":168,"_142":60,"_39":313,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_220":314,"_27":315},{"_44":304,"_32":305,"_30":306,"_140":280,"_42":43,"_41":307,"_142":25,"_39":308,"_13":155,"_34":35,"_36":309,"_24":25,"_147":-5,"_220":310,"_27":311},{"_44":296,"_32":297,"_30":298,"_140":280,"_42":43,"_41":299,"_142":25,"_39":300,"_13":161,"_34":35,"_36":301,"_24":25,"_147":-5,"_220":302,"_27":303},{"_44":288,"_32":289,"_30":290,"_140":280,"_42":43,"_41":291,"_142":25,"_39":292,"_13":168,"_34":35,"_36":293,"_24":25,"_147":-5,"_220":294,"_27":295},{"_44":277,"_32":278,"_30":279,"_140":280,"_42":43,"_41":281,"_142":25,"_39":282,"_13":283,"_34":35,"_36":284,"_24":25,"_147":-5,"_220":285,"_27":286},{"_44":270,"_32":271,"_30":272,"_140":167,"_42":43,"_41":168,"_142":60,"_39":273,"_13":64,"_34":20,"_36":274,"_24":25,"_147":-5,"_220":275,"_27":276},{"_44":262,"_32":263,"_30":264,"_140":167,"_42":43,"_41":265,"_142":25,"_39":266,"_13":161,"_34":35,"_36":267,"_24":25,"_147":-5,"_220":268,"_27":269},{"_44":254,"_32":255,"_30":256,"_140":167,"_42":43,"_41":257,"_142":60,"_39":258,"_13":168,"_34":35,"_36":259,"_24":25,"_147":-5,"_220":260,"_27":261},{"_44":246,"_32":247,"_30":248,"_140":167,"_42":43,"_41":249,"_142":60,"_39":250,"_13":64,"_34":35,"_36":251,"_24":25,"_147":-5,"_220":252,"_27":253},{"_44":238,"_32":239,"_30":240,"_140":167,"_42":43,"_41":241,"_142":60,"_39":242,"_13":174,"_34":35,"_36":243,"_24":25,"_147":-5,"_220":244,"_27":245},{"_44":158,"_32":159,"_30":160,"_140":-5,"_42":43,"_41":161,"_142":60,"_39":235,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_220":236,"_27":237},{"_44":226,"_32":227,"_30":228,"_140":160,"_42":43,"_41":229,"_142":25,"_39":230,"_13":231,"_34":35,"_36":232,"_24":25,"_147":-5,"_220":233,"_27":234},{"_44":150,"_32":151,"_30":152,"_140":-5,"_42":43,"_41":153,"_142":25,"_39":223,"_13":155,"_34":20,"_36":156,"_24":25,"_147":-5,"_220":224,"_27":225},{"_44":138,"_32":139,"_30":58,"_140":-5,"_42":43,"_41":141,"_142":25,"_39":219,"_13":144,"_34":145,"_36":146,"_24":25,"_147":148,"_220":221,"_27":222},["D",1787857208217],"children",[],[139],["D",1787856319670],[],[151],"94cba016-3c69-45d9-8b6a-ce0bc279388e","types.ts","0553d0c0-a235-11f1-94c6-1607ee4eb77e",163,["D",1787857424759],8,"export type BenchmarkStatus = \"running\" | \"completed\" | \"failed\";\nexport type MeasurementStatus = \"ok\" | \"error\";\n\nexport interface BenchmarkConfig {\n label: string;\n modelId: string;\n runsPerProvider: number;\n maxOutputTokens: number;\n providerConcurrency: number;\n requestTimeoutMs: number;\n providers?: string[];\n}\n\nexport interface GatewayEndpoint {\n provider_name: string;\n name?: string;\n has_zdr?: boolean;\n has_no_training?: boolean;\n supports_implicit_caching?: boolean;\n pricing?: Record;\n latency_last_1h?: { p50?: number; p95?: number };\n throughput_last_1h?: { p50?: number; p95?: number };\n tags?: string[];\n [key: string]: unknown;\n}\n\nexport interface BenchmarkRunSummary {\n id: string;\n label: string;\n modelId: string;\n status: BenchmarkStatus;\n startedAt: string;\n completedAt: string | null;\n runsPerProvider: number;\n maxOutputTokens: number;\n providerCount: number;\n successfulMeasurements: number;\n failedMeasurements: number;\n error: string | null;\n}\n\nexport interface BenchmarkMeasurement {\n runId: string;\n provider: string;\n attempt: number;\n status: MeasurementStatus;\n outcome: MeasurementOutcome;\n errorCode: MeasurementErrorCode;\n startedAt: string;\n completedAt: string;\n ttftMs: number | null;\n durationMs: number | null;\n inputTokens: number | null;\n outputTokens: number | null;\n cacheReadTokens: number | null;\n cacheWriteTokens: number | null;\n costUsd: number | null;\n tokensPerSecond: number | null;\n responsePreview: string | null;\n error: string | null;\n providerMetadata: Record | null;\n providerInfo: GatewayEndpoint;\n}\n\nexport type MeasurementOutcome = \"text\" | \"no_text\" | \"error\";\nexport type MeasurementErrorCode =\n | \"no_text\"\n | \"timeout\"\n | \"provider_error\"\n | null;\n\nexport type MeasurementSortKey =\n | \"runStartedAt\"\n | \"provider\"\n | \"attempt\"\n | \"outcome\"\n | \"startedAt\"\n | \"ttftMs\"\n | \"durationMs\"\n | \"tokensPerSecond\"\n | \"inputTokens\"\n | \"outputTokens\"\n | \"cacheReadTokens\"\n | \"cacheWriteTokens\"\n | \"costUsd\"\n | \"message\";\n\nexport type MeasurementSortDirection = \"asc\" | \"desc\";\n\nexport interface MeasurementListItem {\n runId: string;\n runStartedAt: string;\n runStatus: BenchmarkStatus;\n provider: string;\n attempt: number;\n outcome: MeasurementOutcome;\n startedAt: string;\n completedAt: string;\n ttftMs: number | null;\n durationMs: number | null;\n inputTokens: number | null;\n outputTokens: number | null;\n cacheReadTokens: number | null;\n cacheWriteTokens: number | null;\n costUsd: number | null;\n tokensPerSecond: number | null;\n responsePreview: string | null;\n error: string | null;\n}\n\nexport interface MeasurementRunOption {\n id: string;\n startedAt: string;\n status: BenchmarkStatus;\n measurementCount: number;\n}\n\nexport interface MeasurementListResponse {\n items: MeasurementListItem[];\n total: number;\n limit: number;\n offset: number;\n providers: string[];\n attempts: number[];\n runs: MeasurementRunOption[];\n}\n\nexport interface BenchmarkRunDetail extends BenchmarkRunSummary {\n config: BenchmarkConfig;\n endpointSnapshot: GatewayEndpoint[];\n measurements: BenchmarkMeasurement[];\n}\n\nexport interface ProviderResult {\n provider: string;\n runCount: number;\n attempts: number;\n successes: number;\n failures: number;\n successRate: number;\n p50TtftMs: number | null;\n p95TtftMs: number | null;\n p50DurationMs: number | null;\n p95DurationMs: number | null;\n p50TokensPerSecond: number | null;\n p05TokensPerSecond: number | null;\n cacheEligibleAttempts: number;\n cacheHits: number;\n cacheHitRate: number | null;\n effectiveCostPerMillionTokensUsd: number | null;\n}\n\nexport interface BenchmarkResults {\n modelId: string;\n runCount: number;\n attemptCount: number;\n successfulAttempts: number;\n failedAttempts: number;\n firstRunAt: string | null;\n lastCompletedAt: string | null;\n providers: ProviderResult[];\n}\n",[],[159,227],["D",1787848382634],[216],[159],"0921c7f1-3a63-4c80-b035-7a8887bae310","root.tsx","01a04410-0100-7214-ba2d-918a7f307571",44,["D",1787852742071],"/** @jsxImportSource npm:hono@4/jsx */\nimport { raw } from \"npm:hono@4/html\";\nimport { immutableFileUrl } from \"https://esm.town/v/std/utils/index.ts\";\n\nexport function Root() {\n return (\n <>\n {raw(\"\")}\n \n \n \n \n \n \n GLM-5.3-Flash Provider Benchmark\n

Diff · v174