GLM-5.3-Flash Provider Benchmark

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_414":415,"_416":417},"routes/_app.x.$handle.$name",{"_101":100,"_32":46,"_110":91,"_27":111,"_13":14,"_112":-5,"_11":113,"_114":115,"_116":117,"_118":119,"_120":121,"_20":-5,"_122":-5,"_123":-5,"_124":25,"_125":126,"_127":60,"_128":-5,"_129":-5,"_130":131},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_44":45,"_32":46,"_39":47,"_48":-5,"_49":25,"_50":51,"_52":51,"_53":54,"_55":-5,"_18":56,"_57":58,"_59":60,"_61":-5,"_62":-5,"_63":64,"_65":66,"_67":68,"_69":70,"_71":72,"_73":-5,"_74":75,"_76":75,"_77":78,"_79":78,"_80":78,"_81":75,"_82":75,"_83":84,"_85":86,"_87":88,"_89":25},"version",212,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":33,"_34":35,"_36":38,"_39":40,"_41":14,"_42":43,"_24":25},"previous",{"_30":31,"_32":33,"_34":35,"_36":37},"deleted",false,"deletedHere","path","benchmark/execute.ts","latestPath","fileId","519c40a2-a235-11f1-9ecd-1607ee4eb77e","name","execute.ts","fileType","script","content","import { createGateway } from \"npm:@ai-sdk/gateway@3.0.134\";\nimport { streamText } from \"npm:ai@6.0.168\";\nimport {\n createBenchmarkRun,\n findResumableRun,\n finishBenchmarkRun,\n getBenchmarkRun,\n getRecordedAttempts,\n saveMeasurement,\n} from \"../db/queries.ts\";\nimport type {\n BenchmarkConfig,\n BenchmarkMeasurement,\n GatewayEndpoint,\n MeasurementErrorCode,\n} from \"../shared/types.ts\";\nimport { buildMessages } from \"./prompt.ts\";\n\nconst ENDPOINTS_BASE_URL = \"https://ai-gateway.vercel.sh/v1/models\";\n\nfunction errorMessage(error: unknown): string {\n return error instanceof Error ? error.message : String(error);\n}\n\nfunction errorCode(message: string): MeasurementErrorCode {\n return /timed out|timeout|aborted/i.test(message)\n ? \"timeout\"\n : \"provider_error\";\n}\n\nfunction numericValue(value: unknown): number | null {\n if (typeof value === \"number\" && Number.isFinite(value)) return value;\n if (typeof value === \"string\") {\n const parsed = Number(value);\n return Number.isFinite(parsed) ? parsed : null;\n }\n return null;\n}\n\nfunction readGatewayCost(\n metadata: Record | undefined,\n): number | null {\n const gateway = metadata?.gateway;\n if (!gateway || typeof gateway !== \"object\") return null;\n const record = gateway as Record;\n return numericValue(record.cost)\n ?? numericValue(record.costUsd)\n ?? numericValue(record.totalCost);\n}\n\nasync function fetchEndpoints(modelId: string): Promise {\n const response = await fetch(\n `${ENDPOINTS_BASE_URL}/${modelId}/endpoints`,\n );\n if (!response.ok) {\n throw new Error(\n `Endpoint catalogue failed with ${response.status} ${response.statusText}`,\n );\n }\n\n const payload = await response.json() as {\n data?: { endpoints?: GatewayEndpoint[] } | GatewayEndpoint[];\n endpoints?: GatewayEndpoint[];\n };\n const endpoints = Array.isArray(payload)\n ? payload\n : Array.isArray(payload.data)\n ? payload.data\n : payload.data?.endpoints ?? payload.endpoints ?? [];\n\n return endpoints\n .filter((endpoint) => typeof endpoint.provider_name === \"string\")\n .sort((a, b) => a.provider_name.localeCompare(b.provider_name));\n}\n\nasync function runAttempt(input: {\n runId: string;\n config: BenchmarkConfig;\n endpoint: GatewayEndpoint;\n attempt: number;\n apiKey: string;\n}): Promise {\n const { runId, config, endpoint, attempt, apiKey } = input;\n const startedAt = new Date().toISOString();\n const started = performance.now();\n let ttftMs: number | null = null;\n let preview = \"\";\n let streamError: unknown = null;\n\n try {\n const gateway = createGateway({ apiKey });\n const result = streamText({\n model: gateway(config.modelId),\n maxOutputTokens: config.maxOutputTokens,\n maxRetries: 0,\n messages: buildMessages(attempt),\n abortSignal: AbortSignal.timeout(config.requestTimeoutMs),\n providerOptions: {\n gateway: {\n only: [endpoint.provider_name],\n },\n },\n onError({ error }) {\n streamError ??= error;\n console.error(\n endpoint.provider_name + \" attempt \" + attempt +\n \" stream error: \" + errorMessage(error),\n );\n },\n });\n\n for await (const chunk of result.textStream) {\n if (ttftMs === null) ttftMs = performance.now() - started;\n if (preview.length < 800) {\n preview += chunk.slice(0, 800 - preview.length);\n }\n }\n\n if (streamError !== null) throw streamError;\n\n const durationMs = performance.now() - started;\n const usage = await result.usage;\n const providerMetadata = await result.providerMetadata as\n | Record\n | undefined;\n const inputDetails = usage.inputTokenDetails;\n const outputTokens = usage.outputTokens ?? null;\n\n const hasText = preview.trim().length > 0;\n return {\n runId,\n provider: endpoint.provider_name,\n attempt,\n status: hasText ? \"ok\" : \"error\",\n outcome: hasText ? \"text\" : \"no_text\",\n errorCode: hasText ? null : \"no_text\",\n startedAt,\n completedAt: new Date().toISOString(),\n ttftMs,\n durationMs,\n inputTokens: usage.inputTokens ?? null,\n outputTokens,\n cacheReadTokens: inputDetails?.cacheReadTokens ?? null,\n cacheWriteTokens: inputDetails?.cacheWriteTokens ?? null,\n costUsd: readGatewayCost(providerMetadata),\n tokensPerSecond: outputTokens && durationMs > 0\n ? outputTokens / (durationMs / 1_000)\n : null,\n responsePreview: preview || null,\n error: hasText ? null : \"No text output was returned.\",\n providerMetadata: providerMetadata ?? null,\n providerInfo: endpoint,\n };\n } catch (error) {\n const message = errorMessage(streamError ?? error);\n return {\n runId,\n provider: endpoint.provider_name,\n attempt,\n status: \"error\",\n outcome: \"error\",\n errorCode: errorCode(message),\n startedAt,\n completedAt: new Date().toISOString(),\n ttftMs,\n durationMs: performance.now() - started,\n inputTokens: null,\n outputTokens: null,\n cacheReadTokens: null,\n cacheWriteTokens: null,\n costUsd: null,\n tokensPerSecond: null,\n responsePreview: preview || null,\n error: message,\n providerMetadata: null,\n providerInfo: endpoint,\n };\n }\n}\n\nexport async function executeBenchmark(config: BenchmarkConfig) {\n const apiKey = Deno.env.get(\"VERCEL_AI_GATEWAY_API_KEY\");\n if (!apiKey) {\n throw new Error(\"Missing VERCEL_AI_GATEWAY_API_KEY environment variable\");\n }\n\n const configJson = JSON.stringify(config);\n let runId = await findResumableRun(configJson);\n let endpoints: GatewayEndpoint[];\n\n if (runId) {\n const existing = await getBenchmarkRun(runId);\n endpoints = existing?.endpointSnapshot ?? [];\n console.log(`Resuming benchmark ${runId}`);\n } else {\n endpoints = await fetchEndpoints(config.modelId);\n }\n\n if (config.providers?.length) {\n const allowed = new Set(config.providers);\n endpoints = endpoints.filter((endpoint) =>\n allowed.has(endpoint.provider_name)\n );\n }\n if (endpoints.length === 0) {\n throw new Error(\"No matching providers found in the endpoint catalogue\");\n }\n\n if (!runId) {\n runId = crypto.randomUUID();\n await createBenchmarkRun({ id: runId, config, endpoints });\n console.log(`Created benchmark ${runId}`);\n }\n\n try {\n const recorded = await getRecordedAttempts(runId);\n let nextProvider = 0;\n\n async function worker() {\n while (true) {\n const providerIndex = nextProvider++;\n const endpoint = endpoints[providerIndex];\n if (!endpoint) return;\n\n for (let attempt = 1; attempt <= config.runsPerProvider; attempt++) {\n const key = `${endpoint.provider_name}:${attempt}`;\n if (recorded.has(key)) {\n console.log(`Skipping saved attempt ${key}`);\n continue;\n }\n\n console.log(\n `Running ${endpoint.provider_name} attempt ${attempt}/${config.runsPerProvider}`,\n );\n const measurement = await runAttempt({\n runId,\n config,\n endpoint,\n attempt,\n apiKey,\n });\n await saveMeasurement(measurement);\n recorded.add(key);\n console.log(\n `${endpoint.provider_name} attempt ${attempt}: ${measurement.status}`,\n );\n }\n }\n }\n\n const workerCount = Math.max(\n 1,\n Math.min(config.providerConcurrency, endpoints.length),\n );\n await Promise.all(Array.from({ length: workerCount }, () => worker()));\n await finishBenchmarkRun(runId, \"completed\");\n\n const detail = await getBenchmarkRun(runId);\n console.log(\n JSON.stringify({\n runId,\n status: detail?.status,\n providers: detail?.providerCount,\n successfulMeasurements: detail?.successfulMeasurements,\n failedMeasurements: detail?.failedMeasurements,\n }),\n );\n return detail;\n } catch (error) {\n const message = errorMessage(error);\n await finishBenchmarkRun(runId, \"failed\", message);\n throw error;\n }\n}\n","import { createGateway } from \"npm:@ai-sdk/gateway@3.0.134\";\nimport { streamText } from \"npm:ai@6.0.168\";\nimport {\n claimBenchmarkExecution,\n createBenchmarkRun,\n findRecentRunningRun,\n findResumableRun,\n finishBenchmarkRun,\n getBenchmarkRun,\n getRecordedAttempts,\n releaseBenchmarkExecution,\n saveMeasurement,\n} from \"../db/queries.ts\";\nimport type {\n BenchmarkConfig,\n BenchmarkMeasurement,\n GatewayEndpoint,\n MeasurementErrorCode,\n} from \"../shared/types.ts\";\nimport { buildMessages } from \"./prompt.ts\";\n\nconst ENDPOINTS_BASE_URL = \"https://ai-gateway.vercel.sh/v1/models\";\n\nfunction errorMessage(error: unknown): string {\n return error instanceof Error ? error.message : String(error);\n}\n\nfunction errorCode(message: string): MeasurementErrorCode {\n return /timed out|timeout|aborted/i.test(message)\n ? \"timeout\"\n : \"provider_error\";\n}\n\nfunction numericValue(value: unknown): number | null {\n if (typeof value === \"number\" && Number.isFinite(value)) return value;\n if (typeof value === \"string\") {\n const parsed = Number(value);\n return Number.isFinite(parsed) ? parsed : null;\n }\n return null;\n}\n\nfunction readGatewayCost(\n metadata: Record | undefined,\n): number | null {\n const gateway = metadata?.gateway;\n if (!gateway || typeof gateway !== \"object\") return null;\n const record = gateway as Record;\n return numericValue(record.cost)\n ?? numericValue(record.costUsd)\n ?? numericValue(record.totalCost);\n}\n\nasync function fetchEndpoints(modelId: string): Promise {\n const response = await fetch(\n `${ENDPOINTS_BASE_URL}/${modelId}/endpoints`,\n );\n if (!response.ok) {\n throw new Error(\n `Endpoint catalogue failed with ${response.status} ${response.statusText}`,\n );\n }\n\n const payload = await response.json() as {\n data?: { endpoints?: GatewayEndpoint[] } | GatewayEndpoint[];\n endpoints?: GatewayEndpoint[];\n };\n const endpoints = Array.isArray(payload)\n ? payload\n : Array.isArray(payload.data)\n ? payload.data\n : payload.data?.endpoints ?? payload.endpoints ?? [];\n\n return endpoints\n .filter((endpoint) => typeof endpoint.provider_name === \"string\")\n .sort((a, b) => a.provider_name.localeCompare(b.provider_name));\n}\n\nasync function runAttempt(input: {\n runId: string;\n config: BenchmarkConfig;\n endpoint: GatewayEndpoint;\n attempt: number;\n apiKey: string;\n}): Promise {\n const { runId, config, endpoint, attempt, apiKey } = input;\n const startedAt = new Date().toISOString();\n const started = performance.now();\n let ttftMs: number | null = null;\n let preview = \"\";\n let streamError: unknown = null;\n\n try {\n const gateway = createGateway({ apiKey });\n const result = streamText({\n model: gateway(config.modelId),\n maxOutputTokens: config.maxOutputTokens,\n maxRetries: 0,\n messages: buildMessages(attempt),\n abortSignal: AbortSignal.timeout(config.requestTimeoutMs),\n providerOptions: {\n gateway: {\n only: [endpoint.provider_name],\n },\n },\n onError({ error }) {\n streamError ??= error;\n console.error(\n endpoint.provider_name + \" attempt \" + attempt +\n \" stream error: \" + errorMessage(error),\n );\n },\n });\n\n for await (const chunk of result.textStream) {\n if (ttftMs === null) ttftMs = performance.now() - started;\n if (preview.length < 800) {\n preview += chunk.slice(0, 800 - preview.length);\n }\n }\n\n if (streamError !== null) throw streamError;\n\n const durationMs = performance.now() - started;\n const usage = await result.usage;\n const providerMetadata = await result.providerMetadata as\n | Record\n | undefined;\n const inputDetails = usage.inputTokenDetails;\n const outputTokens = usage.outputTokens ?? null;\n\n const hasText = preview.trim().length > 0;\n return {\n runId,\n provider: endpoint.provider_name,\n attempt,\n status: hasText ? \"ok\" : \"error\",\n outcome: hasText ? \"text\" : \"no_text\",\n errorCode: hasText ? null : \"no_text\",\n startedAt,\n completedAt: new Date().toISOString(),\n ttftMs,\n durationMs,\n inputTokens: usage.inputTokens ?? null,\n outputTokens,\n cacheReadTokens: inputDetails?.cacheReadTokens ?? null,\n cacheWriteTokens: inputDetails?.cacheWriteTokens ?? null,\n costUsd: readGatewayCost(providerMetadata),\n tokensPerSecond: outputTokens && durationMs > 0\n ? outputTokens / (durationMs / 1_000)\n : null,\n responsePreview: preview || null,\n error: hasText ? null : \"No text output was returned.\",\n providerMetadata: providerMetadata ?? null,\n providerInfo: endpoint,\n };\n } catch (error) {\n const message = errorMessage(streamError ?? error);\n return {\n runId,\n provider: endpoint.provider_name,\n attempt,\n status: \"error\",\n outcome: \"error\",\n errorCode: errorCode(message),\n startedAt,\n completedAt: new Date().toISOString(),\n ttftMs,\n durationMs: performance.now() - started,\n inputTokens: null,\n outputTokens: null,\n cacheReadTokens: null,\n cacheWriteTokens: null,\n costUsd: null,\n tokensPerSecond: null,\n responsePreview: preview || null,\n error: message,\n providerMetadata: null,\n providerInfo: endpoint,\n };\n }\n}\n\nexport async function executeBenchmark(config: BenchmarkConfig) {\n const apiKey = Deno.env.get(\"VERCEL_AI_GATEWAY_API_KEY\");\n if (!apiKey) {\n throw new Error(\"Missing VERCEL_AI_GATEWAY_API_KEY environment variable\");\n }\n\n const configJson = JSON.stringify(config);\n let runId = await findResumableRun(configJson);\n let endpoints: GatewayEndpoint[];\n\n if (runId) {\n const existing = await getBenchmarkRun(runId);\n endpoints = existing?.endpointSnapshot ?? [];\n console.log(`Resuming benchmark ${runId}`);\n } else {\n endpoints = await fetchEndpoints(config.modelId);\n }\n\n if (config.providers?.length) {\n const allowed = new Set(config.providers);\n endpoints = endpoints.filter((endpoint) =>\n allowed.has(endpoint.provider_name)\n );\n }\n if (endpoints.length === 0) {\n throw new Error(\"No matching providers found in the endpoint catalogue\");\n }\n\n if (!runId) {\n runId = crypto.randomUUID();\n await createBenchmarkRun({ id: runId, config, endpoints });\n console.log(`Created benchmark ${runId}`);\n }\n\n try {\n const recorded = await getRecordedAttempts(runId);\n let nextProvider = 0;\n\n async function worker() {\n while (true) {\n const providerIndex = nextProvider++;\n const endpoint = endpoints[providerIndex];\n if (!endpoint) return;\n\n for (let attempt = 1; attempt <= config.runsPerProvider; attempt++) {\n const key = `${endpoint.provider_name}:${attempt}`;\n if (recorded.has(key)) {\n console.log(`Skipping saved attempt ${key}`);\n continue;\n }\n\n console.log(\n `Running ${endpoint.provider_name} attempt ${attempt}/${config.runsPerProvider}`,\n );\n const measurement = await runAttempt({\n runId,\n config,\n endpoint,\n attempt,\n apiKey,\n });\n await saveMeasurement(measurement);\n recorded.add(key);\n console.log(\n `${endpoint.provider_name} attempt ${attempt}: ${measurement.status}`,\n );\n }\n }\n }\n\n const workerCount = Math.max(\n 1,\n Math.min(config.providerConcurrency, endpoints.length),\n );\n await Promise.all(Array.from({ length: workerCount }, () => worker()));\n await finishBenchmarkRun(runId, \"completed\");\n\n const detail = await getBenchmarkRun(runId);\n console.log(\n JSON.stringify({\n runId,\n status: detail?.status,\n providers: detail?.providerCount,\n successfulMeasurements: detail?.successfulMeasurements,\n failedMeasurements: detail?.failedMeasurements,\n }),\n );\n return detail;\n } catch (error) {\n const message = errorMessage(error);\n await finishBenchmarkRun(runId, \"failed\", message);\n throw error;\n }\n}\n","createdAt",["D",1787859205499],"branchVersion","branchId","24cb14b4-1851-4548-adc6-43a574860a3d","id","d055324c-28f9-4529-80dd-e9bbfbdbeb65","glm53-flash-benchmark",["D",1787848294548],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Run and compare AI Gateway providers","imageUrl","a0bf3b31-15a5-4d5c-880e-4b1e22c9bc18","pinnedFileId","01a04410-0100-7214-ba2d-a172e514af90","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","ba7bb5fb-1abb-4ffb-8f3d-7c244782e5f7","tursoName","d055324c28f9452980dde9bbfbdbeb65","tursoHostname","d055324c28f9452980dde9bbfbdbeb65-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789500119686],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_101":100,"_109":105},"org",{"_101":100,"_102":103,"_104":105,"_18":-5,"_106":107,"_108":60},"branch",{"_13":90,"_32":91,"_44":43,"_92":45,"_93":-5,"_94":-5,"_95":-5,"_96":-5,"_97":-5,"_98":46,"_99":100},"isOwned",338,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","valdottown","handle","displayName","Val Town ","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvdXBsb2FkZWQvaW1nXzJyWnQ5cFE0c3pleVRCSEVTWmxzaXJsTUlGciJ9","tier","business","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_44":45,"_32":46,"_39":409,"_48":-5,"_49":25,"_50":51,"_52":51,"_53":54,"_55":-5,"_18":56,"_57":58,"_59":60,"_61":-5,"_62":-5,"_63":64,"_65":66,"_67":68,"_69":70,"_71":410,"_73":-5,"_74":75,"_76":75,"_77":78,"_79":78,"_80":78,"_81":75,"_82":75,"_83":411,"_85":412,"_87":413,"_89":25},"tree",[193,199,204,216,218,219],"files",[193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219],"counts",{"_183":64,"_184":180,"_185":168,"_186":187,"_188":189},"contents",[132,133,134,135,136,137],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_44":177,"_32":178,"_30":179,"_140":-5,"_42":43,"_41":180,"_142":60,"_39":181,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_27":182},{"_44":171,"_32":172,"_30":173,"_140":-5,"_42":43,"_41":174,"_142":60,"_39":175,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_27":176},{"_44":165,"_32":166,"_30":167,"_140":-5,"_42":43,"_41":168,"_142":60,"_39":169,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_27":170},{"_44":158,"_32":159,"_30":160,"_140":-5,"_42":43,"_41":161,"_142":60,"_39":162,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_27":164},{"_44":150,"_32":151,"_30":152,"_140":-5,"_42":43,"_41":153,"_142":25,"_39":154,"_13":155,"_34":20,"_36":156,"_24":25,"_147":-5,"_27":157},{"_44":138,"_32":139,"_30":58,"_140":-5,"_42":43,"_41":141,"_142":25,"_39":143,"_13":144,"_34":145,"_36":146,"_24":25,"_147":148,"_27":149},"5f0cf0af-ee3a-4bd8-944b-b297596b3da0","index.ts","parentId",196,"latest",["D",1787858406876],8,"http","import { parseVal, serveImmutableFile } from \"https://esm.town/v/std/utils/index.ts\";\nimport { Hono } from \"npm:hono@4\";\nimport { listMeasurements } from \"./db/measurements.ts\";\nimport { getBenchmarkResults } from \"./db/results.ts\";\nimport { Root } from \"./frontend/root.tsx\";\nimport type {\n MeasurementPublicStatus,\n MeasurementSortDirection,\n MeasurementSortKey,\n} from \"./shared/types.ts\";\n\nconst MODEL_ID = \"zai/glm-5.3-flash\";\nconst MEASUREMENT_SORT_KEYS = new Set([\n \"runStartedAt\",\n \"provider\",\n \"attempt\",\n \"status\",\n \"startedAt\",\n \"ttftMs\",\n \"durationMs\",\n \"tokensPerSecond\",\n \"inputTokens\",\n \"outputTokens\",\n \"cacheReadTokens\",\n \"cacheWriteTokens\",\n \"costUsd\",\n \"message\",\n]);\nconst app = new Hono();\n\nfunction measurementSortKey(value: string | undefined): MeasurementSortKey {\n const key = value as MeasurementSortKey | undefined;\n return key && MEASUREMENT_SORT_KEYS.has(key) ? key : \"startedAt\";\n}\n\nfunction measurementStatus(\n value: string | undefined,\n): MeasurementPublicStatus | undefined {\n return value === \"success\" || value === \"error\" ? value : undefined;\n}\n\nfunction positiveInteger(\n value: string | undefined,\n fallback: number,\n maximum: number,\n): number {\n const parsed = Number(value);\n return Number.isInteger(parsed) && parsed > 0\n ? Math.min(parsed, maximum)\n : fallback;\n}\n\nfunction nonNegativeInteger(\n value: string | undefined,\n fallback: number,\n maximum: number,\n): number {\n const parsed = Number(value);\n return Number.isSafeInteger(parsed) && parsed >= 0\n ? Math.min(parsed, maximum)\n : fallback;\n}\n\napp.get(\"/\", (c) => c.html(Root()));\napp.get(\"/measurements\", (c) => c.html(Root()));\n\napp.get(\"/__immutable/*\", (c) => serveImmutableFile(c.req.path));\n\napp.get(\"/api/results\", async (c) => {\n const results = await getBenchmarkResults(MODEL_ID);\n c.header(\"Cache-Control\", \"no-store\");\n c.header(\"X-Content-Type-Options\", \"nosniff\");\n return c.json({ results });\n});\n\napp.get(\"/api/measurements\", async (c) => {\n const attemptValue = c.req.query(\"attempt\");\n const attempt = positiveInteger(attemptValue, 0, 100);\n const direction: MeasurementSortDirection =\n c.req.query(\"direction\") === \"asc\" ? \"asc\" : \"desc\";\n\n const measurements = await listMeasurements({\n modelId: MODEL_ID,\n provider: c.req.query(\"provider\") || undefined,\n status: measurementStatus(c.req.query(\"status\")),\n attempt: attempt > 0 ? attempt : undefined,\n runId: c.req.query(\"run\") || undefined,\n sortKey: measurementSortKey(c.req.query(\"sort\")),\n direction,\n limit: positiveInteger(c.req.query(\"limit\"), 50, 100),\n offset: nonNegativeInteger(c.req.query(\"offset\"), 0, 10_000_000),\n });\n\n c.header(\"Cache-Control\", \"no-store\");\n c.header(\"X-Content-Type-Options\", \"nosniff\");\n return c.json({ measurements });\n});\n\napp.get(\"/source\", (c) => c.redirect(parseVal().links.self.val));\n\napp.onError((err, c) => {\n console.error(\n \"Benchmark request failed:\",\n err instanceof Error ? err.message : String(err),\n );\n c.header(\"Cache-Control\", \"no-store\");\n c.header(\"X-Content-Type-Options\", \"nosniff\");\n return c.req.path.startsWith(\"/api/\")\n ? c.json({ error: \"The request could not be completed.\" }, 500)\n : c.text(\"The page could not be loaded.\", 500);\n});\n\nexport default app.fetch;\n","label","glm53flash",[139],"c2a7bf72-4c04-480b-9f45-575389413779","README.md","01a04410-0100-7214-ba2d-9ca990fb87d3",211,["D",1787859186418],21,"# GLM-5.3-Flash Gateway Benchmark\n\nA public, read-only Val Town page that compares providers serving\n`zai/glm-5.3-flash` through Vercel AI Gateway.\n\n## How results work\n\nEvery provider attempt is one row in the val-scoped\n`benchmark_measurements` SQLite table. Results pool measurements from every\ncompleted scheduled and manual run.\n\nBefore each run, the worker fetches Vercel's live endpoint catalog. It\nbenchmarks every returned provider and stores a normalized provider snapshot.\nProviders newly added by Vercel are included automatically. The full catalog\nalso stays on the run record so an interrupted run resumes against the exact\nprovider set it started with.\n\nThe sortable provider table reports:\n\n- Success rate across all attempts.\n- TTFT p50 and p95 across successful responses.\n- p50 and p5 output tokens per second across successful responses.\n- Cache hits among eligible repeated successful responses.\n- Effective observed cost per million input plus output tokens.\n\nEffective $/MTok is total observed cost divided by total input plus output\ntokens, multiplied by one million. Cached tokens remain part of input tokens,\nwhile their discounted price is reflected in observed cost.\n\np50 is the median. TTFT p95 is the slow latency tail, while tokens/sec p5 is\nthe slow throughput tail. With fewer than 20 observations, nearest-rank p95 and\np5 are the observed maximum and minimum.\n\nThe measurements page exposes only two public outcomes: Success and Error.\nSuccessful rows show a check mark. Error rows show a short, redacted error\nmessage; timeouts include the configured 75 second limit. Raw response text and\nprovider metadata remain private.\n\n## Schedule\n\n`benchmark/scheduled.ts` runs every 15 minutes in UTC. Each execution refreshes\nthe provider catalog, then runs the full benchmark. Requests have a 75 second\ntimeout and automatic SDK retries are disabled.\n\nA SQLite execution lock prevents scheduled and manual starts from overlapping.\nThe lock expires after 90 minutes so an interrupted worker cannot block future\nruns forever. Individual attempts are checkpointed, so an interrupted run can\nresume without duplicating stored measurements.\n\n## Run manually\n\nEdit `benchmark/config.ts` if needed, then open `benchmark/run.ts` in Val\nTown and click **Run**. Public visitors cannot invoke the runner from the HTTP\napp.\n\n## Access\n\nThe Gateway credential is stored as the encrypted\n`VERCEL_AI_GATEWAY_API_KEY` environment variable. The HTTP app exposes only\nread-only routes, so public visitors cannot start paid model requests.\n\n## Structure\n\n- `benchmark/` — configuration, fixed conversation, manual runner, and scheduled interval.\n- `db/` — val-scoped SQLite schema, writes, provider snapshots, and cross-run aggregation.\n- `frontend/` — React interface styled with Twind.\n- `shared/` — shared benchmark and result types.\n",[151],"57180eb9-f846-47ed-9f7f-5ec6aa1d02be","shared","f91bd0d2-a234-11f1-8cec-1607ee4eb77e",4,["D",1787848382634],"directory",[159],"6f24ec24-8f0c-4d0d-b0d3-fbc1bac995ce","frontend","01a04410-0100-7214-ba2d-95062f41a4ba",1,["D",1787848294548],[166],"5e355af5-f49e-4221-87df-a392a9184c4b","db","f73349d0-a234-11f1-922e-1607ee4eb77e",3,["D",1787848379437],[172],"1d67c685-0a4f-421f-a3c8-63f7187dfcee","benchmark","f52e35aa-a234-11f1-971d-1607ee4eb77e",2,["D",1787848376051],[178],"pulls","branches","forks","envVars",{"_190":168,"_192":64},"connections",{"_190":64,"_191":64},"count","unfilledCount","emptyCount",{"_44":177,"_32":178,"_30":179,"_140":-5,"_42":43,"_41":180,"_142":60,"_39":406,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_221":407,"_27":408},{"_44":398,"_32":399,"_30":400,"_140":179,"_42":43,"_41":401,"_142":25,"_39":402,"_13":168,"_34":35,"_36":403,"_24":25,"_147":-5,"_221":404,"_27":405},{"_44":394,"_32":33,"_30":31,"_140":179,"_42":43,"_41":14,"_142":25,"_39":395,"_13":303,"_34":35,"_36":38,"_24":25,"_147":-5,"_221":396,"_27":397},{"_44":385,"_32":386,"_30":387,"_140":179,"_42":43,"_41":388,"_142":25,"_39":389,"_13":180,"_34":390,"_36":391,"_24":25,"_147":-5,"_221":392,"_27":393},{"_44":377,"_32":378,"_30":379,"_140":179,"_42":43,"_41":380,"_142":25,"_39":381,"_13":168,"_34":35,"_36":382,"_24":25,"_147":-5,"_221":383,"_27":384},{"_44":369,"_32":370,"_30":371,"_140":179,"_42":43,"_41":372,"_142":25,"_39":373,"_13":64,"_34":35,"_36":374,"_24":25,"_147":-5,"_221":375,"_27":376},{"_44":171,"_32":172,"_30":173,"_140":-5,"_42":43,"_41":174,"_142":60,"_39":366,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_221":367,"_27":368},{"_44":357,"_32":358,"_30":359,"_140":173,"_42":43,"_41":360,"_142":60,"_39":361,"_13":362,"_34":35,"_36":363,"_24":25,"_147":-5,"_221":364,"_27":365},{"_44":348,"_32":349,"_30":350,"_140":173,"_42":43,"_41":351,"_142":25,"_39":352,"_13":353,"_34":35,"_36":354,"_24":25,"_147":-5,"_221":355,"_27":356},{"_44":339,"_32":340,"_30":341,"_140":173,"_42":43,"_41":342,"_142":25,"_39":343,"_13":344,"_34":35,"_36":345,"_24":25,"_147":-5,"_221":346,"_27":347},{"_44":330,"_32":331,"_30":332,"_140":173,"_42":43,"_41":333,"_142":25,"_39":334,"_13":335,"_34":35,"_36":336,"_24":25,"_147":-5,"_221":337,"_27":338},{"_44":165,"_32":166,"_30":167,"_140":-5,"_42":43,"_41":168,"_142":60,"_39":327,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_221":328,"_27":329},{"_44":323,"_32":289,"_30":282,"_140":167,"_42":43,"_41":168,"_142":60,"_39":324,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_221":325,"_27":326},{"_44":315,"_32":316,"_30":317,"_140":282,"_42":43,"_41":318,"_142":25,"_39":319,"_13":155,"_34":35,"_36":320,"_24":25,"_147":-5,"_221":321,"_27":322},{"_44":307,"_32":308,"_30":309,"_140":282,"_42":43,"_41":310,"_142":25,"_39":311,"_13":180,"_34":35,"_36":312,"_24":25,"_147":-5,"_221":313,"_27":314},{"_44":298,"_32":299,"_30":300,"_140":282,"_42":43,"_41":301,"_142":60,"_39":302,"_13":303,"_34":35,"_36":304,"_24":25,"_147":-5,"_221":305,"_27":306},{"_44":290,"_32":291,"_30":292,"_140":282,"_42":43,"_41":293,"_142":25,"_39":294,"_13":161,"_34":35,"_36":295,"_24":25,"_147":-5,"_221":296,"_27":297},{"_44":279,"_32":280,"_30":281,"_140":282,"_42":43,"_41":283,"_142":25,"_39":284,"_13":285,"_34":35,"_36":286,"_24":25,"_147":-5,"_221":287,"_27":288},{"_44":272,"_32":273,"_30":274,"_140":167,"_42":43,"_41":168,"_142":60,"_39":275,"_13":64,"_34":20,"_36":276,"_24":25,"_147":-5,"_221":277,"_27":278},{"_44":263,"_32":264,"_30":265,"_140":167,"_42":43,"_41":266,"_142":25,"_39":267,"_13":268,"_34":35,"_36":269,"_24":25,"_147":-5,"_221":270,"_27":271},{"_44":255,"_32":256,"_30":257,"_140":167,"_42":43,"_41":258,"_142":60,"_39":259,"_13":168,"_34":35,"_36":260,"_24":25,"_147":-5,"_221":261,"_27":262},{"_44":247,"_32":248,"_30":249,"_140":167,"_42":43,"_41":250,"_142":60,"_39":251,"_13":64,"_34":35,"_36":252,"_24":25,"_147":-5,"_221":253,"_27":254},{"_44":239,"_32":240,"_30":241,"_140":167,"_42":43,"_41":242,"_142":60,"_39":243,"_13":174,"_34":35,"_36":244,"_24":25,"_147":-5,"_221":245,"_27":246},{"_44":158,"_32":159,"_30":160,"_140":-5,"_42":43,"_41":161,"_142":60,"_39":236,"_13":64,"_34":163,"_36":-5,"_24":25,"_147":-5,"_221":237,"_27":238},{"_44":227,"_32":228,"_30":229,"_140":160,"_42":43,"_41":230,"_142":25,"_39":231,"_13":232,"_34":35,"_36":233,"_24":25,"_147":-5,"_221":234,"_27":235},{"_44":150,"_32":151,"_30":152,"_140":-5,"_42":43,"_41":153,"_142":25,"_39":224,"_13":155,"_34":20,"_36":156,"_24":25,"_147":-5,"_221":225,"_27":226},{"_44":138,"_32":139,"_30":58,"_140":-5,"_42":43,"_41":141,"_142":25,"_39":220,"_13":144,"_34":145,"_36":146,"_24":25,"_147":148,"_221":222,"_27":223},["D",1787858406876],"children",[],[139],["D",1787859186418],[],[151],"f267ab42-26d8-4071-a786-eeea5b379d71","types.ts","0553d0c0-a235-11f1-94c6-1607ee4eb77e",194,["D",1787858397606],9,"export type BenchmarkStatus = \"running\" | \"completed\" | \"failed\";\nexport type MeasurementStatus = \"ok\" | \"error\";\n\nexport interface BenchmarkConfig {\n label: string;\n modelId: string;\n runsPerProvider: number;\n maxOutputTokens: number;\n providerConcurrency: number;\n requestTimeoutMs: number;\n providers?: string[];\n}\n\nexport interface GatewayEndpoint {\n provider_name: string;\n name?: string;\n has_zdr?: boolean;\n has_no_training?: boolean;\n supports_implicit_caching?: boolean;\n pricing?: Record;\n latency_last_1h?: { p50?: number; p95?: number };\n throughput_last_1h?: { p50?: number; p95?: number };\n tags?: string[];\n [key: string]: unknown;\n}\n\nexport interface BenchmarkRunSummary {\n id: string;\n label: string;\n modelId: string;\n status: BenchmarkStatus;\n startedAt: string;\n completedAt: string | null;\n runsPerProvider: number;\n maxOutputTokens: number;\n providerCount: number;\n successfulMeasurements: number;\n failedMeasurements: number;\n error: string | null;\n}\n\nexport type StoredMeasurementOutcome = \"text\" | \"no_text\" | \"error\";\nexport type MeasurementErrorCode =\n | \"no_text\"\n | \"timeout\"\n | \"provider_error\"\n | null;\n\nexport interface BenchmarkMeasurement {\n runId: string;\n provider: string;\n attempt: number;\n status: MeasurementStatus;\n outcome: StoredMeasurementOutcome;\n errorCode: MeasurementErrorCode;\n startedAt: string;\n completedAt: string;\n ttftMs: number | null;\n durationMs: number | null;\n inputTokens: number | null;\n outputTokens: number | null;\n cacheReadTokens: number | null;\n cacheWriteTokens: number | null;\n costUsd: number | null;\n tokensPerSecond: number | null;\n responsePreview: string | null;\n error: string | null;\n providerMetadata: Record | null;\n providerInfo: GatewayEndpoint;\n}\n\nexport type MeasurementPublicStatus = \"success\" | \"error\";\n\nexport type MeasurementSortKey =\n | \"runStartedAt\"\n | \"provider\"\n | \"attempt\"\n | \"status\"\n | \"startedAt\"\n | \"ttftMs\"\n | \"durationMs\"\n | \"tokensPerSecond\"\n | \"inputTokens\"\n | \"outputTokens\"\n | \"cacheReadTokens\"\n | \"cacheWriteTokens\"\n | \"costUsd\"\n | \"message\";\n\nexport type MeasurementSortDirection = \"asc\" | \"desc\";\n\nexport interface MeasurementListItem {\n runId: string;\n runStartedAt: string;\n runStatus: BenchmarkStatus;\n provider: string;\n attempt: number;\n status: MeasurementPublicStatus;\n startedAt: string;\n completedAt: string;\n ttftMs: number | null;\n durationMs: number | null;\n inputTokens: number | null;\n outputTokens: number | null;\n cacheReadTokens: number | null;\n cacheWriteTokens: number | null;\n costUsd: number | null;\n tokensPerSecond: number | null;\n error: string | null;\n}\n\nexport interface MeasurementRunOption {\n id: string;\n startedAt: string;\n status: BenchmarkStatus;\n measurementCount: number;\n}\n\nexport interface MeasurementListResponse {\n items: MeasurementListItem[];\n total: number;\n limit: number;\n offset: number;\n providers: string[];\n attempts: number[];\n runs: MeasurementRunOption[];\n}\n\nexport interface BenchmarkRunDetail extends BenchmarkRunSummary {\n config: BenchmarkConfig;\n endpointSnapshot: GatewayEndpoint[];\n measurements: BenchmarkMeasurement[];\n}\n\nexport interface ProviderResult {\n provider: string;\n runCount: number;\n attempts: number;\n successes: number;\n failures: number;\n successRate: number;\n p50TtftMs: number | null;\n p95TtftMs: number | null;\n p50DurationMs: number | null;\n p95DurationMs: number | null;\n p50TokensPerSecond: number | null;\n p05TokensPerSecond: number | null;\n cacheEligibleAttempts: number;\n cacheHits: number;\n cacheHitRate: number | null;\n effectiveCostPerMillionTokensUsd: number | null;\n}\n\nexport interface BenchmarkResults {\n modelId: string;\n runCount: number;\n attemptCount: number;\n successfulAttempts: number;\n failedAttempts: number;\n firstRunAt: string | null;\n lastCompletedAt: string | null;\n providers: ProviderResult[];\n}\n",[],[159,228],["D",1787848382634],[217],[159],"0921c7f1-3a63-4c80-b035-7a8887bae310","root.tsx","01a04410-0100-7214-ba2d-918a7f307571",44,["D",1787852742071],"/** @jsxImportSource npm:hono@4/jsx */\nimport { raw } from \"npm:hono@4/html\";\nimport { immutableFileUrl } from \"https://esm.town/v/std/utils/index.ts\";\n\nexport function Root() {\n return (\n <>\n {raw(\"\")}\n \n \n \n \n \n \n GLM-5.3-Flash Provider Benchmark\n

Diff · v212