[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_233":234,"_235":236},"routes/_app.x.$handle.$name",{"_130":129,"_35":69,"_139":115,"_30":140,"_13":14,"_141":-5,"_11":142,"_143":144,"_145":146,"_147":148,"_149":150,"_23":-5,"_151":-5,"_152":153,"_154":73,"_155":156,"_157":73,"_158":-5,"_159":-5,"_160":161},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_67":68,"_35":69,"_39":70,"_71":-5,"_72":73,"_74":75,"_76":77,"_78":-5,"_79":80,"_18":81,"_82":83,"_84":73,"_85":-5,"_86":-5,"_87":88,"_89":90,"_91":92,"_93":94,"_95":96,"_97":-5,"_98":99,"_100":99,"_101":102,"_103":102,"_104":102,"_105":99,"_106":99,"_107":108,"_109":110,"_111":112,"_113":73},"version",2,"changes",[19,20,21,22],"aiChatInfo","userId",{"_23":59,"_25":60,"_27":28,"_29":28,"_30":61,"_32":61},{"_23":51,"_25":52,"_27":28,"_29":28,"_30":53,"_32":53},{"_23":44,"_25":45,"_27":28,"_29":28,"_30":46,"_32":46},{"_23":24,"_25":26,"_27":28,"_29":28,"_30":31,"_32":31},"file",{"_33":34,"_35":31,"_36":37,"_38":-5,"_39":40,"_41":14,"_42":43,"_27":28},"previous",{"_33":34,"_35":31,"_36":37,"_38":-5},"deleted",true,"deletedHere","path","results","latestPath","fileId","01a0aaec-472f-7548-9493-cf9cfa1c9a7e","name","fileType","directory","content","createdAt",["D",1789574024485],"branchVersion","branchId","785ff6c9-d213-406f-ad60-76a7204bdaf0",{"_33":47,"_35":48,"_36":23,"_38":49,"_39":50,"_41":14,"_42":43,"_27":28},{"_33":47,"_35":48,"_36":23,"_38":49},"results/styles.css","01a0aaec-472f-7548-9493-c99e296a6f30","styles.css","body {\n font-family: system-ui, sans-serif;\n margin: 2rem auto;\n padding: 0 1rem;\n line-height: 1.4;\n}\n\nh2 {\n border-bottom: 1px solid #ccc;\n}\n\nh3 {\n margin: 1.5rem 0 0.5rem;\n}\n\ntable {\n width: 100%;\n border-collapse: collapse;\n}\n\nth,\ntd {\n text-align: left;\n vertical-align: top;\n padding: 0.4rem 0.6rem;\n border-bottom: 1px solid #eee;\n}\n\nth {\n font-size: 0.85rem;\n}\n\ntd.cost {\n font-variant-numeric: tabular-nums;\n white-space: nowrap;\n}\n\n.val-town {\n font-weight: 600;\n color: #0a7;\n}\n\ndetails summary {\n cursor: pointer;\n font-size: 0.85rem;\n}\n\ndetails p {\n margin: 0.5rem 0 0;\n white-space: pre-wrap;\n font-size: 0.9rem;\n}\n",["D",1789574024485],{"_33":54,"_35":55,"_36":56,"_38":57,"_39":58,"_41":14,"_42":43,"_27":28},{"_33":54,"_35":55,"_36":56,"_38":57},"results/sqlite.ts","01a0aaec-472f-7548-9493-c7a6fd5bcac6","sqlite.ts","script","import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\n\n// One-time setup: run this file manually to create the evals table.\nawait sqlite.execute(`\n CREATE TABLE IF NOT EXISTS evals (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n model TEXT NOT NULL,\n query TEXT NOT NULL,\n response TEXT NOT NULL,\n mentions TEXT NOT NULL DEFAULT '[]', -- JSON array of matched keywords\n cost REAL, -- USD, from OpenRouter usage.cost\n created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))\n )\n`);\n\nconsole.log(\"evals table ready\");\n",["D",1789574024485],{"_33":62,"_35":63,"_36":64,"_38":65,"_39":66,"_41":14,"_42":43,"_27":28},{"_33":62,"_35":63,"_36":64,"_38":65},"results/http.tsx","01a0aaec-472f-7548-9493-c302c1ae7542","http.tsx","http","/** @jsxImportSource npm:hono/jsx */\nimport { Hono } from \"npm:hono\";\nimport { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport { serveFile } from \"https://esm.town/v/std/utils/index.ts\";\n\ntype Eval = {\n id: number;\n model: string;\n query: string;\n response: string;\n mentions: string; // JSON array\n cost: number | null;\n created_at: string;\n};\n\nconst app = new Hono();\n\napp.get(\"/styles.css\", () => serveFile(\"/results/styles.css\", import.meta.url));\n\napp.get(\"/\", async (c) => {\n const { rows } = await sqlite.execute(\n `SELECT * FROM evals ORDER BY created_at DESC, id ASC`,\n );\n const evals = rows as unknown as Eval[];\n\n const byDay = new Map>();\n for (const e of evals) {\n const day = e.created_at.slice(0, 10);\n const byQuery = byDay.get(day) ?? new Map();\n byQuery.set(e.query, [...(byQuery.get(e.query) ?? []), e]);\n byDay.set(day, byQuery);\n }\n\n return c.html(\n \n \n OpenRouter evals\n \n \n \n

OpenRouter evals

\n {[...byDay].map(([day, byQuery]) => (\n

{day}

\n {[...byQuery].map(([query, evals]) => (\n

{query}

\n \n \n \n \n \n \n \n \n \n \n {evals.map((e) => )}\n \n

Model	Mentions	Cost	Response

\n ))}\n

\n ))}\n \n ,\n );\n});\n\nfunction EvalRow({ eval: e }: { eval: Eval }) {\n const mentions: string[] = JSON.parse(e.mentions);\n return (\n \n {e.model}\n \n {mentions.map((m, i) => (\n <>\n {m}\n {i < mentions.length - 1 ? \", \" : \"\"}\n \n ))}\n \n {e.cost == null ? \"—\" : `$${e.cost.toFixed(5)}`}\n \n

show

{e.response}

\n \n \n );\n}\n\nexport default app.fetch;\n",["D",1789574024485],"id","beb1ea3e-e3e0-46af-b3bb-b4c4a2ce4ff3","openrouter",["D",1789574006500],"deletedAt","legacyEnvVars",false,"privacy","public","httpPrivacy","restricted","description","imageUrl","https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/3cb991cc-ab5f-45d4-ed11-a055ea494d01/square512?exp=1789636840&sig=b5d1f1279a1ebf291c8a5bd31fb82ffc90883e77c5392651367269cc9b7c8737","6906d0a1-0165-46b9-8717-60dfd935316a","pinnedFileId","01a0aaec-472f-7548-9493-d2553a1915a9","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","019aa807-52c2-709c-928b-2029bae9a558","tursoName","beb1ea3ee3e046afb3bbb4c4a2ce4ff3","tursoHostname","beb1ea3ee3e046afb3bbb4c4a2ce4ff3-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789574488569],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_130":129,"_138":134},"org",{"_130":129,"_131":132,"_133":134,"_18":-5,"_135":136,"_137":28},"branch",{"_13":114,"_35":115,"_67":43,"_116":68,"_117":118,"_119":120,"_121":122,"_123":124,"_125":126,"_127":69,"_128":129},"isOwned",109,"main","projectId","forkedBranchId","73fb214a-24dc-47d0-b133-4e1551bec9d4","forkedBranchVersion",121,"forkedProjectPrivacy","unlisted","forkedProjectId","60f95924-a1c1-4d0f-bcf7-d9c821abf894","forkedProjectOrgId","631c5cff-7089-4994-8427-c7580df789cd","projectName","userHandle","templates","handle","displayName","Templates","avatar","https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/129cab32-3bd7-4c3e-f39b-f12900c76100/square512?exp=1789636840&sig=cdcd3f6735d415932dbbaa8c52ec2aa6dbde901fe0ed082a78f1e68474d8d221","tier","business","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_67":68,"_35":69,"_39":228,"_71":-5,"_72":73,"_74":75,"_76":77,"_78":-5,"_79":80,"_18":81,"_82":83,"_84":73,"_85":-5,"_86":-5,"_87":88,"_89":90,"_91":92,"_93":94,"_95":229,"_97":-5,"_98":99,"_100":99,"_101":102,"_103":102,"_104":102,"_105":99,"_106":99,"_107":230,"_109":231,"_111":232,"_113":73},"tree",[211,212,213,214],"files",[211,212,213,214],"counts",{"_201":88,"_202":176,"_203":88,"_204":205,"_206":207},"contents",[168,169,170,171],"upstreamProject","upstreamBranch",{"_67":118,"_39":162,"_163":164,"_35":115,"_116":124,"_13":120,"_117":-5,"_119":-5,"_165":166,"_71":-5,"_113":73,"_74":122,"_128":167,"_89":126,"_127":69},"emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},["D",1789070688575],"updatedAt",["D",1789497231671],"lockfileKey","1789497231751-esm.town_v_pmillspaugh_openrouter@121-main","pmillspaugh",{"_67":194,"_35":195,"_33":83,"_175":-5,"_42":43,"_41":176,"_177":73,"_39":196,"_13":88,"_36":197,"_38":198,"_27":73,"_180":199,"_30":200},{"_67":188,"_35":189,"_33":190,"_175":-5,"_42":43,"_41":176,"_177":73,"_39":191,"_13":88,"_36":23,"_38":192,"_27":73,"_180":-5,"_30":193},{"_67":182,"_35":183,"_33":184,"_175":-5,"_42":43,"_41":176,"_177":73,"_39":185,"_13":88,"_36":23,"_38":186,"_27":73,"_180":-5,"_30":187},{"_67":172,"_35":173,"_33":174,"_175":-5,"_42":43,"_41":176,"_177":73,"_39":178,"_13":88,"_36":23,"_38":179,"_27":73,"_180":-5,"_30":181},"c889d349-1bc6-45e1-bf58-b061c0637a9a","queries.json","01a0aaec-472f-7548-9493-dff4a32e21d1","parentId",1,"latest",["D",1789574006500],"[\n \"best vibe coding tools\",\n \"how to build an mcp server\",\n \"how to receive webhooks\",\n \"host a cron job\"\n]","label",[173],"8c157dad-7f97-4086-8518-670aa29eb39c","models.json","01a0aaec-472f-7548-9493-d80cd5ce184e",["D",1789574006500],"[\n \"anthropic/claude-haiku-4.5\",\n \"openai/gpt-5.6-luna\",\n \"google/gemini-3.8-flash\",\n \"deepseek/deepseek-v4.1-flash\",\n \"z-ai/glm-5.3-flash\"\n]",[183],"c2385585-5281-499e-adcb-9a2dbbc3ca94","keywords.json","01a0aaec-472f-7548-9493-d6fc2d5c8185",["D",1789574006500],"[\n \"val town\",\n \"cloudflare\",\n \"vercel\",\n \"netlify\",\n \"github\",\n \"aws lambda\",\n \"supabase\",\n \"fly.io\",\n \"railway\",\n \"render\",\n \"deno deploy\",\n \"replit\",\n \"lovable\",\n \"v0\",\n \"bolt\",\n \"cursor\",\n \"windsurf\",\n \"base44\",\n \"codepen\",\n \"glitch\",\n \"zapier\",\n \"retool\",\n \"n8n\",\n \"pipedream\"\n]",[189],"e7dc7405-4475-4cde-a8a2-8ddd3e6c3a6a","evals.ts",["D",1789574006500],"interval","import { OpenRouter } from \"npm:@openrouter/sdk@1.2\";\nimport { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport MODELS from \"./models.json\" with { type: \"json\" };\nimport QUERIES from \"./queries.json\" with { type: \"json\" };\nimport KEYWORDS from \"./keywords.json\" with { type: \"json\" };\n\nconst client = new OpenRouter({\n apiKey: Deno.env.get(\"OPENROUTER_API_KEY\"),\n httpReferer: \"https://std-openrouter.val.run\",\n appTitle: \"pmillspaugh/openrouter\",\n});\n\nexport default async function runEvals() {\n await Promise.all(MODELS.map(async (model) => {\n for (const query of QUERIES) {\n try {\n console.log(`MODEL: ${model} | QUERY: ${query}`);\n const completion = await client.chat.send({\n chatRequest: {\n model,\n messages: [{\n role: \"user\",\n content: `${query} (answer in 1-5 sentences)`,\n }],\n usage: { include: true },\n },\n }, { timeoutMs: 60_000 });\n\n if (completion instanceof ReadableStream) {\n throw new Error(\"Expected a non-streaming response\");\n }\n\n const response = completion.choices[0].message.content;\n if (typeof response !== \"string\") {\n throw new Error(\n `No content: ${JSON.stringify(completion.choices[0])}`,\n );\n }\n\n const mentions = extractMentions(query, response, KEYWORDS);\n const cost = completion.usage?.cost ?? null;\n\n await sqlite.execute({\n sql:\n `INSERT INTO evals (model, query, response, mentions, cost) VALUES (?, ?, ?, ?, ?)`,\n args: [model, query, response, JSON.stringify(mentions), cost],\n });\n } catch (err) {\n console.error(`[${model}] Failed \"${query}\":`, err);\n }\n }\n }));\n}\n\nfunction extractMentions(\n query: string,\n response: string,\n keywords: string[],\n): string[] {\n const normalize = (s: string) => s.toLowerCase().replace(/[^a-z0-9]/g, \"\");\n const needle = normalize(query);\n const haystack = normalize(response);\n return keywords.filter((kw) => {\n const k = normalize(kw);\n return haystack.includes(k) && !needle.includes(k);\n });\n}\n","openrouter-demo",[195],"pulls","branches","forks","envVars",{"_208":176,"_210":88},"connections",{"_208":88,"_209":88},"count","unfilledCount","emptyCount",{"_67":194,"_35":195,"_33":83,"_175":-5,"_42":43,"_41":176,"_177":73,"_39":225,"_13":88,"_36":197,"_38":198,"_27":73,"_180":199,"_216":226,"_30":227},{"_67":188,"_35":189,"_33":190,"_175":-5,"_42":43,"_41":176,"_177":73,"_39":222,"_13":88,"_36":23,"_38":192,"_27":73,"_180":-5,"_216":223,"_30":224},{"_67":182,"_35":183,"_33":184,"_175":-5,"_42":43,"_41":176,"_177":73,"_39":219,"_13":88,"_36":23,"_38":186,"_27":73,"_180":-5,"_216":220,"_30":221},{"_67":172,"_35":173,"_33":174,"_175":-5,"_42":43,"_41":176,"_177":73,"_39":215,"_13":88,"_36":23,"_38":179,"_27":73,"_180":-5,"_216":217,"_30":218},["D",1789574006500],"children",[],[173],["D",1789574006500],[],[183],["D",1789574006500],[],[189],["D",1789574006500],[],[195],["D",1789574006500],["D",1789574488569],{"_130":129,"_138":134},{"_130":129,"_131":132,"_133":134,"_18":-5,"_135":136,"_137":28},{"_13":114,"_35":115,"_67":43,"_116":68,"_117":118,"_119":120,"_121":122,"_123":124,"_125":126,"_127":69,"_128":129},"ctx",{"_107":-5,"_159":266,"_267":268,"_269":270,"_271":73,"_272":73,"_273":73,"_274":28,"_275":-5,"_276":-7,"_277":-7,"_278":88,"_279":73,"_280":73,"_281":-5,"_282":73,"_283":73,"_284":73,"_285":-5},"ENV",{"_237":238,"_239":240,"_241":115,"_242":243,"_244":245,"_246":247,"_248":249,"_250":251,"_252":253,"_254":255,"_256":257,"_258":259,"_260":261,"_262":263,"_264":265},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","c73f21da490985bff19c279eb0e09db24d688e4f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v2