[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8},"actionData","errors","root",{"_105":106,"_107":108},"routes/embed.x.$handle.$name.$",{"_9":10,"_11":12,"_13":14,"_15":16,"_17":18,"_19":20,"_21":-5},"handle","ziggyware","name","free-ai","path","router-core.ts","project",{"_22":45,"_11":12,"_33":46,"_47":-5,"_48":42,"_49":50,"_51":50,"_52":53,"_54":-5,"_55":56,"_57":-5,"_58":42,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":66,"_67":68,"_69":70,"_71":-5,"_72":73,"_74":73,"_75":76,"_77":76,"_78":76,"_79":73,"_80":73,"_81":82,"_83":84,"_85":86,"_87":42},"file",{"_22":23,"_11":14,"_24":25,"_26":-5,"_27":28,"_29":30,"_31":32,"_33":34,"_35":36,"_37":38,"_39":40,"_41":42,"_43":-5,"_13":44},"permalink","https://www.val.town/x/ziggyware/free-ai/code/router-core.ts","extension","id","79642d84-a85e-4c3c-bf5a-bd41f5caeca2","fileId","6a4ef8c6-ab4d-11f1-9c11-1607ee4eb77e","parentId","branchId","1a44bbd4-418c-11f1-af64-42b51c65c3df","branchVersion",1642,"latest",true,"createdAt",["D",1789500412535],"version",40,"fileType","script","content","import { getProviderArray, type ProviderInstance, resolveTargets } from \"./inference-provider.ts\";\nimport { all, bumpProviderStats, providerStatsAll, run, sql } from \"./db.ts\";\n\nexport const MAX_CONCURRENCY = 3;\nconst MAX_FAILS = 2;\nconst BREAK_DURATION_MS = 25_000;\n// Per-attempt ceiling when no caller deadline narrows it further. Was 120_000, then cut to 15_000 to stop\n// one stalling vendor from eating the whole response window — but 15s proved too short for what this app\n// actually sends: live evidence (a Groq 413) confirmed prompts here run 18k+ tokens with up to 16k\n// max_tokens requested, and a live /health comparison showed Anthropic's own success rate fall from 6/7 to\n// 6/23 (nearly all new failures API_STALLED_TIMEOUT) once attempts were being cut off at 15s — a real,\n// legitimately-completing generation for this workload routinely needs more than that. 30s is a genuine\n// tradeoff against breadth (fewer distinct providers fit in one invocation's ~50s safety ceiling), chosen\n// because a provider that's actually working and just needs time is worth far more than three that were\n// never going to succeed anyway getting tried instead.\nexport const HARD_TIMEOUT_MS = 30_000;\nconst RETRYABLE = new Set([408, 409, 425, 429, 500, 502, 503, 504]);\n\nexport let inflight = 0;\n// FIFO admission instead of the previous `while (inflight >= MAX) await sleep(10)` spin: polling burned a\n// timer tick per 10ms per waiter and — worse — admitted waiters in whatever order their timers happened\n// to fire, so under contention a request that arrived later could jump one that had been waiting.\n// Waiters now resolve strictly in arrival order, with zero timers while blocked.\nconst waiters: (() => void)[] = [];\nasync function acquire(): Promise {\n if (inflight < MAX_CONCURRENCY) { inflight++; return; }\n await new Promise((resolve) => waiters.push(resolve));\n inflight++;\n}\nfunction release(): void {\n inflight--;\n const next = waiters.shift();\n if (next) next();\n}\n\n// Config-dead demotion. The breaker deliberately never trips on non-retryable 4xx (a bad key or model id\n// must not oscillate it) — correct, but the consequence, visible in every trail this session, was that a\n// provider returning 404 \"Function not found\", 403 tier_not_allowed or 402 quota-exhausted got a full,\n// fresh attempt on EVERY pass of EVERY request, forever, each one costing a round-trip before the next\n// candidate. Demotion is the missing middle ground: after CFG_DEAD_AFTER consecutive config-class\n// failures the instance is moved to the END of the order (still tried — fail-open is preserved, nothing\n// is ever excluded), for CFG_DEAD_MS, persisted cross-isolate. One success clears it immediately.\nconst CFG_DEAD_AFTER = 2;\nconst CFG_DEAD_MS = 10 * 60_000;\nconst CFG_DEAD_STATUSES = new Set([401, 402, 403, 404]);\n// A 429 is retryable in general, but a 429 whose body says the QUOTA is gone (daily free cap, billing\n// hard-limit) is not a transient rate limit — it's config-dead until the window resets. Observed live on\n// SambaNova / HuggingFace / OpenAI this session.\n// Adversarial case that shaped this regex: Groq's TRANSIENT per-minute 429 reads \"Rate limit reached for\n// model … on tokens per minute (TPM) … Please try again in 2.5s\" — a naive \"limit reached\" match would\n// have demoted a perfectly healthy provider for 10 minutes on a 2-second backoff. So: any explicit\n// short-retry / per-minute phrasing wins outright as transient, and only unmistakably daily/billing\n// wording counts as exhausted.\n//\n// Second adversarial case, found live (a real session pinned groq/gpt-oss-120b and #1 at the front of the\n// plan for hours): the bare \"try again in\" clause matched on ANY retry-after phrasing regardless of the\n// actual wait it named, including \"on tokens per day (TPD) … Please try again in 26m56s\" — a 27-MINUTE\n// daily-cap block, misclassified as transient purely because the sentence happens to also carry a\n// countdown. That kept a provider that would fail identically for the rest of the day sitting at tier-1\n// head priority (never demoted, since !QUOTA_TRANSIENT short-circuited isConfigClass to false) instead of\n// being pushed to the tail after two such failures like any other config-dead instance — costing a real\n// round-trip on literally every attempt, forever, while never getting out of the way for a working\n// provider. Fix: \"try again in\" only counts as transient when the number right after it is a bare\n// seconds value with no minutes/hours component attached (\"try again in 2.5s\", not \"26m56s\" or \"1h2m3s\") —\n// verified against both live daily-cap error bodies (now correctly NOT transient) and the original TPM\n// adversarial case above (still transient, unaffected — it doesn't go through this clause at all).\nconst QUOTA_TRANSIENT = /(?:try again in\\s+(?!\\d+h\\b|\\d+m\\d)\\d+(?:\\.\\d+)?\\s*s\\b)|per minute|per second|\\bTPM\\b|\\bRPM\\b|\\bTPS\\b/i;\nconst QUOTA_EXHAUSTED = /quota|billing|insufficient|credit|exceeded your|per.?day|daily|free-models-per-day|hard limit/i;\nfunction isConfigClass(status: number | undefined, body: string): boolean {\n if (status === undefined) return false;\n if (CFG_DEAD_STATUSES.has(status)) return true;\n return status === 429 && !QUOTA_TRANSIENT.test(body) && QUOTA_EXHAUSTED.test(body);\n}\n\ntype Breaker = {\n failCount: number; state: \"CLOSED\" | \"OPEN\" | \"HALF_OPEN\"; nextAttempt: number;\n requests: number; ok: number; failed: number; lastError: string | null; // counters exist solely for /health's instance roster\n cfgFails: number; demotedUntil: number; demotedReason: string | null;\n fromDb: boolean; // true ⇒ this isolate's OPEN/demoted state was learned from omni_state, not observed here — so a later re-hydrate may retract it\n};\nconst breakers = new Map();\nexport function getBreaker(name: string): Breaker {\n let b = breakers.get(name);\n if (!b) breakers.set(name, b = { failCount: 0, state: \"CLOSED\", nextAttempt: 0, requests: 0, ok: 0, failed: 0, lastError: null, cfgFails: 0, demotedUntil: 0, demotedReason: null, fromDb: false });\n return b;\n}\nexport const isDemoted = (name: string) => getBreaker(name).demotedUntil > Date.now();\nfunction admit(name: string) {\n const b = getBreaker(name);\n if (b.state === \"OPEN\" && Date.now() >= b.nextAttempt) b.state = \"HALF_OPEN\";\n return b.state !== \"OPEN\";\n}\n// `trip`=false records the attempt (roster counters) without touching breaker state — used for 4xx config\n// errors, which fail over but must never oscillate the breaker on a bad key/model id (see `attempt` below).\n// The breaker's open/closed decision stays in-memory-only (per-isolate — no DB read added to the hot\n// path); the counters are additionally persisted (best-effort, never blocks or fails the actual routing\n// decision) so /health reflects reality across isolates instead of whichever one happens to answer a\n// GET. bumpProviderStats failing silently is acceptable: it's diagnostics, not routing logic.\n// Breaker/demotion persistence is written ONLY on a transition (closed→open, open→closed, demoted↔not) —\n// the first version of this wrote on every report(), one extra DB round-trip per attempt for a value that\n// almost never changed. The stats bump is unavoidable (it IS the per-attempt counter); this one wasn't.\nasync function report(name: string, ok: boolean, error?: string, trip = true, configClass = false) {\n const b = getBreaker(name);\n const wasOpen = b.state === \"OPEN\", wasDemoted = b.demotedUntil > Date.now();\n b.fromDb = false; // whatever happens below was observed HERE — local evidence outranks anything hydrated\n b.requests++;\n if (ok) { b.ok++; b.failCount = 0; b.state = \"CLOSED\"; b.cfgFails = 0; b.demotedUntil = 0; b.demotedReason = null; }\n else {\n b.failed++;\n if (error) b.lastError = error;\n if (trip && (++b.failCount >= MAX_FAILS || b.state === \"HALF_OPEN\")) {\n b.state = \"OPEN\"; b.nextAttempt = Date.now() + BREAK_DURATION_MS; b.failCount = 0;\n }\n if (configClass) {\n if (++b.cfgFails >= CFG_DEAD_AFTER) { b.demotedUntil = Date.now() + CFG_DEAD_MS; b.demotedReason = (error ?? \"config error\").slice(0, 160); }\n } else b.cfgFails = 0; // a transient failure between config errors resets the streak — demotion needs a CONSISTENT signal\n }\n await bumpProviderStats(name, ok, error ?? null, Date.now()).catch(() => {});\n const isOpen = b.state === \"OPEN\", nowDemoted = b.demotedUntil > Date.now();\n if (isOpen !== wasOpen || nowDemoted !== wasDemoted) await persistBreaker(name, b).catch(() => {});\n}\nexport const circuitStates = () => getProviderArray().map((p) => ({ name: p.name, state: getBreaker(p.name).state }));\n\n// ────────────────────────────────────────────────────────────────\n// Breaker cross-isolate persistence (item 5). The in-memory `breakers` Map above still governs the\n// admit()/report() decision on THIS isolate with zero added DB read on the hot path — that design choice\n// (see the comment on `report`) is unchanged. What was missing: a fresh isolate started moments after\n// another isolate opened a breaker had no way to know, so /health's `cooling` field and a brand-new\n// isolate's own routing could both treat a genuinely-tripped provider as available again. omni_state\n// (already reserved in db.ts's schema comment for exactly this: \"router cooldowns / dead models\") now\n// carries each OPEN breaker's state so a new isolate can hydrate it once, cheaply, instead of per-attempt.\nfunction persistBreaker(name: string, b: Breaker): Promise {\n const key = `breaker:${name}`;\n const now = Date.now();\n const demoted = b.demotedUntil > now;\n if (b.state === \"OPEN\" || demoted) {\n const value = JSON.stringify({\n state: b.state, nextAttempt: b.nextAttempt, lastError: b.lastError,\n demotedUntil: demoted ? b.demotedUntil : 0, demotedReason: demoted ? b.demotedReason : null,\n });\n return run(sql`INSERT INTO omni_state (key, value, ts) VALUES (${key}, ${value}, ${now})\n ON CONFLICT(key) DO UPDATE SET value = excluded.value, ts = excluded.ts`);\n }\n return run(sql`DELETE FROM omni_state WHERE key = ${key}`);\n}\nlet hydratedAt = 0;\n// Re-read at most once per HYDRATE_TTL_MS. The first version hydrated ONCE per isolate and then never\n// again, which live trails proved wrong within the hour: message #747 (06:03:04) tried nemotron-ultra,\n// cerebras and sambanova first — all three already demoted by a sibling isolate at 06:02:24 — because\n// this isolate had hydrated earlier and considered itself done. A SELECT on a tiny table every few\n// seconds is nothing next to the 2–30s LLM call it precedes. Persisted rows are applied set-only;\n// entries this isolate only ever learned from the DB (fromDb) are retracted when their row disappears\n// (another isolate saw a success and cleared it). Locally observed state is never overwritten.\nconst HYDRATE_TTL_MS = 3_000;\nasync function hydrateBreakers(): Promise {\n const now = Date.now();\n if (now - hydratedAt < HYDRATE_TTL_MS) return;\n hydratedAt = now;\n const r = await all(\"omni_state\", sql`SELECT * FROM omni_state WHERE key LIKE 'breaker:%'`);\n if (!r.ok) return;\n const seen = new Set();\n for (const row of r.value as any[]) {\n try {\n const data = JSON.parse(row.value);\n const name = row.key.slice(\"breaker:\".length);\n seen.add(name);\n const b = getBreaker(name);\n const localOpen = b.state === \"OPEN\" && !b.fromDb, localDemoted = b.demotedUntil > now && !b.fromDb;\n if (!localOpen && data.state === \"OPEN\" && typeof data.nextAttempt === \"number\" && data.nextAttempt > now) {\n b.state = \"OPEN\"; b.nextAttempt = data.nextAttempt; b.lastError = data.lastError ?? b.lastError; b.fromDb = true;\n }\n if (!localDemoted && typeof data.demotedUntil === \"number\" && data.demotedUntil > now) {\n b.demotedUntil = data.demotedUntil; b.demotedReason = data.demotedReason ?? null; b.fromDb = true;\n }\n } catch { /* malformed row — skip it, never let a bad persisted entry block routing */ }\n }\n for (const [name, b] of breakers) {\n if (b.fromDb && !seen.has(name)) { // retracted elsewhere\n if (b.state === \"OPEN\") b.state = \"CLOSED\";\n b.demotedUntil = 0; b.demotedReason = null; b.fromDb = false;\n }\n }\n}\n// Item 2: the \"clear cooldowns\" button in ui-settings.ts (routerReset()) has always called a POST\n// /api/reset that never existed anywhere in router.tsx — silently no-op'd behind a 404 the UI never\n// surfaced. This is the function that route now actually backs: clears every in-memory breaker on THIS\n// isolate and, best-effort, the persisted OPEN state so other isolates don't re-hydrate the cleared\n// breakers back open on their next call.\nexport async function resetBreakers(): Promise<{ cleared: number }> {\n const cleared = breakers.size;\n breakers.clear();\n const r = await all(\"omni_state\", sql`SELECT key FROM omni_state WHERE key LIKE 'breaker:%'`);\n if (r.ok) for (const row of r.value as any[]) await run(sql`DELETE FROM omni_state WHERE key = ${row.key}`).catch(() => {});\n hydratedAt = Date.now(); // nothing left to hydrate — prevents a stale read from racing back in before the next natural trip\n return { cleared };\n}\n// Item 4: provider_stats accumulates forever with no way to tell \"still failing\" from \"was failing before\n// a fix landed\" (the exact ambiguity flagged after this session's stale-binding fix). Distinct from\n// resetBreakers(): this clears the durable request/ok/failed counters, not the live routing decision.\nexport async function resetProviderStats(): Promise {\n await run(sql`DELETE FROM provider_stats`).catch(() => {});\n}\n/** Per-instance roster for /health — the settings panel's \"test / status\" table reads exactly this shape.\n * requests/ok/failed/last_error come from the durable table (accurate across isolates). cooling/demoted\n * are hydrated from the persisted breaker state first, so a breaker another isolate tripped is visible\n * here too — this closes the \"may show ready while a concurrent request elsewhere is avoiding it\" gap\n * the previous version of this comment documented. */\nexport async function getInstanceRoster() {\n await hydrateBreakers();\n const statsRow = await providerStatsAll();\n const stats = new Map((statsRow.ok ? statsRow.value : []).map((r: any) => [r.name, r]));\n const now = Date.now();\n return getProviderArray().map((p) => {\n const b = getBreaker(p.name);\n const s = stats.get(p.name);\n const cooling = b.state === \"OPEN\" ? { ms_left: Math.max(0, b.nextAttempt - now), reason: b.lastError ?? \"circuit open\" } : null;\n const demoted = b.demotedUntil > now ? { ms_left: b.demotedUntil - now, reason: b.demotedReason ?? \"config error\" } : null;\n return {\n name: p.name, key: p.key ? `${p.keyEnv}…${p.key.slice(-4)}` : null, key_issue: p.keyIssue,\n requests: s?.requests ?? 0, ok: s?.ok ?? 0, failed: s?.failed ?? 0,\n cooling, demoted, last_error: b.lastError ?? s?.last_error ?? null,\n };\n });\n}\n\nexport async function getSavedModels(): Promise> {\n const r = await all(\"omni_router_config\", sql`SELECT * FROM omni_router_config`);\n if (!r.ok) return {};\n return Object.fromEntries((r.value as any[]).map((row) => [row.provider, row.selected_model]));\n}\n\nexport type RouteOpts = {\n model?: string; tools?: unknown[]; tool_choice?: unknown;\n temperature?: number; max_tokens?: number; stream?: boolean; response_format?: unknown;\n deadlineMs?: number; // caller's remaining wall-clock budget (ms), from request receipt, for the WHOLE routeInference call — not per-attempt\n};\nexport type RouteHit = { provider: ProviderInstance; model: string; latencyMs: number; res: Response; plan?: string[]; trail?: RouteAttempt[] };\n// One row per provider try (or skip) — the raw material for the routing flowchart (app.tsx's PassMeter\n// turns this into a synthetic \"route\" ToolEvent; ui-timeline.ts draws it as a chain of nodes). Attached to\n// RouteHit on success and to the thrown error on exhaustion, so the flowchart exists in BOTH outcomes —\n// \"what did it actually try\" must answer the same whether the turn ended in an answer or a 503.\nexport type RouteAttempt = {\n provider: string; vendor: string; model: string; pass: number; timeoutMs: number;\n ok: boolean; skipped?: boolean; demoted?: boolean; latencyMs?: number; error?: string; ts: number;\n};\n\n// A 200 OK with an empty message, or a message that's nothing but dots (a known vendor failure\n// signature — truncated/garbage generations some providers return under load instead of a real error),\n// is not a valid answer. Only a plain-string content is checked — array/multi-part content bodies are\n// left alone rather than risk a false positive on a legitimate format we haven't seen. tool_calls-only\n// messages (content legitimately empty/null) are never flagged.\nfunction isEmptyOrDotContent(msg: any): boolean {\n if (Array.isArray(msg?.tool_calls) && msg.tool_calls.length) return false;\n if (typeof msg?.content !== \"string\") return false;\n const stripped = msg.content.replace(/\\s+/g, \"\");\n return stripped.length === 0 || /^\\.+$/.test(stripped);\n}\n\n// 4xx config errors still fail over but don't trip the breaker (no oscillation on a bad key/model id).\nasync function attempt(p: ProviderInstance, model: string, messages: unknown[], o: RouteOpts, timeoutMs: number): Promise {\n // A malformed secret never reaches the network: gemini's `…3, }` key produced 62 identical \"Failed to\n // construct 'Request': … not a valid ByteString\" failures — thrown by the Request constructor, so it\n // had no HTTP status, was classed retryable, tripped the breaker on a 25s cycle and was retried on\n // every single request forever. It is a config error by definition; say so, and let demotion park it.\n if (p.keyIssue) throw Object.assign(new Error(`KEY_MALFORMED (${p.keyEnv}): ${p.keyIssue}`), { retryable: false, configClass: true, status: 0 });\n const payload: Record = { model, messages, stream: !!o.stream };\n if (o.max_tokens && o.max_tokens > 0) payload.max_tokens = o.max_tokens;\n if (typeof o.temperature === \"number\") payload.temperature = o.temperature;\n // tool_choice:\"none\" with a tools array is semantically \"no tools callable\" — identical to sending no\n // tools at all — but Groq rejects the combination with HTTP 400 \"Tool choice is none, but model called a\n // tool\" (seen on every wrap-up pass this session). Send the equivalent form every vendor accepts.\n if (o.tools?.length && o.tool_choice !== \"none\") { payload.tools = o.tools; if (o.tool_choice) payload.tool_choice = o.tool_choice; }\n if (o.response_format) payload.response_format = o.response_format;\n const ctl = new AbortController();\n const timer = setTimeout(() => ctl.abort(), timeoutMs);\n const t0 = performance.now();\n try {\n const headers: Record = { \"Content-Type\": \"application/json\", \"HTTP-Referer\": \"https://val.town\", \"X-Title\": \"Ziggyware-OmniRouter\" };\n if (p.key) headers.Authorization = `Bearer ${p.key}`;\n const res = await fetch(`${p.base}/chat/completions`, { method: \"POST\", headers, body: JSON.stringify(payload), signal: ctl.signal });\n if (!res.ok) {\n const body = await res.text().catch(() => \"\");\n const e: any = new Error(`HTTP ${res.status}: ${body.slice(0, 300)}`);\n e.retryable = RETRYABLE.has(res.status);\n e.status = res.status;\n e.configClass = isConfigClass(res.status, body);\n throw e;\n }\n // Headers arrived within timeoutMs — that phase is done; clear its timer here, exactly like before\n // content validation existed. CONFIRMED REGRESSION (live /health: anthropic's success rate went from\n // 6/7 to 6/23 after this was added, nearly all new failures API_STALLED_TIMEOUT): keeping the SAME\n // timer armed through the full body read punished large, legitimately-slow-but-completing generations\n // (this app requests up to 16k max_tokens against prompts confirmed 18k+ tokens by a live Groq 413) —\n // a completion that needed even a few seconds past a shrunk, late-pass timeoutMs to finish sending was\n // aborted and misreported as a stall. Body read gets its OWN separate, generous, fixed budget instead —\n // independent of the caller's possibly-tiny remaining timeoutMs — so content validation (which does\n // need the full body) can't retroactively punish a provider that already answered in time.\n if (!o.stream) {\n clearTimeout(timer);\n const BODY_READ_TIMEOUT_MS = 20_000;\n let text: string;\n try {\n text = await Promise.race([\n res.text(),\n new Promise((_, reject) =>\n setTimeout(() => reject(Object.assign(new Error(\"API_STALLED_TIMEOUT\"), { retryable: true, name: \"AbortError\" })), BODY_READ_TIMEOUT_MS)\n ),\n ]);\n } catch (e: any) {\n res.body?.cancel().catch(() => {});\n throw e;\n }\n let json: any = null;\n try { json = JSON.parse(text); } catch { /* not JSON — let router-openai.ts's own parse surface the real error */ }\n const msg = json?.choices?.[0]?.message;\n if (json && isEmptyOrDotContent(msg)) {\n const e: any = new Error(`EMPTY_OR_DOT_RESPONSE: ${JSON.stringify(String(msg?.content ?? \"\")).slice(0, 60)}`);\n e.retryable = true;\n throw e;\n }\n return { provider: p, model, latencyMs: Math.round(performance.now() - t0), res: new Response(text, { status: res.status, headers: res.headers }) };\n }\n clearTimeout(timer);\n return { provider: p, model, latencyMs: Math.round(performance.now() - t0), res };\n } catch (e: any) {\n clearTimeout(timer);\n // Root cause of a real observed failure: a provider that reliably STALLS (never errors, just never\n // finishes — nemotron-ultra#2/#3 in a live /health dump: 14 and 1 requests, 0 ok, last_error\n // API_STALLED_TIMEOUT on every one) was structurally EXEMPT from demotion. isConfigClass() only ever\n // fires from the res.ok===false branch above (a real HTTP status), so an AbortError never set\n // configClass — report() then reset cfgFails to 0 every time (line \"else b.cfgFails = 0\"), so the\n // streak needed to demote (CFG_DEAD_AFTER) could never accumulate no matter how many times in a row\n // this exact instance stalled. Each stall still costs up to HARD_TIMEOUT_MS (30s) or\n // BODY_READ_TIMEOUT_MS (20s) of the invocation's 50s wall-clock ceiling — one or two stalling\n // instances sitting at their normal (undemoted) mid-plan position can consume nearly the ENTIRE\n // per-invocation budget, so everything scheduled after them in `targets` (in this case: groq's and\n // openrouter's later key slots) never gets attempted at all, invocation after invocation, because plan\n // order is otherwise stable. Marking this configClass (same as a config-dead 402/403/404) lets it\n // accumulate the same way: `retryable` stays true so the normal 25s breaker still applies unchanged,\n // but after CFG_DEAD_AFTER (2) consecutive stalls with no success in between (any ok=true resets the\n // streak — see report()) it now also gets demoted to the tail for CFG_DEAD_MS, the same relief valve\n // already used for a dead key/model, freeing its prime plan position for providers that answer fast.\n if (e.name === \"AbortError\") { const t: any = new Error(\"API_STALLED_TIMEOUT\"); t.retryable = true; t.configClass = true; throw t; }\n // Thrown before any bytes left the process (Request/URL construction) ⇒ our configuration, not their\n // outage. Retrying a TypeError is never going to produce a different result.\n if (e instanceof TypeError && /construct|Invalid URL|ByteString/i.test(String(e.message))) { e.retryable = false; e.configClass = true; }\n if (e.retryable === undefined) e.retryable = true;\n throw e;\n }\n}\n\n// Depth-first (priority, then key-slot) means a single vendor-wide outage — the vendor itself stalling,\n// not any one key being bad — burns every key slot of THAT vendor before a different vendor is ever\n// tried. Observed directly: openrouter/nemotron-ultra + #1 both API_STALLED_TIMEOUT, consuming the whole\n// deadline, while cerebras/groq/mistral/etc. sat untouched. Breadth-first — one instance per vendor first\n// (in existing priority order), second key-slots only in a later wave — means 2 failed attempts always\n// means 2 DIFFERENT vendors tried, not 2 keys of the one that happens to be down. Vendor-relative priority\n// (which model/vendor is preferred) is unchanged; only the within-vendor key-slot ordering is deferred.\nfunction breadthFirst(targets: ProviderInstance[]): ProviderInstance[] {\n const byVendor = new Map();\n for (const p of targets) {\n if (!byVendor.has(p.vendor)) byVendor.set(p.vendor, []);\n byVendor.get(p.vendor)!.push(p);\n }\n const waves: ProviderInstance[][] = [];\n for (const group of byVendor.values()) group.forEach((p, i) => (waves[i] ??= []).push(p));\n return waves.flat();\n}\n\n// Pure breadthFirst() above overcorrected: a second key set on the SAME provider/model (added for\n// throughput / per-key rate-limit headroom) is a different catalog row's slot only in the trivial\n// single-row case — for a vendor with multiple catalog rows (e.g. openrouter's 3 free models), that\n// alt key's position within its own vendor's flattened list can land many entries deep, which\n// breadthFirst then schedules behind EVERY other vendor's primary. Simulated against the real catalog\n// shape (openrouter, 2 keys/row): the alt key landed at attempt #13, while only ~3 attempts fit inside\n// a real turn deadline before providers that hang to HARD_TIMEOUT_MS exhaust it — so the alt key was\n// structurally almost never reached. Fix: let each vendor's TOP-priority row retry with ONE alt key\n// (SAME_KEY_RETRY_CAP) immediately, before the loop advances to a different vendor — this is the only\n// way to make \"I bought a backup key for this rate-limited provider\" actually work — then fall back to\n// breadthFirst for everything else (other rows of that vendor, and any keys beyond the cap). Confirmed\n// via simulation this bounds the regression: worst case (every attempt times out) still guarantees ≥2\n// DIFFERENT vendors get tried before the deadline dies, vs 3 under pure breadthFirst — a small, bounded\n// cost in exchange for alt keys actually being usable.\nconst SAME_KEY_RETRY_CAP = 1;\n// Tier 2 (below) was added after live diagnostics showed a vendor's OTHER catalog rows (e.g. openrouter's\n// qwen3-coder/gpt-oss, siblings of its top-priority nemotron-ultra) going untried in practice even though\n// tier 1 + breadthFirst() both nominally include them: tier 1 only ever admits ONE row per vendor (the top\n// one, plus its own alt keys), and breadthFirst groups by VENDOR not by catalog row — so a sibling row's\n// primary key ends up scheduled behind every alt-key retry of the vendor's first-tried row, many entries\n// deep, structurally unreachable before the wall clock runs out. Fix: a middle tier that gives each\n// vendor's OTHER rows' PRIMARY key (slot 0 only — not their alt keys, which stay in tier 3) a turn right\n// after tier 1's same-row retries, before any vendor's second key of anything. Deliberately NOT folding all\n// of a vendor's rows into tier 1 — that was considered and rejected: a vendor with many free-tier rows\n// could then burn several attempts against what's often a shared per-vendor daily rate limit (confirmed\n// this session: OpenRouter's \"free models per day\" cap) before a different vendor is ever reached.\nfunction orderTargets(targets: ProviderInstance[]): ProviderInstance[] {\n const byVendor = new Map();\n for (const p of targets) {\n if (!byVendor.has(p.vendor)) byVendor.set(p.vendor, []);\n byVendor.get(p.vendor)!.push(p);\n }\n const baseName = (p: ProviderInstance) => p.name.split(\"#\")[0];\n const heads: ProviderInstance[] = []; // tier 1: each vendor's top row, primary key + capped alt-key retries\n const headSet = new Set();\n const siblings: ProviderInstance[] = []; // tier 2: each vendor's OTHER rows, primary key only\n const siblingSet = new Set();\n for (const group of byVendor.values()) {\n if (!group.length) continue;\n const top = group[0]; // already sorted priority,slot — group[0] is this vendor's best row's primary key\n heads.push(top);\n headSet.add(top);\n let taken = 0;\n for (let j = 1; j < group.length && taken < SAME_KEY_RETRY_CAP; j++) {\n if (baseName(group[j]) !== baseName(top)) break; // only the SAME row's own alt keys qualify — a different model doesn't jump the queue\n heads.push(group[j]);\n headSet.add(group[j]);\n taken++;\n }\n const seenRows = new Set([baseName(top)]);\n for (const p of group) {\n const b = baseName(p);\n if (seenRows.has(b)) continue; // either the top row, or an alt key of a sibling row already claimed below\n seenRows.add(b);\n siblings.push(p);\n siblingSet.add(p);\n }\n }\n const rest = targets.filter((p) => !headSet.has(p) && !siblingSet.has(p));\n return [...heads, ...siblings, ...breadthFirst(rest)];\n}\n\n// Item 9: defensive ceiling on `trail`'s own memory footprint, independent of router-openai.ts's separate\n// last-TRAIL_CAP(60) slice taken when serializing the response. That slice only ever protects the wire\n// payload; the in-memory array itself was unbounded, so a pathological worst case (MAX_FULL_PASSES=10 full\n// laps of a large catalog, every attempt failing) could grow it far past what any consumer ever reads.\n// Trimming from the front keeps the most recent — and therefore most diagnostically relevant — entries.\nconst ROUTE_TRAIL_HARD_CAP = 300;\nfunction pushTrail(trail: RouteAttempt[], entry: RouteAttempt): void {\n trail.push(entry);\n if (trail.length > ROUTE_TRAIL_HARD_CAP) trail.splice(0, trail.length - ROUTE_TRAIL_HARD_CAP);\n}\n\nexport async function routeInference(messages: unknown[], o: RouteOpts = {}): Promise {\n await acquire();\n try {\n await hydrateBreakers();\n const live = getProviderArray();\n if (!live.length) throw Object.assign(new Error(\"NO_PROVIDERS_CONFIGURED\"), { status: 503 });\n const resolved = resolveTargets(o.model, live);\n const ordered = orderTargets(resolved.targets);\n // Demoted (config-dead) instances go to the tail as a block, in their existing relative order. This is a\n // stable partition, not a filter: every instance is still in `targets`, still reached by a full pass,\n // still covered by fail-open. What changes is only that a provider proven dead-by-configuration stops\n // costing a round-trip ahead of the ones that might actually answer.\n const demotedSet = new Set(ordered.filter((p) => isDemoted(p.name)));\n const targets = [...ordered.filter((p) => !demotedSet.has(p)), ...ordered.filter((p) => demotedSet.has(p))];\n const { forcedModel } = resolved;\n if (!targets.length) throw Object.assign(new Error(`UNKNOWN_MODEL: ${o.model}`), { status: 404 });\n const saved = await getSavedModels();\n const modelFor = (p: ProviderInstance) => forcedModel ?? saved[p.name.split(\"#\")[0]] ?? p.fallbackModel;\n const errors: string[] = [];\n const triedNames = new Set(); // distinct providers actually attempted (any pass) — see the exhaustion throw below\n const plan = targets.map((p) => p.name); // full planned order, BEFORE any attempt — the flowchart's \"intended route\"\n const trail: RouteAttempt[] = []; // every attempt AND skip, in the order it actually happened — the flowchart's \"actual route\"\n let currentPass = 1;\n\n // Hard wall-clock ceiling for THIS invocation, independent of whatever the caller's o.deadlineMs\n // says: Val Town's free tier kills the function outright around 60s with NO response sent at all —\n // that's worse than a clean 503. 50s leaves margin for parsing/serialization after the last attempt\n // returns. A caller deadline, when present and smaller, still narrows further (honors the caller's\n // own remaining turn budget) — it can only shrink this ceiling, never extend past it.\n const t0 = Date.now();\n const SAFETY_CEILING_MS = 50_000;\n const callDeadline = o.deadlineMs && o.deadlineMs > 0 ? Math.min(o.deadlineMs, SAFETY_CEILING_MS) : SAFETY_CEILING_MS;\n // Was lowered to 1_200 for breadth, then raised back up: a fast-failing provider (bad key, wrong\n // model, exhausted quota — the majority of what's actually being seen in practice) rejects in well\n // under a second regardless of this floor, so a low floor never helped THOSE. What it did do was let\n // a genuinely slow-but-working provider get attempted with almost no realistic chance to finish,\n // burning the slot for nothing. 5s is still short relative to HARD_TIMEOUT_MS's 30s, but below it a\n // real non-streaming completion for this workload's prompt/output sizes has no realistic chance.\n const MIN_ATTEMPT_MS = 5_000;\n const remaining = () => callDeadline - (Date.now() - t0) - 500; // 500ms margin for overhead after the last attempt\n const attemptOne = async (p: ProviderInstance): Promise => {\n const timeoutMs = Math.min(HARD_TIMEOUT_MS, remaining());\n if (timeoutMs < MIN_ATTEMPT_MS) {\n errors.push(`${p.name}: skipped — ${Math.max(0, timeoutMs)}ms left`);\n pushTrail(trail, { provider: p.name, vendor: p.vendor, model: modelFor(p), pass: currentPass, timeoutMs: Math.max(0, timeoutMs), ok: false, skipped: true, demoted: demotedSet.has(p) || undefined, error: \"out of time\", ts: Date.now() });\n return null;\n }\n const model = modelFor(p);\n const demoted = demotedSet.has(p) || undefined;\n triedNames.add(p.name);\n try {\n const hit = await attempt(p, model, messages, o, timeoutMs);\n await report(p.name, true);\n pushTrail(trail, { provider: p.name, vendor: p.vendor, model, pass: currentPass, timeoutMs, ok: true, demoted, latencyMs: hit.latencyMs, ts: Date.now() });\n return { ...hit, plan, trail };\n } catch (e: any) {\n await report(p.name, false, e.message, !!e.retryable, !!e.configClass);\n errors.push(`${p.name}: ${e.message}`);\n pushTrail(trail, { provider: p.name, vendor: p.vendor, model, pass: currentPass, timeoutMs, ok: false, demoted, error: String(e.message).slice(0, 200), ts: Date.now() });\n console.warn(`[SEVERED] ${p.name} (${e.message})`);\n return null;\n }\n };\n // Never declare exhaustion after a handful: walk the ENTIRE ordered target list every pass (skipping\n // only breaker-open providers, restored below via fail-open if a pass skips everyone), and if the\n // whole list comes up empty with wall-clock room left, run it again — up to MAX_FULL_PASSES — instead\n // of giving up after one lap. A later pass can also pick up a provider whose 25s breaker cooldown\n // (BREAK_DURATION_MS) has since elapsed. In practice, MAX_FULL_PASSES is an upper bound the wall\n // clock rarely lets this reach — walking ~20-30 providers at HARD_TIMEOUT_MS each already consumes\n // the whole SAFETY_CEILING_MS in one pass if most of them stall. The rest of \"at least 10 tries\" is\n // deliberately NOT this function's job: main-script.ts's send loop re-invokes the whole turn (a\n // fresh 60s window each time, so not bounded by this invocation's wall clock) on an EXHAUSTED signal,\n // now guaranteed to happen at least 10 times before it gives up — see its comment for why splitting\n // the guarantee across invocations is the only way to honor it without Val Town killing the function\n // mid-response.\n const MAX_FULL_PASSES = 10;\n for (let pass = 1; pass <= MAX_FULL_PASSES; pass++) {\n currentPass = pass;\n const skipped: ProviderInstance[] = [];\n let attemptedThisPass = 0;\n for (const p of targets) {\n if (remaining() < MIN_ATTEMPT_MS) break; // wall clock is the only thing allowed to cut a pass short\n if (!admit(p.name)) {\n skipped.push(p);\n const b = getBreaker(p.name);\n pushTrail(trail, { provider: p.name, vendor: p.vendor, model: modelFor(p), pass, timeoutMs: 0, ok: false, skipped: true, demoted: demotedSet.has(p) || undefined, error: `circuit open — ${Math.max(0, b.nextAttempt - Date.now())}ms cooldown left`, ts: Date.now() });\n continue;\n }\n attemptedThisPass++;\n const hit = await attemptOne(p);\n if (hit) return hit;\n }\n // Fail-open: a breaker-open provider is skipped for SPEED when a healthier one might work instead,\n // never as a verdict that it's still down. If this pass skipped everyone, nothing was actually\n // attempted — that's not exhaustion, that's a stale verdict. Force one real try through the skipped\n // set before moving on, so \"all providers exhausted\" always means all providers were actually tried.\n if (attemptedThisPass === 0 && skipped.length) {\n for (const p of skipped) {\n if (remaining() < MIN_ATTEMPT_MS) break;\n const hit = await attemptOne(p);\n if (hit) return hit;\n }\n }\n if (remaining() < MIN_ATTEMPT_MS) break; // no room for another full pass\n }\n // Bug this closes: the message unconditionally said \"ALL_PROVIDERS_EXHAUSTED\" whether every configured\n // provider had genuinely been tried, or the wall-clock ceiling (SAFETY_CEILING_MS) cut a pass short\n // after only a handful — the two are completely different situations (one means \"come back later, this\n // key/model needs to recover\"; the other means \"we simply ran out of time this invocation, most of the\n // roster was never reached\") and the text gave no way to tell them apart. A live session mistook a\n // 6-of-N partial pass for a genuine full sweep because the message claimed \"ALL\" either way. `tried`\n // (distinct provider names actually attempted, this call, across every pass) vs `plan.length` (the full\n // ordered target list) makes the distinction explicit and machine-readable, not just prose.\n const triedCount = triedNames.size, totalCount = plan.length;\n const scope = triedCount >= totalCount ? `tried all ${totalCount}` : `ran out of time — tried ${triedCount}/${totalCount}, never reached: ${targets.filter((p) => !triedNames.has(p.name)).map((p) => p.name).join(\", \") || \"(none)\"}`;\n throw Object.assign(new Error(`ALL_PROVIDERS_EXHAUSTED (${scope})\\n${errors.join(\"\\n\")}`), { status: 503, errors, plan, trail, tried: triedCount, total: totalCount });\n } finally { release(); }\n}\n","deleted",false,"label",[14],"1a430fbe-418c-11f1-af64-42b51c65c3df",["D",1777220541701],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Free ai chat with inference router.","imageUrl","userId","2c6d46b0-54dd-11f0-a86c-6fbc2b29331a","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","850db142-aa5e-4914-bceb-1e2e085ae4c5","tursoName","1a430fbe418c11f1af6442b51c65c3df","tursoHostname","1a430fbe418c11f1af6442b51c65c3df-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789508444194],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_9":10,"_104":100},"org",{"_9":10,"_98":-5,"_99":100,"_55":56,"_101":102,"_103":42},"branch",{"_35":88,"_11":89,"_22":28,"_90":45,"_91":-5,"_92":-5,"_93":-5,"_94":-5,"_95":-5,"_96":12,"_97":10},"isOwned",1682,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18yekJHRDE5SGVoUHUzUENQaXY2MTRkT3h0cncifQ","tier","free","canRestrictVals","profileImageUrl","ctx",{"_81":-5,"_138":139,"_140":141,"_142":143,"_144":42,"_145":42,"_146":42,"_147":32,"_148":-5,"_149":-7,"_150":-7,"_151":62,"_152":42,"_153":42,"_154":-5,"_155":42,"_156":42,"_157":42,"_158":-5},"ENV",{"_109":110,"_111":112,"_113":89,"_114":115,"_116":117,"_118":119,"_120":121,"_122":123,"_124":125,"_126":127,"_128":129,"_130":131,"_132":133,"_134":135,"_136":137},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town","orgs",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]