`, favicon, and body of `Root()` to\nmatch your app's existing `index.html`, and stamp anything else your shell\nreferences directly (images, css) with `immutableFileUrl()`.\n\n1. Create `frontend/root.tsx`:\n\n```tsx\n/** @jsxImportSource npm:hono@4/jsx */\nimport { raw } from \"npm:hono@4/html\";\nimport { immutableFileUrl } from \"https://esm.town/v/std/utils/index.ts\";\n\n// The HTML shell for the app. It is never cached, so immutableFileUrl can stamp\n// the current val version into each asset URL — the browser caches those\n// immutably, and publishing the val changes the URLs, refetching exactly once.\nexport function Root() {\n return (\n <>\n {raw(\"<!DOCTYPE html>\")}\n <html lang=\"en\">\n <head>\n <meta charSet=\"UTF-8\" />\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\" />\n <title>React Hono Val Town Starter

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_261":262,"_263":264},"routes/_app.x.$handle.$name",{"_99":98,"_31":44,"_107":89,"_26":108,"_13":14,"_109":-5,"_11":110,"_111":112,"_113":114,"_115":116,"_117":118,"_20":-5,"_119":-5,"_120":-5,"_121":24,"_122":123,"_124":58,"_125":-5,"_126":-5,"_127":128},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_42":43,"_31":44,"_37":45,"_46":-5,"_47":24,"_48":49,"_50":49,"_51":52,"_53":-5,"_18":54,"_55":56,"_57":58,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":66,"_67":68,"_69":70,"_71":-5,"_72":73,"_74":73,"_75":76,"_77":76,"_78":76,"_79":73,"_80":73,"_81":82,"_83":84,"_85":86,"_87":24},"version",40,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":-7,"_23":24,"_25":24,"_26":27,"_28":27},"file",{"_29":30,"_31":32,"_33":34,"_35":36,"_37":38,"_39":14,"_40":41,"_23":24},"previous","deleted",false,"deletedHere","path","frontend/beacon.ts","latestPath","fileId","76111a38-a3da-11f1-8dc5-1607ee4eb77e","name","beacon.ts","fileType","script","content","export const BEACON_ROWS = 6;\nexport const BEACON_COLS = 8;\nexport const BEACON_DATA_BITS = 32;\n\nconst HEADER = [\n [true,true,true,true,true,true,true,true],\n [true,false,true,false,true,false,true,false],\n];\n\nexport function encodeRevision(revision:number):boolean[][] {\n const value = revision >>> 0;\n const data:boolean[][] = [];\n for(let row=0;row<4;row++){\n const cells:boolean[]=[];\n for(let col=0;col<8;col++){\n const bit=31-(row*8+col);\n cells.push(Boolean((value>>>bit)&1));\n }\n data.push(cells);\n }\n return [...HEADER.map(row=>[...row]),...data];\n}\n\nexport function decodeRevision(grid:boolean[][]):number {\n if(grid.length!==BEACON_ROWS||grid.some(row=>row.length!==BEACON_COLS)) throw new Error(\"Expected a 6x8 beacon grid\");\n for(let row=0;row<2;row++)for(let col=0;col<8;col++)if(grid[row][col]!==HEADER[row][col])throw new Error(\"Invalid beacon orientation header\");\n let value=0;\n for(let row=2;row<6;row++)for(let col=0;col<8;col++)value=(value*2)+(grid[row][col]?1:0);\n return value>>>0;\n}\n\nexport function verifyBeaconRoundTrip(values:number[]=[0,1,1842,65535,4294967295]):boolean {\n return values.every(value=>decodeRevision(encodeRevision(value))===(value>>>0));\n}\n","createdAt",["D",1788029410390],"branchVersion","branchId","ec5fed3c-97c0-4273-93d2-1fc6dfd46413","id","546ce605-1508-4081-a1fc-e50474041c15","roverwatch-vision-lab",["D",1787937077281],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Deterministic computer vision test harness","imageUrl","40e69d1a-3254-4858-920b-7b1449761575","pinnedFileId","01a0495a-b924-761c-aebd-32882f84364c","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","54bae3d1-8223-41e5-b22d-81eb28a263c7","tursoName","546ce60515084081a1fce50474041c15","tursoHostname","546ce60515084081a1fce50474041c15-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789501263826],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_99":98,"_106":102},"org",{"_99":98,"_100":-5,"_101":102,"_18":54,"_103":104,"_105":24},"branch",{"_13":88,"_31":89,"_42":41,"_90":43,"_91":-5,"_92":-5,"_93":-5,"_94":-5,"_95":-5,"_96":44,"_97":98},"isOwned",76,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","pchinjr","handle","displayName","avatar","https://images.clerk.dev/oauth_github/img_2OaSpooJk6yZ5KFqyjtIVA4z3SH.jpeg","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_42":43,"_31":44,"_37":256,"_46":-5,"_47":24,"_48":49,"_50":49,"_51":52,"_53":-5,"_18":54,"_55":56,"_57":58,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":66,"_67":68,"_69":257,"_71":-5,"_72":73,"_74":73,"_75":76,"_77":76,"_78":76,"_79":73,"_80":73,"_81":258,"_83":259,"_85":260,"_87":24},"tree",[175,184,185,186],"files",[175,176,177,178,179,180,181,182,183,184,185,186],"counts",{"_165":62,"_166":155,"_167":62,"_168":169,"_170":171},"contents",[129,130,131,132],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_42":159,"_31":160,"_29":161,"_135":-5,"_40":41,"_39":155,"_137":58,"_37":162,"_13":62,"_33":163,"_35":-5,"_23":24,"_142":-5,"_26":164},{"_42":152,"_31":153,"_29":154,"_135":-5,"_40":41,"_39":155,"_137":58,"_37":156,"_13":62,"_33":20,"_35":157,"_23":24,"_142":-5,"_26":158},{"_42":144,"_31":145,"_29":146,"_135":-5,"_40":41,"_39":147,"_137":24,"_37":148,"_13":149,"_33":20,"_35":150,"_23":24,"_142":-5,"_26":151},{"_42":133,"_31":134,"_29":56,"_135":-5,"_40":41,"_39":136,"_137":24,"_37":138,"_13":139,"_33":140,"_35":141,"_23":24,"_142":44,"_26":143},"7e95111c-90fa-48ba-b2cc-335160cd7476","index.ts","parentId",12,"latest",["D",1787949066635],3,"http","import { parseVal, serveImmutableFile } from \"https://esm.town/v/std/utils/index.ts\";\nimport { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport { Hono } from \"npm:hono\";\nimport { Root } from \"./frontend/root.tsx\";\nimport { defaultDifficulty, presets } from \"./frontend/scenarios.ts\";\n\nconst app = new Hono();\n\nawait sqlite.execute(\"CREATE TABLE IF NOT EXISTS vision_lab_state (id INTEGER PRIMARY KEY CHECK (id = 1), payload TEXT NOT NULL, revision INTEGER NOT NULL DEFAULT 1, updated_at TEXT NOT NULL)\");\nawait sqlite.execute(\"CREATE TABLE IF NOT EXISTS vision_lab_transitions (revision INTEGER PRIMARY KEY, scenario_id TEXT NOT NULL, step INTEGER NOT NULL, scheduled_at TEXT NOT NULL, published_at TEXT NOT NULL, expected_json TEXT NOT NULL, state_json TEXT NOT NULL)\");\nawait sqlite.execute(\"CREATE INDEX IF NOT EXISTS idx_vision_lab_transitions_published_at ON vision_lab_transitions(published_at)\");\nawait sqlite.execute(\"CREATE TABLE IF NOT EXISTS vision_lab_observations (id INTEGER PRIMARY KEY AUTOINCREMENT, revision INTEGER NOT NULL, observer_id TEXT NOT NULL, observed_at TEXT NOT NULL, received_at TEXT NOT NULL)\");\n\nconst now = new Date().toISOString();\nconst initial = {\n scenario: presets[0],\n difficulty: defaultDifficulty,\n step: 0,\n scheduledAt: now,\n publishedAt: now,\n revision: 1,\n};\nawait sqlite.execute({\n sql: \"INSERT OR IGNORE INTO vision_lab_state (id,payload,revision,updated_at) VALUES (1,?,?,?)\",\n args: [JSON.stringify(initial), 1, now],\n});\n\nasync function readState() {\n const result = await sqlite.execute(\"SELECT payload, revision, updated_at FROM vision_lab_state WHERE id = 1\");\n const row = result.rows[0] as Record;\n const state = JSON.parse(String(row.payload));\n return {\n ...state,\n scheduledAt: state.scheduledAt || state.startedAt || row.updated_at,\n publishedAt: state.publishedAt || row.updated_at,\n revision: Number(row.revision),\n updatedAt: row.updated_at,\n };\n}\n\nconst bootState = await readState();\nawait sqlite.execute({\n sql: \"INSERT OR IGNORE INTO vision_lab_transitions (revision,scenario_id,step,scheduled_at,published_at,expected_json,state_json) VALUES (?,?,?,?,?,?,?)\",\n args: [\n Number(bootState.revision),\n String(bootState.scenario.id),\n Number(bootState.step || 0),\n String(bootState.scheduledAt),\n String(bootState.publishedAt),\n JSON.stringify(bootState.scenario.expectedResult),\n JSON.stringify(bootState),\n ],\n});\n\nfunction transitionFromRow(row: Record) {\n return {\n revision: Number(row.revision),\n scenarioId: String(row.scenario_id),\n step: Number(row.step),\n scheduledAt: String(row.scheduled_at),\n publishedAt: String(row.published_at),\n expected: JSON.parse(String(row.expected_json)),\n state: JSON.parse(String(row.state_json)),\n };\n}\n\napp.get(\"/\", (c) => c.html(Root()));\napp.get(\"/test\", (c) => c.html(Root()));\napp.get(\"/__immutable/*\", (c) => serveImmutableFile(c.req.path));\napp.get(\"/source\", (c) => c.redirect(parseVal().links.self.val));\n\napp.get(\"/api/state\", async (c) => c.json(await readState()));\napp.get(\"/current.json\", async (c) => c.json(await readState()));\n\napp.get(\"/api/history\", async (c) => {\n const sinceRevision = Math.max(0, Number(c.req.query(\"sinceRevision\") || 0));\n const limit = Math.min(1000, Math.max(1, Number(c.req.query(\"limit\") || 200)));\n const result = await sqlite.execute({\n sql: \"SELECT revision,scenario_id,step,scheduled_at,published_at,expected_json,state_json FROM vision_lab_transitions WHERE revision > ? ORDER BY revision ASC LIMIT ?\",\n args: [sinceRevision, limit],\n });\n const transitions = result.rows.map((row) => transitionFromRow(row as Record));\n return c.json({ groundTruthOnly: true, sinceRevision, count: transitions.length, transitions });\n});\n\napp.get(\"/api/state-at\", async (c) => {\n const at = c.req.query(\"at\");\n if (!at || Number.isNaN(Date.parse(at))) return c.json({ error: \"Valid ISO-8601 at query parameter required\" }, 400);\n const result = await sqlite.execute({\n sql: \"SELECT revision,scenario_id,step,scheduled_at,published_at,expected_json,state_json FROM vision_lab_transitions WHERE published_at <= ? ORDER BY published_at DESC, revision DESC LIMIT 1\",\n args: [new Date(at).toISOString()],\n });\n if (!result.rows.length) return c.json({ error: \"No ground-truth state had been published at that time\" }, 404);\n return c.json({ groundTruthOnly: true, capturedAt: new Date(at).toISOString(), transition: transitionFromRow(result.rows[0] as Record) });\n});\n\napp.get(\"/api/observations\", async (c) => {\n const revision = Math.max(0, Number(c.req.query(\"revision\") || 0));\n const limit = Math.min(1000, Math.max(1, Number(c.req.query(\"limit\") || 200)));\n const result = await sqlite.execute({\n sql: \"SELECT id,revision,observer_id,observed_at,received_at FROM vision_lab_observations WHERE revision >= ? ORDER BY id DESC LIMIT ?\",\n args: [revision, limit],\n });\n return c.json({ groundTruth: false, count: result.rows.length, observations: result.rows });\n});\n\napp.post(\"/api/state\", async (c) => {\n const body = await c.req.json();\n if (!body?.scenario?.id || !body?.scenario?.visualState || !body?.scenario?.expectedResult) {\n return c.json({ error: \"Invalid scenario payload\" }, 400);\n }\n const current = await readState();\n const publishedAt = new Date().toISOString();\n const scheduledAt = body.scheduledAt && !Number.isNaN(Date.parse(body.scheduledAt))\n ? new Date(body.scheduledAt).toISOString()\n : publishedAt;\n const next = {\n scenario: body.scenario,\n difficulty: body.difficulty || defaultDifficulty,\n step: Number(body.step || 0),\n scheduledAt,\n publishedAt,\n revision: Number(current.revision) + 1,\n };\n const encoded = JSON.stringify(next);\n if (encoded.length > 100_000) return c.json({ error: \"Payload too large\" }, 413);\n await sqlite.batch([\n {\n sql: \"INSERT INTO vision_lab_transitions (revision,scenario_id,step,scheduled_at,published_at,expected_json,state_json) VALUES (?,?,?,?,?,?,?)\",\n args: [next.revision, next.scenario.id, next.step, scheduledAt, publishedAt, JSON.stringify(next.scenario.expectedResult), encoded],\n },\n {\n sql: \"UPDATE vision_lab_state SET payload=?, revision=?, updated_at=? WHERE id=1\",\n args: [encoded, next.revision, publishedAt],\n },\n ]);\n return c.json(next);\n});\n\napp.post(\"/api/observations\", async (c) => {\n const body = await c.req.json();\n if (!Number.isInteger(body?.revision) || !body?.observerId || !body?.observedAt || Number.isNaN(Date.parse(body.observedAt))) {\n return c.json({ error: \"revision, observerId, and valid observedAt are required\" }, 400);\n }\n const receivedAt = new Date().toISOString();\n await sqlite.execute({\n sql: \"INSERT INTO vision_lab_observations (revision,observer_id,observed_at,received_at) VALUES (?,?,?,?)\",\n args: [body.revision, String(body.observerId).slice(0, 100), new Date(body.observedAt).toISOString(), receivedAt],\n });\n return c.json({ accepted: true, groundTruth: false, revision: body.revision, receivedAt }, 201);\n});\n\napp.onError((err) => Promise.reject(err));\nexport default app.fetch;","label",[134],"081ba9e1-e175-4a89-827a-a719eb67fabe","README.md","01a0495a-b924-761c-aebd-2ee61846be65",39,["D",1787949411135],21,"# Roverwatch Vision Lab\n\nRoverwatch Vision Lab is a standalone, deterministic visual test harness for developing and evaluating Roverwatch's screenshot-understanding pipeline without loading Overwatch or depending on live gameplay.\n\n- Control Mode: https://roverwatch-vision-lab.val.run/\n- Test Screen Mode: https://roverwatch-vision-lab.val.run/test\n- Current machine-readable state: https://roverwatch-vision-lab.val.run/current.json\n- Val source: https://www.val.town/x/pchinjr/roverwatch-vision-lab/code/\n\nVision Lab is intentionally separate from the main Roverwatch repository. It produces controlled visual stimuli and their expected answers. Roverwatch captures those visuals from Twitch, performs inference, and reports what it detected.\n\n## Why this exists\n\nTesting computer vision against a live game makes failures difficult to reproduce:\n\n- the game state changes constantly;\n- screenshots do not have explicit ground truth;\n- a missed detection may be caused by capture, compression, timing, rendering, or inference;\n- reproducing the exact same visual state is difficult;\n- testing requires a running game and a known game situation.\n\nVision Lab replaces those uncontrolled inputs with stable scenario IDs, known visuals, and explicit expected results. It lets the Roverwatch development process answer smaller questions in order:\n\n1. Can Roverwatch capture the stream?\n2. Can it retrieve and decode the screenshot?\n3. Can it identify a simple shape and color?\n4. Can it associate the inference with the correct active scenario?\n5. Can it write the expected message to Twitch chat?\n6. Can it handle text, numbers, HUD regions, motion, occlusion, noise, and temporal changes?\n7. Can repeated runs be compared objectively?\n\nThe initial end-to-end proof is deliberately simple:\n\n~~~text\nVision Lab displays a green circle\n ↓\nOBS sends the Test Screen to Twitch\n ↓\nRoverwatch captures a frame\n ↓\nThe vision model reports \"green circle\"\n ↓\nRoverwatch reads Vision Lab ground truth\n ↓\nThe result is marked correct or incorrect\n ↓\nRoverwatch writes the result to Twitch chat\n~~~\n\n## Responsibilities and system boundary\n\n### Vision Lab owns\n\n- deterministic scenario definitions;\n- rendering visual test stimuli;\n- a clean OBS-compatible Test Screen;\n- the currently active scenario state;\n- structured expected results;\n- stable scenario and sequence-step IDs;\n- difficulty transformations such as blur, scale, noise, and occlusion;\n- local operator history;\n- authoritative server-side ground-truth history and JSON export;\n- capture-time ground-truth resolution;\n- non-authoritative Test Screen observation records;\n- explicit scheduled, published, observed, and received timestamps.\n\n### Roverwatch owns\n\n- Twitch stream acquisition;\n- frame capture and S3 screenshot storage;\n- screenshot selection and timestamps;\n- model invocation;\n- inference parsing and normalization;\n- Twitch chat output;\n- comparison of actual inference with expected Vision Lab results;\n- evaluation metrics, latency tracking, and operational logs.\n\n### Vision Lab does not currently own\n\n- direct calls into Roverwatch;\n- Twitch credentials or chat authentication;\n- reading screenshots from S3;\n- model inference;\n- automatic scoring;\n- confusion matrices;\n- labeled-image capture;\n- production Overwatch UI artwork.\n\nKeeping this boundary explicit prevents the test harness from becoming coupled to one model, one capture implementation, or one Roverwatch deployment.\n\n## Current capabilities\n\nThe current application supports:\n\n- separate Control and Test Screen modes;\n- cross-browser synchronization between Chrome and OBS;\n- shape and color scenarios;\n- large and small OCR text;\n- numeric percentages;\n- generic game HUD states;\n- multiple simultaneous HUD values;\n- scale, position, contrast, brightness, blur, opacity, noise, background, and occlusion controls;\n- basic motion modes;\n- a timed blank → green circle → blank → red square → blank → blue triangle sequence;\n- seeded random scenario selection;\n- structured ground-truth display and copying;\n- scenario JSON export;\n- local test-session recording and export;\n- keyboard shortcuts;\n- an HTTP endpoint for the currently active state;\n- append-only server-side transition history;\n- capture-time state lookup through `/api/state-at`;\n- separate Test Screen observation history;\n- ground-truth-only server exports.\n\n## Quick start\n\n### Manual browser test\n\n1. Open Control Mode.\n2. Click Open Test Screen.\n3. Place the two windows side by side.\n4. Select Green Circle in Control Mode.\n5. Confirm that the Test Screen shows only a green circle.\n6. Confirm that the Expected Result panel reports shape circle and color green.\n7. Select Red Square and Blue Triangle and confirm that the previous scenario's difficulty settings do not leak into the new one.\n8. Run the baseline timeline and confirm that blank states appear between shapes.\n\n### OBS Browser Source\n\n1. Add a Browser Source in OBS.\n2. Use https://roverwatch-vision-lab.val.run/test as its URL.\n3. Set width to 1920 and height to 1080.\n4. Leave custom CSS empty.\n5. Disable Shutdown source when not visible if sequences must keep running while the source is hidden.\n6. Keep Refresh browser when scene becomes active disabled unless a fresh load is desired.\n7. Change scenarios from Control Mode.\n8. Confirm that the OBS source updates within approximately 500 ms.\n\nThe Test Screen has no controls, requires no mouse input, hides the cursor, fills the viewport, and polls shared state while it is open.\n\n## Architecture\n\n~~~text\nfrontend/scenarios.ts\n Scenario types, defaults, presets, baseline sequence\n ↓\nfrontend/components/App.tsx\n Control Mode, Test Screen, renderer, timeline, session recorder\n ↓\nPOST /api/state\n Atomically appends a ground-truth transition and updates current state\n ↓\nVal-scoped SQLite\n Current state + immutable transitions + separate observations\n ↓\nGET /api/state, /current.json, /api/history, /api/state-at\n ↓\nTest Screen polls every 500 ms, renders the revision, then records observedAt\n~~~\n\n### File layout\n\n| File | Responsibility |\n| --- | --- |\n| index.ts | Hono HTTP entrypoint, routes, SQLite initialization, shared-state reads and writes |\n| frontend/root.tsx | HTML shell, metadata, Twind loader, immutable frontend entry URL |\n| frontend/index.tsx | Mounts the React application |\n| frontend/components/App.tsx | Control Mode, Test Screen, renderers, timeline, random mode, exports |\n| frontend/scenarios.ts | Scenario contracts, presets, default difficulty, baseline sequence |\n| frontend/favicon.svg | Application icon |\n| README.md | Architecture, operations, extension, and Roverwatch integration contract |\n\n### Technology choices\n\n- Val Town hosts the application and HTTP routes.\n- Hono handles routing.\n- React 18.2 renders both modes.\n- Twind provides utility styling without a build process.\n- Val-scoped SQLite stores shared display state.\n- Browser localStorage stores control-only history and seed data.\n- Immutable versioned frontend URLs prevent stale JavaScript after a deployment.\n- Polling is used because Val Town does not accept incoming WebSocket connections.\n\n## Two kinds of state\n\nUnderstanding this distinction is essential for maintenance.\n\n### Shared live state\n\nThe current screen state is stored in the val-scoped SQLite table vision_lab_state. There is exactly one row, with ID 1.\n\n| Column | Meaning |\n| --- | --- |\n| id | Singleton row ID |\n| payload | JSON-encoded active LiveState |\n| revision | Monotonically increasing state revision |\n| updated_at | Server timestamp of the latest update |\n\nShared state exists because Chrome and OBS use separate browser profiles. They cannot reliably share localStorage or BroadcastChannel state. Control Mode therefore publishes to the server, and Test Screen Mode polls the server.\n\nEvery accepted state update is also appended to vision_lab_transitions in the same SQLite batch as the current-state update. A transition can never be edited through the application API. It contains revision, scenario ID, step, scheduled time, server publication time, expected JSON, and the resolved state JSON.\n\nTest Screen observations are written to vision_lab_observations. They are explicitly marked non-ground-truth and never change a transition. Each observation records revision, observer ID, client observation time, and server receipt time.\n\nThe current shared state is intentionally last-write-wins. Opening multiple Control Mode windows means whichever window publishes last controls every connected Test Screen. Historical transitions remain append-only.\n\n### Browser-local state\n\nThe key roverwatch-vision-lab.v1 stores:\n\n- the current random-test seed;\n- recent session history.\n\nThis data is local to the browser profile. It is not visible to OBS and is not intended as authoritative evaluation data until exported.\n\n## Data contracts\n\n### Scenario\n\nA scenario separates what is rendered from what Roverwatch is expected to detect.\n\n~~~ts\ninterface Scenario {\n id: string;\n name: string;\n description: string;\n category: \"shape\" | \"ocr\" | \"hud\" | \"sequence\";\n visualState: {\n kind: \"blank\" | \"shape\" | \"text\" | \"number\" | \"hud\";\n // Renderer-specific fields\n };\n expectedResult: Record;\n difficulty?: Partial;\n}\n~~~\n\nExample:\n\n~~~json\n{\n \"id\": \"shape.green-circle\",\n \"name\": \"Green Circle\",\n \"description\": \"Green Circle recognition baseline\",\n \"category\": \"shape\",\n \"visualState\": {\n \"kind\": \"shape\",\n \"shape\": \"circle\",\n \"color\": \"green\"\n },\n \"expectedResult\": {\n \"scenarioType\": \"shape\",\n \"visible\": true,\n \"shape\": \"circle\",\n \"color\": \"green\"\n }\n}\n~~~\n\n### LiveState\n\nA LiveState is the resolved scenario being shown now.\n\n~~~ts\ninterface LiveState {\n scenario: Scenario;\n difficulty: Difficulty;\n step: number;\n scheduledAt: string;\n publishedAt?: string;\n observedAt?: string;\n revision?: number;\n}\n~~~\n\nThe server assigns publishedAt and the authoritative revision. A Test Screen adds its own observedAt only after it receives that revision. The current-state response also includes updatedAt for compatibility.\n\n### Timestamp semantics\n\n| Timestamp | Clock owner | Meaning | Authoritative use |\n| --- | --- | --- | --- |\n| scheduledAt | Control browser | When the controller intended the state to begin | Sequence intent and controller delay |\n| publishedAt | Vision Lab server | When the ground-truth transition and current state committed atomically | Authoritative state timeline |\n| observedAt | Individual Test Screen | When that client first received the revision | Display-observation latency only |\n| receivedAt | Vision Lab server | When an observation record reached the server | Observation transport diagnostics |\n| Roverwatch capturedAt | Roverwatch capture process | When the evaluated frame was acquired | Lookup key for /api/state-at |\n| Roverwatch inference timestamps | Roverwatch | When inference started and completed | Model and pipeline responsiveness |\n\nThese clocks may be skewed. Use publishedAt to define the authoritative ground-truth timeline. Use capturedAt to resolve a frame against that timeline. Treat observedAt as evidence that a particular Test Screen received a revision, not proof that Twitch had already encoded and delivered it.\n\n### Ground truth versus live decisions\n\nThe transitions table and ground-truth exports contain only scenario intent, resolved visual state, expected result, revision, and authoritative timing. They never contain model output, Twitch chat output, confidence, match status, or pass/fail decisions.\n\nObservations are stored in a separate table and are labeled non-ground-truth. Roverwatch results must remain in Roverwatch storage. Evaluation is a later join:\n\n~~~text\nVision Lab immutable transition\n +\nRoverwatch frame and inference record\n ↓\nComparison record and responsiveness metrics\n~~~\n\nThis separation makes it possible to change scoring logic without rewriting the historical expected record.\n\n\n### Difficulty\n\n~~~ts\ninterface Difficulty {\n scale: number;\n position: \"center\" | \"top-left\" | \"top-right\" |\n \"bottom-left\" | \"bottom-right\" | \"random\";\n contrast: \"high\" | \"medium\" | \"low\";\n brightness: number;\n blur: number;\n opacity: number;\n noise: number;\n background: \"solid\" | \"gradient\" | \"simple\" | \"busy\";\n occlusion: number;\n motion: \"stationary\" | \"horizontal\" | \"vertical\" |\n \"diagonal\" | \"bounce\" | \"random-walk\";\n speed: number;\n}\n~~~\n\n### Ground truth\n\nThe Control Mode ground-truth panel wraps the scenario's expected result with execution context:\n\n~~~json\n{\n \"scenarioId\": \"shape.green-circle\",\n \"scenarioName\": \"Green Circle\",\n \"step\": 0,\n \"revision\": 217,\n \"scheduledAt\": \"2026-08-28T20:33:01.614Z\",\n \"publishedAt\": \"2026-08-28T20:33:07.480Z\",\n \"expected\": {\n \"scenarioType\": \"shape\",\n \"visible\": true,\n \"shape\": \"circle\",\n \"color\": \"green\"\n }\n}\n~~~\n\nGround truth comes from scenario.expectedResult. It is not inferred from the rendered pixels. That separation is what makes the harness useful for evaluation.\n\n## HTTP API\n\nThe application is public and currently uses these routes:\n\n| Method | Route | Purpose |\n| --- | --- | --- |\n| GET | / | Control Mode |\n| GET | /test | Clean Test Screen |\n| GET | /api/state | Current shared LiveState |\n| POST | /api/state | Replace current shared LiveState |\n| GET | /current.json | Public machine-readable alias for current state |\n| GET | /api/history | Append-only ground-truth transitions after an optional revision |\n| GET | /api/state-at?at=... | Ground truth that had been published at a capture timestamp |\n| GET | /api/observations | Non-ground-truth Test Screen observations |\n| POST | /api/observations | Append a Test Screen observation without changing ground truth |\n| GET | /source | Redirect to editable Val source |\n| GET | /__immutable/* | Versioned frontend modules and assets |\n\n### Reading current ground truth\n\n~~~bash\ncurl https://roverwatch-vision-lab.val.run/current.json\n~~~\n\nThe main Roverwatch process should initially treat this endpoint as a debugging and correlation source, not as a production dependency.\n\n### State update validation\n\nPOST requests must contain:\n\n- scenario.id;\n- scenario.visualState;\n- scenario.expectedResult.\n\nPayloads above 100,000 characters are rejected. The server increments revision, stores the complete payload, and returns the saved state.\n\nPublishing a state and appending its transition occur atomically in one SQLite batch. The transition history endpoint is marked groundTruthOnly: true; the observations endpoint is marked groundTruth: false.\n\nFor capture-time correlation, Roverwatch should request:\n\n~~~text\nGET /api/state-at?at=\n~~~\n\nThe endpoint returns the most recent transition whose server-issued publishedAt is less than or equal to the capture time.\n\nAll endpoints are currently public. Do not place secrets, Twitch credentials, private screenshots, user data, or model credentials in scenario state.\n\n## How the renderer works\n\nScenarioStage reads two separate inputs:\n\n1. scenario.visualState selects the content renderer.\n2. difficulty transforms the rendered target and background.\n\nCurrent renderer branches are:\n\n| Kind | Renderer behavior |\n| --- | --- |\n| blank | Renders no target |\n| shape | Renders circle, square, or CSS triangle |\n| text | Renders exact text with size, alignment, and colors |\n| number | Uses the text renderer for numeric recognition |\n| hud | Renders health, ultimate, life state, team count, and enemy count |\n\nDo not derive expected results inside render components. Renderer code controls appearance; scenario definitions control expected meaning.\n\n## Scenario ID conventions\n\nStable IDs are required because Roverwatch logs and evaluation results will eventually join against them.\n\nRecommended format:\n\n~~~text\n.\n~~~\n\nExamples:\n\n~~~text\nshape.green-circle\nshape.red-square\nocr.large-text\nnumber.ult-99\nhud.player-dead\nhud.ult-ready\ndifficulty.occluded-target\nmotion.moving-target\nsequence.baseline.02-green\n~~~\n\nRules:\n\n- IDs should be lowercase and machine-readable.\n- Do not reuse an ID for a semantically different test.\n- Visual refinements that preserve the same intended detection may keep the ID.\n- Changed expected meaning should receive a new ID.\n- Sequence steps need stable IDs, not only numeric indexes.\n- For versioned evaluation sets, add an explicit version namespace later rather than silently changing the old set.\n\n## Preset and difficulty-state rules\n\nPreset difficulty overrides are scenario-local. Selecting a new preset or moving to a new sequence step must:\n\n1. begin with defaultDifficulty;\n2. merge only the new scenario's difficulty overrides;\n3. publish the resolved result.\n\nNever merge a newly selected preset onto the previous scenario's resolved difficulty. Doing that causes state leakage: a moving or occluded preset can make every later scenario move or remain occluded.\n\nManual difficulty controls intentionally modify the active scenario after selection. Selecting another preset resets those controls to defaults plus the new preset overrides.\n\nThis rule is enforced through:\n\n~~~ts\nmerged(nextScenario, defaultDifficulty)\n~~~\n\nTreat it as an invariant when changing scenario-selection, sequence, or random-mode code.\n\n## Baseline sequence\n\nThe initial sequence proves temporal capture and classification:\n\n| Step ID | Duration | Expected visual |\n| --- | ---: | --- |\n| sequence.baseline.01-blank | 8 seconds | Blank |\n| sequence.baseline.02-green | 15 seconds | Green circle |\n| sequence.baseline.03-blank | 8 seconds | Blank |\n| sequence.baseline.04-red | 15 seconds | Red square |\n| sequence.baseline.05-blank | 8 seconds | Blank |\n| sequence.baseline.06-blue | 15 seconds | Blue triangle |\n\nBlank intervals matter. They let Roverwatch demonstrate that it detects absence, does not repeat a stale inference, and can associate frames with state transitions.\n\n## Random mode and reproducibility\n\nRandom mode uses a deterministic seeded pseudo-random generator. Given:\n\n- the same seed;\n- the same preset ordering;\n- the same interval;\n- the same starting point;\n\nit should produce the same selection order.\n\nChanging the preset array order changes the generated scenario sequence even when the seed remains the same. Treat preset ordering as part of the reproducibility contract until random mode records the generated schedule explicitly.\n\n## Test sessions and exports\n\nStarting a session records the current revision. End + export then downloads authoritative transitions after that revision. Export ground truth downloads the available history independently of a session.\n\nThe server export has this shape:\n\n~~~json\n{\n \"groundTruthOnly\": true,\n \"sinceRevision\": 216,\n \"count\": 1,\n \"transitions\": [\n {\n \"revision\": 217,\n \"scenarioId\": \"shape.green-circle\",\n \"step\": 0,\n \"scheduledAt\": \"2026-08-28T20:33:01.614Z\",\n \"publishedAt\": \"2026-08-28T20:33:07.480Z\",\n \"expected\": {\n \"scenarioType\": \"shape\",\n \"visible\": true,\n \"shape\": \"circle\",\n \"color\": \"green\"\n },\n \"state\": {\n \"scenario\": {},\n \"difficulty\": {},\n \"step\": 0,\n \"scheduledAt\": \"2026-08-28T20:33:01.614Z\",\n \"publishedAt\": \"2026-08-28T20:33:07.480Z\",\n \"revision\": 217\n }\n }\n ]\n}\n~~~\n\nExports contain expected state only. Roverwatch inference, chat output, match status, and mismatch decisions must be stored in Roverwatch and joined afterward. This prevents model output from contaminating the source used to judge that output.\n\nBrowser-local operator history remains useful for immediate UI feedback, but responsiveness analysis should use server transitions, Test Screen observations, and Roverwatch capture/inference timestamps.\n\n## Recommended Roverwatch integration\n\n### Phase 1: manual correlation\n\nUse this first.\n\n1. Open Control Mode and Test Screen.\n2. Stream Test Screen through OBS to Twitch.\n3. Select one scenario at a time.\n4. Let Roverwatch capture a frame and run inference.\n5. Inspect Roverwatch logs and Vision Lab's Expected Result panel.\n6. Confirm the model output and Twitch chat message manually.\n\nSuccess criterion: Roverwatch consistently distinguishes blank, green circle, red square, and blue triangle.\n\n### Phase 2: log the Vision Lab state with each inference\n\nWhen a screenshot is selected for inference:\n\n1. Roverwatch records its frame capture timestamp as close to acquisition as possible.\n2. Roverwatch requests /api/state-at?at=<capturedAt>.\n3. Roverwatch records scenarioId, revision, scheduledAt, and publishedAt from the returned transition.\n4. Roverwatch records the frame capture timestamp and S3 object key.\n5. Roverwatch records normalized inference output and model latency.\n6. The combined record is written to Roverwatch logs.\n\nSuggested evaluation record:\n\n~~~ts\ninterface RoverwatchVisionEvaluation {\n frameId: string;\n s3Key: string;\n capturedAt: string;\n inferenceStartedAt: string;\n inferenceCompletedAt: string;\n model: string;\n visionLab: {\n scenarioId: string;\n revision: number;\n startedAt: string;\n updatedAt: string;\n expected: Record;\n };\n actual: Record;\n matched?: boolean;\n mismatchReasons?: string[];\n}\n~~~\n\n### Phase 3: guard against transition races\n\nA frame captured near a scenario change may show the previous state while /current.json already reports the next state.\n\nDo not compare a frame blindly against whatever state is current after inference finishes.\n\nRecommended approach:\n\n1. Read Vision Lab state close to frame capture, not after a slow model call.\n2. Resolve the frame through /api/state-at using the capture timestamp.\n3. Record revision, scheduledAt, and publishedAt.\n4. Use Test Screen observations to estimate controller-to-display delay without treating observations as ground truth.\n5. Add a transition-settling window before scoring a new state.\n6. Mark ambiguous transition-window frames as excluded rather than incorrect.\n\nThe baseline sequence now uses 8-second blank states and 15-second target states. Continue excluding frames near transitions until measured Twitch and capture latency supports a narrower settling window.\n\n### Phase 4: automated scoring\n\nAfter correlation is reliable, normalize expected and actual outputs into a shared result schema and calculate:\n\n- exact-match accuracy;\n- per-scenario accuracy;\n- false positives during blank states;\n- false negatives;\n- confusion pairs;\n- model latency;\n- end-to-end capture-to-chat latency.\n\nKeep raw model output alongside normalized output. Normalization bugs must be distinguishable from model errors.\n\n## Adding a preset with an existing renderer\n\n1. Open frontend/scenarios.ts.\n2. Add an object to presets.\n3. Assign a stable, unique ID.\n4. Define only visual fields used by the selected renderer kind.\n5. Write an explicit, machine-readable expected result.\n6. Add only scenario-specific difficulty overrides.\n7. Select the new preset in Control Mode.\n8. Verify the Test Screen.\n9. Verify /current.json.\n10. Select a normal preset afterward and confirm no difficulty state leaked.\n\nExample:\n\n~~~ts\n{\n id: \"ocr.player-dead\",\n name: \"Player Dead Text\",\n description: \"Large death-state OCR target\",\n category: \"ocr\",\n visualState: {\n kind: \"text\",\n text: \"PLAYER DEAD\",\n fontSize: 128,\n align: \"center\",\n foreground: \"#ffffff\",\n background: \"#070b12\"\n },\n expectedResult: {\n scenarioType: \"text\",\n text: \"PLAYER DEAD\"\n }\n}\n~~~\n\n## Adding a new scenario type\n\nA new scenario type changes both the contract and renderer.\n\n1. Add the new kind to ScenarioKind.\n2. Add its visual fields to the Scenario type, or refactor visualState into a discriminated union.\n3. Add a renderer component in App.tsx.\n4. Add one branch in ScenarioStage.\n5. Create at least one baseline preset.\n6. Define the expected-result schema.\n7. Confirm difficulty transformations still work.\n8. Confirm blank and later presets reset the new type correctly.\n9. Document the expected schema here.\n10. Coordinate with Roverwatch before automated comparison depends on the new schema.\n\nIf scenario types grow significantly, the next maintainability refactor should make visualState a discriminated union and move renderers into separate component files.\n\n## Adding or changing a sequence\n\nSequence steps live in milestoneSequence in frontend/scenarios.ts.\n\nEach step needs:\n\n- a stable step ID;\n- a duration in milliseconds;\n- a scenario definition;\n- an explicit expected result through that scenario.\n\nWhen changing durations, remember that Twitch latency, frame capture cadence, S3 upload delay, polling, and model latency all consume part of the visible window. A three-second state may yield very few usable frames in a delayed pipeline.\n\nFor automated evaluation, prefer longer states until measured end-to-end latency is known.\n\n## Verification checklist\n\nRun this checklist after changing state, scenarios, rendering, or sequences.\n\n### Application\n\n- Control Mode returns HTTP 200.\n- Test Screen returns HTTP 200.\n- Frontend modules return JavaScript, not HTML.\n- /current.json returns valid JSON.\n- /api/history returns append-only ground-truth transitions.\n- /api/state-at resolves the expected revision for a known timestamp.\n- /api/observations remains separate and marked non-ground-truth.\n- The Test Screen has no visible controls.\n- The OBS source updates after Control Mode changes.\n\n### Scenario behavior\n\n- Green Circle renders green and circular.\n- Red Square renders red and square.\n- Blue Triangle renders blue and triangular.\n- Expected Result matches the rendered scenario.\n- Stable IDs appear in /current.json.\n- Blank renders no target.\n- Timeline steps occur in the documented order.\n\n### State isolation regression\n\n- Select Moving Target.\n- Select Green Circle.\n- Confirm motion is stationary.\n- Select Partially Occluded Target.\n- Select Red Square.\n- Confirm occlusion is 0%.\n- Select Busy HUD.\n- Select Blue Triangle.\n- Confirm background is solid and noise is 0.\n- Refresh both Control and Test Screen.\n- Confirm the current state remains internally consistent.\n\n### OBS and Roverwatch\n\n- OBS uses 1920×1080.\n- The source continues while unfocused.\n- Twitch receives the screen.\n- Roverwatch captures the intended frame.\n- Roverwatch logs S3 key, capture time, scenario ID, and revision.\n- Twitch chat output matches the expected result.\n\n## Troubleshooting\n\n### A previous effect remains active\n\nCause: the new scenario was merged onto the previous scenario's resolved difficulty.\n\nFix: make preset, sequence, and random selections resolve from defaultDifficulty, then merge only the new scenario override.\n\n### Refresh preserves an unexpected state\n\nRefresh reads the shared SQLite state. This is intentional so OBS and Control Mode stay synchronized. Select a clean preset or Reset difficulty. If the stored payload is internally inconsistent, inspect the singleton vision_lab_state row.\n\n### Control Mode says offline\n\nCheck:\n\n- GET /api/state;\n- Val Town logs;\n- SQLite availability;\n- browser network errors;\n- whether the val's HTTP access is still public.\n\n### Test Screen does not update\n\n- Confirm Control Mode says synced.\n- Confirm /current.json changes.\n- Wait at least 500 ms.\n- Refresh the OBS Browser Source.\n- Confirm OBS points to /test, not the control URL.\n- Confirm the source was not shut down while hidden.\n\n### OBS shows controls\n\nThe wrong URL is loaded. Use /test.\n\n### Stale JavaScript appears after deployment\n\nThe HTML shell stamps versioned immutable module URLs. Reload the page so it receives the new asset version. Avoid hardcoding an old /__immutable/<version>/... URL.\n\n### Ground truth and screenshot disagree near transitions\n\nThis is probably a correlation race rather than a model error. Increase state duration, exclude transition windows, and log revision plus timestamps at capture time.\n\n### Random runs differ with the same seed\n\nCheck whether preset ordering, interval, starting state, or deployment version changed.\n\n## Security and operational notes\n\n- The val and /current.json are public.\n- The state update endpoint is also currently public.\n- Do not store credentials or sensitive data in scenario payloads.\n- A third party with the URL could change the active state.\n- Before using this beyond a development spike, add update authorization or restrict HTTP access while preserving a safe read path for Roverwatch.\n- The SQLite database is val-scoped. Do not switch the import to organization-global SQLite.\n- The app stores only one current shared state; it is not an audit log.\n- Multiple simultaneous control sessions are last-write-wins.\n- Polling every 500 ms creates ongoing reads from every open Test Screen. Revisit this if usage grows.\n\n## Known limitations\n\n- Motion labels are broader than the current visual implementations; several modes currently share simple CSS animation behavior.\n- The speed control is represented in state but is not fully applied to every motion mode.\n- Random position is currently deterministic and fixed rather than seed-derived per event.\n- Sequence editing is not yet exposed in the UI.\n- Session history is local to one browser.\n- Transition history begins at the state present when the history schema was introduced; earlier revisions were not reconstructable.\n- Ground-truth export is capped at 1,000 transitions per request and does not yet paginate automatically.\n- There is no screenshot capture inside Vision Lab.\n- There is no automatic inference comparison.\n- There is no authentication on state updates.\n- Ground-truth timestamps are not a distributed clock guarantee.\n- Current state polling can observe a transition after the captured Twitch frame was produced.\n\nThese limitations should be treated as explicit backlog items, not assumed capabilities.\n\n## Recommended future work\n\nPriority order for Roverwatch development:\n\n1. Complete the simple-shape Twitch capture and chat spike.\n2. Resolve and log Vision Lab scenario ID and revision for each frame through /api/state-at.\n3. Measure scheduled-to-published, published-to-observed, capture, upload, inference, and chat latency separately.\n4. Add pagination and retention policy for transition and observation history.\n5. Define shared normalized expected/actual result schemas.\n6. Add automated comparison and mismatch reasons in Roverwatch, not in ground-truth history.\n7. Measure capture, upload, inference, and chat latency separately.\n8. Add versioned scenario sets.\n9. Add screenshot labeling and evaluation exports.\n10. Add update authentication before broader use.\n11. Refactor renderers and visualState types as scenario complexity grows.\n12. Add model accuracy, confusion, and latency reporting.\n\n## Maintenance principles\n\n- Preserve the separation between visual state and expected result.\n- Treat scenario IDs as durable API identifiers.\n- Reset scenario-local difficulty when changing scenarios.\n- Keep Test Screen free of controls and metadata.\n- Prefer deterministic behavior over visual novelty.\n- Record raw outputs before normalization.\n- Do not classify transition-race frames as model failures.\n- Keep the main Roverwatch integration read-only until the contract is stable.\n- Document changes to schemas, IDs, timing, and synchronization behavior.\n- Add regression scenarios for every production vision failure worth reproducing.\n\n## Definition of done for the current spike\n\nThe first Roverwatch vision spike is complete when:\n\n- Vision Lab is streamed through OBS to Twitch;\n- Roverwatch captures screenshots from that Twitch stream;\n- Roverwatch recognizes blank, green circle, red square, and blue triangle states;\n- each capture is logged with S3 key, capture timestamp, Vision Lab scenario ID, and revision;\n- Twitch chat receives the normalized detection;\n- repeated runs can reproduce and explain failures;\n- transition-window frames are excluded or labeled ambiguous.\n\nAt that point, the project has proven the full transport and inference loop. More realistic HUD tests should be added only after the simple pipeline is reliable.\n",[145],"e8c03566-51a9-483d-9966-c36948fb0bf3","CHANGELOG.md","01a0495a-b924-761c-aebd-29a0c7fb0190",1,["D",1787937077281],"# Changelog\n\n## 2026-07-30 — Immutable asset caching\n\nFrontend files are now served at version-addressed URLs\n(`/__immutable/45/frontend/index.tsx`) with\n`Cache-Control: public, max-age=31536000, immutable`, so repeat visits load the\nwhole client graph from the browser cache — roughly 3× faster (~665ms → ~157ms\nmeasured). Publishing your val bumps its version, which changes every URL, so\ncaches invalidate automatically. Old-version URLs return 404 after a publish\n(like Next.js build assets); a reload picks up the new version.\n\nWhat changed in the template:\n\n- `frontend/index.html` → [`frontend/root.tsx`](/frontend/root.tsx): the HTML\n shell is now JSX and stamps asset URLs with `immutableFileUrl()`\n- [`index.ts`](/index.ts): serves `Root()` at `/`, and one\n `app.get(\"/__immutable/*\", (c) => serveImmutableFile(c.req.path))` route\n serves all frontend files\n\n### Upgrading an existing remix\n\nYour copy of this template doesn't update automatically. Two options:\n\n**Minimal (most of the benefit):** point both routes at `serveImmutableFile` —\nyour existing bare URLs redirect into versioned space, the new route serves\nthem, and the whole module graph gets cached (costs one redirect per page view):\n\n```ts\nimport { serveImmutableFile } from \"https://esm.town/v/std/utils/index.ts\";\n\napp.get(\"/__immutable/*\", (c) => serveImmutableFile(c.req.path));\napp.get(\"/frontend/**/*\", (c) => serveImmutableFile(c.req.path)); // was serveFile\n```\n\n**Full upgrade (also removes the redirect):** three self-contained steps —\nno other files needed. Adjust the ``, favicon, and body of `Root()` to\nmatch your app's existing `index.html`, and stamp anything else your shell\nreferences directly (images, css) with `immutableFileUrl()`.\n\n1. Create `frontend/root.tsx`:\n\n```tsx\n/** @jsxImportSource npm:hono@4/jsx */\nimport { raw } from \"npm:hono@4/html\";\nimport { immutableFileUrl } from \"https://esm.town/v/std/utils/index.ts\";\n\n// The HTML shell for the app. It is never cached, so immutableFileUrl can stamp\n// the current val version into each asset URL — the browser caches those\n// immutably, and publishing the val changes the URLs, refetching exactly once.\nexport function Root() {\n return (\n <>\n {raw(\"<!DOCTYPE html>\")}\n <html lang=\"en\">\n <head>\n <meta charSet=\"UTF-8\" />\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\" />\n <title>React Hono Val Town Starter\n

Diff · v40