`, favicon, and body of `Root()` to\nmatch your app's existing `index.html`, and stamp anything else your shell\nreferences directly (images, css) with `immutableFileUrl()`.\n\n1. Create `frontend/root.tsx`:\n\n```tsx\n/** @jsxImportSource npm:hono@4/jsx */\nimport { raw } from \"npm:hono@4/html\";\nimport { immutableFileUrl } from \"https://esm.town/v/std/utils/index.ts\";\n\n// The HTML shell for the app. It is never cached, so immutableFileUrl can stamp\n// the current val version into each asset URL — the browser caches those\n// immutably, and publishing the val changes the URLs, refetching exactly once.\nexport function Root() {\n return (\n <>\n {raw(\"<!DOCTYPE html>\")}\n <html lang=\"en\">\n <head>\n <meta charSet=\"UTF-8\" />\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\" />\n <title>React Hono Val Town Starter

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_446":447,"_448":449},"routes/_app.x.$handle.$name",{"_100":99,"_32":46,"_108":90,"_27":109,"_13":14,"_110":-5,"_11":111,"_112":113,"_114":115,"_116":117,"_118":119,"_20":-5,"_120":-5,"_121":-5,"_122":25,"_123":124,"_125":60,"_126":-5,"_127":-5,"_128":129},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_44":45,"_32":46,"_39":47,"_48":-5,"_49":25,"_50":51,"_52":51,"_53":54,"_55":-5,"_18":56,"_57":58,"_59":60,"_61":-5,"_62":-5,"_63":64,"_65":66,"_67":68,"_69":70,"_71":72,"_73":-5,"_74":75,"_76":75,"_77":78,"_79":78,"_80":78,"_81":75,"_82":75,"_83":84,"_85":86,"_87":88,"_89":25},"version",76,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":33,"_34":35,"_36":38,"_39":40,"_41":14,"_42":43,"_24":25},"previous",{"_30":31,"_32":33,"_34":35,"_36":37},"deleted",false,"deletedHere","path","tests/beaconPixelDecoder.test.ts","latestPath","fileId","7a9b49e0-a4af-11f1-bfdf-1607ee4eb77e","name","beaconPixelDecoder.test.ts","fileType","script","content","import { encodeRevision } from \"../frontend/beacon.ts\";\nimport { decodeRevisionFromPixels } from \"../frontend/beaconContract.ts\";\n\nfunction assert(condition:unknown,message:string){if(!condition)throw new Error(message)}\nfunction raster(revision:number,width=1920,height=1080){\n const data=new Uint8ClampedArray(width*height*4);\n const bg=revision%2===0?0:255;\n for(let i=0;i;\n const state = JSON.parse(String(row.payload));\n return { ...state, scheduledAt: state.scheduledAt || state.startedAt || row.updated_at, publishedAt: state.publishedAt || row.updated_at, revision: Number(row.revision), updatedAt: row.updated_at };\n}\n\nconst bootState = await readState();\nawait sqlite.execute({\n sql: \"INSERT OR IGNORE INTO vision_lab_transitions (revision,scenario_id,step,scheduled_at,published_at,expected_json,state_json) VALUES (?,?,?,?,?,?,?)\",\n args: [Number(bootState.revision), String(bootState.scenario.id), Number(bootState.step || 0), String(bootState.scheduledAt), String(bootState.publishedAt), JSON.stringify(bootState.scenario.expectedResult), JSON.stringify(bootState)],\n});\n\nfunction transitionFromRow(row: Record) {\n return { revision: Number(row.revision), scenarioId: String(row.scenario_id), step: Number(row.step), scheduledAt: String(row.scheduled_at), publishedAt: String(row.published_at), expected: JSON.parse(String(row.expected_json)), state: JSON.parse(String(row.state_json)) };\n}\nfunction observationFromRow(row: Record) {\n return {\n id: Number(row.id), revision: Number(row.revision), observerId: String(row.observer_id),\n observedAt: String(row.observed_at), clientReceivedAt: row.client_received_at ? String(row.client_received_at) : null,\n renderedAt: row.rendered_at ? String(row.rendered_at) : null, receivedAt: String(row.received_at),\n pollDurationMs: row.poll_duration_ms == null ? null : Number(row.poll_duration_ms),\n pollIntervalMs: row.poll_interval_ms == null ? null : Number(row.poll_interval_ms),\n route: row.route ? String(row.route) : null, sessionId: row.session_id ? String(row.session_id) : null,\n };\n}\nfunction validDate(value: unknown) {\n return typeof value === \"string\" && !Number.isNaN(Date.parse(value));\n}\n\napp.get(\"/\", (c) => c.html(Root()));\napp.get(\"/test\", (c) => c.html(Root()));\napp.get(\"/latency\", (c) => c.html(Root()));\napp.get(\"/__immutable/*\", (c) => serveImmutableFile(c.req.path));\napp.get(\"/source\", (c) => c.redirect(parseVal().links.self.val));\n\napp.get(\"/api/state\", async (c) => c.json(await readState()));\napp.get(\"/current.json\", async (c) => c.json(await readState()));\napp.get(\"/api/time\", (c) => {\n const serverTime = new Date();\n return c.json({ serverTime: serverTime.toISOString(), unixMs: serverTime.getTime() });\n});\napp.get(\"/api/beacon-contract\", (c) => c.json(beaconPixelContract));\napp.get(\"/api/beacon/:revision\", (c) => {\n const revision = Number(c.req.param(\"revision\"));\n if (!Number.isInteger(revision) || revision < 0 || revision > 0xffffffff) return c.json({ error: \"revision must be an unsigned 32-bit integer\" }, 400);\n const matrix = encodeRevision(revision);\n return c.json({ revision, matrix, decodedRevision: decodeRevision(matrix), selfTestPassed: verifyBeaconRoundTrip() });\n});\n\napp.get(\"/api/history\", async (c) => {\n const sinceRevision = Math.max(0, Number(c.req.query(\"sinceRevision\") || 0));\n const limit = Math.min(1000, Math.max(1, Number(c.req.query(\"limit\") || 200)));\n const result = await sqlite.execute({ sql: \"SELECT revision,scenario_id,step,scheduled_at,published_at,expected_json,state_json FROM vision_lab_transitions WHERE revision > ? ORDER BY revision ASC LIMIT ?\", args: [sinceRevision, limit] });\n const transitions = result.rows.map((row) => transitionFromRow(row as Record));\n return c.json({ groundTruthOnly: true, sinceRevision, count: transitions.length, transitions });\n});\n\napp.get(\"/api/state-at\", async (c) => {\n const at = c.req.query(\"at\");\n if (!at || Number.isNaN(Date.parse(at))) return c.json({ error: \"Valid ISO-8601 at query parameter required\" }, 400);\n const result = await sqlite.execute({ sql: \"SELECT revision,scenario_id,step,scheduled_at,published_at,expected_json,state_json FROM vision_lab_transitions WHERE published_at <= ? ORDER BY published_at DESC, revision DESC LIMIT 1\", args: [new Date(at).toISOString()] });\n if (!result.rows.length) return c.json({ error: \"No ground-truth state had been published at that time\" }, 404);\n return c.json({ groundTruthOnly: true, capturedAt: new Date(at).toISOString(), transition: transitionFromRow(result.rows[0] as Record) });\n});\n\napp.get(\"/api/observations\", async (c) => {\n const revision = Math.max(0, Number(c.req.query(\"revision\") || 0));\n const limit = Math.min(1000, Math.max(1, Number(c.req.query(\"limit\") || 200)));\n const sessionId = c.req.query(\"sessionId\");\n const route = c.req.query(\"route\");\n const clauses = [\"revision >= ?\"];\n const args: (string | number)[] = [revision];\n if (sessionId) { clauses.push(\"session_id = ?\"); args.push(sessionId); }\n if (route) { clauses.push(\"route = ?\"); args.push(route); }\n args.push(limit);\n const result = await sqlite.execute({\n sql: \"SELECT id,revision,observer_id,observed_at,client_received_at,rendered_at,received_at,poll_duration_ms,poll_interval_ms,route,session_id FROM vision_lab_observations WHERE \" + clauses.join(\" AND \") + \" ORDER BY id DESC LIMIT ?\",\n args,\n });\n return c.json({ groundTruth: false, count: result.rows.length, observations: result.rows.map((row) => observationFromRow(row as Record)) });\n});\n\napp.post(\"/api/state\", async (c) => {\n const body = await c.req.json();\n if (!body?.scenario?.id || !body?.scenario?.visualState || !body?.scenario?.expectedResult) return c.json({ error: \"Invalid scenario payload\" }, 400);\n const current = await readState();\n const publishedAt = new Date().toISOString();\n const scheduledAt = validDate(body.scheduledAt) ? new Date(body.scheduledAt).toISOString() : publishedAt;\n const next = { scenario: body.scenario, difficulty: body.difficulty || defaultDifficulty, step: Number(body.step || 0), scheduledAt, publishedAt, revision: Number(current.revision) + 1 };\n const encoded = JSON.stringify(next);\n if (encoded.length > 100_000) return c.json({ error: \"Payload too large\" }, 413);\n await sqlite.batch([\n { sql: \"INSERT INTO vision_lab_transitions (revision,scenario_id,step,scheduled_at,published_at,expected_json,state_json) VALUES (?,?,?,?,?,?,?)\", args: [next.revision, next.scenario.id, next.step, scheduledAt, publishedAt, JSON.stringify(next.scenario.expectedResult), encoded] },\n { sql: \"UPDATE vision_lab_state SET payload=?, revision=?, updated_at=? WHERE id=1\", args: [encoded, next.revision, publishedAt] },\n ]);\n return c.json(next);\n});\n\napp.post(\"/api/observations\", async (c) => {\n const body = await c.req.json();\n const clientReceivedAt = body?.clientReceivedAt || body?.observedAt;\n const renderedAt = body?.renderedAt || body?.observedAt;\n const observedAt = body?.observedAt || renderedAt;\n if (!Number.isInteger(body?.revision) || !body?.observerId || !validDate(clientReceivedAt) || !validDate(renderedAt) || !validDate(observedAt)) {\n return c.json({ error: \"revision, observerId, and valid clientReceivedAt/renderedAt (or observedAt) are required\" }, 400);\n }\n const receivedAt = new Date().toISOString();\n await sqlite.execute({\n sql: \"INSERT INTO vision_lab_observations (revision,observer_id,observed_at,received_at,client_received_at,rendered_at,poll_duration_ms,poll_interval_ms,route,session_id) VALUES (?,?,?,?,?,?,?,?,?,?)\",\n args: [\n body.revision, String(body.observerId).slice(0, 100), new Date(observedAt).toISOString(), receivedAt,\n new Date(clientReceivedAt).toISOString(), new Date(renderedAt).toISOString(),\n Number.isFinite(Number(body.pollDurationMs)) ? Number(body.pollDurationMs) : null,\n POLL_INTERVALS.includes(Number(body.pollIntervalMs)) ? Number(body.pollIntervalMs) : null,\n body.route ? String(body.route).slice(0, 100) : null,\n body.sessionId ? String(body.sessionId).slice(0, 100) : null,\n ],\n });\n return c.json({ accepted: true, groundTruth: false, revision: body.revision, clientReceivedAt: new Date(clientReceivedAt).toISOString(), renderedAt: new Date(renderedAt).toISOString(), receivedAt }, 201);\n});\n\napp.post(\"/api/latency-sessions/start\", async (c) => {\n const body = await c.req.json();\n const pollIntervalMs = Number(body?.pollIntervalMs);\n const transitionIntervalMs = Number(body?.transitionIntervalMs);\n const count = Number(body?.count);\n if (!POLL_INTERVALS.includes(pollIntervalMs) || !TRANSITION_INTERVALS.includes(transitionIntervalMs) || !Number.isInteger(count) || count < 1 || count > 100) {\n return c.json({ error: \"pollIntervalMs, transitionIntervalMs, and count are invalid\" }, 400);\n }\n const current = await readState();\n const sessionId = crypto.randomUUID();\n const startedAt = new Date().toISOString();\n const configuration = { pollIntervalMs, transitionIntervalMs, count };\n await sqlite.execute({\n sql: \"INSERT INTO vision_lab_latency_sessions (session_id,started_at,start_revision,configuration_json,vision_lab_version) VALUES (?,?,?,?,?)\",\n args: [sessionId, startedAt, Number(current.revision), JSON.stringify(configuration), VISION_LAB_VERSION],\n });\n return c.json({ session: { sessionId, startedAt, startingRevision: Number(current.revision), configuration, visionLabVersion: VISION_LAB_VERSION } }, 201);\n});\n\napp.post(\"/api/latency-sessions/:id/stop\", async (c) => {\n const sessionId = c.req.param(\"id\");\n const existing = await sqlite.execute({ sql: \"SELECT session_id FROM vision_lab_latency_sessions WHERE session_id = ?\", args: [sessionId] });\n if (!existing.rows.length) return c.json({ error: \"Latency session not found\" }, 404);\n const current = await readState();\n const endedAt = new Date().toISOString();\n await sqlite.execute({ sql: \"UPDATE vision_lab_latency_sessions SET ended_at = COALESCE(ended_at, ?), end_revision = COALESCE(end_revision, ?) WHERE session_id = ?\", args: [endedAt, Number(current.revision), sessionId] });\n return c.json({ sessionId, endedAt, endingRevision: Number(current.revision) });\n});\n\napp.get(\"/api/latency-sessions/:id/export\", async (c) => {\n const sessionId = c.req.param(\"id\");\n const sessionResult = await sqlite.execute({ sql: \"SELECT session_id,started_at,ended_at,start_revision,end_revision,configuration_json,vision_lab_version FROM vision_lab_latency_sessions WHERE session_id = ?\", args: [sessionId] });\n if (!sessionResult.rows.length) return c.json({ error: \"Latency session not found\" }, 404);\n const row = sessionResult.rows[0] as Record;\n const current = await readState();\n const endRevision = row.end_revision == null ? Number(current.revision) : Number(row.end_revision);\n const transitionsResult = await sqlite.execute({\n sql: \"SELECT revision,scenario_id,step,scheduled_at,published_at,expected_json,state_json FROM vision_lab_transitions WHERE revision > ? AND revision <= ? ORDER BY revision ASC\",\n args: [Number(row.start_revision), endRevision],\n });\n const observationsResult = await sqlite.execute({\n sql: \"SELECT id,revision,observer_id,observed_at,client_received_at,rendered_at,received_at,poll_duration_ms,poll_interval_ms,route,session_id FROM vision_lab_observations WHERE session_id = ? ORDER BY id ASC\",\n args: [sessionId],\n });\n return c.json({\n schemaVersion: VISION_LAB_VERSION, beaconSchemaVersion: BEACON_SCHEMA_VERSION, sessionId, visionLabVersion: String(row.vision_lab_version),\n startedAt: String(row.started_at), endedAt: row.ended_at ? String(row.ended_at) : null,\n startingRevision: Number(row.start_revision), endingRevision: endRevision,\n configuration: JSON.parse(String(row.configuration_json)),\n groundTruth: { authoritative: true, transitions: transitionsResult.rows.map((item) => transitionFromRow(item as Record)) },\n observations: { authoritative: false, records: observationsResult.rows.map((item) => observationFromRow(item as Record)) },\n });\n});\n\napp.onError((err) => Promise.reject(err));\nexport default app.fetch;","label",[137],"d137f001-ac87-4c1b-82fa-008e29b54ef5","README.md","01a0495a-b924-761c-aebd-2ee61846be65",75,["D",1788121004407],23,"# Roverwatch Vision Lab\n\nRoverwatch Vision Lab is a standalone, deterministic visual test harness for developing and evaluating Roverwatch's screenshot-understanding pipeline without loading Overwatch or depending on live gameplay.\n\n- Control Mode: https://roverwatch-vision-lab.val.run/\n- Test Screen Mode: https://roverwatch-vision-lab.val.run/test\n- Current machine-readable state: https://roverwatch-vision-lab.val.run/current.json\n- Val source: https://www.val.town/x/pchinjr/roverwatch-vision-lab/code/\n\nVision Lab is intentionally separate from the main Roverwatch repository. It produces controlled visual stimuli and their expected answers. Roverwatch captures those visuals from Twitch, performs inference, and reports what it detected.\n\n## Why this exists\n\nTesting computer vision against a live game makes failures difficult to reproduce:\n\n- the game state changes constantly;\n- screenshots do not have explicit ground truth;\n- a missed detection may be caused by capture, compression, timing, rendering, or inference;\n- reproducing the exact same visual state is difficult;\n- testing requires a running game and a known game situation.\n\nVision Lab replaces those uncontrolled inputs with stable scenario IDs, known visuals, and explicit expected results. It lets the Roverwatch development process answer smaller questions in order:\n\n1. Can Roverwatch capture the stream?\n2. Can it retrieve and decode the screenshot?\n3. Can it identify a simple shape and color?\n4. Can it associate the inference with the correct active scenario?\n5. Can it write the expected message to Twitch chat?\n6. Can it handle text, numbers, HUD regions, motion, occlusion, noise, and temporal changes?\n7. Can repeated runs be compared objectively?\n\nThe initial end-to-end proof is deliberately simple:\n\n~~~text\nVision Lab displays a green circle\n ↓\nOBS sends the Test Screen to Twitch\n ↓\nRoverwatch captures a frame\n ↓\nThe vision model reports \"green circle\"\n ↓\nRoverwatch reads Vision Lab ground truth\n ↓\nThe result is marked correct or incorrect\n ↓\nRoverwatch writes the result to Twitch chat\n~~~\n\n## Responsibilities and system boundary\n\n### Vision Lab owns\n\n- deterministic scenario definitions;\n- rendering visual test stimuli;\n- a clean OBS-compatible Test Screen;\n- the currently active scenario state;\n- structured expected results;\n- stable scenario and sequence-step IDs;\n- difficulty transformations such as blur, scale, noise, and occlusion;\n- local operator history;\n- authoritative server-side ground-truth history and JSON export;\n- capture-time ground-truth resolution;\n- non-authoritative Test Screen observation records;\n- explicit scheduled, published, observed, and received timestamps.\n\n### Roverwatch owns\n\n- Twitch stream acquisition;\n- frame capture and S3 screenshot storage;\n- screenshot selection and timestamps;\n- model invocation;\n- inference parsing and normalization;\n- Twitch chat output;\n- comparison of actual inference with expected Vision Lab results;\n- evaluation metrics, latency tracking, and operational logs.\n\n### Vision Lab does not currently own\n\n- direct calls into Roverwatch;\n- Twitch credentials or chat authentication;\n- reading screenshots from S3;\n- model inference;\n- automatic scoring;\n- confusion matrices;\n- labeled-image capture;\n- production Overwatch UI artwork.\n\nKeeping this boundary explicit prevents the test harness from becoming coupled to one model, one capture implementation, or one Roverwatch deployment.\n\n## Current capabilities\n\nThe current application supports:\n\n- separate Control and Test Screen modes;\n- cross-browser synchronization between Chrome and OBS;\n- shape and color scenarios;\n- large and small OCR text;\n- numeric percentages;\n- generic game HUD states;\n- multiple simultaneous HUD values;\n- scale, position, contrast, brightness, blur, opacity, noise, background, and occlusion controls;\n- basic motion modes;\n- a timed blank → green circle → blank → red square → blank → blue triangle sequence;\n- seeded random scenario selection;\n- structured ground-truth display and copying;\n- scenario JSON export;\n- local test-session recording and export;\n- keyboard shortcuts;\n- an HTTP endpoint for the currently active state;\n- append-only server-side transition history;\n- capture-time state lookup through `/api/state-at`;\n- separate Test Screen observation history;\n- ground-truth-only server exports.\n\n## Quick start\n\n### Manual browser test\n\n1. Open Control Mode.\n2. Click Open Test Screen.\n3. Place the two windows side by side.\n4. Select Green Circle in Control Mode.\n5. Confirm that the Test Screen shows only a green circle.\n6. Confirm that the Expected Result panel reports shape circle and color green.\n7. Select Red Square and Blue Triangle and confirm that the previous scenario's difficulty settings do not leak into the new one.\n8. Run the baseline timeline and confirm that blank states appear between shapes.\n\n### OBS Browser Source\n\n1. Add a Browser Source in OBS.\n2. Use https://roverwatch-vision-lab.val.run/test as its URL.\n3. Set width to 1920 and height to 1080.\n4. Leave custom CSS empty.\n5. Disable Shutdown source when not visible if sequences must keep running while the source is hidden.\n6. Keep Refresh browser when scene becomes active disabled unless a fresh load is desired.\n7. Change scenarios from Control Mode.\n8. Confirm that the OBS source updates within approximately 500 ms.\n\nThe Test Screen has no controls, requires no mouse input, hides the cursor, fills the viewport, and polls shared state while it is open.\n\n## Architecture\n\n~~~text\nfrontend/scenarios.ts\n Scenario types, defaults, presets, baseline sequence\n ↓\nfrontend/components/App.tsx\n Control Mode, Test Screen, renderer, timeline, session recorder\n ↓\nPOST /api/state\n Atomically appends a ground-truth transition and updates current state\n ↓\nVal-scoped SQLite\n Current state + immutable transitions + separate observations\n ↓\nGET /api/state, /current.json, /api/history, /api/state-at\n ↓\nTest Screen polls every 500 ms, renders the revision, then records observedAt\n~~~\n\n### File layout\n\n| File | Responsibility |\n| --- | --- |\n| index.ts | Hono HTTP entrypoint, routes, SQLite initialization, shared-state reads and writes |\n| frontend/root.tsx | HTML shell, metadata, Twind loader, immutable frontend entry URL |\n| frontend/index.tsx | Mounts the React application |\n| frontend/components/App.tsx | Control Mode, Test Screen, renderers, timeline, random mode, exports |\n| frontend/scenarios.ts | Scenario contracts, presets, default difficulty, baseline sequence |\n| frontend/favicon.svg | Application icon |\n| README.md | Architecture, operations, extension, and Roverwatch integration contract |\n\n### Technology choices\n\n- Val Town hosts the application and HTTP routes.\n- Hono handles routing.\n- React 18.2 renders both modes.\n- Twind provides utility styling without a build process.\n- Val-scoped SQLite stores shared display state.\n- Browser localStorage stores control-only history and seed data.\n- Immutable versioned frontend URLs prevent stale JavaScript after a deployment.\n- Polling is used because Val Town does not accept incoming WebSocket connections.\n\n## Two kinds of state\n\nUnderstanding this distinction is essential for maintenance.\n\n### Shared live state\n\nThe current screen state is stored in the val-scoped SQLite table vision_lab_state. There is exactly one row, with ID 1.\n\n| Column | Meaning |\n| --- | --- |\n| id | Singleton row ID |\n| payload | JSON-encoded active LiveState |\n| revision | Monotonically increasing state revision |\n| updated_at | Server timestamp of the latest update |\n\nShared state exists because Chrome and OBS use separate browser profiles. They cannot reliably share localStorage or BroadcastChannel state. Control Mode therefore publishes to the server, and Test Screen Mode polls the server.\n\nEvery accepted state update is also appended to vision_lab_transitions in the same SQLite batch as the current-state update. A transition can never be edited through the application API. It contains revision, scenario ID, step, scheduled time, server publication time, expected JSON, and the resolved state JSON.\n\nTest Screen observations are written to vision_lab_observations. They are explicitly marked non-ground-truth and never change a transition. Each observation records revision, observer ID, client observation time, and server receipt time.\n\nThe current shared state is intentionally last-write-wins. Opening multiple Control Mode windows means whichever window publishes last controls every connected Test Screen. Historical transitions remain append-only.\n\n### Browser-local state\n\nThe key roverwatch-vision-lab.v1 stores:\n\n- the current random-test seed;\n- recent session history.\n\nThis data is local to the browser profile. It is not visible to OBS and is not intended as authoritative evaluation data until exported.\n\n## Data contracts\n\n### Scenario\n\nA scenario separates what is rendered from what Roverwatch is expected to detect.\n\n~~~ts\ninterface Scenario {\n id: string;\n name: string;\n description: string;\n category: \"shape\" | \"ocr\" | \"hud\" | \"sequence\";\n visualState: {\n kind: \"blank\" | \"shape\" | \"text\" | \"number\" | \"hud\";\n // Renderer-specific fields\n };\n expectedResult: Record;\n difficulty?: Partial;\n}\n~~~\n\nExample:\n\n~~~json\n{\n \"id\": \"shape.green-circle\",\n \"name\": \"Green Circle\",\n \"description\": \"Green Circle recognition baseline\",\n \"category\": \"shape\",\n \"visualState\": {\n \"kind\": \"shape\",\n \"shape\": \"circle\",\n \"color\": \"green\"\n },\n \"expectedResult\": {\n \"scenarioType\": \"shape\",\n \"visible\": true,\n \"shape\": \"circle\",\n \"color\": \"green\"\n }\n}\n~~~\n\n### LiveState\n\nA LiveState is the resolved scenario being shown now.\n\n~~~ts\ninterface LiveState {\n scenario: Scenario;\n difficulty: Difficulty;\n step: number;\n scheduledAt: string;\n publishedAt?: string;\n observedAt?: string;\n revision?: number;\n}\n~~~\n\nThe server assigns publishedAt and the authoritative revision. A Test Screen adds its own observedAt only after it receives that revision. The current-state response also includes updatedAt for compatibility.\n\n### Timestamp semantics\n\n| Timestamp | Clock owner | Meaning | Authoritative use |\n| --- | --- | --- | --- |\n| scheduledAt | Control browser | When the controller intended the state to begin | Sequence intent and controller delay |\n| publishedAt | Vision Lab server | When the ground-truth transition and current state committed atomically | Authoritative state timeline |\n| observedAt | Individual Test Screen | When that client first received the revision | Display-observation latency only |\n| receivedAt | Vision Lab server | When an observation record reached the server | Observation transport diagnostics |\n| Roverwatch capturedAt | Roverwatch capture process | When the evaluated frame was acquired | Lookup key for /api/state-at |\n| Roverwatch inference timestamps | Roverwatch | When inference started and completed | Model and pipeline responsiveness |\n\nThese clocks may be skewed. Use publishedAt to define the authoritative ground-truth timeline. Use capturedAt to resolve a frame against that timeline. Treat observedAt as evidence that a particular Test Screen received a revision, not proof that Twitch had already encoded and delivered it.\n\n### Ground truth versus live decisions\n\nThe transitions table and ground-truth exports contain only scenario intent, resolved visual state, expected result, revision, and authoritative timing. They never contain model output, Twitch chat output, confidence, match status, or pass/fail decisions.\n\nObservations are stored in a separate table and are labeled non-ground-truth. Roverwatch results must remain in Roverwatch storage. Evaluation is a later join:\n\n~~~text\nVision Lab immutable transition\n +\nRoverwatch frame and inference record\n ↓\nComparison record and responsiveness metrics\n~~~\n\nThis separation makes it possible to change scoring logic without rewriting the historical expected record.\n\n\n### Difficulty\n\n~~~ts\ninterface Difficulty {\n scale: number;\n position: \"center\" | \"top-left\" | \"top-right\" |\n \"bottom-left\" | \"bottom-right\" | \"random\";\n contrast: \"high\" | \"medium\" | \"low\";\n brightness: number;\n blur: number;\n opacity: number;\n noise: number;\n background: \"solid\" | \"gradient\" | \"simple\" | \"busy\";\n occlusion: number;\n motion: \"stationary\" | \"horizontal\" | \"vertical\" |\n \"diagonal\" | \"bounce\" | \"random-walk\";\n speed: number;\n}\n~~~\n\n### Ground truth\n\nThe Control Mode ground-truth panel wraps the scenario's expected result with execution context:\n\n~~~json\n{\n \"scenarioId\": \"shape.green-circle\",\n \"scenarioName\": \"Green Circle\",\n \"step\": 0,\n \"revision\": 217,\n \"scheduledAt\": \"2026-08-28T20:33:01.614Z\",\n \"publishedAt\": \"2026-08-28T20:33:07.480Z\",\n \"expected\": {\n \"scenarioType\": \"shape\",\n \"visible\": true,\n \"shape\": \"circle\",\n \"color\": \"green\"\n }\n}\n~~~\n\nGround truth comes from scenario.expectedResult. It is not inferred from the rendered pixels. That separation is what makes the harness useful for evaluation.\n\n## HTTP API\n\nThe application is public and currently uses these routes:\n\n| Method | Route | Purpose |\n| --- | --- | --- |\n| GET | / | Control Mode |\n| GET | /test | Clean Test Screen |\n| GET | /api/state | Current shared LiveState |\n| POST | /api/state | Replace current shared LiveState |\n| GET | /current.json | Public machine-readable alias for current state |\n| GET | /api/history | Append-only ground-truth transitions after an optional revision |\n| GET | /api/state-at?at=... | Ground truth that had been published at a capture timestamp |\n| GET | /api/observations | Non-ground-truth Test Screen observations |\n| POST | /api/observations | Append a Test Screen observation without changing ground truth |\n| GET | /source | Redirect to editable Val source |\n| GET | /__immutable/* | Versioned frontend modules and assets |\n\n### Reading current ground truth\n\n~~~bash\ncurl https://roverwatch-vision-lab.val.run/current.json\n~~~\n\nThe main Roverwatch process should initially treat this endpoint as a debugging and correlation source, not as a production dependency.\n\n### State update validation\n\nPOST requests must contain:\n\n- scenario.id;\n- scenario.visualState;\n- scenario.expectedResult.\n\nPayloads above 100,000 characters are rejected. The server increments revision, stores the complete payload, and returns the saved state.\n\nPublishing a state and appending its transition occur atomically in one SQLite batch. The transition history endpoint is marked groundTruthOnly: true; the observations endpoint is marked groundTruth: false.\n\nFor capture-time correlation, Roverwatch should request:\n\n~~~text\nGET /api/state-at?at=\n~~~\n\nThe endpoint returns the most recent transition whose server-issued publishedAt is less than or equal to the capture time.\n\nAll endpoints are currently public. Do not place secrets, Twitch credentials, private screenshots, user data, or model credentials in scenario state.\n\n## How the renderer works\n\nScenarioStage reads two separate inputs:\n\n1. scenario.visualState selects the content renderer.\n2. difficulty transforms the rendered target and background.\n\nCurrent renderer branches are:\n\n| Kind | Renderer behavior |\n| --- | --- |\n| blank | Renders no target |\n| shape | Renders circle, square, or CSS triangle |\n| text | Renders exact text with size, alignment, and colors |\n| number | Uses the text renderer for numeric recognition |\n| hud | Renders health, ultimate, life state, team count, and enemy count |\n\nDo not derive expected results inside render components. Renderer code controls appearance; scenario definitions control expected meaning.\n\n## Scenario ID conventions\n\nStable IDs are required because Roverwatch logs and evaluation results will eventually join against them.\n\nRecommended format:\n\n~~~text\n.\n~~~\n\nExamples:\n\n~~~text\nshape.green-circle\nshape.red-square\nocr.large-text\nnumber.ult-99\nhud.player-dead\nhud.ult-ready\ndifficulty.occluded-target\nmotion.moving-target\nsequence.baseline.02-green\n~~~\n\nRules:\n\n- IDs should be lowercase and machine-readable.\n- Do not reuse an ID for a semantically different test.\n- Visual refinements that preserve the same intended detection may keep the ID.\n- Changed expected meaning should receive a new ID.\n- Sequence steps need stable IDs, not only numeric indexes.\n- For versioned evaluation sets, add an explicit version namespace later rather than silently changing the old set.\n\n## Preset and difficulty-state rules\n\nPreset difficulty overrides are scenario-local. Selecting a new preset or moving to a new sequence step must:\n\n1. begin with defaultDifficulty;\n2. merge only the new scenario's difficulty overrides;\n3. publish the resolved result.\n\nNever merge a newly selected preset onto the previous scenario's resolved difficulty. Doing that causes state leakage: a moving or occluded preset can make every later scenario move or remain occluded.\n\nManual difficulty controls intentionally modify the active scenario after selection. Selecting another preset resets those controls to defaults plus the new preset overrides.\n\nThis rule is enforced through:\n\n~~~ts\nmerged(nextScenario, defaultDifficulty)\n~~~\n\nTreat it as an invariant when changing scenario-selection, sequence, or random-mode code.\n\n## Baseline sequence\n\nThe initial sequence proves temporal capture and classification:\n\n| Step ID | Duration | Expected visual |\n| --- | ---: | --- |\n| sequence.baseline.01-blank | 8 seconds | Blank |\n| sequence.baseline.02-green | 15 seconds | Green circle |\n| sequence.baseline.03-blank | 8 seconds | Blank |\n| sequence.baseline.04-red | 15 seconds | Red square |\n| sequence.baseline.05-blank | 8 seconds | Blank |\n| sequence.baseline.06-blue | 15 seconds | Blue triangle |\n\nBlank intervals matter. They let Roverwatch demonstrate that it detects absence, does not repeat a stale inference, and can associate frames with state transitions.\n\n## Random mode and reproducibility\n\nRandom mode uses a deterministic seeded pseudo-random generator. Given:\n\n- the same seed;\n- the same preset ordering;\n- the same interval;\n- the same starting point;\n\nit should produce the same selection order.\n\nChanging the preset array order changes the generated scenario sequence even when the seed remains the same. Treat preset ordering as part of the reproducibility contract until random mode records the generated schedule explicitly.\n\n## Test sessions and exports\n\nStarting a session records the current revision. End + export then downloads authoritative transitions after that revision. Export ground truth downloads the available history independently of a session.\n\nThe server export has this shape:\n\n~~~json\n{\n \"groundTruthOnly\": true,\n \"sinceRevision\": 216,\n \"count\": 1,\n \"transitions\": [\n {\n \"revision\": 217,\n \"scenarioId\": \"shape.green-circle\",\n \"step\": 0,\n \"scheduledAt\": \"2026-08-28T20:33:01.614Z\",\n \"publishedAt\": \"2026-08-28T20:33:07.480Z\",\n \"expected\": {\n \"scenarioType\": \"shape\",\n \"visible\": true,\n \"shape\": \"circle\",\n \"color\": \"green\"\n },\n \"state\": {\n \"scenario\": {},\n \"difficulty\": {},\n \"step\": 0,\n \"scheduledAt\": \"2026-08-28T20:33:01.614Z\",\n \"publishedAt\": \"2026-08-28T20:33:07.480Z\",\n \"revision\": 217\n }\n }\n ]\n}\n~~~\n\nExports contain expected state only. Roverwatch inference, chat output, match status, and mismatch decisions must be stored in Roverwatch and joined afterward. This prevents model output from contaminating the source used to judge that output.\n\nBrowser-local operator history remains useful for immediate UI feedback, but responsiveness analysis should use server transitions, Test Screen observations, and Roverwatch capture/inference timestamps.\n\n## Recommended Roverwatch integration\n\n### Phase 1: manual correlation\n\nUse this first.\n\n1. Open Control Mode and Test Screen.\n2. Stream Test Screen through OBS to Twitch.\n3. Select one scenario at a time.\n4. Let Roverwatch capture a frame and run inference.\n5. Inspect Roverwatch logs and Vision Lab's Expected Result panel.\n6. Confirm the model output and Twitch chat message manually.\n\nSuccess criterion: Roverwatch consistently distinguishes blank, green circle, red square, and blue triangle.\n\n### Phase 2: log the Vision Lab state with each inference\n\nWhen a screenshot is selected for inference:\n\n1. Roverwatch records its frame capture timestamp as close to acquisition as possible.\n2. Roverwatch requests /api/state-at?at=<capturedAt>.\n3. Roverwatch records scenarioId, revision, scheduledAt, and publishedAt from the returned transition.\n4. Roverwatch records the frame capture timestamp and S3 object key.\n5. Roverwatch records normalized inference output and model latency.\n6. The combined record is written to Roverwatch logs.\n\nSuggested evaluation record:\n\n~~~ts\ninterface RoverwatchVisionEvaluation {\n frameId: string;\n s3Key: string;\n capturedAt: string;\n inferenceStartedAt: string;\n inferenceCompletedAt: string;\n model: string;\n visionLab: {\n scenarioId: string;\n revision: number;\n startedAt: string;\n updatedAt: string;\n expected: Record;\n };\n actual: Record;\n matched?: boolean;\n mismatchReasons?: string[];\n}\n~~~\n\n### Phase 3: guard against transition races\n\nA frame captured near a scenario change may show the previous state while /current.json already reports the next state.\n\nDo not compare a frame blindly against whatever state is current after inference finishes.\n\nRecommended approach:\n\n1. Read Vision Lab state close to frame capture, not after a slow model call.\n2. Resolve the frame through /api/state-at using the capture timestamp.\n3. Record revision, scheduledAt, and publishedAt.\n4. Use Test Screen observations to estimate controller-to-display delay without treating observations as ground truth.\n5. Add a transition-settling window before scoring a new state.\n6. Mark ambiguous transition-window frames as excluded rather than incorrect.\n\nThe baseline sequence now uses 8-second blank states and 15-second target states. Continue excluding frames near transitions until measured Twitch and capture latency supports a narrower settling window.\n\n### Phase 4: automated scoring\n\nAfter correlation is reliable, normalize expected and actual outputs into a shared result schema and calculate:\n\n- exact-match accuracy;\n- per-scenario accuracy;\n- false positives during blank states;\n- false negatives;\n- confusion pairs;\n- model latency;\n- end-to-end capture-to-chat latency.\n\nKeep raw model output alongside normalized output. Normalization bugs must be distinguishable from model errors.\n\n## Adding a preset with an existing renderer\n\n1. Open frontend/scenarios.ts.\n2. Add an object to presets.\n3. Assign a stable, unique ID.\n4. Define only visual fields used by the selected renderer kind.\n5. Write an explicit, machine-readable expected result.\n6. Add only scenario-specific difficulty overrides.\n7. Select the new preset in Control Mode.\n8. Verify the Test Screen.\n9. Verify /current.json.\n10. Select a normal preset afterward and confirm no difficulty state leaked.\n\nExample:\n\n~~~ts\n{\n id: \"ocr.player-dead\",\n name: \"Player Dead Text\",\n description: \"Large death-state OCR target\",\n category: \"ocr\",\n visualState: {\n kind: \"text\",\n text: \"PLAYER DEAD\",\n fontSize: 128,\n align: \"center\",\n foreground: \"#ffffff\",\n background: \"#070b12\"\n },\n expectedResult: {\n scenarioType: \"text\",\n text: \"PLAYER DEAD\"\n }\n}\n~~~\n\n## Adding a new scenario type\n\nA new scenario type changes both the contract and renderer.\n\n1. Add the new kind to ScenarioKind.\n2. Add its visual fields to the Scenario type, or refactor visualState into a discriminated union.\n3. Add a renderer component in App.tsx.\n4. Add one branch in ScenarioStage.\n5. Create at least one baseline preset.\n6. Define the expected-result schema.\n7. Confirm difficulty transformations still work.\n8. Confirm blank and later presets reset the new type correctly.\n9. Document the expected schema here.\n10. Coordinate with Roverwatch before automated comparison depends on the new schema.\n\nIf scenario types grow significantly, the next maintainability refactor should make visualState a discriminated union and move renderers into separate component files.\n\n## Adding or changing a sequence\n\nSequence steps live in milestoneSequence in frontend/scenarios.ts.\n\nEach step needs:\n\n- a stable step ID;\n- a duration in milliseconds;\n- a scenario definition;\n- an explicit expected result through that scenario.\n\nWhen changing durations, remember that Twitch latency, frame capture cadence, S3 upload delay, polling, and model latency all consume part of the visible window. A three-second state may yield very few usable frames in a delayed pipeline.\n\nFor automated evaluation, prefer longer states until measured end-to-end latency is known.\n\n## Verification checklist\n\nRun this checklist after changing state, scenarios, rendering, or sequences.\n\n### Application\n\n- Control Mode returns HTTP 200.\n- Test Screen returns HTTP 200.\n- Frontend modules return JavaScript, not HTML.\n- /current.json returns valid JSON.\n- /api/history returns append-only ground-truth transitions.\n- /api/state-at resolves the expected revision for a known timestamp.\n- /api/observations remains separate and marked non-ground-truth.\n- The Test Screen has no visible controls.\n- The OBS source updates after Control Mode changes.\n\n### Scenario behavior\n\n- Green Circle renders green and circular.\n- Red Square renders red and square.\n- Blue Triangle renders blue and triangular.\n- Expected Result matches the rendered scenario.\n- Stable IDs appear in /current.json.\n- Blank renders no target.\n- Timeline steps occur in the documented order.\n\n### State isolation regression\n\n- Select Moving Target.\n- Select Green Circle.\n- Confirm motion is stationary.\n- Select Partially Occluded Target.\n- Select Red Square.\n- Confirm occlusion is 0%.\n- Select Busy HUD.\n- Select Blue Triangle.\n- Confirm background is solid and noise is 0.\n- Refresh both Control and Test Screen.\n- Confirm the current state remains internally consistent.\n\n### OBS and Roverwatch\n\n- OBS uses 1920×1080.\n- The source continues while unfocused.\n- Twitch receives the screen.\n- Roverwatch captures the intended frame.\n- Roverwatch logs S3 key, capture time, scenario ID, and revision.\n- Twitch chat output matches the expected result.\n\n## Troubleshooting\n\n### A previous effect remains active\n\nCause: the new scenario was merged onto the previous scenario's resolved difficulty.\n\nFix: make preset, sequence, and random selections resolve from defaultDifficulty, then merge only the new scenario override.\n\n### Refresh preserves an unexpected state\n\nRefresh reads the shared SQLite state. This is intentional so OBS and Control Mode stay synchronized. Select a clean preset or Reset difficulty. If the stored payload is internally inconsistent, inspect the singleton vision_lab_state row.\n\n### Control Mode says offline\n\nCheck:\n\n- GET /api/state;\n- Val Town logs;\n- SQLite availability;\n- browser network errors;\n- whether the val's HTTP access is still public.\n\n### Test Screen does not update\n\n- Confirm Control Mode says synced.\n- Confirm /current.json changes.\n- Wait at least 500 ms.\n- Refresh the OBS Browser Source.\n- Confirm OBS points to /test, not the control URL.\n- Confirm the source was not shut down while hidden.\n\n### OBS shows controls\n\nThe wrong URL is loaded. Use /test.\n\n### Stale JavaScript appears after deployment\n\nThe HTML shell stamps versioned immutable module URLs. Reload the page so it receives the new asset version. Avoid hardcoding an old /__immutable/<version>/... URL.\n\n### Ground truth and screenshot disagree near transitions\n\nThis is probably a correlation race rather than a model error. Increase state duration, exclude transition windows, and log revision plus timestamps at capture time.\n\n### Random runs differ with the same seed\n\nCheck whether preset ordering, interval, starting state, or deployment version changed.\n\n## Security and operational notes\n\n- The val and /current.json are public.\n- The state update endpoint is also currently public.\n- Do not store credentials or sensitive data in scenario payloads.\n- A third party with the URL could change the active state.\n- Before using this beyond a development spike, add update authorization or restrict HTTP access while preserving a safe read path for Roverwatch.\n- The SQLite database is val-scoped. Do not switch the import to organization-global SQLite.\n- The app stores only one current shared state; it is not an audit log.\n- Multiple simultaneous control sessions are last-write-wins.\n- Polling every 500 ms creates ongoing reads from every open Test Screen. Revisit this if usage grows.\n\n## Known limitations\n\n- Motion labels are broader than the current visual implementations; several modes currently share simple CSS animation behavior.\n- The speed control is represented in state but is not fully applied to every motion mode.\n- Random position is currently deterministic and fixed rather than seed-derived per event.\n- Sequence editing is not yet exposed in the UI.\n- Session history is local to one browser.\n- Transition history begins at the state present when the history schema was introduced; earlier revisions were not reconstructable.\n- Ground-truth export is capped at 1,000 transitions per request and does not yet paginate automatically.\n- There is no screenshot capture inside Vision Lab.\n- There is no automatic inference comparison.\n- There is no authentication on state updates.\n- Ground-truth timestamps are not a distributed clock guarantee.\n- Current state polling can observe a transition after the captured Twitch frame was produced.\n\nThese limitations should be treated as explicit backlog items, not assumed capabilities.\n\n## Recommended future work\n\nPriority order for Roverwatch development:\n\n1. Complete the simple-shape Twitch capture and chat spike.\n2. Resolve and log Vision Lab scenario ID and revision for each frame through /api/state-at.\n3. Measure scheduled-to-published, published-to-observed, capture, upload, inference, and chat latency separately.\n4. Add pagination and retention policy for transition and observation history.\n5. Define shared normalized expected/actual result schemas.\n6. Add automated comparison and mismatch reasons in Roverwatch, not in ground-truth history.\n7. Measure capture, upload, inference, and chat latency separately.\n8. Add versioned scenario sets.\n9. Add screenshot labeling and evaluation exports.\n10. Add update authentication before broader use.\n11. Refactor renderers and visualState types as scenario complexity grows.\n12. Add model accuracy, confusion, and latency reporting.\n\n## Maintenance principles\n\n- Preserve the separation between visual state and expected result.\n- Treat scenario IDs as durable API identifiers.\n- Reset scenario-local difficulty when changing scenarios.\n- Keep Test Screen free of controls and metadata.\n- Prefer deterministic behavior over visual novelty.\n- Record raw outputs before normalization.\n- Do not classify transition-race frames as model failures.\n- Keep the main Roverwatch integration read-only until the contract is stable.\n- Document changes to schemas, IDs, timing, and synchronization behavior.\n- Add regression scenarios for every production vision failure worth reproducing.\n\n## Definition of done for the current spike\n\nThe first Roverwatch vision spike is complete when:\n\n- Vision Lab is streamed through OBS to Twitch;\n- Roverwatch captures screenshots from that Twitch stream;\n- Roverwatch recognizes blank, green circle, red square, and blue triangle states;\n- each capture is logged with S3 key, capture timestamp, Vision Lab scenario ID, and revision;\n- Twitch chat receives the normalized detection;\n- repeated runs can reproduce and explain failures;\n- transition-window frames are excluded or labeled ambiguous.\n\nAt that point, the project has proven the full transport and inference loop. More realistic HUD tests should be added only after the simple pipeline is reliable.\n\n\n---\n\n## Latency evaluation mode\n\nLatency evaluation is an additive subsystem for measuring how quickly a published Vision Lab revision reaches the browser, is committed by React, appears in a rendered frame, and is later captured by Roverwatch. It does not replace Control Mode, Test Screen, the immutable transition history, or the normal scenario catalog.\n\nUse these URLs:\n\n| URL | Purpose | Visible controls |\n| --- | --- | --- |\n| `/` | Control Mode, including latency-session controls and diagnostics | Yes |\n| `/test` | Normal scenario output for OBS and recognition tests | No |\n| `/latency?poll=100` | High-contrast latency target with revision beacon | No |\n| `/api/time` | Server clock sample for offset estimation | N/A |\n| `/api/beacon/1842` | Encoder/decoder diagnostic for one revision | N/A |\n\nLatency Mode defaults to a 100 ms poll interval. The only supported intervals are 100, 250, and 500 ms. Invalid query values fall back to 100 ms.\n\n### Why latency mode is separate\n\nThe normal Test Screen intentionally renders arbitrary shapes, text, HUDs, motion, noise, and occlusion. Those features are useful for recognition testing but make transport-latency measurement harder to decode. Latency Mode instead renders a deterministic full-screen signal whose identity is recoverable from a screenshot.\n\nGround truth and observations remain separate:\n\n- `vision_lab_transitions` is the authoritative, append-only record of what the server published.\n- `vision_lab_observations` is diagnostic evidence reported by a browser. It can be late, duplicated, absent, or affected by an incorrect client clock.\n- Roverwatch frame, inference, and chat timestamps belong in Roverwatch. They are joined to Vision Lab data after capture.\n\nNever promote an observation to ground truth, and never write Roverwatch's live decision into the transition history.\n\n### Timing model\n\nThe timestamps have deliberately narrow meanings:\n\n| Timestamp | Assigned by | Meaning | Authoritative |\n| --- | --- | --- | --- |\n| `scheduledAt` | Controller | When the controller intended the transition to occur | No |\n| `publishedAt` | Vision Lab server | When the new revision was atomically persisted and published | Yes, for server publication |\n| `clientReceivedAt` | Latency browser | When the state response containing a new revision was handled | Diagnostic |\n| `renderedAt` | Latency browser | After React committed that revision and two animation frames completed | Diagnostic |\n| `receivedAt` | Vision Lab server | When the browser observation POST reached the server | Diagnostic |\n| `capturedAt` | Roverwatch | When Roverwatch acquired the frame | Authoritative within Roverwatch |\n\nThe browser records `renderedAt` with a double `requestAnimationFrame`: the first callback follows the React commit; the second gives the browser an opportunity to paint that committed DOM. This is a practical paint proxy, not proof that OBS or Twitch has already captured the pixels.\n\nUseful derived intervals are:\n\n- controller scheduling delay: `publishedAt - scheduledAt`\n- poll delivery delay: `clientReceivedAt - publishedAt`\n- browser commit/paint delay: `renderedAt - clientReceivedAt`\n- observation return delay: `receivedAt - renderedAt`\n- display-to-capture delay: `capturedAt - renderedAt`\n- publication-to-capture responsiveness: `capturedAt - publishedAt`\n\nCross-machine subtraction is only meaningful after clock-offset handling. `/api/time` returns `serverTime` and `unixMs`. A client can estimate offset by sampling its local clock immediately before and after the request, using the midpoint as the approximate local time corresponding to the server response. Record round-trip time and prefer the lowest-RTT sample. This estimates offset; it is not clock synchronization.\n\n### Revision beacon specification\n\nEvery Latency Mode frame contains two representations of the same revision:\n\n1. a large OCR label, `REVISION `;\n2. a 6-row by 8-column binary beacon.\n\nThe binary layout is stable and versioned by the latency export schema:\n\n| Row | Bits | Purpose |\n| ---: | --- | --- |\n| 0 | `11111111` | solid orientation/header row |\n| 1 | `10101010` | alternating orientation/header row |\n| 2 | bits 31..24 | revision, most-significant byte |\n| 3 | bits 23..16 | revision |\n| 4 | bits 15..8 | revision |\n| 5 | bits 7..0 | revision, least-significant byte |\n\nBlack cells represent 1 and white cells represent 0. The value is an unsigned 32-bit integer in big-endian bit order. The page background alternates black and white on every revision, so even a downscaled or blurred video stream exposes a strong whole-frame transition.\n\nThe shared module `frontend/beacon.ts` contains `encodeRevision`, `decodeRevision`, and `verifyBeaconRoundTrip`. The diagnostic route `/api/beacon/:revision` runs the same codec and reports `decodedRevision` plus `selfTestPassed`.\n\nA screenshot decoder should:\n\n1. crop or rectify the beacon rectangle;\n2. locate the two header rows and correct orientation;\n3. sample the center of each data cell, avoiding grid boundaries;\n4. classify black as 1 and white as 0;\n5. concatenate rows 2 through 5;\n6. decode the 32-bit unsigned big-endian value;\n7. compare it with OCR when OCR is available;\n8. mark the frame ambiguous if the two methods disagree.\n\nDo not infer revision from background parity alone; parity only reveals whether a revision is odd or even.\n\n### Polling and failure behavior\n\nLatency Mode uses recursive `setTimeout`, scheduled only after the preceding fetch finishes. Requests therefore never overlap. This matters at a 100 ms interval: `setInterval` would otherwise accumulate concurrent requests when a response takes longer than the interval.\n\nOn a poll failure, the page:\n\n- keeps the last successfully rendered revision;\n- logs the failure to the browser console;\n- schedules the next attempt after the configured interval;\n- does not fabricate a new observation.\n\nA new observation is posted only when the revision changes. Reloading the page creates or reuses a session-scoped observer ID and may report the current revision once.\n\n### Running a latency sequence\n\nControl Mode has a **Latency evaluation** panel.\n\n1. Select a browser poll interval: 100, 250, or 500 ms.\n2. Select a transition interval: 250, 500, 1000, or 2000 ms.\n3. Select a transition count from 1 through 100.\n4. Open the supplied Latency Screen URL in a normal browser or OBS Browser Source.\n5. Start the sequence.\n6. Keep the latency target visible for the duration of the run.\n7. Stop early if needed, or allow the configured count to finish.\n8. Export the session JSON.\n\nThe controller publishes one immutable revision per step. Each latency scenario includes `latencySessionId`, `sequenceId`, and `sequenceIndex` in its expected result so joins remain explicit. The panel also displays the current revision and the latest `clientReceivedAt`, `renderedAt`, and server `receivedAt`.\n\nA 250 ms transition interval with 500 ms polling will necessarily skip some intermediate revisions. That is an expected experimental outcome, not a history failure: the server transition export still contains every published revision, while the browser observation set contains only revisions it actually received and rendered.\n\n### Latency session API\n\nStart a session:\n\n~~~http\nPOST /api/latency-sessions/start\nContent-Type: application/json\n\n{\n \"pollIntervalMs\": 100,\n \"transitionIntervalMs\": 500,\n \"count\": 20\n}\n~~~\n\nStop a session:\n\n~~~http\nPOST /api/latency-sessions//stop\n~~~\n\nExport a session:\n\n~~~http\nGET /api/latency-sessions//export\n~~~\n\nThe export keeps authoritative transitions and non-authoritative observations in separate objects:\n\n~~~json\n{\n \"schemaVersion\": \"vision-lab-latency-v1\",\n \"sessionId\": \"uuid\",\n \"visionLabVersion\": \"vision-lab-latency-v1\",\n \"startedAt\": \"ISO-8601\",\n \"endedAt\": \"ISO-8601 or null\",\n \"startingRevision\": 100,\n \"endingRevision\": 120,\n \"configuration\": {\n \"pollIntervalMs\": 100,\n \"transitionIntervalMs\": 500,\n \"count\": 20\n },\n \"groundTruth\": {\n \"authoritative\": true,\n \"transitions\": []\n },\n \"observations\": {\n \"authoritative\": false,\n \"records\": []\n }\n}\n~~~\n\nAn export made before Stop uses the current revision as its provisional ending revision and leaves `endedAt` null. For a finalized evaluation artifact, stop the session first.\n\n### Database schema additions\n\n`vision_lab_observations` retains its original columns for backward compatibility and adds:\n\n| Column | Type | Notes |\n| --- | --- | --- |\n| `client_received_at` | TEXT | ISO-8601 browser response-handling time |\n| `rendered_at` | TEXT | ISO-8601 double-rAF paint proxy |\n| `poll_duration_ms` | REAL | Fetch duration measured by the browser |\n| `poll_interval_ms` | INTEGER | 100, 250, or 500 |\n| `route` | TEXT | Normally `/latency`; helps separate Test Screen observations |\n| `session_id` | TEXT | Optional latency-session join key |\n\n`vision_lab_latency_sessions` stores session identity, start/end timestamps, revision bounds, configuration JSON, and the Vision Lab schema/version identifier. Schema changes are additive. Startup migrations tolerate columns that already exist, so redeployment does not destroy history.\n\nLegacy Test Screen posts containing only `observedAt` remain accepted. For those posts, the server uses `observedAt` as the compatibility value for the newer browser timestamps. New latency clients should always send the explicit fields.\n\n### Roverwatch correlation workflow\n\nFor each captured latency frame, Roverwatch should store:\n\n~~~ts\ninterface RoverwatchLatencyFrame {\n sessionId: string;\n frameId: string;\n capturedAt: string;\n s3Key?: string;\n beaconRevision?: number;\n ocrRevision?: number;\n decodeStatus: \"matched\" | \"beacon-only\" | \"ocr-only\" | \"ambiguous\" | \"unreadable\";\n inferenceStartedAt?: string;\n inferenceCompletedAt?: string;\n chatPublishedAt?: string;\n}\n~~~\n\nAfter the run:\n\n1. decode every captured frame;\n2. load the finalized Vision Lab latency export;\n3. join frames to `groundTruth.transitions` by revision;\n4. join browser diagnostics from `observations.records` by revision and session ID;\n5. preserve frames with unreadable or conflicting codes as explicit decode failures;\n6. calculate separate distributions for publication-to-browser, browser-to-capture, inference, and chat;\n7. report skipped revisions separately from incorrectly decoded revisions.\n\nA revision present in ground truth but absent from observations means the Latency Screen did not report rendering it. A revision present in observations but absent from the session's revision range indicates a session-tagging or export-boundary problem. Neither condition should be silently discarded.\n\n### OBS setup for latency runs\n\nUse a dedicated OBS Browser Source:\n\n- URL: `https://roverwatch-vision-lab.val.run/latency?poll=100`\n- width: 1920\n- height: 1080\n- no custom CSS that changes contrast, scale, or visibility\n- keep the source active during the entire session\n- disable source shutdown while hidden if switching scenes\n\nDo not crop the beacon out of the frame. If the Twitch or capture pipeline rescales video, verify the beacon remains large enough for center-cell sampling before collecting a full dataset.\n\n### Latency troubleshooting\n\n#### High `publishedAt -> clientReceivedAt`\n\nCheck the configured polling interval, request RTT, background-tab throttling, OBS Browser Source activity, server logs, and whether the previous request was slow. Expected polling wait alone ranges from nearly zero to roughly one poll interval.\n\n#### High `clientReceivedAt -> renderedAt`\n\nCheck main-thread contention, browser/OBS CPU load, React errors, GPU pressure, and frame throttling. The double-rAF measurement depends on the browser continuing to produce animation frames.\n\n#### High `renderedAt -> capturedAt`\n\nThe delay is outside Vision Lab's browser commit path. Inspect OBS capture, encoding, Twitch ingest/playback buffering, Roverwatch sampling cadence, and clock offset.\n\n#### Missing revisions\n\nCompare transition interval with poll interval first. A polling browser cannot guarantee observation of every state when transitions are faster than polling. Use the immutable transition list as the complete publication record.\n\n#### Beacon and OCR disagree\n\nTreat the frame as ambiguous. Check crop alignment, orientation, compression, thresholding, partial-frame capture during a transition, and whether OCR read a stale/blurred label.\n\n#### Observation timestamps appear out of order\n\nBrowser timestamps and server timestamps may come from different clocks. Estimate clock offset with repeated `/api/time` samples and retain RTT. Do not rewrite the raw timestamps in the export.\n\n### Latency verification checklist\n\n- `/latency?poll=100` returns HTTP 200 and contains no controls.\n- The whole-screen background alternates on each revision.\n- The OCR label and binary beacon encode the same revision.\n- `/api/beacon/0`, `/api/beacon/1`, `/api/beacon/1842`, and `/api/beacon/4294967295` round-trip.\n- Browser polling never has more than one state request in flight.\n- A changed revision produces exactly one observation per open latency client.\n- `clientReceivedAt <= renderedAt` for a client observation.\n- `receivedAt` is assigned by the server and is not accepted from the client.\n- Session export contains all authoritative transitions in the revision range.\n- Session export keeps observations in a separately labeled non-authoritative collection.\n- Existing `/`, `/test`, `/current.json`, `/api/history`, `/api/state-at`, and legacy observation posts still work.\n- Moving Target followed by a normal preset resets motion to stationary.\n- Partially Occluded Target followed by a normal preset resets occlusion to zero.\n\n### Maintenance rules for latency code\n\n- Keep the beacon layout stable within a schema version. Introduce a new version before changing rows, bit order, colors, or headers.\n- Keep `publishedAt` server-assigned inside the atomic transition write.\n- Keep `renderedAt` tied to a revision-specific React commit and double-rAF.\n- Keep polling single-flight; do not replace recursive scheduling with an unguarded interval.\n- Preserve raw timestamps and raw decoded values.\n- Keep latency exports self-describing with configuration, revision bounds, and version.\n- Keep experimental diagnostics out of authoritative ground truth.\n- Add codec round-trip cases whenever the beacon implementation changes.\n- Verify old routes and legacy observations after every schema migration.\n\n\n---\n\n# Beacon Pixel Contract\n\nThis section is the definitive, immutable pixel protocol for `vision-lab-latency-v1`. It supersedes any earlier geometry implied by responsive CSS. A decoder can be implemented from this section and `GET /api/beacon-contract` without reading renderer code.\n\n## Canonical frame and beacon rectangle\n\nThe canonical latency frame is exactly 1920×1080 pixels. Coordinates use a top-left origin, with x increasing rightward and y increasing downward.\n\n| Property | 1920×1080 source pixels | Normalized | Uniform 1280×720 |\n| --- | ---: | ---: | ---: |\n| x | 240 | 0.125 | 160 |\n| y | 300 | 0.2777777777777778 | 200 |\n| width | 1440 | 0.75 | 960 |\n| height | 720 | 0.6666666666666666 | 480 |\n\nNormalization is against the complete frame: `x/1920`, `y/1080`, `width/1920`, and `height/1080`. The primary contract is the canonical/normalized geometry. The 1280×720 values are the exact result of uniform 2/3 scaling.\n\nThe latency renderer is a full-frame SVG with `viewBox=\"0 0 1920 1080\"`. A complete 16:9 capture therefore preserves this coordinate system under uniform scaling.\n\n## Grid geometry\n\nThe beacon contains 6 rows and 8 columns. Rows are indexed top-to-bottom as 0–5. Columns are indexed left-to-right as 0–7.\n\nAt 1920×1080:\n\n| Property | Pixels |\n| --- | ---: |\n| border width | 16 |\n| outer padding after border | 12 |\n| first cell x | 268 |\n| first cell y | 328 |\n| cell width | 166 |\n| cell height | 104 |\n| horizontal gap | 8 |\n| vertical gap | 8 |\n| column stride | 174 |\n| row stride | 112 |\n\nThe canonical rectangle for cell `(row, column)` is:\n\n~~~text\ncellX = 268 + column * 174\ncellY = 328 + row * 112\ncellWidth = 166\ncellHeight = 104\n~~~\n\nThe canonical center sample is:\n\n~~~text\nsampleX = 268 + column * 174 + 83\nsampleY = 328 + row * 112 + 52\n~~~\n\nFor any supported complete 16:9 frame of width `W` and height `H`:\n\n~~~text\nscaledSampleX = sampleX * W / 1920\nscaledSampleY = sampleY * H / 1080\npixelX = floor(scaledSampleX)\npixelY = floor(scaledSampleY)\n~~~\n\nAt 1280×720, border width is 10.6667 px, padding is 8 px, horizontal gap is 5.3333 px, vertical gap is 5.3333 px, cell size is 110.6667×69.3333 px, and the first cell begins at (178.6667, 218.6667).\n\nConcrete acceptance example: row 3, column 5 has canonical rectangle `x=1138, y=664, width=166, height=104` and center `(1221,716)`. At 1280×720 its rectangle is approximately `x=758.6667, y=442.6667, width=110.6667, height=69.3333`; its floating center is `(814,477.3333)`, so the sampled raster pixel is `(814,477)`.\n\nOnly center-point sampling is normative. Decoders must not sample borders, padding, gaps, or cell edges.\n\n## Colors and classification\n\nCanonical sRGB colors are:\n\n| Element | CSS color | RGB |\n| --- | --- | --- |\n| bit 1 cell | `#000000` | (0,0,0) |\n| bit 0 cell | `#ffffff` | (255,255,255) |\n| beacon border, padding, and gaps | `#808080` | (128,128,128) |\n| even-revision page background | `#000000` | (0,0,0) |\n| odd-revision page background | `#ffffff` | (255,255,255) |\n\nThe fixed gray beacon surround keeps the grid boundary distinguishable from either alternating page background. Page-background parity is diagnostic only and must not be used to recover the full revision.\n\nClassify each center sample with sRGB luminance:\n\n~~~text\nluminance = 0.2126 * R + 0.7152 * G + 0.0722 * B\n~~~\n\nThe thresholds are normative:\n\n~~~text\nluminance <= 96 => black => bit 1\nluminance >= 160 => white => bit 0\n96 < luminance < 160 => ambiguous cell\n~~~\n\nIf any sampled cell is ambiguous, return `ambiguous-cell` and no revision. Do not round an ambiguous sample toward either bit.\n\n## Bit ordering and validation\n\nRows and columns are read in row-major order: top-to-bottom, then left-to-right.\n\n~~~text\nrow 0: 11111111 orientation/header\nrow 1: 10101010 orientation/header\nrow 2: bits 31..24 left-to-right\nrow 3: bits 23..16 left-to-right\nrow 4: bits 15..8 left-to-right\nrow 5: bits 7..0 left-to-right\n~~~\n\nThe payload is an unsigned 32-bit integer. Bit 31 is the most-significant bit and bit 0 is the least-significant bit. Byte and bit order are big-endian/MSB-first.\n\nValidation fields are intentionally absent:\n\n~~~text\nchecksum: none\nparity: none\ncomplement bits: none\nerror correction: none\n~~~\n\nHeader mismatch returns `invalid-header`; it must never return a revision. Future validation or layout changes require a new schema version.\n\nRevision 1842 is:\n\n~~~text\ndecimal: 1842\nhexadecimal: 0x00000732\nbinary: 00000000 00000000 00000111 00110010\n\nrow 2: 00000000\nrow 3: 00000000\nrow 4: 00000111\nrow 5: 00110010\n~~~\n\n## Supported and unsupported transformations\n\nThe following are validated by committed fixtures:\n\n- complete 1920×1080 lossless PNG;\n- complete 1280×720 LANCZOS-resized PNG;\n- complete 1280×720 JPEG at quality 75 with 4:2:0 chroma subsampling.\n\nUniform resizing of a complete 16:9 frame is supported when coordinates are scaled from the canonical frame. JPEG compression comparable to the quality-75 fixtures is supported.\n\nThe following are unsupported and must not be silently decoded:\n\n- a cropped frame or cropped beacon;\n- non-uniform aspect-ratio distortion;\n- letterboxing unless the bars are removed and the active 16:9 image bounds are known;\n- rotation or perspective distortion;\n- a relocated beacon;\n- minor blur beyond what happens incidentally in the validated resize/JPEG fixtures;\n- layouts with a frame aspect ratio differing from 16:9 by more than 0.001.\n\n## Reference implementation and fixtures\n\nThe logical bit codec remains in `frontend/beacon.ts`.\n\nThe independent pixel contract and reference pixel decoder are in:\n\n~~~text\nfrontend/beaconContract.ts\n~~~\n\nThe decoder returns `decoded`, `invalid-header`, `ambiguous-cell`, `out-of-bounds`, or `unsupported-layout`, and never returns a revision for malformed input.\n\nRound-trip and corruption tests are in:\n\n~~~text\ntests/beaconPixelDecoder.test.ts\n~~~\n\nByte-exact committed fixtures are in:\n\n~~~text\ntest-fixtures/beacon-v1/\n~~~\n\nVal Town project files are text-backed, so PNG/JPEG bytes are committed as `.base64` files. Decode them with `base64 --decode .base64 > `. The fixture README lists SHA-256 hashes for every reconstructed image.\n\nThe fixture matrix contains revisions 0, 1, 1842, and 4294967295 in all three validated forms: canonical source PNG, resized 1280×720 PNG, and compressed 1280×720 quality-75 JPEG.\n\n## Machine-readable contract and version safety\n\n`GET /api/beacon-contract` returns the exact canonical frame, beacon rectangle, normalized rectangle, 1280×720 rectangle, grid geometry, colors, payload ordering, validation policy, center-sampling formula, luminance thresholds, and transformation support.\n\nLatency-session exports contain both:\n\n~~~json\n{\n \"schemaVersion\": \"vision-lab-latency-v1\",\n \"beaconSchemaVersion\": \"vision-lab-latency-v1\"\n}\n~~~\n\n`vision-lab-latency-v1` is now immutable. Any future change to dimensions, placement, cell geometry, headers, payload meaning, bit order, colors, thresholds, validation rules, or supported geometry must introduce a new beacon schema such as `vision-lab-latency-v2`. Do not modify v1 in place.\n",[148],"e8c03566-51a9-483d-9966-c36948fb0bf3","CHANGELOG.md","01a0495a-b924-761c-aebd-29a0c7fb0190",1,["D",1787937077281],"# Changelog\n\n## 2026-07-30 — Immutable asset caching\n\nFrontend files are now served at version-addressed URLs\n(`/__immutable/45/frontend/index.tsx`) with\n`Cache-Control: public, max-age=31536000, immutable`, so repeat visits load the\nwhole client graph from the browser cache — roughly 3× faster (~665ms → ~157ms\nmeasured). Publishing your val bumps its version, which changes every URL, so\ncaches invalidate automatically. Old-version URLs return 404 after a publish\n(like Next.js build assets); a reload picks up the new version.\n\nWhat changed in the template:\n\n- `frontend/index.html` → [`frontend/root.tsx`](/frontend/root.tsx): the HTML\n shell is now JSX and stamps asset URLs with `immutableFileUrl()`\n- [`index.ts`](/index.ts): serves `Root()` at `/`, and one\n `app.get(\"/__immutable/*\", (c) => serveImmutableFile(c.req.path))` route\n serves all frontend files\n\n### Upgrading an existing remix\n\nYour copy of this template doesn't update automatically. Two options:\n\n**Minimal (most of the benefit):** point both routes at `serveImmutableFile` —\nyour existing bare URLs redirect into versioned space, the new route serves\nthem, and the whole module graph gets cached (costs one redirect per page view):\n\n```ts\nimport { serveImmutableFile } from \"https://esm.town/v/std/utils/index.ts\";\n\napp.get(\"/__immutable/*\", (c) => serveImmutableFile(c.req.path));\napp.get(\"/frontend/**/*\", (c) => serveImmutableFile(c.req.path)); // was serveFile\n```\n\n**Full upgrade (also removes the redirect):** three self-contained steps —\nno other files needed. Adjust the ``, favicon, and body of `Root()` to\nmatch your app's existing `index.html`, and stamp anything else your shell\nreferences directly (images, css) with `immutableFileUrl()`.\n\n1. Create `frontend/root.tsx`:\n\n```tsx\n/** @jsxImportSource npm:hono@4/jsx */\nimport { raw } from \"npm:hono@4/html\";\nimport { immutableFileUrl } from \"https://esm.town/v/std/utils/index.ts\";\n\n// The HTML shell for the app. It is never cached, so immutableFileUrl can stamp\n// the current val version into each asset URL — the browser caches those\n// immutably, and publishing the val changes the URLs, refetching exactly once.\nexport function Root() {\n return (\n <>\n {raw(\"<!DOCTYPE html>\")}\n <html lang=\"en\">\n <head>\n <meta charSet=\"UTF-8\" />\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\" />\n <title>React Hono Val Town Starter\n

Diff · v76