OpenAI Realtime API Voice Agent

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_261":262,"_263":264},"routes/_app.x.$handle.$name",{"_99":98,"_32":45,"_108":86,"_27":109,"_13":14,"_110":-5,"_11":111,"_112":113,"_114":115,"_116":117,"_118":119,"_20":-5,"_120":121,"_122":123,"_124":25,"_125":126,"_127":59,"_128":-5,"_129":-5,"_130":131},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":57,"_58":59,"_60":-5,"_61":-5,"_62":63,"_64":65,"_66":-5,"_67":-5,"_68":-5,"_69":-5,"_70":71,"_72":71,"_73":74,"_75":74,"_76":74,"_77":71,"_78":71,"_79":80,"_81":82,"_83":84,"_85":25},"version",3,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":33,"_34":20,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":33,"_34":20,"_35":36},"deleted",false,"deletedHere","path","frontend/index.html","latestPath","fileId","019e419b-c582-779d-87fe-e2f2fcae3c28","name","index.html","fileType","content","\n\n \n \n \n OpenAI Realtime API Voice Agent\n \n \n \n \n \n\n

\n\n

\n\n \n \n","\n\n \n \n \n Voice Video Clipper\n \n \n \n \n\n

\n \n 📹 Click to upload a video (or drop one anywhere on this page)\n \n \n

\n No video loaded\n \n \n

\n\n

\n Clips\n \n

No clips yet. Upload a video, click \"Start talking\", and tell me what to clip.

\n\n

\n\n \n \n\n","createdAt",["D",1779217383520],"branchVersion","branchId","e84effa0-53b4-11f1-978a-ee650bb23af1","id","e848c068-53b4-11f1-978a-ee650bb23af1","voice-video-clipper",["D",1779217188136],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Sample app for the OpenAI Realtime API","imageUrl","9319c175-2593-4f88-ba21-04f2ef7b1c28","pinnedFileId","019e419b-c582-779d-87ff-06103d749f1b","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","d663a026-6449-42f6-9156-1d4a2d64d38c","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_99":98,"_107":103},"org",{"_99":98,"_100":101,"_102":103,"_18":55,"_104":105,"_106":25},"branch",{"_13":14,"_32":86,"_43":42,"_87":44,"_88":89,"_90":91,"_92":50,"_93":94,"_95":65,"_96":45,"_97":98},"isOwned","main","projectId","forkedBranchId","9851274e-539b-11f1-b21e-ee650bb23af1","forkedBranchVersion",9,"forkedProjectPrivacy","forkedProjectId","984a3ce0-539b-11f1-b21e-ee650bb23af1","forkedProjectOrgId","projectName","userHandle","dcm31","handle","displayName","Charlie","avatar","https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/3e904faa-3764-4629-c475-f561a49b0000/square512?exp=1789577999&sig=ccae2bd9e85fa66acf31b16bebb92bf56e22818ff77cfbe2634e1b690ccd40dc","tier","pro","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_43":44,"_32":45,"_38":257,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":57,"_58":59,"_60":-5,"_61":-5,"_62":63,"_64":65,"_66":-5,"_67":-5,"_68":-5,"_69":-5,"_70":71,"_72":71,"_73":74,"_75":74,"_76":74,"_77":71,"_78":71,"_79":258,"_81":259,"_83":260,"_85":25},"tree",[190,192,198,199,200],"files",[190,191,192,193,194,195,196,197,198,199,200],"counts",{"_179":63,"_180":150,"_181":63,"_182":183,"_184":185},"contents",[141,142,143,144,145],"upstreamProject",{"_43":94,"_32":139,"_52":53,"_54":-5,"_49":50,"_79":140},"upstreamBranch",{"_43":89,"_38":132,"_133":134,"_32":86,"_87":94,"_13":91,"_88":135,"_90":136,"_137":138,"_47":-5,"_85":25,"_49":50,"_97":98,"_64":65,"_96":139},"emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},["D",1779206316505],"updatedAt",["D",1779209326110],"53c6585e-373c-11f1-83b9-42b51c65c3df",50,"lockfileKey","1779209355978-esm.town_v_dcm31_hello-realtime-townie3@9-main","hello-realtime-townie3",{"_99":98,"_107":103},{"_43":174,"_32":175,"_30":176,"_149":-5,"_41":42,"_40":150,"_151":59,"_38":177,"_13":63,"_34":172,"_35":-5,"_24":25,"_155":-5,"_27":178},{"_43":168,"_32":169,"_30":170,"_149":-5,"_41":42,"_40":150,"_151":59,"_38":171,"_13":63,"_34":172,"_35":-5,"_24":25,"_155":-5,"_27":173},{"_43":162,"_32":163,"_30":164,"_149":-5,"_41":42,"_40":150,"_151":59,"_38":165,"_13":63,"_34":20,"_35":166,"_24":25,"_155":-5,"_27":167},{"_43":157,"_32":158,"_30":57,"_149":-5,"_41":42,"_40":150,"_151":59,"_38":159,"_13":63,"_34":153,"_35":160,"_24":25,"_155":-5,"_27":161},{"_43":146,"_32":147,"_30":148,"_149":-5,"_41":42,"_40":150,"_151":59,"_38":152,"_13":63,"_34":153,"_35":154,"_24":25,"_155":-5,"_27":156},"e859b0bc-53b4-11f1-978a-ee650bb23af1","oauth-test.tsx","019e419b-c582-779d-87ff-092435708fcd","parentId",1,"latest",["D",1779217188136],"http","/** @jsxImportSource https://esm.sh/hono/jsx */\nimport { Hono } from \"https://esm.sh/hono\";\nimport {\n getOAuthUserData,\n oauthMiddleware,\n} from \"https://esm.town/v/std/oauth/middleware.ts\";\n\nconst app = new Hono();\n\napp.get(\"/\", async (c) => {\n const session = await getOAuthUserData(c.req.raw);\n\n return c.html(\n \n \n {session?.user\n ?

Logged in as {session.user.username}

\n : Log in}\n \n ,\n );\n});\n\nexport default oauthMiddleware(app.fetch);\n","label",[147],"e8599460-53b4-11f1-978a-ee650bb23af1","main.tsx",["D",1779217188136],"import { Hono } from \"npm:hono\";\nimport { blob } from \"https://esm.town/v/std/blob\";\nimport { parseVal, serveFile } from \"https://esm.town/v/std/utils/index.ts\";\nimport {\n getOAuthUserData,\n oauthMiddleware,\n} from \"https://esm.town/v/std/oauth/middleware.ts\";\nimport rtc from \"./routes/rtc.ts\";\nimport sip from \"./routes/sip.ts\";\nimport observer from \"./routes/observer.ts\";\n\nconst SOURCE_URL = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\")).links.self.val;\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.route(\"/sip\", sip);\napp.route(\"/observer\", observer);\napp.get(\"/source\", (c) => c.redirect(SOURCE_URL));\n\n// Lightweight session probe used by the frontend to render login state.\napp.get(\"/me\", async (c) => {\n const session = await getOAuthUserData(c.req.raw);\n if (!session) return c.json({ loggedIn: false });\n return c.json({\n loggedIn: true,\n username: session.user.username,\n profileImageUrl: session.user.links.profileImageUrl,\n });\n});\n\napp.get(\"/\", () => serveFile(\"/frontend/index.html\", import.meta.url));\napp.get(\"/favicon.ico\", async (c) => {\n const entry = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\"));\n const file = await blob.get(`${entry.name}/favicon.ico`);\n if (!file) {\n return c.text(\"favicon not found\", 404);\n }\n return new Response(file.body, {\n headers: {\n \"Content-Type\": \"image/x-icon\",\n \"Cache-Control\": \"public, max-age=31536000, immutable\",\n },\n });\n});\n\napp.onError((err) => Promise.reject(err));\n\n// Wrap the app with std/oauth so users log in with their Val Town account.\n// The user's access token is then passed through to the embedded Val Town\n// MCP server (api.val.town/v3/mcp) so it can act on their behalf.\n//\n// Valid scopes (from /oauth/.well-known/openid-configuration):\n// openid, offline_access, profile, email,\n// user_r, user_rw, val_r, val_rw, project_rw,\n// blob_rw, sqlite_rw, email_rw, telemetry_r, telemetry_rw\n//\n// Note: if you change scopes and login starts failing with what looks like a\n// redirect loop, delete the cached OAuth client blob:\n// blob key: oauth_client:/auth/callback\n// (val.town's OAuth issuer can invalidate clients that attempted invalid\n// scopes, and std/oauth caches the dead client_id in blob storage forever.)\n//\n// Available routes added by the middleware:\n// GET /auth/login – start the OAuth flow\n// GET /auth/callback – OAuth callback (handled internally)\n// POST /auth/logout – clear the session cookie\n// GET /auth/logo – Val Town logo SVG (use ?theme=light for dark bg)\nexport default oauthMiddleware(\n {\n scopes: [\n \"openid\",\n \"offline_access\",\n \"profile\",\n \"user_r\",\n \"val_rw\",\n \"project_rw\",\n \"blob_rw\",\n \"sqlite_rw\",\n \"telemetry_rw\",\n ],\n },\n app.fetch,\n);\n",[158],"e8598150-53b4-11f1-978a-ee650bb23af1","README.md","019e419b-c582-779d-87ff-007f3537dd11",["D",1779217188136],"# hello-realtime\n\n**Hello Realtime** is a OpenAI Realtime app that supports both WebRTC and SIP\n(telephone) users. You can access the app via WebRTC at\n[hello-realtime.val.run](https://hello-realtime.val.run), or via SIP by calling\n425-800-0042.\n\nThis demo shows off the new SIP API, the new all-in-one WebRTC API, and the new\nserver-side websocket interface.\n\nIf you remix the app, you'll just need to pop in your own `OPENAI_API_KEY` (from\n[platform.openai.com](https://platform.openai.com)), and if you want SIP, the\n`OPENAI_SIGNING_SECRET`.\n\n## Architecture\n\n1. **WebRTC Flow**:\n - Browser connects to frontend\n - creates WebRTC offer\n - `/rtc` endpoint handles SDP negotiation with OpenAI\n - observer established to monitor session\n2. **SIP Flow**:\n - Phone calls triggers webhook\n - `/sip` endpoint verifies webhook and accepts call\n - observer established to monitor session\n\n## API Endpoints\n\n- `GET /` - serves the WebRTC-based app\n- `POST /rtc` - creates a new WebRTC-based Realtime API session\n - Supports optional `model` URL parameter to override the default model\n - Example: `POST /rtc?model=gpt-4o-realtime-preview`\n- `POST /sip` - handles an incoming Realtime API SIP call\n- `POST /observer` - internal endpoint to establish a Realtime API websocket for\n session control\n\n## Project Structure\n\n```\n├── frontend/ \n│ └── index.html # HTML/JS application for WebRTC\n├── main.tsx # Main entrypoint for the val.town app\n└── routes/\n |\n ├── agent.ts # Realtime API session configuration and tools\n ├── observer.ts # Server-side session control for SIP & WebRTC\n ├── rtc.ts # WebRTC session setup\n ├── sip.ts # SIP webhook handling and session setup\n └── utils.ts # Various HTTP helpers\n```\n\n## File Links\n\n- [`frontend/index.html`](./frontend/index.html)\n- [`main.tsx`](./main.tsx)\n- [`routes/agent.ts`](./routes/agent.ts)\n- [`routes/observer.ts`](./routes/observer.ts)\n- [`routes/rtc.ts`](./routes/rtc.ts)\n- [`routes/sip.ts`](./routes/sip.ts)\n- [`routes/utils.ts`](./routes/utils.ts)",[163],"e8597d9a-53b4-11f1-978a-ee650bb23af1","routes","019e419b-c582-779d-87fe-fd695d1a1239",["D",1779217188136],"directory",[169],"e8590a4a-53b4-11f1-978a-ee650bb23af1","frontend","019e419b-c582-779d-87fe-e53cc091d3b4",["D",1779217188136],[175],"pulls","branches","forks","envVars",{"_186":188,"_189":150},"connections",{"_186":63,"_187":63},"count","unfilledCount",2,"emptyCount",{"_43":174,"_32":175,"_30":176,"_149":-5,"_41":42,"_40":150,"_151":59,"_38":254,"_13":63,"_34":172,"_35":-5,"_24":25,"_155":-5,"_202":255,"_27":256},{"_43":250,"_32":33,"_30":31,"_149":176,"_41":42,"_40":14,"_151":59,"_38":251,"_13":150,"_34":20,"_35":37,"_24":25,"_155":-5,"_202":252,"_27":253},{"_43":168,"_32":169,"_30":170,"_149":-5,"_41":42,"_40":150,"_151":59,"_38":247,"_13":63,"_34":172,"_35":-5,"_24":25,"_155":-5,"_202":248,"_27":249},{"_43":240,"_32":241,"_30":242,"_149":170,"_41":42,"_40":188,"_151":59,"_38":243,"_13":150,"_34":215,"_35":244,"_24":25,"_155":-5,"_202":245,"_27":246},{"_43":233,"_32":234,"_30":235,"_149":170,"_41":42,"_40":150,"_151":59,"_38":236,"_13":63,"_34":215,"_35":237,"_24":25,"_155":-5,"_202":238,"_27":239},{"_43":226,"_32":227,"_30":228,"_149":170,"_41":42,"_40":150,"_151":59,"_38":229,"_13":63,"_34":215,"_35":230,"_24":25,"_155":-5,"_202":231,"_27":232},{"_43":219,"_32":220,"_30":221,"_149":170,"_41":42,"_40":150,"_151":59,"_38":222,"_13":63,"_34":215,"_35":223,"_24":25,"_155":-5,"_202":224,"_27":225},{"_43":211,"_32":212,"_30":213,"_149":170,"_41":42,"_40":150,"_151":59,"_38":214,"_13":63,"_34":215,"_35":216,"_24":25,"_155":-5,"_202":217,"_27":218},{"_43":162,"_32":163,"_30":164,"_149":-5,"_41":42,"_40":150,"_151":59,"_38":208,"_13":63,"_34":20,"_35":166,"_24":25,"_155":-5,"_202":209,"_27":210},{"_43":157,"_32":158,"_30":57,"_149":-5,"_41":42,"_40":150,"_151":59,"_38":205,"_13":63,"_34":153,"_35":160,"_24":25,"_155":-5,"_202":206,"_27":207},{"_43":146,"_32":147,"_30":148,"_149":-5,"_41":42,"_40":150,"_151":59,"_38":201,"_13":63,"_34":153,"_35":154,"_24":25,"_155":-5,"_202":203,"_27":204},["D",1779217188136],"children",[],[147],["D",1779217188136],[],[158],["D",1779217188136],[],[163],"e85974a8-53b4-11f1-978a-ee650bb23af1","utils.ts","019e419b-c582-779d-87fe-fb249fb0a835",["D",1779217188136],"script","const REALTIME_BASE_URL = \"https://api.openai.com/v1/realtime\";\nconst OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\n// Builds the URL for a Realtime API endpoint.\nexport function makeUrl(callId?: string, action?: string) {\n let url = REALTIME_BASE_URL;\n if (!callId) {\n // start call (POST)\n url += \"/calls\";\n } else if (action) {\n // accept, reject, hangup (POST)\n url += `/calls/${callId}/${action}`;\n } else {\n // observer (web socket)\n url += `?call_id=${callId}`;\n }\n return url;\n}\n\n// Builds the headers for a Realtime API endpoint.\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\n// Debug helper for printing Realtime API errors.\nexport async function getErrorText(resp: Response) {\n const errBody = await resp.text().catch(() => \"\");\n return `${resp.status} ${resp.statusText}: ${errBody}`;\n}",[],[169,212],"e8595dec-53b4-11f1-978a-ee650bb23af1","sip.ts","019e419b-c582-779d-87fe-f5896ac1ce40",["D",1779217188136],"import { Hono } from \"npm:hono\";\nimport { Webhook } from \"npm:svix\";\nimport { makeSession } from \"./agent.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst sip = new Hono();\n\n// POST /sip : webhook endpoint\nsip.post(\"/\", async (c) => {\n // Verify the webhook.\n const OPENAI_SIGNING_SECRET = Deno.env.get(\"OPENAI_SIGNING_SECRET\");\n if (!OPENAI_SIGNING_SECRET) {\n console.error(\"🔴 webhook secret not configured\");\n return c.text(\"Internal error\", 500);\n }\n const webhook = new Webhook(OPENAI_SIGNING_SECRET);\n const bodyStr = await c.req.text();\n let callId: string | undefined;\n let toNumber: string | undefined;\n try {\n const headers = Object.fromEntries(c.req.raw.headers.entries());\n const payload: any = webhook.verify(bodyStr, headers);\n callId = payload.data.call_id;\n toNumber = getToNumber(payload.data.sip_headers);\n const fromNumber = getFromNumber(payload.data.sip_headers);\n console.log(\"✅ verified webhook:\", callId, fromNumber, toNumber);\n } catch (error) {\n console.error(`🔴 verify failed: ${error}`);\n return c.text(\"Invalid signature\", 401);\n }\n\n // Accept the call. SIP calls come from phone users — there's no logged-in\n // Val Town user, so we don't attach an MCP token. The agent will run with\n // only the built-in `display_text` function tool (and no val.town MCP).\n const model = getModelForNumber(toNumber);\n const url = makeUrl(callId, \"accept\");\n const headers = makeHeaders(\"application/json\");\n const body = JSON.stringify(makeSession(model));\n const opts = { method: \"POST\", headers, body };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n console.error(`🔴 accept failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n console.log(\"✅ call accepted:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n return c.newResponse(null);\n});\n\nconst getToNumber = headers => getNumberFromSipHeader(headers, \"to\");\nconst getFromNumber = headers => getNumberFromSipHeader(headers, \"from\");\n\nfunction getNumberFromSipHeader(headers: any, name: string) {\n const header = getSipHeader(headers, name);\n const match = header?.match(/sip:\\+?(\\d+)@/);\n return match ? `+${match[1]}` : undefined;\n}\n\nfunction getSipHeader(headers: any, name: string) {\n return headers.find((h) => h.name.toLowerCase() === name)?.value;\n}\n\nfunction getModelForNumber(number: string) {\n const map = { \"+14258000074\": \"gpt-realtime-mini\" };\n return map[number];\n}\n\nexport default sip;",[],[169,220],"e859423a-53b4-11f1-978a-ee650bb23af1","rtc.ts","019e419b-c582-779d-87fe-f015dab949c8",["D",1779217188136],"import { Hono } from \"npm:hono\";\nimport { getOAuthUserData } from \"https://esm.town/v/std/oauth/middleware.ts\";\nimport { makeSession } from \"./agent.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new call\nrtc.post(\"/\", async (c) => {\n // Require a logged-in Val Town user — their OAuth access token is used\n // to authenticate the embedded MCP tool against api.val.town.\n const session = await getOAuthUserData(c.req.raw);\n if (!session) {\n return c.text(\"Unauthorized — log in at /auth/login first\", 401);\n }\n\n // Extract model parameter from URL if provided\n const model = c.req.query(\"model\");\n if (model !== undefined) {\n console.log(\"🔧 Model parameter received:\", model);\n }\n\n // Parse video flag from query (?video=true)\n const video = c.req.query(\"video\") === \"true\";\n if (video) {\n console.log(\"📹 Video mode enabled\");\n }\n\n console.log(`🔑 starting call for @${session.user.username}`);\n\n // Create the call.\n const fd = new FormData();\n fd.set(\"sdp\", await c.req.text());\n fd.set(\n \"session\",\n JSON.stringify(makeSession(model, video, session.accessToken)),\n );\n const opts = { method: \"POST\", headers: makeHeaders(), body: fd };\n const resp = await fetch(makeUrl(), opts);\n if (!resp.ok) {\n console.error(`🔴 start call failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n\n const contentType = resp.headers.get(\"Content-Type\");\n const location = resp.headers.get(\"Location\");\n const callId = location?.split(\"/\").pop();\n console.log(\"✅ call created:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n const responseHeaders = {\n \"Content-Type\": contentType,\n \"Location\": location,\n };\n return c.newResponse(resp.body, { headers: responseHeaders });\n});\n\nexport default rtc;\n",[],[169,227],"e8592fc0-53b4-11f1-978a-ee650bb23af1","observer.ts","019e419b-c582-779d-87fe-ef89ac69ce5d",["D",1779217188136],"import { Hono } from \"npm:hono\";\nimport WebSocket from \"npm:ws\";\nimport { createHandler } from \"./agent.ts\";\nimport { makeHeaders, makeUrl } from \"./utils.ts\";\n\nexport interface ObserverHandler {\n onInit(): void;\n onTool(name: string, args: any): void;\n onMcpCallCompleted(name: string, output: string | null): void;\n onOutputAudioStarted(): void;\n onOutputAudioStopped(): void;\n onOutputAudioCleared(): void;\n}\n\nexport class ObserverClient {\n private ws: WebSocket;\n private closeResolve: () => void;\n private closePromise: Promise;\n private handler: ObserverHandler;\n\n constructor(callId: string) {\n const url = makeUrl(callId);\n this.ws = new WebSocket(url, { headers: makeHeaders() });\n this.ws.on(\"open\", () => this.handleOpen());\n this.ws.on(\"message\", (data) => this.handleMessage(JSON.parse(data)));\n this.ws.on(\"error\", (err) => this.handleError(err));\n this.ws.on(\"close\", (ev) => this.handleClose(ev));\n this.closePromise = new Promise((resolve) =>\n this.closeResolve = resolve\n );\n this.handler = createHandler(this, callId);\n }\n createResponse() {\n this.sendMessage({ type: \"response.create\" });\n }\n async waitForClose() {\n await this.closePromise;\n }\n private sendMessage(message: any) {\n this.ws.send(JSON.stringify(message));\n }\n private handleOpen() {\n console.log(\"✅ websocket connected\");\n this.handler.onInit();\n }\n private async handleMessage(message: any) {\n if (!message.type.endsWith(\".delta\")) { // ignore verbose messages\n console.log(message);\n }\n switch (message.type) {\n case \"response.function_call_arguments.done\":\n this.handler.onTool(message.name, message.arguments);\n break;\n case \"response.output_item.done\":\n if (message.item?.type === \"mcp_call\") {\n this.handler.onMcpCallCompleted(message.item.name, message.item.output);\n }\n break;\n case \"output_audio_buffer.started\":\n this.handler.onOutputAudioStarted();\n break;\n case \"output_audio_buffer.stopped\":\n this.handler.onOutputAudioStopped();\n break;\n case \"output_audio_buffer.cleared\":\n this.handler.onOutputAudioCleared();\n break;\n }\n }\n private handleError(err: Error) {\n console.error(`🔴 websocket failed: ${err.message}`);\n this.closeResolve();\n }\n private handleClose(e: Event) {\n console.log(`websocket closed: ${e}`);\n this.closeResolve();\n }\n}\n\nconst observer = new Hono();\n\n// POST /observer/:callId : self-connect to establish a web socket\nobserver.post(\"/:callId\", async (c) => {\n const callId = c.req.param(\"callId\");\n const client = new ObserverClient(callId);\n await client.waitForClose();\n return c.newResponse(null);\n});\n\nexport default observer;",[],[169,234],"1c0923a2-53b5-11f1-978d-ee650bb23af1","agent.ts","019e419b-c582-779d-87fe-e9ee7c10f210",["D",1779217274951],"import { ObserverClient, ObserverHandler } from \"./observer.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst MODEL = \"gpt-realtime-2\";\nconst TRANSCRIBE_MODEL = \"gpt-4o-transcribe\";\nconst NR_TYPE = \"near_field\";\nconst VOICE = \"marin\";\n\nconst SYSTEM_PROMPT =\n `You are Clipper, a voice-controlled video editing assistant.\n\nThe user has uploaded a video in the browser and wants to make clips from it\nusing their voice. You can call tools to add clips, play clips back, clear\nclips, or check the current state of the video player. All clipping is\nnon-destructive — clips are just numbered markers (start, end, optional label)\nthat the user can play back.\n\nBe terse. The user is watching the video; long-winded responses get in the way.\nConfirm actions in one short sentence. Don't list every clip back unless asked.\n\nTimes are always in absolute seconds from the start of the video. If the user\nsays \"right here\", \"now\", \"from where I am\", or anything positional, call\nget_video_state first to read currentTime, then use that number. If the user\nsays \"for N seconds\" without an end time, treat that as a clip starting at the\ncurrent playhead and ending N seconds later.\n\nAfter you add a clip, briefly confirm: \"Clip : s to s.\"\nWhen playing a clip, say \"Playing clip \" or similar — one short line.\nIf no video is loaded yet, tell the user to upload one first.\n\nTools:\n- get_video_state: current time, duration, paused, filename, and clips list.\n- add_clip(start, end, label?): record a new clip in seconds.\n- play_clip(number): jump to and play a specific clip (1-indexed).\n- play_all: play all clips in order.\n- clear_clips: remove all clips.\n`;\n\n// Builds the declarative session configuration for a Realtime API session.\n// `_accessToken` is accepted for backward compat with the rtc route but unused —\n// this val doesn't need any MCP tools.\nexport function makeSession(\n modelOverride?: string,\n _video?: boolean,\n _accessToken?: string,\n) {\n const model = modelOverride || MODEL;\n\n const tools: any[] = [\n {\n type: \"function\",\n name: \"get_video_state\",\n description:\n \"Read the current state of the video player: currentTime (seconds), duration (seconds), paused (bool), filename (string), and the existing clips list. Call this whenever the user references 'now', 'here', or any positional language that needs the current playhead.\",\n parameters: { type: \"object\", properties: {} },\n },\n {\n type: \"function\",\n name: \"add_clip\",\n description:\n \"Add a new clip to the clips list. Times are absolute seconds from the start of the video. The clip is numbered automatically in order of creation. Use a short label if the user describes the clip ('intro', 'punchline', etc.).\",\n parameters: {\n type: \"object\",\n properties: {\n start: {\n type: \"number\",\n description: \"Clip start time in seconds.\",\n },\n end: { type: \"number\", description: \"Clip end time in seconds.\" },\n label: { type: \"string\", description: \"Optional short label.\" },\n },\n required: [\"start\", \"end\"],\n },\n },\n {\n type: \"function\",\n name: \"play_clip\",\n description: \"Play a specific clip by its number (1-indexed).\",\n parameters: {\n type: \"object\",\n properties: {\n number: {\n type: \"integer\",\n description: \"The clip number to play, starting from 1.\",\n },\n },\n required: [\"number\"],\n },\n },\n {\n type: \"function\",\n name: \"play_all\",\n description: \"Play through all clips in order.\",\n parameters: { type: \"object\", properties: {} },\n },\n {\n type: \"function\",\n name: \"clear_clips\",\n description: \"Remove all existing clips and start over.\",\n parameters: { type: \"object\", properties: {} },\n },\n ];\n\n return {\n type: \"realtime\",\n model,\n instructions: SYSTEM_PROMPT,\n audio: {\n input: {\n noise_reduction: { type: NR_TYPE },\n transcription: { model: TRANSCRIBE_MODEL },\n },\n output: { voice: VOICE },\n },\n tools,\n };\n}\n\n// AgentHandler implements the agent runtime behavior (tool calling, etc).\n// Tool calls are actually handled in the browser over the data channel — the\n// server-side observer just logs them.\nclass AgentHandler implements ObserverHandler {\n private callEnding = false;\n constructor(private client: ObserverClient, private callId: string) {}\n onInit() {}\n async onTool(name: string, _args: any) {\n console.log(\"🔧 function tool call:\", name, _args);\n }\n async onMcpCallCompleted(name: string, output: string | null) {\n console.log(\"🔧 MCP call completed:\", name, output?.slice(0, 200));\n this.client.createResponse();\n }\n async onOutputAudioStarted() {}\n async onOutputAudioStopped() {\n if (this.callEnding) await this.hangup();\n }\n async onOutputAudioCleared() {}\n private async hangup() {\n const url = makeUrl(this.callId, \"hangup\");\n const resp = await fetch(url, { method: \"POST\", headers: makeHeaders() });\n if (!resp.ok) {\n console.error(`🔴 end call failed: ${await getErrorText(resp)}`);\n return;\n }\n console.log(\"✅ call ended\");\n }\n}\n\nexport function createHandler(client: ObserverClient, callId: string) {\n return new AgentHandler(client, callId);\n}\n",[],[169,241],["D",1779217188136],[193,194,195,196,197],[169],"5cbe45d0-53b5-11f1-aa3f-ee650bb23af1",["D",1779217383520],[],[175,33],["D",1779217188136],[191],[175],["D",1779217188136],{"_99":98,"_107":103},{"_99":98,"_100":101,"_102":103,"_18":55,"_104":105,"_106":25},{"_13":14,"_32":86,"_43":42,"_87":44,"_88":89,"_90":91,"_92":50,"_93":94,"_95":65,"_96":45,"_97":98},"ctx",{"_79":-5,"_129":294,"_295":296,"_297":298,"_299":25,"_300":25,"_301":25,"_302":59,"_303":-5,"_304":-7,"_305":-7,"_306":63,"_307":25,"_308":25,"_309":-5,"_310":25,"_311":25,"_312":25,"_313":-5},"ENV",{"_265":266,"_267":268,"_269":86,"_270":271,"_272":273,"_274":275,"_276":277,"_278":279,"_280":281,"_282":283,"_284":285,"_286":287,"_288":289,"_290":291,"_292":293},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Realtime Agent

Voice Video Clipper

\n Clips\n ▶ Play all\n

\n Clips\n \n

Diff · v3