OpenAI Realtime API Voice Agent

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_270":271,"_272":273},"routes/_app.x.$handle.$name",{"_111":110,"_38":52,"_119":97,"_33":120,"_13":14,"_121":-5,"_11":122,"_123":124,"_125":126,"_127":128,"_129":130,"_26":-5,"_131":132,"_133":134,"_135":31,"_136":137,"_138":66,"_139":-5,"_140":-5,"_141":142},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":18,"_19":-5},"project",{"_50":51,"_38":52,"_45":53,"_54":-5,"_55":31,"_56":57,"_58":57,"_59":60,"_61":-5,"_19":62,"_63":64,"_65":66,"_67":-5,"_68":-5,"_69":70,"_71":72,"_73":74,"_75":76,"_77":78,"_79":-5,"_80":81,"_82":81,"_83":84,"_85":84,"_86":84,"_87":81,"_88":81,"_89":90,"_91":92,"_93":94,"_95":31},"version",28,"changes",[25],"aiChatInfo",{"_20":21,"_22":-5,"_23":-5,"_24":-5},"userId","aiChatMessageId","ab2c33bf-db8a-4a23-b63f-a1528c9ae75a","aiChatThreadId","aiChatThreadUserId","aiChatThreadTitle",{"_26":27,"_28":29,"_30":31,"_32":31,"_33":34,"_35":34},"file",{"_36":37,"_38":39,"_40":41,"_42":44,"_45":46,"_47":14,"_48":49,"_30":31},"previous",{"_36":37,"_38":39,"_40":41,"_42":43},"deleted",false,"deletedHere","path","routes/agent.ts","latestPath","fileId","019d8705-54fa-7761-a068-8d4dc61ae936","name","agent.ts","fileType","script","content","import { ObserverClient, ObserverHandler } from \"./observer.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst MODEL = \"gpt-realtime-1.5\";\nconst TRANSCRIBE_MODEL = \"gpt-4o-transcribe\";\nconst NR_TYPE = \"near_field\";\n\nconst towniePrompt =\n await (await fetch(\"https://www.val.town/townie/system-prompt\"))\n .text();\n\nconst VOICE = \"marin\";\n\n// Builds the declarative session configuration for a Realtime API session.\nexport function makeSession(modelOverride?: string) {\n const model = modelOverride || MODEL;\n return {\n type: \"realtime\",\n model,\n instructions: towniePrompt,\n audio: {\n input: {\n noise_reduction: { type: NR_TYPE },\n transcription: { model: TRANSCRIBE_MODEL },\n },\n output: { voice: VOICE },\n },\n tools: [{\n type: \"mcp\",\n server_label: \"val_town_mcp\",\n server_url: \"https://api.val.town/v3/mcp\",\n require_approval: \"never\",\n headers: {\n \"Authorization\": `Bearer ${Deno.env.get(\"VAL_TOWN_API_KEY_2\")}`,\n },\n }],\n };\n}\n\n// AgentHandler implements the agent runtime behavior (tool calling, etc).\nclass AgentHandler implements ObserverHandler {\n private audioActive = false;\n private callEnding = false;\n constructor(private client: ObserverClient, private callId: string) {}\n onInit() {\n // Trigger a response immediately.\n // this.client.createResponse();\n }\n async onTool(name: string, _args: any) {\n console.log(\"🔧 function tool call:\", name, _args);\n }\n async onMcpCallCompleted(name: string, output: string | null) {\n console.log(\"🔧 MCP call completed:\", name, output?.slice(0, 200));\n // After an MCP tool call completes, the model has the results in context\n // but won't automatically generate a follow-up response. We need to\n // explicitly trigger a new response so it speaks the results.\n this.client.createResponse();\n }\n async onOutputAudioStarted() {\n this.audioActive = true;\n }\n async onOutputAudioStopped() {\n // Shut down the call and websocket once audio has finished spooling.\n this.audioActive = false;\n if (this.callEnding) {\n await this.hangup();\n }\n }\n async onOutputAudioCleared() {\n this.audioActive = false;\n }\n private async hangup() {\n const url = makeUrl(this.callId, \"hangup\");\n const resp = await fetch(url, { method: \"POST\", headers: makeHeaders() });\n if (!resp.ok) {\n console.error(`🔴 end call failed: ${await getErrorText(resp)}`);\n return;\n }\n\n console.log(\"✅ call ended\");\n }\n}\n\n// Creates the runtime handler for the session.\nexport function createHandler(client: ObserverClient, callId: string) {\n return new AgentHandler(client, callId);\n}","import { ObserverClient, ObserverHandler } from \"./observer.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst MODEL = \"gpt-realtime-1.5\";\nconst TRANSCRIBE_MODEL = \"gpt-4o-transcribe\";\nconst NR_TYPE = \"near_field\";\n\nconst towniePrompt =\n await (await fetch(\"https://www.val.town/townie/system-prompt\"))\n .text();\n\nconst VOICE = \"marin\";\n\n// Builds the declarative session configuration for a Realtime API session.\nexport function makeSession(modelOverride?: string, video?: boolean) {\n const model = modelOverride || MODEL;\n const session: Record = {\n type: \"realtime\",\n model,\n instructions: towniePrompt,\n audio: {\n input: {\n noise_reduction: { type: NR_TYPE },\n transcription: { model: TRANSCRIBE_MODEL },\n },\n output: { voice: VOICE },\n },\n tools: [{\n type: \"mcp\",\n server_label: \"val_town_mcp\",\n server_url: \"https://api.val.town/v3/mcp\",\n require_approval: \"never\",\n headers: {\n \"Authorization\": `Bearer ${Deno.env.get(\"VAL_TOWN_API_KEY_2\")}`,\n },\n }],\n };\n\n if (video) {\n session.video = { input: true };\n }\n\n return session;\n}\n\n// AgentHandler implements the agent runtime behavior (tool calling, etc).\nclass AgentHandler implements ObserverHandler {\n private audioActive = false;\n private callEnding = false;\n constructor(private client: ObserverClient, private callId: string) {}\n onInit() {\n // Trigger a response immediately.\n // this.client.createResponse();\n }\n async onTool(name: string, _args: any) {\n console.log(\"🔧 function tool call:\", name, _args);\n }\n async onMcpCallCompleted(name: string, output: string | null) {\n console.log(\"🔧 MCP call completed:\", name, output?.slice(0, 200));\n // After an MCP tool call completes, the model has the results in context\n // but won't automatically generate a follow-up response. We need to\n // explicitly trigger a new response so it speaks the results.\n this.client.createResponse();\n }\n async onOutputAudioStarted() {\n this.audioActive = true;\n }\n async onOutputAudioStopped() {\n // Shut down the call and websocket once audio has finished spooling.\n this.audioActive = false;\n if (this.callEnding) {\n await this.hangup();\n }\n }\n async onOutputAudioCleared() {\n this.audioActive = false;\n }\n private async hangup() {\n const url = makeUrl(this.callId, \"hangup\");\n const resp = await fetch(url, { method: \"POST\", headers: makeHeaders() });\n if (!resp.ok) {\n console.error(`🔴 end call failed: ${await getErrorText(resp)}`);\n return;\n }\n\n console.log(\"✅ call ended\");\n }\n}\n\n// Creates the runtime handler for the session.\nexport function createHandler(client: ObserverClient, callId: string) {\n return new AgentHandler(client, callId);\n}","createdAt",["D",1776165205633],"branchVersion","branchId","53c6585e-373c-11f1-83b9-42b51c65c3df","id","53c0860e-373c-11f1-83b9-42b51c65c3df","hello-realtime-townie",["D",1776086766746],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Sample app for the OpenAI Realtime API","imageUrl","a0bf3b31-15a5-4d5c-880e-4b1e22c9bc18","pinnedFileId","019d8705-54fa-7761-a068-abcc39ccfece","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","e81e7335-6d4c-4764-a902-d0962bbfa9f7","tursoName","53c0860e373c11f183b942b51c65c3df","tursoHostname","53c0860e373c11f183b942b51c65c3df-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789500689328],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_111":110,"_118":114},"org",{"_111":110,"_112":-5,"_113":114,"_19":62,"_115":116,"_117":31},"branch",{"_13":96,"_38":97,"_50":49,"_98":51,"_99":100,"_101":102,"_103":57,"_104":105,"_106":107,"_108":52,"_109":110},"isOwned",50,"main","projectId","forkedBranchId","be33420e-792a-11f0-bfa3-0224a6c84d84","forkedBranchVersion",375,"forkedProjectPrivacy","forkedProjectId","be2e762a-792a-11f0-bfa3-0224a6c84d84","forkedProjectOrgId","9c125825-8df1-4029-a26d-29049e9bd5a3","projectName","userHandle","stevekrouse","handle","displayName","avatar","https://images.clerk.dev/uploaded/img_2PqHa2Gsy93xQrjh2w78Xu0cChW.jpeg","tier","pro","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_50":51,"_38":52,"_45":265,"_54":-5,"_55":31,"_56":57,"_58":57,"_59":60,"_61":-5,"_19":62,"_63":64,"_65":66,"_67":-5,"_68":-5,"_69":70,"_71":72,"_73":74,"_75":76,"_77":266,"_79":-5,"_80":81,"_82":81,"_83":84,"_85":84,"_86":84,"_87":81,"_88":81,"_89":267,"_91":268,"_93":269,"_95":31},"tree",[199,201,207,208],"files",[199,200,201,202,203,204,205,206,207,208],"counts",{"_187":162,"_188":189,"_190":191,"_192":193,"_194":195},"contents",[155,156,157,158],"upstreamProject",{"_50":105,"_38":152,"_59":60,"_61":-5,"_56":57,"_89":153},"upstreamBranch",{"_50":100,"_45":143,"_144":145,"_38":97,"_98":105,"_13":146,"_99":147,"_101":148,"_149":150,"_54":-5,"_95":31,"_56":57,"_109":151,"_71":107,"_108":152},"emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},["D",1755188493466],"updatedAt",["D",1778178264384],379,"e004ac78-4230-11f0-834a-76b3cceeab13",47,"lockfileKey","1778178275619-esm.town_v_jubertioai_hello-realtime@379-main","jubertioai","hello-realtime",{"_111":151,"_118":154},"https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/61901958-12bd-49a3-43e9-2f67f1617200/square512?exp=1789570799&sig=616111f0677b4a4ab67bf546207d7d26f480315db3de8da944cc4ad5af25c76a",{"_50":182,"_38":183,"_36":184,"_161":-5,"_48":49,"_47":162,"_163":66,"_45":185,"_13":70,"_40":180,"_42":-5,"_30":31,"_167":-5,"_33":186},{"_50":176,"_38":177,"_36":178,"_161":-5,"_48":49,"_47":162,"_163":66,"_45":179,"_13":70,"_40":180,"_42":-5,"_30":31,"_167":-5,"_33":181},{"_50":170,"_38":171,"_36":172,"_161":-5,"_48":49,"_47":162,"_163":66,"_45":173,"_13":70,"_40":26,"_42":174,"_30":31,"_167":-5,"_33":175},{"_50":159,"_38":160,"_36":64,"_161":-5,"_48":49,"_47":162,"_163":31,"_45":164,"_13":70,"_40":165,"_42":166,"_30":31,"_167":168,"_33":169},"53cbc8d4-373c-11f1-83b9-42b51c65c3df","main.tsx","parentId",1,"latest",["D",1776086766746],"http","import { Hono } from \"npm:hono\";\nimport { blob } from \"https://esm.town/v/std/blob\";\nimport { parseVal, serveFile } from \"https://esm.town/v/std/utils/index.ts\";\n// import mcp from \"./routes/mcp.ts\";\nimport rtc from \"./routes/rtc.ts\";\nimport sip from \"./routes/sip.ts\";\nimport observer from \"./routes/observer.ts\";\n\nconst SOURCE_URL = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\")).links.self.val;\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.route(\"/sip\", sip);\napp.route(\"/observer\", observer);\n// app.route(\"/mcp\", mcp);\napp.get(\"/source\", (c) => c.redirect(SOURCE_URL));\n\napp.get(\"/\", () => serveFile(\"/frontend/index.html\", import.meta.url));\napp.get(\"/favicon.ico\", async (c) => {\n const entry = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\"));\n const file = await blob.get(`${entry.name}/favicon.ico`);\n if (!file) {\n return c.text(\"favicon not found\", 404);\n }\n return new Response(file.body, {\n headers: {\n \"Content-Type\": \"image/x-icon\",\n \"Cache-Control\": \"public, max-age=31536000, immutable\",\n },\n });\n}); \n \nexport default app.fetch;","label","townie-realtime",[160],"53cbbc86-373c-11f1-83b9-42b51c65c3df","README.md","019d8705-54fa-7761-a068-a5181925a024",["D",1776086766746],"# hello-realtime\n\n**Hello Realtime** is a OpenAI Realtime app that supports both WebRTC and SIP\n(telephone) users. You can access the app via WebRTC at\n[hello-realtime.val.run](https://hello-realtime.val.run), or via SIP by calling\n425-800-0042.\n\nThis demo shows off the new SIP API, the new all-in-one WebRTC API, and the new\nserver-side websocket interface.\n\nIf you remix the app, you'll just need to pop in your own `OPENAI_API_KEY` (from\n[platform.openai.com](https://platform.openai.com)), and if you want SIP, the\n`OPENAI_SIGNING_SECRET`.\n\n## Architecture\n\n1. **WebRTC Flow**:\n - Browser connects to frontend\n - creates WebRTC offer\n - `/rtc` endpoint handles SDP negotiation with OpenAI\n - observer established to monitor session\n2. **SIP Flow**:\n - Phone calls triggers webhook\n - `/sip` endpoint verifies webhook and accepts call\n - observer established to monitor session\n\n## API Endpoints\n\n- `GET /` - serves the WebRTC-based app\n- `POST /rtc` - creates a new WebRTC-based Realtime API session\n - Supports optional `model` URL parameter to override the default model\n - Example: `POST /rtc?model=gpt-4o-realtime-preview`\n- `POST /sip` - handles an incoming Realtime API SIP call\n- `POST /observer` - internal endpoint to establish a Realtime API websocket for\n session control\n\n## Project Structure\n\n```\n├── frontend/ \n│ └── index.html # HTML/JS application for WebRTC\n├── main.tsx # Main entrypoint for the val.town app\n└── routes/\n |\n ├── agent.ts # Realtime API session configuration and tools\n ├── observer.ts # Server-side session control for SIP & WebRTC\n ├── rtc.ts # WebRTC session setup\n ├── sip.ts # SIP webhook handling and session setup\n └── utils.ts # Various HTTP helpers\n```\n\n## File Links\n\n- [`frontend/index.html`](./frontend/index.html)\n- [`main.tsx`](./main.tsx)\n- [`routes/agent.ts`](./routes/agent.ts)\n- [`routes/observer.ts`](./routes/observer.ts)\n- [`routes/rtc.ts`](./routes/rtc.ts)\n- [`routes/sip.ts`](./routes/sip.ts)\n- [`routes/utils.ts`](./routes/utils.ts)",[171],"53cbbb0a-373c-11f1-83b9-42b51c65c3df","routes","019d8705-54fa-7761-a068-a3313eec009e",["D",1776086766746],"directory",[177],"53cb7bae-373c-11f1-83b9-42b51c65c3df","frontend","019d8705-54fa-7761-a068-88c6188f174a",["D",1776086766746],[183],"pulls","branches",2,"forks",3,"envVars",{"_196":189,"_198":70},"connections",{"_196":70,"_197":70},"count","unfilledCount","emptyCount",{"_50":182,"_38":183,"_36":184,"_161":-5,"_48":49,"_47":162,"_163":66,"_45":262,"_13":70,"_40":180,"_42":-5,"_30":31,"_167":-5,"_210":263,"_33":264},{"_50":254,"_38":255,"_36":256,"_161":184,"_48":49,"_47":257,"_163":31,"_45":258,"_13":162,"_40":26,"_42":259,"_30":31,"_167":-5,"_210":260,"_33":261},{"_50":176,"_38":177,"_36":178,"_161":-5,"_48":49,"_47":162,"_163":66,"_45":251,"_13":70,"_40":180,"_42":-5,"_30":31,"_167":-5,"_210":252,"_33":253},{"_50":246,"_38":39,"_36":37,"_161":178,"_48":49,"_47":14,"_163":31,"_45":247,"_13":248,"_40":41,"_42":44,"_30":31,"_167":-5,"_210":249,"_33":250},{"_50":238,"_38":239,"_36":240,"_161":178,"_48":49,"_47":241,"_163":66,"_45":242,"_13":189,"_40":41,"_42":243,"_30":31,"_167":-5,"_210":244,"_33":245},{"_50":230,"_38":231,"_36":232,"_161":178,"_48":49,"_47":233,"_163":31,"_45":234,"_13":162,"_40":41,"_42":235,"_30":31,"_167":-5,"_210":236,"_33":237},{"_50":223,"_38":224,"_36":225,"_161":178,"_48":49,"_47":162,"_163":31,"_45":226,"_13":70,"_40":41,"_42":227,"_30":31,"_167":-5,"_210":228,"_33":229},{"_50":216,"_38":217,"_36":218,"_161":178,"_48":49,"_47":162,"_163":66,"_45":219,"_13":70,"_40":41,"_42":220,"_30":31,"_167":-5,"_210":221,"_33":222},{"_50":170,"_38":171,"_36":172,"_161":-5,"_48":49,"_47":162,"_163":66,"_45":213,"_13":70,"_40":26,"_42":174,"_30":31,"_167":-5,"_210":214,"_33":215},{"_50":159,"_38":160,"_36":64,"_161":-5,"_48":49,"_47":162,"_163":31,"_45":209,"_13":70,"_40":165,"_42":166,"_30":31,"_167":168,"_210":211,"_33":212},["D",1776086766746],"children",[],[160],["D",1776086766746],[],[171],"53cbb448-373c-11f1-83b9-42b51c65c3df","utils.ts","019d8705-54fa-7761-a068-9c94857ce1ed",["D",1776086766746],"const REALTIME_BASE_URL = \"https://api.openai.com/v1/realtime\";\nconst OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\n// Builds the URL for a Realtime API endpoint.\nexport function makeUrl(callId?: string, action?: string) {\n let url = REALTIME_BASE_URL;\n if (!callId) {\n // start call (POST)\n url += \"/calls\";\n } else if (action) {\n // accept, reject, hangup (POST)\n url += `/calls/${callId}/${action}`;\n } else {\n // observer (web socket)\n url += `?call_id=${callId}`;\n }\n return url;\n}\n\n// Builds the headers for a Realtime API endpoint.\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\n// Debug helper for printing Realtime API errors.\nexport async function getErrorText(resp: Response) {\n const errBody = await resp.text().catch(() => \"\");\n return `${resp.status} ${resp.statusText}: ${errBody}`;\n}",[],[177,217],"53cba44e-373c-11f1-83b9-42b51c65c3df","sip.ts","019d8705-54fa-7761-a068-9bc0320eecb7",["D",1776086766746],"import { Hono } from \"npm:hono\";\nimport { Webhook } from \"npm:svix\";\nimport { makeSession } from \"./agent.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst sip = new Hono();\n\n// POST /sip : webhook endpoint\nsip.post(\"/\", async (c) => {\n // Verify the webhook.\n const OPENAI_SIGNING_SECRET = Deno.env.get(\"OPENAI_SIGNING_SECRET\");\n if (!OPENAI_SIGNING_SECRET) {\n console.error(\"🔴 webhook secret not configured\");\n return c.text(\"Internal error\", 500);\n }\n const webhook = new Webhook(OPENAI_SIGNING_SECRET);\n const bodyStr = await c.req.text();\n let callId: string | undefined;\n let toNumber: string | undefined;\n try {\n const headers = Object.fromEntries(c.req.raw.headers.entries());\n const payload: any = webhook.verify(bodyStr, headers);\n callId = payload.data.call_id;\n toNumber = getToNumber(payload.data.sip_headers);\n const fromNumber = getFromNumber(payload.data.sip_headers);\n console.log(\"✅ verified webhook:\", callId, fromNumber, toNumber);\n } catch (error) {\n console.error(`🔴 verify failed: ${error}`);\n return c.text(\"Invalid signature\", 401);\n }\n\n // Accept the call.\n const model = getModelForNumber(toNumber);\n const url = makeUrl(callId, \"accept\");\n const headers = makeHeaders(\"application/json\");\n const body = JSON.stringify(makeSession(model));\n const opts = { method: \"POST\", headers, body };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n console.error(`🔴 accept failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n console.log(\"✅ call accepted:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n return c.newResponse(null);\n});\n\nconst getToNumber = headers => getNumberFromSipHeader(headers, \"to\");\nconst getFromNumber = headers => getNumberFromSipHeader(headers, \"from\");\n\nfunction getNumberFromSipHeader(headers: any, name: string) {\n const header = getSipHeader(headers, name);\n const match = header?.match(/sip:\\+?(\\d+)@/);\n return match ? `+${match[1]}` : undefined;\n}\n\nfunction getSipHeader(headers: any, name: string) {\n return headers.find((h) => h.name.toLowerCase() === name)?.value;\n}\n\nfunction getModelForNumber(number: string) {\n const map = { \"+14258000074\": \"gpt-realtime-mini\" };\n return map[number];\n}\n\nexport default sip;",[],[177,224],"eed15cac-37f2-11f1-8358-42b51c65c3df","rtc.ts","019d8705-54fa-7761-a068-976e7d341d86",27,["D",1776165195323],"import { Hono } from \"npm:hono\";\nimport { makeSession } from \"./agent.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new call\nrtc.post(\"/\", async (c) => {\n // Extract model parameter from URL if provided\n const model = c.req.query(\"model\");\n if (model !== undefined) {\n console.log(\"🔧 Model parameter received:\", model);\n }\n\n // Parse video flag from query (?video=true)\n const video = c.req.query(\"video\") === \"true\";\n if (video) {\n console.log(\"📹 Video mode enabled\");\n }\n \n // Create the call.\n const fd = new FormData();\n fd.set(\"sdp\", await c.req.text());\n fd.set(\"session\", JSON.stringify(makeSession(model, video)));\n const opts = { method: \"POST\", headers: makeHeaders(), body: fd };\n const resp = await fetch(makeUrl(), opts);\n if (!resp.ok) {\n console.error(`🔴 start call failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n\n const contentType = resp.headers.get(\"Content-Type\");\n const location = resp.headers.get(\"Location\");\n const callId = location?.split(\"/\").pop();\n console.log(\"✅ call created:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n const responseHeaders = {\n \"Content-Type\": contentType,\n \"Location\": location,\n };\n return c.newResponse(resp.body, { headers: responseHeaders });\n});\n\nexport default rtc;",[],[177,231],"7c29d172-37f0-11f1-880b-42b51c65c3df","observer.ts","019d8705-54fa-7761-a068-91275444e97e",24,["D",1776164143983],"import { Hono } from \"npm:hono\";\nimport WebSocket from \"npm:ws\";\nimport { createHandler } from \"./agent.ts\";\nimport { makeHeaders, makeUrl } from \"./utils.ts\";\n\nexport interface ObserverHandler {\n onInit(): void;\n onTool(name: string, args: any): void;\n onMcpCallCompleted(name: string, output: string | null): void;\n onOutputAudioStarted(): void;\n onOutputAudioStopped(): void;\n onOutputAudioCleared(): void;\n}\n\nexport class ObserverClient {\n private ws: WebSocket;\n private closeResolve: () => void;\n private closePromise: Promise;\n private handler: ObserverHandler;\n\n constructor(callId: string) {\n const url = makeUrl(callId);\n this.ws = new WebSocket(url, { headers: makeHeaders() });\n this.ws.on(\"open\", () => this.handleOpen());\n this.ws.on(\"message\", (data) => this.handleMessage(JSON.parse(data)));\n this.ws.on(\"error\", (err) => this.handleError(err));\n this.ws.on(\"close\", (ev) => this.handleClose(ev));\n this.closePromise = new Promise((resolve) =>\n this.closeResolve = resolve\n );\n this.handler = createHandler(this, callId);\n }\n createResponse() {\n this.sendMessage({ type: \"response.create\" });\n }\n async waitForClose() {\n await this.closePromise;\n }\n private sendMessage(message: any) {\n this.ws.send(JSON.stringify(message));\n }\n private handleOpen() {\n console.log(\"✅ websocket connected\");\n this.handler.onInit();\n }\n private async handleMessage(message: any) {\n if (!message.type.endsWith(\".delta\")) { // ignore verbose messages\n console.log(message);\n }\n switch (message.type) {\n case \"response.function_call_arguments.done\":\n this.handler.onTool(message.name, message.arguments);\n break;\n case \"response.output_item.done\":\n if (message.item?.type === \"mcp_call\") {\n this.handler.onMcpCallCompleted(message.item.name, message.item.output);\n }\n break;\n case \"output_audio_buffer.started\":\n this.handler.onOutputAudioStarted();\n break;\n case \"output_audio_buffer.stopped\":\n this.handler.onOutputAudioStopped();\n break;\n case \"output_audio_buffer.cleared\":\n this.handler.onOutputAudioCleared();\n break;\n }\n }\n private handleError(err: Error) {\n console.error(`🔴 websocket failed: ${err.message}`);\n this.closeResolve();\n }\n private handleClose(e: Event) {\n console.log(`websocket closed: ${e}`);\n this.closeResolve();\n }\n}\n\nconst observer = new Hono();\n\n// POST /observer/:callId : self-connect to establish a web socket\nobserver.post(\"/:callId\", async (c) => {\n const callId = c.req.param(\"callId\");\n const client = new ObserverClient(callId);\n await client.waitForClose();\n return c.newResponse(null);\n});\n\nexport default observer;",[],[177,239],"f4f4c7fe-37f2-11f1-8601-42b51c65c3df",["D",1776165205633],16,[],[177,39],["D",1776086766746],[202,203,204,205,206],[177],"ea457876-37f2-11f1-adac-42b51c65c3df","index.html","019d8705-54fa-7761-a068-8455826e2ed1",26,["D",1776165187700],"\n\n \n \n \n OpenAI Realtime API Voice Agent\n \n \n \n \n\n

\n\n

\n\n \n \n",[],[183,255],["D",1776086766746],[200],[183],["D",1776086766746],["D",1789500689328],{"_111":110,"_118":114},{"_111":110,"_112":-5,"_113":114,"_19":62,"_115":116,"_117":31},{"_13":96,"_38":97,"_50":49,"_98":51,"_99":100,"_101":102,"_103":57,"_104":105,"_106":107,"_108":52,"_109":110},"ctx",{"_89":-5,"_140":303,"_304":305,"_306":307,"_308":31,"_309":31,"_310":31,"_311":66,"_312":-5,"_313":-7,"_314":-7,"_315":70,"_316":31,"_317":31,"_318":-5,"_319":31,"_320":31,"_321":31,"_322":-5},"ENV",{"_274":275,"_276":277,"_278":97,"_279":280,"_281":282,"_283":284,"_285":286,"_287":288,"_289":290,"_291":292,"_293":294,"_295":296,"_297":298,"_299":300,"_301":302},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Realtime Agent

Diff · v28