OpenAI Realtime API Voice Agent

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_270":271,"_272":273},"routes/_app.x.$handle.$name",{"_114":113,"_33":55,"_122":100,"_28":123,"_13":14,"_124":-5,"_11":125,"_126":127,"_128":129,"_130":131,"_132":133,"_21":-5,"_134":135,"_136":137,"_138":26,"_139":140,"_141":69,"_142":-5,"_143":-5,"_144":145},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_53":54,"_33":55,"_40":56,"_57":-5,"_58":26,"_59":60,"_61":60,"_62":63,"_64":-5,"_18":65,"_66":67,"_68":69,"_70":-5,"_71":-5,"_72":73,"_74":75,"_76":77,"_78":79,"_80":81,"_82":-5,"_83":84,"_85":84,"_86":87,"_88":87,"_89":87,"_90":84,"_91":84,"_92":93,"_94":95,"_96":97,"_98":26},"version",31,"changes",[19,20],"aiChatInfo","userId",{"_21":45,"_23":46,"_25":26,"_27":26,"_28":47,"_30":47},{"_21":22,"_23":24,"_25":26,"_27":26,"_28":29,"_30":29},"file",{"_31":32,"_33":34,"_35":36,"_37":39,"_40":41,"_42":14,"_43":44,"_25":26},"previous",{"_31":32,"_33":34,"_35":36,"_37":38},"deleted",false,"deletedHere","path","routes/agent.ts","latestPath","fileId","019d8705-54fa-7761-a068-8d4dc61ae936","name","agent.ts","fileType","script","content","import { ObserverClient, ObserverHandler } from \"./observer.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst MODEL = \"gpt-realtime-1.5\";\nconst TRANSCRIBE_MODEL = \"gpt-4o-transcribe\";\nconst NR_TYPE = \"near_field\";\n\nconst towniePrompt =\n await (await fetch(\"https://www.val.town/townie/system-prompt\"))\n .text();\n\nconst VOICE = \"marin\";\n\n// Builds the declarative session configuration for a Realtime API session.\n// The video parameter is accepted but not added to the session config —\n// video input works automatically via the WebRTC video track; the model\n// receives video frames without any special session-level flag.\nexport function makeSession(modelOverride?: string, _video?: boolean) {\n const model = modelOverride || MODEL;\n return {\n type: \"realtime\",\n model,\n instructions: towniePrompt,\n audio: {\n input: {\n noise_reduction: { type: NR_TYPE },\n transcription: { model: TRANSCRIBE_MODEL },\n },\n output: { voice: VOICE },\n },\n tools: [{\n type: \"mcp\",\n server_label: \"val_town_mcp\",\n server_url: \"https://api.val.town/v3/mcp\",\n require_approval: \"never\",\n headers: {\n \"Authorization\": `Bearer ${Deno.env.get(\"VAL_TOWN_API_KEY_2\")}`,\n },\n }],\n };\n}\n\n// AgentHandler implements the agent runtime behavior (tool calling, etc).\nclass AgentHandler implements ObserverHandler {\n private audioActive = false;\n private callEnding = false;\n constructor(private client: ObserverClient, private callId: string) {}\n onInit() {\n // Trigger a response immediately.\n // this.client.createResponse();\n }\n async onTool(name: string, _args: any) {\n console.log(\"🔧 function tool call:\", name, _args);\n }\n async onMcpCallCompleted(name: string, output: string | null) {\n console.log(\"🔧 MCP call completed:\", name, output?.slice(0, 200));\n // After an MCP tool call completes, the model has the results in context\n // but won't automatically generate a follow-up response. We need to\n // explicitly trigger a new response so it speaks the results.\n this.client.createResponse();\n }\n async onOutputAudioStarted() {\n this.audioActive = true;\n }\n async onOutputAudioStopped() {\n // Shut down the call and websocket once audio has finished spooling.\n this.audioActive = false;\n if (this.callEnding) {\n await this.hangup();\n }\n }\n async onOutputAudioCleared() {\n this.audioActive = false;\n }\n private async hangup() {\n const url = makeUrl(this.callId, \"hangup\");\n const resp = await fetch(url, { method: \"POST\", headers: makeHeaders() });\n if (!resp.ok) {\n console.error(`🔴 end call failed: ${await getErrorText(resp)}`);\n return;\n }\n\n console.log(\"✅ call ended\");\n }\n}\n\n// Creates the runtime handler for the session.\nexport function createHandler(client: ObserverClient, callId: string) {\n return new AgentHandler(client, callId);\n}","import { ObserverClient, ObserverHandler } from \"./observer.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst MODEL = \"gpt-realtime-1.5\";\nconst TRANSCRIBE_MODEL = \"gpt-4o-transcribe\";\nconst NR_TYPE = \"near_field\";\n\nconst towniePrompt =\n await (await fetch(\"https://www.val.town/townie/system-prompt\"))\n .text();\n\nconst VOICE = \"marin\";\n\n// Builds the declarative session configuration for a Realtime API session.\n// The video parameter is accepted but not added to the session config —\n// video input works automatically via the WebRTC video track; the model\n// receives video frames without any special session-level flag.\nexport function makeSession(modelOverride?: string, _video?: boolean) {\n const model = modelOverride || MODEL;\n return {\n type: \"realtime\",\n model,\n instructions: towniePrompt,\n audio: {\n input: {\n noise_reduction: { type: NR_TYPE },\n transcription: { model: TRANSCRIBE_MODEL },\n },\n output: { voice: VOICE },\n },\n tools: [\n {\n type: \"mcp\",\n server_label: \"val_town_mcp\",\n server_url: \"https://api.val.town/v3/mcp\",\n require_approval: \"never\",\n headers: {\n \"Authorization\": `Bearer ${Deno.env.get(\"VAL_TOWN_API_KEY_2\")}`,\n },\n },\n {\n type: \"function\",\n name: \"display_text\",\n description:\n \"Display formatted text to the user's screen. Use this to show code snippets, links, lists, or any information that is better read than heard. The content supports full markdown including links, code blocks, bold, italic, lists, etc. Always use this tool when sharing URLs, code, or structured information.\",\n parameters: {\n type: \"object\",\n properties: {\n title: {\n type: \"string\",\n description: \"A short title for the displayed content\",\n },\n content: {\n type: \"string\",\n description:\n \"The markdown-formatted text content to display. Include links as [text](url), code as ```lang blocks, etc.\",\n },\n },\n required: [\"content\"],\n },\n },\n ],\n };\n}\n\n// AgentHandler implements the agent runtime behavior (tool calling, etc).\nclass AgentHandler implements ObserverHandler {\n private audioActive = false;\n private callEnding = false;\n constructor(private client: ObserverClient, private callId: string) {}\n onInit() {\n // Trigger a response immediately.\n // this.client.createResponse();\n }\n async onTool(name: string, _args: any) {\n console.log(\"🔧 function tool call:\", name, _args);\n }\n async onMcpCallCompleted(name: string, output: string | null) {\n console.log(\"🔧 MCP call completed:\", name, output?.slice(0, 200));\n // After an MCP tool call completes, the model has the results in context\n // but won't automatically generate a follow-up response. We need to\n // explicitly trigger a new response so it speaks the results.\n this.client.createResponse();\n }\n async onOutputAudioStarted() {\n this.audioActive = true;\n }\n async onOutputAudioStopped() {\n // Shut down the call and websocket once audio has finished spooling.\n this.audioActive = false;\n if (this.callEnding) {\n await this.hangup();\n }\n }\n async onOutputAudioCleared() {\n this.audioActive = false;\n }\n private async hangup() {\n const url = makeUrl(this.callId, \"hangup\");\n const resp = await fetch(url, { method: \"POST\", headers: makeHeaders() });\n if (!resp.ok) {\n console.error(`🔴 end call failed: ${await getErrorText(resp)}`);\n return;\n }\n\n console.log(\"✅ call ended\");\n }\n}\n\n// Creates the runtime handler for the session.\nexport function createHandler(client: ObserverClient, callId: string) {\n return new AgentHandler(client, callId);\n}","createdAt",["D",1776170817022],"branchVersion","branchId","53c6585e-373c-11f1-83b9-42b51c65c3df",{"_31":48,"_33":49,"_35":21,"_37":51,"_40":52,"_42":14,"_43":44,"_25":26},{"_31":48,"_33":49,"_35":21,"_37":50},"frontend/index.html","019d8705-54fa-7761-a068-8455826e2ed1","index.html","\n\n \n \n \n OpenAI Realtime API Voice Agent\n \n \n \n \n\n

\n\n

\n\n \n \n","\n\n \n \n \n OpenAI Realtime API Voice Agent\n \n \n \n \n \n\n

\n\n

\n\n \n \n",["D",1776170817022],"id","53c0860e-373c-11f1-83b9-42b51c65c3df","hello-realtime-townie",["D",1776086766746],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Sample app for the OpenAI Realtime API","imageUrl","a0bf3b31-15a5-4d5c-880e-4b1e22c9bc18","pinnedFileId","019d8705-54fa-7761-a068-abcc39ccfece","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","e81e7335-6d4c-4764-a902-d0962bbfa9f7","tursoName","53c0860e373c11f183b942b51c65c3df","tursoHostname","53c0860e373c11f183b942b51c65c3df-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789500689328],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_114":113,"_121":117},"org",{"_114":113,"_115":-5,"_116":117,"_18":65,"_118":119,"_120":26},"branch",{"_13":99,"_33":100,"_53":44,"_101":54,"_102":103,"_104":105,"_106":60,"_107":108,"_109":110,"_111":55,"_112":113},"isOwned",50,"main","projectId","forkedBranchId","be33420e-792a-11f0-bfa3-0224a6c84d84","forkedBranchVersion",375,"forkedProjectPrivacy","forkedProjectId","be2e762a-792a-11f0-bfa3-0224a6c84d84","forkedProjectOrgId","9c125825-8df1-4029-a26d-29049e9bd5a3","projectName","userHandle","stevekrouse","handle","displayName","avatar","https://images.clerk.dev/uploaded/img_2PqHa2Gsy93xQrjh2w78Xu0cChW.jpeg","tier","pro","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_53":54,"_33":55,"_40":265,"_57":-5,"_58":26,"_59":60,"_61":60,"_62":63,"_64":-5,"_18":65,"_66":67,"_68":69,"_70":-5,"_71":-5,"_72":73,"_74":75,"_76":77,"_78":79,"_80":266,"_82":-5,"_83":84,"_85":84,"_86":87,"_88":87,"_89":87,"_90":84,"_91":84,"_92":267,"_94":268,"_96":269,"_98":26},"tree",[203,205,211,212],"files",[203,204,205,206,207,208,209,210,211,212],"counts",{"_191":168,"_192":193,"_194":195,"_196":197,"_198":199},"contents",[158,159,160,161],"upstreamProject",{"_53":108,"_33":155,"_62":63,"_64":-5,"_59":60,"_92":156},"upstreamBranch",{"_53":103,"_40":146,"_147":148,"_33":100,"_101":108,"_13":149,"_102":150,"_104":151,"_152":153,"_57":-5,"_98":26,"_59":60,"_112":154,"_74":110,"_111":155},"emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},["D",1755188493466],"updatedAt",["D",1778178264384],379,"e004ac78-4230-11f0-834a-76b3cceeab13",47,"lockfileKey","1778178275619-esm.town_v_jubertioai_hello-realtime@379-main","jubertioai","hello-realtime",{"_114":154,"_121":157},"https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/61901958-12bd-49a3-43e9-2f67f1617200/square512?exp=1789549199&sig=c8fcde5d26ad4cf40ff1ad5a8c0321304f62c6a6a36d7f592fa053a5c65d6783",{"_53":186,"_33":187,"_31":188,"_164":-5,"_43":44,"_42":168,"_166":69,"_40":189,"_13":73,"_35":184,"_37":-5,"_25":26,"_171":-5,"_28":190},{"_53":180,"_33":181,"_31":182,"_164":-5,"_43":44,"_42":168,"_166":69,"_40":183,"_13":73,"_35":184,"_37":-5,"_25":26,"_171":-5,"_28":185},{"_53":174,"_33":175,"_31":176,"_164":-5,"_43":44,"_42":168,"_166":69,"_40":177,"_13":73,"_35":21,"_37":178,"_25":26,"_171":-5,"_28":179},{"_53":162,"_33":163,"_31":67,"_164":-5,"_43":44,"_42":165,"_166":26,"_40":167,"_13":168,"_35":169,"_37":170,"_25":26,"_171":172,"_28":173},"1ec452d8-37f4-11f1-baeb-42b51c65c3df","main.tsx","parentId",29,"latest",["D",1776165705281],1,"http","import { Hono } from \"npm:hono\";\nimport { blob } from \"https://esm.town/v/std/blob\";\nimport { parseVal, serveFile } from \"https://esm.town/v/std/utils/index.ts\";\n// import mcp from \"./routes/mcp.ts\";\nimport rtc from \"./routes/rtc.ts\";\nimport sip from \"./routes/sip.ts\";\nimport observer from \"./routes/observer.ts\";\n\nconst SOURCE_URL = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\")).links.self.val;\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.route(\"/sip\", sip);\napp.route(\"/observer\", observer);\n// app.route(\"/mcp\", mcp);\napp.get(\"/source\", (c) => c.redirect(SOURCE_URL));\n\napp.get(\"/\", () => serveFile(\"/frontend/index.html\", import.meta.url));\napp.get(\"/favicon.ico\", async (c) => {\n const entry = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\"));\n const file = await blob.get(`${entry.name}/favicon.ico`);\n if (!file) {\n return c.text(\"favicon not found\", 404);\n }\n return new Response(file.body, {\n headers: {\n \"Content-Type\": \"image/x-icon\",\n \"Cache-Control\": \"public, max-age=31536000, immutable\",\n },\n });\n});\n\napp.onError((err) => Promise.reject(err));\n\nexport default app.fetch;","label","townie-realtime",[163],"53cbbc86-373c-11f1-83b9-42b51c65c3df","README.md","019d8705-54fa-7761-a068-a5181925a024",["D",1776086766746],"# hello-realtime\n\n**Hello Realtime** is a OpenAI Realtime app that supports both WebRTC and SIP\n(telephone) users. You can access the app via WebRTC at\n[hello-realtime.val.run](https://hello-realtime.val.run), or via SIP by calling\n425-800-0042.\n\nThis demo shows off the new SIP API, the new all-in-one WebRTC API, and the new\nserver-side websocket interface.\n\nIf you remix the app, you'll just need to pop in your own `OPENAI_API_KEY` (from\n[platform.openai.com](https://platform.openai.com)), and if you want SIP, the\n`OPENAI_SIGNING_SECRET`.\n\n## Architecture\n\n1. **WebRTC Flow**:\n - Browser connects to frontend\n - creates WebRTC offer\n - `/rtc` endpoint handles SDP negotiation with OpenAI\n - observer established to monitor session\n2. **SIP Flow**:\n - Phone calls triggers webhook\n - `/sip` endpoint verifies webhook and accepts call\n - observer established to monitor session\n\n## API Endpoints\n\n- `GET /` - serves the WebRTC-based app\n- `POST /rtc` - creates a new WebRTC-based Realtime API session\n - Supports optional `model` URL parameter to override the default model\n - Example: `POST /rtc?model=gpt-4o-realtime-preview`\n- `POST /sip` - handles an incoming Realtime API SIP call\n- `POST /observer` - internal endpoint to establish a Realtime API websocket for\n session control\n\n## Project Structure\n\n```\n├── frontend/ \n│ └── index.html # HTML/JS application for WebRTC\n├── main.tsx # Main entrypoint for the val.town app\n└── routes/\n |\n ├── agent.ts # Realtime API session configuration and tools\n ├── observer.ts # Server-side session control for SIP & WebRTC\n ├── rtc.ts # WebRTC session setup\n ├── sip.ts # SIP webhook handling and session setup\n └── utils.ts # Various HTTP helpers\n```\n\n## File Links\n\n- [`frontend/index.html`](./frontend/index.html)\n- [`main.tsx`](./main.tsx)\n- [`routes/agent.ts`](./routes/agent.ts)\n- [`routes/observer.ts`](./routes/observer.ts)\n- [`routes/rtc.ts`](./routes/rtc.ts)\n- [`routes/sip.ts`](./routes/sip.ts)\n- [`routes/utils.ts`](./routes/utils.ts)",[175],"53cbbb0a-373c-11f1-83b9-42b51c65c3df","routes","019d8705-54fa-7761-a068-a3313eec009e",["D",1776086766746],"directory",[181],"53cb7bae-373c-11f1-83b9-42b51c65c3df","frontend","019d8705-54fa-7761-a068-88c6188f174a",["D",1776086766746],[187],"pulls","branches",2,"forks",3,"envVars",{"_200":193,"_202":73},"connections",{"_200":73,"_201":73},"count","unfilledCount","emptyCount",{"_53":186,"_33":187,"_31":188,"_164":-5,"_43":44,"_42":168,"_166":69,"_40":262,"_13":73,"_35":184,"_37":-5,"_25":26,"_171":-5,"_214":263,"_28":264},{"_53":258,"_33":49,"_31":48,"_164":188,"_43":44,"_42":14,"_166":26,"_40":259,"_13":193,"_35":21,"_37":51,"_25":26,"_171":-5,"_214":260,"_28":261},{"_53":180,"_33":181,"_31":182,"_164":-5,"_43":44,"_42":168,"_166":69,"_40":255,"_13":73,"_35":184,"_37":-5,"_25":26,"_171":-5,"_214":256,"_28":257},{"_53":250,"_33":34,"_31":32,"_164":182,"_43":44,"_42":14,"_166":26,"_40":251,"_13":252,"_35":36,"_37":39,"_25":26,"_171":-5,"_214":253,"_28":254},{"_53":242,"_33":243,"_31":244,"_164":182,"_43":44,"_42":245,"_166":69,"_40":246,"_13":193,"_35":36,"_37":247,"_25":26,"_171":-5,"_214":248,"_28":249},{"_53":234,"_33":235,"_31":236,"_164":182,"_43":44,"_42":237,"_166":26,"_40":238,"_13":168,"_35":36,"_37":239,"_25":26,"_171":-5,"_214":240,"_28":241},{"_53":227,"_33":228,"_31":229,"_164":182,"_43":44,"_42":168,"_166":26,"_40":230,"_13":73,"_35":36,"_37":231,"_25":26,"_171":-5,"_214":232,"_28":233},{"_53":220,"_33":221,"_31":222,"_164":182,"_43":44,"_42":168,"_166":69,"_40":223,"_13":73,"_35":36,"_37":224,"_25":26,"_171":-5,"_214":225,"_28":226},{"_53":174,"_33":175,"_31":176,"_164":-5,"_43":44,"_42":168,"_166":69,"_40":217,"_13":73,"_35":21,"_37":178,"_25":26,"_171":-5,"_214":218,"_28":219},{"_53":162,"_33":163,"_31":67,"_164":-5,"_43":44,"_42":165,"_166":26,"_40":213,"_13":168,"_35":169,"_37":170,"_25":26,"_171":172,"_214":215,"_28":216},["D",1776165705281],"children",[],[163],["D",1776086766746],[],[175],"53cbb448-373c-11f1-83b9-42b51c65c3df","utils.ts","019d8705-54fa-7761-a068-9c94857ce1ed",["D",1776086766746],"const REALTIME_BASE_URL = \"https://api.openai.com/v1/realtime\";\nconst OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\n// Builds the URL for a Realtime API endpoint.\nexport function makeUrl(callId?: string, action?: string) {\n let url = REALTIME_BASE_URL;\n if (!callId) {\n // start call (POST)\n url += \"/calls\";\n } else if (action) {\n // accept, reject, hangup (POST)\n url += `/calls/${callId}/${action}`;\n } else {\n // observer (web socket)\n url += `?call_id=${callId}`;\n }\n return url;\n}\n\n// Builds the headers for a Realtime API endpoint.\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\n// Debug helper for printing Realtime API errors.\nexport async function getErrorText(resp: Response) {\n const errBody = await resp.text().catch(() => \"\");\n return `${resp.status} ${resp.statusText}: ${errBody}`;\n}",[],[181,221],"53cba44e-373c-11f1-83b9-42b51c65c3df","sip.ts","019d8705-54fa-7761-a068-9bc0320eecb7",["D",1776086766746],"import { Hono } from \"npm:hono\";\nimport { Webhook } from \"npm:svix\";\nimport { makeSession } from \"./agent.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst sip = new Hono();\n\n// POST /sip : webhook endpoint\nsip.post(\"/\", async (c) => {\n // Verify the webhook.\n const OPENAI_SIGNING_SECRET = Deno.env.get(\"OPENAI_SIGNING_SECRET\");\n if (!OPENAI_SIGNING_SECRET) {\n console.error(\"🔴 webhook secret not configured\");\n return c.text(\"Internal error\", 500);\n }\n const webhook = new Webhook(OPENAI_SIGNING_SECRET);\n const bodyStr = await c.req.text();\n let callId: string | undefined;\n let toNumber: string | undefined;\n try {\n const headers = Object.fromEntries(c.req.raw.headers.entries());\n const payload: any = webhook.verify(bodyStr, headers);\n callId = payload.data.call_id;\n toNumber = getToNumber(payload.data.sip_headers);\n const fromNumber = getFromNumber(payload.data.sip_headers);\n console.log(\"✅ verified webhook:\", callId, fromNumber, toNumber);\n } catch (error) {\n console.error(`🔴 verify failed: ${error}`);\n return c.text(\"Invalid signature\", 401);\n }\n\n // Accept the call.\n const model = getModelForNumber(toNumber);\n const url = makeUrl(callId, \"accept\");\n const headers = makeHeaders(\"application/json\");\n const body = JSON.stringify(makeSession(model));\n const opts = { method: \"POST\", headers, body };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n console.error(`🔴 accept failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n console.log(\"✅ call accepted:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n return c.newResponse(null);\n});\n\nconst getToNumber = headers => getNumberFromSipHeader(headers, \"to\");\nconst getFromNumber = headers => getNumberFromSipHeader(headers, \"from\");\n\nfunction getNumberFromSipHeader(headers: any, name: string) {\n const header = getSipHeader(headers, name);\n const match = header?.match(/sip:\\+?(\\d+)@/);\n return match ? `+${match[1]}` : undefined;\n}\n\nfunction getSipHeader(headers: any, name: string) {\n return headers.find((h) => h.name.toLowerCase() === name)?.value;\n}\n\nfunction getModelForNumber(number: string) {\n const map = { \"+14258000074\": \"gpt-realtime-mini\" };\n return map[number];\n}\n\nexport default sip;",[],[181,228],"eed15cac-37f2-11f1-8358-42b51c65c3df","rtc.ts","019d8705-54fa-7761-a068-976e7d341d86",27,["D",1776165195323],"import { Hono } from \"npm:hono\";\nimport { makeSession } from \"./agent.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new call\nrtc.post(\"/\", async (c) => {\n // Extract model parameter from URL if provided\n const model = c.req.query(\"model\");\n if (model !== undefined) {\n console.log(\"🔧 Model parameter received:\", model);\n }\n\n // Parse video flag from query (?video=true)\n const video = c.req.query(\"video\") === \"true\";\n if (video) {\n console.log(\"📹 Video mode enabled\");\n }\n \n // Create the call.\n const fd = new FormData();\n fd.set(\"sdp\", await c.req.text());\n fd.set(\"session\", JSON.stringify(makeSession(model, video)));\n const opts = { method: \"POST\", headers: makeHeaders(), body: fd };\n const resp = await fetch(makeUrl(), opts);\n if (!resp.ok) {\n console.error(`🔴 start call failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n\n const contentType = resp.headers.get(\"Content-Type\");\n const location = resp.headers.get(\"Location\");\n const callId = location?.split(\"/\").pop();\n console.log(\"✅ call created:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n const responseHeaders = {\n \"Content-Type\": contentType,\n \"Location\": location,\n };\n return c.newResponse(resp.body, { headers: responseHeaders });\n});\n\nexport default rtc;",[],[181,235],"7c29d172-37f0-11f1-880b-42b51c65c3df","observer.ts","019d8705-54fa-7761-a068-91275444e97e",24,["D",1776164143983],"import { Hono } from \"npm:hono\";\nimport WebSocket from \"npm:ws\";\nimport { createHandler } from \"./agent.ts\";\nimport { makeHeaders, makeUrl } from \"./utils.ts\";\n\nexport interface ObserverHandler {\n onInit(): void;\n onTool(name: string, args: any): void;\n onMcpCallCompleted(name: string, output: string | null): void;\n onOutputAudioStarted(): void;\n onOutputAudioStopped(): void;\n onOutputAudioCleared(): void;\n}\n\nexport class ObserverClient {\n private ws: WebSocket;\n private closeResolve: () => void;\n private closePromise: Promise;\n private handler: ObserverHandler;\n\n constructor(callId: string) {\n const url = makeUrl(callId);\n this.ws = new WebSocket(url, { headers: makeHeaders() });\n this.ws.on(\"open\", () => this.handleOpen());\n this.ws.on(\"message\", (data) => this.handleMessage(JSON.parse(data)));\n this.ws.on(\"error\", (err) => this.handleError(err));\n this.ws.on(\"close\", (ev) => this.handleClose(ev));\n this.closePromise = new Promise((resolve) =>\n this.closeResolve = resolve\n );\n this.handler = createHandler(this, callId);\n }\n createResponse() {\n this.sendMessage({ type: \"response.create\" });\n }\n async waitForClose() {\n await this.closePromise;\n }\n private sendMessage(message: any) {\n this.ws.send(JSON.stringify(message));\n }\n private handleOpen() {\n console.log(\"✅ websocket connected\");\n this.handler.onInit();\n }\n private async handleMessage(message: any) {\n if (!message.type.endsWith(\".delta\")) { // ignore verbose messages\n console.log(message);\n }\n switch (message.type) {\n case \"response.function_call_arguments.done\":\n this.handler.onTool(message.name, message.arguments);\n break;\n case \"response.output_item.done\":\n if (message.item?.type === \"mcp_call\") {\n this.handler.onMcpCallCompleted(message.item.name, message.item.output);\n }\n break;\n case \"output_audio_buffer.started\":\n this.handler.onOutputAudioStarted();\n break;\n case \"output_audio_buffer.stopped\":\n this.handler.onOutputAudioStopped();\n break;\n case \"output_audio_buffer.cleared\":\n this.handler.onOutputAudioCleared();\n break;\n }\n }\n private handleError(err: Error) {\n console.error(`🔴 websocket failed: ${err.message}`);\n this.closeResolve();\n }\n private handleClose(e: Event) {\n console.log(`websocket closed: ${e}`);\n this.closeResolve();\n }\n}\n\nconst observer = new Hono();\n\n// POST /observer/:callId : self-connect to establish a web socket\nobserver.post(\"/:callId\", async (c) => {\n const callId = c.req.param(\"callId\");\n const client = new ObserverClient(callId);\n await client.waitForClose();\n return c.newResponse(null);\n});\n\nexport default observer;",[],[181,243],"059b59ee-3800-11f1-a538-42b51c65c3df",["D",1776170817022],18,[],[181,34],["D",1776086766746],[206,207,208,209,210],[181],"059adc62-3800-11f1-a538-42b51c65c3df",["D",1776170817022],[],[187,49],["D",1776086766746],[204],[187],["D",1776086766746],["D",1789500689328],{"_114":113,"_121":117},{"_114":113,"_115":-5,"_116":117,"_18":65,"_118":119,"_120":26},{"_13":99,"_33":100,"_53":44,"_101":54,"_102":103,"_104":105,"_106":60,"_107":108,"_109":110,"_111":55,"_112":113},"ctx",{"_92":-5,"_143":303,"_304":305,"_306":307,"_308":26,"_309":26,"_310":26,"_311":69,"_312":-5,"_313":-7,"_314":-7,"_315":73,"_316":26,"_317":26,"_318":-5,"_319":26,"_320":26,"_321":26,"_322":-5},"ENV",{"_274":275,"_276":277,"_278":100,"_279":280,"_281":282,"_283":284,"_285":286,"_287":288,"_289":290,"_291":292,"_293":294,"_295":296,"_297":298,"_299":300,"_301":302},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Realtime Agent

Realtime Agent

Diff · v31