OpenAI Realtime API Voice Agent

\n\n

\n\n \n \n\n",[],[185,192],[185],{"_103":201,"_95":202,"_157":203,"_159":-5,"_160":138,"_161":177,"_162":187,"_105":204,"_99":32,"_164":165,"_166":-5,"_167":22,"_168":-5,"_169":205,"_98":240},"6d0d7df6-84d1-11f0-b251-0224a6c84d84","routes","0198f5be-4c71-73ea-a6c2-b5ff9a959440",["D",1756469545982],[206,215,223,232],{"_103":207,"_95":208,"_157":209,"_159":203,"_160":138,"_161":177,"_162":22,"_105":210,"_99":32,"_164":211,"_166":212,"_167":22,"_168":-5,"_169":213,"_98":214},"6d0d71c6-84d1-11f0-b251-0224a6c84d84","observer.ts","0198f5be-4c71-73ea-a6c2-a6772b8df5d1",["D",1756469545982],"script","import { Hono } from \"npm:hono\";\nimport WebSocket from \"npm:ws\";\nimport { makeHeaders } from \"./utils.ts\";\n\nconst observer = new Hono();\n\n// POST /observer/:callId : self-connect to establish a web socket\nobserver.post(\"/:callId\", async (c) => {\n const callId = c.req.param(\"callId\");\n const url = `wss://api.openai.com/v1/realtime?call_id=${callId}`;\n const ws = new WebSocket(url, { headers: makeHeaders() });\n ws.on(\"open\", () => {\n console.log(\"✅ websocket connected\");\n setTimeout(() => ws.send(JSON.stringify({ type: \"response.create\" })), 250);\n });\n ws.on(\"message\", (data /*, isBinary */) => {\n const message = JSON.parse(data);\n if (message.type !== \"response.audio_transcript.delta\") {\n console.log(message);\n }\n });\n ws.on(\"error\", (err: Error) => {\n console.error(`🔴 websocket failed: ${err.message}`);\n });\n\n // Respond immediately; WS continues in background\n return c.newResponse(null);\n});\n\nexport default observer;",[],[202,208],{"_103":216,"_95":217,"_157":218,"_159":203,"_160":138,"_161":177,"_162":22,"_105":219,"_99":32,"_164":211,"_166":220,"_167":22,"_168":-5,"_169":221,"_98":222},"6d0d76d0-84d1-11f0-b251-0224a6c84d84","rtc.ts","0198f5be-4c71-73ea-a6c2-ab5f9aa7c46d",["D",1756469545982],"import { Hono } from \"npm:hono\";\nimport { makeHeaders, makeSession } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new call\nrtc.post(\"/\", async (c) => {\n // Create the call.\n const url = \"https://api.openai.com/v1/realtime/calls\";\n const headers = makeHeaders();\n const fd = new FormData();\n fd.set(\"sdp\", await c.req.text());\n fd.set(\"session\", JSON.stringify(makeSession()));\n const opts = { method: \"POST\", headers, body: fd };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n const errText = await resp.text().catch(() => \"\");\n console.error(`🔴 start call failed: ${resp.status} ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n\n const contentType = resp.headers.get(\"Content-Type\");\n const location = resp.headers.get(\"Location\");\n const callId = location?.split(\"/\").pop();\n console.log(\"✅ call created:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" });\n\n return c.newResponse(resp.body, {\n headers: { \"Content-Type\": contentType },\n });\n});\n\nexport default rtc;",[],[202,217],{"_103":224,"_95":225,"_157":226,"_159":203,"_160":138,"_161":227,"_162":22,"_105":228,"_99":32,"_164":211,"_166":229,"_167":22,"_168":-5,"_169":230,"_98":231},"8c5f3496-84d7-11f0-9274-0224a6c84d84","sip.ts","8c5f3536-84d7-11f0-9274-0224a6c84d84",10,["D",1756472175629],"import { Hono } from \"npm:hono\";\nimport { Webhook } from \"npm:svix\";\nimport { makeHeaders, makeSession } from \"./utils.ts\";\n\nconst sip = new Hono();\n\n// POST /sip : webhook endpoint\nsip.post(\"/\", async (c) => {\n // Verify the webhook.\n const OPENAI_SIGNING_SECRET = Deno.env.get(\"OPENAI_SIGNING_SECRET\");\n if (!OPENAI_SIGNING_SECRET) {\n console.error(\"🔴 webhook secret not configured\");\n return c.text(\"Internal error\", 500);\n }\n const webhook = new Webhook(OPENAI_SIGNING_SECRET);\n const bodyStr = await c.req.text();\n let callId: string | undefined;\n try {\n const headers = Object.fromEntries(c.req.raw.headers.entries());\n const payload: any = webhook.verify(bodyStr, headers);\n callId = payload.data.call_id;\n console.log(\"✅ verified webhook:\", callId);\n } catch (error) {\n console.error(`🔴 verify failed: ${error}`);\n return c.text(\"Invalid signature\", 401);\n }\n\n // Accept the call.\n const url = `https://api.openai.com/v1/realtime/calls/${callId}/accept`;\n const headers = makeHeaders(\"application/json\");\n const body = JSON.stringify(makeSession());\n const opts = { method: \"POST\", headers, body };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n const errText = await resp.text().catch(() => \"\");\n console.error(`🔴 accept failed: ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n console.log(\"✅ call accepted:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer trigger failed:\", e));\n\n return c.newResponse(null);\n});\n\nexport default sip;",[],[202,225],{"_103":233,"_95":234,"_157":235,"_159":203,"_160":138,"_161":177,"_162":22,"_105":236,"_99":32,"_164":211,"_166":237,"_167":22,"_168":-5,"_169":238,"_98":239},"6d0d7a9a-84d1-11f0-b251-0224a6c84d84","utils.ts","0198f5be-4c71-73ea-a6c2-b2aa1b7e0116",["D",1756469545982],"const MODEL = \"gpt-realtime\";\nconst INSTRUCTIONS = `\n Greet the user in English, and thank them for trying the new OpenAI Realtime API.\n Give them a brief summary based on the list below, and then ask if they have any questions.\n Answer questions using the information below. For questions outside this scope,\n apologize and say you can't help with that.\n ---\n Short summary:\n - The Realtime API is now generally available, and has improved instruction following, voice naturalness, and audio quality.\n There are two new voices and it's easier to develop for. There's also a new telephony integration for phone scenarios.\n Full list of improvements:\n - generally available (no longer in beta)\n - two new voices (Cedar and Marin)\n - improved instruction following\n - enhanced voice naturalness\n - higher audio quality\n - improved handling of alphanumerics (eg, properly understanding credit card and phone numbers)\n - support for the OpenAI Prompts API\n - support for MCP-based tools\n - auto-truncation to reduce context size\n - native telephony support, making it simple to connect voice calls to existing Realtime API applications\n - when using WebRTC, you can connect without needing an ephemeral token\n - when using WebRTC, you can now control sessions (including tool calls) from the server\n - when using WebRTC, you can now send video to the model, and it can respond based on what it sees\n - many helpful bugfixes!\n`;\nconst VOICE = \"marin\";\n\nconst OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\nexport function makeSession() {\n return {\n type: \"realtime\",\n model: MODEL,\n instructions: INSTRUCTIONS,\n audio: {\n input: { noise_reduction: { type: \"near_field\" } },\n output: { voice: VOICE },\n },\n };\n}",[],[202,234],[202],{"_103":242,"_95":243,"_157":244,"_159":-5,"_160":138,"_161":245,"_162":187,"_105":246,"_99":32,"_164":178,"_166":247,"_167":22,"_168":-5,"_169":248,"_98":249},"f29cbe12-84d3-11f0-bb6d-0224a6c84d84",".vtignore","f29cbec6-84d3-11f0-bb6d-0224a6c84d84",5,["D",1756470629152],".git\n.vscode\n.cursorrules\n.DS_Store\nnode_modules\nvendor",[],[243],{"_103":251,"_95":252,"_157":253,"_159":-5,"_160":138,"_161":254,"_162":22,"_105":255,"_99":32,"_164":178,"_166":256,"_167":22,"_168":-5,"_169":257,"_98":258},"f29f72ce-84d3-11f0-9fef-0224a6c84d84","CLAUDE.md","f29f7364-84d3-11f0-9fef-0224a6c84d84",6,["D",1756470629156],"# CLAUDE.md\n\nThis file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.\n\n## Project Overview\n\nHello Realtime is an OpenAI Realtime API demo application supporting WebRTC users. It's built on Val Town's Deno runtime with TypeScript and uses the Hono framework for routing.\n\n## Commands\n\n### Development\n- **Run locally**: Deploy to Val Town platform - no local dev server\n- **Test deployment**: Visit https://hello-realtime.val.run for WebRTC\n\n## Architecture\n\n### Core Components\n\n1. **Main Entry Point** (`main.tsx`)\n - Hono app with route mounting\n - Routes: `/rtc`, `/observer`\n - Serves frontend from `/frontend/index.html`\n\n2. **WebRTC Route** (`routes/rtc.ts`)\n - Creates OpenAI Realtime calls via POST to `/rtc`\n - Handles SDP offer/answer exchange\n - Triggers observer WebSocket connection\n\n3. **Observer Route** (`routes/observer.ts`)\n - Establishes WebSocket connection to OpenAI\n - Monitors call events and transcripts\n - Auto-sends `response.create` after connection\n\n4. **Utilities** (`routes/utils.ts`)\n - `makeHeaders()`: Adds OpenAI API authentication\n - `makeSession()`: Configures Realtime session with model, instructions, and voice settings\n\n### Frontend\n- Single-page vanilla JavaScript application (`frontend/index.html`)\n- WebRTC peer connection management\n- Real-time status indicators and logging\n- No external dependencies or build process\n\n## Environment Variables\n\nRequired:\n- `OPENAI_API_KEY`: For all OpenAI API calls\n\n## Val Town Specifics\n\n- Uses Val Town's file reading utilities from `https://esm.town/v/std/utils@85-main/index.ts`\n- Deno runtime with TypeScript support\n- No local development server - deploy directly to Val Town\n- Import npm packages via `npm:` prefix (e.g., `npm:hono`)\n\n## Key Implementation Details\n\n- OpenAI Realtime model: `gpt-realtime`\n- Voice: `marin` (configurable in utils.ts)\n- Audio configuration includes noise reduction for near-field devices\n- WebSocket messages are logged except for audio transcript deltas\n- Observer automatically triggers 250ms after connection",[],[252],{"_103":260,"_95":261,"_157":262,"_159":-5,"_160":138,"_161":263,"_162":22,"_105":264,"_99":177,"_164":178,"_166":265,"_167":22,"_168":-5,"_169":266,"_98":267},"f2b0819a-84d3-11f0-8ae7-0224a6c84d84","README.md","0198f5be-4c71-73ea-a6c2-b95cbba37441",8,["D",1756470629299],"# hello-realtime\n\nHello Realtime is a complete OpenAI Realtime application that supports WebRTC users.\nYou can access the app via WebRTC at https://hello-realtime.val.run.\n\nThis demo shows off the new all-in-one WebRTC API and the new server-side websocket interface.\n\nIf you remix the app, you'll just need to pop in your own OPENAI_API_KEY (from platform.openai.com).",[],[261],{"_103":269,"_95":270,"_157":271,"_159":-5,"_160":138,"_161":100,"_162":187,"_105":272,"_99":177,"_164":178,"_166":273,"_167":22,"_168":-5,"_169":274,"_98":275},"8c7aff00-84d7-11f0-b6b5-0224a6c84d84","deno.json","f294aaa6-84d3-11f0-a56b-0224a6c84d84",["D",1756472175730],"{\n \"$schema\": \"https://raw.githubusercontent.com/denoland/deno/348900b8b79f4a434cab4c74b3bc8d4d2fa8ee74/cli/schemas/config-file.v1.json\",\n \"lock\": false,\n \"compilerOptions\": {\n \"noImplicitAny\": false,\n \"strict\": false,\n \"types\": [\n \"https://www.val.town/types/valtown.d.ts\"\n ],\n \"lib\": [\n \"dom\",\n \"dom.iterable\",\n \"dom.asynciterable\",\n \"deno.ns\",\n \"deno.unstable\"\n ]\n },\n \"lint\": {\n \"files\": {\n \"include\": [\n \"deno:/https/esm.town/**/*\"\n ]\n },\n \"rules\": {\n \"exclude\": [\n \"no-explicit-any\"\n ]\n }\n },\n \"node_modules_dir\": false,\n \"experimental\": {\n \"unstable-node-globals\": true,\n \"unstable-temporal\": true,\n \"unstable-worker-options\": true,\n \"unstable-sloppy-imports\": true\n }\n}\n",[],[270],{"_103":277,"_95":278,"_157":279,"_159":-5,"_160":138,"_161":280,"_162":22,"_105":281,"_99":118,"_164":282,"_166":283,"_167":22,"_168":96,"_169":284,"_98":285},"8c66a85c-84d7-11f0-9ca5-0224a6c84d84","main.tsx","0198f5be-4c71-73ea-a6c2-be6162e31cb7",11,["D",1756472175674],"http","import { Hono } from \"npm:hono\";\nimport {\n readFile,\n serveFile,\n} from \"https://esm.town/v/std/utils@85-main/index.ts\";\n// import mcp from \"./routes/mcp.ts\";\nimport rtc from \"./routes/rtc.ts\";\nimport sip from \"./routes/sip.ts\";\nimport observer from \"./routes/observer.ts\";\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.route(\"/sip\", sip);\napp.route(\"/observer\", observer);\n// app.route(\"/mcp\", mcp);\n\napp.get(\"/\", async (c) => {\n const html = await readFile(\"/frontend/index.html\", import.meta.url);\n return c.html(html);\n});\n\nexport default app.fetch;\n",[],[278],"files",[154,171,183,190,200,206,215,223,232,241,250,259,268,276],"counts",{"_290":32,"_291":177,"_292":293,"_294":177},"pulls","branches","forks",3,"envVars","contents",[297,300,303,306,309,312,315,318],{"_103":155,"_95":156,"_157":158,"_159":-5,"_160":138,"_161":118,"_162":22,"_105":298,"_99":32,"_164":165,"_166":-5,"_167":22,"_168":-5,"_98":299},["D",1756470628950],[156],{"_103":184,"_95":185,"_157":186,"_159":-5,"_160":138,"_161":177,"_162":187,"_105":301,"_99":32,"_164":165,"_166":-5,"_167":22,"_168":-5,"_98":302},["D",1756469545982],[185],{"_103":201,"_95":202,"_157":203,"_159":-5,"_160":138,"_161":177,"_162":187,"_105":304,"_99":32,"_164":165,"_166":-5,"_167":22,"_168":-5,"_98":305},["D",1756469545982],[202],{"_103":242,"_95":243,"_157":244,"_159":-5,"_160":138,"_161":245,"_162":187,"_105":307,"_99":32,"_164":178,"_166":247,"_167":22,"_168":-5,"_98":308},["D",1756470629152],[243],{"_103":251,"_95":252,"_157":253,"_159":-5,"_160":138,"_161":254,"_162":22,"_105":310,"_99":32,"_164":178,"_166":256,"_167":22,"_168":-5,"_98":311},["D",1756470629156],[252],{"_103":260,"_95":261,"_157":262,"_159":-5,"_160":138,"_161":263,"_162":22,"_105":313,"_99":177,"_164":178,"_166":265,"_167":22,"_168":-5,"_98":314},["D",1756470629299],[261],{"_103":269,"_95":270,"_157":271,"_159":-5,"_160":138,"_161":100,"_162":187,"_105":316,"_99":177,"_164":178,"_166":273,"_167":22,"_168":-5,"_98":317},["D",1756472175730],[270],{"_103":277,"_95":278,"_157":279,"_159":-5,"_160":138,"_161":280,"_162":22,"_105":319,"_99":118,"_164":282,"_166":283,"_167":22,"_168":96,"_98":320},["D",1756472175674],[278],"upstreamProject",{"_103":146,"_95":323,"_111":324,"_112":-5,"_109":110,"_23":325},"hello-realtime","Sample app for the OpenAI Realtime API",{"_93":326,"_130":327},"jubertioai","https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/61901958-12bd-49a3-43e9-2f67f1617200/square512","upstreamBranch",{"_103":141,"_105":330,"_331":332,"_95":43,"_139":146,"_99":333,"_140":334,"_142":335,"_336":337,"_107":-5,"_151":22,"_109":110,"_150":326,"_16":148,"_149":323},["D",1755188493466],"updatedAt",["D",1757729497613],297,"e004ac78-4230-11f0-834a-76b3cceeab13",47,"lockfileKey","1757729506929-esm.town_v_jubertioai_hello-realtime@297-main","emptyEnvVars","isNew","flags",[],"routes/_app.x.$handle.$name.branch.$branch.version.$version.code.$",{"_93":94,"_95":96,"_97":43,"_99":100,"_98":72,"_101":344,"_152":349,"_286":392,"_178":-7,"_295":393,"_410":411,"_414":22,"_415":-5,"_416":417,"_418":419,"_420":421},{"_103":104,"_95":96,"_105":345,"_107":-5,"_108":22,"_109":110,"_111":-5,"_112":-5,"_14":113,"_114":-5,"_115":22,"_116":-5,"_117":118,"_16":119,"_120":121,"_122":121,"_123":124,"_125":124,"_126":124,"_127":121,"_128":121,"_23":346,"_132":347,"_135":348,"_151":22},["D",1756469545982],{"_93":94,"_130":131},{"_93":94,"_134":131},{"_99":137,"_95":43,"_103":138,"_139":104,"_140":141,"_142":143,"_144":110,"_145":146,"_147":148,"_149":96,"_150":94},[350,356,362,377,380,383,386,389],{"_103":155,"_95":156,"_157":158,"_159":-5,"_160":138,"_161":118,"_162":22,"_105":163,"_99":32,"_164":165,"_166":-5,"_167":22,"_168":-5,"_169":351,"_98":355},[352],{"_103":172,"_95":173,"_157":174,"_159":158,"_160":138,"_161":175,"_162":22,"_105":176,"_99":177,"_164":178,"_166":179,"_167":22,"_168":-5,"_169":353,"_98":354},[],[156,173],[156],{"_103":184,"_95":185,"_157":186,"_159":-5,"_160":138,"_161":177,"_162":187,"_105":188,"_99":32,"_164":165,"_166":-5,"_167":22,"_168":-5,"_169":357,"_98":361},[358],{"_103":191,"_95":192,"_157":193,"_159":186,"_160":138,"_161":194,"_162":22,"_105":195,"_99":177,"_164":178,"_166":196,"_167":22,"_168":-5,"_169":359,"_98":360},[],[185,192],[185],{"_103":201,"_95":202,"_157":203,"_159":-5,"_160":138,"_161":177,"_162":187,"_105":204,"_99":32,"_164":165,"_166":-5,"_167":22,"_168":-5,"_169":363,"_98":376},[364,367,370,373],{"_103":207,"_95":208,"_157":209,"_159":203,"_160":138,"_161":177,"_162":22,"_105":210,"_99":32,"_164":211,"_166":212,"_167":22,"_168":-5,"_169":365,"_98":366},[],[202,208],{"_103":216,"_95":217,"_157":218,"_159":203,"_160":138,"_161":177,"_162":22,"_105":219,"_99":32,"_164":211,"_166":220,"_167":22,"_168":-5,"_169":368,"_98":369},[],[202,217],{"_103":224,"_95":225,"_157":226,"_159":203,"_160":138,"_161":227,"_162":22,"_105":228,"_99":32,"_164":211,"_166":229,"_167":22,"_168":-5,"_169":371,"_98":372},[],[202,225],{"_103":233,"_95":234,"_157":235,"_159":203,"_160":138,"_161":177,"_162":22,"_105":236,"_99":32,"_164":211,"_166":237,"_167":22,"_168":-5,"_169":374,"_98":375},[],[202,234],[202],{"_103":242,"_95":243,"_157":244,"_159":-5,"_160":138,"_161":245,"_162":187,"_105":246,"_99":32,"_164":178,"_166":247,"_167":22,"_168":-5,"_169":378,"_98":379},[],[243],{"_103":251,"_95":252,"_157":253,"_159":-5,"_160":138,"_161":254,"_162":22,"_105":255,"_99":32,"_164":178,"_166":256,"_167":22,"_168":-5,"_169":381,"_98":382},[],[252],{"_103":260,"_95":261,"_157":262,"_159":-5,"_160":138,"_161":263,"_162":22,"_105":264,"_99":177,"_164":178,"_166":265,"_167":22,"_168":-5,"_169":384,"_98":385},[],[261],{"_103":269,"_95":270,"_157":271,"_159":-5,"_160":138,"_161":100,"_162":187,"_105":272,"_99":177,"_164":178,"_166":273,"_167":22,"_168":-5,"_169":387,"_98":388},[],[270],{"_103":277,"_95":278,"_157":279,"_159":-5,"_160":138,"_161":280,"_162":22,"_105":281,"_99":118,"_164":282,"_166":283,"_167":22,"_168":96,"_169":390,"_98":391},[],[278],[350,352,356,358,362,364,367,370,373,377,380,383,386,389],[394,396,398,400,402,404,406,408],{"_103":155,"_95":156,"_157":158,"_159":-5,"_160":138,"_161":118,"_162":22,"_105":298,"_99":32,"_164":165,"_166":-5,"_167":22,"_168":-5,"_98":395},[156],{"_103":184,"_95":185,"_157":186,"_159":-5,"_160":138,"_161":177,"_162":187,"_105":301,"_99":32,"_164":165,"_166":-5,"_167":22,"_168":-5,"_98":397},[185],{"_103":201,"_95":202,"_157":203,"_159":-5,"_160":138,"_161":177,"_162":187,"_105":304,"_99":32,"_164":165,"_166":-5,"_167":22,"_168":-5,"_98":399},[202],{"_103":242,"_95":243,"_157":244,"_159":-5,"_160":138,"_161":245,"_162":187,"_105":307,"_99":32,"_164":178,"_166":247,"_167":22,"_168":-5,"_98":401},[243],{"_103":251,"_95":252,"_157":253,"_159":-5,"_160":138,"_161":254,"_162":22,"_105":310,"_99":32,"_164":178,"_166":256,"_167":22,"_168":-5,"_98":403},[252],{"_103":260,"_95":261,"_157":262,"_159":-5,"_160":138,"_161":263,"_162":22,"_105":313,"_99":177,"_164":178,"_166":265,"_167":22,"_168":-5,"_98":405},[261],{"_103":269,"_95":270,"_157":271,"_159":-5,"_160":138,"_161":100,"_162":187,"_105":316,"_99":177,"_164":178,"_166":273,"_167":22,"_168":-5,"_98":407},[270],{"_103":277,"_95":278,"_157":279,"_159":-5,"_160":138,"_161":280,"_162":22,"_105":319,"_99":118,"_164":282,"_166":283,"_167":22,"_168":96,"_98":409},[278],"upstream",{"_103":141,"_105":412,"_331":413,"_95":43,"_139":146,"_99":333,"_140":334,"_142":335,"_336":337,"_107":-5,"_151":22,"_109":110,"_150":326,"_16":148,"_149":323},["D",1755188493466],["D",1757729497613],"showCompareCallout","pull","ahead",53,"behind",102,"templates",[],"actionData","errors"]

Realtime Agent

yisding

hello-realtime-video