OpenAI Realtime API Voice Agent

\n\n

\n\n \n \n",[],[156,174],[156],{"_103":184,"_95":185,"_157":186,"_159":-5,"_160":138,"_161":118,"_162":163,"_105":187,"_99":32,"_165":166,"_167":-5,"_168":22,"_169":-5,"_170":188,"_98":222},"43bc72a0-87f0-11f0-ba02-0224a6c84d84","routes","01990a31-913b-7345-a24a-e8c1bf3b5dc2",["D",1756812644593],[189,198,206,214],{"_103":190,"_95":191,"_157":192,"_159":186,"_160":138,"_161":118,"_162":163,"_105":193,"_99":32,"_165":194,"_167":195,"_168":22,"_169":-5,"_170":196,"_98":197},"43bc638c-87f0-11f0-ba02-0224a6c84d84","observer.ts","01990a31-913b-7345-a24a-da6bc15ddc7e",["D",1756812644593],"script","import { Hono } from \"npm:hono\";\nimport WebSocket from \"npm:ws\";\nimport { makeHeaders } from \"./utils.ts\";\n\nconst observer = new Hono();\n\n// POST /observer/:callId : self-connect to establish a web socket\nobserver.post(\"/:callId\", async (c) => {\n const callId = c.req.param(\"callId\");\n const url = `wss://api.openai.com/v1/realtime?call_id=${callId}`;\n const ws = new WebSocket(url, { headers: makeHeaders() });\n ws.on(\"open\", () => {\n console.log(\"✅ websocket connected\");\n setTimeout(() => ws.send(JSON.stringify({ type: \"response.create\" })), 250);\n });\n ws.on(\"message\", (data /*, isBinary */) => {\n const message = JSON.parse(data);\n if (message.type !== \"response.audio_transcript.delta\") {\n console.log(message);\n }\n });\n ws.on(\"error\", (err: Error) => {\n console.error(`🔴 websocket failed: ${err.message}`);\n });\n\n // Respond immediately; WS continues in background\n return c.newResponse(null);\n});\n\nexport default observer;",[],[185,191],{"_103":199,"_95":200,"_157":201,"_159":186,"_160":138,"_161":100,"_162":22,"_105":202,"_99":118,"_165":194,"_167":203,"_168":22,"_169":-5,"_170":204,"_98":205},"3982972c-87f2-11f0-a196-0224a6c84d84","rtc.ts","01990a31-913b-7345-a24a-deedb45ded9b",["D",1756813486509],"import { Hono } from \"npm:hono\";\nimport { makeHeaders, makeSession } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new call\nrtc.post(\"/\", async (c) => {\n // Check if MCP mode is requested via query parameter\n const useMCP = c.req.query(\"mcp\") === \"true\";\n const selfUrl = new URL(c.req.url);\n const mcpUrl = useMCP ? `${selfUrl.origin}/mcp/agent/demo-agent` : undefined;\n \n // Create the call.\n const url = \"https://api.openai.com/v1/realtime/calls\";\n const headers = makeHeaders();\n const fd = new FormData();\n fd.set(\"sdp\", await c.req.text());\n fd.set(\"session\", JSON.stringify(makeSession(useMCP, mcpUrl)));\n const opts = { method: \"POST\", headers, body: fd };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n const errText = await resp.text().catch(() => \"\");\n console.error(`🔴 start call failed: ${resp.status} ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n\n const contentType = resp.headers.get(\"Content-Type\");\n const location = resp.headers.get(\"Location\");\n const callId = location?.split(\"/\").pop();\n console.log(\"✅ call created:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch(() => {}); // catch to not crash the process\n\n return c.newResponse(resp.body, {\n headers: { \"Content-Type\": contentType },\n });\n});\n\nexport default rtc;",[],[185,200],{"_103":207,"_95":208,"_157":209,"_159":186,"_160":138,"_161":118,"_162":163,"_105":210,"_99":32,"_165":194,"_167":211,"_168":22,"_169":-5,"_170":212,"_98":213},"43bc6e18-87f0-11f0-ba02-0224a6c84d84","sip.ts","01990a31-913b-7345-a24a-e34aa31ad6da",["D",1756812644593],"import { Hono } from \"npm:hono\";\nimport { Webhook } from \"npm:svix\";\nimport { makeHeaders, makeSession } from \"./utils.ts\";\n\nconst sip = new Hono();\n\n// POST /sip : webhook endpoint\nsip.post(\"/\", async (c) => {\n // Verify the webhook.\n const OPENAI_SIGNING_SECRET = Deno.env.get(\"OPENAI_SIGNING_SECRET\");\n if (!OPENAI_SIGNING_SECRET) {\n console.error(\"🔴 webhook secret not configured\");\n return c.text(\"Internal error\", 500);\n }\n const webhook = new Webhook(OPENAI_SIGNING_SECRET);\n const bodyStr = await c.req.text();\n let callId: string | undefined;\n try {\n const headers = Object.fromEntries(c.req.raw.headers.entries());\n const payload: any = webhook.verify(bodyStr, headers);\n callId = payload.data.call_id;\n console.log(\"✅ verified webhook:\", callId);\n } catch (error) {\n console.error(`🔴 verify failed: ${error}`);\n return c.text(\"Invalid signature\", 401);\n }\n\n // Accept the call.\n const url = `https://api.openai.com/v1/realtime/calls/${callId}/accept`;\n const headers = makeHeaders(\"application/json\");\n const body = JSON.stringify(makeSession());\n const opts = { method: \"POST\", headers, body };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n const errText = await resp.text().catch(() => \"\");\n console.error(`🔴 accept failed: ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n console.log(\"✅ call accepted:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer trigger failed:\", e));\n\n return c.newResponse(null);\n});\n\nexport default sip;",[],[185,208],{"_103":215,"_95":216,"_157":217,"_159":186,"_160":138,"_161":118,"_162":22,"_105":218,"_99":32,"_165":194,"_167":219,"_168":22,"_169":-5,"_170":220,"_98":221},"43bc6fb2-87f0-11f0-ba02-0224a6c84d84","utils.ts","01990a31-913b-7345-a24a-e588c8e5287c",["D",1756812644593],"const MODEL = \"gpt-realtime\";\nconst INSTRUCTIONS = `\n Greet the user in English, and thank them for trying the new OpenAI Realtime API.\n Give them a brief summary based on the list below, and then ask if they have any questions.\n Answer questions using the information below. For questions outside this scope,\n apologize and say you can't help with that.\n ---\n Short summary:\n - The Realtime API is now generally available, and has improved instruction following, voice naturalness, and audio quality.\n There are two new voices and it's easier to develop for. There's also a new telephony integration for phone scenarios.\n Full list of improvements:\n - generally available (no longer in beta)\n - two new voices (Cedar and Marin)\n - improved instruction following\n - enhanced voice naturalness\n - higher audio quality\n - improved handling of alphanumerics (eg, properly understanding credit card and phone numbers)\n - support for the OpenAI Prompts API\n - support for MCP-based tools\n - auto-truncation to reduce context size\n - native telephony support, making it simple to connect voice calls to existing Realtime API applications\n - when using WebRTC, you can connect without needing an ephemeral token\n - when using WebRTC, you can now control sessions (including tool calls) from the server\n - when using WebRTC, you can now send video to the model, and it can respond based on what it sees\n - many helpful bugfixes!\n`;\nconst VOICE = \"marin\";\n\nconst OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\nexport function makeSession() {\n return {\n type: \"realtime\",\n model: MODEL,\n instructions: INSTRUCTIONS,\n audio: {\n input: { noise_reduction: { type: \"near_field\" } },\n output: { voice: VOICE },\n },\n };\n}",[],[185,216],[185],{"_103":224,"_95":225,"_157":226,"_159":-5,"_160":138,"_161":227,"_162":163,"_105":228,"_99":32,"_165":178,"_167":229,"_168":22,"_169":-5,"_170":230,"_98":231},"3971d5ae-87f2-11f0-b5ac-0224a6c84d84",".vtignore","3971d6f8-87f2-11f0-b5ac-0224a6c84d84",3,["D",1756813486401],".git\n.vscode\n.cursorrules\n.DS_Store\nnode_modules\nvendor",[],[225],{"_103":233,"_95":234,"_157":235,"_159":-5,"_160":138,"_161":236,"_162":22,"_105":237,"_99":118,"_165":178,"_167":238,"_168":22,"_169":-5,"_170":239,"_98":240},"397c1514-87f2-11f0-a4ac-0224a6c84d84","README.md","01990a31-913b-7345-a24a-ef358561a0b2",5,["D",1756813486457],"# hello-mcp\n\n**Hello MCP** is a remix of OpenAI's hello-realtime demo that adds Model Context Protocol (MCP) support for server-side tool execution.\n\nThis demo showcases:\n- WebRTC-based voice conversations with OpenAI's Realtime API\n- **MCP (Model Context Protocol)** for secure server-side tool execution\n- A demo `getFavoriteFood` tool to demonstrate MCP functionality\n- Toggle between standard and MCP-enabled modes\n\nIf you remix the app, you'll just need to pop in your own `OPENAI_API_KEY` (from\n[platform.openai.com](https://platform.openai.com)).\n\n## What's New: MCP Support\n\n### Model Context Protocol (MCP)\nMCP enables server-side tool execution, keeping your business logic and API keys secure on the server rather than exposing them to the client.\n\n### Demo Tool: getFavoriteFood\nTry asking questions like:\n- \"What's Thomas's favorite food?\"\n- \"What does Brian like to eat?\"\n- \"Tell me about Yana's favorite food\"\n\nThe tool knows about: Thomas, Brian, Emil, Yana, Alex, Max, Justin, and Sam.\n\n## Architecture\n\n1. **Standard WebRTC Flow**:\n - Browser connects to frontend\n - Creates WebRTC offer\n - `/rtc` endpoint handles SDP negotiation with OpenAI\n - Observer established to monitor session\n\n2. **MCP-Enhanced Flow** (when enabled):\n - Same WebRTC setup as above\n - OpenAI discovers tools via MCP protocol (`/mcp/agent/:agentId`)\n - Tools execute server-side when invoked\n - Results returned through MCP protocol\n - Client triggers response generation after tool completion\n\n## API Endpoints\n- `GET /` - serves the WebRTC-based app with MCP toggle\n- `POST /rtc` - creates a new WebRTC-based Realtime API session (with optional `?mcp=true`)\n- `POST /sip` - handles an incoming Realtime API SIP call\n- `POST /observer` - internal endpoint to establish a Realtime API websocket for session control\n- `POST /mcp/agent/:agentId` - MCP protocol endpoint for tool discovery and execution\n- `GET /mcp/agent/:agentId/tools` - Debug endpoint to list available tools\n\n## Project Structure\n```\n├── frontend/ \n│ └── index.html # HTML/JS application for WebRTC with MCP support\n├── main.tsx # Main entrypoint for the val.town app\n└── routes/\n |── mcp.ts # MCP protocol handler with getFavoriteFood tool\n |── observer.ts # Server-side session control for SIP & WebRTC\n |── rtc.ts # WebRTC session setup with MCP configuration\n |── sip.ts # SIP webhook handling and session setup\n └── utils.ts # Realtime API session configuration with MCP support\n```\n\n## How to Test MCP\n\n1. **Start the app** - Click \"Start\" and allow microphone access\n2. **Check MCP is enabled** - Ensure \"Use MCP Tools\" checkbox is checked\n3. **Ask about favorite foods** - Try \"What's Thomas's favorite food?\"\n4. **Watch the logs** - You'll see MCP events including tool discovery and execution\n5. **Try without MCP** - Uncheck the box and restart to see the difference\n\n## MCP Protocol Details\n\nThe MCP implementation follows the JSON-RPC 2.0 specification:\n\n### Initialize\n```json\n{\n \"method\": \"initialize\",\n \"params\": { \"protocolVersion\": \"2025-06-18\" }\n}\n```\n\n### Tools List\n```json\n{\n \"method\": \"tools/list\",\n \"result\": { \"tools\": [...] }\n}\n```\n\n### Tool Call\n```json\n{\n \"method\": \"tools/call\",\n \"params\": {\n \"name\": \"getFavoriteFood\",\n \"arguments\": { \"name\": \"Thomas\" }\n }\n}\n```",[],[234],{"_103":242,"_95":243,"_157":244,"_159":-5,"_160":138,"_161":245,"_162":163,"_105":246,"_99":32,"_165":178,"_167":247,"_168":22,"_169":-5,"_170":248,"_98":249},"396e1a5e-87f2-11f0-98c7-0224a6c84d84","deno.json","396e1afe-87f2-11f0-98c7-0224a6c84d84",2,["D",1756813486377],"{\n \"$schema\": \"https://raw.githubusercontent.com/denoland/deno/348900b8b79f4a434cab4c74b3bc8d4d2fa8ee74/cli/schemas/config-file.v1.json\",\n \"lock\": false,\n \"compilerOptions\": {\n \"noImplicitAny\": false,\n \"strict\": false,\n \"types\": [\n \"https://www.val.town/types/valtown.d.ts\"\n ],\n \"lib\": [\n \"dom\",\n \"dom.iterable\",\n \"dom.asynciterable\",\n \"deno.ns\",\n \"deno.unstable\"\n ]\n },\n \"lint\": {\n \"files\": {\n \"include\": [\n \"deno:/https/esm.town/**/*\"\n ]\n },\n \"rules\": {\n \"exclude\": [\n \"no-explicit-any\"\n ]\n }\n },\n \"node_modules_dir\": false,\n \"experimental\": {\n \"unstable-node-globals\": true,\n \"unstable-temporal\": true,\n \"unstable-worker-options\": true,\n \"unstable-sloppy-imports\": true\n }\n}",[],[243],{"_103":251,"_95":252,"_157":253,"_159":-5,"_160":138,"_161":118,"_162":22,"_105":254,"_99":32,"_165":255,"_167":256,"_168":22,"_169":-5,"_170":257,"_98":258},"43bc7868-87f0-11f0-ba02-0224a6c84d84","main.tsx","01990a31-913b-7345-a24a-f20a587c5fee",["D",1756812644593],"http","import { Hono } from \"npm:hono\";\nimport { parseVal, serveFile } from \"https://esm.town/v/std/utils/index.ts\";\n// import mcp from \"./routes/mcp.ts\";\nimport rtc from \"./routes/rtc.ts\";\nimport sip from \"./routes/sip.ts\";\nimport observer from \"./routes/observer.ts\";\n\nconst SOURCE_URL = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\")).links.self.val;\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.route(\"/sip\", sip);\napp.route(\"/observer\", observer);\n// app.route(\"/mcp\", mcp);\napp.get(\"/source\", (c) => c.redirect(SOURCE_URL));\n\napp.get(\"/\", () => serveFile(\"/frontend/index.html\"));\n\nexport default app.fetch;",[],[252],"files",[154,172,183,189,198,206,214,223,232,241,250],"counts",{"_263":32,"_264":118,"_265":245,"_266":118},"pulls","branches","forks","envVars","contents",[269,272,275,278,281,284],{"_103":155,"_95":156,"_157":158,"_159":-5,"_160":138,"_161":118,"_162":163,"_105":270,"_99":32,"_165":166,"_167":-5,"_168":22,"_169":-5,"_98":271},["D",1756812644593],[156],{"_103":184,"_95":185,"_157":186,"_159":-5,"_160":138,"_161":118,"_162":163,"_105":273,"_99":32,"_165":166,"_167":-5,"_168":22,"_169":-5,"_98":274},["D",1756812644593],[185],{"_103":224,"_95":225,"_157":226,"_159":-5,"_160":138,"_161":227,"_162":163,"_105":276,"_99":32,"_165":178,"_167":229,"_168":22,"_169":-5,"_98":277},["D",1756813486401],[225],{"_103":233,"_95":234,"_157":235,"_159":-5,"_160":138,"_161":236,"_162":22,"_105":279,"_99":118,"_165":178,"_167":238,"_168":22,"_169":-5,"_98":280},["D",1756813486457],[234],{"_103":242,"_95":243,"_157":244,"_159":-5,"_160":138,"_161":245,"_162":163,"_105":282,"_99":32,"_165":178,"_167":247,"_168":22,"_169":-5,"_98":283},["D",1756813486377],[243],{"_103":251,"_95":252,"_157":253,"_159":-5,"_160":138,"_161":118,"_162":22,"_105":285,"_99":32,"_165":255,"_167":256,"_168":22,"_169":-5,"_98":286},["D",1756812644593],[252],"upstreamProject",{"_103":146,"_95":289,"_111":290,"_112":-5,"_109":110,"_23":291},"hello-realtime","Sample app for the OpenAI Realtime API",{"_93":292,"_130":293},"jubertioai","https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/61901958-12bd-49a3-43e9-2f67f1617200/square512","upstreamBranch",{"_103":141,"_105":296,"_297":298,"_95":43,"_139":146,"_99":299,"_140":300,"_142":301,"_302":303,"_107":-5,"_151":22,"_109":110,"_150":292,"_16":148,"_149":289},["D",1755188493466],"updatedAt",["D",1757729497613],297,"e004ac78-4230-11f0-834a-76b3cceeab13",47,"lockfileKey","1757729506929-esm.town_v_jubertioai_hello-realtime@297-main","emptyEnvVars","isNew","flags",[],"routes/_app.x.$handle.$name.branch.$branch.version.$version.code.$",{"_93":94,"_95":96,"_97":43,"_99":100,"_98":72,"_101":310,"_152":315,"_259":349,"_178":-7,"_267":350,"_363":364,"_367":22,"_368":-5,"_369":370,"_371":372,"_373":374},{"_103":104,"_95":96,"_105":311,"_107":-5,"_108":22,"_109":110,"_111":-5,"_112":-5,"_14":113,"_114":-5,"_115":22,"_116":-5,"_117":118,"_16":119,"_120":121,"_122":121,"_123":124,"_125":124,"_126":124,"_127":121,"_128":121,"_23":312,"_132":313,"_135":314,"_151":22},["D",1756812644593],{"_93":94,"_130":131},{"_93":94,"_134":131},{"_99":137,"_95":43,"_103":138,"_139":104,"_140":141,"_142":143,"_144":110,"_145":146,"_147":148,"_149":96,"_150":94},[316,322,337,340,343,346],{"_103":155,"_95":156,"_157":158,"_159":-5,"_160":138,"_161":118,"_162":163,"_105":164,"_99":32,"_165":166,"_167":-5,"_168":22,"_169":-5,"_170":317,"_98":321},[318],{"_103":173,"_95":174,"_157":175,"_159":158,"_160":138,"_161":176,"_162":163,"_105":177,"_99":118,"_165":178,"_167":179,"_168":22,"_169":-5,"_170":319,"_98":320},[],[156,174],[156],{"_103":184,"_95":185,"_157":186,"_159":-5,"_160":138,"_161":118,"_162":163,"_105":187,"_99":32,"_165":166,"_167":-5,"_168":22,"_169":-5,"_170":323,"_98":336},[324,327,330,333],{"_103":190,"_95":191,"_157":192,"_159":186,"_160":138,"_161":118,"_162":163,"_105":193,"_99":32,"_165":194,"_167":195,"_168":22,"_169":-5,"_170":325,"_98":326},[],[185,191],{"_103":199,"_95":200,"_157":201,"_159":186,"_160":138,"_161":100,"_162":22,"_105":202,"_99":118,"_165":194,"_167":203,"_168":22,"_169":-5,"_170":328,"_98":329},[],[185,200],{"_103":207,"_95":208,"_157":209,"_159":186,"_160":138,"_161":118,"_162":163,"_105":210,"_99":32,"_165":194,"_167":211,"_168":22,"_169":-5,"_170":331,"_98":332},[],[185,208],{"_103":215,"_95":216,"_157":217,"_159":186,"_160":138,"_161":118,"_162":22,"_105":218,"_99":32,"_165":194,"_167":219,"_168":22,"_169":-5,"_170":334,"_98":335},[],[185,216],[185],{"_103":224,"_95":225,"_157":226,"_159":-5,"_160":138,"_161":227,"_162":163,"_105":228,"_99":32,"_165":178,"_167":229,"_168":22,"_169":-5,"_170":338,"_98":339},[],[225],{"_103":233,"_95":234,"_157":235,"_159":-5,"_160":138,"_161":236,"_162":22,"_105":237,"_99":118,"_165":178,"_167":238,"_168":22,"_169":-5,"_170":341,"_98":342},[],[234],{"_103":242,"_95":243,"_157":244,"_159":-5,"_160":138,"_161":245,"_162":163,"_105":246,"_99":32,"_165":178,"_167":247,"_168":22,"_169":-5,"_170":344,"_98":345},[],[243],{"_103":251,"_95":252,"_157":253,"_159":-5,"_160":138,"_161":118,"_162":22,"_105":254,"_99":32,"_165":255,"_167":256,"_168":22,"_169":-5,"_170":347,"_98":348},[],[252],[316,318,322,324,327,330,333,337,340,343,346],[351,353,355,357,359,361],{"_103":155,"_95":156,"_157":158,"_159":-5,"_160":138,"_161":118,"_162":163,"_105":270,"_99":32,"_165":166,"_167":-5,"_168":22,"_169":-5,"_98":352},[156],{"_103":184,"_95":185,"_157":186,"_159":-5,"_160":138,"_161":118,"_162":163,"_105":273,"_99":32,"_165":166,"_167":-5,"_168":22,"_169":-5,"_98":354},[185],{"_103":224,"_95":225,"_157":226,"_159":-5,"_160":138,"_161":227,"_162":163,"_105":276,"_99":32,"_165":178,"_167":229,"_168":22,"_169":-5,"_98":356},[225],{"_103":233,"_95":234,"_157":235,"_159":-5,"_160":138,"_161":236,"_162":22,"_105":279,"_99":118,"_165":178,"_167":238,"_168":22,"_169":-5,"_98":358},[234],{"_103":242,"_95":243,"_157":244,"_159":-5,"_160":138,"_161":245,"_162":163,"_105":282,"_99":32,"_165":178,"_167":247,"_168":22,"_169":-5,"_98":360},[243],{"_103":251,"_95":252,"_157":253,"_159":-5,"_160":138,"_161":118,"_162":22,"_105":285,"_99":32,"_165":255,"_167":256,"_168":22,"_169":-5,"_98":362},[252],"upstream",{"_103":141,"_105":365,"_297":366,"_95":43,"_139":146,"_99":299,"_140":300,"_142":301,"_302":303,"_107":-5,"_151":22,"_109":110,"_150":292,"_16":148,"_149":289},["D",1755188493466],["D",1757729497613],"showCompareCallout","pull","ahead",22,"behind",100,"templates",[],"actionData","errors"]

Realtime Agent with MCP