OpenAI Realtime API Voice Agent

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_279":280,"_281":282},"routes/_app.x.$handle.$name",{"_101":100,"_32":46,"_109":87,"_27":110,"_13":14,"_111":-5,"_11":112,"_113":114,"_115":116,"_117":118,"_119":120,"_20":-5,"_121":122,"_123":124,"_125":25,"_126":127,"_128":59,"_129":-5,"_130":-5,"_131":132},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_44":45,"_32":46,"_39":47,"_48":-5,"_49":25,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":57,"_58":59,"_60":59,"_61":-5,"_62":63,"_64":65,"_66":-5,"_67":-5,"_68":-5,"_69":-5,"_70":71,"_72":71,"_73":74,"_75":74,"_76":74,"_77":71,"_78":71,"_79":80,"_81":82,"_83":84,"_85":25},"version",100,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":33,"_34":35,"_36":38,"_39":40,"_41":14,"_42":43,"_24":25},"previous",{"_30":31,"_32":33,"_34":35,"_36":37},"deleted",false,"deletedHere","path","routes/mcp.ts","latestPath","fileId","0199a69a-800d-76c4-a76a-23ea07f65a22","name","mcp.ts","fileType","script","content","import { Hono } from \"npm:hono\";\nimport { StreamableHTTPTransport } from \"npm:@hono/mcp\";\nimport { McpServer } from \"npm:@modelcontextprotocol/sdk/server/mcp.js\";\nimport { z } from \"npm:zod\";\n\nconst mcp = new Hono();\nconst server = new McpServer({ name: \"hello-mcp\", version: \"1.0.0\" });\n\n// MCP entrypoint over HTTP.\n\nmcp.all(\"/\", async (c) => {\n const transport = new StreamableHTTPTransport();\n await server.connect(transport);\n const res = await transport.handleRequest(c);\n if (res.status >= 400) {\n console.error(\"MCP HTTP error:\", res.status);\n }\n return res;\n});\n\nfunction makeJsonResponse(data: any) {\n return { content: [{ type: \"json\", data }] };\n}\n\nfunction makeErrorResponse(message: string) {\n return { content: [{ type: \"text\", text: message }], isError: true };\n}\n\nfunction wrap(fn: Function, args: any) {\n try {\n return makeJsonResponse(fn(args));\n } catch (e) {\n return makeErrorResponse(e.message);\n }\n}\n\n//----- Demo tool -----\n\nfunction getFavoriteFood({ name }: { name: string }) {\n const favoriteFood = favoriteFoods[name.toLowerCase()];\n if (!favoriteFood) {\n const knownFoods = Object.keys(favoriteFoods).join(\", \");\n const message =\n `I don't know ${name}'s favorite food. I know about: ${knownFoods}.`;\n throw new Error(message);\n }\n\n return { favoriteFood };\n}\n\nconst favoriteFoods: Record = {\n thomas: \"alsacian choucroute\",\n brian: \"trail mix\",\n emil: \"nutella\",\n yana: \"french macarons\",\n alex: \"brownies\",\n max: \"feta cheese\",\n justin: \"sushi\",\n sam: \"pizza\",\n};\n\nfunction unixNow() {\n return Math.floor(Date.now() / 1000);\n}\n\nasync function getRecentTopStories(\n { hours = 24, limit = 5 }: { hours?: number; limit?: number },\n) {\n const seconds = (hours || 24) * 60 * 60;\n const cutoff = unixNow() - seconds;\n const ids = await fetch(\n \"https://hacker-news.firebaseio.com/v0/topstories.json\",\n ).then((r) => r.json());\n const stories: any[] = [];\n for (const id of ids) {\n if (stories.length >= Math.max(1, Math.min(limit || 5, 20))) break;\n const item = await fetch(\n `https://hacker-news.firebaseio.com/v0/item/${id}.json`,\n ).then((r) => r.json());\n if (item && item.time >= cutoff) {\n stories.push({\n id: item.id,\n title: item.title,\n url: item.url,\n score: item.score,\n by: item.by,\n time: item.time,\n hnLink: `https://news.ycombinator.com/item?id=${item.id}`,\n });\n }\n }\n return { stories };\n}\n\nserver.tool(\n \"getRecentTopStories\",\n \"Returns the top Hacker News stories from the last X hours.\",\n z.object({\n hours: z.number().min(1).max(720).default(24),\n limit: z.number().min(1).max(20).default(5),\n }),\n async (args) => {\n console.log(\"getRecentTopStories tool CALLED with:\", args);\n try {\n const result = await getRecentTopStories(args);\n console.log(\"getRecentTopStories RESULT:\", result);\n return { content: [{ type: \"json\", data: result }] }; // IMPORTANT\n } catch (e: any) {\n console.error(\"getRecentTopStories ERROR:\", e);\n return {\n content: [{ type: \"text\", text: e?.message || \"Unknown error\" }],\n isError: true,\n };\n }\n },\n);\n\nserver.tool(\n \"getFavoriteFood\",\n \"Return a person's favorite food (known: Thomas, Brian, Emil, Yana, Alex, Max, Justin, Sam). Use this tool whenever the user asks about favorite foods, e.g. 'What's Thomas's favorite food?'\",\n z.object({\n name: z.string().describe(\"The person's name to look up\"),\n }),\n (x) => wrap(getFavoriteFood, x),\n);\n\nexport default mcp;","import { Hono } from \"npm:hono\";\nimport { StreamableHTTPTransport } from \"npm:@hono/mcp\";\nimport { McpServer } from \"npm:@modelcontextprotocol/sdk/server/mcp.js\";\nimport { z } from \"npm:zod\";\n\nconst mcp = new Hono();\nconst server = new McpServer({ name: \"hello-mcp\", version: \"1.0.0\" });\n\n// MCP entrypoint over HTTP.\nmcp.all(\"/\", async (c) => {\n console.log(\"MCP endpoint hit!\");\n const transport = new StreamableHTTPTransport();\n await server.connect(transport);\n return transport.handleRequest(c);\n});\n\nfunction makeJsonResponse(data: any) {\n return { content: [{ type: \"json\", data }] };\n}\n\nfunction makeErrorResponse(message: string) {\n return { content: [{ type: \"text\", text: message }], isError: true };\n}\n\nfunction wrap(fn: Function, args: any) {\n try {\n return makeJsonResponse(fn(args));\n } catch (e) {\n return makeErrorResponse(e.message);\n }\n}\n\n// ----- Demo tool -----\n// server.tool(\n// \"getFavoriteFood\",\n// \"Returns the favorite food for specific people\",\n// z.object({\n// name: z.string().describe(\"The person's name to look up\"),\n// }),\n// (x) => wrap(getFavoriteFood, x),\n// );\n\n// const favoriteFoods: Record = {\n// thomas: \"alsacian choucroute\",\n// brian: \"trail mix\",\n// emil: \"nutella\",\n// yana: \"french macarons\",\n// alex: \"brownies\",\n// max: \"feta cheese\",\n// justin: \"sushi\",\n// sam: \"pizza\",\n// };\n\n// function getFavoriteFood({ name }: { name: string }) {\n// const favoriteFood = favoriteFoods[name.toLowerCase()];\n// if (!favoriteFood) {\n// const knownFoods = Object.keys(favoriteFoods).join(\", \");\n// const message =\n// `I don't know ${name}'s favorite food. I know about: ${knownFoods}.`;\n// throw new Error(message);\n// }\n\n// return { favoriteFood };\n// }\n\nfunction unixNow() {\n return Math.floor(Date.now() / 1000);\n}\n\nasync function getRecentTopStories(\n { hours = 24, limit = 5 }: { hours?: number; limit?: number },\n) {\n const seconds = (hours || 24) * 60 * 60;\n const cutoff = unixNow() - seconds;\n const ids = await fetch(\n \"https://hacker-news.firebaseio.com/v0/topstories.json\",\n ).then((r) => r.json());\n const stories: any[] = [];\n for (const id of ids) {\n if (stories.length >= Math.max(1, Math.min(limit || 5, 20))) break;\n const item = await fetch(\n `https://hacker-news.firebaseio.com/v0/item/${id}.json`,\n ).then((r) => r.json());\n if (item && item.time >= cutoff) {\n stories.push({\n id: item.id,\n title: item.title,\n url: item.url,\n score: item.score,\n by: item.by,\n time: item.time,\n hnLink: `https://news.ycombinator.com/item?id=${item.id}`,\n });\n }\n }\n return { stories };\n}\n\nserver.tool(\n \"getRecentTopStories\",\n \"Returns the top Hacker News stories from the last X hours.\",\n z.object({\n hours: z.number()\n .min(1).max(720).default(24)\n .describe(\n \"How many hours ago to include (default 24, max 720, i.e. 30 days)\",\n ),\n limit: z.number()\n .min(1).max(20).default(5)\n .describe(\"How many stories to return (1-20, default 5)\"),\n }),\n async (args) => {\n try {\n const result = await getRecentTopStories(args);\n return makeJsonResponse(result);\n } catch (e) {\n return makeErrorResponse(e?.message || \"Unknown error\");\n }\n },\n);\n\nexport default mcp;","createdAt",["D",1759697703211],"branchVersion","branchId","06a8b4aa-9fce-11f0-b51b-0224a6c84d84","id","06a39c86-9fce-11f0-b51b-0224a6c84d84","hello-mcp-2",["D",1759436767161],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","8b915eb8-46fa-461f-b203-599b3142d43c","pinnedFileId","0199a69a-800d-76c4-a76a-478db1b3b7b1","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","a6c542de-5cae-4c1f-b173-5164bdee6141","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_101":100,"_108":104},"org",{"_101":100,"_102":-5,"_103":104,"_18":55,"_105":106,"_107":25},"branch",{"_13":86,"_32":87,"_44":43,"_88":45,"_89":90,"_91":92,"_93":51,"_94":95,"_96":97,"_98":46,"_99":100},"isOwned",102,"main","projectId","forkedBranchId","85f61a28-8b8c-11f0-8edb-0224a6c84d84","forkedBranchVersion",36,"forkedProjectPrivacy","forkedProjectId","85f1b29e-8b8c-11f0-8edb-0224a6c84d84","forkedProjectOrgId","9c125825-8df1-4029-a26d-29049e9bd5a3","projectName","userHandle","darefail","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18ybWtvQU1DZnc1MXdOWFhhcWMycFhiVVVaak4ifQ","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_44":45,"_32":46,"_39":275,"_48":-5,"_49":25,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":57,"_58":59,"_60":59,"_61":-5,"_62":63,"_64":65,"_66":-5,"_67":-5,"_68":-5,"_69":-5,"_70":71,"_72":71,"_73":74,"_75":74,"_76":74,"_77":71,"_78":71,"_79":276,"_81":277,"_83":278,"_85":25},"tree",[200,202,208,209,210,211],"files",[200,201,202,203,204,205,206,207,208,209,210,211],"counts",{"_190":63,"_191":153,"_192":63,"_193":194,"_195":196},"contents",[144,145,146,147,148,149],"upstreamProject",{"_44":95,"_32":141,"_53":-5,"_54":-5,"_50":51,"_79":142},"upstreamBranch",{"_44":90,"_39":133,"_134":135,"_32":87,"_88":95,"_13":92,"_89":136,"_91":137,"_138":139,"_48":-5,"_85":25,"_50":51,"_99":140,"_64":97,"_98":141},"emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},["D",1757209610704],"updatedAt",["D",1757226403907],"43b7750c-87f0-11f0-ba02-0224a6c84d84",23,"lockfileKey","1764099635145-esm.town_v_jubertioai_hello-mcp@36-main","jubertioai","hello-mcp",{"_101":140,"_108":143},"https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/61901958-12bd-49a3-43e9-2f67f1617200/square512?exp=1789590967&sig=07a92cdc35585351d81aa6e7581426c2f5f5b4386819f90b0ecd6a651081f0e5",{"_44":185,"_32":186,"_30":187,"_152":-5,"_42":43,"_41":153,"_154":59,"_39":188,"_13":63,"_34":183,"_36":-5,"_24":25,"_158":-5,"_27":189},{"_44":179,"_32":180,"_30":181,"_152":-5,"_42":43,"_41":153,"_154":59,"_39":182,"_13":63,"_34":183,"_36":-5,"_24":25,"_158":-5,"_27":184},{"_44":173,"_32":174,"_30":175,"_152":-5,"_42":43,"_41":153,"_154":59,"_39":176,"_13":63,"_34":20,"_36":177,"_24":25,"_158":-5,"_27":178},{"_44":166,"_32":167,"_30":168,"_152":-5,"_42":43,"_41":169,"_154":59,"_39":170,"_13":153,"_34":20,"_36":171,"_24":25,"_158":-5,"_27":172},{"_44":160,"_32":161,"_30":162,"_152":-5,"_42":43,"_41":153,"_154":59,"_39":163,"_13":63,"_34":20,"_36":164,"_24":25,"_158":-5,"_27":165},{"_44":150,"_32":151,"_30":57,"_152":-5,"_42":43,"_41":153,"_154":59,"_39":155,"_13":63,"_34":156,"_36":157,"_24":25,"_158":-5,"_27":159},"06ac2cf2-9fce-11f0-b51b-0224a6c84d84","main.tsx","parentId",1,"latest",["D",1759436767161],"http","import { Hono } from \"npm:hono\";\nimport { parseVal, serveFile } from \"https://esm.town/v/std/utils/index.ts\";\nimport mcp from \"./routes/mcp.ts\";\nimport rtc from \"./routes/rtc.ts\";\nimport sip from \"./routes/sip.ts\";\nimport observer from \"./routes/observer.ts\";\n\nconst SOURCE_URL = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\")).links.self.val;\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.route(\"/sip\", sip);\napp.route(\"/observer\", observer);\napp.route(\"/mcp\", mcp);\napp.get(\"/source\", (c) => c.redirect(SOURCE_URL));\n\napp.get(\"/\", () => serveFile(\"/frontend/index.html\"));\n\nexport default app.fetch;","label",[151],"06ac2b4e-9fce-11f0-b51b-0224a6c84d84","deno.json","0199a69a-800d-76c4-a76a-40c301cd0ce2",["D",1759436767161],"{\n \"$schema\": \"https://raw.githubusercontent.com/denoland/deno/348900b8b79f4a434cab4c74b3bc8d4d2fa8ee74/cli/schemas/config-file.v1.json\",\n \"lock\": false,\n \"compilerOptions\": {\n \"noImplicitAny\": false,\n \"strict\": false,\n \"types\": [\n \"https://www.val.town/types/valtown.d.ts\"\n ],\n \"lib\": [\n \"dom\",\n \"dom.iterable\",\n \"dom.asynciterable\",\n \"deno.ns\",\n \"deno.unstable\"\n ]\n },\n \"lint\": {\n \"files\": {\n \"include\": [\n \"deno:/https/esm.town/**/*\"\n ]\n },\n \"rules\": {\n \"exclude\": [\n \"no-explicit-any\"\n ]\n }\n },\n \"node_modules_dir\": false,\n \"experimental\": {\n \"unstable-node-globals\": true,\n \"unstable-temporal\": true,\n \"unstable-worker-options\": true,\n \"unstable-sloppy-imports\": true\n }\n}",[161],"47faadc0-a215-11f0-8823-0224a6c84d84","README.md","0199a69a-800d-76c4-a76a-3e175d951709",7,["D",1759687273396],"# hello-mcp\n\n**Hello MCP** is a remix of OpenAI's hello-realtime demo that adds Model Context Protocol (MCP) support for server-side tool execution.\n\nThis demo showcases:\n- WebRTC-based voice conversations with OpenAI's Realtime API\n- **MCP (Model Context Protocol)** for secure server-side tool execution\n- A demo `getFavoriteFood` tool to demonstrate MCP functionality\n- Toggle between standard and MCP-enabled modes\n\nIf you remix the app, you'll just need to pop in your own `OPENAI_API_KEY` (from\n[platform.openai.com](https://platform.openai.com)).\n\n## What's New: MCP Support\n\n### Model Context Protocol (MCP)\nMCP enables server-side tool execution, keeping your business logic and API keys secure on the server rather than exposing them to the client.\n\n### Demo Tool: getFavoriteFood\nTry asking questions like:\n- \"What's Thomas's favorite food?\"\n- \"What does Brian like to eat?\"\n- \"Tell me about Yana's favorite food\"\n\nThe tool knows about: Thomas, Brian, Emil, Yana, Alex, Max, Justin, and Sam.\n\n## Architecture\n\n1. **Standard WebRTC Flow**:\n - Browser connects to frontend\n - Creates WebRTC offer\n - `/rtc` endpoint handles SDP negotiation with OpenAI\n - Observer established to monitor session\n\n2. **MCP-Enhanced Flow** (when enabled):\n - Same WebRTC setup as above\n - OpenAI discovers tools via MCP protocol (`/mcp`)\n - Tools execute server-side when invoked\n - Results returned through MCP protocol\n - Client triggers response generation after tool completion\n\n## API Endpoints\n- `GET /` - serves the WebRTC-based app with MCP toggle\n- `POST /rtc` - creates a new WebRTC-based Realtime API session (with optional `?mcp=true`)\n- `POST /sip` - handles an incoming Realtime API SIP call\n- `POST /observer` - internal endpoint to establish a Realtime API websocket for session control\n- `POST /mcp` - MCP protocol endpoint for tool discovery and execution\n- `GET /mcp/tools` - Debug endpoint to list available tools\n\n## Project Structure\n```\n├── frontend/ \n│ └── index.html # HTML/JS application for WebRTC with MCP support\n├── main.tsx # Main entrypoint for the val.town app\n└── routes/\n |── mcp.ts # MCP protocol handler with getFavoriteFood tool\n |── observer.ts # Server-side session control for SIP & WebRTC\n |── rtc.ts # WebRTC session setup with MCP configuration\n |── sip.ts # SIP webhook handling and session setup\n └── utils.ts # Realtime API session configuration with MCP support\n```\n\n## How to Test MCP\n\n1. **Start the app** - Click \"Start\" and allow microphone access\n2. **Check MCP is enabled** - Ensure \"Use MCP Tools\" checkbox is checked\n3. **Ask about favorite foods** - Try \"What's Thomas's favorite food?\"\n4. **Watch the logs** - You'll see MCP events including tool discovery and execution\n5. **Try without MCP** - Uncheck the box and restart to see the difference\n\n## MCP Protocol Details\n\nThe MCP implementation follows the JSON-RPC 2.0 specification:\n\n### Initialize\n```json\n{\n \"method\": \"initialize\",\n \"params\": { \"protocolVersion\": \"2025-06-18\" }\n}\n```\n\n### Tools List\n```json\n{\n \"method\": \"tools/list\",\n \"result\": { \"tools\": [...] }\n}\n```\n\n### Tool Call\n```json\n{\n \"method\": \"tools/call\",\n \"params\": {\n \"name\": \"getFavoriteFood\",\n \"arguments\": { \"name\": \"Thomas\" }\n }\n}\n```",[167],"06ac25fe-9fce-11f0-b51b-0224a6c84d84",".vtignore","0199a69a-800d-76c4-a76a-3817211ffd66",["D",1759436767161],".git\n.vscode\n.cursorrules\n.DS_Store\nnode_modules\nvendor",[174],"06ac20cc-9fce-11f0-b51b-0224a6c84d84","routes","0199a69a-800d-76c4-a76a-364ec52a8b81",["D",1759436767161],"directory",[180],"06ac167c-9fce-11f0-b51b-0224a6c84d84","frontend","0199a69a-800d-76c4-a76a-1d9777c97ecc",["D",1759436767161],[186],"pulls","branches","forks","envVars",{"_197":153,"_199":63},"connections",{"_197":63,"_198":63},"count","unfilledCount","emptyCount",{"_44":185,"_32":186,"_30":187,"_152":-5,"_42":43,"_41":153,"_154":59,"_39":272,"_13":63,"_34":183,"_36":-5,"_24":25,"_158":-5,"_213":273,"_27":274},{"_44":265,"_32":266,"_30":267,"_152":187,"_42":43,"_41":153,"_154":59,"_39":268,"_13":63,"_34":20,"_36":269,"_24":25,"_158":-5,"_213":270,"_27":271},{"_44":179,"_32":180,"_30":181,"_152":-5,"_42":43,"_41":153,"_154":59,"_39":262,"_13":63,"_34":183,"_36":-5,"_24":25,"_158":-5,"_213":263,"_27":264},{"_44":257,"_32":33,"_30":31,"_152":181,"_42":43,"_41":14,"_154":25,"_39":258,"_13":259,"_34":35,"_36":38,"_24":25,"_158":-5,"_213":260,"_27":261},{"_44":248,"_32":249,"_30":250,"_152":181,"_42":43,"_41":251,"_154":25,"_39":252,"_13":253,"_34":35,"_36":254,"_24":25,"_158":-5,"_213":255,"_27":256},{"_44":241,"_32":242,"_30":243,"_152":181,"_42":43,"_41":153,"_154":59,"_39":244,"_13":63,"_34":35,"_36":245,"_24":25,"_158":-5,"_213":246,"_27":247},{"_44":234,"_32":235,"_30":236,"_152":181,"_42":43,"_41":153,"_154":59,"_39":237,"_13":63,"_34":35,"_36":238,"_24":25,"_158":-5,"_213":239,"_27":240},{"_44":225,"_32":226,"_30":227,"_152":181,"_42":43,"_41":228,"_154":25,"_39":229,"_13":230,"_34":35,"_36":231,"_24":25,"_158":-5,"_213":232,"_27":233},{"_44":173,"_32":174,"_30":175,"_152":-5,"_42":43,"_41":153,"_154":59,"_39":222,"_13":63,"_34":20,"_36":177,"_24":25,"_158":-5,"_213":223,"_27":224},{"_44":166,"_32":167,"_30":168,"_152":-5,"_42":43,"_41":169,"_154":59,"_39":219,"_13":153,"_34":20,"_36":171,"_24":25,"_158":-5,"_213":220,"_27":221},{"_44":160,"_32":161,"_30":162,"_152":-5,"_42":43,"_41":153,"_154":59,"_39":216,"_13":63,"_34":20,"_36":164,"_24":25,"_158":-5,"_213":217,"_27":218},{"_44":150,"_32":151,"_30":57,"_152":-5,"_42":43,"_41":153,"_154":59,"_39":212,"_13":63,"_34":156,"_36":157,"_24":25,"_158":-5,"_213":214,"_27":215},["D",1759436767161],"children",[],[151],["D",1759436767161],[],[161],["D",1759687273396],[],[167],["D",1759436767161],[],[174],"145c60c6-a229-11f0-9ade-0224a6c84d84","utils.ts","0199a69a-800d-76c4-a76a-3107deafae5d",80,["D",1759695776731],28,"const MODEL = \"gpt-realtime\";\nconst INSTRUCTIONS = `\n You are an assistant with access to tools for favorite foods and Hacker News stories.\nAlways use the provided tools when asked about a person's favorite food or about news/stories from Hacker News.\nAfter using a tool, summarize the result in your response.\nGreet users in English when the conversation starts.\n`;\n\n//const VOICE = \"cedar\"; //male tenor\n//const VOICE = \"marin\"; //female alto\n//const VOICE = \"alloy\"; //female contralto\nconst VOICE = \"ash\"; //male\tbaritone\n//const VOICE = \"ballad\";// male\ttenor\n//const VOICE = \"coral\";//female soprano\n//const VOICE = \"echo\"; //male tenor\n//const VOICE = \"fable\"; //female alto\n//const VOICE = \"onyx\";//male base\n//const VOICE = \"nova\";//female alto\n//const VOICE = \"sage\";//female soprano\n//const VOICE = \"shimmer\";//female contralto\n//const VOICE = \"verse\"; //male tenor\n\nconst OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\nexport function makeSession(mcpUrl?: string) {\n const session: any = {\n type: \"realtime\",\n model: MODEL,\n instructions: INSTRUCTIONS,\n audio: {\n input: { noise_reduction: { type: \"near_field\" } },\n output: { voice: VOICE },\n },\n };\n\n // Add MCP configuration if requested\n if (mcpUrl) {\n session.tools = [{\n type: \"mcp\",\n server_label: \"hello-mcp-demo\",\n server_url: mcpUrl,\n authorization: \"demo-token-12345\", // In production, use a secure token\n require_approval: \"never\",\n }];\n }\n\n return session;\n}",[],[180,226],"06ac1c76-9fce-11f0-b51b-0224a6c84d84","sip.ts","0199a69a-800d-76c4-a76a-2f2a066a5d4b",["D",1759436767161],"import { Hono } from \"npm:hono\";\nimport { Webhook } from \"npm:svix\";\nimport { makeHeaders, makeSession } from \"./utils.ts\";\n\nconst sip = new Hono();\n\n// POST /sip : webhook endpoint\nsip.post(\"/\", async (c) => {\n // Verify the webhook.\n const OPENAI_SIGNING_SECRET = Deno.env.get(\"OPENAI_SIGNING_SECRET\");\n if (!OPENAI_SIGNING_SECRET) {\n console.error(\"🔴 webhook secret not configured\");\n return c.text(\"Internal error\", 500);\n }\n const webhook = new Webhook(OPENAI_SIGNING_SECRET);\n const bodyStr = await c.req.text();\n let callId: string | undefined;\n try {\n const headers = Object.fromEntries(c.req.raw.headers.entries());\n const payload: any = webhook.verify(bodyStr, headers);\n callId = payload.data.call_id;\n console.log(\"✅ verified webhook:\", callId);\n } catch (error) {\n console.error(`🔴 verify failed: ${error}`);\n return c.text(\"Invalid signature\", 401);\n }\n\n // Accept the call.\n const url = `https://api.openai.com/v1/realtime/calls/${callId}/accept`;\n const headers = makeHeaders(\"application/json\");\n const body = JSON.stringify(makeSession());\n const opts = { method: \"POST\", headers, body };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n const errText = await resp.text().catch(() => \"\");\n console.error(`🔴 accept failed: ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n console.log(\"✅ call accepted:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer trigger failed:\", e));\n\n return c.newResponse(null);\n});\n\nexport default sip;",[],[180,235],"06ac1b40-9fce-11f0-b51b-0224a6c84d84","rtc.ts","0199a69a-800d-76c4-a76a-299e95a0e04a",["D",1759436767161],"import { Hono } from \"npm:hono\";\nimport { makeHeaders, makeSession } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new call\nrtc.post(\"/\", async (c) => {\n // Check if MCP mode is requested via query parameter\n const useMCP = c.req.query(\"mcp\") === \"true\";\n const selfUrl = new URL(c.req.url);\n const mcpUrl = useMCP ? `${selfUrl.origin}/mcp` : undefined;\n\n // Create the call.\n const url = \"https://api.openai.com/v1/realtime/calls\";\n const headers = makeHeaders();\n const fd = new FormData();\n fd.set(\"sdp\", await c.req.text());\n fd.set(\"session\", JSON.stringify(makeSession(mcpUrl)));\n const opts = { method: \"POST\", headers, body: fd };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n const errText = await resp.text().catch(() => \"\");\n console.error(`🔴 start call failed: ${resp.status} ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n\n const contentType = resp.headers.get(\"Content-Type\");\n const location = resp.headers.get(\"Location\");\n const callId = location?.split(\"/\").pop();\n console.log(\"✅ call created:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch(() => {}); // catch to not crash the process\n\n return c.newResponse(resp.body, {\n headers: { \"Content-Type\": contentType },\n });\n});\n\nexport default rtc;",[],[180,242],"0a03bdce-a22c-11f0-a817-0224a6c84d84","observer.ts","0199a69a-800d-76c4-a76a-26ae019037e2",99,["D",1759697047860],13,"import { Hono } from \"npm:hono\";\nimport WebSocket from \"npm:ws\";\nimport { makeHeaders } from \"./utils.ts\";\n\nconst observer = new Hono();\n\n// POST /observer/:callId : self-connect to establish a web socket\nobserver.post(\"/:callId\", async (c) => {\n const callId = c.req.param(\"callId\");\n if (!callId) return c.text(\"Missing callId\", 400);\n\n const url = \"wss://api.openai.com/v1/realtime?call_id=${callId}\";\n const ws = new WebSocket(url, { headers: makeHeaders() });\n\n let inFlight = false;\n\n ws.on(\"open\", () => {\n console.log(\"✅ websocket connected\");\n // No initial response.create — wait for response.required\n });\n\n ws.on(\"error\", (err: Error) => {\n console.error(`🔴 websocket failed: ${err.message}`);\n // If an error occurs, allow a new turn later\n inFlight = false;\n });\n\n ws.on(\"message\", (data) => {\n let message: any;\n try {\n const s = typeof data === \"string\" ? data : data.toString();\n message = JSON.parse(s);\n } catch (e) {\n console.error(\"🔴 failed to parse WS frame:\", e);\n return;\n }\n\n // Keep logs readable by skipping the very chatty transcript deltas\n if (message.type !== \"response.output_audio_transcript.delta\") {\n console.log(\"WS:\", message.type, message);\n }\n\n // Start a new response whenever the model asks\n if (\n message.type === \"response.required\" && !inFlight &&\n ws.readyState === WebSocket.OPEN\n ) {\n inFlight = true;\n ws.send(JSON.stringify({ type: \"response.create\" }));\n }\n\n // Consider these as end-of-turn or abort signals; allow the next turn\n const endOfTurnTypes = new Set([\n \"response.completed\",\n \"response.canceled\",\n \"error\",\n ]);\n if (endOfTurnTypes.has(message.type)) {\n inFlight = false;\n }\n\n // Extra visibility if an MCP call failed\n if (message.type === \"response.mcp_call.failed\") {\n console.error(\"🔴 MCP call failed:\", message);\n // Let the turn proceed; model will typically emit response.required next\n inFlight = false;\n }\n });\n\n // Respond immediately; WS continues in background\n return c.newResponse(null);\n});\n\nexport default observer;",[],[180,249],"90a1bf06-a22d-11f0-92a2-0224a6c84d84",["D",1759697703211],58,[],[180,33],["D",1759436767161],[203,204,205,206,207],[180],"06abf91c-9fce-11f0-b51b-0224a6c84d84","index.html","0199a69a-800d-76c4-a76a-18998ae6b2a3",["D",1759436767161],"\n\n \n \n \n OpenAI Realtime API Voice Agent\n \n \n \n \n\n

\n\n

\n\n \n \n",[],[186,266],["D",1759436767161],[201],[186],["D",1759436767161],{"_101":100,"_108":104},{"_101":100,"_102":-5,"_103":104,"_18":55,"_105":106,"_107":25},{"_13":86,"_32":87,"_44":43,"_88":45,"_89":90,"_91":92,"_93":51,"_94":95,"_96":97,"_98":46,"_99":100},"ctx",{"_79":-5,"_130":312,"_313":314,"_315":316,"_317":25,"_318":25,"_319":25,"_320":59,"_321":-5,"_322":-7,"_323":-7,"_324":63,"_325":25,"_326":25,"_327":-5,"_328":25,"_329":25,"_330":25,"_331":-5},"ENV",{"_283":284,"_285":286,"_287":87,"_288":289,"_290":291,"_292":293,"_294":295,"_296":297,"_298":299,"_300":301,"_302":303,"_304":305,"_306":307,"_308":309,"_310":311},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","811aa22498c9b0b5a47e8d778de579cdb990f332","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Realtime Agent with MCP

Diff · v100