OpenAI Realtime API Voice Agent

\n\n

\n\n \n\n \n \n",[],[160,177],[160],{"_104":186,"_96":187,"_161":188,"_163":-5,"_164":141,"_165":140,"_166":118,"_106":189,"_101":32,"_168":169,"_170":-5,"_171":22,"_172":-5,"_173":190,"_99":216},"e33e54ea-aa2a-11f0-a60e-0224a6c84d84","routes","0199ea84-4b3b-702e-b795-77daa3f12d25",["D",1760576162545],[191,200,208],{"_104":192,"_96":193,"_161":194,"_163":188,"_164":141,"_165":140,"_166":118,"_106":195,"_101":32,"_168":196,"_170":197,"_171":22,"_172":-5,"_173":198,"_99":199},"e33e5094-aa2a-11f0-a60e-0224a6c84d84","observer.ts","0199ea84-4b3b-702e-b795-683faafa1371",["D",1760576162545],"script","import { Hono } from \"npm:hono\";\nimport WebSocket from \"npm:ws\";\nimport { makeHeaders } from \"./utils.ts\";\n\nconst observer = new Hono();\n\n// POST /observer/:callId : self-connect to establish a web socket\nobserver.post(\"/:callId\", async (c) => {\n const callId = c.req.param(\"callId\");\n const url = `wss://api.openai.com/v1/realtime?call_id=${callId}`;\n const ws = new WebSocket(url, { headers: makeHeaders() });\n ws.on(\"open\", () => {\n console.log(\"✅ websocket connected\");\n setTimeout(() => ws.send(JSON.stringify({ type: \"response.create\" })), 250);\n });\n ws.on(\"message\", (data /*, isBinary */) => {\n const message = JSON.parse(data);\n if (message.type !== \"response.audio_transcript.delta\") {\n console.log(message);\n }\n });\n ws.on(\"error\", (err: Error) => {\n console.error(`🔴 websocket failed: ${err.message}`);\n });\n\n // Respond immediately; WS continues in background\n return c.newResponse(null);\n});\n\nexport default observer;\n",[],[187,193],{"_104":201,"_96":202,"_161":203,"_163":188,"_164":141,"_165":140,"_166":118,"_106":204,"_101":32,"_168":196,"_170":205,"_171":22,"_172":-5,"_173":206,"_99":207},"e33e52a6-aa2a-11f0-a60e-0224a6c84d84","rtc.ts","0199ea84-4b3b-702e-b795-6e011747c225",["D",1760576162545],"import { Hono } from \"npm:hono\";\nimport { makeHeaders, makeSession } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new call\nrtc.post(\"/\", async (c) => {\n // Parse video flag from query (?video=true)\n const reqUrl = new URL(c.req.url);\n const video = reqUrl.searchParams.get(\"video\") === \"true\";\n\n // Create the call.\n const url = \"https://api.openai.com/v1/realtime/calls\";\n const headers = makeHeaders();\n const fd = new FormData();\n fd.set(\"sdp\", await c.req.text());\n fd.set(\"session\", JSON.stringify(makeSession(video)));\n const opts = { method: \"POST\", headers, body: fd };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n const errText = await resp.text().catch(() => \"\");\n console.error(`🔴 start call failed: ${resp.status} ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n\n const contentType = resp.headers.get(\"Content-Type\");\n const location = resp.headers.get(\"Location\");\n const callId = location?.split(\"/\").pop();\n console.log(\"✅ call created:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" });\n\n const responseHeaders = {\n \"Content-Type\": contentType,\n \"Location\": location,\n };\n return c.newResponse(resp.body, { headers: responseHeaders }); \n});\n\nexport default rtc;",[],[187,202],{"_104":209,"_96":210,"_161":211,"_163":188,"_164":141,"_165":140,"_166":118,"_106":212,"_101":32,"_168":196,"_170":213,"_171":22,"_172":-5,"_173":214,"_99":215},"e33e53be-aa2a-11f0-a60e-0224a6c84d84","utils.ts","0199ea84-4b3b-702e-b795-704cc279e2e9",["D",1760576162545],"const MODEL = \"gpt-realtime\";\nconst INSTRUCTIONS = `Greet the user in English.`;\nconst VOICE = \"marin\";\n\nconst OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\nexport function makeSession(video: boolean) {\n const session = {\n type: \"realtime\",\n model: MODEL,\n instructions: INSTRUCTIONS,\n audio: {\n input: { noise_reduction: { type: \"near_field\" } },\n output: { voice: VOICE },\n },\n };\n\n return session;\n}",[],[187,210],[187],{"_104":218,"_96":219,"_161":220,"_163":-5,"_164":141,"_165":140,"_166":118,"_106":221,"_101":32,"_168":180,"_170":222,"_171":22,"_172":-5,"_173":223,"_99":224},"e33e55ee-aa2a-11f0-a60e-0224a6c84d84",".gitignore","0199ea84-4b3b-702e-b795-795582ec3ba8",["D",1760576162545],"node_modules",[],[219],{"_104":226,"_96":227,"_161":228,"_163":-5,"_164":141,"_165":140,"_166":118,"_106":229,"_101":32,"_168":180,"_170":230,"_171":22,"_172":-5,"_173":231,"_99":232},"e33e56e8-aa2a-11f0-a60e-0224a6c84d84",".vtignore","0199ea84-4b3b-702e-b795-7cf2f2ef40bb",["D",1760576162545],".git\n.vscode\n.cursorrules\n.DS_Store\nnode_modules\nvendor",[],[227],{"_104":234,"_96":235,"_161":236,"_163":-5,"_164":141,"_165":140,"_166":118,"_106":237,"_101":32,"_168":180,"_170":238,"_171":22,"_172":-5,"_173":239,"_99":240},"e33e57ec-aa2a-11f0-a60e-0224a6c84d84","README.md","0199ea84-4b3b-702e-b795-802a11dd2aa1",["D",1760576162545],"# hello-realtime-video\n\nHello Realtime is a complete OpenAI Realtime application that supports WebRTC\nusers. You can access the app via WebRTC at\nhttps://hello-realtime-video.val.run.\n\nThis demo shows off the new all-in-one WebRTC API and the new server-side\nwebsocket interface.\n\nIf you remix the app, you'll just need to pop in your own OPENAI_API_KEY (from\nplatform.openai.com).\n",[],[235],{"_104":242,"_96":243,"_161":244,"_163":-5,"_164":141,"_165":140,"_166":118,"_106":245,"_101":32,"_168":180,"_170":246,"_171":22,"_172":-5,"_173":247,"_99":248},"e33e58f0-aa2a-11f0-a60e-0224a6c84d84","deno.json","0199ea84-4b3b-702e-b795-85cc7f2098ef",["D",1760576162545],"{\n \"$schema\": \"https://raw.githubusercontent.com/denoland/deno/348900b8b79f4a434cab4c74b3bc8d4d2fa8ee74/cli/schemas/config-file.v1.json\",\n \"lock\": false,\n \"compilerOptions\": {\n \"noImplicitAny\": false,\n \"strict\": false,\n \"types\": [\n \"https://www.val.town/types/valtown.d.ts\"\n ],\n \"lib\": [\n \"dom\",\n \"dom.iterable\",\n \"dom.asynciterable\",\n \"deno.ns\",\n \"deno.unstable\"\n ]\n },\n \"lint\": {\n \"files\": {\n \"include\": [\n \"deno:/https/esm.town/**/*\"\n ]\n },\n \"rules\": {\n \"exclude\": [\n \"no-explicit-any\"\n ]\n }\n },\n \"node_modules_dir\": false,\n \"experimental\": {\n \"unstable-node-globals\": true,\n \"unstable-temporal\": true,\n \"unstable-worker-options\": true,\n \"unstable-sloppy-imports\": true\n }\n}\n",[],[243],{"_104":250,"_96":100,"_161":116,"_163":-5,"_164":141,"_165":140,"_166":118,"_106":251,"_101":32,"_168":252,"_170":253,"_171":22,"_172":-5,"_173":254,"_99":255},"e33e5a12-aa2a-11f0-a60e-0224a6c84d84",["D",1760576162545],"http","import { Hono } from \"npm:hono\";\nimport { readFile } from \"https://esm.town/v/std/utils@85-main/index.ts\";\n// import mcp from \"./routes/mcp.ts\";\nimport rtc from \"./routes/rtc.ts\";\nimport observer from \"./routes/observer.ts\";\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.route(\"/observer\", observer);\n// app.route(\"/mcp\", mcp);\n\napp.get(\"/\", async (c) => {\n const html = await readFile(\"/frontend/index.html\", import.meta.url);\n return c.html(html);\n});\n\nexport default app.fetch;\n",[],[100],"files",[158,175,185,191,200,208,217,225,233,241,249],"counts",{"_260":32,"_261":140,"_262":32,"_263":264},"pulls","branches","forks","envVars",{"_265":140,"_266":32},"count","emptyCount","contents",[],{"_104":250,"_164":141,"_161":116,"_96":100,"_170":253,"_168":252,"_101":32,"_163":-5,"_106":270,"_165":140,"_172":-5,"_99":271},["D",1760576162545],[100],"upstreamProject",{"_104":150,"_96":274,"_112":-5,"_113":-5,"_110":148,"_23":275},"hello-realtime-video",{"_94":276,"_133":277},"jubertioai","https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/61901958-12bd-49a3-43e9-2f67f1617200/square512","upstreamBranch",{"_104":144,"_106":280,"_281":282,"_96":44,"_142":150,"_101":146,"_143":283,"_145":284,"_285":286,"_108":-5,"_155":22,"_110":148,"_154":276,"_16":152,"_153":274},["D",1756945107313],"updatedAt",["D",1757017261874],"6d08270c-84d1-11f0-b251-0224a6c84d84",54,"lockfileKey","1764564115925-esm.town_v_jubertioai_hello-realtime-video@7-main","emptyEnvVars","isNew","flags",[],"routes/_app.x.$handle.$name.branch.$branch.code.$",{"_94":95,"_96":97,"_98":44,"_101":-7,"_99":100,"_102":293,"_156":298,"_256":332,"_180":333,"_267":336,"_337":338,"_341":-5,"_342":32,"_343":32,"_344":345},{"_104":105,"_96":97,"_106":294,"_108":-5,"_109":22,"_110":111,"_112":-5,"_113":-5,"_14":114,"_115":116,"_117":118,"_119":118,"_120":-5,"_121":32,"_16":122,"_123":124,"_125":124,"_126":127,"_128":127,"_129":127,"_130":124,"_131":124,"_23":295,"_135":296,"_138":297,"_155":22},["D",1760576162545],{"_94":95,"_133":134},{"_94":95,"_137":134,"_14":114},{"_101":140,"_96":44,"_104":141,"_142":105,"_143":144,"_145":146,"_147":148,"_149":150,"_151":152,"_153":97,"_154":95},[299,305,317,320,323,326,329],{"_104":159,"_96":160,"_161":162,"_163":-5,"_164":141,"_165":140,"_166":118,"_106":167,"_101":32,"_168":169,"_170":-5,"_171":22,"_172":-5,"_173":300,"_99":304},[301],{"_104":176,"_96":177,"_161":178,"_163":162,"_164":141,"_165":140,"_166":118,"_106":179,"_101":32,"_168":180,"_170":181,"_171":22,"_172":-5,"_173":302,"_99":303},[],[160,177],[160],{"_104":186,"_96":187,"_161":188,"_163":-5,"_164":141,"_165":140,"_166":118,"_106":189,"_101":32,"_168":169,"_170":-5,"_171":22,"_172":-5,"_173":306,"_99":316},[307,310,313],{"_104":192,"_96":193,"_161":194,"_163":188,"_164":141,"_165":140,"_166":118,"_106":195,"_101":32,"_168":196,"_170":197,"_171":22,"_172":-5,"_173":308,"_99":309},[],[187,193],{"_104":201,"_96":202,"_161":203,"_163":188,"_164":141,"_165":140,"_166":118,"_106":204,"_101":32,"_168":196,"_170":205,"_171":22,"_172":-5,"_173":311,"_99":312},[],[187,202],{"_104":209,"_96":210,"_161":211,"_163":188,"_164":141,"_165":140,"_166":118,"_106":212,"_101":32,"_168":196,"_170":213,"_171":22,"_172":-5,"_173":314,"_99":315},[],[187,210],[187],{"_104":218,"_96":219,"_161":220,"_163":-5,"_164":141,"_165":140,"_166":118,"_106":221,"_101":32,"_168":180,"_170":222,"_171":22,"_172":-5,"_173":318,"_99":319},[],[219],{"_104":226,"_96":227,"_161":228,"_163":-5,"_164":141,"_165":140,"_166":118,"_106":229,"_101":32,"_168":180,"_170":230,"_171":22,"_172":-5,"_173":321,"_99":322},[],[227],{"_104":234,"_96":235,"_161":236,"_163":-5,"_164":141,"_165":140,"_166":118,"_106":237,"_101":32,"_168":180,"_170":238,"_171":22,"_172":-5,"_173":324,"_99":325},[],[235],{"_104":242,"_96":243,"_161":244,"_163":-5,"_164":141,"_165":140,"_166":118,"_106":245,"_101":32,"_168":180,"_170":246,"_171":22,"_172":-5,"_173":327,"_99":328},[],[243],{"_104":250,"_96":100,"_161":116,"_163":-5,"_164":141,"_165":140,"_166":118,"_106":251,"_101":32,"_168":252,"_170":253,"_171":22,"_172":-5,"_173":330,"_99":331},[],[100],[299,301,305,307,310,313,317,320,323,326,329],{"_104":250,"_164":141,"_161":116,"_96":100,"_170":253,"_168":252,"_101":32,"_163":-5,"_106":334,"_165":140,"_172":-5,"_99":335},["D",1760576162545],[100],[],"upstream",{"_104":144,"_106":339,"_281":340,"_96":44,"_142":150,"_101":146,"_143":283,"_145":284,"_285":286,"_108":-5,"_155":22,"_110":148,"_154":276,"_16":152,"_153":274},["D",1756945107313],["D",1757017261874],"pull","ahead","behind","templates",[346,363],{"_104":347,"_96":348,"_106":349,"_108":-5,"_109":22,"_110":148,"_112":350,"_113":351,"_14":352,"_115":353,"_117":118,"_119":-5,"_120":354,"_121":355,"_16":356,"_123":124,"_125":124,"_126":127,"_128":127,"_129":127,"_130":124,"_131":124,"_23":357,"_135":360,"_138":361,"_155":22},"8e7382ae-d835-11ef-bbbf-e6cdfca9ef9f","markdownBlogStarter",["D",1737491000655],"Starter template for a markdown blog","https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/44a31e5c-3e8a-48a8-d067-5785c0b08200/square512","a0bf3b31-15a5-4d5c-880e-4b1e22c9bc18","a4a9ad14-d835-11ef-8351-e6cdfca9ef9f",167,12,"e81e7335-6d4c-4764-a902-d0962bbfa9f7",{"_94":358,"_133":359},"stevekrouse","https://images.clerk.dev/uploaded/img_2PqHa2Gsy93xQrjh2w78Xu0cChW.jpeg",{"_94":358,"_137":359,"_14":352},{"_104":362,"_96":44,"_101":354,"_142":347,"_143":-5,"_145":-5},"8e73c264-d835-11ef-bbbf-e6cdfca9ef9f",{"_104":364,"_96":365,"_106":366,"_108":-5,"_109":22,"_110":148,"_112":367,"_113":-5,"_14":368,"_115":369,"_117":118,"_119":-5,"_120":370,"_121":371,"_16":372,"_123":124,"_125":124,"_126":127,"_128":127,"_129":127,"_130":124,"_131":124,"_23":373,"_135":376,"_138":377,"_155":22},"96102714-f528-11ef-aa68-569c3dd06744","reactHonoStarter",["D",1740674013689],"Starter template with client-side React & Hono server","c9143560-63b6-444f-8f9f-e2647fabe09f","a9b2b20a-f528-11ef-9e98-569c3dd06744",39,14,"c606f87f-4d7e-48c8-9c39-ef5ab94aec49",{"_94":374,"_133":375},"std","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvdXBsb2FkZWQvaW1nXzJVaVhpQ1BId3dscVpBbkdZVWlMNjJtcjkxWS5wbmcifQ",{"_94":374,"_137":375,"_14":-5},{"_104":378,"_96":44,"_101":379,"_142":364,"_143":-5,"_145":-5},"961078ea-f528-11ef-aa68-569c3dd06744",72,"actionData","errors"]

Realtime Agent

shikharkwatra

test-realtime-video