OpenAI Realtime API Voice Agent

\n\n

\n\n \n \n",[],[155,173],[155],{"_102":184,"_95":185,"_156":186,"_158":-5,"_159":137,"_160":117,"_161":162,"_104":187,"_99":32,"_164":165,"_166":-5,"_167":22,"_168":-5,"_169":188,"_98":227},"a6d4939e-843b-11f0-a497-0224a6c84d84","routes","0198f1e8-bcdb-709c-8377-6a376a4311f0",["D",1756405218407],[189,199,209,217],{"_102":190,"_95":191,"_156":192,"_158":186,"_159":137,"_160":136,"_161":162,"_104":193,"_99":194,"_164":195,"_166":196,"_167":22,"_168":-5,"_169":197,"_98":198},"62f8d010-8b50-11f0-8b67-0224a6c84d84","observer.ts","0198f1e8-bcdb-709c-8377-58bd29583a96",["D",1757183782247],5,"script","import { Hono } from \"npm:hono\";\nimport WebSocket from \"npm:ws\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst observer = new Hono();\n\n// POST /observer/:callId : self-connect to establish a web socket\nobserver.post(\"/:callId\", async (c) => {\n const callId = c.req.param(\"callId\");\n const url = makeUrl(callId);\n const ws = new WebSocket(url, { headers: makeHeaders() });\n ws.on(\"open\", () => handleOpen(ws, callId));\n ws.on(\"message\", (data) => handleMessage(ws, callId, JSON.parse(data)));\n ws.on(\"error\", (err) => handleError(ws, callId, err));\n\n // Block forever to keep the socket alive.\n await new Promise(() => {});\n return c.newResponse(null);\n});\n\nfunction handleOpen(ws: WebSocket, _callId: string) {\n console.log(\"✅ websocket connected\");\n // Have the model start talking immediately\n // const message = { type: \"response.create\" };\n // setTimeout(() => ws.send(JSON.stringify(message)), 250);\n}\n\nasync function handleMessage(_ws: WebSocket, callId: string, message: any) {\n if (message.type !== \"response.output_audio_transcript.delta\") {\n console.log(message);\n }\n if (message.type == \"response.function_call_arguments.done\") {\n if (message.name == \"end_call\") {\n doHangup(callId);\n }\n }\n}\n\nasync function doHangup(callId: string) {\n const url = makeUrl(callId, \"hangup\");\n console.log(`? hangup: ${url}`);\n const resp = await fetch(url, { method: \"POST\", headers: makeHeaders() });\n if (!resp.ok) {\n console.error(\n `🔴 end call with id '${callId}'' failed: ${await getErrorText(resp)}`,\n );\n return;\n }\n\n console.log(\"✅ call ended\");\n}\n\nfunction handleError(_ws: WebSocket, _callId: string, err: Error) {\n console.error(`🔴 websocket failed: ${err.message}`);\n}\n\nexport default observer;",[],[185,191],{"_102":200,"_95":201,"_156":202,"_158":186,"_159":137,"_160":203,"_161":162,"_104":204,"_99":205,"_164":195,"_166":206,"_167":22,"_168":-5,"_169":207,"_98":208},"34b317de-8b4f-11f0-8154-0224a6c84d84","rtc.ts","0198f1e8-bcdb-709c-8377-5f90981b2f78",22,["D",1757183275113],4,"import { Hono } from \"npm:hono\";\nimport { getErrorText, makeHeaders, makeSession, makeUrl } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new call\nrtc.post(\"/\", async (c) => {\n // Create the call.\n const headers = makeHeaders();\n const fd = new FormData();\n fd.set(\"sdp\", await c.req.text());\n fd.set(\"session\", JSON.stringify(makeSession()));\n const opts = { method: \"POST\", headers: makeHeaders(), body: fd };\n const resp = await fetch(makeUrl(), opts);\n if (!resp.ok) {\n console.error(`🔴 start call failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n\n const contentType = resp.headers.get(\"Content-Type\");\n const location = resp.headers.get(\"Location\");\n const callId = location?.split(\"/\").pop();\n console.log(\"✅ call created:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n const responseHeaders = {\n \"Content-Type\": contentType,\n \"Location\": location,\n };\n return c.newResponse(resp.body, { headers: responseHeaders });\n});\n\nexport default rtc;",[],[185,201],{"_102":210,"_95":211,"_156":212,"_158":186,"_159":137,"_160":203,"_161":162,"_104":213,"_99":117,"_164":195,"_166":214,"_167":22,"_168":-5,"_169":215,"_98":216},"34b31ac2-8b4f-11f0-8154-0224a6c84d84","sip.ts","0198f1e8-bcdb-709c-8377-62fec7493c1b",["D",1757183275113],"import { Hono } from \"npm:hono\";\nimport { Webhook } from \"npm:svix\";\nimport { getErrorText, makeHeaders, makeSession, makeUrl } from \"./utils.ts\";\n\nconst sip = new Hono();\n\n// POST /sip : webhook endpoint\nsip.post(\"/\", async (c) => {\n // Verify the webhook.\n const OPENAI_SIGNING_SECRET = Deno.env.get(\"OPENAI_SIGNING_SECRET\");\n if (!OPENAI_SIGNING_SECRET) {\n console.error(\"🔴 webhook secret not configured\");\n return c.text(\"Internal error\", 500);\n }\n const webhook = new Webhook(OPENAI_SIGNING_SECRET);\n const bodyStr = await c.req.text();\n let callId: string | undefined;\n try {\n const headers = Object.fromEntries(c.req.raw.headers.entries());\n const payload: any = webhook.verify(bodyStr, headers);\n callId = payload.data.call_id;\n console.log(\"✅ verified webhook:\", callId);\n } catch (error) {\n console.error(`🔴 verify failed: ${error}`);\n return c.text(\"Invalid signature\", 401);\n }\n\n // Accept the call.\n const url = makeUrl(callId, \"accept\");\n const headers = makeHeaders(\"application/json\");\n const body = JSON.stringify(makeSession());\n const opts = { method: \"POST\", headers, body };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n console.error(`🔴 accept failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n console.log(\"✅ call accepted:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n return c.newResponse(null);\n});\n\nexport default sip;",[],[185,211],{"_102":218,"_95":219,"_156":220,"_158":186,"_159":137,"_160":221,"_161":162,"_104":222,"_99":223,"_164":195,"_166":224,"_167":22,"_168":-5,"_169":225,"_98":226},"f6983f96-8b4f-11f0-8b67-0224a6c84d84","utils.ts","0198f1e8-bcdb-709c-8377-67d25a0eb80c",25,["D",1757183600419],2,"const REALTIME_BASE_URL = \"https://api.openai.com/v1/realtime\";\nconst MODEL = \"gpt-realtime\";\nconst INSTRUCTIONS = ``;\nconst VOICE = \"marin\";\nconst HANGUP_TOOL = {\n type: \"function\",\n name: \"end_call\",\n description: `\n Use this function to hang up the call when the user says goodbye \n or otherwise indicates they are about to end the call.`,\n};\n\nconst OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\nexport function makeUrl(callId?: string, action?: string) {\n let url = REALTIME_BASE_URL;\n if (!callId) {\n // start call (POST)\n url += \"/calls\";\n } else if (action) {\n // accept, reject, hangup (POST)\n url += `/calls/${callId}/${action}`;\n } else {\n // observer (web socket)\n url += `?call_id=${callId}`;\n }\n return url;\n}\n\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\nexport function makeSession() {\n return {\n type: \"realtime\",\n model: MODEL,\n instructions: INSTRUCTIONS,\n audio: {\n input: { noise_reduction: { type: \"near_field\" } },\n output: { voice: VOICE },\n },\n tools: [HANGUP_TOOL],\n };\n}\n\nexport async function getErrorText(resp: Response) {\n const errBody = await resp.text().catch(() => \"\");\n return `${resp.status} ${resp.statusText}: ${errBody}`;\n}",[],[185,219],[185],{"_102":229,"_95":230,"_156":231,"_158":-5,"_159":137,"_160":232,"_161":162,"_104":233,"_99":177,"_164":178,"_166":234,"_167":22,"_168":-5,"_169":235,"_98":236},"1c19fa04-8a90-11f0-8753-0224a6c84d84","README.md","0198f1e8-bcdb-709c-8377-6c6788b812bb",21,["D",1757101199974],"# hello-realtime\n\n**Hello Realtime** is a OpenAI Realtime app that supports both WebRTC and SIP\n(telephone) users. You can access the app via WebRTC at\n[hello-realtime.val.run](https://hello-realtime.val.run), or via SIP by calling 425-800-0042.\n\nThis demo shows off the new SIP API, the new all-in-one WebRTC API, and the new\nserver-side websocket interface.\n\nIf you remix the app, you'll just need to pop in your own `OPENAI_API_KEY` (from\n[platform.openai.com](https://platform.openai.com)), and if you want SIP, the `OPENAI_SIGNING_SECRET`.\n\n## Architecture\n\n1. **WebRTC Flow**:\n - Browser connects to frontend\n - creates WebRTC offer\n - `/rtc` endpoint handles SDP negotiation with OpenAI\n - observer established to monitor session\n2. **SIP Flow**:\n - Phone calls triggers webhook\n - `/sip` endpoint verifies webhook and accepts call\n - observer established to monitor session\n\n## API Endpoints\n- `GET /` - serves the WebRTC-based app\n- `POST /rtc` - creates a new WebRTC-based Realtime API session\n- `POST /sip` - handles an incoming Realtime API SIP call\n- `POST /observer` - internal endpoint to establish a Realtime API websocket for session control\n\n## Project Structure\n```\n\n├── frontend/ \n│ └── index.html # HTML/JS application for WebRTC\n├── main.tsx # Main entrypoint for the val.town app\n└── routes/\n |── observer.ts # Server-side session control for SIP & WebRTC\n |── rtc.ts # WebRTC session setup\n |── sip.ts # SIP webhook handling and session setup\n └── utils.ts # Realtime API session configuration and helpers\n```\n\n- [`frontend/index.html`](./frontend/index.html)\n- [`main.tsx`](./main.tsx)\n- [`routes/observer.ts`](./routes/observer.ts)\n- [`routes/rtc.ts`](./routes/rtc.ts)\n- [`routes/sip.ts`](./routes/sip.ts)\n- [`routes/utils.ts`](./routes/utils.ts)",[],[230],{"_102":238,"_95":239,"_156":240,"_158":-5,"_159":137,"_160":241,"_161":162,"_104":242,"_99":243,"_164":244,"_166":245,"_167":22,"_168":-5,"_169":246,"_98":247},"b379fc64-86e9-11f0-bb6e-0224a6c84d84","main.tsx","0198f1e8-bcdb-709c-8377-708b32e71bb4",20,["D",1756699874499],9,"http","import { Hono } from \"npm:hono\";\nimport { parseVal, serveFile } from \"https://esm.town/v/std/utils/index.ts\";\n// import mcp from \"./routes/mcp.ts\";\nimport rtc from \"./routes/rtc.ts\";\nimport sip from \"./routes/sip.ts\";\nimport observer from \"./routes/observer.ts\";\n\nconst SOURCE_URL = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\")).links.self.val;\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.route(\"/sip\", sip);\napp.route(\"/observer\", observer);\n// app.route(\"/mcp\", mcp);\napp.get(\"/source\", (c) => c.redirect(SOURCE_URL));\n\napp.get(\"/\", () => serveFile(\"/frontend/index.html\"));\n\nexport default app.fetch;",[],[239],"files",[153,171,183,189,199,209,217,228,237],"counts",{"_252":32,"_253":117,"_254":117,"_255":256},"pulls","branches","forks","envVars",{"_257":117,"_258":32},"count","emptyCount","contents",[261,264,267,270],{"_102":154,"_95":155,"_156":157,"_158":-5,"_159":137,"_160":117,"_161":162,"_104":262,"_99":32,"_164":165,"_166":-5,"_167":22,"_168":-5,"_98":263},["D",1756405218407],[155],{"_102":184,"_95":185,"_156":186,"_158":-5,"_159":137,"_160":117,"_161":162,"_104":265,"_99":32,"_164":165,"_166":-5,"_167":22,"_168":-5,"_98":266},["D",1756405218407],[185],{"_102":229,"_95":230,"_156":231,"_158":-5,"_159":137,"_160":232,"_161":162,"_104":268,"_99":177,"_164":178,"_166":234,"_167":22,"_168":-5,"_98":269},["D",1757101199974],[230],{"_102":238,"_95":239,"_156":240,"_158":-5,"_159":137,"_160":241,"_161":162,"_104":271,"_99":243,"_164":244,"_166":245,"_167":22,"_168":-5,"_98":272},["D",1756699874499],[239],"upstreamProject",{"_102":145,"_95":96,"_110":275,"_111":-5,"_108":109,"_23":276},"Sample app for the OpenAI Realtime API",{"_93":277,"_129":278},"jubertioai","https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/61901958-12bd-49a3-43e9-2f67f1617200/square512","upstreamBranch",{"_102":140,"_104":281,"_282":283,"_95":43,"_138":145,"_99":284,"_139":285,"_141":286,"_287":288,"_106":-5,"_150":22,"_108":109,"_149":277,"_16":147,"_148":96},["D",1755188493466],"updatedAt",["D",1760849643067],372,"e004ac78-4230-11f0-834a-76b3cceeab13",47,"lockfileKey","1760849654205-esm.town_v_jubertioai_hello-realtime@372-main","emptyEnvVars","isNew","flags",[],"routes/_app.x.$handle.$name.environment-variables",{"_93":94,"_95":96,"_100":295,"_255":300},{"_102":103,"_95":96,"_104":296,"_106":-5,"_107":22,"_108":109,"_110":-5,"_111":-5,"_14":112,"_113":-5,"_114":22,"_115":-5,"_116":117,"_16":118,"_119":120,"_121":120,"_122":123,"_124":123,"_125":123,"_126":120,"_127":120,"_23":297,"_131":298,"_134":299,"_150":22},["D",1756405218407],{"_93":94,"_129":130},{"_93":94,"_133":130,"_14":112},{"_99":136,"_95":43,"_102":137,"_138":103,"_139":140,"_141":142,"_143":109,"_144":145,"_146":147,"_148":96,"_149":94},[301],{"_302":303,"_104":304,"_282":305,"_110":306,"_307":22},"key","OPENAI_API_KEY",["D",1756405218407],["D",1756405286018],"https://platform.openai.com/settings/organization/api-keys","isEmpty","actionData","errors"]

Realtime Agent

stevekrouse

hello-realtime

Environment variables