OpenAI Realtime API Voice Agent

\n\n

\n\n \n \n",[],[156,174],[156],{"_103":185,"_95":186,"_157":187,"_159":-5,"_160":138,"_161":118,"_162":163,"_105":188,"_100":32,"_165":166,"_167":-5,"_168":22,"_169":-5,"_170":189,"_98":228},"a6d4939e-843b-11f0-a497-0224a6c84d84","routes","0198f1e8-bcdb-709c-8377-6a376a4311f0",["D",1756405218407],[190,200,210,218],{"_103":191,"_95":192,"_157":193,"_159":187,"_160":138,"_161":137,"_162":163,"_105":194,"_100":195,"_165":196,"_167":197,"_168":22,"_169":-5,"_170":198,"_98":199},"62f8d010-8b50-11f0-8b67-0224a6c84d84","observer.ts","0198f1e8-bcdb-709c-8377-58bd29583a96",["D",1757183782247],5,"script","import { Hono } from \"npm:hono\";\nimport WebSocket from \"npm:ws\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst observer = new Hono();\n\n// POST /observer/:callId : self-connect to establish a web socket\nobserver.post(\"/:callId\", async (c) => {\n const callId = c.req.param(\"callId\");\n const url = makeUrl(callId);\n const ws = new WebSocket(url, { headers: makeHeaders() });\n ws.on(\"open\", () => handleOpen(ws, callId));\n ws.on(\"message\", (data) => handleMessage(ws, callId, JSON.parse(data)));\n ws.on(\"error\", (err) => handleError(ws, callId, err));\n\n // Block forever to keep the socket alive.\n await new Promise(() => {});\n return c.newResponse(null);\n});\n\nfunction handleOpen(ws: WebSocket, _callId: string) {\n console.log(\"✅ websocket connected\");\n // Have the model start talking immediately\n // const message = { type: \"response.create\" };\n // setTimeout(() => ws.send(JSON.stringify(message)), 250);\n}\n\nasync function handleMessage(_ws: WebSocket, callId: string, message: any) {\n if (message.type !== \"response.output_audio_transcript.delta\") {\n console.log(message);\n }\n if (message.type == \"response.function_call_arguments.done\") {\n if (message.name == \"end_call\") {\n doHangup(callId);\n }\n }\n}\n\nasync function doHangup(callId: string) {\n const url = makeUrl(callId, \"hangup\");\n console.log(`? hangup: ${url}`);\n const resp = await fetch(url, { method: \"POST\", headers: makeHeaders() });\n if (!resp.ok) {\n console.error(\n `🔴 end call with id '${callId}'' failed: ${await getErrorText(resp)}`,\n );\n return;\n }\n\n console.log(\"✅ call ended\");\n}\n\nfunction handleError(_ws: WebSocket, _callId: string, err: Error) {\n console.error(`🔴 websocket failed: ${err.message}`);\n}\n\nexport default observer;",[],[186,192],{"_103":201,"_95":202,"_157":203,"_159":187,"_160":138,"_161":204,"_162":163,"_105":205,"_100":206,"_165":196,"_167":207,"_168":22,"_169":-5,"_170":208,"_98":209},"34b317de-8b4f-11f0-8154-0224a6c84d84","rtc.ts","0198f1e8-bcdb-709c-8377-5f90981b2f78",22,["D",1757183275113],4,"import { Hono } from \"npm:hono\";\nimport { getErrorText, makeHeaders, makeSession, makeUrl } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new call\nrtc.post(\"/\", async (c) => {\n // Create the call.\n const headers = makeHeaders();\n const fd = new FormData();\n fd.set(\"sdp\", await c.req.text());\n fd.set(\"session\", JSON.stringify(makeSession()));\n const opts = { method: \"POST\", headers: makeHeaders(), body: fd };\n const resp = await fetch(makeUrl(), opts);\n if (!resp.ok) {\n console.error(`🔴 start call failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n\n const contentType = resp.headers.get(\"Content-Type\");\n const location = resp.headers.get(\"Location\");\n const callId = location?.split(\"/\").pop();\n console.log(\"✅ call created:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n const responseHeaders = {\n \"Content-Type\": contentType,\n \"Location\": location,\n };\n return c.newResponse(resp.body, { headers: responseHeaders });\n});\n\nexport default rtc;",[],[186,202],{"_103":211,"_95":212,"_157":213,"_159":187,"_160":138,"_161":204,"_162":163,"_105":214,"_100":118,"_165":196,"_167":215,"_168":22,"_169":-5,"_170":216,"_98":217},"34b31ac2-8b4f-11f0-8154-0224a6c84d84","sip.ts","0198f1e8-bcdb-709c-8377-62fec7493c1b",["D",1757183275113],"import { Hono } from \"npm:hono\";\nimport { Webhook } from \"npm:svix\";\nimport { getErrorText, makeHeaders, makeSession, makeUrl } from \"./utils.ts\";\n\nconst sip = new Hono();\n\n// POST /sip : webhook endpoint\nsip.post(\"/\", async (c) => {\n // Verify the webhook.\n const OPENAI_SIGNING_SECRET = Deno.env.get(\"OPENAI_SIGNING_SECRET\");\n if (!OPENAI_SIGNING_SECRET) {\n console.error(\"🔴 webhook secret not configured\");\n return c.text(\"Internal error\", 500);\n }\n const webhook = new Webhook(OPENAI_SIGNING_SECRET);\n const bodyStr = await c.req.text();\n let callId: string | undefined;\n try {\n const headers = Object.fromEntries(c.req.raw.headers.entries());\n const payload: any = webhook.verify(bodyStr, headers);\n callId = payload.data.call_id;\n console.log(\"✅ verified webhook:\", callId);\n } catch (error) {\n console.error(`🔴 verify failed: ${error}`);\n return c.text(\"Invalid signature\", 401);\n }\n\n // Accept the call.\n const url = makeUrl(callId, \"accept\");\n const headers = makeHeaders(\"application/json\");\n const body = JSON.stringify(makeSession());\n const opts = { method: \"POST\", headers, body };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n console.error(`🔴 accept failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n console.log(\"✅ call accepted:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n return c.newResponse(null);\n});\n\nexport default sip;",[],[186,212],{"_103":219,"_95":220,"_157":221,"_159":187,"_160":138,"_161":222,"_162":163,"_105":223,"_100":224,"_165":196,"_167":225,"_168":22,"_169":-5,"_170":226,"_98":227},"f6983f96-8b4f-11f0-8b67-0224a6c84d84","utils.ts","0198f1e8-bcdb-709c-8377-67d25a0eb80c",25,["D",1757183600419],2,"const REALTIME_BASE_URL = \"https://api.openai.com/v1/realtime\";\nconst MODEL = \"gpt-realtime\";\nconst INSTRUCTIONS = ``;\nconst VOICE = \"marin\";\nconst HANGUP_TOOL = {\n type: \"function\",\n name: \"end_call\",\n description: `\n Use this function to hang up the call when the user says goodbye \n or otherwise indicates they are about to end the call.`,\n};\n\nconst OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\nexport function makeUrl(callId?: string, action?: string) {\n let url = REALTIME_BASE_URL;\n if (!callId) {\n // start call (POST)\n url += \"/calls\";\n } else if (action) {\n // accept, reject, hangup (POST)\n url += `/calls/${callId}/${action}`;\n } else {\n // observer (web socket)\n url += `?call_id=${callId}`;\n }\n return url;\n}\n\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\nexport function makeSession() {\n return {\n type: \"realtime\",\n model: MODEL,\n instructions: INSTRUCTIONS,\n audio: {\n input: { noise_reduction: { type: \"near_field\" } },\n output: { voice: VOICE },\n },\n tools: [HANGUP_TOOL],\n };\n}\n\nexport async function getErrorText(resp: Response) {\n const errBody = await resp.text().catch(() => \"\");\n return `${resp.status} ${resp.statusText}: ${errBody}`;\n}",[],[186,220],[186],{"_103":230,"_95":231,"_157":232,"_159":-5,"_160":138,"_161":233,"_162":163,"_105":234,"_100":178,"_165":179,"_167":235,"_168":22,"_169":-5,"_170":236,"_98":237},"1c19fa04-8a90-11f0-8753-0224a6c84d84","README.md","0198f1e8-bcdb-709c-8377-6c6788b812bb",21,["D",1757101199974],"# hello-realtime\n\n**Hello Realtime** is a OpenAI Realtime app that supports both WebRTC and SIP\n(telephone) users. You can access the app via WebRTC at\n[hello-realtime.val.run](https://hello-realtime.val.run), or via SIP by calling 425-800-0042.\n\nThis demo shows off the new SIP API, the new all-in-one WebRTC API, and the new\nserver-side websocket interface.\n\nIf you remix the app, you'll just need to pop in your own `OPENAI_API_KEY` (from\n[platform.openai.com](https://platform.openai.com)), and if you want SIP, the `OPENAI_SIGNING_SECRET`.\n\n## Architecture\n\n1. **WebRTC Flow**:\n - Browser connects to frontend\n - creates WebRTC offer\n - `/rtc` endpoint handles SDP negotiation with OpenAI\n - observer established to monitor session\n2. **SIP Flow**:\n - Phone calls triggers webhook\n - `/sip` endpoint verifies webhook and accepts call\n - observer established to monitor session\n\n## API Endpoints\n- `GET /` - serves the WebRTC-based app\n- `POST /rtc` - creates a new WebRTC-based Realtime API session\n- `POST /sip` - handles an incoming Realtime API SIP call\n- `POST /observer` - internal endpoint to establish a Realtime API websocket for session control\n\n## Project Structure\n```\n\n├── frontend/ \n│ └── index.html # HTML/JS application for WebRTC\n├── main.tsx # Main entrypoint for the val.town app\n└── routes/\n |── observer.ts # Server-side session control for SIP & WebRTC\n |── rtc.ts # WebRTC session setup\n |── sip.ts # SIP webhook handling and session setup\n └── utils.ts # Realtime API session configuration and helpers\n```\n\n- [`frontend/index.html`](./frontend/index.html)\n- [`main.tsx`](./main.tsx)\n- [`routes/observer.ts`](./routes/observer.ts)\n- [`routes/rtc.ts`](./routes/rtc.ts)\n- [`routes/sip.ts`](./routes/sip.ts)\n- [`routes/utils.ts`](./routes/utils.ts)",[],[231],{"_103":239,"_95":240,"_157":241,"_159":-5,"_160":138,"_161":242,"_162":163,"_105":243,"_100":244,"_165":245,"_167":246,"_168":22,"_169":-5,"_170":247,"_98":248},"b379fc64-86e9-11f0-bb6e-0224a6c84d84","main.tsx","0198f1e8-bcdb-709c-8377-708b32e71bb4",20,["D",1756699874499],9,"http","import { Hono } from \"npm:hono\";\nimport { parseVal, serveFile } from \"https://esm.town/v/std/utils/index.ts\";\n// import mcp from \"./routes/mcp.ts\";\nimport rtc from \"./routes/rtc.ts\";\nimport sip from \"./routes/sip.ts\";\nimport observer from \"./routes/observer.ts\";\n\nconst SOURCE_URL = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\")).links.self.val;\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.route(\"/sip\", sip);\napp.route(\"/observer\", observer);\n// app.route(\"/mcp\", mcp);\napp.get(\"/source\", (c) => c.redirect(SOURCE_URL));\n\napp.get(\"/\", () => serveFile(\"/frontend/index.html\"));\n\nexport default app.fetch;",[],[240],"files",[154,172,184,190,200,210,218,229,238],"counts",{"_253":32,"_254":118,"_255":118,"_256":118},"pulls","branches","forks","envVars","contents",[],{"_103":173,"_160":138,"_157":175,"_95":174,"_167":180,"_165":179,"_100":178,"_159":158,"_105":260,"_161":176,"_169":-5,"_98":261},["D",1757183484678],[156,174],"upstreamProject",{"_103":146,"_95":96,"_111":264,"_112":-5,"_109":110,"_23":265},"Sample app for the OpenAI Realtime API",{"_93":266,"_130":267},"jubertioai","https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/61901958-12bd-49a3-43e9-2f67f1617200/square512","upstreamBranch",{"_103":141,"_105":270,"_271":272,"_95":43,"_139":146,"_100":273,"_140":274,"_142":275,"_276":277,"_107":-5,"_151":22,"_109":110,"_150":266,"_16":148,"_149":96},["D",1755188493466],"updatedAt",["D",1757729497613],297,"e004ac78-4230-11f0-834a-76b3cceeab13",47,"lockfileKey","1757729506929-esm.town_v_jubertioai_hello-realtime@297-main","emptyEnvVars","isNew","flags",[],"routes/_app.x.$handle.$name.branch.$branch.history_.$",{"_93":94,"_95":96,"_97":43,"_98":99,"_101":284,"_179":289,"_292":293,"_333":334,"_345":-7,"_346":32,"_347":22},{"_103":104,"_95":96,"_105":285,"_107":-5,"_108":22,"_109":110,"_111":-5,"_112":-5,"_14":113,"_114":-5,"_115":22,"_116":-5,"_117":118,"_16":119,"_120":121,"_122":121,"_123":124,"_125":124,"_126":124,"_127":121,"_128":121,"_23":286,"_132":287,"_135":288,"_151":22},["D",1756405218407],{"_93":94,"_130":131},{"_93":94,"_134":131},{"_100":137,"_95":43,"_103":138,"_139":104,"_140":141,"_142":143,"_144":110,"_145":146,"_147":148,"_149":96,"_150":94},{"_103":173,"_95":174,"_157":175,"_159":158,"_160":138,"_161":176,"_162":163,"_105":290,"_100":178,"_165":179,"_167":180,"_168":22,"_169":-5,"_98":291},["D",1757183484678],[156,174],"history",[294,296,301,306,310,314,319,324,329],{"_103":173,"_160":138,"_157":175,"_95":174,"_167":180,"_165":179,"_100":178,"_159":158,"_105":295,"_161":176,"_169":-5},["D",1757183484678],{"_103":297,"_160":138,"_157":175,"_95":174,"_167":180,"_165":179,"_100":298,"_159":158,"_105":299,"_161":300,"_169":-5},"a67e4ce4-8b4f-11f0-b9be-0224a6c84d84",7,["D",1757183466031],23,{"_103":302,"_160":138,"_157":175,"_95":174,"_167":303,"_165":179,"_100":304,"_159":158,"_105":305,"_161":204,"_169":-5},"34b3026c-8b4f-11f0-8154-0224a6c84d84","\n\n \n \n \n OpenAI Realtime API Voice Agent\n \n \n \n \n\n

\n\n

\n\n \n \n",6,["D",1757183275113],{"_103":307,"_160":138,"_157":175,"_95":174,"_167":308,"_165":179,"_100":195,"_159":158,"_105":309,"_161":233,"_169":-5},"1c1a0288-8a90-11f0-8753-0224a6c84d84","\n\n \n \n \n OpenAI Realtime API Voice Agent\n \n \n \n \n\n

\n\n

\n\n \n \n",["D",1757101199974],{"_103":311,"_160":138,"_157":175,"_95":174,"_167":312,"_165":179,"_100":206,"_159":158,"_105":313,"_161":242,"_169":-5},"b379f098-86e9-11f0-bb6e-0224a6c84d84","\n\n \n \n \n OpenAI Realtime API Voice Agent\n \n \n \n \n\n

\n\n

\n\n \n \n",["D",1756699874499],{"_103":315,"_160":138,"_157":175,"_95":174,"_167":312,"_165":179,"_100":316,"_159":158,"_105":317,"_161":318,"_169":-5},"99da75d4-84f7-11f0-9346-0224a6c84d84",3,["D",1756485942140],15,{"_103":320,"_160":138,"_157":175,"_95":174,"_167":321,"_165":179,"_100":224,"_159":158,"_105":322,"_161":323,"_169":-5},"60ea1338-84f7-11f0-a8b5-0224a6c84d84","\n\n \n \n \n OpenAI Realtime API Voice Agent\n \n \n \n \n\n

\n\n

\n\n \n \n",["D",1756485846608],13,{"_103":325,"_160":138,"_157":175,"_95":174,"_167":326,"_165":179,"_100":118,"_159":158,"_105":327,"_161":328,"_169":-5},"5dacf8f2-84f7-11f0-a553-0224a6c84d84","\n\n \n \n \n OpenAI Realtime API Voice Agent\n \n \n \n \n\n

\n\n

\n\n \n \n",["D",1756485841168],12,{"_103":330,"_160":138,"_157":175,"_95":174,"_167":331,"_165":179,"_100":32,"_159":158,"_105":332,"_161":118,"_169":-5},"a6d47102-843b-11f0-a497-0224a6c84d84","\n\n \n \n \n OpenAI Realtime API Voice Agent\n \n \n \n \n\n

\n\n

\n\n \n \n",["D",1756405218407],"versions",[335,337,338,339,340,341,342,343,344],{"_179":294,"_336":296},"previous",{"_179":296,"_336":301},{"_179":301,"_336":306},{"_179":306,"_336":310},{"_179":310,"_336":314},{"_179":314,"_336":319},{"_179":319,"_336":324},{"_179":324,"_336":329},{"_179":329,"_336":-5},"versionNumber","page","hasNext","actionData","errors"]

Realtime Agent

Realtime Agent

Realtime Agent

Realtime Agent

Realtime Agent

Realtime Agent

Realtime Agent

stevekrouse

hello-realtime

History for index.html