OpenAI Realtime API Voice Agent

\n\n

\n\n \n \n",[18,68],"pulls","branches","forks","envVars",{"_180":182,"_183":40},"connections",{"_180":26,"_181":26},"count","unfilledCount",2,"emptyCount",{"_33":161,"_13":18,"_66":74,"_43":-5,"_35":36,"_162":40,"_163":24,"_37":254,"_39":26,"_69":75,"_165":-5,"_70":30,"_166":-5,"_199":255,"_17":256},{"_33":169,"_13":68,"_66":67,"_43":74,"_35":36,"_162":40,"_163":24,"_37":251,"_39":26,"_69":21,"_165":171,"_70":30,"_166":-5,"_199":252,"_17":253},{"_33":247,"_13":218,"_66":212,"_43":-5,"_35":36,"_162":40,"_163":24,"_37":248,"_39":26,"_69":75,"_165":-5,"_70":30,"_166":-5,"_199":249,"_17":250},{"_33":240,"_13":241,"_66":242,"_43":212,"_35":36,"_162":110,"_163":24,"_37":243,"_39":182,"_69":214,"_165":244,"_70":30,"_166":-5,"_199":245,"_17":246},{"_33":233,"_13":234,"_66":235,"_43":212,"_35":36,"_162":40,"_163":24,"_37":236,"_39":26,"_69":214,"_165":237,"_70":30,"_166":-5,"_199":238,"_17":239},{"_33":226,"_13":227,"_66":228,"_43":212,"_35":36,"_162":40,"_163":24,"_37":229,"_39":26,"_69":214,"_165":230,"_70":30,"_166":-5,"_199":231,"_17":232},{"_33":219,"_13":220,"_66":221,"_43":212,"_35":36,"_162":40,"_163":24,"_37":222,"_39":26,"_69":214,"_165":223,"_70":30,"_166":-5,"_199":224,"_17":225},{"_33":209,"_13":210,"_66":211,"_43":212,"_35":36,"_162":40,"_163":24,"_37":213,"_39":26,"_69":214,"_165":215,"_70":30,"_166":-5,"_199":216,"_17":217},{"_33":202,"_13":203,"_66":204,"_43":-5,"_35":36,"_162":40,"_163":24,"_37":205,"_39":26,"_69":21,"_165":206,"_70":30,"_166":-5,"_199":207,"_17":208},{"_33":194,"_13":195,"_66":86,"_43":-5,"_35":36,"_162":40,"_163":24,"_37":196,"_39":26,"_69":197,"_165":198,"_70":30,"_166":-5,"_199":200,"_17":201},"e7085d28-8e1e-11f0-a66e-0224a6c84d84","main.tsx",["D",1757492382377],"http","import { Hono } from \"npm:hono\";\nimport { parseVal, serveFile } from \"https://esm.town/v/std/utils/index.ts\";\n// import mcp from \"./routes/mcp.ts\";\nimport rtc from \"./routes/rtc.ts\";\nimport sip from \"./routes/sip.ts\";\nimport observer from \"./routes/observer.ts\";\n\nconst SOURCE_URL = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\")).links.self.val;\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.route(\"/sip\", sip);\napp.route(\"/observer\", observer);\n// app.route(\"/mcp\", mcp);\napp.get(\"/source\", (c) => c.redirect(SOURCE_URL));\n\napp.get(\"/\", () => serveFile(\"/frontend/index.html\"));\n\nexport default app.fetch;","children",[],[195],"e7085a8a-8e1e-11f0-a66e-0224a6c84d84","README.md","019932b5-8afa-7298-b0b7-04c55fa0746d",["D",1757492382377],"# hello-realtime\n\n**Hello Realtime** is a OpenAI Realtime app that supports both WebRTC and SIP\n(telephone) users. You can access the app via WebRTC at\n[hello-realtime.val.run](https://hello-realtime.val.run), or via SIP by calling\n425-800-0042.\n\nThis demo shows off the new SIP API, the new all-in-one WebRTC API, and the new\nserver-side websocket interface.\n\nIf you remix the app, you'll just need to pop in your own `OPENAI_API_KEY` (from\n[platform.openai.com](https://platform.openai.com)), and if you want SIP, the\n`OPENAI_SIGNING_SECRET`.\n\n## Architecture\n\n1. **WebRTC Flow**:\n - Browser connects to frontend\n - creates WebRTC offer\n - `/rtc` endpoint handles SDP negotiation with OpenAI\n - observer established to monitor session\n2. **SIP Flow**:\n - Phone calls triggers webhook\n - `/sip` endpoint verifies webhook and accepts call\n - observer established to monitor session\n\n## API Endpoints\n\n- `GET /` - serves the WebRTC-based app\n- `POST /rtc` - creates a new WebRTC-based Realtime API session\n- `POST /sip` - handles an incoming Realtime API SIP call\n- `POST /observer` - internal endpoint to establish a Realtime API websocket for\n session control\n\n## Project Structure\n\n```\n├── frontend/ \n│ └── index.html # HTML/JS application for WebRTC\n├── main.tsx # Main entrypoint for the val.town app\n└── routes/\n |\n ├── agent.ts # Realtime API session configuration and tools\n ├── observer.ts # Server-side session control for SIP & WebRTC\n ├── rtc.ts # WebRTC session setup\n ├── sip.ts # SIP webhook handling and session setup\n └── utils.ts # Various HTTP helpers\n```\n\n## File Links\n\n- [`frontend/index.html`](./frontend/index.html)\n- [`main.tsx`](./main.tsx)\n- [`routes/agent.ts`](./routes/agent.ts)\n- [`routes/observer.ts`](./routes/observer.ts)\n- [`routes/rtc.ts`](./routes/rtc.ts)\n- [`routes/sip.ts`](./routes/sip.ts)\n- [`routes/utils.ts`](./routes/utils.ts)",[],[203],"e7085544-8e1e-11f0-a66e-0224a6c84d84","utils.ts","019932b5-8afa-7298-b0b6-fdc76bf3b919","019932b5-8afa-7298-b0b7-0393fd22400c",["D",1757492382377],"script","const REALTIME_BASE_URL = \"https://api.openai.com/v1/realtime\";\nconst OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\n// Builds the URL for a Realtime API endpoint.\nexport function makeUrl(callId?: string, action?: string) {\n let url = REALTIME_BASE_URL;\n if (!callId) {\n // start call (POST)\n url += \"/calls\";\n } else if (action) {\n // accept, reject, hangup (POST)\n url += `/calls/${callId}/${action}`;\n } else {\n // observer (web socket)\n url += `?call_id=${callId}`;\n }\n return url;\n}\n\n// Builds the headers for a Realtime API endpoint.\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\n// Debug helper for printing Realtime API errors.\nexport async function getErrorText(resp: Response) {\n const errBody = await resp.text().catch(() => \"\");\n return `${resp.status} ${resp.statusText}: ${errBody}`;\n}",[],[218,210],"routes","e7085440-8e1e-11f0-a66e-0224a6c84d84","sip.ts","019932b5-8afa-7298-b0b6-f95f0d965b11",["D",1757492382377],"import { Hono } from \"npm:hono\";\nimport { Webhook } from \"npm:svix\";\nimport { makeSession } from \"./agent.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst sip = new Hono();\n\n// POST /sip : webhook endpoint\nsip.post(\"/\", async (c) => {\n // Verify the webhook.\n const OPENAI_SIGNING_SECRET = Deno.env.get(\"OPENAI_SIGNING_SECRET\");\n if (!OPENAI_SIGNING_SECRET) {\n console.error(\"🔴 webhook secret not configured\");\n return c.text(\"Internal error\", 500);\n }\n const webhook = new Webhook(OPENAI_SIGNING_SECRET);\n const bodyStr = await c.req.text();\n let callId: string | undefined;\n try {\n const headers = Object.fromEntries(c.req.raw.headers.entries());\n const payload: any = webhook.verify(bodyStr, headers);\n callId = payload.data.call_id;\n console.log(\"✅ verified webhook:\", callId);\n } catch (error) {\n console.error(`🔴 verify failed: ${error}`);\n return c.text(\"Invalid signature\", 401);\n }\n\n // Accept the call.\n const url = makeUrl(callId, \"accept\");\n const headers = makeHeaders(\"application/json\");\n const body = JSON.stringify(makeSession());\n const opts = { method: \"POST\", headers, body };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n console.error(`🔴 accept failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n console.log(\"✅ call accepted:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n return c.newResponse(null);\n});\n\nexport default sip;",[],[218,220],"e70852f6-8e1e-11f0-a66e-0224a6c84d84","rtc.ts","019932b5-8afa-7298-b0b6-f63129b59562",["D",1757492382377],"import { Hono } from \"npm:hono\";\nimport { makeSession } from \"./agent.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new call\nrtc.post(\"/\", async (c) => {\n // Create the call.\n const headers = makeHeaders();\n const fd = new FormData();\n fd.set(\"sdp\", await c.req.text());\n fd.set(\"session\", JSON.stringify(makeSession()));\n const opts = { method: \"POST\", headers: makeHeaders(), body: fd };\n const resp = await fetch(makeUrl(), opts);\n if (!resp.ok) {\n console.error(`🔴 start call failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n\n const contentType = resp.headers.get(\"Content-Type\");\n const location = resp.headers.get(\"Location\");\n const callId = location?.split(\"/\").pop();\n console.log(\"✅ call created:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n const responseHeaders = {\n \"Content-Type\": contentType,\n \"Location\": location,\n };\n return c.newResponse(resp.body, { headers: responseHeaders });\n});\n\nexport default rtc;",[],[218,227],"e7085062-8e1e-11f0-a66e-0224a6c84d84","observer.ts","019932b5-8afa-7298-b0b6-f1502d78ef49",["D",1757492382377],"import { Hono } from \"npm:hono\";\nimport WebSocket from \"npm:ws\";\nimport { invokeTool } from \"./agent.ts\";\nimport { makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst observer = new Hono();\n\nclass ObserverClient {\n private ws: WebSocket;\n private ending = false;\n private closeResolve: () => void;\n private closePromise: Promise;\n\n constructor(private callId: string) {\n const url = makeUrl(callId);\n this.ws = new WebSocket(url, { headers: makeHeaders() });\n this.ws.on(\"open\", () => this.handleOpen());\n this.ws.on(\"message\", (data) => this.handleMessage(JSON.parse(data)));\n this.ws.on(\"error\", (err) => this.handleError(err));\n this.ws.on(\"close\", (ev) => this.handleClose(ev));\n this.closePromise = new Promise((resolve) =>\n this.closeResolve = resolve\n );\n }\n async waitForClose() {\n await this.closePromise;\n }\n private handleOpen() {\n console.log(\"✅ websocket connected\");\n const message = { type: \"response.create\" };\n setTimeout(() => this.ws.send(JSON.stringify(message)), 250);\n }\n private async handleMessage(message: any) {\n if (message.type !== \"response.output_audio_transcript.delta\") {\n console.log(message);\n }\n if (message.type == \"response.function_call_arguments.done\") {\n if (message.name == \"end_call\") {\n this.ending = true;\n }\n } else if (message.type == \"output_audio_buffer.stopped\") {\n if (this.ending) {\n await invokeTool(this.callId, \"end_call\", {});\n console.log(\"websocket closing\");\n this.ws.close();\n }\n }\n }\n private handleError(err: Error) {\n console.error(`🔴 websocket failed: ${err.message}`);\n this.closeResolve();\n }\n private handleClose(e: Event) {\n console.log(`websocket closed: ${e}`);\n this.closeResolve();\n }\n}\n\n// POST /observer/:callId : self-connect to establish a web socket\nobserver.post(\"/:callId\", async (c) => {\n const callId = c.req.param(\"callId\");\n const client = new ObserverClient(callId);\n await client.waitForClose();\n return c.newResponse(null);\n});\n\nexport default observer;",[],[218,234],"0b7af834-8e22-11f0-9cef-0224a6c84d84","agent.ts","019932b5-8afa-7298-b0b6-ec884f1e23ef",["D",1757493732101],"import { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst MODEL = \"gpt-realtime\";\nconst INSTRUCTIONS = `\n Greet the user in English, and thank them wanting to list on goldenpages.ie.\n Give them a brief summary based on the list below, and then ask if they have any questions.\n Answer questions using the information below. For questions outside this scope,\n apologize and say you can't help with that.\n ---\n Short summary:\n - Any Irish registered business can avail of a free listing per location for free.\n We neeed Business Name, Business Address, Business eircode , Business Phone Number, Business E mail and business Type .\n Full list of available Fields:\n - Short Description\n -Long Description\n - Photos\n - Videos\n - Service Location\n - Website\n - Business Type\n - Business Name\n - Business Address\n - Business Eircode\n - Business Phone Number\n - Business Email\n - Business Hours';\nconst VOICE = \"marin\";\nconst HANGUP_TOOL = {\n type: \"function\",\n name: \"end_call\",\n description: `;\nconst VOICE = \"marin\";\nconst HANGUP_TOOL = {\n type: \"function\",\n name: \"end_call\",\n description: `\n Use this function to hang up the call when the user says goodbye \n or otherwise indicates they are about to end the call.`,\n};\n// Builds the session configuration for a Realtime API session.\nexport function makeSession() {\n return {\n type: \"realtime\",\n model: MODEL,\n instructions: INSTRUCTIONS,\n audio: {\n input: { noise_reduction: { type: \"near_field\" } },\n output: { voice: VOICE },\n },\n tools: [HANGUP_TOOL],\n };\n}\n\n// Handles a Realtime API tool call.\nexport async function invokeTool(callId: string, name: string, args: any) {\n if (name == HANGUP_TOOL.name) {\n await doHangup(callId);\n }\n}\n\nasync function doHangup(callId: string) {\n const url = makeUrl(callId, \"hangup\");\n const resp = await fetch(url, { method: \"POST\", headers: makeHeaders() });\n if (!resp.ok) {\n console.error(`🔴 end call failed: ${await getErrorText(resp)}`);\n return;\n }\n\n console.log(\"✅ call ended\");\n}",[],[218,241],"e7085940-8e1e-11f0-a66e-0224a6c84d84",["D",1757492382377],[187,188,189,190,191],[218],["D",1757492382377],[],[18,68],["D",1757492382377],[185],[18],["D",1757492382377],{"_11":12,"_126":73},{"_11":12,"_122":-5,"_72":73,"_31":71,"_123":124,"_125":30},{"_39":110,"_13":16,"_33":36,"_111":77,"_112":63,"_113":114,"_115":81,"_116":117,"_118":119,"_120":14,"_121":12},"ctx",{"_52":-5,"_147":294,"_295":296,"_297":298,"_299":30,"_300":30,"_301":30,"_302":24,"_303":-5,"_304":-7,"_305":-7,"_306":26,"_307":30,"_308":30,"_309":-5,"_310":30,"_311":30,"_312":30,"_313":-5},"ENV",{"_265":266,"_267":268,"_269":16,"_270":271,"_272":273,"_274":275,"_276":277,"_278":279,"_280":281,"_282":283,"_284":285,"_286":287,"_288":289,"_290":291,"_292":293},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","811aa22498c9b0b5a47e8d778de579cdb990f332","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Realtime Agent

History · frontend