OpenAI Realtime API Voice Agent

\n\n

\n\n \n \n",[],[155,174],[155],{"_103":183,"_95":184,"_156":185,"_158":-5,"_159":137,"_160":161,"_162":163,"_105":186,"_100":32,"_165":166,"_167":-5,"_168":22,"_169":-5,"_170":187,"_98":222},"3fec50c2-8508-11f0-a1fe-0224a6c84d84","routes","0198f725-9761-70a8-bb33-ed541bc4dfaa",["D",1756493092620],[188,197,205,213],{"_103":189,"_95":190,"_156":191,"_158":185,"_159":137,"_160":161,"_162":163,"_105":192,"_100":32,"_165":193,"_167":194,"_168":22,"_169":-5,"_170":195,"_98":196},"3fec471c-8508-11f0-a1fe-0224a6c84d84","observer.ts","0198f725-9761-70a8-bb33-dd018e79b788",["D",1756493092620],"script","import { Hono } from \"npm:hono\";\nimport WebSocket from \"npm:ws\";\nimport { makeHeaders } from \"./utils.ts\";\n\nconst observer = new Hono();\n\n// POST /observer/:callId : self-connect to establish a web socket\nobserver.post(\"/:callId\", async (c) => {\n const callId = c.req.param(\"callId\");\n const url = `wss://api.openai.com/v1/realtime?call_id=${callId}`;\n const ws = new WebSocket(url, { headers: makeHeaders() });\n ws.on(\"open\", () => {\n console.log(\"✅ websocket connected\");\n setTimeout(() => ws.send(JSON.stringify({ type: \"response.create\" })), 250);\n });\n ws.on(\"message\", (data /*, isBinary */) => {\n const message = JSON.parse(data);\n if (message.type !== \"response.audio_transcript.delta\") {\n console.log(message);\n }\n });\n ws.on(\"error\", (err: Error) => {\n console.error(`🔴 websocket failed: ${err.message}`);\n });\n\n // Respond immediately; WS continues in background\n return c.newResponse(null);\n});\n\nexport default observer;",[],[184,190],{"_103":198,"_95":199,"_156":200,"_158":185,"_159":137,"_160":161,"_162":163,"_105":201,"_100":32,"_165":193,"_167":202,"_168":22,"_169":-5,"_170":203,"_98":204},"3fec4a78-8508-11f0-a1fe-0224a6c84d84","rtc.ts","0198f725-9761-70a8-bb33-e3cb8a4465f9",["D",1756493092620],"import { Hono } from \"npm:hono\";\nimport { makeHeaders, makeSession } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new call\nrtc.post(\"/\", async (c) => {\n // Create the call.\n const url = \"https://api.openai.com/v1/realtime/calls\";\n const headers = makeHeaders();\n const fd = new FormData();\n fd.set(\"sdp\", await c.req.text());\n fd.set(\"session\", JSON.stringify(makeSession()));\n const opts = { method: \"POST\", headers, body: fd };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n const errText = await resp.text().catch(() => \"\");\n console.error(`🔴 start call failed: ${resp.status} ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n\n const contentType = resp.headers.get(\"Content-Type\");\n const location = resp.headers.get(\"Location\");\n const callId = location?.split(\"/\").pop();\n console.log(\"✅ call created:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" });\n\n return c.newResponse(resp.body, {\n headers: { \"Content-Type\": contentType },\n });\n});\n\nexport default rtc;",[],[184,199],{"_103":206,"_95":207,"_156":208,"_158":185,"_159":137,"_160":161,"_162":163,"_105":209,"_100":32,"_165":193,"_167":210,"_168":22,"_169":-5,"_170":211,"_98":212},"3fec4c3a-8508-11f0-a1fe-0224a6c84d84","sip.ts","0198f725-9761-70a8-bb33-e72059582aa8",["D",1756493092620],"import { Hono } from \"npm:hono\";\nimport { Webhook } from \"npm:svix\";\nimport { makeHeaders, makeSession } from \"./utils.ts\";\n\nconst sip = new Hono();\n\n// POST /sip : webhook endpoint\nsip.post(\"/\", async (c) => {\n // Verify the webhook.\n const OPENAI_SIGNING_SECRET = Deno.env.get(\"OPENAI_SIGNING_SECRET\");\n if (!OPENAI_SIGNING_SECRET) {\n console.error(\"🔴 webhook secret not configured\");\n return c.text(\"Internal error\", 500);\n }\n const webhook = new Webhook(OPENAI_SIGNING_SECRET);\n const bodyStr = await c.req.text();\n let callId: string | undefined;\n try {\n const headers = Object.fromEntries(c.req.raw.headers.entries());\n const payload: any = webhook.verify(bodyStr, headers);\n callId = payload.data.call_id;\n console.log(\"✅ verified webhook:\", callId);\n } catch (error) {\n console.error(`🔴 verify failed: ${error}`);\n return c.text(\"Invalid signature\", 401);\n }\n\n // Accept the call.\n const url = `https://api.openai.com/v1/realtime/calls/${callId}/accept`;\n const headers = makeHeaders(\"application/json\");\n const body = JSON.stringify(makeSession());\n const opts = { method: \"POST\", headers, body };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n const errText = await resp.text().catch(() => \"\");\n console.error(`🔴 accept failed: ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n console.log(\"✅ call accepted:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer trigger failed:\", e));\n\n return c.newResponse(null);\n});\n\nexport default sip;",[],[184,207],{"_103":214,"_95":215,"_156":216,"_158":185,"_159":137,"_160":217,"_162":163,"_105":218,"_100":161,"_165":193,"_167":219,"_168":22,"_169":-5,"_170":220,"_98":221},"d63b3674-8508-11f0-a900-0224a6c84d84","utils.ts","0198f725-9761-70a8-bb33-ea8763655db6",2,["D",1756493344882],"const MODEL = \"gpt-realtime\";\nconst INSTRUCTIONS = `\nWrite the instructions prompt, for an English - Kannada teacher. \nYou'll start with simple instructions that are used in day-to-day life, say it in English, then in Kannada, and then you'll explain the words used. \nThen you'll ask a question in Kannada to the user, based on what they just learned. \nIf their response is correct, move on to the next phrase. \nIf response is incorrect, tell them what's wrong. \nThen continue quizzing in an alternate version of the question, until they get it right. \nBefore you begin the conversation, create a set of phrases (about 30 or so), by asking the user their current proficiency and assessing them. \nWe want conversational Kannada - someone who's new to the state should be able to navigate daily life easily with this. \nWhen speaking to the user, use a mix of Kannada and English. Mix up phrases from both language in a way a bilingual person would naturally do.`;\nconst VOICE = \"marin\";\n\nconst OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\nexport function makeSession() {\n return {\n type: \"realtime\",\n model: MODEL,\n instructions: INSTRUCTIONS,\n audio: {\n input: { noise_reduction: { type: \"near_field\" } },\n output: { voice: VOICE },\n },\n };\n}",[],[184,215],[184],{"_103":224,"_95":225,"_156":226,"_158":-5,"_159":137,"_160":161,"_162":163,"_105":227,"_100":32,"_165":177,"_167":228,"_168":22,"_169":-5,"_170":229,"_98":230},"3fec5234-8508-11f0-a1fe-0224a6c84d84","README.md","0198f725-9761-70a8-bb33-f127a4ca8732",["D",1756493092620],"# hello-realtime\n\nHello Realtime is a complete OpenAI Realtime application that supports both WebRTC and SIP users.\nYou can access the app via WebRTC at https://hello-realtime.val.run, or via SIP by calling 425-800-0042.\n\nThis demo shows off the new SIP API, the new all-in-one WebRTC API, and the new server-side websocket interface.\n\nIf you remix the app, you'll just need to pop in your own OPENAI_API_KEY (from platform.openai.com), and also the OPENAI_SIGNING_SECRET if you want to use the SIP interface.",[],[225],{"_103":232,"_95":233,"_156":234,"_158":-5,"_159":137,"_160":161,"_162":163,"_105":235,"_100":32,"_165":236,"_167":237,"_168":22,"_169":-5,"_170":238,"_98":239},"3fec53ce-8508-11f0-a1fe-0224a6c84d84","main.tsx","0198f725-9761-70a8-bb33-f679a7d74b15",["D",1756493092620],"http","import { Hono } from \"npm:hono\";\nimport {\n readFile,\n serveFile,\n} from \"https://esm.town/v/std/utils@85-main/index.ts\";\n// import mcp from \"./routes/mcp.ts\";\nimport rtc from \"./routes/rtc.ts\";\nimport sip from \"./routes/sip.ts\";\nimport observer from \"./routes/observer.ts\";\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.route(\"/sip\", sip);\napp.route(\"/observer\", observer);\n// app.route(\"/mcp\", mcp);\n\napp.get(\"/\", async (c) => {\n const html = await readFile(\"/frontend/index.html\", import.meta.url);\n return c.html(html);\n});\n\nexport default app.fetch;",[],[233],"files",[153,172,182,188,197,205,213,223,231],"counts",{"_244":32,"_245":161,"_246":32,"_247":217},"pulls","branches","forks","envVars","contents",[],{"_103":214,"_159":137,"_156":216,"_95":215,"_167":219,"_165":193,"_100":161,"_158":185,"_105":251,"_160":217,"_169":-5,"_98":252},["D",1756493344882],[184,215],"upstreamProject",{"_103":145,"_95":255,"_111":256,"_112":-5,"_109":110,"_23":257},"hello-realtime","Sample app for the OpenAI Realtime API",{"_93":258,"_129":259},"jubertioai","https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/61901958-12bd-49a3-43e9-2f67f1617200/square512","upstreamBranch",{"_103":140,"_105":262,"_263":264,"_95":43,"_138":145,"_100":265,"_139":266,"_141":267,"_268":269,"_107":-5,"_150":22,"_109":110,"_149":258,"_16":147,"_148":255},["D",1755188493466],"updatedAt",["D",1757729497613],297,"e004ac78-4230-11f0-834a-76b3cceeab13",47,"lockfileKey","1757729506929-esm.town_v_jubertioai_hello-realtime@297-main","emptyEnvVars","isNew","flags",[],"routes/_app.x.$handle.$name.branch.$branch.code.$",{"_93":94,"_95":96,"_97":43,"_100":-7,"_98":99,"_101":276,"_151":281,"_240":309,"_177":310,"_248":313,"_314":315,"_318":22,"_319":-5,"_320":217,"_321":322,"_323":324},{"_103":104,"_95":96,"_105":277,"_107":-5,"_108":22,"_109":110,"_111":-5,"_112":-5,"_14":113,"_114":-5,"_115":22,"_116":-5,"_117":32,"_16":118,"_119":120,"_121":120,"_122":123,"_124":123,"_125":123,"_126":120,"_127":120,"_23":278,"_131":279,"_134":280,"_150":22},["D",1756493092620],{"_93":94,"_129":130},{"_93":94,"_133":130},{"_100":136,"_95":43,"_103":137,"_138":104,"_139":140,"_141":142,"_143":110,"_144":145,"_146":147,"_148":96,"_149":94},[282,288,303,306],{"_103":154,"_95":155,"_156":157,"_158":-5,"_159":137,"_160":161,"_162":163,"_105":164,"_100":32,"_165":166,"_167":-5,"_168":22,"_169":-5,"_170":283,"_98":287},[284],{"_103":173,"_95":174,"_156":175,"_158":157,"_159":137,"_160":136,"_162":163,"_105":176,"_100":161,"_165":177,"_167":178,"_168":22,"_169":-5,"_170":285,"_98":286},[],[155,174],[155],{"_103":183,"_95":184,"_156":185,"_158":-5,"_159":137,"_160":161,"_162":163,"_105":186,"_100":32,"_165":166,"_167":-5,"_168":22,"_169":-5,"_170":289,"_98":302},[290,293,296,299],{"_103":189,"_95":190,"_156":191,"_158":185,"_159":137,"_160":161,"_162":163,"_105":192,"_100":32,"_165":193,"_167":194,"_168":22,"_169":-5,"_170":291,"_98":292},[],[184,190],{"_103":198,"_95":199,"_156":200,"_158":185,"_159":137,"_160":161,"_162":163,"_105":201,"_100":32,"_165":193,"_167":202,"_168":22,"_169":-5,"_170":294,"_98":295},[],[184,199],{"_103":206,"_95":207,"_156":208,"_158":185,"_159":137,"_160":161,"_162":163,"_105":209,"_100":32,"_165":193,"_167":210,"_168":22,"_169":-5,"_170":297,"_98":298},[],[184,207],{"_103":214,"_95":215,"_156":216,"_158":185,"_159":137,"_160":217,"_162":163,"_105":218,"_100":161,"_165":193,"_167":219,"_168":22,"_169":-5,"_170":300,"_98":301},[],[184,215],[184],{"_103":224,"_95":225,"_156":226,"_158":-5,"_159":137,"_160":161,"_162":163,"_105":227,"_100":32,"_165":177,"_167":228,"_168":22,"_169":-5,"_170":304,"_98":305},[],[225],{"_103":232,"_95":233,"_156":234,"_158":-5,"_159":137,"_160":161,"_162":163,"_105":235,"_100":32,"_165":236,"_167":237,"_168":22,"_169":-5,"_170":307,"_98":308},[],[233],[282,284,288,290,293,296,299,303,306],{"_103":214,"_159":137,"_156":216,"_95":215,"_167":219,"_165":193,"_100":161,"_158":185,"_105":311,"_160":217,"_169":-5,"_98":312},["D",1756493344882],[184,215],[],"upstream",{"_103":140,"_105":316,"_263":317,"_95":43,"_138":145,"_100":265,"_139":266,"_141":267,"_268":269,"_107":-5,"_150":22,"_109":110,"_149":258,"_16":147,"_148":255},["D",1755188493466],["D",1757729497613],"showCompareCallout","pull","ahead","behind",102,"templates",[],"actionData","errors"]

Learn Kannada through conversations