OpenAI Realtime API Voice Agent

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_292":293,"_294":295},"routes/_app.x.$handle.$name",{"_11":12,"_13":14,"_15":16,"_18":19,"_17":-7,"_217":-5,"_20":218,"_22":219,"_220":221,"_222":223,"_26":224,"_24":225,"_226":227,"_228":229,"_230":53,"_231":232,"_233":73,"_234":-5,"_235":-5,"_236":237},"routes/_app.x.$handle.$name.branch.$branch.code.$",{"_11":12,"_13":14,"_15":16,"_17":-7,"_18":19,"_20":21,"_22":23,"_24":25,"_26":27,"_28":29,"_30":-5,"_31":32,"_33":34,"_35":-5},"handle","jubertioai","name","hello-ambient","branchName","main","version","path","frontend","project",{"_36":174,"_13":14,"_38":175,"_51":-5,"_176":53,"_54":55,"_177":55,"_178":-5,"_179":-5,"_180":181,"_182":92,"_183":73,"_184":73,"_185":-5,"_186":75,"_57":58,"_187":-5,"_188":-5,"_189":-5,"_190":-5,"_191":192,"_193":192,"_194":195,"_196":195,"_197":195,"_198":192,"_199":192,"_200":201,"_202":203,"_204":205,"_52":53},"tree",[86,87,88,89],"file",{"_36":82,"_13":19,"_64":67,"_66":-5,"_68":69,"_70":71,"_72":73,"_38":83,"_17":75,"_76":84,"_77":-5,"_79":53,"_80":-5,"_18":85},"contents",[61],"upstream",{"_36":37,"_38":39,"_40":41,"_13":16,"_42":43,"_17":44,"_45":46,"_47":48,"_49":50,"_51":-5,"_52":53,"_54":55,"_56":12,"_57":58,"_59":60},"pull","ahead",109,"behind",7,"envVars","id","be33420e-792a-11f0-bfa3-0224a6c84d84","createdAt",["D",1755188493466],"updatedAt",["D",1778178264384],"projectId","be2e762a-792a-11f0-bfa3-0224a6c84d84",379,"forkedBranchId","e004ac78-4230-11f0-834a-76b3cceeab13","forkedBranchVersion",47,"lockfileKey","1778178275619-esm.town_v_jubertioai_hello-realtime@379-main","deletedAt","isOwned",false,"privacy","public","userHandle","orgId","9c125825-8df1-4029-a26d-29049e9bd5a3","projectName","hello-realtime",{"_36":62,"_13":63,"_64":65,"_66":67,"_68":69,"_70":71,"_72":73,"_38":74,"_17":75,"_76":24,"_77":78,"_79":53,"_80":-5,"_18":81},"ab1f4e06-bb74-11f0-bfa0-42dde27851f2","index.html","fileId","019a5bd1-1741-76ad-96c4-7a26a0c5f6b8","parentId","019a5bd1-1741-76ad-96c4-7f83ca3a792b","branchId","ab1c7e74-bb74-11f0-bfa0-42dde27851f2","branchVersion",1,"latest",true,["D",1762477020968],0,"fileType","content","\n\n \n \n \n OpenAI Realtime API Voice Agent\n \n \n \n \n\n

\n\n

\n\n \n \n","deleted","label",[19,63],"ab1f692c-bb74-11f0-bfa0-42dde27851f2",["D",1762477020968],"directory",[19],{"_36":82,"_13":19,"_64":67,"_66":-5,"_68":69,"_70":71,"_72":73,"_38":167,"_17":75,"_76":84,"_77":-5,"_79":53,"_80":-5,"_96":168,"_18":169},{"_36":106,"_13":107,"_64":108,"_66":-5,"_68":69,"_70":71,"_72":73,"_38":109,"_17":75,"_76":84,"_77":-5,"_79":53,"_80":-5,"_96":110,"_18":111},{"_36":99,"_13":100,"_64":101,"_66":-5,"_68":69,"_70":71,"_72":73,"_38":102,"_17":75,"_76":24,"_77":103,"_79":53,"_80":-5,"_96":104,"_18":105},{"_36":90,"_13":91,"_64":92,"_66":-5,"_68":69,"_70":71,"_72":73,"_38":93,"_17":75,"_76":94,"_77":95,"_79":53,"_80":14,"_96":97,"_18":98},"ab1f7b9c-bb74-11f0-bfa0-42dde27851f2","main.tsx","019a5bd1-1741-76ad-96c4-9fef8c5b0894",["D",1762477020968],"http","import { Hono } from \"npm:hono\";\nimport { blob } from \"https://esm.town/v/std/blob\";\nimport { parseVal, serveFile } from \"https://esm.town/v/std/utils/index.ts\";\n// import mcp from \"./routes/mcp.ts\";\nimport rtc from \"./routes/rtc.ts\";\nimport sip from \"./routes/sip.ts\";\nimport observer from \"./routes/observer.ts\";\n\nconst SOURCE_URL = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\")).links.self.val;\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.route(\"/sip\", sip);\napp.route(\"/observer\", observer);\n// app.route(\"/mcp\", mcp);\napp.get(\"/source\", (c) => c.redirect(SOURCE_URL));\n\napp.get(\"/\", () => serveFile(\"/frontend/index.html\", import.meta.url));\napp.get(\"/favicon.ico\", async (c) => {\n const entry = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\"));\n const file = await blob.get(`${entry.name}/favicon.ico`);\n if (!file) {\n return c.text(\"favicon not found\", 404);\n }\n return new Response(file.body, {\n headers: {\n \"Content-Type\": \"image/x-icon\",\n \"Cache-Control\": \"public, max-age=31536000, immutable\",\n },\n });\n}); \n \nexport default app.fetch;","children",[],[91],"ab1f7958-bb74-11f0-bfa0-42dde27851f2","README.md","019a5bd1-1741-76ad-96c4-9b24f7bc5552",["D",1762477020968],"# hello-realtime\n\n**Hello Realtime** is a OpenAI Realtime app that supports both WebRTC and SIP\n(telephone) users. You can access the app via WebRTC at\n[hello-realtime.val.run](https://hello-realtime.val.run), or via SIP by calling\n425-800-0042.\n\nThis demo shows off the new SIP API, the new all-in-one WebRTC API, and the new\nserver-side websocket interface.\n\nIf you remix the app, you'll just need to pop in your own `OPENAI_API_KEY` (from\n[platform.openai.com](https://platform.openai.com)), and if you want SIP, the\n`OPENAI_SIGNING_SECRET`.\n\n## Architecture\n\n1. **WebRTC Flow**:\n - Browser connects to frontend\n - creates WebRTC offer\n - `/rtc` endpoint handles SDP negotiation with OpenAI\n - observer established to monitor session\n2. **SIP Flow**:\n - Phone calls triggers webhook\n - `/sip` endpoint verifies webhook and accepts call\n - observer established to monitor session\n\n## API Endpoints\n\n- `GET /` - serves the WebRTC-based app\n- `POST /rtc` - creates a new WebRTC-based Realtime API session\n - Supports optional `model` URL parameter to override the default model\n - Example: `POST /rtc?model=gpt-4o-realtime-preview`\n- `POST /sip` - handles an incoming Realtime API SIP call\n- `POST /observer` - internal endpoint to establish a Realtime API websocket for\n session control\n\n## Project Structure\n\n```\n├── frontend/ \n│ └── index.html # HTML/JS application for WebRTC\n├── main.tsx # Main entrypoint for the val.town app\n└── routes/\n |\n ├── agent.ts # Realtime API session configuration and tools\n ├── observer.ts # Server-side session control for SIP & WebRTC\n ├── rtc.ts # WebRTC session setup\n ├── sip.ts # SIP webhook handling and session setup\n └── utils.ts # Various HTTP helpers\n```\n\n## File Links\n\n- [`frontend/index.html`](./frontend/index.html)\n- [`main.tsx`](./main.tsx)\n- [`routes/agent.ts`](./routes/agent.ts)\n- [`routes/observer.ts`](./routes/observer.ts)\n- [`routes/rtc.ts`](./routes/rtc.ts)\n- [`routes/sip.ts`](./routes/sip.ts)\n- [`routes/utils.ts`](./routes/utils.ts)",[],[100],"ab1f784a-bb74-11f0-bfa0-42dde27851f2","routes","019a5bd1-1741-76ad-96c4-969fe8b2c77a",["D",1762477020968],[112,113,114,115,116,117],[107],{"_36":158,"_13":159,"_64":160,"_66":108,"_68":69,"_70":161,"_72":73,"_38":162,"_17":163,"_76":122,"_77":164,"_79":53,"_80":-5,"_96":165,"_18":166},{"_36":149,"_13":150,"_64":151,"_66":108,"_68":69,"_70":152,"_72":73,"_38":153,"_17":154,"_76":122,"_77":155,"_79":53,"_80":-5,"_96":156,"_18":157},{"_36":142,"_13":143,"_64":144,"_66":108,"_68":69,"_70":71,"_72":73,"_38":145,"_17":75,"_76":122,"_77":146,"_79":53,"_80":-5,"_96":147,"_18":148},{"_36":135,"_13":136,"_64":137,"_66":108,"_68":69,"_70":71,"_72":73,"_38":138,"_17":75,"_76":122,"_77":139,"_79":53,"_80":-5,"_96":140,"_18":141},{"_36":126,"_13":127,"_64":128,"_66":108,"_68":69,"_70":129,"_72":73,"_38":130,"_17":131,"_76":122,"_77":132,"_79":53,"_80":-5,"_96":133,"_18":134},{"_36":118,"_13":119,"_64":120,"_66":108,"_68":69,"_70":71,"_72":73,"_38":121,"_17":75,"_76":122,"_77":123,"_79":53,"_80":-5,"_96":124,"_18":125},"ab1f75f2-bb74-11f0-bfa0-42dde27851f2","utils.ts","019a5bd1-1741-76ad-96c4-918a43fca263",["D",1762477020968],"script","const REALTIME_BASE_URL = \"https://api.openai.com/v1/realtime\";\nconst OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\n// Builds the URL for a Realtime API endpoint.\nexport function makeUrl(callId?: string, action?: string) {\n let url = REALTIME_BASE_URL;\n if (!callId) {\n // start call (POST)\n url += \"/calls\";\n } else if (action) {\n // accept, reject, hangup (POST)\n url += `/calls/${callId}/${action}`;\n } else {\n // observer (web socket)\n url += `?call_id=${callId}`;\n }\n return url;\n}\n\n// Builds the headers for a Realtime API endpoint.\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\n// Debug helper for printing Realtime API errors.\nexport async function getErrorText(resp: Response) {\n const errBody = await resp.text().catch(() => \"\");\n return `${resp.status} ${resp.statusText}: ${errBody}`;\n}",[],[107,119],"5404061a-bf8a-11f0-af84-42dde27851f2","sound.ts","c16a11e4-bdd8-11f0-969c-42dde27851f2",110,["D",1762926128433],62,"export async function searchFreesound(query: string): Promise {\n const apiKey = Deno.env.get(\"FREESOUND_API_KEY\");\n if (!apiKey) {\n throw new Error(\"Freesound API key not configured\");\n }\n\n const searchUrl = `https://freesound.org/apiv2/search/text/?query=${\n encodeURIComponent(query)\n }&token=${apiKey}&fields=id,name,username,duration&page_size=1`;\n const searchResponse = await fetch(searchUrl);\n if (!searchResponse.ok) {\n throw new Error(`Search failed: ${searchResponse.status}`);\n }\n\n const searchData = await searchResponse.json();\n if (!searchData.results || searchData.results.length === 0) {\n throw new Error(\"No results found\");\n }\n\n const sound = searchData.results[0];\n const detailUrl =\n `https://freesound.org/apiv2/sounds/${sound.id}/?token=${apiKey}&fields=id,name,username,duration,previews`;\n const detailResponse = await fetch(detailUrl);\n if (!detailResponse.ok) {\n throw new Error(`Detail fetch failed: ${detailResponse.status}`);\n }\n\n const detailData = await detailResponse.json();\n const previewUrl = detailData.previews[\"preview-hq-mp3\"] ||\n detailData.previews[\"preview-lq-mp3\"] ||\n detailData.previews[\"preview-hq-ogg\"] ||\n detailData.previews[\"preview-lq-ogg\"];\n if (!previewUrl) {\n throw new Error(\"No preview URL\");\n }\n\n return previewUrl;\n}\n\nimport { MPEGDecoder } from \"npm:mpg123-decoder\";\nimport * as Speex from \"npm:speex-resampler\";\nconst SpeexResampler = Speex.default.default;\n// OR (equivalent):\n// const { default: { default: SpeexResampler } } = Speex;\nawait SpeexResampler.initPromise;\n\nexport async function mp3ToPcm24k(\n url: string,\n onChunk: (pcm: Int16Array) => void,\n opts: { chunkMs?: number } = {},\n) {\n const chunkMs = opts.chunkMs ?? 1000;\n\n // 1. Fetch MP3\n const resp = await fetch(url);\n if (!resp.ok) throw new Error(\"MP3 fetch failed\");\n const mp3Bytes = new Uint8Array(await resp.arrayBuffer());\n\n // 2. Decode with mpg123\n const decoder = new MPEGDecoder();\n await decoder.ready;\n const result = decoder.decode(mp3Bytes);\n\n const { channelData, sampleRate } = result;\n const channels = channelData.length; // 1 or 2 typically\n\n // 3) Downmix channels → Float32 PCM\n const monoF32 = downmixToMono(channelData);\n\n // 4) Downsample to 24 kHz\n const monoS16 = floatToS16(monoF32);\n const monoBytes = new Uint8Array(\n monoS16.buffer,\n monoS16.byteOffset,\n monoS16.byteLength,\n );\n\n // 4) Initialize and run Speex Resampler\n const resampler = new SpeexResampler(1, sampleRate, 24000, 5);\n const resampledBytes = await resampler.processChunk(monoBytes);\n\n // 5) Convert back to Int16\n const mono24 = new Int16Array(\n resampledBytes.buffer,\n resampledBytes.byteOffset,\n resampledBytes.byteLength / 2,\n );\n\n // 6) Chunk by milliseconds (default 20ms)\n const framesPerChunk = Math.floor(24000 * (chunkMs / 1000));\n const samplesPerChunk = framesPerChunk * channels;\n\n for (let i = 0; i < mono24.length; i += samplesPerChunk) {\n const chunk = mono24.subarray(\n i,\n Math.min(i + samplesPerChunk, mono24.length),\n );\n onChunk(chunk);\n }\n}\n\n// --------------------------------------------------------------\n// Helpers\n// --------------------------------------------------------------\n\nfunction interleave(channels: Float32Array[]): Float32Array {\n if (channels.length === 1) return channels[0]; // mono passthrough\n\n const frames = channels[0].length;\n const ch = channels.length;\n const out = new Float32Array(frames * ch);\n\n let o = 0;\n for (let i = 0; i < frames; i++) {\n for (let c = 0; c < ch; c++) {\n out[o++] = channels[c][i];\n }\n }\n return out;\n}\n\nfunction downmixToMono(channels: Float32Array[]): Float32Array {\n if (channels.length === 1) return channels[0]; // mono passthrough\n\n const frames = channels[0].length;\n const ch = channels.length;\n const out = new Float32Array(frames);\n\n const scale = 1 / ch; // linear-average downmix\n\n for (let i = 0; i < frames; i++) {\n let sum = 0;\n for (let c = 0; c < ch; c++) {\n sum += channels[c][i];\n }\n out[i] = sum * scale;\n }\n\n return out;\n}\n\nfunction floatToS16(f32: Float32Array): Int16Array {\n const out = new Int16Array(f32.length);\n for (let i = 0; i < f32.length; i++) {\n const v = Math.max(-1, Math.min(1, f32[i]));\n out[i] = v < 0 ? v * 0x8000 : v * 0x7fff;\n }\n return out;\n}\n\n/*\nmp3ToPcm24k(\n \"https://cdn.freesound.org/previews/451/451531_612689-hq.mp3\",\n (x) => {\n console.log(x);\n },\n);*/",[],[107,127],"ab1f7322-bb74-11f0-bfa0-42dde27851f2","sip.ts","019a5bd1-1741-76ad-96c4-8edf4a3099e5",["D",1762477020968],"import { Hono } from \"npm:hono\";\nimport { Webhook } from \"npm:svix\";\nimport { makeSession } from \"./agent.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst sip = new Hono();\n\n// POST /sip : webhook endpoint\nsip.post(\"/\", async (c) => {\n // Verify the webhook.\n const OPENAI_SIGNING_SECRET = Deno.env.get(\"OPENAI_SIGNING_SECRET\");\n if (!OPENAI_SIGNING_SECRET) {\n console.error(\"🔴 webhook secret not configured\");\n return c.text(\"Internal error\", 500);\n }\n const webhook = new Webhook(OPENAI_SIGNING_SECRET);\n const bodyStr = await c.req.text();\n let callId: string | undefined;\n let toNumber: string | undefined;\n try {\n const headers = Object.fromEntries(c.req.raw.headers.entries());\n const payload: any = webhook.verify(bodyStr, headers);\n callId = payload.data.call_id;\n toNumber = getToNumber(payload.data.sip_headers);\n const fromNumber = getFromNumber(payload.data.sip_headers);\n console.log(\"✅ verified webhook:\", callId, fromNumber, toNumber);\n } catch (error) {\n console.error(`🔴 verify failed: ${error}`);\n return c.text(\"Invalid signature\", 401);\n }\n\n // Accept the call.\n const model = getModelForNumber(toNumber);\n const url = makeUrl(callId, \"accept\");\n const headers = makeHeaders(\"application/json\");\n const body = JSON.stringify(makeSession(model));\n const opts = { method: \"POST\", headers, body };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n console.error(`🔴 accept failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n console.log(\"✅ call accepted:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n return c.newResponse(null);\n});\n\nconst getToNumber = headers => getNumberFromSipHeader(headers, \"to\");\nconst getFromNumber = headers => getNumberFromSipHeader(headers, \"from\");\n\nfunction getNumberFromSipHeader(headers: any, name: string) {\n const header = getSipHeader(headers, name);\n const match = header?.match(/sip:\\+?(\\d+)@/);\n return match ? `+${match[1]}` : undefined;\n}\n\nfunction getSipHeader(headers: any, name: string) {\n return headers.find((h) => h.name.toLowerCase() === name)?.value;\n}\n\nfunction getModelForNumber(number: string) {\n const map = { \"+14258000074\": \"gpt-realtime-mini\" };\n return map[number];\n}\n\nexport default sip;",[],[107,136],"ab1f70ca-bb74-11f0-bfa0-42dde27851f2","rtc.ts","019a5bd1-1741-76ad-96c4-8848e15ad164",["D",1762477020968],"import { Hono } from \"npm:hono\";\nimport { makeSession } from \"./agent.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new call\nrtc.post(\"/\", async (c) => {\n // Extract model parameter from URL if provided\n const model = c.req.query(\"model\");\n if (model !== undefined) {\n console.log(\"🔧 Model parameter received:\", model);\n }\n \n // Create the call.\n const fd = new FormData();\n fd.set(\"sdp\", await c.req.text());\n fd.set(\"session\", JSON.stringify(makeSession(model)));\n const opts = { method: \"POST\", headers: makeHeaders(), body: fd };\n const resp = await fetch(makeUrl(), opts);\n if (!resp.ok) {\n console.error(`🔴 start call failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n\n const contentType = resp.headers.get(\"Content-Type\");\n const location = resp.headers.get(\"Location\");\n const callId = location?.split(\"/\").pop();\n console.log(\"✅ call created:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n const responseHeaders = {\n \"Content-Type\": contentType,\n \"Location\": location,\n };\n return c.newResponse(resp.body, { headers: responseHeaders });\n});\n\nexport default rtc;",[],[107,143],"956d7316-bf85-11f0-8d95-42dde27851f2","observer.ts","019a5bd1-1741-76ad-96c4-852ef73d2815",95,["D",1762924090693],9,"import { Hono } from \"npm:hono\";\nimport WebSocket from \"npm:ws\";\nimport { createHandler } from \"./agent.ts\";\nimport { makeHeaders, makeUrl } from \"./utils.ts\";\n\nexport interface ObserverHandler {\n onInit(): void;\n onTool(name: string, args: any, callId: string): void;\n onOutputAudioStarted(): void;\n onOutputAudioStopped(): void;\n onOutputAudioCleared(): void;\n}\n\nexport class ObserverClient {\n private ws: WebSocket;\n private closeResolve: () => void;\n private closePromise: Promise;\n private handler: ObserverHandler;\n\n constructor(callId: string) {\n const url = makeUrl(callId);\n this.ws = new WebSocket(url, { headers: makeHeaders() });\n this.ws.on(\"open\", () => this.handleOpen());\n this.ws.on(\"message\", (data) => this.handleMessage(JSON.parse(data)));\n this.ws.on(\"error\", (err) => this.handleError(err));\n this.ws.on(\"close\", (ev) => this.handleClose(ev));\n this.closePromise = new Promise((resolve) =>\n this.closeResolve = resolve\n );\n this.handler = createHandler(this, callId);\n }\n createFunctionCallOutput(functionCallId: string, output: any) {\n const item = {\n type: \"function_call_output\",\n call_id: functionCallId,\n output: JSON.stringify(output),\n };\n this.sendMessage({ type: \"conversation.item.create\", item });\n }\n createResponse() {\n this.sendMessage({ type: \"response.create\" });\n }\n createOutputAudioBuffer(name: string) {\n this.sendMessage({ type: \"output_audio_buffer.create\", name });\n }\n deleteOutputAudioBuffer(name: string) {\n this.sendMessage({ type: \"output_audio_buffer.delete\", name });\n }\n appendOutputAudio(name: string, pcm: Int16Array) {\n const uint8 = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);\n const audio = btoa(String.fromCharCode(...uint8));\n this.sendMessage({ type: \"output_audio_buffer.append\", name, audio });\n }\n async waitForClose() {\n await this.closePromise;\n }\n private sendMessage(message: any) {\n this.ws.send(JSON.stringify(message));\n }\n private handleOpen() {\n console.log(\"✅ websocket connected\");\n this.handler.onInit();\n }\n private async handleMessage(message: any) {\n if (!message.type.endsWith(\".delta\")) { // ignore verbose messages\n console.log(message);\n }\n switch (message.type) {\n case \"response.function_call_arguments.done\":\n this.handler.onTool(\n message.name,\n JSON.parse(message.arguments),\n message.call_id,\n );\n break;\n case \"output_audio_buffer.started\":\n this.handler.onOutputAudioStarted();\n break;\n case \"output_audio_buffer.stopped\":\n this.handler.onOutputAudioStopped();\n break;\n case \"output_audio_buffer.cleared\":\n this.handler.onOutputAudioCleared();\n break;\n }\n }\n private handleError(err: Error) {\n console.error(`🔴 websocket failed: ${err.message}`);\n this.closeResolve();\n }\n private handleClose(e: Event) {\n console.log(`websocket closed: ${e}`);\n this.closeResolve();\n }\n}\n\nconst observer = new Hono();\n\n// POST /observer/:callId : self-connect to establish a web socket\nobserver.post(\"/:callId\", async (c) => {\n const callId = c.req.param(\"callId\");\n const client = new ObserverClient(callId);\n await client.waitForClose();\n return c.newResponse(null);\n});\n\nexport default observer;",[],[107,150],"1c66c50c-bf86-11f0-96f7-42dde27851f2","agent.ts","019a5bd1-1741-76ad-96c4-8125dc25488c",99,["D",1762924317124],37,"import { ObserverClient, ObserverHandler } from \"./observer.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\nimport { mp3ToPcm24k, searchFreesound } from \"./sound.ts\";\n\nconst MODEL = \"gpt-realtime\";\nconst TRANSCRIBE_MODEL = \"gpt-4o-transcribe\";\nconst NR_TYPE = \"near_field\";\nconst INSTRUCTIONS = `\n Greet the user with a slow, calming voice and encourage them to close their eyes and listen.\n Ask them to think of a relaxing sound. When they mention the name of a sound or an \n environment, invoke the play_sound function with the name of the sound they are looking\n for. When you call the play_sound function, tell the user that you're loading that\n sonic environment.\n When the user says goodbye or you think the call is over, say a brief goodbye and then\n invoke the end_call function.\n`;\nconst VOICE = \"marin\";\nconst HANGUP_TOOL = {\n type: \"function\",\n name: \"end_call\",\n description: `\n Use this function to hang up the call when the user says goodbye \n or otherwise indicates they are about to end the call.`,\n};\nconst PLAY_SOUND_TOOL = {\n type: \"function\",\n name: \"play_sound\",\n description: `\n Use this function to look up a sound effect by name and start\n playing it out as background audio.`,\n parameters: {\n type: \"object\",\n properties: {\n query: {\n type: \"string\",\n description: \"A search query for the Freesound API.\",\n },\n },\n required: [\"query\"],\n },\n};\nconst BUFFER_NAME = \"background\";\n\n// Builds the declarative session configuration for a Realtime API session.\nexport function makeSession(modelOverride?: string) {\n const model = modelOverride || MODEL;\n const instructions = INSTRUCTIONS.replace(\"{{model}}\", model);\n return {\n type: \"realtime\",\n model,\n instructions,\n audio: {\n input: {\n noise_reduction: { type: NR_TYPE },\n transcription: { model: TRANSCRIBE_MODEL },\n },\n output: { voice: VOICE },\n },\n tools: [HANGUP_TOOL, PLAY_SOUND_TOOL],\n };\n}\n\n// AgentHandler implements the agent runtime behavior (tool calling, etc).\nclass AgentHandler implements ObserverHandler {\n private audioActive = false;\n private callEnding = false;\n constructor(private client: ObserverClient, private callId: string) {}\n onInit() {\n // Trigger a response immediately.\n this.client.createResponse();\n /*const x = {\n type: \"response.function_call_arguments.done\",\n event_id: \"event_CaC6edkeJLkTTMM1LaaOk\",\n response_id: \"resp_CaC6bN3JBzHmvqDcNJnBP\",\n item_id: \"item_CaC6eNjnRrQngK6z6kWUG\",\n output_index: 1,\n call_id: \"call_Ipl4jJE60yC6HSVH\",\n name: \"play_sound\",\n arguments: '{ \\n \"query\": \"gentle ocean waves\" \\n} \\n',\n };*/\n //this.onTool(x.name, JSON.parse(x.arguments));\n }\n async onTool(name: string, args: any, toolCallId: string) {\n switch (name) {\n case HANGUP_TOOL.name:\n await this.onHangupTool(args, toolCallId);\n break;\n case PLAY_SOUND_TOOL.name:\n await this.onPlaySoundTool(args, toolCallId);\n break;\n }\n }\n async onHangupTool(args: any, toolCallId: string) {\n if (this.audioActive) {\n this.callEnding = true;\n } else {\n await this.hangup();\n }\n }\n async onPlaySoundTool(args: any, toolCallId: string) {\n console.log(`Playing sound: ${args.query}`);\n let url: string;\n try {\n url = await searchFreesound(args.query);\n } catch (e) {\n const output = { \"error\": e.message };\n this.client.createFunctionCallOutput(toolCallId, output);\n this.client.createResponse();\n return;\n }\n console.log(`URL: ${url}`);\n this.client.deleteOutputAudioBuffer(BUFFER_NAME);\n this.client.createOutputAudioBuffer(BUFFER_NAME);\n mp3ToPcm24k(url, (pcm: Int16Array) => {\n this.client.appendOutputAudio(BUFFER_NAME, pcm);\n });\n }\n async onOutputAudioStarted() {\n this.audioActive = true;\n }\n async onOutputAudioStopped() {\n // Shut down the call and websocket once audio has finished spooling.\n this.audioActive = false;\n if (this.callEnding) {\n await this.hangup();\n }\n }\n async onOutputAudioCleared() {\n this.audioActive = false;\n }\n private async hangup() {\n const url = makeUrl(this.callId, \"hangup\");\n const resp = await fetch(url, { method: \"POST\", headers: makeHeaders() });\n if (!resp.ok) {\n console.error(`🔴 end call failed: ${await getErrorText(resp)}`);\n return;\n }\n\n console.log(\"✅ call ended\");\n }\n}\n\n// Creates the runtime handler for the session.\nexport function createHandler(client: ObserverClient, callId: string) {\n return new AgentHandler(client, callId);\n}",[],[107,159],["D",1762477020968],[170],[19],{"_36":62,"_13":63,"_64":65,"_66":67,"_68":69,"_70":71,"_72":73,"_38":171,"_17":75,"_76":24,"_77":78,"_79":53,"_80":-5,"_96":172,"_18":173},["D",1762477020968],[],[19,63],"ab1ab602-bb74-11f0-bfa0-42dde27851f2",["D",1762477020968],"legacyEnvVars","httpPrivacy","description","imageUrl","userId","dbe13bc0-4230-11f0-95cd-ab37681351fa","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_11":12,"_216":212},"org",{"_11":12,"_210":-5,"_211":212,"_180":181,"_213":214,"_215":53},"branch",{"_17":129,"_13":16,"_36":69,"_42":174,"_45":37,"_47":206,"_207":55,"_208":43,"_209":58,"_59":14,"_56":12},372,"forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","displayName","avatar","https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/61901958-12bd-49a3-43e9-2f67f1617200/square512?exp=1789577999&sig=b97f38dc06e3aeec48990c5845e88d533f6ad0ed0a65fabc30d122c3d26e42cc","tier","pro","canRestrictVals","profileImageUrl","adoptOrgId",{"_36":174,"_13":14,"_38":288,"_51":-5,"_176":53,"_54":55,"_177":55,"_178":-5,"_179":-5,"_180":181,"_182":92,"_183":73,"_184":73,"_185":-5,"_186":75,"_57":58,"_187":-5,"_188":-5,"_189":-5,"_190":-5,"_191":192,"_193":192,"_194":195,"_196":195,"_197":195,"_198":192,"_199":192,"_200":289,"_202":290,"_204":291,"_52":53},[255,257,264,265],"files",[255,256,257,258,259,260,261,262,263,264,265],"counts",{"_245":75,"_246":71,"_247":75,"_35":248,"_249":250},[243],{"_36":82,"_13":19,"_64":67,"_66":-5,"_68":69,"_70":71,"_72":73,"_38":83,"_17":75,"_76":84,"_77":-5,"_79":53,"_80":-5,"_18":242},"upstreamProject",{"_36":43,"_13":60,"_178":240,"_179":-5,"_54":55,"_200":241},"upstreamBranch",{"_36":37,"_38":238,"_40":239,"_13":16,"_42":43,"_17":44,"_45":46,"_47":48,"_49":50,"_51":-5,"_52":53,"_54":55,"_56":12,"_57":58,"_59":60},"emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},["D",1755188493466],["D",1778178264384],"Sample app for the OpenAI Realtime API",{"_11":12,"_216":212},[19],{"_36":62,"_13":63,"_64":65,"_66":67,"_68":69,"_70":71,"_72":73,"_38":74,"_17":75,"_76":24,"_77":78,"_79":53,"_80":-5,"_18":244},[19,63],"pulls","branches","forks",{"_251":253,"_254":75},"connections",{"_251":75,"_252":75},"count","unfilledCount",3,"emptyCount",{"_36":82,"_13":19,"_64":67,"_66":-5,"_68":69,"_70":71,"_72":73,"_38":167,"_17":75,"_76":84,"_77":-5,"_79":53,"_80":-5,"_96":286,"_18":287},{"_36":62,"_13":63,"_64":65,"_66":67,"_68":69,"_70":71,"_72":73,"_38":171,"_17":75,"_76":24,"_77":78,"_79":53,"_80":-5,"_96":284,"_18":285},{"_36":106,"_13":107,"_64":108,"_66":-5,"_68":69,"_70":71,"_72":73,"_38":109,"_17":75,"_76":84,"_77":-5,"_79":53,"_80":-5,"_96":282,"_18":283},{"_36":158,"_13":159,"_64":160,"_66":108,"_68":69,"_70":161,"_72":73,"_38":162,"_17":163,"_76":122,"_77":164,"_79":53,"_80":-5,"_96":280,"_18":281},{"_36":149,"_13":150,"_64":151,"_66":108,"_68":69,"_70":152,"_72":73,"_38":153,"_17":154,"_76":122,"_77":155,"_79":53,"_80":-5,"_96":278,"_18":279},{"_36":142,"_13":143,"_64":144,"_66":108,"_68":69,"_70":71,"_72":73,"_38":145,"_17":75,"_76":122,"_77":146,"_79":53,"_80":-5,"_96":276,"_18":277},{"_36":135,"_13":136,"_64":137,"_66":108,"_68":69,"_70":71,"_72":73,"_38":138,"_17":75,"_76":122,"_77":139,"_79":53,"_80":-5,"_96":274,"_18":275},{"_36":126,"_13":127,"_64":128,"_66":108,"_68":69,"_70":129,"_72":73,"_38":130,"_17":131,"_76":122,"_77":132,"_79":53,"_80":-5,"_96":272,"_18":273},{"_36":118,"_13":119,"_64":120,"_66":108,"_68":69,"_70":71,"_72":73,"_38":121,"_17":75,"_76":122,"_77":123,"_79":53,"_80":-5,"_96":270,"_18":271},{"_36":99,"_13":100,"_64":101,"_66":-5,"_68":69,"_70":71,"_72":73,"_38":102,"_17":75,"_76":24,"_77":103,"_79":53,"_80":-5,"_96":268,"_18":269},{"_36":90,"_13":91,"_64":92,"_66":-5,"_68":69,"_70":71,"_72":73,"_38":93,"_17":75,"_76":94,"_77":95,"_79":53,"_80":14,"_96":266,"_18":267},[],[91],[],[100],[],[107,119],[],[107,127],[],[107,136],[],[107,143],[],[107,150],[],[107,159],[258,259,260,261,262,263],[107],[],[19,63],[256],[19],["D",1762477020968],{"_11":12,"_216":212},{"_11":12,"_210":-5,"_211":212,"_180":181,"_213":214,"_215":53},{"_17":129,"_13":16,"_36":69,"_42":174,"_45":37,"_47":206,"_207":55,"_208":43,"_209":58,"_59":14,"_56":12},"ctx",{"_200":-5,"_235":325,"_326":327,"_328":329,"_330":53,"_331":53,"_332":53,"_333":73,"_334":-5,"_335":-7,"_336":-7,"_337":75,"_338":53,"_339":53,"_340":-5,"_341":53,"_342":53,"_343":53,"_344":-5},"ENV",{"_296":297,"_298":299,"_300":16,"_301":302,"_303":304,"_305":306,"_307":308,"_309":310,"_311":312,"_313":314,"_315":316,"_317":318,"_319":320,"_321":322,"_323":324},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Realtime Agent