OpenAI GPT-Live Voice Agent

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_256":257,"_258":259},"routes/_app.x.$handle.$name",{"_99":98,"_32":45,"_107":86,"_27":108,"_13":14,"_109":-5,"_11":110,"_111":112,"_113":114,"_115":116,"_117":118,"_20":-5,"_119":120,"_121":122,"_123":25,"_124":125,"_126":58,"_127":-5,"_128":-5,"_129":130},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":31,"_57":58,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":79,"_80":81,"_82":83,"_84":25},"version",18,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","main.tsx","latestPath","fileId","01a08d52-52ea-7194-86c2-05042be094d8","name","fileType","http","content","import { ObserverClient, ObserverHandler } from \"./observer.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst MODEL = \"gpt-live-1\";\nconst BACKEND_MODEL = \"gpt-5.6-luna\";\nconst VOICE = \"marin\";\nconst DATE = new Date().toLocaleDateString();\n\n// Instructions for the GPT-Live voice model: conversation style + when to delegate.\nconst INSTRUCTIONS = `\n As soon as the session starts, without waiting for the user to speak, greet the user in English\n and tell them that they're using the GPT-Live API. Mention that GPT-Live is a full duplex model\n that can listen while it speaks, for more natural conversation, and can also delegate to GPT 5.6\n and tools like web search when more infomration is needed.\n \n When the user says goodbye or you think the call is over, say a brief goodbye and then\n ask the backend to end the call.\n The current date is ${DATE}.\n`;\n\n// Instructions for the delegated Responses backend: business rules + tool use.\nconst BACKEND_INSTRUCTIONS = `\n You support a live voice conversation. Keep results short and factual.\n When asked to end the call, call the end_call function.\n The current date is ${DATE}.\n`;\n\nconst HANGUP_TOOL = {\n type: \"function\",\n name: \"end_call\",\n description: `\n Use this function to hang up the call when the user says goodbye\n or otherwise indicates they are about to end the call.`,\n parameters: { type: \"object\", properties: {}, additionalProperties: false },\n};\n\nconst SEARCH_TOOL = {\n type: \"web_search\",\n};\n\nconst BACKEND = {\n type: \"responses\",\n responses: {\n model: BACKEND_MODEL,\n instructions: BACKEND_INSTRUCTIONS,\n tools: [HANGUP_TOOL, SEARCH_TOOL],\n tool_choice: \"auto\",\n parallel_tool_calls: false,\n },\n};\n\n// Builds the declarative session configuration for a GPT-Live session.\nexport function makeSession() {\n return {\n model: MODEL,\n instructions: INSTRUCTIONS,\n audio: { output: { voice: VOICE } },\n delegation: BACKEND,\n };\n}\n\n// AgentHandler implements the agent runtime behavior (tool calling, etc).\nclass AgentHandler implements ObserverHandler {\n private audioActive = false;\n private callEnding = false;\n constructor(private client: ObserverClient, private sessionId: string) {}\n onInit() {\n // Nothing to do: the greeting is baked into the session instructions.\n }\n async onTool(name: string, _args: any, callId: string) {\n if (name == HANGUP_TOOL.name) {\n // Return the tool result and let the backend response finish.\n this.client.sendToolResult(callId, JSON.stringify({ status: \"ok\" }));\n this.client.createResponse();\n // If audio is playing, defer the hangup until it finishes.\n // Otherwise, end the call now.\n if (this.audioActive) {\n this.callEnding = true;\n } else {\n await this.hangup();\n }\n }\n }\n async onOutputAudioStarted() {\n this.audioActive = true;\n }\n async onOutputAudioStopped() {\n // Shut down the call and websocket once audio has finished spooling.\n this.audioActive = false;\n if (this.callEnding) {\n await this.hangup();\n }\n }\n async onOutputAudioCleared() {\n this.audioActive = false;\n }\n private async hangup() {\n const url = makeUrl(this.sessionId, \"hangup\");\n const resp = await fetch(url, { method: \"POST\", headers: makeHeaders() });\n if (!resp.ok) {\n console.error(`🔴 end call failed: ${await getErrorText(resp)}`);\n return;\n }\n\n console.log(\"✅ call ended\");\n }\n}\n\n// Creates the runtime handler for the session.\nexport function createHandler(client: ObserverClient, sessionId: string) {\n return new AgentHandler(client, sessionId);\n}","import { ObserverClient, ObserverHandler } from \"./observer.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst MODEL = \"gpt-live-1\";\nconst BACKEND_MODEL = \"gpt-5.6-luna\";\nconst VOICE = \"marin\";\nconst DATE = new Date().toLocaleDateString();\n\n// Instructions for the GPT-Live voice model: conversation style + when to delegate.\nconst INSTRUCTIONS = `\n As soon as the session starts, without waiting for the user to speak, greet the user in English\n and tell them that they're using the GPT-Live API. Mention that GPT-Live is a full duplex model\n that can listen while it speaks, for more natural conversation, and it can also delegate to GPT 5.6\n and use tools like web search when more infomration is needed.\n \n When the user says goodbye or you think the call is over, say a brief goodbye and then\n ask the backend to end the call.\n The current date is ${DATE}.\n`;\n\n// Instructions for the delegated Responses backend: business rules + tool use.\nconst BACKEND_INSTRUCTIONS = `\n You support a live voice conversation. Keep results short and factual.\n When asked to end the call, call the end_call function.\n The current date is ${DATE}.\n`;\n\nconst HANGUP_TOOL = {\n type: \"function\",\n name: \"end_call\",\n description: `\n Use this function to hang up the call when the user says goodbye\n or otherwise indicates they are about to end the call.`,\n parameters: { type: \"object\", properties: {}, additionalProperties: false },\n};\n\nconst SEARCH_TOOL = {\n type: \"web_search\",\n};\n\nconst BACKEND = {\n type: \"responses\",\n responses: {\n model: BACKEND_MODEL,\n instructions: BACKEND_INSTRUCTIONS,\n tools: [HANGUP_TOOL, SEARCH_TOOL],\n tool_choice: \"auto\",\n parallel_tool_calls: false,\n },\n};\n\n// Builds the declarative session configuration for a GPT-Live session.\nexport function makeSession() {\n return {\n model: MODEL,\n instructions: INSTRUCTIONS,\n audio: { output: { voice: VOICE } },\n delegation: BACKEND,\n };\n}\n\n// AgentHandler implements the agent runtime behavior (tool calling, etc).\nclass AgentHandler implements ObserverHandler {\n private audioActive = false;\n private callEnding = false;\n constructor(private client: ObserverClient, private sessionId: string) {}\n onInit() {\n // Nothing to do: the greeting is baked into the session instructions.\n }\n async onTool(name: string, _args: any, callId: string) {\n if (name == HANGUP_TOOL.name) {\n // Return the tool result and let the backend response finish.\n this.client.sendToolResult(callId, JSON.stringify({ status: \"ok\" }));\n this.client.createResponse();\n // If audio is playing, defer the hangup until it finishes.\n // Otherwise, end the call now.\n if (this.audioActive) {\n this.callEnding = true;\n } else {\n await this.hangup();\n }\n }\n }\n async onOutputAudioStarted() {\n this.audioActive = true;\n }\n async onOutputAudioStopped() {\n // Shut down the call and websocket once audio has finished spooling.\n this.audioActive = false;\n if (this.callEnding) {\n await this.hangup();\n }\n }\n async onOutputAudioCleared() {\n this.audioActive = false;\n }\n private async hangup() {\n const url = makeUrl(this.sessionId, \"hangup\");\n const resp = await fetch(url, { method: \"POST\", headers: makeHeaders() });\n if (!resp.ok) {\n console.error(`🔴 end call failed: ${await getErrorText(resp)}`);\n return;\n }\n\n console.log(\"✅ call ended\");\n }\n}\n\n// Creates the runtime handler for the session.\nexport function createHandler(client: ObserverClient, sessionId: string) {\n return new AgentHandler(client, sessionId);\n}","createdAt",["D",1789097257981],"branchVersion","branchId","5d994488-44c2-4ba1-8f86-50810fea3ff5","id","5a239725-73df-432f-af30-67e6eff074b8","hello-live",["D",1789077377613],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Sample app for the OpenAI Realtime API","imageUrl","dbe13bc0-4230-11f0-95cd-ab37681351fa","pinnedFileId","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",1,"orgId","9c125825-8df1-4029-a26d-29049e9bd5a3","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_99":98,"_106":102},"org",{"_99":98,"_100":-5,"_101":102,"_18":55,"_103":104,"_105":25},"branch",{"_13":85,"_32":86,"_43":42,"_87":44,"_88":89,"_90":91,"_92":50,"_93":94,"_95":64,"_96":45,"_97":98},"isOwned",31,"main","projectId","forkedBranchId","be33420e-792a-11f0-bfa3-0224a6c84d84","forkedBranchVersion",379,"forkedProjectPrivacy","forkedProjectId","be2e762a-792a-11f0-bfa3-0224a6c84d84","forkedProjectOrgId","projectName","userHandle","jubertioai","handle","displayName","avatar","https://imagedelivery.net/iHX6Ovru0O7AjmyT5yZRoA/61901958-12bd-49a3-43e9-2f67f1617200/square512?exp=1789662666&sig=98c93e55c195cdecbb282234b17fe749e5d030c6584d3844d5dde88c4237eb38","tier","pro","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_43":44,"_32":45,"_38":252,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":31,"_57":58,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":253,"_80":254,"_82":255,"_84":25},"tree",[181,183,189,190],"files",[181,182,183,184,185,186,187,188,189,190],"counts",{"_170":162,"_171":62,"_172":173,"_174":175,"_176":177},"contents",[140,141,142,143],"upstreamProject",{"_43":94,"_32":138,"_52":53,"_54":-5,"_49":50,"_78":139},"upstreamBranch",{"_43":89,"_38":131,"_132":133,"_32":86,"_87":94,"_13":91,"_88":134,"_90":135,"_136":137,"_47":-5,"_84":25,"_49":50,"_97":98,"_63":64,"_96":138},"emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},["D",1755188493466],"updatedAt",["D",1778178264384],"e004ac78-4230-11f0-834a-76b3cceeab13",47,"lockfileKey","1778178275619-esm.town_v_jubertioai_hello-realtime@379-main","hello-realtime",{"_99":98,"_106":102},{"_43":165,"_32":166,"_30":167,"_145":-5,"_41":42,"_40":62,"_146":58,"_38":168,"_13":162,"_33":163,"_35":-5,"_24":25,"_149":-5,"_27":169},{"_43":158,"_32":159,"_30":160,"_145":-5,"_41":42,"_40":62,"_146":58,"_38":161,"_13":162,"_33":163,"_35":-5,"_24":25,"_149":-5,"_27":164},{"_43":151,"_32":152,"_30":153,"_145":-5,"_41":42,"_40":154,"_146":25,"_38":155,"_13":62,"_33":20,"_35":156,"_24":25,"_149":-5,"_27":157},{"_43":144,"_32":28,"_30":31,"_145":-5,"_41":42,"_40":14,"_146":25,"_38":147,"_13":148,"_33":34,"_35":37,"_24":25,"_149":45,"_27":150},"f0a1b21d-fbf6-4682-a36a-2bec3593c841","parentId","latest",["D",1789097257981],4,"label",[28],"48e04926-7987-4b8a-bbf6-f4b71cf836a7","README.md","01a08d52-52ea-7194-86c2-02433b79a9a3",13,["D",1789095811926],"# hello-live\n\n**Hello Live** is an OpenAI GPT-Live app that supports both WebRTC and SIP\n(telephone) users. You can access the app via WebRTC at\n[hello-live.val.run](https://hello-live.val.run), or via SIP by calling\n425-800-0073.\n\nThis demo shows off the GPT-Live WebRTC session API, the SIP webhook flow, and\nthe server-side sideband websocket used to observe and control a session. It\nuses **Responses delegation**: `gpt-live-1` handles the spoken conversation\nwhile a backend Responses model (`gpt-5.6-luna`) handles reasoning and tool\ncalls (e.g. `end_call`).\n\nIf you remix the app, you'll just need to pop in your own `OPENAI_API_KEY` (from\n[platform.openai.com](https://platform.openai.com)), and if you want SIP, the\n`OPENAI_SIGNING_SECRET`. For SIP you also need to subscribe your project's\nwebhook to the `live.transport.incoming` event.\n\n## Architecture\n\n1. **WebRTC Flow**:\n - Browser connects to frontend and creates a WebRTC offer\n - `/rtc` endpoint posts the offer + session config to\n `POST /v1/live/sessions` and returns the SDP answer + session ID\n - sideband observer attached to monitor the session\n2. **SIP Flow**:\n - Phone call triggers the `live.transport.incoming` webhook\n - `/sip` endpoint verifies the webhook and accepts the call with the session\n config\n - sideband observer attached to monitor the session\n3. **Delegation**:\n - GPT-Live delegates tool use to the Responses backend\n - Function calls arrive on the sideband wrapped in `response.event`\n - The observer returns results with `response.item.create` and continues the\n backend with `response.create`\n\n```mermaid\nsequenceDiagram\n participant B as Browser / Phone\n participant V as hello-live (val)\n participant L as GPT-Live\n participant R as Responses backend\n\n B->>V: SDP offer (WebRTC) / webhook (SIP)\n V->>L: POST /v1/live/sessions (or /accept)\n L-->>V: session.id + SDP answer\n V-->>B: SDP answer\n V->>L: attach sideband WS\n L-->>B: audio (greeting)\n B-->>L: \"goodbye\"\n L->>R: delegate\n R-->>L: end_call function call\n L-->>V: response.event (function_call)\n V->>L: response.item.create + response.create\n V->>L: POST /hangup\n L-->>V: session.closed\n```\n\n## API Endpoints\n\n- `GET /` - serves the WebRTC-based app\n- `POST /rtc` - creates a new GPT-Live WebRTC session; body is the SDP offer,\n response is `{ session: { id }, transport: { type: \"webrtc\", sdp } }`\n- `POST /sip` - handles the `live.transport.incoming` webhook for an incoming\n SIP call\n- `POST /observer/:sessionId` - internal endpoint to attach a GPT-Live sideband\n websocket for session control\n\n## Project Structure\n\n```\n├── frontend/\n│ └── index.html # HTML/JS application for WebRTC\n├── main.tsx # Main entrypoint for the val.town app\n└── routes/\n ├── agent.ts # GPT-Live session configuration, delegation, and tools\n ├── observer.ts # Server-side sideband control for SIP & WebRTC\n ├── rtc.ts # WebRTC session setup\n ├── sip.ts # SIP webhook handling and session setup\n └── utils.ts # Various HTTP helpers\n```\n\n## File Links\n\n- [`frontend/index.html`](./frontend/index.html)\n- [`main.tsx`](./main.tsx)\n- [`routes/agent.ts`](./routes/agent.ts)\n- [`routes/observer.ts`](./routes/observer.ts)\n- [`routes/rtc.ts`](./routes/rtc.ts)\n- [`routes/sip.ts`](./routes/sip.ts)\n- [`routes/utils.ts`](./routes/utils.ts)\n\n## TODO\n\n- Drive `audioActive` / `callEnding` from reflected `session.output_audio.delta`\n events so hangup waits for the goodbye to finish playing\n- Send `session.close` from the browser on Stop and wait for `session.closed`\n- Render transcript deltas as captions\n",[152],"8d58d695-053e-4f33-867c-e98e00d8a0ce","routes","01a08d52-52ea-7194-86c1-fcfc58d46e7b",["D",1789077377613],0,"directory",[159],"41ed3c9e-89d8-4495-9e9b-43f929ce1d40","frontend","01a08d52-52ea-7194-86c1-e708aa8c7466",["D",1789077377613],[166],"pulls","branches","forks",2,"envVars",{"_178":173,"_180":162},"connections",{"_178":162,"_179":162},"count","unfilledCount","emptyCount",{"_43":165,"_32":166,"_30":167,"_145":-5,"_41":42,"_40":62,"_146":58,"_38":249,"_13":162,"_33":163,"_35":-5,"_24":25,"_149":-5,"_192":250,"_27":251},{"_43":241,"_32":242,"_30":243,"_145":167,"_41":42,"_40":244,"_146":58,"_38":245,"_13":217,"_33":20,"_35":246,"_24":25,"_149":-5,"_192":247,"_27":248},{"_43":158,"_32":159,"_30":160,"_145":-5,"_41":42,"_40":62,"_146":58,"_38":238,"_13":162,"_33":163,"_35":-5,"_24":25,"_149":-5,"_192":239,"_27":240},{"_43":230,"_32":231,"_30":232,"_145":160,"_41":42,"_40":233,"_146":25,"_38":234,"_13":173,"_33":202,"_35":235,"_24":25,"_149":-5,"_192":236,"_27":237},{"_43":222,"_32":223,"_30":224,"_145":160,"_41":42,"_40":225,"_146":25,"_38":226,"_13":173,"_33":202,"_35":227,"_24":25,"_149":-5,"_192":228,"_27":229},{"_43":214,"_32":215,"_30":216,"_145":160,"_41":42,"_40":217,"_146":58,"_38":218,"_13":62,"_33":202,"_35":219,"_24":25,"_149":-5,"_192":220,"_27":221},{"_43":206,"_32":207,"_30":208,"_145":160,"_41":42,"_40":209,"_146":58,"_38":210,"_13":62,"_33":202,"_35":211,"_24":25,"_149":-5,"_192":212,"_27":213},{"_43":198,"_32":199,"_30":200,"_145":160,"_41":42,"_40":173,"_146":58,"_38":201,"_13":62,"_33":202,"_35":203,"_24":25,"_149":-5,"_192":204,"_27":205},{"_43":151,"_32":152,"_30":153,"_145":-5,"_41":42,"_40":154,"_146":25,"_38":195,"_13":62,"_33":20,"_35":156,"_24":25,"_149":-5,"_192":196,"_27":197},{"_43":144,"_32":28,"_30":31,"_145":-5,"_41":42,"_40":14,"_146":25,"_38":191,"_13":148,"_33":34,"_35":37,"_24":25,"_149":45,"_192":193,"_27":194},["D",1789097257981],"children",[],[28],["D",1789095811926],[],[152],"3c1e587b-f850-47fd-b510-ba9f1f670b00","utils.ts","01a08d52-52ea-7194-86c1-fa926adeaebf",["D",1789089295919],"script","const LIVE_BASE_URL = \"https://api.openai.com/v1/live/sessions\";\nconst LIVE_WS_URL = \"wss://api.openai.com/v1/live/sessions\";\nconst OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\n// Builds the URL for a GPT-Live API endpoint.\nexport function makeUrl(sessionId?: string, action?: string) {\n if (!sessionId) {\n // create session (POST, WebRTC)\n return LIVE_BASE_URL;\n }\n if (action) {\n // accept, reject, hangup, refer (POST)\n return `${LIVE_BASE_URL}/${sessionId}/${action}`;\n }\n // sideband observer (web socket)\n return `${LIVE_WS_URL}/${sessionId}/attach`;\n}\n\n// Builds the headers for a GPT-Live API endpoint.\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\n// Debug helper for printing GPT-Live API errors.\nexport async function getErrorText(resp: Response) {\n const errBody = await resp.text().catch(() => \"\");\n return `${resp.status} ${resp.statusText}: ${errBody}`;\n}\n",[],[159,199],"987a429d-ec66-4046-9f1e-91220d6c2a49","sip.ts","01a08d52-52ea-7194-86c1-f5950ae11dab",6,["D",1789089296094],"import { Hono } from \"npm:hono\";\nimport { Webhook } from \"npm:svix\";\nimport { makeSession } from \"./agent.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst sip = new Hono();\n\n// Webhook event types that signal an incoming GPT-Live SIP call.\nconst INCOMING_EVENTS = [\"live.transport.incoming\", \"live.call.incoming\"];\n\n// POST /sip : webhook endpoint\nsip.post(\"/\", async (c) => {\n // Verify the webhook.\n const OPENAI_SIGNING_SECRET = Deno.env.get(\"OPENAI_SIGNING_SECRET\");\n if (!OPENAI_SIGNING_SECRET) {\n console.error(\"🔴 webhook secret not configured\");\n return c.text(\"Internal error\", 500);\n }\n const webhook = new Webhook(OPENAI_SIGNING_SECRET);\n const bodyStr = await c.req.text();\n let sessionId: string | undefined;\n try {\n const headers = Object.fromEntries(c.req.raw.headers.entries());\n const payload: any = webhook.verify(bodyStr, headers);\n\n // Only act on GPT-Live incoming-call events. The same call may also emit a\n // legacy realtime.call.incoming event; acknowledge it but do nothing.\n if (!INCOMING_EVENTS.includes(payload.type)) {\n console.log(\"ℹ️ ignoring webhook event:\", payload.type);\n return c.newResponse(null);\n }\n if (payload.data.type && payload.data.type !== \"sip\") {\n console.log(\"ℹ️ ignoring non-SIP transport:\", payload.data.type);\n return c.newResponse(null);\n }\n\n sessionId = payload.data.session_id;\n const toNumber = getToNumber(payload.data.sip_headers);\n const fromNumber = getFromNumber(payload.data.sip_headers);\n console.log(\"✅ verified webhook:\", sessionId, fromNumber, toNumber);\n } catch (error) {\n console.error(`🔴 verify failed: ${error}`);\n return c.text(\"Invalid signature\", 401);\n }\n\n // Accept the call.\n const url = makeUrl(sessionId, \"accept\");\n const headers = makeHeaders(\"application/json\");\n const body = JSON.stringify({ session: makeSession() });\n const opts = { method: \"POST\", headers, body };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n console.error(`🔴 accept failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n console.log(\"✅ call accepted:\", sessionId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${sessionId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n return c.newResponse(null);\n});\n\nconst getToNumber = (headers) => getNumberFromSipHeader(headers, \"to\");\nconst getFromNumber = (headers) => getNumberFromSipHeader(headers, \"from\");\n\nfunction getNumberFromSipHeader(headers: any, name: string) {\n const header = getSipHeader(headers, name);\n const match = header?.match(/sip:\\+?(\\d+)@/);\n return match ? `+${match[1]}` : undefined;\n}\n\nfunction getSipHeader(headers: any, name: string) {\n return headers?.find((h) => h.name.toLowerCase() === name)?.value;\n}\n\nexport default sip;\n",[],[159,207],"79541d33-5428-4e63-9cf5-e80266ec9e0c","rtc.ts","01a08d52-52ea-7194-86c1-f11dcf89267e",5,["D",1789089296042],"import { Hono } from \"npm:hono\";\nimport { makeSession } from \"./agent.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new GPT-Live WebRTC session\nrtc.post(\"/\", async (c) => {\n // Create the session: JSON body with the session config and the SDP offer.\n const body = JSON.stringify({\n session: makeSession(),\n transport: { type: \"webrtc\", sdp: await c.req.text() },\n });\n const opts = { method: \"POST\", headers: makeHeaders(\"application/json\"), body };\n const resp = await fetch(makeUrl(), opts);\n if (!resp.ok) {\n console.error(`🔴 start session failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n\n // Response: { session: { id }, transport: { type: \"webrtc\", sdp } }\n const result = await resp.json();\n const sessionId = result?.session?.id;\n console.log(\"✅ session created:\", sessionId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${sessionId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n return c.json(result, 201);\n});\n\nexport default rtc;\n",[],[159,215],"1acf792a-7420-462e-9a6f-495ff18bd171","observer.ts","01a08d52-52ea-7194-86c1-ef7b6e72cd3b",12,["D",1789090261169],"import { Hono } from \"npm:hono\";\nimport WebSocket from \"npm:ws\";\nimport { createHandler } from \"./agent.ts\";\nimport { makeHeaders, makeUrl } from \"./utils.ts\";\n\nexport interface ObserverHandler {\n onInit(): void;\n onTool(name: string, args: any, callId: string): void;\n onOutputAudioStarted(): void;\n onOutputAudioStopped(): void;\n onOutputAudioCleared(): void;\n}\n\n// ObserverClient attaches a sideband websocket to an existing GPT-Live session.\nexport class ObserverClient {\n private ws: WebSocket;\n private closeResolve: () => void;\n private closePromise: Promise;\n private handler: ObserverHandler;\n\n constructor(sessionId: string) {\n const url = makeUrl(sessionId);\n this.ws = new WebSocket(url, { headers: makeHeaders() });\n this.ws.on(\"open\", () => this.handleOpen());\n this.ws.on(\"message\", (data) => this.handleMessage(JSON.parse(data)));\n this.ws.on(\"error\", (err) => this.handleError(err));\n this.ws.on(\"close\", (ev) => this.handleClose(ev));\n this.closePromise = new Promise((resolve) =>\n this.closeResolve = resolve\n );\n this.handler = createHandler(this, sessionId);\n }\n // Starts or continues delegated backend (Responses) work.\n createResponse() {\n this.sendMessage({ type: \"response.create\" });\n }\n // Returns a function call result to the delegated backend.\n sendToolResult(callId: string, output: string) {\n this.sendMessage({\n type: \"response.item.create\",\n item: { type: \"function_call_output\", call_id: callId, output },\n });\n }\n async waitForClose() {\n await this.closePromise;\n }\n private sendMessage(message: any) {\n this.ws.send(JSON.stringify(message));\n }\n private handleOpen() {\n // The session is already running; do not send session.start.\n console.log(\"✅ websocket connected\");\n this.handler.onInit();\n }\n private async handleMessage(message: any) {\n // Ignore verbose messages: top-level deltas, nested Responses deltas,\n // and reflected input audio.\n const isVerbose = message.type?.endsWith(\".delta\") ||\n message.event?.type?.endsWith(\".delta\") ||\n message.type === \"session.input_audio.append\";\n if (!isVerbose) {\n console.log(message);\n }\n switch (message.type) {\n case \"response.event\": {\n // Nested Responses events arrive wrapped in an envelope.\n const inner = message.event;\n if (\n inner?.type === \"response.output_item.done\" &&\n inner.item?.type === \"function_call\"\n ) {\n this.handler.onTool(\n inner.item.name,\n inner.item.arguments,\n inner.item.call_id,\n );\n }\n break;\n }\n case \"session.closed\":\n console.log(\n `session closed: reason=${message.reason} usage=${\n JSON.stringify(message.usage)\n }`,\n );\n break;\n case \"error\":\n console.error(`🔴 session error: ${JSON.stringify(message.error)}`);\n break;\n }\n }\n private handleError(err: Error) {\n console.error(`🔴 websocket failed: ${err.message}`);\n this.closeResolve();\n }\n private handleClose(e: Event) {\n console.log(`websocket closed: ${e}`);\n this.closeResolve();\n }\n}\n\nconst observer = new Hono();\n\n// POST /observer/:sessionId : self-connect to establish a sideband web socket\nobserver.post(\"/:sessionId\", async (c) => {\n const sessionId = c.req.param(\"sessionId\");\n const client = new ObserverClient(sessionId);\n await client.waitForClose();\n return c.newResponse(null);\n});\n\nexport default observer;\n",[],[159,223],"b5f3bacb-e62e-4990-ae0f-df17c863803f","agent.ts","01a08d52-52ea-7194-86c1-ea635a7ced70",16,["D",1789096725583],"import { ObserverClient, ObserverHandler } from \"./observer.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst MODEL = \"gpt-live-1\";\nconst BACKEND_MODEL = \"gpt-5.6-luna\";\nconst VOICE = \"marin\";\nconst DATE = new Date().toLocaleDateString();\n\n// Instructions for the GPT-Live voice model: conversation style + when to delegate.\nconst INSTRUCTIONS = `\n As soon as the session starts, without waiting for the user to speak, greet the user in English\n and tell them that they're using the GPT-Live API.\n Give them a very brief summary of the benefits of GPT-Live based on the details below,\n and then ask if they have any questions.\n When the user says goodbye or you think the call is over, say a brief goodbye and then\n ask the backend to end the call.\n The current date is ${DATE}.\n ---\n GPT-Live is full duplex: it can listen while it speaks, so conversations feel natural.\n It handles the spoken conversation while a separate backend model handles reasoning and tools,\n so the conversation keeps flowing while that work runs in the background.\n`;\n\n// Instructions for the delegated Responses backend: business rules + tool use.\nconst BACKEND_INSTRUCTIONS = `\n You support a live voice conversation. Keep results short and factual.\n When asked to end the call, call the end_call function.\n The current date is ${DATE}.\n`;\n\nconst HANGUP_TOOL = {\n type: \"function\",\n name: \"end_call\",\n description: `\n Use this function to hang up the call when the user says goodbye\n or otherwise indicates they are about to end the call.`,\n parameters: { type: \"object\", properties: {}, additionalProperties: false },\n};\n\nconst SEARCH_TOOL = {\n type: \"web_search\",\n};\n\nconst BACKEND = {\n type: \"responses\",\n responses: {\n model: BACKEND_MODEL,\n instructions: BACKEND_INSTRUCTIONS,\n tools: [HANGUP_TOOL, SEARCH_TOOL],\n tool_choice: \"auto\",\n parallel_tool_calls: false,\n },\n};\n\n// Builds the declarative session configuration for a GPT-Live session.\nexport function makeSession() {\n return {\n model: MODEL,\n instructions: INSTRUCTIONS,\n audio: { output: { voice: VOICE } },\n delegation: BACKEND,\n };\n}\n\n// AgentHandler implements the agent runtime behavior (tool calling, etc).\nclass AgentHandler implements ObserverHandler {\n private audioActive = false;\n private callEnding = false;\n constructor(private client: ObserverClient, private sessionId: string) {}\n onInit() {\n // Nothing to do: the greeting is baked into the session instructions.\n }\n async onTool(name: string, _args: any, callId: string) {\n if (name == HANGUP_TOOL.name) {\n // Return the tool result and let the backend response finish.\n this.client.sendToolResult(callId, JSON.stringify({ status: \"ok\" }));\n this.client.createResponse();\n // If audio is playing, defer the hangup until it finishes.\n // Otherwise, end the call now.\n if (this.audioActive) {\n this.callEnding = true;\n } else {\n await this.hangup();\n }\n }\n }\n async onOutputAudioStarted() {\n this.audioActive = true;\n }\n async onOutputAudioStopped() {\n // Shut down the call and websocket once audio has finished spooling.\n this.audioActive = false;\n if (this.callEnding) {\n await this.hangup();\n }\n }\n async onOutputAudioCleared() {\n this.audioActive = false;\n }\n private async hangup() {\n const url = makeUrl(this.sessionId, \"hangup\");\n const resp = await fetch(url, { method: \"POST\", headers: makeHeaders() });\n if (!resp.ok) {\n console.error(`🔴 end call failed: ${await getErrorText(resp)}`);\n return;\n }\n\n console.log(\"✅ call ended\");\n }\n}\n\n// Creates the runtime handler for the session.\nexport function createHandler(client: ObserverClient, sessionId: string) {\n return new AgentHandler(client, sessionId);\n}",[],[159,231],["D",1789077377613],[184,185,186,187,188],[159],"e176d513-0c31-4ddb-9253-907244e88783","index.html","01a08d52-52ea-7194-86c1-e2418c390e45",11,["D",1789089316618],"\n\n \n \n \n OpenAI GPT-Live Voice Agent\n \n \n \n \n\n

\n\n

\n\n \n \n",[],[166,242],["D",1789077377613],[182],[166],["D",1789077377613],{"_99":98,"_106":102},{"_99":98,"_100":-5,"_101":102,"_18":55,"_103":104,"_105":25},{"_13":85,"_32":86,"_43":42,"_87":44,"_88":89,"_90":91,"_92":50,"_93":94,"_95":64,"_96":45,"_97":98},"ctx",{"_78":-5,"_128":289,"_290":291,"_292":293,"_294":25,"_295":25,"_296":25,"_297":58,"_298":-5,"_299":-7,"_300":-7,"_301":162,"_302":25,"_303":25,"_304":-5,"_305":25,"_306":25,"_307":25,"_308":-5},"ENV",{"_260":261,"_262":263,"_264":86,"_265":266,"_267":268,"_269":270,"_271":272,"_273":274,"_275":276,"_277":278,"_279":280,"_281":282,"_283":284,"_285":286,"_287":288},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","899124c68db8496248a17ee95f074b807335ea7b","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

GPT-Live Agent

Diff · v18