OpenAI GPT-Live Voice Agent

\n\n

\n\n \n \n",[18,68],"pulls","branches","forks","envVars",{"_180":182,"_183":182},"connections",{"_180":26,"_181":26},"count","unfilledCount",2,"emptyCount",{"_33":161,"_13":18,"_66":74,"_43":-5,"_35":36,"_164":40,"_169":24,"_37":248,"_39":26,"_69":75,"_162":-5,"_70":30,"_165":-5,"_200":249,"_17":250},{"_33":168,"_13":68,"_66":67,"_43":74,"_35":36,"_164":40,"_169":24,"_37":245,"_39":26,"_69":21,"_162":171,"_70":30,"_165":-5,"_200":246,"_17":247},{"_33":241,"_13":219,"_66":213,"_43":-5,"_35":36,"_164":40,"_169":24,"_37":242,"_39":26,"_69":75,"_162":-5,"_70":30,"_165":-5,"_200":243,"_17":244},{"_33":234,"_13":235,"_66":236,"_43":213,"_35":36,"_164":40,"_169":24,"_37":237,"_39":26,"_69":215,"_162":238,"_70":30,"_165":-5,"_200":239,"_17":240},{"_33":227,"_13":228,"_66":229,"_43":213,"_35":36,"_164":40,"_169":24,"_37":230,"_39":26,"_69":215,"_162":231,"_70":30,"_165":-5,"_200":232,"_17":233},{"_33":220,"_13":221,"_66":222,"_43":213,"_35":36,"_164":40,"_169":24,"_37":223,"_39":26,"_69":215,"_162":224,"_70":30,"_165":-5,"_200":225,"_17":226},{"_33":210,"_13":211,"_66":212,"_43":213,"_35":36,"_164":40,"_169":24,"_37":214,"_39":26,"_69":215,"_162":216,"_70":30,"_165":-5,"_200":217,"_17":218},{"_33":203,"_13":204,"_66":205,"_43":-5,"_35":36,"_164":111,"_169":24,"_37":206,"_39":40,"_69":21,"_162":207,"_70":30,"_165":-5,"_200":208,"_17":209},{"_33":193,"_13":194,"_66":87,"_43":-5,"_35":36,"_164":195,"_169":24,"_37":196,"_39":182,"_69":197,"_162":198,"_70":30,"_165":199,"_200":201,"_17":202},"41bc99bf-9bc9-4a4a-9dad-27094d25a781","main.tsx",3,["D",1789134513163],"http","import { Hono } from \"npm:hono\";\nimport { blob } from \"https://esm.town/v/std/blob\";\nimport { parseVal, serveFile } from \"https://esm.town/v/std/utils/index.ts\";\n// import mcp from \"./routes/mcp.ts\";\nimport rtc from \"./routes/rtc.ts\";\nimport observer from \"./routes/observer.ts\";\n\nconst SOURCE_URL = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\")).links.self.val;\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.route(\"/observer\", observer);\n// app.route(\"/mcp\", mcp);\napp.get(\"/source\", (c) => c.redirect(SOURCE_URL));\n\napp.get(\"/\", () => serveFile(\"/frontend/index.html\", import.meta.url));\napp.get(\"/favicon.ico\", async (c) => {\n const entry = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\"));\n const file = await blob.get(`${entry.name}/favicon.ico`);\n if (!file) {\n return c.text(\"favicon not found\", 404);\n }\n return new Response(file.body, {\n headers: {\n \"Content-Type\": \"image/x-icon\",\n \"Cache-Control\": \"public, max-age=31536000, immutable\",\n },\n });\n});\n\nexport default app.fetch;\n","hello-live-42","children",[],[194],"1ed8ad71-593e-46f2-90e0-0ec30ab83b1e","README.md","01a090b9-2a2d-76c2-a217-de6028994993",["D",1789134529183],"# hello-live\n\n**Hello Live** is an OpenAI GPT-Live app that supports WebRTC voice calls. You\ncan access the app at [hello-live.val.run](https://hello-live.val.run).\n\nThis demo shows off the GPT-Live WebRTC session API and the server-side sideband\nwebsocket used to observe and control a session. It uses **Responses\ndelegation**: `gpt-live-1` handles the spoken conversation while a backend\nResponses model (`gpt-5.6-luna`) handles reasoning and tool calls (e.g.\n`end_call`). Demo calls are limited to 3 minutes.\n\nIf you remix the app, you'll just need to pop in your own `OPENAI_API_KEY` (from\n[platform.openai.com](https://platform.openai.com)).\n\n## Architecture\n\n1. **WebRTC Flow**:\n - Browser connects to frontend and creates a WebRTC offer\n - `/rtc` endpoint posts the offer + session config to\n `POST /v1/live/sessions` and returns the SDP answer + session ID\n - sideband observer attached to monitor the session\n2. **Delegation**:\n - GPT-Live delegates tool use to the Responses backend\n - Function calls arrive on the sideband wrapped in `response.event`\n - The observer returns results with `response.item.create` and continues the\n backend with `response.create`\n\n```mermaid\nsequenceDiagram\n participant B as Browser\n participant V as hello-live (val)\n participant L as GPT-Live\n participant R as Responses backend\n\n B->>V: SDP offer (WebRTC)\n V->>L: POST /v1/live/sessions\n L-->>V: session.id + SDP answer\n V-->>B: SDP answer\n V->>L: attach sideband WS\n L-->>B: audio (greeting)\n B-->>L: \"goodbye\"\n L->>R: delegate\n R-->>L: end_call function call\n L-->>V: response.event (function_call)\n V->>L: response.item.create + response.create\n V->>L: POST /hangup\n L-->>V: session.closed\n```\n\n## API Endpoints\n\n- `GET /` - serves the WebRTC-based app\n- `POST /rtc` - creates a new GPT-Live WebRTC session; body is the SDP offer,\n response is `{ session: { id }, transport: { type: \"webrtc\", sdp } }`\n- `POST /observer/:sessionId` - internal endpoint to attach a GPT-Live sideband\n websocket for session control\n\n## Project Structure\n\n```\n├── frontend/\n│ └── index.html # HTML/JS application for WebRTC\n├── main.tsx # Main entrypoint for the val.town app\n└── routes/\n ├── agent.ts # GPT-Live session configuration, delegation, and tools\n ├── observer.ts # Server-side sideband control for WebRTC\n ├── rtc.ts # WebRTC session setup\n └── utils.ts # Various HTTP helpers\n```\n\n## File Links\n\n- [`frontend/index.html`](./frontend/index.html)\n- [`main.tsx`](./main.tsx)\n- [`routes/agent.ts`](./routes/agent.ts)\n- [`routes/observer.ts`](./routes/observer.ts)\n- [`routes/rtc.ts`](./routes/rtc.ts)\n- [`routes/utils.ts`](./routes/utils.ts)\n\n## TODO\n\n- Drive `audioActive` / `callEnding` from reflected `session.output_audio.delta`\n events so hangup waits for the goodbye to finish playing\n- Send `session.close` from the browser on Stop and wait for `session.closed`\n- Render transcript deltas as captions\n",[],[204],"cfefedbd-89ae-41be-a515-39c0854962bd","utils.ts","01a090b9-2a2d-76c2-a217-d5a57881496c","01a090b9-2a2d-76c2-a217-da3b745137b9",["D",1789134449113],"script","const LIVE_BASE_URL = \"https://api.openai.com/v1/live/sessions\";\nconst LIVE_WS_URL = \"wss://api.openai.com/v1/live/sessions\";\nconst OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\n// Builds the URL for a GPT-Live API endpoint.\nexport function makeUrl(sessionId?: string, action?: string) {\n if (!sessionId) {\n // create session (POST, WebRTC)\n return LIVE_BASE_URL;\n }\n if (action) {\n // accept, reject, hangup, refer (POST)\n return `${LIVE_BASE_URL}/${sessionId}/${action}`;\n }\n // sideband observer (web socket)\n return `${LIVE_WS_URL}/${sessionId}/attach`;\n}\n\n// Builds the headers for a GPT-Live API endpoint.\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\n// Debug helper for printing GPT-Live API errors.\nexport async function getErrorText(resp: Response) {\n const errBody = await resp.text().catch(() => \"\");\n return `${resp.status} ${resp.statusText}: ${errBody}`;\n}\n",[],[219,211],"routes","5d437b85-5440-4871-9b1f-6767e016bccf","rtc.ts","01a090b9-2a2d-76c2-a217-cc930931a340",["D",1789134449113],"import { Hono } from \"npm:hono\";\nimport { makeSession } from \"./agent.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new GPT-Live WebRTC session\nrtc.post(\"/\", async (c) => {\n // Create the session: JSON body with the session config and the SDP offer.\n const body = JSON.stringify({\n session: makeSession(),\n transport: { type: \"webrtc\", sdp: await c.req.text() },\n });\n const opts = { method: \"POST\", headers: makeHeaders(\"application/json\"), body };\n const resp = await fetch(makeUrl(), opts);\n if (!resp.ok) {\n console.error(`🔴 start session failed: ${await getErrorText(resp)}`);\n return c.text(\"Internal error\", 500);\n }\n\n // Response: { session: { id }, transport: { type: \"webrtc\", sdp } }\n const result = await resp.json();\n const sessionId = result?.session?.id;\n console.log(\"✅ session created:\", sessionId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${sessionId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer start failed:\", e));\n\n return c.json(result, 201);\n});\n\nexport default rtc;\n",[],[219,221],"52c630d3-af7f-4b57-945e-17198c2f334b","observer.ts","01a090b9-2a2d-76c2-a217-c8fa43abd328",["D",1789134449113],"import { Hono } from \"npm:hono\";\nimport WebSocket from \"npm:ws\";\nimport { createHandler } from \"./agent.ts\";\nimport { makeHeaders, makeUrl } from \"./utils.ts\";\n\nexport interface ObserverHandler {\n onInit(): void;\n onClosed(): void;\n onTool(name: string, args: any, callId: string): void;\n onOutputAudioStarted(): void;\n onOutputAudioStopped(): void;\n onOutputAudioCleared(): void;\n}\n\n// ObserverClient attaches a sideband websocket to an existing GPT-Live session.\nexport class ObserverClient {\n private ws: WebSocket;\n private closeResolve: () => void;\n private closePromise: Promise;\n private handler: ObserverHandler;\n\n constructor(sessionId: string) {\n const url = makeUrl(sessionId);\n this.ws = new WebSocket(url, { headers: makeHeaders() });\n this.ws.on(\"open\", () => this.handleOpen());\n this.ws.on(\"message\", (data) => this.handleMessage(JSON.parse(data)));\n this.ws.on(\"error\", (err) => this.handleError(err));\n this.ws.on(\"close\", (ev) => this.handleClose(ev));\n this.closePromise = new Promise((resolve) =>\n this.closeResolve = resolve\n );\n this.handler = createHandler(this, sessionId);\n }\n // Starts or continues delegated backend (Responses) work.\n createResponse() {\n this.sendMessage({ type: \"response.create\" });\n }\n // Returns a function call result to the delegated backend.\n sendToolResult(callId: string, output: string) {\n this.sendMessage({\n type: \"response.item.create\",\n item: { type: \"function_call_output\", call_id: callId, output },\n });\n }\n async waitForClose() {\n await this.closePromise;\n }\n private sendMessage(message: any) {\n this.ws.send(JSON.stringify(message));\n }\n private handleOpen() {\n // The session is already running; do not send session.start.\n console.log(\"✅ websocket connected\");\n this.handler.onInit();\n }\n private async handleMessage(message: any) {\n // Ignore verbose messages: top-level deltas, nested Responses deltas,\n // and reflected input audio.\n const isVerbose = message.type?.endsWith(\".delta\") ||\n message.event?.type?.endsWith(\".delta\") ||\n message.type === \"session.input_audio.append\";\n if (!isVerbose) {\n console.log(message);\n }\n switch (message.type) {\n case \"response.event\": {\n // Nested Responses events arrive wrapped in an envelope.\n const inner = message.event;\n if (\n inner?.type === \"response.output_item.done\" &&\n inner.item?.type === \"function_call\"\n ) {\n this.handler.onTool(\n inner.item.name,\n inner.item.arguments,\n inner.item.call_id,\n );\n }\n break;\n }\n case \"session.closed\":\n console.log(\n `session closed: reason=${message.reason} usage=${\n JSON.stringify(message.usage)\n }`,\n );\n break;\n case \"error\":\n console.error(`🔴 session error: ${JSON.stringify(message.error)}`);\n break;\n }\n }\n private handleError(err: Error) {\n console.error(`🔴 websocket failed: ${err.message}`);\n this.handler.onClosed();\n this.closeResolve();\n }\n private handleClose(e: Event) {\n console.log(`websocket closed: ${e}`);\n this.handler.onClosed();\n this.closeResolve();\n }\n}\n\nconst observer = new Hono();\n\n// POST /observer/:sessionId : self-connect to establish a sideband web socket\nobserver.post(\"/:sessionId\", async (c) => {\n const sessionId = c.req.param(\"sessionId\");\n const client = new ObserverClient(sessionId);\n await client.waitForClose();\n return c.newResponse(null);\n});\n\nexport default observer;\n",[],[219,228],"9092e558-b373-4e26-93a6-1743717f2a5b","agent.ts","01a090b9-2a2d-76c2-a217-c4bf4db31b1b",["D",1789134449113],"import { ObserverClient, ObserverHandler } from \"./observer.ts\";\nimport { getErrorText, makeHeaders, makeUrl } from \"./utils.ts\";\n\nconst MODEL = \"gpt-live-1\";\nconst BACKEND_MODEL = \"gpt-5.6-luna\";\nconst VOICE = \"marin\";\nconst DATE = new Date().toLocaleDateString();\nconst MAX_CALL_MS = 180 * 1000; // hard cap on call length to prevent abuse\n\n// Instructions for the GPT-Live voice model: conversation style + when to delegate.\nconst INSTRUCTIONS = `\n As soon as the session starts, without waiting for the user to speak, greet the user in English\n and tell them that you're an AI assistant powered by the GPT-Live API. Mention that you can\n simultaneously listen and speak, for more natural conversation, and that you can also invoke\n GPT 5.6 or tools like web search when more information is needed. Then ask if there's anything\n you can help with.\n\n When the user says goodbye or you think the call is over, say a brief goodbye and then\n ask the backend to end the call.\n The current date is ${DATE}.\n`;\n\n// Instructions for the delegated Responses backend: business rules + tool use.\nconst BACKEND_INSTRUCTIONS = `\n You support a live voice conversation. Keep results short and factual.\n When asked to end the call, call the end_call function.\n The current date is ${DATE}.\n`;\n\nconst HANGUP_TOOL = {\n type: \"function\",\n name: \"end_call\",\n description: `\n Use this function to hang up the call when the user says goodbye\n or otherwise indicates they are about to end the call.`,\n parameters: { type: \"object\", properties: {}, additionalProperties: false },\n};\n\nconst SEARCH_TOOL = {\n type: \"web_search\",\n};\n\nconst BACKEND = {\n type: \"responses\",\n responses: {\n model: BACKEND_MODEL,\n instructions: BACKEND_INSTRUCTIONS,\n tools: [HANGUP_TOOL, SEARCH_TOOL],\n tool_choice: \"auto\",\n parallel_tool_calls: false,\n },\n};\n\n// Builds the declarative session configuration for a GPT-Live session.\nexport function makeSession() {\n return {\n model: MODEL,\n instructions: INSTRUCTIONS,\n audio: { output: { voice: VOICE } },\n delegation: BACKEND,\n };\n}\n\n// AgentHandler implements the agent runtime behavior (tool calling, etc).\nclass AgentHandler implements ObserverHandler {\n private audioActive = false;\n private callEnding = false;\n private timeout?: ReturnType;\n constructor(private client: ObserverClient, private sessionId: string) {}\n onInit() {\n // The greeting is baked into the session instructions; just arm the\n // max-duration timer.\n this.timeout = setTimeout(() => {\n console.log(`⏰ call exceeded ${MAX_CALL_MS / 1000}s, hanging up`);\n this.hangup();\n }, MAX_CALL_MS);\n }\n onClosed() {\n clearTimeout(this.timeout);\n }\n async onTool(name: string, _args: any, callId: string) {\n if (name == HANGUP_TOOL.name) {\n // Return the tool result and let the backend response finish.\n this.client.sendToolResult(callId, JSON.stringify({ status: \"ok\" }));\n this.client.createResponse();\n // If audio is playing, defer the hangup until it finishes.\n // Otherwise, end the call now.\n if (this.audioActive) {\n this.callEnding = true;\n } else {\n await this.hangup();\n }\n }\n }\n async onOutputAudioStarted() {\n this.audioActive = true;\n }\n async onOutputAudioStopped() {\n // Shut down the call and websocket once audio has finished spooling.\n this.audioActive = false;\n if (this.callEnding) {\n await this.hangup();\n }\n }\n async onOutputAudioCleared() {\n this.audioActive = false;\n }\n private async hangup() {\n clearTimeout(this.timeout);\n const url = makeUrl(this.sessionId, \"hangup\");\n const resp = await fetch(url, { method: \"POST\", headers: makeHeaders() });\n if (!resp.ok) {\n console.error(`🔴 end call failed: ${await getErrorText(resp)}`);\n return;\n }\n\n console.log(\"✅ call ended\");\n }\n}\n\n// Creates the runtime handler for the session.\nexport function createHandler(client: ObserverClient, sessionId: string) {\n return new AgentHandler(client, sessionId);\n}",[],[219,235],"18ba3975-f1cf-4737-b6f6-960aa0c2d1b7",["D",1789134449113],[187,188,189,190],[219],["D",1789134449113],[],[18,68],["D",1789134449113],[185],[18],["D",1789134449113],{"_11":12,"_127":73},{"_11":12,"_122":123,"_72":73,"_31":71,"_124":125,"_126":30},{"_39":111,"_13":16,"_33":36,"_112":77,"_113":63,"_114":64,"_115":81,"_116":117,"_118":119,"_120":14,"_121":12},"ctx",{"_52":-5,"_148":288,"_289":290,"_291":292,"_293":30,"_294":30,"_295":30,"_296":24,"_297":-5,"_298":-7,"_299":-7,"_300":26,"_301":30,"_302":30,"_303":-5,"_304":30,"_305":30,"_306":30,"_307":-5},"ENV",{"_259":260,"_261":262,"_263":16,"_264":265,"_266":267,"_268":269,"_270":271,"_272":273,"_274":275,"_276":277,"_278":279,"_280":281,"_282":283,"_284":285,"_286":287},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","c73f21da490985bff19c279eb0e09db24d688e4f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

GPT-Live Agent

History · frontend