OpenAI Realtime Transcription

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_405":406,"_407":408},"routes/_app.x.$handle.$name",{"_100":99,"_32":46,"_108":87,"_27":109,"_13":14,"_110":-5,"_11":111,"_112":113,"_114":115,"_116":117,"_118":119,"_20":-5,"_120":121,"_122":123,"_124":25,"_125":126,"_127":59,"_128":-5,"_129":-5,"_130":131},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_44":45,"_32":46,"_39":47,"_48":-5,"_49":25,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":57,"_58":59,"_60":59,"_61":-5,"_62":63,"_64":65,"_66":-5,"_67":-5,"_68":-5,"_69":-5,"_70":71,"_72":71,"_73":74,"_75":74,"_76":74,"_77":71,"_78":71,"_79":80,"_81":82,"_83":84,"_85":25},"version",41,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":33,"_34":35,"_36":38,"_39":40,"_41":14,"_42":43,"_24":25},"previous",{"_30":31,"_32":33,"_34":35,"_36":37},"deleted",false,"deletedHere","path","routes/rtc.ts","latestPath","fileId","01990bce-c638-714e-97a4-07196c7003e4","name","rtc.ts","fileType","script","content","import { Hono } from \"npm:hono\";\nimport { makeHeaders, makeTranscriptionSession } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new transcription call\nrtc.post(\"/\", async (c) => {\n // Get transcription configuration from query params\n const model = c.req.query(\"model\") as any || \"gpt-4o-transcribe\";\n const language = c.req.query(\"language\") || \"en\";\n const vad = c.req.query(\"vad\") !== \"false\"; // default true\n const logprobs = c.req.query(\"logprobs\") === \"true\"; // default false\n \n // Create the transcription session config\n const sessionConfig = makeTranscriptionSession({\n model,\n language,\n enableVAD: vad,\n includeLogprobs: logprobs\n });\n\n // Step 1: Get ephemeral token from transcription_sessions endpoint\n const sessionUrl = \"https://api.openai.com/v1/realtime/transcription_sessions\";\n const sessionResp = await fetch(sessionUrl, {\n method: \"POST\",\n headers: {\n ...makeHeaders(),\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify(sessionConfig),\n });\n \n if (!sessionResp.ok) {\n const errText = await sessionResp.text().catch(() => \"\");\n console.error(`🔴 Failed to create transcription session: ${sessionResp.status} ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n \n const sessionData = await sessionResp.json();\n const clientSecret = sessionData.client_secret.value;\n console.log(\"✅ Got transcription session token\");\n \n // Step 2: Use the token to establish WebRTC connection\n const realtimeUrl = \"https://api.openai.com/v1/realtime\";\n const sdpResp = await fetch(realtimeUrl, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${clientSecret}`,\n \"Content-Type\": \"application/sdp\",\n },\n body: await c.req.text(),\n });\n \n if (!sdpResp.ok) {\n const errText = await sdpResp.text().catch(() => \"\");\n console.error(`🔴 Failed to establish WebRTC connection: ${sdpResp.status} ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n \n const contentType = sdpResp.headers.get(\"Content-Type\");\n const location = sdpResp.headers.get(\"Location\");\n const callId = location?.split(\"/\").pop();\n console.log(\"✅ transcription call created:\", callId);\n\n // Kick off observer in the background to monitor transcription events\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch(() => {}); // catch to not crash the process\n\n return c.newResponse(sdpResp.body, {\n headers: { \"Content-Type\": contentType },\n });\n});\n\nexport default rtc;","import { Hono } from \"npm:hono\";\nimport { makeHeaders, makeTranscriptionSession } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new transcription call\nrtc.post(\"/\", async (c) => {\n // Get transcription configuration from query params\n const sessionConfig = makeTranscriptionSession({\n model: c.req.query(\"model\") as any || \"gpt-4o-transcribe\",\n language: c.req.query(\"language\") || \"en\",\n enableVAD: c.req.query(\"vad\") !== \"false\"\n });\n\n // Step 1: Get ephemeral token from transcription_sessions endpoint\n const sessionUrl = \"https://api.openai.com/v1/realtime/transcription_sessions\";\n const sessionResp = await fetch(sessionUrl, {\n method: \"POST\",\n headers: {\n ...makeHeaders(),\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify(sessionConfig),\n });\n \n if (!sessionResp.ok) {\n const errText = await sessionResp.text().catch(() => \"\");\n console.error(`🔴 Failed to create transcription session: ${sessionResp.status} ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n \n const sessionData = await sessionResp.json();\n const clientSecret = sessionData.client_secret.value;\n console.log(\"✅ Got transcription session token\");\n \n // Step 2: Use the token to establish WebRTC connection\n const realtimeUrl = \"https://api.openai.com/v1/realtime\";\n const sdpResp = await fetch(realtimeUrl, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${clientSecret}`,\n \"Content-Type\": \"application/sdp\",\n },\n body: await c.req.text(),\n });\n \n if (!sdpResp.ok) {\n const errText = await sdpResp.text().catch(() => \"\");\n console.error(`🔴 Failed to establish WebRTC connection: ${sdpResp.status} ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n \n return c.newResponse(sdpResp.body, {\n headers: { \"Content-Type\": sdpResp.headers.get(\"Content-Type\") },\n });\n});\n\nexport default rtc;","createdAt",["D",1756953072888],"branchVersion","branchId","509ca3b6-882f-11f0-9954-0224a6c84d84","id","50991944-882f-11f0-9954-0224a6c84d84","hello-transcription",["D",1756839724543],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","0198f8d1-d70f-76ea-a55f-2b5ba506023d","pinnedFileId","01990bce-c638-714e-97a4-20de8c449fb9","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","0198f8d1-d70f-76ea-a55f-2ed16889c1b0","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_100":99,"_107":103},"org",{"_100":99,"_101":-5,"_102":103,"_18":55,"_104":105,"_106":25},"branch",{"_13":86,"_32":87,"_44":43,"_88":45,"_89":90,"_91":92,"_93":51,"_94":95,"_96":65,"_97":46,"_98":99},"isOwned",45,"main","projectId","forkedBranchId","43b7750c-87f0-11f0-ba02-0224a6c84d84","forkedBranchVersion",15,"forkedProjectPrivacy","forkedProjectId","43b401c4-87f0-11f0-ba02-0224a6c84d84","forkedProjectOrgId","projectName","userHandle","emcho","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18zMXpKcFhORzY4VDlCZlRBZWUybTNFMmV3bDYifQ","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_44":45,"_32":46,"_39":401,"_48":-5,"_49":25,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":57,"_58":59,"_60":59,"_61":-5,"_62":63,"_64":65,"_66":-5,"_67":-5,"_68":-5,"_69":-5,"_70":71,"_72":71,"_73":74,"_75":74,"_76":74,"_77":71,"_78":71,"_79":402,"_81":403,"_83":404,"_85":25},"tree",[229,231,242,244,248,249,250,251,252,253],"files",[229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253],"counts",{"_219":63,"_220":166,"_221":166,"_222":223,"_224":225},"contents",[142,143,144,145,146,147,148,149,150,151],"upstreamProject",{"_44":95,"_32":140,"_53":-5,"_54":-5,"_50":51,"_79":141},"upstreamBranch",{"_44":90,"_39":132,"_133":134,"_32":87,"_88":95,"_13":135,"_89":136,"_91":137,"_138":139,"_48":-5,"_85":25,"_50":51,"_98":99,"_64":65,"_97":140},"emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},["D",1756812644593],"updatedAt",["D",1757114821629],23,"be33420e-792a-11f0-bfa3-0224a6c84d84",197,"lockfileKey","1764516869134-esm.town_v_emcho_hello-mcp@23-main","hello-mcp",{"_100":99,"_107":103},{"_44":213,"_32":214,"_30":215,"_154":-5,"_42":43,"_41":216,"_156":59,"_39":217,"_13":63,"_34":200,"_36":-5,"_24":25,"_161":-5,"_27":218},{"_44":207,"_32":208,"_30":209,"_154":-5,"_42":43,"_41":210,"_156":59,"_39":211,"_13":63,"_34":200,"_36":-5,"_24":25,"_161":-5,"_27":212},{"_44":202,"_32":203,"_30":204,"_154":-5,"_42":43,"_41":166,"_156":59,"_39":205,"_13":63,"_34":200,"_36":-5,"_24":25,"_161":-5,"_27":206},{"_44":196,"_32":197,"_30":198,"_154":-5,"_42":43,"_41":166,"_156":59,"_39":199,"_13":63,"_34":200,"_36":-5,"_24":25,"_161":-5,"_27":201},{"_44":189,"_32":190,"_30":191,"_154":-5,"_42":43,"_41":192,"_156":59,"_39":193,"_13":63,"_34":20,"_36":194,"_24":25,"_161":-5,"_27":195},{"_44":183,"_32":184,"_30":185,"_154":-5,"_42":43,"_41":166,"_156":59,"_39":186,"_13":63,"_34":20,"_36":187,"_24":25,"_161":-5,"_27":188},{"_44":177,"_32":178,"_30":179,"_154":-5,"_42":43,"_41":158,"_156":59,"_39":180,"_13":63,"_34":20,"_36":181,"_24":25,"_161":-5,"_27":182},{"_44":170,"_32":171,"_30":172,"_154":-5,"_42":43,"_41":173,"_156":59,"_39":174,"_13":166,"_34":20,"_36":175,"_24":25,"_161":-5,"_27":176},{"_44":163,"_32":164,"_30":165,"_154":-5,"_42":43,"_41":166,"_156":59,"_39":167,"_13":63,"_34":20,"_36":168,"_24":25,"_161":-5,"_27":169},{"_44":152,"_32":153,"_30":57,"_154":-5,"_42":43,"_41":155,"_156":59,"_39":157,"_13":158,"_34":159,"_36":160,"_24":25,"_161":-5,"_27":162},"3980aa2a-8937-11f0-b673-0224a6c84d84","main.tsx","parentId",40,"latest",["D",1756953072886],2,"http","import { Hono } from \"npm:hono\";\nimport { parseVal, serveFile } from \"https://esm.town/v/std/utils/index.ts\";\nimport rtc from \"./routes/rtc.ts\";\n\nconst SOURCE_URL = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\"))?.links?.self?.val || \"#\";\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.get(\"/source\", (c) => c.redirect(SOURCE_URL));\napp.get(\"/\", () => serveFile(\"/frontend/index.html\"));\n\nexport default app.fetch;","label",[153],"50a3ea9a-882f-11f0-9954-0224a6c84d84","deno.json","01990bce-c638-714e-97a4-1fa9e6167c76",1,["D",1756839724543],"{\n \"$schema\": \"https://raw.githubusercontent.com/denoland/deno/348900b8b79f4a434cab4c74b3bc8d4d2fa8ee74/cli/schemas/config-file.v1.json\",\n \"lock\": false,\n \"compilerOptions\": {\n \"noImplicitAny\": false,\n \"strict\": false,\n \"types\": [\n \"https://www.val.town/types/valtown.d.ts\"\n ],\n \"lib\": [\n \"dom\",\n \"dom.iterable\",\n \"dom.asynciterable\",\n \"deno.ns\",\n \"deno.unstable\"\n ]\n },\n \"lint\": {\n \"files\": {\n \"include\": [\n \"deno:/https/esm.town/**/*\"\n ]\n },\n \"rules\": {\n \"exclude\": [\n \"no-explicit-any\"\n ]\n }\n },\n \"node_modules_dir\": false,\n \"experimental\": {\n \"unstable-node-globals\": true,\n \"unstable-temporal\": true,\n \"unstable-worker-options\": true,\n \"unstable-sloppy-imports\": true\n }\n}",[164],"1a739988-8830-11f0-a821-0224a6c84d84","README.md","01990bce-c638-714e-97a4-1a67f3821e53",10,["D",1756840062842],"# hello-transcription\n\nReal-time speech transcription using OpenAI's Realtime API - a demonstration of transcription-only mode without AI responses.\n\n## Features\n\n- **Real-time Transcription**: Live speech-to-text conversion as you speak\n- **Multiple Models**: Choose between GPT-4o Transcribe, GPT-4o Mini Transcribe, or Whisper-1\n- **Language Support**: Transcribe in multiple languages (English, Spanish, French, German, Italian, Portuguese, Chinese, Japanese, Korean)\n- **Voice Activity Detection (VAD)**: Automatic detection of speech segments\n- **Logprobs Support**: Optional confidence scores for transcriptions\n- **Split View**: See transcriptions and event logs side-by-side\n\n## How It Works\n\nThis app uses OpenAI's Realtime API in transcription-only mode:\n1. Your voice is captured via WebRTC\n2. Audio is streamed to OpenAI's transcription service\n3. Transcriptions are returned in real-time\n4. No AI responses are generated (transcription only)\n\n## Model Differences\n\n- **GPT-4o Transcribe**: Streaming transcription with incremental updates\n- **GPT-4o Mini Transcribe**: Smaller model, streaming transcription \n- **Whisper-1**: Complete transcription after each speech segment (no streaming)\n\n## Configuration Options\n\n- **Model**: Select the transcription model\n- **Language**: Choose the primary language for better accuracy\n- **VAD**: Enable/disable automatic voice activity detection\n- **Logprobs**: Include confidence scores (for advanced use)\n\n## Usage\n\n1. Click \"Start\" to begin transcription\n2. Allow microphone access when prompted\n3. Start speaking - transcriptions appear in real-time\n4. Partial transcriptions update as you speak\n5. Final transcriptions are marked in green\n6. Click \"Stop\" to end the session\n\n## API Endpoints\n\n- `GET /` - Serves the transcription interface\n- `POST /rtc` - Creates a WebRTC transcription session\n- `POST /observer/:callId` - WebSocket observer for transcription events\n\n## Environment Variables\n\nSet in your Val Town environment:\n- `OPENAI_API_KEY` - Your OpenAI API key (required)\n\n## Local Development\n\n```bash\n# Install Deno\ncurl -fsSL https://deno.land/install.sh | sh\n\n# Run locally\ndeno run --allow-all main.tsx\n```\n\n## Val Town Deployment\n\n1. Fork/remix this val on Val Town\n2. Add your `OPENAI_API_KEY` to Val Town secrets\n3. Your app will be available at `https://[your-val-name].val.run`\n\n## Technical Details\n\nThe app uses OpenAI's Realtime API in transcription mode:\n- Session type: `transcription` (not `realtime`)\n- Audio format: PCM16\n- Noise reduction: Near-field (optimized for close microphones)\n- WebRTC data channel for receiving transcription events\n\n## Event Flow\n\n1. `input_audio_buffer.committed` - Audio chunk received\n2. `conversation.item.input_audio_transcription.delta` - Partial transcription\n3. `conversation.item.input_audio_transcription.completed` - Final transcription\n\n## Credits\n\nBuilt with OpenAI's Realtime API for transcription-only use cases.",[171],"1a3463bc-8830-11f0-a44c-0224a6c84d84","CLAUDE.md","1a34648e-8830-11f0-a44c-0224a6c84d84",["D",1756840062786],"# Hello-Transcription - OpenAI Realtime API Transcription Demo\n\n## 🎯 Project Overview\n\nHello-Transcription demonstrates the transcription-only mode of OpenAI's Realtime API. Unlike the conversational mode, this implementation focuses purely on speech-to-text conversion without generating AI responses, making it ideal for subtitles, live captions, meeting transcriptions, and other transcription-focused use cases.\n\n**Created:** September 2, 2025 \n**Platform:** Val Town \n**API:** OpenAI Realtime API (Transcription Mode) \n**Key Feature:** Real-time streaming transcription with multiple model support\n\n## 🏗️ Technical Stack\n\n- **Runtime:** Deno (Val Town platform)\n- **Framework:** Hono (lightweight web framework)\n- **Transcription:** OpenAI Realtime API in transcription mode\n- **Connection:** WebRTC with data channel for events\n- **Frontend:** Vanilla JavaScript with split-view interface\n- **Models:** GPT-4o Transcribe, GPT-4o Mini Transcribe, Whisper-1\n\n## 📁 Project Structure\n\n```\nhello-transcription/\n├── frontend/\n│ └── index.html # Split-view transcription interface\n├── routes/\n│ ├── rtc.ts # WebRTC session setup for transcription\n│ ├── observer.ts # WebSocket observer for transcription events\n│ └── utils.ts # Transcription session configuration\n├── main.tsx # Main entry point\n├── deno.json # Deno configuration\n├── README.md # User documentation\n└── CLAUDE.md # This file - technical documentation\n```\n\n## 🔑 Core Concepts\n\n### Transcription vs Conversation Mode\n\nThe Realtime API supports two distinct modes:\n\n1. **Conversation Mode** (`type: \"realtime\"`):\n - Two-way interaction with AI\n - User speaks → AI responds\n - Used in hello-realtime and hello-mcp\n\n2. **Transcription Mode** (`type: \"transcription\"`):\n - One-way speech-to-text only\n - User speaks → Text output\n - No AI responses generated\n - Lower latency, lower cost\n - This demo uses transcription mode\n\n### Transcription Session Object\n\n```typescript\n{\n type: \"transcription\",\n input_audio_format: \"pcm16\",\n input_audio_transcription: {\n model: \"gpt-4o-transcribe\", // or \"gpt-4o-mini-transcribe\", \"whisper-1\"\n prompt: \"\", // Optional context hint\n language: \"en\" // ISO-639-1 language code\n },\n turn_detection: {\n type: \"server_vad\",\n threshold: 0.5,\n prefix_padding_ms: 300,\n silence_duration_ms: 500\n },\n input_audio_noise_reduction: {\n type: \"near_field\" // or \"far_field\", null\n },\n include: [\"item.input_audio_transcription.logprobs\"] // Optional\n}\n```\n\n## 🛠️ Key Components\n\n### 1. Utils Configuration (`/routes/utils.ts`)\n\nHandles transcription session configuration with sensible defaults:\n\n```typescript\nexport function makeTranscriptionSession(config: TranscriptionConfig = {}) {\n const {\n model = \"gpt-4o-transcribe\",\n language = \"en\",\n prompt = \"\",\n enableVAD = true,\n noiseReduction = \"near_field\",\n includeLogprobs = false\n } = config;\n \n // Build session object...\n}\n```\n\n**Key Configuration Options:**\n- **model**: Transcription model selection\n- **language**: Primary language for better accuracy\n- **prompt**: Context hints (e.g., \"Expect medical terminology\")\n- **enableVAD**: Automatic voice activity detection\n- **noiseReduction**: Audio preprocessing type\n- **includeLogprobs**: Confidence scores for words\n\n### 2. RTC Route Handler (`/routes/rtc.ts`)\n\nCreates WebRTC sessions specifically for transcription:\n\n```typescript\n// Get config from query params\nconst model = c.req.query(\"model\") || \"gpt-4o-transcribe\";\nconst language = c.req.query(\"language\") || \"en\";\nconst vad = c.req.query(\"vad\") !== \"false\";\nconst logprobs = c.req.query(\"logprobs\") === \"true\";\n\n// Create transcription session\nconst sessionConfig = makeTranscriptionSession({\n model,\n language,\n enableVAD: vad,\n includeLogprobs: logprobs\n});\n```\n\n**Important:** Uses `type: \"transcription\"` not `type: \"realtime\"`\n\n### 3. Observer WebSocket (`/routes/observer.ts`)\n\nMonitors transcription events via server-side WebSocket:\n\n```typescript\nws.onmessage = (event) => {\n const data = JSON.parse(event.data);\n \n if (data.type === \"conversation.item.input_audio_transcription.delta\") {\n // Streaming partial transcription\n console.log(`📝 Transcription delta: \"${data.delta}\"`);\n } else if (data.type === \"conversation.item.input_audio_transcription.completed\") {\n // Final transcription for segment\n console.log(`✅ Transcription completed: \"${data.transcript}\"`);\n }\n};\n```\n\n### 4. Frontend Interface (`/frontend/index.html`)\n\nSplit-view interface with real-time transcription display:\n\n#### Left Panel - Transcriptions\n- Shows live transcription stream\n- Partial transcriptions update in real-time\n- Final transcriptions marked with green border\n- Each segment timestamped\n\n#### Right Panel - Event Logs\n- Technical event stream\n- Debug information\n- Connection status\n\n#### Data Channel Handling\n```javascript\ndataChannel.onmessage = (event) => {\n const data = JSON.parse(event.data);\n \n if (data.type === \"conversation.item.input_audio_transcription.delta\") {\n // Update partial transcription\n addTranscript(data.item_id, data.delta, false);\n } else if (data.type === \"conversation.item.input_audio_transcription.completed\") {\n // Mark transcription as final\n addTranscript(data.item_id, data.transcript, true);\n }\n};\n```\n\n## 📊 Model Comparison\n\n### GPT-4o Transcribe\n- **Streaming:** Yes - incremental updates via delta events\n- **Latency:** Low\n- **Accuracy:** High\n- **Use Case:** Live subtitles, real-time captions\n\n### GPT-4o Mini Transcribe\n- **Streaming:** Yes - incremental updates\n- **Latency:** Very low\n- **Accuracy:** Good\n- **Use Case:** Quick transcriptions, lower cost\n\n### Whisper-1\n- **Streaming:** No - complete segments only\n- **Latency:** Higher (waits for complete utterance)\n- **Accuracy:** Very high\n- **Use Case:** High-accuracy transcriptions, post-processing\n\n## 🔄 Event Flow\n\n### Transcription Event Sequence\n\n1. **Audio Input**\n ```\n User speaks → Microphone → WebRTC → OpenAI\n ```\n\n2. **VAD Processing** (if enabled)\n ```\n Voice detected → Buffer audio → Silence detected → Commit buffer\n ```\n\n3. **Transcription Events**\n ```\n input_audio_buffer.committed\n ↓\n conversation.item.input_audio_transcription.delta (streaming models)\n ↓\n conversation.item.input_audio_transcription.completed\n ```\n\n### Event Types\n\n#### Delta Event (Streaming)\n```json\n{\n \"type\": \"conversation.item.input_audio_transcription.delta\",\n \"item_id\": \"item_003\",\n \"content_index\": 0,\n \"delta\": \"Hello, how\"\n}\n```\n\n#### Completed Event\n```json\n{\n \"type\": \"conversation.item.input_audio_transcription.completed\",\n \"item_id\": \"item_003\",\n \"content_index\": 0,\n \"transcript\": \"Hello, how are you today?\"\n}\n```\n\n## ⚙️ Configuration Details\n\n### Voice Activity Detection (VAD)\n\nVAD automatically detects speech segments:\n\n```typescript\nturn_detection: {\n type: \"server_vad\",\n threshold: 0.5, // Sensitivity (0-1)\n prefix_padding_ms: 300, // Audio before speech\n silence_duration_ms: 500 // Silence to end segment\n}\n```\n\n**VAD Disabled:**\n```typescript\nturn_detection: null // Manual control required\n```\n\n### Noise Reduction\n\nThree noise reduction modes:\n\n1. **near_field**: Optimized for close microphones (default)\n2. **far_field**: For distant microphones/speakers\n3. **null**: No noise reduction\n\n### Language Configuration\n\nISO-639-1 language codes improve accuracy:\n- \"en\" - English\n- \"es\" - Spanish \n- \"fr\" - French\n- \"de\" - German\n- \"zh\" - Chinese\n- \"ja\" - Japanese\n\n### Logprobs (Confidence Scores)\n\nWhen enabled, provides word-level confidence:\n\n```typescript\ninclude: [\"item.input_audio_transcription.logprobs\"]\n```\n\nReturns probability scores for each transcribed word, useful for:\n- Highlighting uncertain words\n- Quality assessment\n- Post-processing decisions\n\n## 🧪 Testing Guide\n\n### Local Development\n\n1. **Setup Environment**\n ```bash\n # Create .env file\n echo \"OPENAI_API_KEY=sk-...\" > .env\n \n # Install Deno\n curl -fsSL https://deno.land/install.sh | sh\n ```\n\n2. **Run Development Server**\n ```bash\n # With auto-reload\n deno run --watch --allow-all main.tsx\n \n # Or standard\n deno run --allow-all main.tsx\n ```\n\n3. **Test Transcription**\n - Open http://localhost:8000\n - Select model and language\n - Click \"Start\"\n - Speak clearly\n - Watch transcriptions appear\n\n### Testing Different Models\n\n1. **Test Streaming (GPT-4o)**\n - Select \"GPT-4o Transcribe\"\n - Speak continuously\n - Notice incremental updates\n\n2. **Test Non-Streaming (Whisper-1)**\n - Select \"Whisper-1\"\n - Speak, then pause\n - Notice complete segments only\n\n3. **Test VAD**\n - Enable VAD\n - Speak with pauses\n - Notice automatic segmentation\n\n4. **Test Without VAD**\n - Disable VAD\n - Requires manual commit (not implemented in this demo)\n\n## 🐛 Common Issues & Solutions\n\n### Issue: No transcriptions appearing\n**Solutions:**\n- Check microphone permissions\n- Verify OPENAI_API_KEY is set\n- Check browser console for errors\n- Ensure WebRTC connection established\n\n### Issue: Transcriptions cut off mid-sentence\n**Solutions:**\n- Adjust VAD silence_duration_ms (longer = fewer cuts)\n- Try different VAD threshold\n- Consider using different model\n\n### Issue: Poor transcription accuracy\n**Solutions:**\n- Set correct language parameter\n- Use appropriate noise reduction setting\n- Provide context via prompt parameter\n- Try higher accuracy model (Whisper-1)\n\n### Issue: High latency\n**Solutions:**\n- Use GPT-4o Mini for lower latency\n- Check network connection\n- Consider disabling logprobs\n\n## 📈 Performance Characteristics\n\n### Latency Comparison\n- **GPT-4o Mini**: ~100-200ms first token\n- **GPT-4o**: ~150-250ms first token \n- **Whisper-1**: ~500-1000ms (full segment)\n\n### Throughput\n- Handles real-time audio (16kHz PCM16)\n- Multiple concurrent sessions supported\n- No buffering required for streaming models\n\n### Cost Optimization\n- Transcription-only mode is cheaper than conversation mode\n- No AI inference costs\n- GPT-4o Mini most cost-effective\n- Whisper-1 for batch/accuracy needs\n\n## 🔒 Security Considerations\n\n### Current Implementation\n- API key stored in environment variable\n- No authentication on endpoints\n- No rate limiting\n- Single-tenant design\n\n### Production Recommendations\n1. **Authentication**: Add user authentication\n2. **Rate Limiting**: Implement per-user limits\n3. **CORS**: Configure appropriate origins\n4. **Monitoring**: Track usage and errors\n5. **Encryption**: Ensure HTTPS only\n6. **Token Security**: Use short-lived tokens\n\n## 🚀 Deployment\n\n### Val Town Deployment\n\n1. **Create/Remix Val**\n ```bash\n vt remix emcho/hello-transcription my-transcription\n ```\n\n2. **Set Environment**\n - Add `OPENAI_API_KEY` in Val Town secrets\n\n3. **Deploy**\n ```bash\n vt push\n ```\n\n4. **Access**\n - URL: `https://[your-val-name].val.run`\n\n### Environment Variables\n- `OPENAI_API_KEY` - Required for OpenAI API access\n\n## 📝 Future Enhancements\n\n### Potential Features\n\n1. **Recording & Export**\n - Save transcriptions to file\n - Export as SRT/VTT subtitles\n - Download audio recordings\n\n2. **Advanced Controls**\n - Manual VAD control\n - Custom VAD parameters UI\n - Prompt templates\n\n3. **Multi-Stream**\n - Multiple speaker support\n - Speaker diarization\n - Parallel transcriptions\n\n4. **Post-Processing**\n - Punctuation enhancement\n - Grammar correction\n - Translation\n\n5. **Visualization**\n - Audio waveform display\n - VAD activity indicator\n - Confidence heat map\n\n6. **Integration**\n - Webhook support\n - Real-time API streaming\n - Database storage\n\n## 🔗 References\n\n### Documentation\n- [OpenAI Realtime Transcription Guide](https://platform.openai.com/docs/guides/realtime-transcription)\n- [Realtime API Reference](https://platform.openai.com/docs/api-reference/realtime)\n- [Voice Activity Detection Guide](https://platform.openai.com/docs/guides/realtime-vad)\n- [Val Town Documentation](https://docs.val.town)\n\n### Related Projects\n- [hello-realtime](https://hello-realtime.val.run) - Conversation mode demo\n- [hello-mcp](https://github.com/hello-mcp) - MCP tool execution demo\n\n### Key Differences from Conversation Mode\n1. Session type: `transcription` vs `realtime`\n2. No AI responses generated\n3. Different event types\n4. Lower latency and cost\n5. Focused on speech-to-text only\n\n## 💡 Implementation Notes\n\n### Critical Discoveries\n1. **Session Type**: Must use `type: \"transcription\"` not `type: \"realtime\"`\n2. **Event Names**: Different from conversation mode events\n3. **Model Behavior**: Whisper doesn't stream, GPT-4o models do\n4. **VAD Impact**: Significantly affects transcription segmentation\n5. **Language Setting**: Dramatically improves accuracy for non-English\n\n### Best Practices\n1. Always specify language for non-English content\n2. Use near_field noise reduction for headset mics\n3. Enable VAD for natural speech segmentation\n4. Choose model based on latency vs accuracy needs\n5. Monitor item_id for proper segment ordering\n\n## 🎯 Summary\n\nHello-Transcription successfully demonstrates the transcription-only capabilities of OpenAI's Realtime API. Key achievements:\n\n1. **Pure Transcription**: No AI responses, focused solely on speech-to-text\n2. **Model Flexibility**: Support for three different transcription models\n3. **Real-time Streaming**: Live transcription updates for supported models\n4. **Configuration Options**: VAD, noise reduction, language, logprobs\n5. **Clean Interface**: Split-view design for transcriptions and logs\n\nThis implementation serves as a foundation for building transcription-focused applications like live captioning, meeting transcription, subtitle generation, and accessibility tools.",[178],"50a3e2c0-882f-11f0-9954-0224a6c84d84",".vtignore","01990bce-c638-714e-97a4-157f5fb8f430",["D",1756839724543],".git\n.vscode\n.cursorrules\n.DS_Store\nnode_modules\nvendor",[184],"39792548-8937-11f0-82b5-0224a6c84d84",".gitattributes","397925fc-8937-11f0-82b5-0224a6c84d84",39,["D",1756953072887],"# Auto detect text files and perform LF normalization\n* text=auto\n",[190],"50a3e18a-882f-11f0-9954-0224a6c84d84","routes","01990bce-c638-714e-97a4-1048c460355e",["D",1756839724543],"directory",[197],"50a3d50a-882f-11f0-9954-0224a6c84d84","frontend","01990bce-c637-779b-aded-96ab9fc690df",["D",1756839724543],[203],"38dfaf46-8935-11f0-9d99-0224a6c84d84","docs","38dfaffa-8935-11f0-9d99-0224a6c84d84",22,["D",1756952212884],[208],"38d12796-8935-11f0-b12f-0224a6c84d84",".claude","38d1285e-8935-11f0-b12f-0224a6c84d84",21,["D",1756952212789],[214],"pulls","branches","forks","envVars",{"_226":166,"_228":63},"connections",{"_226":63,"_227":63},"count","unfilledCount","emptyCount",{"_44":213,"_32":214,"_30":215,"_154":-5,"_42":43,"_41":216,"_156":59,"_39":398,"_13":63,"_34":200,"_36":-5,"_24":25,"_161":-5,"_255":399,"_27":400},{"_44":390,"_32":391,"_30":392,"_154":215,"_42":43,"_41":393,"_156":59,"_39":394,"_13":63,"_34":20,"_36":395,"_24":25,"_161":-5,"_255":396,"_27":397},{"_44":207,"_32":208,"_30":209,"_154":-5,"_42":43,"_41":210,"_156":59,"_39":387,"_13":63,"_34":200,"_36":-5,"_24":25,"_161":-5,"_255":388,"_27":389},{"_44":383,"_32":318,"_30":312,"_154":209,"_42":43,"_41":135,"_156":59,"_39":384,"_13":63,"_34":200,"_36":-5,"_24":25,"_161":-5,"_255":385,"_27":386},{"_44":375,"_32":376,"_30":377,"_154":312,"_42":43,"_41":378,"_156":59,"_39":379,"_13":63,"_34":20,"_36":380,"_24":25,"_161":-5,"_255":381,"_27":382},{"_44":367,"_32":368,"_30":369,"_154":312,"_42":43,"_41":370,"_156":59,"_39":371,"_13":63,"_34":20,"_36":372,"_24":25,"_161":-5,"_255":373,"_27":374},{"_44":359,"_32":360,"_30":361,"_154":312,"_42":43,"_41":362,"_156":59,"_39":363,"_13":63,"_34":20,"_36":364,"_24":25,"_161":-5,"_255":365,"_27":366},{"_44":351,"_32":352,"_30":353,"_154":312,"_42":43,"_41":354,"_156":59,"_39":355,"_13":63,"_34":20,"_36":356,"_24":25,"_161":-5,"_255":357,"_27":358},{"_44":343,"_32":344,"_30":345,"_154":312,"_42":43,"_41":346,"_156":59,"_39":347,"_13":63,"_34":20,"_36":348,"_24":25,"_161":-5,"_255":349,"_27":350},{"_44":335,"_32":336,"_30":337,"_154":312,"_42":43,"_41":338,"_156":59,"_39":339,"_13":63,"_34":20,"_36":340,"_24":25,"_161":-5,"_255":341,"_27":342},{"_44":327,"_32":328,"_30":329,"_154":312,"_42":43,"_41":330,"_156":59,"_39":331,"_13":63,"_34":20,"_36":332,"_24":25,"_161":-5,"_255":333,"_27":334},{"_44":319,"_32":320,"_30":321,"_154":312,"_42":43,"_41":322,"_156":59,"_39":323,"_13":63,"_34":20,"_36":324,"_24":25,"_161":-5,"_255":325,"_27":326},{"_44":309,"_32":310,"_30":311,"_154":312,"_42":43,"_41":313,"_156":59,"_39":314,"_13":63,"_34":20,"_36":315,"_24":25,"_161":-5,"_255":316,"_27":317},{"_44":202,"_32":203,"_30":204,"_154":-5,"_42":43,"_41":166,"_156":59,"_39":306,"_13":63,"_34":200,"_36":-5,"_24":25,"_161":-5,"_255":307,"_27":308},{"_44":298,"_32":299,"_30":300,"_154":204,"_42":43,"_41":301,"_156":25,"_39":302,"_13":284,"_34":20,"_36":303,"_24":25,"_161":-5,"_255":304,"_27":305},{"_44":196,"_32":197,"_30":198,"_154":-5,"_42":43,"_41":166,"_156":59,"_39":295,"_13":63,"_34":200,"_36":-5,"_24":25,"_161":-5,"_255":296,"_27":297},{"_44":287,"_32":288,"_30":289,"_154":198,"_42":43,"_41":290,"_156":25,"_39":291,"_13":166,"_34":35,"_36":292,"_24":25,"_161":-5,"_255":293,"_27":294},{"_44":282,"_32":33,"_30":31,"_154":198,"_42":43,"_41":14,"_156":25,"_39":283,"_13":284,"_34":35,"_36":38,"_24":25,"_161":-5,"_255":285,"_27":286},{"_44":273,"_32":274,"_30":275,"_154":198,"_42":43,"_41":276,"_156":25,"_39":277,"_13":278,"_34":35,"_36":279,"_24":25,"_161":-5,"_255":280,"_27":281},{"_44":189,"_32":190,"_30":191,"_154":-5,"_42":43,"_41":192,"_156":59,"_39":270,"_13":63,"_34":20,"_36":194,"_24":25,"_161":-5,"_255":271,"_27":272},{"_44":183,"_32":184,"_30":185,"_154":-5,"_42":43,"_41":166,"_156":59,"_39":267,"_13":63,"_34":20,"_36":187,"_24":25,"_161":-5,"_255":268,"_27":269},{"_44":177,"_32":178,"_30":179,"_154":-5,"_42":43,"_41":158,"_156":59,"_39":264,"_13":63,"_34":20,"_36":181,"_24":25,"_161":-5,"_255":265,"_27":266},{"_44":170,"_32":171,"_30":172,"_154":-5,"_42":43,"_41":173,"_156":59,"_39":261,"_13":166,"_34":20,"_36":175,"_24":25,"_161":-5,"_255":262,"_27":263},{"_44":163,"_32":164,"_30":165,"_154":-5,"_42":43,"_41":166,"_156":59,"_39":258,"_13":63,"_34":20,"_36":168,"_24":25,"_161":-5,"_255":259,"_27":260},{"_44":152,"_32":153,"_30":57,"_154":-5,"_42":43,"_41":155,"_156":59,"_39":254,"_13":158,"_34":159,"_36":160,"_24":25,"_161":-5,"_255":256,"_27":257},["D",1756953072886],"children",[],[153],["D",1756839724543],[],[164],["D",1756840062842],[],[171],["D",1756840062786],[],[178],["D",1756839724543],[],[184],["D",1756953072887],[],[190],"3ae73d5e-8935-11f0-a20a-0224a6c84d84","utils.ts","01990bce-c638-714e-97a4-0caf2e9a1c1c",32,["D",1756952216289],9,"const OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n \"openai-beta\": \"realtime-v1\",\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\nexport interface TranscriptionConfig {\n model?: \"gpt-4o-transcribe\" | \"gpt-4o-mini-transcribe\" | \"whisper-1\";\n language?: string;\n prompt?: string;\n enableVAD?: boolean;\n noiseReduction?: \"near_field\" | \"far_field\" | null;\n includeLogprobs?: boolean;\n}\n\nexport function makeTranscriptionSession(config: TranscriptionConfig = {}) {\n const {\n model = \"gpt-4o-transcribe\",\n language = \"en\",\n prompt = \"\",\n enableVAD = true,\n noiseReduction = \"near_field\",\n includeLogprobs = false\n } = config;\n\n // Create transcription session config (based on Justin Uberti's working implementation)\n const session: any = {\n input_audio_transcription: {\n model,\n language\n }\n };\n\n // Add prompt if provided\n if (prompt) {\n session.input_audio_transcription.prompt = prompt;\n }\n\n // Configure VAD\n if (enableVAD) {\n session.turn_detection = {\n type: \"server_vad\"\n };\n } else {\n session.turn_detection = null;\n }\n\n return session;\n}\n\nexport function makeSessionUpdateMessage(config: TranscriptionConfig = {}) {\n const {\n model = \"gpt-4o-transcribe\",\n language = \"en\",\n prompt = \"\",\n enableVAD = true,\n noiseReduction = \"near_field\",\n includeLogprobs = false\n } = config;\n\n // Create session update message to enable transcription\n const update: any = {\n type: \"session.update\",\n input_audio_transcription: {\n model,\n prompt,\n language\n }\n };\n\n // Add include for logprobs if requested\n if (includeLogprobs) {\n update.include = [\"item.input_audio_transcription.logprobs\"];\n }\n\n return update;\n}",[],[197,274],"39860664-8937-11f0-8510-0224a6c84d84",["D",1756953072888],5,[],[197,33],"1a514ed2-8830-11f0-8241-0224a6c84d84","observer.ts","01990bce-c638-714e-97a4-029a75bebd75",6,["D",1756840062935],"import { Hono } from \"npm:hono\";\nimport { makeHeaders } from \"./utils.ts\";\n\nconst observer = new Hono();\n\n// POST /observer/:callId : establish websocket to observe transcription events\nobserver.post(\"/:callId\", async (c) => {\n const callId = c.req.param(\"callId\");\n console.log(`📡 Starting observer for transcription call: ${callId}`);\n \n // Connect to OpenAI WebSocket for this call\n const wsUrl = `wss://api.openai.com/v1/realtime?call_id=${callId}`;\n const ws = new WebSocket(wsUrl, {\n headers: makeHeaders(),\n });\n\n ws.onopen = () => {\n console.log(`✅ Observer connected for call: ${callId}`);\n };\n\n ws.onmessage = (event) => {\n try {\n const data = JSON.parse(event.data);\n \n // Log transcription-specific events\n if (data.type === \"conversation.item.input_audio_transcription.delta\") {\n console.log(`📝 Transcription delta [${data.item_id}]: \"${data.delta}\"`);\n } else if (data.type === \"conversation.item.input_audio_transcription.completed\") {\n console.log(`✅ Transcription completed [${data.item_id}]: \"${data.transcript}\"`);\n } else if (data.type === \"input_audio_buffer.committed\") {\n console.log(`🎤 Audio buffer committed [${data.item_id}]`);\n } else if (data.type === \"error\") {\n console.error(`❌ Error: ${data.error?.message || JSON.stringify(data.error)}`);\n }\n \n // Log other important events at debug level\n if (data.type?.startsWith(\"transcription_session\")) {\n console.log(`🔧 Session event: ${data.type}`);\n }\n } catch (e) {\n console.error(\"Failed to parse WebSocket message:\", e);\n }\n };\n\n ws.onerror = (error) => {\n console.error(`❌ WebSocket error for call ${callId}:`, error);\n };\n\n ws.onclose = () => {\n console.log(`👋 Observer disconnected for call: ${callId}`);\n };\n\n // Return immediately (observer runs in background)\n return c.text(\"Observer started\", 200);\n});\n\nexport default observer;",[],[197,288],["D",1756839724543],[245,246,247],[197],"3ac9b248-8935-11f0-a247-0224a6c84d84","index.html","01990bce-c637-779b-aded-90c65c11e441",30,["D",1756952216094],"\n\n \n \n \n OpenAI Realtime Transcription\n \n \n \n \n\n

Transcription

\n Start speaking to see transcription here...\n

Event Logs

\n\n

\n\n \n \n",[],[203,299],["D",1756839724543],[243],[203],"3af92d0c-8935-11f0-9287-0224a6c84d84","tools-connectors-mcp.md","3af92dfc-8935-11f0-9287-0224a6c84d84","38edc7a2-8935-11f0-98af-0224a6c84d84",33,["D",1756952213122],"Connectors and MCP servers\n\nBeta\n\n==================================\n\nUse connectors and remote MCP servers to give models new capabilities.\n\nIn addition to tools you make available to the model with [function calling](/docs/guides/function-calling), you can give models new capabilities using **connectors** and **remote MCP servers**. These tools give the model the ability to connect to and control external services when needed to respond to a user's prompt. These tool calls can either be allowed automatically, or restricted with explicit approval required by you as the developer.\n\n* **Connectors** are OpenAI-maintained MCP wrappers for popular services like Google Workspace or Dropbox, like the connectors available in [ChatGPT](https://chatgpt.com).\n* **Remote MCP servers** can be any server on the public Internet that implements a remote [Model Context Protocol](https://modelcontextprotocol.io/introduction) (MCP) server.\n\nThis guide will show how to use both remote MCP servers and connectors to give the model access to new capabilities.\n\nQuickstart\n----------\n\nCheck out the examples below to see how remote MCP servers and connectors work through the [Responses API](/docs/api-reference/responses/create). Both connectors and remote MCP servers can be used with the `mcp` built-in tool type.\n\nUsing remote MCP servers\n\nRemote MCP servers require a `server_url`. Depending on the server, you may also need an OAuth `authorization` parameter containing an access token.\n\nUsing a remote MCP server in the Responses API\n\n```bash\ncurl https://api.openai.com/v1/responses \\\n-H \"Content-Type: application/json\" \\\n-H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n-d '{\n \"model\": \"gpt-5\",\n \"tools\": [\n {\n \"type\": \"mcp\",\n \"server_label\": \"dmcp\",\n \"server_description\": \"A Dungeons and Dragons MCP server to assist with dice rolling.\",\n \"server_url\": \"https://dmcp-server.deno.dev/sse\",\n \"require_approval\": \"never\"\n }\n ],\n \"input\": \"Roll 2d4+1\"\n }'\n```\n\n```javascript\nimport OpenAI from \"openai\";\nconst client = new OpenAI();\n\nconst resp = await client.responses.create({\n model: \"gpt-5\",\n tools: [\n {\n type: \"mcp\",\n server_label: \"dmcp\",\n server_description: \"A Dungeons and Dragons MCP server to assist with dice rolling.\",\n server_url: \"https://dmcp-server.deno.dev/sse\",\n require_approval: \"never\",\n },\n ],\n input: \"Roll 2d4+1\",\n});\n\nconsole.log(resp.output_text);\n```\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI()\n\nresp = client.responses.create(\n model=\"gpt-5\",\n tools=[\n {\n \"type\": \"mcp\",\n \"server_label\": \"dmcp\",\n \"server_description\": \"A Dungeons and Dragons MCP server to assist with dice rolling.\",\n \"server_url\": \"https://dmcp-server.deno.dev/sse\",\n \"require_approval\": \"never\",\n },\n ],\n input=\"Roll 2d4+1\",\n)\n\nprint(resp.output_text)\n```\n\nIt is very important that developers trust any remote MCP server they use with the Responses API. A malicious server can exfiltrate sensitive data from anything that enters the model's context. Carefully review the **Risks and Safety** section below before using this tool.\n\nUsing connectors\n\nConnectors require a `connector_id` parameter, and an OAuth access token provided by your application in the `authorization` parameter.\n\nUsing connectors in the Responses API\n\n```bash\ncurl https://api.openai.com/v1/responses \\\n-H \"Content-Type: application/json\" \\\n-H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n-d '{\n \"model\": \"gpt-5\",\n \"tools\": [\n {\n \"type\": \"mcp\",\n \"server_label\": \"Dropbox\",\n \"connector_id\": \"connector_dropbox\",\n \"authorization\": \"\",\n \"require_approval\": \"never\"\n }\n ],\n \"input\": \"Summarize the Q2 earnings report.\"\n }'\n```\n\n```javascript\nimport OpenAI from \"openai\";\nconst client = new OpenAI();\n\nconst resp = await client.responses.create({\n model: \"gpt-5\",\n tools: [\n {\n type: \"mcp\",\n server_label: \"Dropbox\",\n connector_id: \"connector_dropbox\",\n authorization: \"\",\n require_approval: \"never\",\n },\n ],\n input: \"Summarize the Q2 earnings report.\",\n});\n\nconsole.log(resp.output_text);\n```\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI()\n\nresp = client.responses.create(\n model=\"gpt-5\",\n tools=[\n {\n \"type\": \"mcp\",\n \"server_label\": \"Dropbox\",\n \"connector_id\": \"connector_dropbox\",\n \"authorization\": \"\",\n \"require_approval\": \"never\",\n },\n ],\n input=\"Summarize the Q2 earnings report.\",\n)\n\nprint(resp.output_text)\n```\n\nThe API will return new items in the `output` array of the model response. If the model decides to use a Connector or MCP server, it will first make a request to list available tools from the server, which will create a `mcp_list_tools` output item. From the simple remote MCP server example above, it contains only one tool definition:\n\n```json\n{\n \"id\": \"mcpl_68a6102a4968819c8177b05584dd627b0679e572a900e618\",\n \"type\": \"mcp_list_tools\",\n \"server_label\": \"dmcp\",\n \"tools\": [\n {\n \"annotations\": null,\n \"description\": \"Given a string of text describing a dice roll...\",\n \"input_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"diceRollExpression\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\"diceRollExpression\"],\n \"additionalProperties\": false\n },\n \"name\": \"roll\"\n }\n ]\n}\n```\n\nIf the model decides to call one of the available tools from the MCP server, you will also find a `mcp_call` output which will show what the model sent to the MCP tool, and what the MCP tool sent back as output.\n\n```json\n{\n \"id\": \"mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618\",\n \"type\": \"mcp_call\",\n \"approval_request_id\": null,\n \"arguments\": \"{\\\"diceRollExpression\\\":\\\"2d4 + 1\\\"}\",\n \"error\": null,\n \"name\": \"roll\",\n \"output\": \"4\",\n \"server_label\": \"dmcp\"\n}\n```\n\nRead on in the guide below to learn more about how the MCP tool works, how to filter available tools, and how to handle tool call approval requests.\n\nHow it works\n------------\n\nThe MCP tool (for both remote MCP servers and connectors) is available in the [Responses API](/docs/api-reference/responses/create) in most recent models. Check MCP tool compatibility for your model [here](/docs/models). When you're using the MCP tool, you only pay for [tokens](/docs/pricing) used when importing tool definitions or making tool calls. There are no additional fees involved per tool call.\n\nBelow, we'll step through the process the API takes when calling an MCP tool.\n\n### Step 1: Listing available tools\n\nWhen you specify a remote MCP server in the `tools` parameter, the API will attempt to get a list of tools from the server. The Responses API works with remote MCP servers that support either the Streamable HTTP or the HTTP/SSE transport protocols.\n\nIf successful in retrieving the list of tools, a new `mcp_list_tools` output item will appear in the model response output. The `tools` property of this object will show the tools that were successfully imported.\n\n```json\n{\n \"id\": \"mcpl_68a6102a4968819c8177b05584dd627b0679e572a900e618\",\n \"type\": \"mcp_list_tools\",\n \"server_label\": \"dmcp\",\n \"tools\": [\n {\n \"annotations\": null,\n \"description\": \"Given a string of text describing a dice roll...\",\n \"input_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"diceRollExpression\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\"diceRollExpression\"],\n \"additionalProperties\": false\n },\n \"name\": \"roll\"\n }\n ]\n}\n```\n\nAs long as the `mcp_list_tools` item is present in the context of an API request, the API will not fetch a list of tools from the MCP server again at each turn in a [conversation](/docs/guides/conversation-state). We recommend you keep this item in the model's context as part of every conversation or workflow execution to optimize for latency.\n\n#### Filtering tools\n\nSome MCP servers can have dozens of tools, and exposing many tools to the model can result in high cost and latency. If you're only interested in a subset of tools an MCP server exposes, you can use the `allowed_tools` parameter to only import those tools.\n\nConstrain allowed tools\n\n```bash\ncurl https://api.openai.com/v1/responses \\\n-H \"Content-Type: application/json\" \\\n-H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n-d '{\n \"model\": \"gpt-5\",\n \"tools\": [\n {\n \"type\": \"mcp\",\n \"server_label\": \"dmcp\",\n \"server_description\": \"A Dungeons and Dragons MCP server to assist with dice rolling.\",\n \"server_url\": \"https://dmcp-server.deno.dev/sse\",\n \"require_approval\": \"never\",\n \"allowed_tools\": [\"roll\"]\n }\n ],\n \"input\": \"Roll 2d4+1\"\n }'\n```\n\n```javascript\nimport OpenAI from \"openai\";\nconst client = new OpenAI();\n\nconst resp = await client.responses.create({\n model: \"gpt-5\",\n tools: [{\n type: \"mcp\",\n server_label: \"dmcp\",\n server_description: \"A Dungeons and Dragons MCP server to assist with dice rolling.\",\n server_url: \"https://dmcp-server.deno.dev/sse\",\n require_approval: \"never\",\n allowed_tools: [\"roll\"],\n }],\n input: \"Roll 2d4+1\",\n});\n\nconsole.log(resp.output_text);\n```\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI()\n\nresp = client.responses.create(\n model=\"gpt-5\",\n tools=[{\n \"type\": \"mcp\",\n \"server_label\": \"dmcp\",\n \"server_description\": \"A Dungeons and Dragons MCP server to assist with dice rolling.\",\n \"server_url\": \"https://dmcp-server.deno.dev/sse\",\n \"require_approval\": \"never\",\n \"allowed_tools\": [\"roll\"],\n }],\n input=\"Roll 2d4+1\",\n)\n\nprint(resp.output_text)\n```\n\n### Step 2: Calling tools\n\nOnce the model has access to these tool definitions, it may choose to call them depending on what's in the model's context. When the model decides to call an MCP tool, the API will make an request to the remote MCP server to call the tool and put its output into the model's context. This creates an `mcp_call` item which looks like this:\n\n```json\n{\n \"id\": \"mcp_68a6102d8948819c9b1490d36d5ffa4a0679e572a900e618\",\n \"type\": \"mcp_call\",\n \"approval_request_id\": null,\n \"arguments\": \"{\\\"diceRollExpression\\\":\\\"2d4 + 1\\\"}\",\n \"error\": null,\n \"name\": \"roll\",\n \"output\": \"4\",\n \"server_label\": \"dmcp\"\n}\n```\n\nThis item includes both the arguments the model decided to use for this tool call, and the `output` that the remote MCP server returned. All models can choose to make multiple MCP tool calls, so you may see several of these items generated in a single API request.\n\nFailed tool calls will populate the error field of this item with MCP protocol errors, MCP tool execution errors, or general connectivity errors. The MCP errors are documented in the MCP spec [here](https://modelcontextprotocol.io/specification/2025-03-26/server/tools#error-handling).\n\n#### Approvals\n\nBy default, OpenAI will request your approval before any data is shared with a connector or remote MCP server. Approvals help you maintain control and visibility over what data is being sent to an MCP server. We highly recommend that you carefully review (and optionally log) all data being shared with a remote MCP server. A request for an approval to make an MCP tool call creates a `mcp_approval_request` item in the Response's output that looks like this:\n\n```json\n{\n \"id\": \"mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339\",\n \"type\": \"mcp_approval_request\",\n \"arguments\": \"{\\\"diceRollExpression\\\":\\\"2d4 + 1\\\"}\",\n \"name\": \"roll\",\n \"server_label\": \"dmcp\"\n}\n```\n\nYou can then respond to this by creating a new Response object and appending an `mcp_approval_response` item to it.\n\nApproving the use of tools in an API request\n\n```bash\ncurl https://api.openai.com/v1/responses \\\n-H \"Content-Type: application/json\" \\\n-H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n-d '{\n \"model\": \"gpt-5\",\n \"tools\": [\n {\n \"type\": \"mcp\",\n \"server_label\": \"dmcp\",\n \"server_description\": \"A Dungeons and Dragons MCP server to assist with dice rolling.\",\n \"server_url\": \"https://dmcp-server.deno.dev/sse\",\n \"require_approval\": \"always\",\n }\n ],\n \"previous_response_id\": \"resp_682d498bdefc81918b4a6aa477bfafd904ad1e533afccbfa\",\n \"input\": [{\n \"type\": \"mcp_approval_response\",\n \"approve\": true,\n \"approval_request_id\": \"mcpr_682d498e3bd4819196a0ce1664f8e77b04ad1e533afccbfa\"\n }]\n }'\n```\n\n```javascript\nimport OpenAI from \"openai\";\nconst client = new OpenAI();\n\nconst resp = await client.responses.create({\n model: \"gpt-5\",\n tools: [{\n type: \"mcp\",\n server_label: \"dmcp\",\n server_description: \"A Dungeons and Dragons MCP server to assist with dice rolling.\",\n server_url: \"https://dmcp-server.deno.dev/sse\",\n require_approval: \"always\",\n }],\n previous_response_id: \"resp_682d498bdefc81918b4a6aa477bfafd904ad1e533afccbfa\",\n input: [{\n type: \"mcp_approval_response\",\n approve: true,\n approval_request_id: \"mcpr_682d498e3bd4819196a0ce1664f8e77b04ad1e533afccbfa\"\n }],\n});\n\nconsole.log(resp.output_text);\n```\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI()\n\nresp = client.responses.create(\n model=\"gpt-5\",\n tools=[{\n \"type\": \"mcp\",\n \"server_label\": \"dmcp\",\n \"server_description\": \"A Dungeons and Dragons MCP server to assist with dice rolling.\",\n \"server_url\": \"https://dmcp-server.deno.dev/sse\",\n \"require_approval\": \"always\",\n }],\n previous_response_id=\"resp_682d498bdefc81918b4a6aa477bfafd904ad1e533afccbfa\",\n input=[{\n \"type\": \"mcp_approval_response\",\n \"approve\": True,\n \"approval_request_id\": \"mcpr_682d498e3bd4819196a0ce1664f8e77b04ad1e533afccbfa\"\n }],\n)\n\nprint(resp.output_text)\n```\n\nHere we're using the `previous_response_id` parameter to chain this new Response, with the previous Response that generated the approval request. But you can also pass back the [outputs from one response, as inputs into another](/docs/guides/conversation-state#manually-manage-conversation-state) for maximum control over what enter's the model's context.\n\nIf and when you feel comfortable trusting a remote MCP server, you can choose to skip the approvals for reduced latency. To do this, you can set the `require_approval` parameter of the MCP tool to an object listing just the tools you'd like to skip approvals for like shown below, or set it to the value `'never'` to skip approvals for all tools in that remote MCP server.\n\nNever require approval for some tools\n\n```bash\ncurl https://api.openai.com/v1/responses \\\n-H \"Content-Type: application/json\" \\\n-H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n-d '{\n \"model\": \"gpt-5\",\n \"tools\": [\n {\n \"type\": \"mcp\",\n \"server_label\": \"deepwiki\",\n \"server_url\": \"https://mcp.deepwiki.com/mcp\",\n \"require_approval\": {\n \"never\": {\n \"tool_names\": [\"ask_question\", \"read_wiki_structure\"]\n }\n }\n }\n ],\n \"input\": \"What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?\"\n }'\n```\n\n```javascript\nimport OpenAI from \"openai\";\nconst client = new OpenAI();\n\nconst resp = await client.responses.create({\n model: \"gpt-5\",\n tools: [\n {\n type: \"mcp\",\n server_label: \"deepwiki\",\n server_url: \"https://mcp.deepwiki.com/mcp\",\n require_approval: {\n never: {\n tool_names: [\"ask_question\", \"read_wiki_structure\"]\n }\n }\n },\n ],\n input: \"What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?\",\n});\n\nconsole.log(resp.output_text);\n```\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI()\n\nresp = client.responses.create(\n model=\"gpt-5\",\n tools=[\n {\n \"type\": \"mcp\",\n \"server_label\": \"deepwiki\",\n \"server_url\": \"https://mcp.deepwiki.com/mcp\",\n \"require_approval\": {\n \"never\": {\n \"tool_names\": [\"ask_question\", \"read_wiki_structure\"]\n }\n }\n },\n ],\n input=\"What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?\",\n)\n\nprint(resp.output_text)\n```\n\nAuthentication\n--------------\n\nUnlike the [example MCP server we used above](https://dash.deno.com/playground/dmcp-server), most other MCP servers require authentication. The most common scheme is an OAuth access token. Provide this token using the `authorization` field of the MCP tool:\n\nUse Stripe MCP tool\n\n```bash\ncurl https://api.openai.com/v1/responses \\\n-H \"Content-Type: application/json\" \\\n-H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n-d '{\n \"model\": \"gpt-5\",\n \"input\": \"Create a payment link for $20\",\n \"tools\": [\n {\n \"type\": \"mcp\",\n \"server_label\": \"stripe\",\n \"server_url\": \"https://mcp.stripe.com\",\n \"authorization\": \"$STRIPE_OAUTH_ACCESS_TOKEN\"\n }\n ]\n }'\n```\n\n```javascript\nimport OpenAI from \"openai\";\nconst client = new OpenAI();\n\nconst resp = await client.responses.create({\n model: \"gpt-5\",\n input: \"Create a payment link for $20\",\n tools: [\n {\n type: \"mcp\",\n server_label: \"stripe\",\n server_url: \"https://mcp.stripe.com\",\n authorization: \"$STRIPE_OAUTH_ACCESS_TOKEN\"\n }\n ]\n});\n\nconsole.log(resp.output_text);\n```\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI()\n\nresp = client.responses.create(\n model=\"gpt-5\",\n input=\"Create a payment link for $20\",\n tools=[\n {\n \"type\": \"mcp\",\n \"server_label\": \"stripe\",\n \"server_url\": \"https://mcp.stripe.com\",\n \"authorization\": \"$STRIPE_OAUTH_ACCESS_TOKEN\"\n }\n ]\n)\n\nprint(resp.output_text)\n```\n\nTo prevent the leakage of sensitive tokens, the Responses API does not store the value you provide in the `authorization` field. This value will also not be visible in the Response object created. Additionally, because some remote MCP servers generate authenticated URLs, we also discard the _path_ portion of the `server_url` in our responses (i.e. `example.com/mcp` becomes `example.com`). Because of this, you must send the full path of the MCP `server_url` and the `authorization` value in every Responses API creation request you make.\n\nConnectors\n----------\n\nThe Responses API has built-in support for a limited set of connectors to third-party services. These connectors let you pull in context from popular applications, like Dropbox and Gmail, to allow the model to interact with popular services.\n\nConnectors can be used in the same way as remote MCP servers. Both let an OpenAI model access additional third-party tools in an API request. However, instead of passing a `server_url` as you would to call a remote MCP server, you pass a `connector_id` which uniquely identifies a connector available in the API.\n\n### Available connectors\n\n* Dropbox: `connector_dropbox`\n* Gmail: `connector_gmail`\n* Google Calendar: `connector_googlecalendar`\n* Google Drive: `connector_googledrive`\n* Microsoft Teams: `connector_microsoftteams`\n* Outlook Calendar: `connector_outlookcalendar`\n* Outlook Email: `connector_outlookemail`\n* SharePoint: `connector_sharepoint`\n\nWe prioritized services that don't have official remote MCP servers. GitHub, for instance, has an official MCP server you can connect to by passing `https://api.githubcopilot.com/mcp/` to the `server_url` field in the MCP tool.\n\n### Authorizing a connector\n\nIn the `authorization` field, pass in an OAuth access token. OAuth client registration and authorization must be handled separately by your application.\n\nFor testing purposes, you can use Google's [OAuth 2.0 Playground](https://developers.google.com/oauthplayground/) to generate temporary access tokens that you can use in an API request.\n\nTo use the playground to test the connectors API functionality, start by entering:\n\n```text\nhttps://www.googleapis.com/auth/calendar.events\n```\n\nThis authorization scope will enable the API to read Google Calendar events. In the UI under \"Step 1: Select and authorize APIs\".\n\nAfter authorizing the application with your Google account, you will come to \"Step 2: Exchange authorization code for tokens\". This will generate an access token you can use in an API request using the Google Calendar connector:\n\nUse the Google Calendar connector\n\n```bash\ncurl https://api.openai.com/v1/responses \\\n -H \"Content-Type: application/json\" \\\n -H \"Authorization: Bearer $OPENAI_API_KEY\" \\\n -d '{\n \"model\": \"gpt-5\",\n \"tools\": [\n {\n \"type\": \"mcp\",\n \"server_label\": \"google_calendar\",\n \"connector_id\": \"connector_googlecalendar\",\n \"authorization\": \"ya29.A0AS3H6...\",\n \"require_approval\": \"never\"\n }\n ],\n \"input\": \"What is on my Google Calendar for today?\"\n }'\n```\n\n```javascript\nimport OpenAI from \"openai\";\nconst client = new OpenAI();\n\nconst resp = await client.responses.create({\n model: \"gpt-5\",\n tools: [\n {\n type: \"mcp\",\n server_label: \"google_calendar\",\n connector_id: \"connector_googlecalendar\",\n authorization: \"ya29.A0AS3H6...\",\n require_approval: \"never\",\n },\n ],\n input: \"What's on my Google Calendar for today?\",\n});\n\nconsole.log(resp.output_text);\n```\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI()\n\nresp = client.responses.create(\n model=\"gpt-5\",\n tools=[\n {\n \"type\": \"mcp\",\n \"server_label\": \"google_calendar\",\n \"connector_id\": \"connector_googlecalendar\",\n \"authorization\": \"ya29.A0AS3H6...\",\n \"require_approval\": \"never\",\n },\n ],\n input=\"What's on my Google Calendar for today?\",\n)\n\nprint(resp.output_text)\n```\n\nAn MCP tool call from a Connector will look the same as an MCP tool call from a remote MCP server, using the `mcp_call` output item type. In this case, both the arguments to and the response from the Connector are JSON strings:\n\n```json\n{\n \"id\": \"mcp_68a62ae1c93c81a2b98c29340aa3ed8800e9b63986850588\",\n \"type\": \"mcp_call\",\n \"approval_request_id\": null,\n \"arguments\": \"{\\\"time_min\\\":\\\"2025-08-20T00:00:00\\\",\\\"time_max\\\":\\\"2025-08-21T00:00:00\\\",\\\"timezone_str\\\":null,\\\"max_results\\\":50,\\\"query\\\":null,\\\"calendar_id\\\":null,\\\"next_page_token\\\":null}\",\n \"error\": null,\n \"name\": \"search_events\",\n \"output\": \"{\\\"events\\\": [{\\\"id\\\": \\\"2n8ni54ani58pc3ii6soelupcs_20250820\\\", \\\"summary\\\": \\\"Home\\\", \\\"location\\\": null, \\\"start\\\": \\\"2025-08-20T00:00:00\\\", \\\"end\\\": \\\"2025-08-21T00:00:00\\\", \\\"url\\\": \\\"https://www.google.com/calendar/event?eid=Mm44bmk1NGFuaTU4cGMzaWk2c29lbHVwY3NfMjAyNTA4MjAga3doaW5uZXJ5QG9wZW5haS5jb20&ctz=America/Los_Angeles\\\", \\\"description\\\": \\\"\\\\n\\\\n\\\", \\\"transparency\\\": \\\"transparent\\\", \\\"display_url\\\": \\\"https://www.google.com/calendar/event?eid=Mm44bmk1NGFuaTU4cGMzaWk2c29lbHVwY3NfMjAyNTA4MjAga3doaW5uZXJ5QG9wZW5haS5jb20&ctz=America/Los_Angeles\\\", \\\"display_title\\\": \\\"Home\\\"}], \\\"next_page_token\\\": null}\",\n \"server_label\": \"Google_Calendar\"\n}\n```\n\n### Available tools in each connector\n\nThe available tools depend on which scopes your OAuth token has available to it. Expand the tables below to see what tools you can use when connecting to each application.\n\nDropbox\n\n||\n|search|Search Dropbox for files that match a query|files.metadata.read, account_info.read|\n|fetch|Fetch a file by path with optional raw download|files.content.read|\n|search_files|Search Dropbox files and return results|files.metadata.read, account_info.read|\n|fetch_file|Retrieve a file's text or raw content|files.content.read, account_info.read|\n|list_recent_files|Return the most recently modified files accessible to the user|files.metadata.read, account_info.read|\n|get_profile|Retrieve the Dropbox profile of the current user|account_info.read|\n\nGmail\n\n||\n|get_profile|Return the current Gmail user's profile|userinfo.email, userinfo.profile|\n|search_emails|Search Gmail for emails matching a query or label|gmail.modify|\n|search_email_ids|Retrieve Gmail message IDs matching a search|gmail.modify|\n|get_recent_emails|Return the most recently received Gmail messages|gmail.modify|\n|read_email|Fetch a single Gmail message including its body|gmail.modify|\n|batch_read_email|Read multiple Gmail messages in one call|gmail.modify|\n\nGoogle Calendar\n\n||\n|get_profile|Return the current Calendar user's profile|userinfo.email, userinfo.profile|\n|search|Search Calendar events within an optional time window|calendar.events|\n|fetch|Get details for a single Calendar event|calendar.events|\n|search_events|Look up Calendar events using filters|calendar.events|\n|read_event|Read a Google Calendar event by ID|calendar.events|\n\nGoogle Drive\n\n||\n|get_profile|Return the current Drive user's profile|userinfo.email, userinfo.profile|\n|list_drives|List shared drives accessible to the user|drive.readonly|\n|search|Search Drive files using a query|drive.readonly|\n|recent_documents|Return the most recently modified documents|drive.readonly|\n|fetch|Download the content of a Drive file|drive.readonly|\n\nMicrosoft Teams\n\n||\n|search|Search Microsoft Teams chats and channel messages|Chat.Read, ChannelMessage.Read.All|\n|fetch|Fetch a Teams message by path|Chat.Read, ChannelMessage.Read.All|\n|get_chat_members|List the members of a Teams chat|Chat.Read|\n|get_profile|Return the authenticated Teams user's profile|User.Read|\n\nOutlook Calendar\n\n||\n|search_events|Search Outlook Calendar events with date filters|Calendars.Read|\n|fetch_event|Retrieve details for a single event|Calendars.Read|\n|fetch_events_batch|Retrieve multiple events in one call|Calendars.Read|\n|list_events|List calendar events within a date range|Calendars.Read|\n|get_profile|Retrieve the current user's profile|User.Read|\n\nOutlook Email\n\n||\n|get_profile|Return profile info for the Outlook account|User.Read|\n|list_messages|Retrieve Outlook emails from a folder|Mail.Read|\n|search_messages|Search Outlook emails with optional filters|Mail.Read|\n|get_recent_emails|Return the most recently received emails|Mail.Read|\n|fetch_message|Fetch a single email by ID|Mail.Read|\n|fetch_messages_batch|Retrieve multiple emails in one request|Mail.Read|\n\nSharepoint\n\n||\n|get_site|Resolve a SharePoint site by hostname and path|Sites.Read.All|\n|search|Search SharePoint/OneDrive documents by keyword|Sites.Read.All, Files.Read.All|\n|list_recent_documents|Return recently accessed documents|Files.Read.All|\n|fetch|Fetch content from a Graph file download URL|Files.Read.All|\n|get_profile|Retrieve the current user's profile|User.Read|\n\nRisks and safety\n----------------\n\nThe MCP tool permits you to connect OpenAI models to external services. This is a powerful feature that comes with some risks.\n\nFor connectors, there is a risk of potentially sending sensitive data to OpenAI, or allowing models read access to potentially sensitive data in those services.\n\nRemote MCP servers carry those same risks, but also have not been verified by OpenAI. These servers can allow models to access, send, and receive data, and take action in these services. All MCP servers are third-party services that are subject to their own terms and conditions.\n\nIf you come across a malicious MCP server, please report it to `security@openai.com`.\n\nBelow are some best practices to consider when integrating connectors and remote MCP servers.\n\n#### Prompt injection\n\n[Prompt injection](https://chatgpt.com/?prompt=what%20is%20prompt%20injection?) is an important security consideration in any LLM application, and is especially true when you give the model access to MCP servers and connectors which can access sensitive data or take action. Use these tools with appropriate caution and mitigations if the prompt for the model contains user-provided content.\n\n#### Always require approval for sensitive actions\n\nUse the available configurations of the `require_approval` and `allowed_tools` parameters to ensure that any sensitive actions require an approval flow.\n\n#### URLs within MCP tool calls and outputs\n\nIt can be dangerous to request URLs or embed image URLs provided by tool call outputs either from connectors or remote MCP servers. Ensure that you trust the domains and services providing those URLs before embedding or otherwise using them in your application code.\n\n#### Connecting to trusted servers\n\nPick official servers hosted by the service providers themselves (e.g. we recommend connecting to the Stripe server hosted by Stripe themselves on mcp.stripe.com, instead of a Stripe MCP server hosted by a third party). Because there aren't too many official remote MCP servers today, you may be tempted to use a MCP server hosted by an organization that doesn't operate that server and simply proxies request to that service via your API. If you must do this, be extra careful in doing your due diligence on these \"aggregators\", and carefully review how they use your data.\n\n#### Log and review data being shared with third party MCP servers.\n\nBecause MCP servers define their own tool definitions, they may request for data that you may not always be comfortable sharing with the host of that MCP server. Because of this, the MCP tool in the Responses API defaults to requiring approvals of each MCP tool call being made. When developing your application, review the type of data being shared with these MCP servers carefully and robustly. Once you gain confidence in your trust of this MCP server, you can skip these approvals for more performant execution.\n\nWe also recommend logging any data sent to MCP servers. If you're using the Responses API with `store=true`, these data are already logged via the API for 30 days unless Zero Data Retention is enabled for your organization. You may also want to log these data in your own systems and perform periodic reviews on this to ensure data is being shared per your expectations.\n\nMalicious MCP servers may include hidden instructions (prompt injections) designed to make OpenAI models behave unexpectedly. While OpenAI has implemented built-in safeguards to help detect and block these threats, it's essential to carefully review inputs and outputs, and ensure connections are established only with trusted servers.\n\nMCP servers may update tool behavior unexpectedly, potentially leading to unintended or malicious behavior.\n\n#### Implications on Zero Data Retention and Data Residency\n\nThe MCP tool is compatible with Zero Data Retention and Data Residency, but it's important to note that MCP servers are third-party services, and data sent to an MCP server is subject to their data retention and data residency policies.\n\nIn other words, if you're an organization with Data Residency in Europe, OpenAI will limit inference and storage of Customer Content to take place in Europe up until the point communication or data is sent to the MCP server. It is your responsibility to ensure that the MCP server also adheres to any Zero Data Retention or Data Residency requirements you may have. Learn more about Zero Data Retention and Data Residency [here](/docs/guides/your-data).\n\nUsage notes\n-----------\n\n||\n|ResponsesChat CompletionsAssistants|Tier 1200 RPMTier 2 and 31000 RPMTier 4 and 52000 RPM|PricingZDR and data residency|\n\nWas this page useful?",[],[208,318,310],"guides","39f932d0-8935-11f0-98af-0224a6c84d84","realtime.md","39f93366-8935-11f0-98af-0224a6c84d84",28,["D",1756952214731],"Realtime API\n============\n\nBuild low-latency, multimodal LLM applications with the Realtime API.\n\nThe OpenAI Realtime API enables low-latency communication with [models](/docs/models) that natively support speech-to-speech interactions as well as multimodal inputs (audio, images, and text) and outputs (audio and text). These APIs can also be used for [realtime audio transcription](/docs/guides/realtime-transcription).\n\nVoice agents\n------------\n\nOne of the most common use cases for the Realtime API is building voice agents for speech-to-speech model interactions in the browser. Our recommended starting point for these types of applications is the [Agents SDK for TypeScript](https://openai.github.io/openai-agents-js/guides/voice-agents/), which uses a [WebRTC connection](/docs/guides/realtime-webrtc) to the Realtime model in the browser, and [WebSocket](/docs/guides/realtime-websocket) when used on the server.\n\n```js\nimport { RealtimeAgent, RealtimeSession } from \"@openai/agents/realtime\";\n\nconst agent = new RealtimeAgent({\n name: \"Assistant\",\n instructions: \"You are a helpful assistant.\",\n});\n\nconst session = new RealtimeSession(agent);\n\n// Automatically connects your microphone and audio output\nawait session.connect({\n apiKey: \"\",\n});\n```\n\n[\n\nVoice Agent Quickstart\n\nFollow the voice agent quickstart to build Realtime agents in the browser.\n\n](https://openai.github.io/openai-agents-js/guides/voice-agents/quickstart/)\n\nTo use the Realtime API directly outside the context of voice agents, check out the other connection options below.\n\nConnection methods\n------------------\n\nWhile building [voice agents with the Agents SDK](https://openai.github.io/openai-agents-js/guides/voice-agents/) is the fastest path to one specific type of application, the Realtime API provides an entire suite of flexible tools for a variety of use cases.\n\nThere are three primary supported interfaces for the Realtime API:\n\n[\n\nWebRTC connection\n\nIdeal for browser and client-side interactions with a Realtime model.\n\n](/docs/guides/realtime-webrtc)[\n\nWebSocket connection\n\nIdeal for middle tier server-side applications with consistent low-latency network connections.\n\n](/docs/guides/realtime-websocket)[\n\nSIP connection\n\nIdeal for VoIP telephony connections.\n\n](/docs/guides/realtime-sip)\n\nDepending on how you'd like to connect to a Realtime model, check out one of the connection guides above to get started. You'll learn how to initialize a Realtime session, and how to interact with a Realtime model using client and server events.\n\nAPI Usage\n---------\n\nOnce connected to a realtime model using one of the methods above, learn how to interact with the model in these usage guides.\n\n* **[Prompting guide](/docs/guides/realtime-models-prompting):** learn tips and best practices for prompting and steering Realtime models.\n* **[Inputs and outputs](/docs/guides/realtime-inputs-outputs):** Learn how to pass audio, text, and image inputs to the model, and how to receive audio and text back.\n* **[Managing conversations](/docs/guides/realtime-conversations):** Learn about the Realtime session lifecycle and the key events that happen during a conversation.\n* **[Webhooks and server-side controls](/docs/guides/realtime-server-controls):** Learn how you can control a Realtime session on the server to call tools and implement guardrails.\n* **[Function calling](/docs/guides/realtime-function-calling):** Give the realtime model access to call custom code in your own application when appropriate.\n* **[MCP servers](/docs/guides/realtime-mcp):** Give realtime models access to new capabilities via Model Context Protocol (MCP) servers.\n* **[Realtime audio transcription](/docs/guides/realtime-transcription):** Transcribe audio streams in real time over a WebSocket connection.\n\nWas this page useful?",[],[208,318,320],"39b0a236-8935-11f0-8c91-0224a6c84d84","realtime-websocket.md","39b0a2ea-8935-11f0-8c91-0224a6c84d84",27,["D",1756952213879],"Realtime API with WebSocket\n===========================\n\nConnect to the Realtime API using WebSockets on a server.\n\n[WebSockets](https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API) are a broadly supported API for realtime data transfer, and a great choice for connecting to the OpenAI Realtime API in server-to-server applications. For browser and mobile clients, we recommend connecting via [WebRTC](/docs/guides/realtime-webrtc).\n\nIn a server-to-server integration with Realtime, your backend system will connect via WebSocket directly to the Realtime API. You can use a [standard API key](/settings/organization/api-keys) to authenticate this connection, since the token will only be available on your secure backend server.\n\n![connect directly to realtime API](https://openaidevs.retool.com/api/file/464d4334-c467-4862-901b-d0c6847f003a)\n\nConnect via WebSocket\n---------------------\n\nBelow are several examples of connecting via WebSocket to the Realtime API. In addition to using the WebSocket URL below, you will also need to pass an authentication header using your OpenAI API key.\n\nIt is possible to use WebSocket in browsers with an ephemeral API token as shown in the [WebRTC connection guide](/docs/guides/realtime-webrtc), but if you are connecting from a client like a browser or mobile app, WebRTC will be a more robust solution in most cases.\n\nws module (Node.js)\n\nConnect using the ws module (Node.js)\n\n```javascript\nimport WebSocket from \"ws\";\n\nconst url = \"wss://api.openai.com/v1/realtime?model=gpt-realtime\";\nconst ws = new WebSocket(url, {\n headers: {\n Authorization: \"Bearer \" + process.env.OPENAI_API_KEY,\n },\n});\n\nws.on(\"open\", function open() {\n console.log(\"Connected to server.\");\n});\n\nws.on(\"message\", function incoming(message) {\n console.log(JSON.parse(message.toString()));\n});\n```\n\nwebsocket-client (Python)\n\nConnect with websocket-client (Python)\n\n```python\n# example requires websocket-client library:\n# pip install websocket-client\n\nimport os\nimport json\nimport websocket\n\nOPENAI_API_KEY = os.environ.get(\"OPENAI_API_KEY\")\n\nurl = \"wss://api.openai.com/v1/realtime?model=gpt-realtime\"\nheaders = [\"Authorization: Bearer \" + OPENAI_API_KEY]\n\ndef on_open(ws):\n print(\"Connected to server.\")\n\ndef on_message(ws, message):\n data = json.loads(message)\n print(\"Received event:\", json.dumps(data, indent=2))\n\nws = websocket.WebSocketApp(\n url,\n header=headers,\n on_open=on_open,\n on_message=on_message,\n)\n\nws.run_forever()\n```\n\nWebSocket (browsers)\n\nConnect with standard WebSocket (browsers)\n\n```javascript\n/*\nNote that in client-side environments like web browsers, we recommend\nusing WebRTC instead. It is possible, however, to use the standard \nWebSocket interface in browser-like environments like Deno and \nCloudflare Workers.\n*/\n\nconst ws = new WebSocket(\n \"wss://api.openai.com/v1/realtime?model=gpt-realtime\",\n [\n \"realtime\",\n // Auth\n \"openai-insecure-api-key.\" + OPENAI_API_KEY, \n // Optional\n \"openai-organization.\" + OPENAI_ORG_ID,\n \"openai-project.\" + OPENAI_PROJECT_ID,\n ]\n);\n\nws.on(\"open\", function open() {\n console.log(\"Connected to server.\");\n});\n\nws.on(\"message\", function incoming(message) {\n console.log(message.data);\n});\n```\n\nSending and receiving events\n----------------------------\n\nRealtime API sessions are managed using a combination of [client-sent events](/docs/api-reference/realtime_client_events/session) emitted by you as the developer, and [server-sent events](/docs/api-reference/realtime_server_events/error) created by the Realtime API to indicate session lifecycle events.\n\nOver a WebSocket, you will both send and receive JSON-serialized events as strings of text, as in this Node.js example below (the same principles apply for other WebSocket libraries):\n\n```javascript\nimport WebSocket from \"ws\";\n\nconst url = \"wss://api.openai.com/v1/realtime?model=gpt-realtime\";\nconst ws = new WebSocket(url, {\n headers: {\n Authorization: \"Bearer \" + process.env.OPENAI_API_KEY,\n },\n});\n\nws.on(\"open\", function open() {\n console.log(\"Connected to server.\");\n\n // Send client events over the WebSocket once connected\n ws.send(\n JSON.stringify({\n type: \"session.update\",\n session: {\n type: \"realtime\",\n instructions: \"Be extra nice today!\",\n },\n })\n );\n});\n\n// Listen for and parse server events\nws.on(\"message\", function incoming(message) {\n console.log(JSON.parse(message.toString()));\n});\n```\n\nThe WebSocket interface is perhaps the lowest-level interface available to interact with a Realtime model, where you will be responsible for both sending and processing Base64-encoded audio chunks over the socket connection.\n\nTo learn how to send and receive audio over Websockets, refer to the [Realtime conversations guide](/docs/guides/realtime-conversations#handling-audio-with-websockets).\n\nWas this page useful?",[],[208,318,328],"3907d908-8935-11f0-9ad7-0224a6c84d84","realtime-webrtc.md","3907d9a8-8935-11f0-9ad7-0224a6c84d84",25,["D",1756952213152],"Realtime API with WebRTC\n========================\n\nConnect to the Realtime API using WebRTC.\n\n[WebRTC](https://webrtc.org/) is a powerful set of standard interfaces for building real-time applications. The OpenAI Realtime API supports connecting to realtime models through a WebRTC peer connection.\n\nFor browser-based speech-to-speech voice applications, we recommend starting with the [Agents SDK for TypeScript](https://openai.github.io/openai-agents-js/guides/voice-agents/quickstart/), which provides higher-level helpers and APIs for managing Realtime sessions. The WebRTC interface is powerful and flexible, but lower level than the Agents SDK.\n\nWhen connecting to a Realtime model from the client (like a web browser or mobile device), we recommend using WebRTC rather than WebSocket for more consistent performance.\n\nOverview\n--------\n\nConnecting to the Realtime API via WebRTC requires a backend server that can generate an ephemeral API key that can be safely used in a client-side environment (like a browser).\n\nThe process for initializing a WebRTC connection using an ephemeral API key is as follows:\n\n1. A browser makes a request to a developer-controlled server to mint an ephemeral API key.\n2. The developer's server uses a [standard API key](/settings/organization/api-keys) to request an ephemeral key from the [OpenAI REST API](/docs/api-reference/realtime-sessions), and returns that new key to the browser.\n3. The browser uses the ephemeral key to authenticate a session directly with the OpenAI Realtime API as a [WebRTC peer connection](https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection).\n\n![connect to realtime via WebRTC](https://openaidevs.retool.com/api/file/55b47800-9aaf-48b9-90d5-793ab227ddd3)\n\nInitializing a peer connection\n------------------------------\n\nIn the browser, you can use standard WebRTC APIs to initialize a peer connection object. The code below requires a server-side endpoint that generates an ephemeral API token (which we'll see code for in a moment).\n\n```javascript\n// Get a session token for OpenAI Realtime API\nconst tokenResponse = await fetch(\"/token\");\nconst data = await tokenResponse.json();\nconst EPHEMERAL_KEY = data.value;\n\n// Create a peer connection\nconst pc = new RTCPeerConnection();\n\n// Set up to play remote audio from the model\naudioElement.current = document.createElement(\"audio\");\naudioElement.current.autoplay = true;\npc.ontrack = (e) => (audioElement.current.srcObject = e.streams[0]);\n\n// Add local audio track for microphone input in the browser\nconst ms = await navigator.mediaDevices.getUserMedia({\n audio: true,\n});\npc.addTrack(ms.getTracks()[0]);\n\n// Set up data channel for sending and receiving events\nconst dc = pc.createDataChannel(\"oai-events\");\n\n// Start the session using the Session Description Protocol (SDP)\nconst offer = await pc.createOffer();\nawait pc.setLocalDescription(offer);\n\nconst baseUrl = \"https://api.openai.com/v1/realtime/calls\";\nconst model = \"gpt-realtime\";\nconst sdpResponse = await fetch(`${baseUrl}?model=${model}`, {\n method: \"POST\",\n body: offer.sdp,\n headers: {\n Authorization: `Bearer ${EPHEMERAL_KEY}`,\n \"Content-Type\": \"application/sdp\",\n },\n});\n\nconst answer = {\n type: \"answer\",\n sdp: await sdpResponse.text(),\n};\nawait pc.setRemoteDescription(answer);\n```\n\nThe WebRTC APIs provide rich controls for handling media streams and input devices. For more guidance on building user interfaces on top of WebRTC, [refer to the docs on MDN](https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API).\n\nSee the next section for details on implementing the server-side component that powers the `/token` endpoint used in the code above.\n\nCreating an ephemeral token\n---------------------------\n\nTo create an ephemeral token to use on the client-side, you will need to build a small server-side application (or integrate with an existing one) to make an [OpenAI REST API](/docs/api-reference/realtime-sessions) request for an ephemeral key. You will use a [standard API key](/settings/organization/api-keys) to authenticate this request on your backend server.\n\nBelow is an example of a simple Node.js [express](https://expressjs.com/) server which mints an ephemeral API key using the REST API:\n\n```javascript\nimport express from \"express\";\n\nconst app = express();\n\nconst sessionConfig = JSON.stringify({\n session: {\n type: \"realtime\",\n model: \"gpt-realtime\",\n audio: {\n output: {\n voice: \"marin\",\n },\n },\n },\n});\n\n// An endpoint which would work with the client code above - it returns\n// the contents of a REST API request to this protected endpoint\napp.get(\"/token\", async (req, res) => {\n try {\n const response = await fetch(\n \"https://api.openai.com/v1/realtime/client_secrets\",\n {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${apiKey}`,\n \"Content-Type\": \"application/json\",\n },\n body: sessionConfig,\n }\n );\n\n const data = await response.json();\n res.json(data);\n } catch (error) {\n console.error(\"Token generation error:\", error);\n res.status(500).json({ error: \"Failed to generate token\" });\n }\n});\n\napp.listen(3000);\n```\n\nYou can create a server endpoint like this one on any platform that can send and receive HTTP requests. Just ensure that **you only use standard OpenAI API keys on the server, not in the browser.**\n\nSending and receiving events\n----------------------------\n\nRealtime API sessions are managed using a combination of [client-sent events](/docs/api-reference/realtime_client_events/session) emitted by you as the developer, and [server-sent events](/docs/api-reference/realtime_server_events/error) created by the Realtime API to indicate session lifecycle events.\n\nWhen connecting to a Realtime model via WebRTC, you will not have to handle audio events to hear model responses in the same granular way you must with [WebSockets](/docs/guides/realtime-websocket). The WebRTC peer connection object, if configured as above, will do much of that work for you.\n\nTo send and receive other client and server events, you will use the WebRTC peer connection's [data channel](https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Using_data_channels).\n\n```javascript\n// This is the data channel set up in the browser code above...\nconst dc = pc.createDataChannel(\"oai-events\");\n\n// Listen for server events\ndc.addEventListener(\"message\", (e) => {\n const event = JSON.parse(e.data);\n console.log(event);\n});\n\n// Send client events\nconst event = {\n type: \"conversation.item.create\",\n item: {\n type: \"message\",\n role: \"user\",\n content: [\n {\n type: \"input_text\",\n text: \"hello there!\",\n },\n ],\n },\n};\ndc.send(JSON.stringify(event));\n```\n\nTo learn more about managing Realtime conversations, refer to the [Realtime conversations guide](/docs/guides/realtime-conversations).\n\n[\n\nRealtime Console\n\nCheck out the WebRTC Realtime API in this light weight example app.\n\n/Users/emcho/devel/openai-realtime-console\n\nWas this page useful?",[],[208,318,336],"397729b6-8935-11f0-9096-0224a6c84d84","realtime-transcription.md","39772a88-8935-11f0-9096-0224a6c84d84",26,["D",1756952213233],"Realtime transcription\n======================\n\nLearn how to transcribe audio in real-time with the Realtime API.\n\nYou can use the Realtime API for transcription-only use cases, either with input from a microphone or from a file. For example, you can use it to generate subtitles or transcripts in real-time. With the transcription-only mode, the model will not generate responses.\n\nIf you want the model to produce responses, you can use the Realtime API in [speech-to-speech conversation mode](/docs/guides/realtime-conversations).\n\nRealtime transcription sessions\n-------------------------------\n\nTo use the Realtime API for transcription, you need to create a transcription session, connecting via [WebSockets](/docs/guides/realtime?use-case=transcription#connect-with-websockets) or [WebRTC](/docs/guides/realtime?use-case=transcription#connect-with-webrtc).\n\nUnlike the regular Realtime API sessions for conversations, the transcription sessions typically don't contain responses from the model.\n\nThe transcription session object is also different from regular Realtime API sessions:\n\n```json\n{\n object: \"realtime.transcription_session\",\n id: string,\n input_audio_format: string,\n input_audio_transcription: [{\n model: string,\n prompt: string,\n language: string\n }],\n turn_detection: {\n type: \"server_vad\",\n threshold: float,\n prefix_padding_ms: integer,\n silence_duration_ms: integer,\n } | null,\n input_audio_noise_reduction: {\n type: \"near_field\" | \"far_field\"\n },\n include: list[string] | null\n}\n```\n\nSome of the additional properties transcription sessions support are:\n\n* `input_audio_transcription.model`: The transcription model to use, currently `gpt-4o-transcribe`, `gpt-4o-mini-transcribe`, and `whisper-1` are supported\n* `input_audio_transcription.prompt`: The prompt to use for the transcription, to guide the model (e.g. \"Expect words related to technology\")\n* `input_audio_transcription.language`: The language to use for the transcription, ideally in ISO-639-1 format (e.g. \"en\", \"fr\"...) to improve accuracy and latency\n* `input_audio_noise_reduction`: The noise reduction configuration to use for the transcription\n* `include`: The list of properties to include in the transcription events\n\nPossible values for the input audio format are: `pcm16` (default), `g711_ulaw` and `g711_alaw`.\n\nYou can find more information about the transcription session object in the [API reference](/docs/api-reference/realtime-sessions/transcription_session_object).\n\nHandling transcriptions\n-----------------------\n\nWhen using the Realtime API for transcription, you can listen for the `conversation.item.input_audio_transcription.delta` and `conversation.item.input_audio_transcription.completed` events.\n\nFor `whisper-1` the `delta` event will contain full turn transcript, same as `completed` event. For `gpt-4o-transcribe` and `gpt-4o-mini-transcribe` the `delta` event will contain incremental transcripts as they are streamed out from the model.\n\nHere is an example transcription delta event:\n\n```json\n{\n \"event_id\": \"event_2122\",\n \"type\": \"conversation.item.input_audio_transcription.delta\",\n \"item_id\": \"item_003\",\n \"content_index\": 0,\n \"delta\": \"Hello,\"\n}\n```\n\nHere is an example transcription completion event:\n\n```json\n{\n \"event_id\": \"event_2122\",\n \"type\": \"conversation.item.input_audio_transcription.completed\",\n \"item_id\": \"item_003\",\n \"content_index\": 0,\n \"transcript\": \"Hello, how are you?\"\n}\n```\n\nNote that ordering between completion events from different speech turns is not guaranteed. You should use `item_id` to match these events to the `input_audio_buffer.committed` events and use `input_audio_buffer.committed.previous_item_id` to handle the ordering.\n\nTo send audio data to the transcription session, you can use the `input_audio_buffer.append` event.\n\nYou have 2 options:\n\n* Use a streaming microphone input\n* Stream data from a wav file\n\nVoice activity detection\n------------------------\n\nThe Realtime API supports automatic voice activity detection (VAD). Enabled by default, VAD will control when the input audio buffer is committed, therefore when transcription begins.\n\nRead more about configuring VAD in our [Voice Activity Detection](/docs/guides/realtime-vad) guide.\n\nYou can also disable VAD by setting the `turn_detection` property to `null`, and control when to commit the input audio on your end.\n\nAdditional configurations\n-------------------------\n\n### Noise reduction\n\nYou can use the `input_audio_noise_reduction` property to configure how to handle noise reduction in the audio stream.\n\nThe possible values are:\n\n* `near_field`: Use near-field noise reduction.\n* `far_field`: Use far-field noise reduction.\n* `null`: Disable noise reduction.\n\nThe default value is `near_field`, and you can disable noise reduction by setting the property to `null`.\n\n### Using logprobs\n\nYou can use the `include` property to include logprobs in the transcription events, using `item.input_audio_transcription.logprobs`.\n\nThose logprobs can be used to calculate the confidence score of the transcription.\n\n```json\n{\n \"type\": \"transcription_session.update\",\n \"input_audio_format\": \"pcm16\",\n \"input_audio_transcription\": {\n \"model\": \"gpt-4o-transcribe\",\n \"prompt\": \"\",\n \"language\": \"\"\n },\n \"turn_detection\": {\n \"type\": \"server_vad\",\n \"threshold\": 0.5,\n \"prefix_padding_ms\": 300,\n \"silence_duration_ms\": 500,\n },\n \"input_audio_noise_reduction\": {\n \"type\": \"near_field\"\n },\n \"include\": [ \n \"item.input_audio_transcription.logprobs\",\n ],\n}\n```\n\nWas this page useful?",[],[208,318,344],"3d411d36-8935-11f0-a247-0224a6c84d84","realtime-sip.md","3d411dfe-8935-11f0-a247-0224a6c84d84",36,["D",1756952217949],"Realtime API with SIP\n=====================\n\nConnect to the Realtime API using SIP.\n\n[SIP](https://en.wikipedia.org/wiki/Session_Initiation_Protocol) is a protocol used to make phone calls over the internet. With SIP and the Realtime API you can direct incoming phone calls to the API.\n\nOverview\n--------\n\nIf you want to connect a phone number to the Realtime API, use a SIP trunking provider (e.g., Twilio). This is a service that converts your phone call to IP traffic. After you purchase a phone number from your SIP trunking provider, follow the instructions below.\n\nStart by creating a [webhook](/docs/guides/webhooks) for incoming calls, at platform.openai.com. Then, point your SIP trunk at the OpenAI SIP endpoint, using the project ID for which you configured the webhook, e.g., `sip:$PROJECT_ID@sip.api.openai.com;transport=tls`. To find your `$PROJECT_ID`, go to your \\[settings\\] > **General**. The page displays the project ID. It should have a `proj_` prefix.\n\nWhen OpenAI receives SIP traffic associated with your project, the webhook that you configured will be fired. The event fired will be a [`realtime.call.incoming`](/docs/api-reference/webhook_events/realtime/call/incoming) event.\n\nThis webhook lets you accept or reject the call. When accepting the call, you'll provide the configuration (instructions, voice, etc) for the Realtime API session. Once established, you can set up a web socket and monitor the session as usual. The APIs to accept, reject, and monitor the call are documented below.\n\nConnection details\n------------------\n\nURIs used for interacting with Realtime API and SIP:\n\n|SIP URI|sip:$PROJECT_ID@sip.api.openai.com;transport=tls|\n|Accept URI|https://api.openai.com/v1/realtime/calls/$CALL_ID/accept|\n|Reject URI|https://api.openai.com/v1/realtime/calls/$CALL_ID/reject|\n|Refer URI|https://api.openai.com/v1/realtime/calls/$CALL_ID/refer|\n|Events URI|wss://api.openai.com/v1/realtime?call_id=$CALL_ID|\n\nFind your `$CALL_ID` in the `call_id` field in data object present in the webhook. See an example in the next section.\n\nHandle the webhook\n------------------\n\nThe following is an example of a `realtime.call.incoming` handler. It accepts the call and then logs all the events from the Realtime API.\n\nPython\n\nPython\n\n```python\nfrom flask import Flask, request, Response, jsonify, make_response\nfrom openai import OpenAI, InvalidWebhookSignatureError\nimport asyncio\nimport json\nimport os\nimport requests\nimport time\nimport threading\nimport websockets\n\napp = Flask(__name__)\nclient = OpenAI(\n webhook_secret=os.environ[\"OPENAI_WEBHOOK_SECRET\"]\n)\n\nAUTH_HEADER = {\n \"Authorization\": \"Bearer \" + os.getenv(\"OPENAI_API_KEY\")\n}\n\ncall_accept = {\n \"type\": \"realtime\",\n \"instructions\": \"You are a support agent.\",\n \"model\": \"gpt-4o-realtime-preview-2024-12-17\",\n}\n\nresponse_create = {\n \"type\": \"response.create\",\n \"response\": {\n \"instructions\": (\n \"Say to the user 'Thank you for calling, how can I help you'\"\n )\n },\n}\n\nasync def websocket_task(call_id):\n try:\n async with websockets.connect(\n \"wss://api.openai.com/v1/realtime?call_id=\" + call_id,\n additional_headers=AUTH_HEADER,\n ) as websocket:\n await websocket.send(json.dumps(response_create))\n\n while True:\n response = await websocket.recv()\n print(f\"Received from WebSocket: {response}\")\n except Exception as e:\n print(f\"WebSocket error: {e}\")\n\n@app.route(\"/\", methods=[\"POST\"])\ndef webhook():\n try:\n event = client.webhooks.unwrap(request.data, request.headers)\n\n if event.type == \"realtime.call.incoming\":\n requests.post(\n \"https://api.openai.com/v1/realtime/calls/\"\n + event.data.call_id\n + \"/accept\",\n headers={**AUTH_HEADER, \"Content-Type\": \"application/json\"},\n json=call_accept,\n )\n threading.Thread(\n target=lambda: asyncio.run(\n websocket_task(event.data.call_id)\n ),\n daemon=True,\n ).start()\n return Response(status=200)\n except InvalidWebhookSignatureError as e:\n print(\"Invalid signature\", e)\n return Response(\"Invalid signature\", status=400)\n\nif __name__ == \"__main__\":\n app.run(port=8000)\n```\n\nRedirect the call\n-----------------\n\nIt's also possible to redirect the call to another number. During the call, make a POST to the `refer` endpoint:\n\n|URL|https://api.openai.com/v1/realtime/calls/$CALL_ID/refer|\n|Payload|JSON with one key target_uriThis is the value used in the Refer-To. You can use Tel-URI for example tel:+14152909007|\n|Headers|Authorization: Bearer YOUR_API_KEYSubstitute YOUR_API_KEY with a standard API key|\n\nNext steps\n----------\n\nNow that you've connected over SIP, use the left navigation or click into these pages to start building your realtime application.\n\n* [Using realtime models](/docs/guides/realtime-models-prompting)\n* [Managing conversations](/docs/guides/realtime-conversations)\n* [Webhooks and server-side controls](/docs/guides/realtime-server-controls)\n* [Realtime transcription](/docs/guides/realtime-transcription)\n\nWas this page useful?",[],[208,318,352],"3bab9e88-8935-11f0-8603-0224a6c84d84","realtime-server-controls.md","3bab9f50-8935-11f0-8603-0224a6c84d84",34,["D",1756952214272],"Webhooks and server-side controls\n=================================\n\nUse webhooks and server-side controls with the Realtime API.\n\nThe Realtime API allows clients to connect directly to the API server via WebRTC or SIP. However, you'll most likely want tool use and other business logic to reside on your application server to keep this logic private and client-agnostic.\n\nKeep tool use, business logic, and other details secure on the server side by connecting over a “sideband” control channel. We now have sideband options for both SIP and WebRTC connections.\n\nWith WebRTC\n-----------\n\nWhen [establishing a peer connection](/docs/guides/realtime-webrtc), you receive an SDP response from the Realtime API to configure the connection. If you used the sample code from the WebRTC guide, that looks something like this:\n\n```javascript\nconst baseUrl = \"https://api.openai.com/v1/realtime/calls\";\nconst model = \"gpt-realtime\";\nconst sdpResponse = await fetch(`${baseUrl}?model=${model}`, {\n method: \"POST\",\n body: offer.sdp,\n headers: {\n Authorization: `Bearer ${EPHEMERAL_KEY}`,\n \"Content-Type\": \"application/sdp\",\n },\n});\n```\n\nThe SDP response will contain a `Location` header that has a unique call ID that can be used on the server to establish a WebSocket connection to that same Realtime session.\n\n```javascript\nconst location = sdpResponse.headers.get(\"Location\");\nconst callId = location?.split(\"/\").pop();\nconsole.log(callId);\n// rtc_u1_9c6574da8b8a41a18da9308f4ad974ce\n```\n\nOn a server, you can then [listen for events and configure the session](/docs/guides/realtime-conversations) just as you would from the client, using the ID from this URL:\n\n```javascript\nimport WebSocket from \"ws\";\n\n// You'll need to get the call ID from the browser to your\n// server somehow:\nconst callId = \"rtc_u1_9c6574da8b8a41a18da9308f4ad974ce\";\n\n// Connect to a WebSocket for the in-progress call\nconst url = \"wss://api.openai.com/v1/realtime?call_id=\" + callId;\nconst ws = new WebSocket(url, {\n headers: {\n Authorization: \"Bearer \" + process.env.OPENAI_API_KEY,\n },\n});\n\nws.on(\"open\", function open() {\n console.log(\"Connected to server.\");\n\n // Send client events over the WebSocket once connected\n ws.send(\n JSON.stringify({\n type: \"session.update\",\n session: {\n type: \"realtime\",\n instructions: \"Be extra nice today!\",\n },\n })\n );\n});\n\n// Listen for and parse server events\nws.on(\"message\", function incoming(message) {\n console.log(JSON.parse(message.toString()));\n});\n```\n\nIn this way, you are able to add tools, monitor sessions, and carry out business logic on the server instead of needing to configure those actions on the client.\n\n### With SIP\n\n1. A user connects to OpenAI via phone over SIP.\n2. OpenAI sends a webhook to your application’s backend webhook URL, notifying your app of the state of the session.\n\n```text\nPOST https://my_website.com/webhook_endpoint\nuser-agent: OpenAI/1.0 (+https://platform.openai.com/docs/webhooks)\ncontent-type: application/json\nwebhook-id: wh_685342e6c53c8190a1be43f081506c52 # unique id for idempotency\nwebhook-timestamp: 1750287078 # timestamp of delivery attempt\nwebhook-signature: v1,K5oZfzN95Z9UVu1EsfQmfVNQhnkZ2pj9o9NDN/H/pI4=\n\n{\n\"object\": \"event\",\n\"id\": \"evt_685343a1381c819085d44c354e1b330e\",\n\"type\": \"realtime.call.incoming\",\n\"created_at\": 1750287018, // Unix timestamp\n\"data\": {\n\"call_id\": \"some_unique_id\",\n\"sip_headers\": [\n{ \"name\": \"From\", \"value\": \"sip:+142555512112@sip.example.com\" },\n{ \"name\": \"To\", \"value\": \"sip:+18005551212@sip.example.com\" },\n{ \"name\": \"Call-ID\", \"value\": \"03782086-4ce9-44bf-8b0d-4e303d2cc590\"}\n],\n}\n}\n```\n\n3. The application server opens a WebSocket connection to the Realtime API using the `call_id` value provided in the webhook. This `call_id` looks like this: `wss://api.openai.com/v1/realtime?call_id={callId}`. The WebSocket connection will live for the life of the SIP call.\n\nWas this page useful?",[],[208,318,360],"3c3fd6a2-8935-11f0-a46b-0224a6c84d84","realtime-models-prompting.md","3c3fd774-8935-11f0-a46b-0224a6c84d84",35,["D",1756952217909],"Using realtime models\n=====================\n\nUse realtime models and prompting effectively.\n\nRealtime models are post-trained for specific customer use cases. In response to your feedback, the latest speech-to-speech model works differently from previous models. Use this guide to understand and get the most out of it.\n\nMeet the models\n---------------\n\nOur most advanced speech-to-speech model is [gpt-realtime](/docs/models/gpt-realtime).\n\nThis model shows improvements in following complex instructions, calling tools, and producing speech that sounds natural and expressive. For more information, see the [announcement blog post](https://openai.com/index/introducing-gpt-realtime/).\n\nUpdate your session to use a prompt\n-----------------------------------\n\nAfter you initiate a session over [WebRTC](/docs/guides/realtime-webrtc), [WebSocket](/docs/guides/realtime-websocket), or [SIP](/docs/guides/realtime-sip), the client and model are connected. The server will send a [session.created](/docs/api-reference/realtime-server-events/session/created) event to confirm. Now it's a matter of prompting.\n\n### Basic prompt update\n\n1. Create a basic audio prompt in [the dashboard](/audio/realtime).\n \n If you don't know where to start, experiment with the prompt fields until you find something interesting. You can always manage, iterate on, and version your prompts later.\n \n2. Update your realtime session to use the prompt you created. Provide its prompt ID in a `session.update` client event:\n \n\nUpdate the system instructions used by the model in this session\n\n```javascript\nconst event = {\n type: \"session.update\",\n session: {\n type: \"realtime\",\n model: \"gpt-realtime\",\n // Lock the output to audio (add \"text\" if you also want text)\n output_modalities: [\"audio\"],\n audio: {\n input: {\n format: \"pcm16\",\n turn_detection: { type: \"semantic_vad\", create_response: true }\n },\n output: {\n format: \"g711_ulaw\",\n voice: \"alloy\",\n speed: 1.0\n }\n },\n // Use a server-stored prompt by ID. Optionally pin a version and pass variables.\n prompt: {\n id: \"pmpt_123\", // your stored prompt ID\n // version: \"89\", // optional: pin a specific version\n variables: {\n city: \"Paris\" // example variable used by your prompt\n }\n },\n // You can still set direct session fields; these override prompt fields if they overlap:\n instructions: \"Speak clearly and briefly. Confirm understanding before taking actions.\"\n },\n};\n\n// WebRTC data channel and WebSocket both have .send()\ndataChannel.send(JSON.stringify(event));\n```\n\n```python\nevent = {\n \"type\": \"session.update\",\n session: {\n type: \"realtime\",\n model: \"gpt-realtime\",\n // Lock the output to audio (add \"text\" if you also want text)\n output_modalities: [\"audio\"],\n audio: {\n input: {\n format: \"pcm16\",\n turn_detection: { type: \"semantic_vad\", create_response: true }\n },\n output: {\n format: \"g711_ulaw\",\n voice: \"alloy\",\n speed: 1.0\n }\n },\n // Use a server-stored prompt by ID. Optionally pin a version and pass variables.\n prompt: {\n id: \"pmpt_123\", // your stored prompt ID\n // version: \"89\", // optional: pin a specific version\n variables: {\n city: \"Paris\" // example variable used by your prompt\n }\n },\n // You can still set direct session fields; these override prompt fields if they overlap:\n instructions: \"Speak clearly and briefly. Confirm understanding before taking actions.\"\n }\n}\nws.send(json.dumps(event))\n```\n\nWhen the session's updated, the server emits a [session.updated](/docs/api-reference/realtime-server-events/session/updated) event with the new state of the session. You can update the session any time.\n\n### Changing prompt mid-call\n\nTo update the session mid-call (to swap prompt version or variables, or override instructions), send the update over the same data channel you're using:\n\n```text\n// Example: switch to a specific prompt version and change a variable\ndc.send(JSON.stringify({\n type: \"session.update\",\n session: {\n prompt: {\n id: \"pmpt_123\",\n version: \"89\",\n variables: {\n city: \"Berlin\"\n }\n }\n }\n}));\n\n// Example: override instructions (note: direct session fields take precedence over Prompt fields)\ndc.send(JSON.stringify({\n type: \"session.update\",\n session: {\n instructions: \"Speak faster and keep answers under two sentences.\"\n }\n}));\n```\n\nPrompting gpt-realtime\n----------------------\n\nHere are top tips for prompting the realtime speech-to-speech model. For a more in-depth guide to prompting, see the [realtime prompting cookbook](https://cookbook.openai.com/examples/realtime_prompting_guide).\n\n### General usage tips\n\n* **Iterate relentlessly**. Small wording changes can make or break behavior.\n \n Example: Swapping “inaudible” → “unintelligible” improved noisy input handling.\n \n* **Use bullets over paragraphs**. Clear, short bullets outperform long paragraphs.\n \n* **Guide with examples**. The model strongly follows onto sample phrases.\n \n* **Be precise**. Ambiguity and conflicting instructions degrade performance, similar to GPT-5.\n \n* **Control language**. Pin output to a target language if you see drift.\n \n* **Reduce repetition**. Add a variety rule to reduce robotic phrasing.\n \n* **Use all caps for emphasis**: Capitalize key rules to makes them stand out to the model.\n \n* **Convert non-text rules to text**: The model responds better to clearly written text.\n \n Example: Instead of writing, \"IF x > 3 THEN ESCALATE\", write, \"IF MORE THAN THREE FAILURES THEN ESCALATE.\"\n \n\n### Structure your prompt\n\nOrganize your prompt to help the model understand context and stay consistent across turns.\n\nUse clear, labeled sections in your system prompt so the model can find and follow them. Keep each section focused on one thing.\n\n```text\n# Role & Objective — who you are and what “success” means\n# Personality & Tone — the voice and style to maintain\n# Context — retrieved context, relevant info\n# Reference Pronunciations — phonetic guides for tricky words\n# Tools — names, usage rules, and preambles\n# Instructions / Rules — do’s, don’ts, and approach\n# Conversation Flow — states, goals, and transitions\n# Safety & Escalation — fallback and handoff logic\n```\n\nThis format also makes it easier for you to iterate and modify problematic sections.\n\nTo make this system prompt your own, add domain-specific sections (e.g., Compliance, Brand Policy) and remove sections you don’t need. In each section, provide instructions and other information for the model to respond correctly. See specifics below.\n\nPractical tips for prompting realtime models\n--------------------------------------------\n\nHere are 10 tips for creating effective, consistently performing prompts with gpt-realtime. These are just an overview. For more details and full system prompt examples, see the [realtime prompting cookbook](https://cookbook.openai.com/examples/realtime_prompting_guide).\n\n#### 1\\. Be precise. Kill conflicts.\n\nThe new realtime model is very good at instruction following. However, that also means small wording changes or unclear instructions can shift behavior in meaningful ways. Inspect and iterate on your system prompt to try different phrasing and fix instruction contradictions.\n\nIn one experiment we ran, changing the word \"inaudible\" to \"unintelligble\" in instructions for handling noisy inputs significantly improved the model's performance.\n\nAfter your first attempt at a system prompt, have an LLM review it for ambiguity or conflicts.\n\n#### 2\\. Bullets > paragraphs.\n\nRealtime models follow short bullet points better than long paragraphs.\n\nBefore (harder to follow):\n\n```text\nWhen you can’t clearly hear the user, don’t proceed. If there’s background noise or you only caught part of the sentence, pause and ask them politely to repeat themselves in their preferred language, and make sure you keep the conversation in the same language as the user.\n```\n\nAfter (easier to follow):\n\n```text\nOnly respond to clear audio or text.\n\nIf audio is unclear/partial/noisy/silent, ask for clarification in `{preferred_language}`.\n\nContinue in the same language as the user if intelligible.\n```\n\n#### 3\\. Handle unclear audio.\n\nThe realtime model is good at following instructions on how to handle unclear audio. Spell out what to do when audio isn’t usable.\n\n```text\n## Unclear audio\n- Always respond in the same language the user is speaking in, if intelligible.\n- Only respond to clear audio or text.\n- If the user's audio is not clear (e.g., ambiguous input/background noise/silent/unintelligible) or if you did not fully hear or understand the user, ask for clarification using {preferred_language} phrases.\n\nSample clarification phrases (parameterize with {preferred_language}):\n\n- “Sorry, I didn’t catch that—could you say it again?”\n- “There’s some background noise. Please repeat the last part.”\n- “I only heard part of that. What did you say after ___?”\n```\n\n#### 4\\. Constrain the model to one language.\n\nIf you see the model switching languages in an unhelpful way, add a dedicated \"Language\" section in your prompt. Make sure it doesn’t conflict with other rules. By default, mirroring the user’s language works well.\n\nHere's a simple way to mirror the user's language:\n\n```text\n## Language\nLanguage matching: Respond in the same language as the user unless directed otherwise.\nFor non-English, start with the same standard accent/dialect the user uses.\n```\n\nHere's an example of an English-only constraint:\n\n```text\n## Language\n- The conversation will be only in English.\n- Do not respond in any other language, even if the user asks.\n- If the user speaks another language, politely explain that support is limited to English.\n```\n\nIn a language teaching application, your language and conversation sections might look like this:\n\n```text\n## Language\n### Explanations\nUse English when explaining grammar, vocabulary, or cultural context.\n\n### Conversation\nSpeak in French when conducting practice, giving examples, or engaging in dialogue.\n```\n\nYou can also control dialect for a more consistent personality:\n\n```text\n## Language\nResponse only in argentine spanish.\n```\n\n#### 5\\. Provide sample phrases and flow snippets.\n\nThe model learns style from examples. Give short, varied samples for common conversation moments.\n\nFor example, you might give this high-level shape of conversation flow to the model:\n\n```text\nGreeting → Discover → Verify → Diagnose → Resolve → Confirm/Close. Advance only when criteria in each phase are met.\n```\n\nAnd then provide prompt guidance for each section. For example, here's how you might instruct for the greeting section:\n\n```text\n## Conversation flow — Greeting\nGoal: Set tone and invite the reason for calling.\n\nHow to respond:\n- Identify as ACME Internet Support.\n- Keep it brief; invite the caller’s goal.\n\nSample phrases (vary, don’t always reuse):\n- “Thanks for calling ACME Internet—how can I help today?”\n- “You’ve reached ACME Support. What’s going on with your service?”\n- “Hi there—tell me what you’d like help with.”\n\nExit when: Caller states an initial goal or symptom.\n```\n\n#### 6\\. Avoid robotic repetition.\n\nIf responses sound repetitive or robotic, include an explicit variety instruction. This can sometimes happen when using sample phrases.\n\n```text\n## Variety\n- Do not repeat the same sentence twice. Vary your responses so it doesn't sound robotic.\n```\n\n#### 7\\. Use capitalized text to emphasize instructions.\n\nLike many LLMs, using capitalization for important rules can help the model to understand and follow those rules. It's also helpful to convert non-text rules (such as numerical conditions) into text before capitalization.\n\nInstead of:\n\n```text\n## Rules\n- If [func.return_value] > 0, respond 1 to the user.\n```\n\nUse:\n\n```text\n## Rules\n- IF [func.return_value] IS BIGGER THAN 0, RESPOND 1 TO THE USER.\n```\n\n#### 8\\. Help the model use tools.\n\nThe model's use of tools can alter the experience—how much they rely on user confirmation vs. taking action, what they say while they make the tool call, which rules they follow for each specific tool, etc.\n\nOne way to prompt for tool usage is to use preambles. Good preambles instruct the model to give the user some feedback about what it's doing before it makes the tool call, so the user always knows what's going on.\n\nHere's an example:\n\n```text\n# Tools\n- Before any tool call, say one short line like “I’m checking that now.” Then call the tool immediately.\n```\n\nYou can include sample phrases for preambles to add variety and better tailor to your use case.\n\nThere are several other ways to improve the model's behavior when performing tool calls and keeping the conversation going with the user. Ideally, the model is calling the right tools proactively, checking for confirmation for any important write actions, and keeping the user informed along the way. For more specifics, see the [realtime prompting cookbook](https://cookbook.openai.com/examples/realtime_prompting_guide).\n\n#### 9\\. Use LLMs to improve your prompt.\n\nLLMs are great at finding what's going wrong in your prompt. Use ChatGPT or the API to get a model's review of your current realtime prompt and get help improving it.\n\nWhether your prompt is working well or not, here's a prompt you can run to get a model's review:\n\n```text\n## Role & Objective\nYou are a **Prompt-Critique Expert**.\nExamine a user-supplied LLM prompt and surface any weaknesses following the instructions below.\n\n## Instructions\nReview the prompt that is meant for an LLM to follow and identify the following issues:\n- Ambiguity: Could any wording be interpreted in more than one way?\n- Lacking Definitions: Are there any class labels, terms, or concepts that are not defined that might be misinterpreted by an LLM?\n- Conflicting, missing, or vague instructions: Are directions incomplete or contradictory?\n- Unstated assumptions: Does the prompt assume the model has to be able to do something that is not explicitly stated?\n\n## Do **NOT** list issues of the following types:\n- Invent new instructions, tool calls, or external information. You do not know what tools need to be added that are missing.\n- Issues that you are not sure about.\n\n## Output Format\n\n# Issues\n- Numbered list; include brief quote snippets.\n\n# Improvements\n- Numbered list; provide the revised lines you would change and how you would changed them.\n\n# Revised Prompt\n- Revised prompt where you have applied all your improvements surgically with minimal edits to the original prompt\n```\n\nUse this template as a starting point for troubleshooting a recurring issue:\n\n```text\nHere's my current prompt to an LLM:\n[BEGIN OF CURRENT PROMPT]\n{CURRENT_PROMPT}\n[END OF CURRENT PROMPT]\n\nBut I see this issue happening from the LLM:\n[BEGIN OF ISSUE]\n{ISSUE}\n[END OF ISSUE]\nCan you provide some variants of the prompt so that the model can better understand the constraints to alleviate the issue?\n```\n\n#### 10\\. Help users resolve issues faster.\n\nTwo frustrating user experiences are slow, mechanical voice agents and the inability to escalate. Help users faster by providing instructions in your system prompt for speed and escalation.\n\nIn the personality and tone section of your system prompt, add pacing instructions to get the model to quicken its support:\n\n```text\n# Personality & Tone\n## Personality\nFriendly, calm and approachable expert customer service assistant.\n\n## Tone\nTone: Warm, concise, confident, never fawning.\n\n## Length\n2–3 sentences per turn.\n\n## Pacing\nDeliver your audio response fast, but do not sound rushed. Do not modify the content of your response, only increase speaking speed for the same response.\n```\n\nOften with realtime voice agents, having a reliable way to escalate to a human is important. In a safety and escalation section, modify the instructions on WHEN to escalate depending on your use case. Here's an example:\n\n```text\n# Safety & Escalation\nWhen to escalate (no extra troubleshooting):\n- Safety risk (self-harm, threats, harassment)\n- User explicitly asks for a human\n- Severe dissatisfaction (e.g., “extremely frustrated,” repeated complaints, profanity)\n- **2** failed tool attempts on the same task **or** **3** consecutive no-match/no-input events\n- Out-of-scope or restricted (e.g., real-time news, financial/legal/medical advice)\n\nWhat to say at the same time of calling the escalate_to_human tool (MANDATORY):\n- “Thanks for your patience—**I’m connecting you with a specialist now**.”\n- Then call the tool: `escalate_to_human`\n\nExamples that would require escalation:\n- “This is the third time the reset didn’t work. Just get me a person.”\n- “I am extremely frustrated!”\n```\n\nFurther reading\n---------------\n\nThis guide is long but not exhaustive! For more in a specific area, see the following resources:\n\n* [Realtime prompting cookbook](https://cookbook.openai.com/examples/realtime_prompting_guide): Full prompt examples and a deep dive into when and how to use them\n* [Inputs and outputs](/docs/guides/realtime-inputs-outputs): Text and audio input requirements and output options\n* [Managing conversations](/docs/guides/realtime-conversations): Learn to manage a conversation for the duration of a realtime session\n* [Webhooks and server-side controls](/docs/guides/realtime-server-controls): Create a sideband channel to separate sensitive server-side logic from an untrusted client\n* [Function calling](/docs/guides/realtime-function-calling): How to call functions in your realtime app\n* [MCP servers](/docs/guides/realtime-mcp): How to use MCP servers to access additional tools in realtime apps\n* [Realtime transcription](/docs/guides/realtime-transcription): How to transcribe audio with the Realtime API\n* [Voice agents](https://openai.github.io/openai-agents-js/guides/voice-agents/quickstart/): A quickstart for building a voice agent with the Agents SDK\n\nWas this page useful?",[],[208,318,368],"3a3db9b4-8935-11f0-bb34-0224a6c84d84","realtime-conversations.md","3a3dba68-8935-11f0-bb34-0224a6c84d84",29,["D",1756952215183],"Realtime conversations\n======================\n\nLearn how to manage Realtime speech-to-speech conversations.\n\nOnce you have connected to the Realtime API through either [WebRTC](/docs/guides/realtime-webrtc) or [WebSocket](/docs/guides/realtime-websocket), you can call a Realtime model (such as [gpt-4o-realtime-preview](/docs/models/gpt-4o-realtime-preview)) to have speech-to-speech conversations. Doing so will require you to **send client events** to initiate actions, and **listen for server events** to respond to actions taken by the Realtime API.\n\nThis guide will walk through the event flows required to use model capabilities like audio and text generation and function calling, and how to think about the state of a Realtime Session.\n\nIf you do not need to have a conversation with the model, meaning you don't expect any response, you can use the Realtime API in [transcription mode](/docs/guides/realtime-transcription).\n\nRealtime speech-to-speech sessions\n----------------------------------\n\nA Realtime Session is a stateful interaction between the model and a connected client. The key components of the session are:\n\n* The **Session** object, which controls the parameters of the interaction, like the model being used, the voice used to generate output, and other configuration.\n* A **Conversation**, which represents user input Items and model output Items generated during the current session.\n* **Responses**, which are model-generated audio or text Items that are added to the Conversation.\n\n**Input audio buffer and WebSockets**\n\nIf you are using WebRTC, much of the media handling required to send and receive audio from the model is assisted by WebRTC APIs.\n\n \n\nIf you are using WebSockets for audio, you will need to manually interact with the **input audio buffer** by sending audio to the server, sent with JSON events with base64-encoded audio.\n\nAll these components together make up a Realtime Session. You will use client events to update the state of the session, and listen for server events to react to state changes within the session.\n\n![diagram realtime state](https://openaidevs.retool.com/api/file/11fe71d2-611e-4a26-a587-881719a90e56)\n\nSession lifecycle events\n------------------------\n\nAfter initiating a session via either [WebRTC](/docs/guides/realtime-webrtc) or [WebSockets](/docs/guides/realtime-websockets), the server will send a [`session.created`](/docs/api-reference/realtime-server-events/session/created) event indicating the session is ready. On the client, you can update the current session configuration with the [`session.update`](/docs/api-reference/realtime-client-events/session/update) event. Most session properties can be updated at any time, except for the `voice` the model uses for audio output, after the model has responded with audio once during the session. The maximum duration of a Realtime session is **30 minutes**.\n\nThe following example shows updating the session with a `session.update` client event. See the [WebRTC](/docs/guides/realtime-webrtc#sending-and-receiving-events) or [WebSocket](/docs/guides/realtime-websocket#sending-and-receiving-events) guide for more on sending client events over these channels.\n\nUpdate the system instructions used by the model in this session\n\n```javascript\nconst event = {\n type: \"session.update\",\n session: {\n type: \"realtime\",\n model: \"gpt-realtime\",\n // Lock the output to audio (add \"text\" if you also want text)\n output_modalities: [\"audio\"],\n audio: {\n input: {\n format: \"pcm16\",\n turn_detection: { type: \"semantic_vad\", create_response: true }\n },\n output: {\n format: \"g711_ulaw\",\n voice: \"alloy\",\n speed: 1.0\n }\n },\n // Use a server-stored prompt by ID. Optionally pin a version and pass variables.\n prompt: {\n id: \"pmpt_123\", // your stored prompt ID\n // version: \"89\", // optional: pin a specific version\n variables: {\n city: \"Paris\" // example variable used by your prompt\n }\n },\n // You can still set direct session fields; these override prompt fields if they overlap:\n instructions: \"Speak clearly and briefly. Confirm understanding before taking actions.\"\n },\n};\n\n// WebRTC data channel and WebSocket both have .send()\ndataChannel.send(JSON.stringify(event));\n```\n\n```python\nevent = {\n \"type\": \"session.update\",\n session: {\n type: \"realtime\",\n model: \"gpt-realtime\",\n // Lock the output to audio (add \"text\" if you also want text)\n output_modalities: [\"audio\"],\n audio: {\n input: {\n format: \"pcm16\",\n turn_detection: { type: \"semantic_vad\", create_response: true }\n },\n output: {\n format: \"g711_ulaw\",\n voice: \"alloy\",\n speed: 1.0\n }\n },\n // Use a server-stored prompt by ID. Optionally pin a version and pass variables.\n prompt: {\n id: \"pmpt_123\", // your stored prompt ID\n // version: \"89\", // optional: pin a specific version\n variables: {\n city: \"Paris\" // example variable used by your prompt\n }\n },\n // You can still set direct session fields; these override prompt fields if they overlap:\n instructions: \"Speak clearly and briefly. Confirm understanding before taking actions.\"\n }\n}\nws.send(json.dumps(event))\n```\n\nWhen the session has been updated, the server will emit a [`session.updated`](/docs/api-reference/realtime-server-events/session/updated) event with the new state of the session.\n\n||\n|session.update|session.createdsession.updated|\n\nText inputs and outputs\n-----------------------\n\nTo generate text with a Realtime model, you can add text inputs to the current conversation, ask the model to generate a response, and listen for server-sent events indicating the progress of the model's response. In order to generate text, the [session must be configured](/docs/api-reference/realtime-client-events/session/update) with the `text` modality (this is true by default).\n\nCreate a new text conversation item using the [`conversation.item.create`](/docs/api-reference/realtime-client-events/conversation/item/create) client event. This is similar to sending a [user message (prompt) in Chat Completions](/docs/guides/text-generation) in the REST API.\n\nCreate a conversation item with user input\n\n```javascript\nconst event = {\n type: \"conversation.item.create\",\n item: {\n type: \"message\",\n role: \"user\",\n content: [\n {\n type: \"input_text\",\n text: \"What Prince album sold the most copies?\",\n }\n ]\n },\n};\n\n// WebRTC data channel and WebSocket both have .send()\ndataChannel.send(JSON.stringify(event));\n```\n\n```python\nevent = {\n \"type\": \"conversation.item.create\",\n \"item\": {\n \"type\": \"message\",\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"What Prince album sold the most copies?\",\n }\n ]\n }\n}\nws.send(json.dumps(event))\n```\n\nAfter adding the user message to the conversation, send the [`response.create`](/docs/api-reference/realtime-client-events/response/create) event to initiate a response from the model. If both audio and text are enabled for the current session, the model will respond with both audio and text content. If you'd like to generate text only, you can specify that when sending the `response.create` client event, as shown below.\n\nGenerate a text-only response\n\n```javascript\nconst event = {\n type: \"response.create\",\n response: {\n modalities: [ \"text\" ]\n },\n};\n\n// WebRTC data channel and WebSocket both have .send()\ndataChannel.send(JSON.stringify(event));\n```\n\n```python\nevent = {\n \"type\": \"response.create\",\n \"response\": {\n \"modalities\": [ \"text\" ]\n }\n}\nws.send(json.dumps(event))\n```\n\nWhen the response is completely finished, the server will emit the [`response.done`](/docs/api-reference/realtime-server-events/response/done) event. This event will contain the full text generated by the model, as shown below.\n\nListen for response.done to see the final results\n\n```javascript\nfunction handleEvent(e) {\n const serverEvent = JSON.parse(e.data);\n if (serverEvent.type === \"response.done\") {\n console.log(serverEvent.response.output[0]);\n }\n}\n\n// Listen for server messages (WebRTC)\ndataChannel.addEventListener(\"message\", handleEvent);\n\n// Listen for server messages (WebSocket)\n// ws.on(\"message\", handleEvent);\n```\n\n```python\ndef on_message(ws, message):\n server_event = json.loads(message)\n if server_event.type == \"response.done\":\n print(server_event.response.output[0])\n```\n\nWhile the model response is being generated, the server will emit a number of lifecycle events during the process. You can listen for these events, such as [`response.text.delta`](/docs/api-reference/realtime-server-events/response/text/delta), to provide realtime feedback to users as the response is generated. A full listing of the events emitted by there server are found below under **related server events**. They are provided in the rough order of when they are emitted, along with relevant client-side events for text generation.\n\n||\n|conversation.item.createresponse.create|conversation.item.createdresponse.createdresponse.output_item.addedresponse.content_part.addedresponse.text.deltaresponse.text.doneresponse.content_part.doneresponse.output_item.doneresponse.donerate_limits.updated|\n\nAudio inputs and outputs\n------------------------\n\nOne of the most powerful features of the Realtime API is voice-to-voice interaction with the model, without an intermediate text-to-speech or speech-to-text step. This enables lower latency for voice interfaces, and gives the model more data to work with around the tone and inflection of voice input.\n\n### Voice options\n\nRealtime sessions can be configured to use one of several built‑in voices when producing audio output. You can set the `voice` on session creation (or on a `response.create`) to control how the model sounds. Current voice options are `alloy`, `ash`, `ballad`, `coral`, `echo`, `sage`, `shimmer`, and `verse`. Once the model has emitted audio in a session, the `voice` cannot be modified for that session.\n\n### Handling audio with WebRTC\n\nIf you are connecting to the Realtime API using WebRTC, the Realtime API is acting as a [peer connection](https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection) to your client. Audio output from the model is delivered to your client as a remote media stream. Audio input to the model is collected using audio devices ([`getUserMedia`](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia)), and media streams are added as tracks to to the peer connection.\n\nThe example code from the [WebRTC connection guide](/docs/guides/realtime-webrtc) shows a basic example of configuring both local and remote audio using browser APIs:\n\n```javascript\n// Create a peer connection\nconst pc = new RTCPeerConnection();\n\n// Set up to play remote audio from the model\nconst audioEl = document.createElement(\"audio\");\naudioEl.autoplay = true;\npc.ontrack = e => audioEl.srcObject = e.streams[0];\n\n// Add local audio track for microphone input in the browser\nconst ms = await navigator.mediaDevices.getUserMedia({\n audio: true\n});\npc.addTrack(ms.getTracks()[0]);\n```\n\nThe snippet above enables simple interaction with the Realtime API, but there's much more that can be done. For more examples of different kinds of user interfaces, check out the [WebRTC samples](https://github.com/webrtc/samples) repository. Live demos of these samples can also be [found here](https://webrtc.github.io/samples/).\n\nUsing [media captures and streams](https://developer.mozilla.org/en-US/docs/Web/API/Media_Capture_and_Streams_API) in the browser enables you to do things like mute and unmute microphones, select which device to collect input from, and more.\n\n### Client and server events for audio in WebRTC\n\nBy default, WebRTC clients don't need to send any client events to the Realtime API before sending audio inputs. Once a local audio track is added to the peer connection, your users can just start talking!\n\nHowever, WebRTC clients still receive a number of server-sent lifecycle events as audio is moving back and forth between client and server over the peer connection. Examples include:\n\n* When input is sent over the local media track, you will receive [`input_audio_buffer.speech_started`](/docs/api-reference/realtime-server-events/input_audio_buffer/speech_started) events from the server.\n* When local audio input stops, you'll receive the [`input_audio_buffer.speech_stopped`](/docs/api-reference/realtime-server-events/input_audio_buffer/speech_started) event.\n* You'll receive [delta events for the in-progress audio transcript](/docs/api-reference/realtime-server-events/response/audio_transcript/delta).\n* You'll receive a [`response.done`](/docs/api-reference/realtime-server-events/response/done) event when the model has transcribed and completed sending a response.\n\nManipulating WebRTC APIs for media streams may give you all the control you need. However, it may occasionally be necessary to use lower-level interfaces for audio input and output. Refer to the WebSockets section below for more information and a listing of events required for granular audio input handling.\n\n### Handling audio with WebSockets\n\nWhen sending and receiving audio over a WebSocket, you will have a bit more work to do in order to send media from the client, and receive media from the server. Below, you'll find a table describing the flow of events during a WebSocket session that are necessary to send and receive audio over the WebSocket.\n\nThe events below are given in lifecycle order, though some events (like the `delta` events) may happen concurrently.\n\n||\n|Session initialization|session.update|session.createdsession.updated|\n|User audio input|conversation.item.create (send whole audio message)input_audio_buffer.append (stream audio in chunks)input_audio_buffer.commit (used when VAD is disabled)response.create (used when VAD is disabled)|input_audio_buffer.speech_startedinput_audio_buffer.speech_stoppedinput_audio_buffer.committed|\n|Server audio output|input_audio_buffer.clear (used when VAD is disabled)|conversation.item.createdresponse.createdresponse.output_item.createdresponse.content_part.addedresponse.audio.deltaresponse.audio_transcript.deltaresponse.text.deltaresponse.audio.doneresponse.audio_transcript.doneresponse.text.doneresponse.content_part.doneresponse.output_item.doneresponse.donerate_limits.updated|\n\n### Streaming audio input to the server\n\nTo stream audio input to the server, you can use the [`input_audio_buffer.append`](/docs/api-reference/realtime-client-events/input_audio_buffer/append) client event. This event requires you to send chunks of **Base64-encoded audio bytes** to the Realtime API over the socket. Each chunk cannot exceed 15 MB in size.\n\nThe format of the input chunks can be configured either for the entire session, or per response.\n\n* Session: `session.input_audio_format` in [`session.update`](/docs/api-reference/realtime-client-events/session/update)\n* Response: `response.input_audio_format` in [`response.create`](/docs/api-reference/realtime-client-events/response/create)\n\nAppend audio input bytes to the conversation\n\n```javascript\nimport fs from 'fs';\nimport decodeAudio from 'audio-decode';\n\n// Converts Float32Array of audio data to PCM16 ArrayBuffer\nfunction floatTo16BitPCM(float32Array) {\n const buffer = new ArrayBuffer(float32Array.length * 2);\n const view = new DataView(buffer);\n let offset = 0;\n for (let i = 0; i < float32Array.length; i++, offset += 2) {\n let s = Math.max(-1, Math.min(1, float32Array[i]));\n view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);\n }\n return buffer;\n}\n\n// Converts a Float32Array to base64-encoded PCM16 data\nbase64EncodeAudio(float32Array) {\n const arrayBuffer = floatTo16BitPCM(float32Array);\n let binary = '';\n let bytes = new Uint8Array(arrayBuffer);\n const chunkSize = 0x8000; // 32KB chunk size\n for (let i = 0; i < bytes.length; i += chunkSize) {\n let chunk = bytes.subarray(i, i + chunkSize);\n binary += String.fromCharCode.apply(null, chunk);\n }\n return btoa(binary);\n}\n\n// Fills the audio buffer with the contents of three files,\n// then asks the model to generate a response.\nconst files = [\n './path/to/sample1.wav',\n './path/to/sample2.wav',\n './path/to/sample3.wav'\n];\n\nfor (const filename of files) {\n const audioFile = fs.readFileSync(filename);\n const audioBuffer = await decodeAudio(audioFile);\n const channelData = audioBuffer.getChannelData(0);\n const base64Chunk = base64EncodeAudio(channelData);\n ws.send(JSON.stringify({\n type: 'input_audio_buffer.append',\n audio: base64Chunk\n }));\n});\n\nws.send(JSON.stringify({type: 'input_audio_buffer.commit'}));\nws.send(JSON.stringify({type: 'response.create'}));\n```\n\n```python\nimport base64\nimport json\nimport struct\nimport soundfile as sf\nfrom websocket import create_connection\n\n# ... create websocket-client named ws ...\n\ndef float_to_16bit_pcm(float32_array):\n clipped = [max(-1.0, min(1.0, x)) for x in float32_array]\n pcm16 = b''.join(struct.pack(' 1 else data\n base64_chunk = base64_encode_audio(channel_data)\n \n # Send the client event\n event = {\n \"type\": \"input_audio_buffer.append\",\n \"audio\": base64_chunk\n }\n ws.send(json.dumps(event))\n```\n\n### Send full audio messages\n\nIt is also possible to create conversation messages that are full audio recordings. Use the [`conversation.item.create`](/docs/api-reference/realtime-client-events/conversation/item/create) client event to create messages with `input_audio` content.\n\nCreate full audio input conversation items\n\n```javascript\nconst fullAudio = \"\";\n\nconst event = {\n type: \"conversation.item.create\",\n item: {\n type: \"message\",\n role: \"user\",\n content: [\n {\n type: \"input_audio\",\n audio: fullAudio,\n },\n ],\n },\n};\n\n// WebRTC data channel and WebSocket both have .send()\ndataChannel.send(JSON.stringify(event));\n```\n\n```python\nfullAudio = \"\"\n\nevent = {\n \"type\": \"conversation.item.create\",\n \"item\": {\n \"type\": \"message\",\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_audio\",\n \"audio\": fullAudio,\n }\n ],\n },\n}\n\nws.send(json.dumps(event))\n```\n\n### Working with audio output from a WebSocket\n\n**To play output audio back on a client device like a web browser, we recommend using WebRTC rather than WebSockets**. WebRTC will be more robust sending media to client devices over uncertain network conditions.\n\nBut to work with audio output in server-to-server applications using a WebSocket, you will need to listen for [`response.audio.delta`](/docs/api-reference/realtime-server-events/response/audio/delta) events containing the Base64-encoded chunks of audio data from the model. You will either need to buffer these chunks and write them out to a file, or maybe immediately stream them to another source like [a phone call with Twilio](https://www.twilio.com/en-us/blog/twilio-openai-realtime-api-launch-integration).\n\nNote that the [`response.audio.done`](/docs/api-reference/realtime-server-events/response/audio/done) and [`response.done`](/docs/api-reference/realtime-server-events/response/done) events won't actually contain audio data in them - just audio content transcriptions. To get the actual bytes, you'll need to listen for the [`response.audio.delta`](/docs/api-reference/realtime-server-events/response/audio/delta) events.\n\nThe format of the output chunks can be configured either for the entire session, or per response.\n\n* Session: `session.output_audio_format` in [`session.update`](/docs/api-reference/realtime-client-events/session/update)\n* Response: `response.output_audio_format` in [`response.create`](/docs/api-reference/realtime-client-events/response/create)\n\nListen for response.audio.delta events\n\n```javascript\nfunction handleEvent(e) {\n const serverEvent = JSON.parse(e.data);\n if (serverEvent.type === \"response.audio.delta\") {\n // Access Base64-encoded audio chunks\n // console.log(serverEvent.delta);\n }\n}\n\n// Listen for server messages (WebSocket)\nws.on(\"message\", handleEvent);\n```\n\n```python\ndef on_message(ws, message):\n server_event = json.loads(message)\n if server_event.type == \"response.audio.delta\":\n # Access Base64-encoded audio chunks:\n # print(server_event.delta)\n```\n\nVoice activity detection\n------------------------\n\nBy default, Realtime sessions have **voice activity detection (VAD)** enabled, which means the API will determine when the user has started or stopped speaking and respond automatically.\n\nRead more about how to configure VAD in our [voice activity detection](/docs/guides/realtime-vad) guide.\n\n### Disable VAD\n\nVAD can be disabled by setting `turn_detection` to `null` with the [`session.update`](/docs/api-reference/realtime-client-events/session/update) client event. This can be useful for interfaces where you would like to take granular control over audio input, like [push to talk](https://en.wikipedia.org/wiki/Push-to-talk) interfaces.\n\nWhen VAD is disabled, the client will have to manually emit some additional client events to trigger audio responses:\n\n* Manually send [`input_audio_buffer.commit`](/docs/api-reference/realtime-client-events/input_audio_buffer/commit), which will create a new user input item for the conversation.\n* Manually send [`response.create`](/docs/api-reference/realtime-client-events/response/create) to trigger an audio response from the model.\n* Send [`input_audio_buffer.clear`](/docs/api-reference/realtime-client-events/input_audio_buffer/clear) before beginning a new user input.\n\n### Keep VAD, but disable automatic responses\n\nIf you would like to keep VAD mode enabled, but would just like to retain the ability to manually decide when a response is generated, you can set `turn_detection.interrupt_response` and `turn_detection.create_response` to `false` with the [`session.update`](/docs/api-reference/realtime-client-events/session/update) client event. This will retain all the behavior of VAD but not automatically create new Responses. Clients can trigger these manually with a [`response.create`](/docs/api-reference/realtime-client-events/response/create) event.\n\nThis can be useful for moderation or input validation or RAG patterns, where you're comfortable trading a bit more latency in the interaction for control over inputs.\n\nCreate responses outside the default conversation\n-------------------------------------------------\n\nBy default, all responses generated during a session are added to the session's conversation state (the \"default conversation\"). However, you may want to generate model responses outside the context of the session's default conversation, or have multiple responses generated concurrently. You might also want to have more granular control over which conversation items are considered while the model generates a response (e.g. only the last N number of turns).\n\nGenerating \"out-of-band\" responses which are not added to the default conversation state is possible by setting the `response.conversation` field to the string `none` when creating a response with the [`response.create`](/docs/api-reference/realtime-client-events/response/create) client event.\n\nWhen creating an out-of-band response, you will probably also want some way to identify which server-sent events pertain to this response. You can provide `metadata` for your model response that will help you identify which response is being generated for this client-sent event.\n\nCreate an out-of-band model response\n\n```javascript\nconst prompt = `\nAnalyze the conversation so far. If it is related to support, output\n\"support\". If it is related to sales, output \"sales\".\n`;\n\nconst event = {\n type: \"response.create\",\n response: {\n // Setting to \"none\" indicates the response is out of band\n // and will not be added to the default conversation\n conversation: \"none\",\n\n // Set metadata to help identify responses sent back from the model\n metadata: { topic: \"classification\" },\n \n // Set any other available response fields\n modalities: [ \"text\" ],\n instructions: prompt,\n },\n};\n\n// WebRTC data channel and WebSocket both have .send()\ndataChannel.send(JSON.stringify(event));\n```\n\n```python\nprompt = \"\"\"\nAnalyze the conversation so far. If it is related to support, output\n\"support\". If it is related to sales, output \"sales\".\n\"\"\"\n\nevent = {\n \"type\": \"response.create\",\n \"response\": {\n # Setting to \"none\" indicates the response is out of band,\n # and will not be added to the default conversation\n \"conversation\": \"none\",\n\n # Set metadata to help identify responses sent back from the model\n \"metadata\": { \"topic\": \"classification\" },\n\n # Set any other available response fields\n \"modalities\": [ \"text\" ],\n \"instructions\": prompt,\n },\n}\n\nws.send(json.dumps(event))\n```\n\nNow, when you listen for the [`response.done`](/docs/api-reference/realtime-server-events/response/done) server event, you can identify the result of your out-of-band response.\n\nCreate an out-of-band model response\n\n```javascript\nfunction handleEvent(e) {\n const serverEvent = JSON.parse(e.data);\n if (\n serverEvent.type === \"response.done\" &&\n serverEvent.response.metadata?.topic === \"classification\"\n ) {\n // this server event pertained to our OOB model response\n console.log(serverEvent.response.output[0]);\n }\n}\n\n// Listen for server messages (WebRTC)\ndataChannel.addEventListener(\"message\", handleEvent);\n\n// Listen for server messages (WebSocket)\n// ws.on(\"message\", handleEvent);\n```\n\n```python\ndef on_message(ws, message):\n server_event = json.loads(message)\n topic = \"\"\n\n # See if metadata is present\n try:\n topic = server_event.response.metadata.topic\n except AttributeError:\n print(\"topic not set\")\n \n if server_event.type == \"response.done\" and topic == \"classification\":\n # this server event pertained to our OOB model response\n print(server_event.response.output[0])\n```\n\n### Create a custom context for responses\n\nYou can also construct a custom context that the model will use to generate a response, outside the default/current conversation. This can be done using the `input` array on a [`response.create`](/docs/api-reference/realtime-client-events/response/create) client event. You can use new inputs, or reference existing input items in the conversation by ID.\n\nListen for out-of-band model response with custom context\n\n```javascript\nconst event = {\n type: \"response.create\",\n response: {\n conversation: \"none\",\n metadata: { topic: \"pizza\" },\n modalities: [ \"text\" ],\n\n // Create a custom input array for this request with whatever context\n // is appropriate\n input: [\n // potentially include existing conversation items:\n {\n type: \"item_reference\",\n id: \"some_conversation_item_id\"\n },\n {\n type: \"message\",\n role: \"user\",\n content: [\n {\n type: \"input_text\",\n text: \"Is it okay to put pineapple on pizza?\",\n },\n ],\n },\n ],\n },\n};\n\n// WebRTC data channel and WebSocket both have .send()\ndataChannel.send(JSON.stringify(event));\n```\n\n```python\nevent = {\n \"type\": \"response.create\",\n \"response\": {\n \"conversation\": \"none\",\n \"metadata\": { \"topic\": \"pizza\" },\n \"modalities\": [ \"text\" ],\n\n # Create a custom input array for this request with whatever \n # context is appropriate\n \"input\": [\n # potentially include existing conversation items:\n {\n \"type\": \"item_reference\",\n \"id\": \"some_conversation_item_id\"\n },\n\n # include new content as well\n {\n \"type\": \"message\",\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"Is it okay to put pineapple on pizza?\",\n }\n ],\n }\n ],\n },\n}\n\nws.send(json.dumps(event))\n```\n\n### Create responses with no context\n\nYou can also insert responses into the default conversation, ignoring all other instructions and context. Do this by setting `input` to an empty array.\n\nInsert no-context model responses into the default conversation\n\n```javascript\nconst prompt = `\nSay exactly the following:\nI'm a little teapot, short and stout! \nThis is my handle, this is my spout!\n`;\n\nconst event = {\n type: \"response.create\",\n response: {\n // An empty input array removes existing context\n input: [],\n instructions: prompt,\n },\n};\n\n// WebRTC data channel and WebSocket both have .send()\ndataChannel.send(JSON.stringify(event));\n```\n\n```python\nprompt = \"\"\"\nSay exactly the following:\nI'm a little teapot, short and stout! \nThis is my handle, this is my spout!\n\"\"\"\n\nevent = {\n \"type\": \"response.create\",\n \"response\": {\n # An empty input array removes all prior context\n \"input\": [],\n \"instructions\": prompt,\n },\n}\n\nws.send(json.dumps(event))\n```\n\nFunction calling\n----------------\n\nThe Realtime models also support **function calling**, which enables you to execute custom code to extend the capabilities of the model. Here's how it works at a high level:\n\n1. When [updating the session](/docs/api-reference/realtime-client-events/session/update) or [creating a response](/docs/api-reference/realtime-client-events/response/create), you can specify a list of available functions for the model to call.\n2. If when processing input, the model determines it should make a function call, it will add items to the conversation representing arguments to a function call.\n3. When the client detects conversation items that contain function call arguments, it will execute custom code using those arguments\n4. When the custom code has been executed, the client will create new conversation items that contain the output of the function call, and ask the model to respond.\n\nLet's see how this would work in practice by adding a callable function that will provide today's horoscope to users of the model. We'll show the shape of the client event objects that need to be sent, and what the server will emit in turn.\n\n### Configure callable functions\n\nFirst, we must give the model a selection of functions it can call based on user input. Available functions can be configured either at the session level, or the individual response level.\n\n* Session: `session.tools` property in [`session.update`](/docs/api-reference/realtime-client-events/session/update)\n* Response: `response.tools` property in [`response.create`](/docs/api-reference/realtime-client-events/response/create)\n\nHere's an example client event payload for a `session.update` that configures a horoscope generation function, that takes a single argument (the astrological sign for which the horoscope should be generated):\n\n[`session.update`](/docs/api-reference/realtime-client-events/session/update)\n\n```json\n{\n \"type\": \"session.update\",\n \"session\": {\n \"tools\": [\n {\n \"type\": \"function\",\n \"name\": \"generate_horoscope\",\n \"description\": \"Give today's horoscope for an astrological sign.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"sign\": {\n \"type\": \"string\",\n \"description\": \"The sign for the horoscope.\",\n \"enum\": [\n \"Aries\",\n \"Taurus\",\n \"Gemini\",\n \"Cancer\",\n \"Leo\",\n \"Virgo\",\n \"Libra\",\n \"Scorpio\",\n \"Sagittarius\",\n \"Capricorn\",\n \"Aquarius\",\n \"Pisces\"\n ]\n }\n },\n \"required\": [\"sign\"]\n }\n }\n ],\n \"tool_choice\": \"auto\",\n }\n}\n```\n\nThe `description` fields for the function and the parameters help the model choose whether or not to call the function, and what data to include in each parameter. If the model receives input that indicates the user wants their horoscope, it will call this function with a `sign` parameter.\n\n### Detect when the model wants to call a function\n\nBased on inputs to the model, the model may decide to call a function in order to generate the best response. Let's say our application adds the following conversation item and attempts to generate a response:\n\n[`conversation.item.create`](/docs/api-reference/realtime-client-events/conversation/item/create)\n\n```json\n{\n \"type\": \"conversation.item.create\",\n \"item\": {\n \"type\": \"message\",\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"What is my horoscope? I am an aquarius.\"\n }\n ]\n }\n}\n```\n\nFollowed by a client event to generate a response:\n\n[`response.create`](/docs/api-reference/realtime-client-events/response/create)\n\n```json\n{\n \"type\": \"response.create\"\n}\n```\n\nInstead of immediately returning a text or audio response, the model will instead generate a response that contains the arguments that should be passed to a function in the developer's application. You can listen for realtime updates to function call arguments using the [`response.function_call_arguments.delta`](/docs/api-reference/realtime-server-events/response/function_call_arguments/delta) server event, but `response.done` will also have the complete data we need to call our function.\n\n[`response.done`](/docs/api-reference/realtime-server-events/response/done)\n\n```json\n{\n \"type\": \"response.done\",\n \"event_id\": \"event_AeqLA8iR6FK20L4XZs2P6\",\n \"response\": {\n \"object\": \"realtime.response\",\n \"id\": \"resp_AeqL8XwMUOri9OhcQJIu9\",\n \"status\": \"completed\",\n \"status_details\": null,\n \"output\": [\n {\n \"object\": \"realtime.item\",\n \"id\": \"item_AeqL8gmRWDn9bIsUM2T35\",\n \"type\": \"function_call\",\n \"status\": \"completed\",\n \"name\": \"generate_horoscope\",\n \"call_id\": \"call_sHlR7iaFwQ2YQOqm\",\n \"arguments\": \"{\\\"sign\\\":\\\"Aquarius\\\"}\"\n }\n ],\n \"usage\": {\n \"total_tokens\": 541,\n \"input_tokens\": 521,\n \"output_tokens\": 20,\n \"input_token_details\": {\n \"text_tokens\": 292,\n \"audio_tokens\": 229,\n \"cached_tokens\": 0,\n \"cached_tokens_details\": { \"text_tokens\": 0, \"audio_tokens\": 0 }\n },\n \"output_token_details\": {\n \"text_tokens\": 20,\n \"audio_tokens\": 0\n }\n },\n \"metadata\": null\n }\n}\n```\n\nIn the JSON emitted by the server, we can detect that the model wants to call a custom function:\n\n|Property|Function calling purpose|\n|---|---|\n|response.output[0].type|When set to function_call, indicates this response contains arguments for a named function call.|\n|response.output[0].name|The name of the configured function to call, in this case generate_horoscope|\n|response.output[0].arguments|A JSON string containing arguments to the function. In our case, \"{\\\"sign\\\":\\\"Aquarius\\\"}\".|\n|response.output[0].call_id|A system-generated ID for this function call - you will need this ID to pass a function call result back to the model.|\n\nGiven this information, we can execute code in our application to generate the horoscope, and then provide that information back to the model so it can generate a response.\n\n### Provide the results of a function call to the model\n\nUpon receiving a response from the model with arguments to a function call, your application can execute code that satisfies the function call. This could be anything you want, like talking to external APIs or accessing databases.\n\nOnce you are ready to give the model the results of your custom code, you can create a new conversation item containing the result via the `conversation.item.create` client event.\n\n[`conversation.item.create`](/docs/api-reference/realtime-client-events/conversation/item/create)\n\n```json\n{\n \"type\": \"conversation.item.create\",\n \"item\": {\n \"type\": \"function_call_output\",\n \"call_id\": \"call_sHlR7iaFwQ2YQOqm\",\n \"output\": \"{\\\"horoscope\\\": \\\"You will soon meet a new friend.\\\"}\"\n }\n}\n```\n\n* The conversation item type is `function_call_output`\n* `item.call_id` is the same ID we got back in the `response.done` event above\n* `item.output` is a JSON string containing the results of our function call\n\nOnce we have added the conversation item containing our function call results, we again emit the `response.create` event from the client. This will trigger a model response using the data from the function call.\n\n[`response.create`](/docs/api-reference/realtime-client-events/response/create)\n\n```json\n{\n \"type\": \"response.create\"\n}\n```\n\nError handling\n--------------\n\nThe [`error`](/docs/api-reference/realtime-server-events/error) event is emitted by the server whenever an error condition is encountered on the server during the session. Occasionally, these errors can be traced to a client event that was emitted by your application.\n\nUnlike HTTP requests and responses, where a response is implicitly tied to a request from the client, we need to use an `event_id` property on client events to know when one of them has triggered an error condition on the server. This technique is shown in the code below, where the client attempts to emit an unsupported event type.\n\n```javascript\nconst event = {\n event_id: \"my_awesome_event\",\n type: \"scooby.dooby.doo\",\n};\n\ndataChannel.send(JSON.stringify(event));\n```\n\nThis unsuccessful event sent from the client will emit an error event like the following:\n\n```json\n{\n \"type\": \"invalid_request_error\",\n \"code\": \"invalid_value\",\n \"message\": \"Invalid value: 'scooby.dooby.doo' ...\",\n \"param\": \"type\",\n \"event_id\": \"my_awesome_event\"\n}\n```\n\nWas this page useful?",[],[208,318,376],"38edc6ee-8935-11f0-98af-0224a6c84d84",["D",1756952212978],[233,234,235,236,237,238,239,240,241],[208,318],["D",1756952212884],[232],[208],"390439f6-8935-11f0-939c-0224a6c84d84","settings.local.json","39043ac8-8935-11f0-939c-0224a6c84d84",24,["D",1756952213071],"{\n \"permissions\": {\n \"allow\": [\n \"Read(//Users/emcho/devel/juberti-demos/realtime/**)\"\n ],\n \"deny\": [],\n \"ask\": []\n }\n}",[],[214,391],["D",1756952212789],[230],[214],["D",1756839724543],{"_100":99,"_107":103},{"_100":99,"_101":-5,"_102":103,"_18":55,"_104":105,"_106":25},{"_13":86,"_32":87,"_44":43,"_88":45,"_89":90,"_91":92,"_93":51,"_94":95,"_96":65,"_97":46,"_98":99},"ctx",{"_79":-5,"_129":438,"_439":440,"_441":442,"_443":25,"_444":25,"_445":25,"_446":59,"_447":-5,"_448":-7,"_449":-7,"_450":63,"_451":25,"_452":25,"_453":-5,"_454":25,"_455":25,"_456":25,"_457":-5},"ENV",{"_409":410,"_411":412,"_413":87,"_414":415,"_416":417,"_418":419,"_420":421,"_422":423,"_424":425,"_426":427,"_428":429,"_430":431,"_432":433,"_434":435,"_436":437},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v41