📝 OpenAI Realtime Transcription

\n Transcribe audio in real-time using OpenAI's Realtime API. \n View source\n

\n\n

\n Model\n \n

\n \n

\n Language\n \n

\n \n

\n VAD\n \n

\n \n

\n \n \n \n

\n\n \n\n \n\n",[],[280,287],[280],{"_102":297,"_95":298,"_154":299,"_156":-5,"_157":136,"_158":282,"_160":161,"_104":300,"_99":32,"_163":164,"_165":-5,"_166":22,"_167":-5,"_168":301,"_98":320},"50a3e18a-882f-11f0-9954-0224a6c84d84","routes","01990bce-c638-714e-97a4-1048c460355e",["D",1756839724543],[302,311],{"_102":303,"_95":304,"_154":305,"_156":299,"_157":136,"_158":135,"_160":161,"_104":306,"_99":291,"_163":307,"_165":308,"_166":22,"_167":-5,"_168":309,"_98":310},"f3f05396-8a37-11f0-a608-0224a6c84d84","rtc.ts","01990bce-c638-714e-97a4-07196c7003e4",["D",1757063336876],"script","import { Hono } from \"npm:hono\";\nimport { makeHeaders, makeTranscriptionSession } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new transcription call\nrtc.post(\"/\", async (c) => {\n // Get transcription configuration from query params\n const sessionConfig = makeTranscriptionSession({\n model: c.req.query(\"model\") as any || \"gpt-4o-transcribe\",\n language: c.req.query(\"language\") || \"en\",\n enableVAD: c.req.query(\"vad\") !== \"false\",\n });\n\n const realtimeUrl = \"https://api.openai.com/v1/realtime/calls\";\n const formData = new FormData();\n formData.set(\"sdp\", await c.req.text());\n formData.set(\"session\", JSON.stringify(sessionConfig));\n const sdpResp = await fetch(realtimeUrl, {\n method: \"POST\",\n headers: makeHeaders(),\n body: formData,\n });\n\n if (!sdpResp.ok) {\n const errText = await sdpResp.text().catch(() => \"\");\n console.error(\n `🔴 Failed to establish WebRTC connection: ${sdpResp.status} ${errText}`,\n );\n return c.text(\"Internal error\", 500);\n }\n\n return c.newResponse(sdpResp.body, {\n headers: { \"Content-Type\": sdpResp.headers.get(\"Content-Type\") },\n });\n});\n\nexport default rtc;",[],[298,304],{"_102":312,"_95":313,"_154":314,"_156":299,"_157":136,"_158":135,"_160":161,"_104":315,"_99":316,"_163":307,"_165":317,"_166":22,"_167":-5,"_168":318,"_98":319},"f3f05968-8a37-11f0-a608-0224a6c84d84","utils.ts","01990bce-c638-714e-97a4-0caf2e9a1c1c",["D",1757063336876],11,"const OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\nexport interface TranscriptionConfig {\n model?: \"gpt-4o-transcribe\" | \"gpt-4o-mini-transcribe\" | \"whisper-1\";\n language?: string;\n enableVAD?: boolean;\n}\n\nexport function makeTranscriptionSession(config: TranscriptionConfig = {}) {\n const {\n model = \"gpt-4o-transcribe\",\n language = \"en\",\n enableVAD = true,\n } = config;\n\n // Create transcription session config (based on Justin Uberti's working implementation)\n return {\n type: \"transcription\",\n audio: {\n input: {\n transcription: { model, language },\n turn_detection: enableVAD ? { type: \"server_vad\" } : null,\n },\n },\n };\n}",[],[298,313],[298],{"_102":322,"_95":323,"_154":324,"_156":-5,"_157":136,"_158":325,"_160":161,"_104":326,"_99":32,"_163":176,"_165":327,"_166":22,"_167":-5,"_168":328,"_98":329},"39792548-8937-11f0-82b5-0224a6c84d84",".gitattributes","397925fc-8937-11f0-82b5-0224a6c84d84",39,["D",1756953072887],"# Auto detect text files and perform LF normalization\n* text=auto\n",[],[323],{"_102":331,"_95":332,"_154":333,"_156":-5,"_157":136,"_158":282,"_160":161,"_104":334,"_99":32,"_163":176,"_165":335,"_166":22,"_167":-5,"_168":336,"_98":337},"50a3e2c0-882f-11f0-9954-0224a6c84d84",".vtignore","01990bce-c638-714e-97a4-157f5fb8f430",["D",1756839724543],".git\n.vscode\n.cursorrules\n.DS_Store\nnode_modules\nvendor",[],[332],{"_102":339,"_95":340,"_154":341,"_156":-5,"_157":136,"_158":342,"_160":161,"_104":343,"_99":32,"_163":176,"_165":344,"_166":22,"_167":-5,"_168":345,"_98":346},"1a3463bc-8830-11f0-a44c-0224a6c84d84","CLAUDE.md","1a34648e-8830-11f0-a44c-0224a6c84d84",2,["D",1756840062786],"# Hello-Transcription - OpenAI Realtime API Transcription Demo\n\n## 🎯 Project Overview\n\nHello-Transcription demonstrates the transcription-only mode of OpenAI's Realtime API. Unlike the conversational mode, this implementation focuses purely on speech-to-text conversion without generating AI responses, making it ideal for subtitles, live captions, meeting transcriptions, and other transcription-focused use cases.\n\n**Created:** September 2, 2025 \n**Platform:** Val Town \n**API:** OpenAI Realtime API (Transcription Mode) \n**Key Feature:** Real-time streaming transcription with multiple model support\n\n## 🏗️ Technical Stack\n\n- **Runtime:** Deno (Val Town platform)\n- **Framework:** Hono (lightweight web framework)\n- **Transcription:** OpenAI Realtime API in transcription mode\n- **Connection:** WebRTC with data channel for events\n- **Frontend:** Vanilla JavaScript with split-view interface\n- **Models:** GPT-4o Transcribe, GPT-4o Mini Transcribe, Whisper-1\n\n## 📁 Project Structure\n\n```\nhello-transcription/\n├── frontend/\n│ └── index.html # Split-view transcription interface\n├── routes/\n│ ├── rtc.ts # WebRTC session setup for transcription\n│ ├── observer.ts # WebSocket observer for transcription events\n│ └── utils.ts # Transcription session configuration\n├── main.tsx # Main entry point\n├── deno.json # Deno configuration\n├── README.md # User documentation\n└── CLAUDE.md # This file - technical documentation\n```\n\n## 🔑 Core Concepts\n\n### Transcription vs Conversation Mode\n\nThe Realtime API supports two distinct modes:\n\n1. **Conversation Mode** (`type: \"realtime\"`):\n - Two-way interaction with AI\n - User speaks → AI responds\n - Used in hello-realtime and hello-mcp\n\n2. **Transcription Mode** (`type: \"transcription\"`):\n - One-way speech-to-text only\n - User speaks → Text output\n - No AI responses generated\n - Lower latency, lower cost\n - This demo uses transcription mode\n\n### Transcription Session Object\n\n```typescript\n{\n type: \"transcription\",\n input_audio_format: \"pcm16\",\n input_audio_transcription: {\n model: \"gpt-4o-transcribe\", // or \"gpt-4o-mini-transcribe\", \"whisper-1\"\n prompt: \"\", // Optional context hint\n language: \"en\" // ISO-639-1 language code\n },\n turn_detection: {\n type: \"server_vad\",\n threshold: 0.5,\n prefix_padding_ms: 300,\n silence_duration_ms: 500\n },\n input_audio_noise_reduction: {\n type: \"near_field\" // or \"far_field\", null\n },\n include: [\"item.input_audio_transcription.logprobs\"] // Optional\n}\n```\n\n## 🛠️ Key Components\n\n### 1. Utils Configuration (`/routes/utils.ts`)\n\nHandles transcription session configuration with sensible defaults:\n\n```typescript\nexport function makeTranscriptionSession(config: TranscriptionConfig = {}) {\n const {\n model = \"gpt-4o-transcribe\",\n language = \"en\",\n prompt = \"\",\n enableVAD = true,\n noiseReduction = \"near_field\",\n includeLogprobs = false\n } = config;\n \n // Build session object...\n}\n```\n\n**Key Configuration Options:**\n- **model**: Transcription model selection\n- **language**: Primary language for better accuracy\n- **prompt**: Context hints (e.g., \"Expect medical terminology\")\n- **enableVAD**: Automatic voice activity detection\n- **noiseReduction**: Audio preprocessing type\n- **includeLogprobs**: Confidence scores for words\n\n### 2. RTC Route Handler (`/routes/rtc.ts`)\n\nCreates WebRTC sessions specifically for transcription:\n\n```typescript\n// Get config from query params\nconst model = c.req.query(\"model\") || \"gpt-4o-transcribe\";\nconst language = c.req.query(\"language\") || \"en\";\nconst vad = c.req.query(\"vad\") !== \"false\";\nconst logprobs = c.req.query(\"logprobs\") === \"true\";\n\n// Create transcription session\nconst sessionConfig = makeTranscriptionSession({\n model,\n language,\n enableVAD: vad,\n includeLogprobs: logprobs\n});\n```\n\n**Important:** Uses `type: \"transcription\"` not `type: \"realtime\"`\n\n### 3. Observer WebSocket (`/routes/observer.ts`)\n\nMonitors transcription events via server-side WebSocket:\n\n```typescript\nws.onmessage = (event) => {\n const data = JSON.parse(event.data);\n \n if (data.type === \"conversation.item.input_audio_transcription.delta\") {\n // Streaming partial transcription\n console.log(`📝 Transcription delta: \"${data.delta}\"`);\n } else if (data.type === \"conversation.item.input_audio_transcription.completed\") {\n // Final transcription for segment\n console.log(`✅ Transcription completed: \"${data.transcript}\"`);\n }\n};\n```\n\n### 4. Frontend Interface (`/frontend/index.html`)\n\nSplit-view interface with real-time transcription display:\n\n#### Left Panel - Transcriptions\n- Shows live transcription stream\n- Partial transcriptions update in real-time\n- Final transcriptions marked with green border\n- Each segment timestamped\n\n#### Right Panel - Event Logs\n- Technical event stream\n- Debug information\n- Connection status\n\n#### Data Channel Handling\n```javascript\ndataChannel.onmessage = (event) => {\n const data = JSON.parse(event.data);\n \n if (data.type === \"conversation.item.input_audio_transcription.delta\") {\n // Update partial transcription\n addTranscript(data.item_id, data.delta, false);\n } else if (data.type === \"conversation.item.input_audio_transcription.completed\") {\n // Mark transcription as final\n addTranscript(data.item_id, data.transcript, true);\n }\n};\n```\n\n## 📊 Model Comparison\n\n### GPT-4o Transcribe\n- **Streaming:** Yes - incremental updates via delta events\n- **Latency:** Low\n- **Accuracy:** High\n- **Use Case:** Live subtitles, real-time captions\n\n### GPT-4o Mini Transcribe\n- **Streaming:** Yes - incremental updates\n- **Latency:** Very low\n- **Accuracy:** Good\n- **Use Case:** Quick transcriptions, lower cost\n\n### Whisper-1\n- **Streaming:** No - complete segments only\n- **Latency:** Higher (waits for complete utterance)\n- **Accuracy:** Very high\n- **Use Case:** High-accuracy transcriptions, post-processing\n\n## 🔄 Event Flow\n\n### Transcription Event Sequence\n\n1. **Audio Input**\n ```\n User speaks → Microphone → WebRTC → OpenAI\n ```\n\n2. **VAD Processing** (if enabled)\n ```\n Voice detected → Buffer audio → Silence detected → Commit buffer\n ```\n\n3. **Transcription Events**\n ```\n input_audio_buffer.committed\n ↓\n conversation.item.input_audio_transcription.delta (streaming models)\n ↓\n conversation.item.input_audio_transcription.completed\n ```\n\n### Event Types\n\n#### Delta Event (Streaming)\n```json\n{\n \"type\": \"conversation.item.input_audio_transcription.delta\",\n \"item_id\": \"item_003\",\n \"content_index\": 0,\n \"delta\": \"Hello, how\"\n}\n```\n\n#### Completed Event\n```json\n{\n \"type\": \"conversation.item.input_audio_transcription.completed\",\n \"item_id\": \"item_003\",\n \"content_index\": 0,\n \"transcript\": \"Hello, how are you today?\"\n}\n```\n\n## ⚙️ Configuration Details\n\n### Voice Activity Detection (VAD)\n\nVAD automatically detects speech segments:\n\n```typescript\nturn_detection: {\n type: \"server_vad\",\n threshold: 0.5, // Sensitivity (0-1)\n prefix_padding_ms: 300, // Audio before speech\n silence_duration_ms: 500 // Silence to end segment\n}\n```\n\n**VAD Disabled:**\n```typescript\nturn_detection: null // Manual control required\n```\n\n### Noise Reduction\n\nThree noise reduction modes:\n\n1. **near_field**: Optimized for close microphones (default)\n2. **far_field**: For distant microphones/speakers\n3. **null**: No noise reduction\n\n### Language Configuration\n\nISO-639-1 language codes improve accuracy:\n- \"en\" - English\n- \"es\" - Spanish \n- \"fr\" - French\n- \"de\" - German\n- \"zh\" - Chinese\n- \"ja\" - Japanese\n\n### Logprobs (Confidence Scores)\n\nWhen enabled, provides word-level confidence:\n\n```typescript\ninclude: [\"item.input_audio_transcription.logprobs\"]\n```\n\nReturns probability scores for each transcribed word, useful for:\n- Highlighting uncertain words\n- Quality assessment\n- Post-processing decisions\n\n## 🧪 Testing Guide\n\n### Local Development\n\n1. **Setup Environment**\n ```bash\n # Create .env file\n echo \"OPENAI_API_KEY=sk-...\" > .env\n \n # Install Deno\n curl -fsSL https://deno.land/install.sh | sh\n ```\n\n2. **Run Development Server**\n ```bash\n # With auto-reload\n deno run --watch --allow-all main.tsx\n \n # Or standard\n deno run --allow-all main.tsx\n ```\n\n3. **Test Transcription**\n - Open http://localhost:8000\n - Select model and language\n - Click \"Start\"\n - Speak clearly\n - Watch transcriptions appear\n\n### Testing Different Models\n\n1. **Test Streaming (GPT-4o)**\n - Select \"GPT-4o Transcribe\"\n - Speak continuously\n - Notice incremental updates\n\n2. **Test Non-Streaming (Whisper-1)**\n - Select \"Whisper-1\"\n - Speak, then pause\n - Notice complete segments only\n\n3. **Test VAD**\n - Enable VAD\n - Speak with pauses\n - Notice automatic segmentation\n\n4. **Test Without VAD**\n - Disable VAD\n - Requires manual commit (not implemented in this demo)\n\n## 🐛 Common Issues & Solutions\n\n### Issue: No transcriptions appearing\n**Solutions:**\n- Check microphone permissions\n- Verify OPENAI_API_KEY is set\n- Check browser console for errors\n- Ensure WebRTC connection established\n\n### Issue: Transcriptions cut off mid-sentence\n**Solutions:**\n- Adjust VAD silence_duration_ms (longer = fewer cuts)\n- Try different VAD threshold\n- Consider using different model\n\n### Issue: Poor transcription accuracy\n**Solutions:**\n- Set correct language parameter\n- Use appropriate noise reduction setting\n- Provide context via prompt parameter\n- Try higher accuracy model (Whisper-1)\n\n### Issue: High latency\n**Solutions:**\n- Use GPT-4o Mini for lower latency\n- Check network connection\n- Consider disabling logprobs\n\n## 📈 Performance Characteristics\n\n### Latency Comparison\n- **GPT-4o Mini**: ~100-200ms first token\n- **GPT-4o**: ~150-250ms first token \n- **Whisper-1**: ~500-1000ms (full segment)\n\n### Throughput\n- Handles real-time audio (16kHz PCM16)\n- Multiple concurrent sessions supported\n- No buffering required for streaming models\n\n### Cost Optimization\n- Transcription-only mode is cheaper than conversation mode\n- No AI inference costs\n- GPT-4o Mini most cost-effective\n- Whisper-1 for batch/accuracy needs\n\n## 🔒 Security Considerations\n\n### Current Implementation\n- API key stored in environment variable\n- No authentication on endpoints\n- No rate limiting\n- Single-tenant design\n\n### Production Recommendations\n1. **Authentication**: Add user authentication\n2. **Rate Limiting**: Implement per-user limits\n3. **CORS**: Configure appropriate origins\n4. **Monitoring**: Track usage and errors\n5. **Encryption**: Ensure HTTPS only\n6. **Token Security**: Use short-lived tokens\n\n## 🚀 Deployment\n\n### Val Town Deployment\n\n1. **Create/Remix Val**\n ```bash\n vt remix emcho/hello-transcription my-transcription\n ```\n\n2. **Set Environment**\n - Add `OPENAI_API_KEY` in Val Town secrets\n\n3. **Deploy**\n ```bash\n vt push\n ```\n\n4. **Access**\n - URL: `https://[your-val-name].val.run`\n\n### Environment Variables\n- `OPENAI_API_KEY` - Required for OpenAI API access\n\n## 📝 Future Enhancements\n\n### Potential Features\n\n1. **Recording & Export**\n - Save transcriptions to file\n - Export as SRT/VTT subtitles\n - Download audio recordings\n\n2. **Advanced Controls**\n - Manual VAD control\n - Custom VAD parameters UI\n - Prompt templates\n\n3. **Multi-Stream**\n - Multiple speaker support\n - Speaker diarization\n - Parallel transcriptions\n\n4. **Post-Processing**\n - Punctuation enhancement\n - Grammar correction\n - Translation\n\n5. **Visualization**\n - Audio waveform display\n - VAD activity indicator\n - Confidence heat map\n\n6. **Integration**\n - Webhook support\n - Real-time API streaming\n - Database storage\n\n## 🔗 References\n\n### Documentation\n- [OpenAI Realtime Transcription Guide](https://platform.openai.com/docs/guides/realtime-transcription)\n- [Realtime API Reference](https://platform.openai.com/docs/api-reference/realtime)\n- [Voice Activity Detection Guide](https://platform.openai.com/docs/guides/realtime-vad)\n- [Val Town Documentation](https://docs.val.town)\n\n### Related Projects\n- [hello-realtime](https://hello-realtime.val.run) - Conversation mode demo\n- [hello-mcp](https://github.com/hello-mcp) - MCP tool execution demo\n\n### Key Differences from Conversation Mode\n1. Session type: `transcription` vs `realtime`\n2. No AI responses generated\n3. Different event types\n4. Lower latency and cost\n5. Focused on speech-to-text only\n\n## 💡 Implementation Notes\n\n### Critical Discoveries\n1. **Session Type**: Must use `type: \"transcription\"` not `type: \"realtime\"`\n2. **Event Names**: Different from conversation mode events\n3. **Model Behavior**: Whisper doesn't stream, GPT-4o models do\n4. **VAD Impact**: Significantly affects transcription segmentation\n5. **Language Setting**: Dramatically improves accuracy for non-English\n\n### Best Practices\n1. Always specify language for non-English content\n2. Use near_field noise reduction for headset mics\n3. Enable VAD for natural speech segmentation\n4. Choose model based on latency vs accuracy needs\n5. Monitor item_id for proper segment ordering\n\n## 🎯 Summary\n\nHello-Transcription successfully demonstrates the transcription-only capabilities of OpenAI's Realtime API. Key achievements:\n\n1. **Pure Transcription**: No AI responses, focused solely on speech-to-text\n2. **Model Flexibility**: Support for three different transcription models\n3. **Real-time Streaming**: Live transcription updates for supported models\n4. **Configuration Options**: VAD, noise reduction, language, logprobs\n5. **Clean Interface**: Split-view design for transcriptions and logs\n\nThis implementation serves as a foundation for building transcription-focused applications like live captioning, meeting transcription, subtitle generation, and accessibility tools.",[],[340],{"_102":348,"_95":349,"_154":350,"_156":-5,"_157":136,"_158":351,"_160":161,"_104":352,"_99":282,"_163":176,"_165":353,"_166":22,"_167":-5,"_168":354,"_98":355},"1a739988-8830-11f0-a821-0224a6c84d84","README.md","01990bce-c638-714e-97a4-1a67f3821e53",10,["D",1756840062842],"# hello-transcription\n\nReal-time speech transcription using OpenAI's Realtime API - a demonstration of transcription-only mode without AI responses.\n\n## Features\n\n- **Real-time Transcription**: Live speech-to-text conversion as you speak\n- **Multiple Models**: Choose between GPT-4o Transcribe, GPT-4o Mini Transcribe, or Whisper-1\n- **Language Support**: Transcribe in multiple languages (English, Spanish, French, German, Italian, Portuguese, Chinese, Japanese, Korean)\n- **Voice Activity Detection (VAD)**: Automatic detection of speech segments\n- **Logprobs Support**: Optional confidence scores for transcriptions\n- **Split View**: See transcriptions and event logs side-by-side\n\n## How It Works\n\nThis app uses OpenAI's Realtime API in transcription-only mode:\n1. Your voice is captured via WebRTC\n2. Audio is streamed to OpenAI's transcription service\n3. Transcriptions are returned in real-time\n4. No AI responses are generated (transcription only)\n\n## Model Differences\n\n- **GPT-4o Transcribe**: Streaming transcription with incremental updates\n- **GPT-4o Mini Transcribe**: Smaller model, streaming transcription \n- **Whisper-1**: Complete transcription after each speech segment (no streaming)\n\n## Configuration Options\n\n- **Model**: Select the transcription model\n- **Language**: Choose the primary language for better accuracy\n- **VAD**: Enable/disable automatic voice activity detection\n- **Logprobs**: Include confidence scores (for advanced use)\n\n## Usage\n\n1. Click \"Start\" to begin transcription\n2. Allow microphone access when prompted\n3. Start speaking - transcriptions appear in real-time\n4. Partial transcriptions update as you speak\n5. Final transcriptions are marked in green\n6. Click \"Stop\" to end the session\n\n## API Endpoints\n\n- `GET /` - Serves the transcription interface\n- `POST /rtc` - Creates a WebRTC transcription session\n- `POST /observer/:callId` - WebSocket observer for transcription events\n\n## Environment Variables\n\nSet in your Val Town environment:\n- `OPENAI_API_KEY` - Your OpenAI API key (required)\n\n## Local Development\n\n```bash\n# Install Deno\ncurl -fsSL https://deno.land/install.sh | sh\n\n# Run locally\ndeno run --allow-all main.tsx\n```\n\n## Val Town Deployment\n\n1. Fork/remix this val on Val Town\n2. Add your `OPENAI_API_KEY` to Val Town secrets\n3. Your app will be available at `https://[your-val-name].val.run`\n\n## Technical Details\n\nThe app uses OpenAI's Realtime API in transcription mode:\n- Session type: `transcription` (not `realtime`)\n- Audio format: PCM16\n- Noise reduction: Near-field (optimized for close microphones)\n- WebRTC data channel for receiving transcription events\n\n## Event Flow\n\n1. `input_audio_buffer.committed` - Audio chunk received\n2. `conversation.item.input_audio_transcription.delta` - Partial transcription\n3. `conversation.item.input_audio_transcription.completed` - Final transcription\n\n## Credits\n\nBuilt with OpenAI's Realtime API for transcription-only use cases.",[],[349],{"_102":357,"_95":358,"_154":359,"_156":-5,"_157":136,"_158":282,"_160":161,"_104":360,"_99":32,"_163":176,"_165":361,"_166":22,"_167":-5,"_168":362,"_98":363},"50a3ea9a-882f-11f0-9954-0224a6c84d84","deno.json","01990bce-c638-714e-97a4-1fa9e6167c76",["D",1756839724543],"{\n \"$schema\": \"https://raw.githubusercontent.com/denoland/deno/348900b8b79f4a434cab4c74b3bc8d4d2fa8ee74/cli/schemas/config-file.v1.json\",\n \"lock\": false,\n \"compilerOptions\": {\n \"noImplicitAny\": false,\n \"strict\": false,\n \"types\": [\n \"https://www.val.town/types/valtown.d.ts\"\n ],\n \"lib\": [\n \"dom\",\n \"dom.iterable\",\n \"dom.asynciterable\",\n \"deno.ns\",\n \"deno.unstable\"\n ]\n },\n \"lint\": {\n \"files\": {\n \"include\": [\n \"deno:/https/esm.town/**/*\"\n ]\n },\n \"rules\": {\n \"exclude\": [\n \"no-explicit-any\"\n ]\n }\n },\n \"node_modules_dir\": false,\n \"experimental\": {\n \"unstable-node-globals\": true,\n \"unstable-temporal\": true,\n \"unstable-worker-options\": true,\n \"unstable-sloppy-imports\": true\n }\n}",[],[358],{"_102":365,"_95":366,"_154":367,"_156":-5,"_157":136,"_158":368,"_160":161,"_104":369,"_99":342,"_163":370,"_165":371,"_166":22,"_167":-5,"_168":372,"_98":373},"3980aa2a-8937-11f0-b673-0224a6c84d84","main.tsx","01990bce-c638-714e-97a4-20de8c449fb9",40,["D",1756953072886],"http","import { Hono } from \"npm:hono\";\nimport { parseVal, serveFile } from \"https://esm.town/v/std/utils/index.ts\";\nimport rtc from \"./routes/rtc.ts\";\n\nconst SOURCE_URL = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\"))?.links?.self?.val || \"#\";\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.get(\"/source\", (c) => c.redirect(SOURCE_URL));\napp.get(\"/\", () => serveFile(\"/frontend/index.html\"));\n\nexport default app.fetch;",[],[366],"files",[151,170,181,188,195,204,213,222,231,240,249,258,267,278,285,296,302,311,321,330,338,347,356,364],"counts",{"_378":32,"_379":282,"_380":282,"_381":282},"pulls","branches","forks","envVars","contents",[384,387,390,393,396,399,402,405,408,411],{"_102":152,"_95":153,"_154":155,"_156":-5,"_157":136,"_158":159,"_160":161,"_104":385,"_99":32,"_163":164,"_165":-5,"_166":22,"_167":-5,"_98":386},["D",1756952212789],[153],{"_102":182,"_95":183,"_154":184,"_156":-5,"_157":136,"_158":185,"_160":161,"_104":388,"_99":32,"_163":164,"_165":-5,"_166":22,"_167":-5,"_98":389},["D",1756952212884],[183],{"_102":279,"_95":280,"_154":281,"_156":-5,"_157":136,"_158":282,"_160":161,"_104":391,"_99":32,"_163":164,"_165":-5,"_166":22,"_167":-5,"_98":392},["D",1756839724543],[280],{"_102":297,"_95":298,"_154":299,"_156":-5,"_157":136,"_158":282,"_160":161,"_104":394,"_99":32,"_163":164,"_165":-5,"_166":22,"_167":-5,"_98":395},["D",1756839724543],[298],{"_102":322,"_95":323,"_154":324,"_156":-5,"_157":136,"_158":325,"_160":161,"_104":397,"_99":32,"_163":176,"_165":327,"_166":22,"_167":-5,"_98":398},["D",1756953072887],[323],{"_102":331,"_95":332,"_154":333,"_156":-5,"_157":136,"_158":282,"_160":161,"_104":400,"_99":32,"_163":176,"_165":335,"_166":22,"_167":-5,"_98":401},["D",1756839724543],[332],{"_102":339,"_95":340,"_154":341,"_156":-5,"_157":136,"_158":342,"_160":161,"_104":403,"_99":32,"_163":176,"_165":344,"_166":22,"_167":-5,"_98":404},["D",1756840062786],[340],{"_102":348,"_95":349,"_154":350,"_156":-5,"_157":136,"_158":351,"_160":161,"_104":406,"_99":282,"_163":176,"_165":353,"_166":22,"_167":-5,"_98":407},["D",1756840062842],[349],{"_102":357,"_95":358,"_154":359,"_156":-5,"_157":136,"_158":282,"_160":161,"_104":409,"_99":32,"_163":176,"_165":361,"_166":22,"_167":-5,"_98":410},["D",1756839724543],[358],{"_102":365,"_95":366,"_154":367,"_156":-5,"_157":136,"_158":368,"_160":161,"_104":412,"_99":342,"_163":370,"_165":371,"_166":22,"_167":-5,"_98":413},["D",1756953072886],[366],"upstreamProject",{"_102":144,"_95":416,"_110":-5,"_111":-5,"_108":109,"_23":417},"hello-mcp",{"_93":94,"_128":129},"upstreamBranch",{"_102":139,"_104":420,"_421":422,"_95":43,"_137":144,"_99":192,"_138":423,"_140":424,"_425":426,"_106":-5,"_148":22,"_108":109,"_147":94,"_16":117,"_146":416},["D",1756812644593],"updatedAt",["D",1757114821629],"be33420e-792a-11f0-bfa3-0224a6c84d84",197,"lockfileKey","1757114823817-esm.town_v_emcho_hello-mcp@23-main","emptyEnvVars","isNew","flags",[],"routes/_app.x.$handle.$name.compare.$branch",{"_93":94,"_95":96,"_97":43,"_100":433,"_438":439,"_440":441,"_442":443,"_616":161,"_617":437,"_618":619,"_622":-5},{"_102":103,"_95":96,"_104":434,"_106":-5,"_107":22,"_108":109,"_110":-5,"_111":-5,"_14":112,"_113":-5,"_114":22,"_115":-5,"_116":32,"_16":117,"_118":119,"_120":119,"_121":122,"_123":122,"_124":122,"_125":119,"_126":119,"_23":435,"_130":436,"_133":437,"_148":22},["D",1756839724543],{"_93":94,"_128":129},{"_93":94,"_132":129},{"_99":135,"_95":43,"_102":136,"_137":103,"_138":139,"_140":141,"_142":109,"_143":144,"_145":117,"_146":96,"_147":94},"ahead",44,"behind",8,"diff",{"_444":445,"_166":517,"_550":551},"added",[446,452,457,461,465,468,472,477,482,487,492,497,502,507,512],{"_98":153,"_447":-7,"_448":449},"targetFile","sourceFile",{"_102":152,"_95":153,"_154":155,"_156":-5,"_157":136,"_158":159,"_160":161,"_104":162,"_99":32,"_163":164,"_165":-5,"_166":22,"_167":-5,"_168":-7,"_98":450,"_451":-5},[153],"contentHash",{"_98":453,"_447":-7,"_448":454},".claude/settings.local.json",{"_102":171,"_95":172,"_154":173,"_156":155,"_157":136,"_158":174,"_160":161,"_104":175,"_99":32,"_163":176,"_165":177,"_166":22,"_167":-5,"_168":-7,"_98":455,"_451":456},[153,172],"70874fd38c1e559b87d00c28989d15926450e9a37aec792396eea6fd928ac51d",{"_98":323,"_447":-7,"_448":458},{"_102":322,"_95":323,"_154":324,"_156":-5,"_157":136,"_158":325,"_160":161,"_104":326,"_99":32,"_163":176,"_165":327,"_166":22,"_167":-5,"_168":-7,"_98":459,"_451":460},[323],"1a1dbe176bc233b499d35a57db7513f2941c99ab9759f177830c9149be99005b",{"_98":340,"_447":-7,"_448":462},{"_102":339,"_95":340,"_154":341,"_156":-5,"_157":136,"_158":342,"_160":161,"_104":343,"_99":32,"_163":176,"_165":344,"_166":22,"_167":-5,"_168":-7,"_98":463,"_451":464},[340],"18fda0b9a3682649c48bb0c231cfdddfb07c76279e9f9fc6ad11b14e225f1736",{"_98":183,"_447":-7,"_448":466},{"_102":182,"_95":183,"_154":184,"_156":-5,"_157":136,"_158":185,"_160":161,"_104":186,"_99":32,"_163":164,"_165":-5,"_166":22,"_167":-5,"_168":-7,"_98":467,"_451":-5},[183],{"_98":469,"_447":-7,"_448":470},"docs/guides",{"_102":189,"_95":190,"_154":191,"_156":184,"_157":136,"_158":192,"_160":161,"_104":193,"_99":32,"_163":164,"_165":-5,"_166":22,"_167":-5,"_168":-7,"_98":471,"_451":-5},[183,190],{"_98":473,"_447":-7,"_448":474},"docs/guides/realtime-conversations.md",{"_102":196,"_95":197,"_154":198,"_156":191,"_157":136,"_158":199,"_160":161,"_104":200,"_99":32,"_163":176,"_165":201,"_166":22,"_167":-5,"_168":-7,"_98":475,"_451":476},[183,190,197],"d856e82302fbc299dde775eb16485dc26c651d8375b3be0d8270edf3bcef59dc",{"_98":478,"_447":-7,"_448":479},"docs/guides/realtime-models-prompting.md",{"_102":205,"_95":206,"_154":207,"_156":191,"_157":136,"_158":208,"_160":161,"_104":209,"_99":32,"_163":176,"_165":210,"_166":22,"_167":-5,"_168":-7,"_98":480,"_451":481},[183,190,206],"fc07dd01e874e23f8f23327630d62ebc78098b1a2124c9e06972ec9efa3433bd",{"_98":483,"_447":-7,"_448":484},"docs/guides/realtime-server-controls.md",{"_102":214,"_95":215,"_154":216,"_156":191,"_157":136,"_158":217,"_160":161,"_104":218,"_99":32,"_163":176,"_165":219,"_166":22,"_167":-5,"_168":-7,"_98":485,"_451":486},[183,190,215],"e591968270d96b70f4ef3cde0c29ea73d56e51e9b6e263b78d94bbb8c6010384",{"_98":488,"_447":-7,"_448":489},"docs/guides/realtime-sip.md",{"_102":223,"_95":224,"_154":225,"_156":191,"_157":136,"_158":226,"_160":161,"_104":227,"_99":32,"_163":176,"_165":228,"_166":22,"_167":-5,"_168":-7,"_98":490,"_451":491},[183,190,224],"d7acde075ba2a43f16da66b03cabdf0ffa096bdbaa816001bf9b9f5e3afe1733",{"_98":493,"_447":-7,"_448":494},"docs/guides/realtime-transcription.md",{"_102":232,"_95":233,"_154":234,"_156":191,"_157":136,"_158":235,"_160":161,"_104":236,"_99":32,"_163":176,"_165":237,"_166":22,"_167":-5,"_168":-7,"_98":495,"_451":496},[183,190,233],"1ea451a2a9147bc3123f6cee64c64481e70eff54f862d645aa9ed9c3586f594e",{"_98":498,"_447":-7,"_448":499},"docs/guides/realtime-webrtc.md",{"_102":241,"_95":242,"_154":243,"_156":191,"_157":136,"_158":244,"_160":161,"_104":245,"_99":32,"_163":176,"_165":246,"_166":22,"_167":-5,"_168":-7,"_98":500,"_451":501},[183,190,242],"c9ed20fcd9dc4c100635976a4d32d43c1e97cc9a21b7cf3e7089f422940103b2",{"_98":503,"_447":-7,"_448":504},"docs/guides/realtime-websocket.md",{"_102":250,"_95":251,"_154":252,"_156":191,"_157":136,"_158":253,"_160":161,"_104":254,"_99":32,"_163":176,"_165":255,"_166":22,"_167":-5,"_168":-7,"_98":505,"_451":506},[183,190,251],"8e0cc6fb202972818d28382fb72d8119707e11bec25588fdae21cb87e4819aac",{"_98":508,"_447":-7,"_448":509},"docs/guides/realtime.md",{"_102":259,"_95":260,"_154":261,"_156":191,"_157":136,"_158":262,"_160":161,"_104":263,"_99":32,"_163":176,"_165":264,"_166":22,"_167":-5,"_168":-7,"_98":510,"_451":511},[183,190,260],"62e04d01b2faf73ae6154a930a91218a2434dc0c6afc61aa8167703aef840a0b",{"_98":513,"_447":-7,"_448":514},"docs/guides/tools-connectors-mcp.md",{"_102":268,"_95":269,"_154":270,"_156":191,"_157":136,"_158":271,"_160":161,"_104":272,"_99":32,"_163":176,"_165":273,"_166":22,"_167":-5,"_168":-7,"_98":515,"_451":516},[183,190,269],"c7a303a5bc46ce1a356482a2550c358a9867b12cf6645f7b1f00c535b7220c99",[518,530,540],{"_98":519,"_447":520,"_448":-7},"routes/mcp.ts",{"_102":521,"_95":522,"_154":523,"_156":524,"_157":139,"_158":525,"_160":161,"_104":526,"_99":282,"_163":307,"_165":527,"_166":22,"_167":-5,"_168":-7,"_98":528,"_451":529},"6f332422-87f7-11f0-89f5-0224a6c84d84","mcp.ts","399f9e08-87f2-11f0-bf71-0224a6c84d84","01990a31-913b-7345-a24a-e8c1bf3b5dc2",13,["D",1756815724070],"import { Hono } from \"npm:hono\";\n\nconst mcp = new Hono();\n\n// Demo tool: getFavoriteFood with some test data\nconst favoriteFoods: Record = {\n \"thomas\": \"alsacian choucroute\",\n \"brian\": \"trail mix\", \n \"emil\": \"nutella\",\n \"yana\": \"french macarons\",\n \"alex\": \"brownies\",\n \"max\": \"feta cheese\",\n \"justin\": \"sushi\",\n \"sam\": \"pizza\"\n};\n\n// MCP Tools definition\nconst tools = [\n {\n name: \"getFavoriteFood\",\n description: \"Returns the favorite food for specific people\",\n inputSchema: {\n type: \"object\",\n properties: {\n name: {\n type: \"string\",\n description: \"The person's name to look up\"\n }\n },\n required: [\"name\"]\n }\n }\n];\n\n// Handle MCP JSON-RPC requests\nmcp.post(\"/\", async (c) => {\n const body = await c.req.json();\n \n console.log(\"MCP Request:\", body);\n\n // Handle initialize method\n if (body.method === \"initialize\") {\n const clientVersion = body.params?.protocolVersion || \"2025-06-18\";\n return c.json({\n jsonrpc: \"2.0\",\n id: body.id,\n result: {\n protocolVersion: clientVersion,\n capabilities: {\n tools: {\n listChanged: true // Critical: tells OpenAI to fetch tools\n }\n },\n serverInfo: {\n name: \"hello-mcp\",\n version: \"1.0.0\"\n }\n }\n });\n }\n\n // Handle tools/list method\n if (body.method === \"tools/list\") {\n return c.json({\n jsonrpc: \"2.0\",\n id: body.id,\n result: {\n tools: tools\n }\n });\n }\n\n // Handle tools/call method\n if (body.method === \"tools/call\") {\n const { name, arguments: args } = body.params || {};\n \n if (name === \"getFavoriteFood\") {\n const personName = args?.name?.toLowerCase() || \"\";\n const favoriteFood = favoriteFoods[personName];\n \n const result = favoriteFood \n ? {\n success: true,\n data: {\n person: args.name,\n favoriteFood: favoriteFood\n }\n }\n : {\n success: false,\n error: `I don't know ${args.name}'s favorite food. I know about: ${Object.keys(favoriteFoods).join(\", \")}`\n };\n \n return c.json({\n jsonrpc: \"2.0\",\n id: body.id,\n result: {\n content: [\n {\n type: \"text\",\n text: JSON.stringify(result)\n }\n ]\n }\n });\n }\n \n // Tool not found\n return c.json({\n jsonrpc: \"2.0\",\n id: body.id,\n error: {\n code: -32601,\n message: `Tool not found: ${name}`\n }\n });\n }\n\n // Handle unknown notifications (like notifications/initialized)\n if (body.method?.startsWith(\"notifications/\")) {\n return c.json({\n jsonrpc: \"2.0\",\n id: body.id,\n result: {}\n });\n }\n\n // Unknown method\n return c.json({\n jsonrpc: \"2.0\",\n id: body.id,\n error: {\n code: -32601,\n message: `Method not found: ${body.method}`\n }\n });\n});\n\n// Debug endpoint to list tools\nmcp.get(\"/tools\", async (c) => {\n return c.json({\n tools: tools\n });\n});\n\nexport default mcp;",[298,522],"81c323c1750e8d37dfee51f8a53eae1c2b47eb6ffb11d8d38653bb245f3c1f79",{"_98":531,"_447":532,"_448":-7},"routes/observer.ts",{"_102":533,"_95":534,"_154":535,"_156":524,"_157":139,"_158":282,"_160":161,"_104":536,"_99":32,"_163":307,"_165":537,"_166":22,"_167":-5,"_168":-7,"_98":538,"_451":539},"43bc638c-87f0-11f0-ba02-0224a6c84d84","observer.ts","01990a31-913b-7345-a24a-da6bc15ddc7e",["D",1756812644593],"import { Hono } from \"npm:hono\";\nimport WebSocket from \"npm:ws\";\nimport { makeHeaders } from \"./utils.ts\";\n\nconst observer = new Hono();\n\n// POST /observer/:callId : self-connect to establish a web socket\nobserver.post(\"/:callId\", async (c) => {\n const callId = c.req.param(\"callId\");\n const url = `wss://api.openai.com/v1/realtime?call_id=${callId}`;\n const ws = new WebSocket(url, { headers: makeHeaders() });\n ws.on(\"open\", () => {\n console.log(\"✅ websocket connected\");\n setTimeout(() => ws.send(JSON.stringify({ type: \"response.create\" })), 250);\n });\n ws.on(\"message\", (data /*, isBinary */) => {\n const message = JSON.parse(data);\n if (message.type !== \"response.audio_transcript.delta\") {\n console.log(message);\n }\n });\n ws.on(\"error\", (err: Error) => {\n console.error(`🔴 websocket failed: ${err.message}`);\n });\n\n // Respond immediately; WS continues in background\n return c.newResponse(null);\n});\n\nexport default observer;",[298,534],"873651bfd5ccb31259cd76c0fbf7b18b5221ac2bad00768a88a08e39db4eb676",{"_98":541,"_447":542,"_448":-7},"routes/sip.ts",{"_102":543,"_95":544,"_154":545,"_156":524,"_157":139,"_158":282,"_160":161,"_104":546,"_99":32,"_163":307,"_165":547,"_166":22,"_167":-5,"_168":-7,"_98":548,"_451":549},"43bc6e18-87f0-11f0-ba02-0224a6c84d84","sip.ts","01990a31-913b-7345-a24a-e34aa31ad6da",["D",1756812644593],"import { Hono } from \"npm:hono\";\nimport { Webhook } from \"npm:svix\";\nimport { makeHeaders, makeSession } from \"./utils.ts\";\n\nconst sip = new Hono();\n\n// POST /sip : webhook endpoint\nsip.post(\"/\", async (c) => {\n // Verify the webhook.\n const OPENAI_SIGNING_SECRET = Deno.env.get(\"OPENAI_SIGNING_SECRET\");\n if (!OPENAI_SIGNING_SECRET) {\n console.error(\"🔴 webhook secret not configured\");\n return c.text(\"Internal error\", 500);\n }\n const webhook = new Webhook(OPENAI_SIGNING_SECRET);\n const bodyStr = await c.req.text();\n let callId: string | undefined;\n try {\n const headers = Object.fromEntries(c.req.raw.headers.entries());\n const payload: any = webhook.verify(bodyStr, headers);\n callId = payload.data.call_id;\n console.log(\"✅ verified webhook:\", callId);\n } catch (error) {\n console.error(`🔴 verify failed: ${error}`);\n return c.text(\"Invalid signature\", 401);\n }\n\n // Accept the call.\n const url = `https://api.openai.com/v1/realtime/calls/${callId}/accept`;\n const headers = makeHeaders(\"application/json\");\n const body = JSON.stringify(makeSession());\n const opts = { method: \"POST\", headers, body };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n const errText = await resp.text().catch(() => \"\");\n console.error(`🔴 accept failed: ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n console.log(\"✅ call accepted:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch((e) => console.error(\"observer trigger failed:\", e));\n\n return c.newResponse(null);\n});\n\nexport default sip;",[298,544],"08325571281452e7dca6453ea1792bed39680ee3efe8125f3623d7c23b8a5eed","modified",[552,564,577,589,603],{"_98":349,"_447":553,"_448":561},{"_102":554,"_95":349,"_154":555,"_156":-5,"_157":139,"_158":141,"_160":161,"_104":556,"_99":557,"_163":176,"_165":558,"_166":22,"_167":-5,"_168":-7,"_98":559,"_451":560},"6f3f92c0-87f7-11f0-8245-0224a6c84d84","01990a31-913b-7345-a24a-ef358561a0b2",["D",1756815724072],4,"# hello-mcp\n\n**Hello MCP** is a remix of OpenAI's hello-realtime demo that adds Model Context Protocol (MCP) support for server-side tool execution.\n\nThis demo showcases:\n- WebRTC-based voice conversations with OpenAI's Realtime API\n- **MCP (Model Context Protocol)** for secure server-side tool execution\n- A demo `getFavoriteFood` tool to demonstrate MCP functionality\n- Toggle between standard and MCP-enabled modes\n\nIf you remix the app, you'll just need to pop in your own `OPENAI_API_KEY` (from\n[platform.openai.com](https://platform.openai.com)).\n\n## What's New: MCP Support\n\n### Model Context Protocol (MCP)\nMCP enables server-side tool execution, keeping your business logic and API keys secure on the server rather than exposing them to the client.\n\n### Demo Tool: getFavoriteFood\nTry asking questions like:\n- \"What's Thomas's favorite food?\"\n- \"What does Brian like to eat?\"\n- \"Tell me about Yana's favorite food\"\n\nThe tool knows about: Thomas, Brian, Emil, Yana, Alex, Max, Justin, and Sam.\n\n## Architecture\n\n1. **Standard WebRTC Flow**:\n - Browser connects to frontend\n - Creates WebRTC offer\n - `/rtc` endpoint handles SDP negotiation with OpenAI\n - Observer established to monitor session\n\n2. **MCP-Enhanced Flow** (when enabled):\n - Same WebRTC setup as above\n - OpenAI discovers tools via MCP protocol (`/mcp`)\n - Tools execute server-side when invoked\n - Results returned through MCP protocol\n - Client triggers response generation after tool completion\n\n## API Endpoints\n- `GET /` - serves the WebRTC-based app with MCP toggle\n- `POST /rtc` - creates a new WebRTC-based Realtime API session (with optional `?mcp=true`)\n- `POST /sip` - handles an incoming Realtime API SIP call\n- `POST /observer` - internal endpoint to establish a Realtime API websocket for session control\n- `POST /mcp` - MCP protocol endpoint for tool discovery and execution\n- `GET /mcp/tools` - Debug endpoint to list available tools\n\n## Project Structure\n```\n├── frontend/ \n│ └── index.html # HTML/JS application for WebRTC with MCP support\n├── main.tsx # Main entrypoint for the val.town app\n└── routes/\n |── mcp.ts # MCP protocol handler with getFavoriteFood tool\n |── observer.ts # Server-side session control for SIP & WebRTC\n |── rtc.ts # WebRTC session setup with MCP configuration\n |── sip.ts # SIP webhook handling and session setup\n └── utils.ts # Realtime API session configuration with MCP support\n```\n\n## How to Test MCP\n\n1. **Start the app** - Click \"Start\" and allow microphone access\n2. **Check MCP is enabled** - Ensure \"Use MCP Tools\" checkbox is checked\n3. **Ask about favorite foods** - Try \"What's Thomas's favorite food?\"\n4. **Watch the logs** - You'll see MCP events including tool discovery and execution\n5. **Try without MCP** - Uncheck the box and restart to see the difference\n\n## MCP Protocol Details\n\nThe MCP implementation follows the JSON-RPC 2.0 specification:\n\n### Initialize\n```json\n{\n \"method\": \"initialize\",\n \"params\": { \"protocolVersion\": \"2025-06-18\" }\n}\n```\n\n### Tools List\n```json\n{\n \"method\": \"tools/list\",\n \"result\": { \"tools\": [...] }\n}\n```\n\n### Tool Call\n```json\n{\n \"method\": \"tools/call\",\n \"params\": {\n \"name\": \"getFavoriteFood\",\n \"arguments\": { \"name\": \"Thomas\" }\n }\n}\n```",[349],"f7a154deb240cd56f531f1afffdbb8132ece3c3a5a6cad457312f3fc497fd291",{"_102":348,"_95":349,"_154":350,"_156":-5,"_157":136,"_158":351,"_160":161,"_104":352,"_99":282,"_163":176,"_165":353,"_166":22,"_167":-5,"_168":-7,"_98":562,"_451":563},[349],"e66448222334554767aefc73ea436ef8dfa2588a90e1685d9f31373b08394075",{"_98":565,"_447":566,"_448":574},"frontend/index.html",{"_102":567,"_95":287,"_154":568,"_156":569,"_157":139,"_158":557,"_160":161,"_104":570,"_99":282,"_163":176,"_165":571,"_166":22,"_167":-5,"_168":-7,"_98":572,"_451":573},"39770f2e-87f2-11f0-ba62-0224a6c84d84","01990a31-913b-7345-a24a-d2ada2d7b141","01990a31-913b-7345-a24a-d4bd6155983b",["D",1756813486419],"\n\n \n \n \n OpenAI Realtime API Voice Agent\n \n \n \n \n\n

\n\n

\n\n \n \n",[280,287],"3d5b7974ef11252747bcc521b13cdeb1cb89666822115fcdf5e84998345065cd",{"_102":286,"_95":287,"_154":288,"_156":281,"_157":136,"_158":289,"_160":161,"_104":290,"_99":291,"_163":176,"_165":292,"_166":22,"_167":-5,"_168":-7,"_98":575,"_451":576},[280,287],"a01e0ba14c616c694ef3d02f7c84c1aa85951438edbb57738efe7fcdc582b085",{"_98":366,"_447":578,"_448":586},{"_102":579,"_95":366,"_154":580,"_156":-5,"_157":139,"_158":581,"_160":161,"_104":582,"_99":282,"_163":370,"_165":583,"_166":22,"_167":-5,"_168":-7,"_98":584,"_451":585},"39d77a08-87f2-11f0-8c99-0224a6c84d84","01990a31-913b-7345-a24a-f20a587c5fee",9,["D",1756813486382],"import { Hono } from \"npm:hono\";\nimport { parseVal, serveFile } from \"https://esm.town/v/std/utils/index.ts\";\nimport mcp from \"./routes/mcp.ts\";\nimport rtc from \"./routes/rtc.ts\";\nimport sip from \"./routes/sip.ts\";\nimport observer from \"./routes/observer.ts\";\n\nconst SOURCE_URL = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\")).links.self.val;\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.route(\"/sip\", sip);\napp.route(\"/observer\", observer);\napp.route(\"/mcp\", mcp);\napp.get(\"/source\", (c) => c.redirect(SOURCE_URL));\n\napp.get(\"/\", () => serveFile(\"/frontend/index.html\"));\n\nexport default app.fetch;",[366],"b974694c2be6de0b2bd2639c101a3a06a836fde28e51f4a85ba7785386b30600",{"_102":365,"_95":366,"_154":367,"_156":-5,"_157":136,"_158":368,"_160":161,"_104":369,"_99":342,"_163":370,"_165":371,"_166":22,"_167":-5,"_168":-7,"_98":587,"_451":588},[366],"6767d1b6ffb8a934862144e54e0ff4da8a043a92cc83bf8d2955c57ec1253c53",{"_98":590,"_447":591,"_448":600},"routes/rtc.ts",{"_102":592,"_95":304,"_154":593,"_156":524,"_157":139,"_158":594,"_160":161,"_104":595,"_99":596,"_163":307,"_165":597,"_166":22,"_167":-5,"_168":-7,"_98":598,"_451":599},"6f3897d6-87f7-11f0-9f04-0224a6c84d84","01990a31-913b-7345-a24a-deedb45ded9b",14,["D",1756815724071],3,"import { Hono } from \"npm:hono\";\nimport { makeHeaders, makeSession } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new call\nrtc.post(\"/\", async (c) => {\n // Check if MCP mode is requested via query parameter\n const useMCP = c.req.query(\"mcp\") === \"true\";\n const selfUrl = new URL(c.req.url);\n const mcpUrl = useMCP ? `${selfUrl.origin}/mcp` : undefined;\n \n // Create the call.\n const url = \"https://api.openai.com/v1/realtime/calls\";\n const headers = makeHeaders();\n const fd = new FormData();\n fd.set(\"sdp\", await c.req.text());\n fd.set(\"session\", JSON.stringify(makeSession(useMCP, mcpUrl)));\n const opts = { method: \"POST\", headers, body: fd };\n const resp = await fetch(url, opts);\n if (!resp.ok) {\n const errText = await resp.text().catch(() => \"\");\n console.error(`🔴 start call failed: ${resp.status} ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n\n const contentType = resp.headers.get(\"Content-Type\");\n const location = resp.headers.get(\"Location\");\n const callId = location?.split(\"/\").pop();\n console.log(\"✅ call created:\", callId);\n\n // Kick off observer in the background (fire-and-forget).\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch(() => {}); // catch to not crash the process\n\n return c.newResponse(resp.body, {\n headers: { \"Content-Type\": contentType },\n });\n});\n\nexport default rtc;",[298,304],"59cfcb06ec77ff555f737039acc0a2dd85c6a803bb0ac98b94c9ac9c78bce180",{"_102":303,"_95":304,"_154":305,"_156":299,"_157":136,"_158":135,"_160":161,"_104":306,"_99":291,"_163":307,"_165":308,"_166":22,"_167":-5,"_168":-7,"_98":601,"_451":602},[298,304],"8cf96d6c8e541e9e60738a79defdf467b5008f1549b48c8bce31f215cbd55582",{"_98":604,"_447":605,"_448":613},"routes/utils.ts",{"_102":606,"_95":313,"_154":607,"_156":524,"_157":139,"_158":608,"_160":22,"_104":609,"_99":282,"_163":307,"_165":610,"_166":22,"_167":-5,"_168":-7,"_98":611,"_451":612},"398663c0-87f2-11f0-ba62-0224a6c84d84","01990a31-913b-7345-a24a-e588c8e5287c",7,["D",1756813486533],"const MODEL = \"gpt-realtime\";\nconst INSTRUCTIONS = `\n Greet the user in English, and thank them for trying the new OpenAI Realtime API.\n Give them a brief summary based on the list below, and then ask if they have any questions.\n Answer questions using the information below. For questions outside this scope,\n apologize and say you can't help with that.\n ---\n Short summary:\n - The Realtime API is now generally available, and has improved instruction following, voice naturalness, and audio quality.\n There are two new voices and it's easier to develop for. There's also a new telephony integration for phone scenarios.\n Full list of improvements:\n - generally available (no longer in beta)\n - two new voices (Cedar and Marin)\n - improved instruction following\n - enhanced voice naturalness\n - higher audio quality\n - improved handling of alphanumerics (eg, properly understanding credit card and phone numbers)\n - support for the OpenAI Prompts API\n - support for MCP-based tools\n - auto-truncation to reduce context size\n - native telephony support, making it simple to connect voice calls to existing Realtime API applications\n - when using WebRTC, you can connect without needing an ephemeral token\n - when using WebRTC, you can now control sessions (including tool calls) from the server\n - when using WebRTC, you can now send video to the model, and it can respond based on what it sees\n - many helpful bugfixes!\n`;\nconst VOICE = \"marin\";\n\nconst OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\nexport function makeSession(useMCP?: boolean, mcpUrl?: string) {\n const session: any = {\n type: \"realtime\",\n model: MODEL,\n instructions: INSTRUCTIONS,\n audio: {\n input: { noise_reduction: { type: \"near_field\" } },\n output: { voice: VOICE },\n },\n };\n\n // Add MCP configuration if requested\n if (useMCP && mcpUrl) {\n session.tools = [{\n type: \"mcp\",\n server_label: \"hello-mcp-demo\",\n server_url: mcpUrl,\n authorization: \"demo-token-12345\", // In production, use a secure token\n require_approval: \"never\"\n }];\n }\n\n return session;\n}",[298,313],"ccf1a76af1f776c3c10a0375db4189780da7e090754ed3f90257593c7a5030cc",{"_102":312,"_95":313,"_154":314,"_156":299,"_157":136,"_158":135,"_160":161,"_104":315,"_99":316,"_163":307,"_165":317,"_166":22,"_167":-5,"_168":-7,"_98":614,"_451":615},[298,313],"8012a0d4a31bba97909dfa19bb2612edba33c3db035b6f352257721d5e9c04db","hasChanges","sourceBranch","targetBranch",{"_102":139,"_104":620,"_421":621,"_95":43,"_137":144,"_99":192,"_138":423,"_140":424,"_425":426,"_106":-5,"_148":22,"_108":109,"_147":94,"_16":117,"_146":416},["D",1756812644593],["D",1757114821629],"pull","actionData","errors"]

📝 OpenAI Realtime Transcription

Realtime Agent with MCP

Mergehello-transcription/mainintohello-mcp/main

README.md+66-79

frontend/index.html+228-425

Hmain.tsx+1-8

routes/rtc.ts+26-29

routes/utils.ts+20-48

.claude

.claude/settings.local.json+9-0

.gitattributes+2-0

CLAUDE.md+508-0

docs

docs/guides

docs/guides/realtime-conversations.md+919-0

docs/guides/realtime-models-prompting.md+445-0

docs/guides/realtime-server-controls.md+108-0

docs/guides/realtime-sip.md+137-0

docs/guides/realtime-transcription.md+148-0

docs/guides/realtime-webrtc.md+178-0

docs/guides/realtime-websocket.md+151-0

docs/guides/realtime.md+81-0

docs/guides/tools-connectors-mcp.md+813-0

routes/mcp.ts+0-146

routes/observer.ts+0-30

routes/sip.ts+0-49