OpenAI Realtime Transcription

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_272":273,"_274":275},"routes/_app.x.$handle.$name",{"_100":99,"_32":46,"_108":87,"_27":109,"_13":14,"_110":-5,"_11":111,"_112":113,"_114":115,"_116":117,"_118":119,"_20":-5,"_120":121,"_122":123,"_124":25,"_125":126,"_127":59,"_128":-5,"_129":-5,"_130":131},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_44":45,"_32":46,"_39":47,"_48":-5,"_49":25,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":57,"_58":59,"_60":59,"_61":-5,"_62":63,"_64":65,"_66":-5,"_67":-5,"_68":-5,"_69":-5,"_70":71,"_72":71,"_73":74,"_75":74,"_76":74,"_77":71,"_78":71,"_79":80,"_81":82,"_83":84,"_85":25},"version",16,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":33,"_34":35,"_36":38,"_39":40,"_41":14,"_42":43,"_24":25},"previous",{"_30":31,"_32":33,"_34":35,"_36":37},"deleted",false,"deletedHere","path","routes/utils.ts","latestPath","fileId","01990bce-c638-714e-97a4-0caf2e9a1c1c","name","utils.ts","fileType","script","content","const OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\nexport interface TranscriptionConfig {\n model?: \"gpt-4o-transcribe\" | \"gpt-4o-mini-transcribe\" | \"whisper-1\";\n language?: string;\n prompt?: string;\n enableVAD?: boolean;\n noiseReduction?: \"near_field\" | \"far_field\" | null;\n includeLogprobs?: boolean;\n}\n\nexport function makeTranscriptionSession(config: TranscriptionConfig = {}) {\n const {\n model = \"gpt-4o-transcribe\",\n language = \"en\",\n prompt = \"\",\n enableVAD = true,\n noiseReduction = \"near_field\",\n includeLogprobs = false\n } = config;\n\n // Create a transcription-only session (no AI responses)\n const session: any = {\n type: \"realtime\",\n model: \"gpt-realtime\",\n // Disable responses by setting max_response_output_tokens to 1\n max_response_output_tokens: 1,\n instructions: \"Never respond. Only transcribe.\",\n turn_detection: null, // We'll set this below if VAD enabled\n audio: {\n input: {\n format: \"pcm16\",\n transcription: {\n model,\n prompt,\n language\n }\n }\n }\n };\n\n // Configure VAD if enabled\n if (enableVAD) {\n session.turn_detection = {\n type: \"server_vad\",\n threshold: 0.5,\n prefix_padding_ms: 300,\n silence_duration_ms: 500\n };\n } else {\n session.turn_detection = null;\n }\n\n // Configure noise reduction\n if (noiseReduction) {\n session.audio.input.noise_reduction = {\n type: noiseReduction\n };\n }\n\n // Include logprobs if requested\n if (includeLogprobs) {\n session.include = [\"item.input_audio_transcription.logprobs\"];\n }\n\n return session;\n}","const OPENAI_API_KEY = Deno.env.get(\"OPENAI_API_KEY\");\nif (!OPENAI_API_KEY) {\n throw new Error(\"🔴 OpenAI API key not configured\");\n}\n\nexport function makeHeaders(contentType?: string) {\n const obj: Record = {\n Authorization: `Bearer ${OPENAI_API_KEY}`,\n };\n if (contentType) obj[\"Content-Type\"] = contentType;\n return obj;\n}\n\nexport interface TranscriptionConfig {\n model?: \"gpt-4o-transcribe\" | \"gpt-4o-mini-transcribe\" | \"whisper-1\";\n language?: string;\n prompt?: string;\n enableVAD?: boolean;\n noiseReduction?: \"near_field\" | \"far_field\" | null;\n includeLogprobs?: boolean;\n}\n\nexport function makeTranscriptionSession(config: TranscriptionConfig = {}) {\n const {\n model = \"gpt-4o-transcribe\",\n language = \"en\",\n prompt = \"\",\n enableVAD = true,\n noiseReduction = \"near_field\",\n includeLogprobs = false\n } = config;\n\n // Create a transcription session following the documentation structure\n const session: any = {\n type: \"realtime\",\n model: \"gpt-realtime\",\n instructions: \"You are a transcription service. Do not respond with audio.\",\n // Add input_audio_transcription at root level as per docs\n input_audio_transcription: {\n model,\n prompt,\n language\n },\n audio: {\n input: {\n format: \"pcm16\"\n }\n }\n };\n\n // Configure VAD if enabled\n if (enableVAD) {\n session.turn_detection = {\n type: \"server_vad\",\n threshold: 0.5,\n prefix_padding_ms: 300,\n silence_duration_ms: 500\n };\n } else {\n session.turn_detection = null;\n }\n\n // Configure noise reduction at root level as per docs\n if (noiseReduction) {\n session.input_audio_noise_reduction = {\n type: noiseReduction\n };\n }\n\n // Include logprobs if requested\n if (includeLogprobs) {\n session.include = [\"item.input_audio_transcription.logprobs\"];\n }\n\n return session;\n}","createdAt",["D",1756840899853],"branchVersion","branchId","509ca3b6-882f-11f0-9954-0224a6c84d84","id","50991944-882f-11f0-9954-0224a6c84d84","hello-transcription",["D",1756839724543],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","0198f8d1-d70f-76ea-a55f-2b5ba506023d","pinnedFileId","01990bce-c638-714e-97a4-20de8c449fb9","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","0198f8d1-d70f-76ea-a55f-2ed16889c1b0","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_100":99,"_107":103},"org",{"_100":99,"_101":-5,"_102":103,"_18":55,"_104":105,"_106":25},"branch",{"_13":86,"_32":87,"_44":43,"_88":45,"_89":90,"_91":92,"_93":51,"_94":95,"_96":65,"_97":46,"_98":99},"isOwned",45,"main","projectId","forkedBranchId","43b7750c-87f0-11f0-ba02-0224a6c84d84","forkedBranchVersion",15,"forkedProjectPrivacy","forkedProjectId","43b401c4-87f0-11f0-ba02-0224a6c84d84","forkedProjectOrgId","projectName","userHandle","emcho","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18zMXpKcFhORzY4VDlCZlRBZWUybTNFMmV3bDYifQ","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_44":45,"_32":46,"_39":268,"_48":-5,"_49":25,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":57,"_58":59,"_60":59,"_61":-5,"_62":63,"_64":65,"_66":-5,"_67":-5,"_68":-5,"_69":-5,"_70":71,"_72":71,"_73":74,"_75":74,"_76":74,"_77":71,"_78":71,"_79":269,"_81":270,"_83":271,"_85":25},"tree",[207,209,213,214,215,216,217],"files",[207,208,209,210,211,212,213,214,215,216,217],"counts",{"_197":63,"_198":155,"_199":155,"_200":201,"_202":203},"contents",[142,143,144,145,146,147,148],"upstreamProject",{"_44":95,"_32":140,"_53":-5,"_54":-5,"_50":51,"_79":141},"upstreamBranch",{"_44":90,"_39":132,"_133":134,"_32":87,"_88":95,"_13":135,"_89":136,"_91":137,"_138":139,"_48":-5,"_85":25,"_50":51,"_98":99,"_64":65,"_97":140},"emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},["D",1756812644593],"updatedAt",["D",1757114821629],23,"be33420e-792a-11f0-bfa3-0224a6c84d84",197,"lockfileKey","1764516869134-esm.town_v_emcho_hello-mcp@23-main","hello-mcp",{"_100":99,"_107":103},{"_44":192,"_32":193,"_30":194,"_151":-5,"_42":43,"_41":155,"_153":59,"_39":195,"_13":63,"_34":190,"_36":-5,"_24":25,"_158":-5,"_27":196},{"_44":186,"_32":187,"_30":188,"_151":-5,"_42":43,"_41":155,"_153":59,"_39":189,"_13":63,"_34":190,"_36":-5,"_24":25,"_158":-5,"_27":191},{"_44":180,"_32":181,"_30":182,"_151":-5,"_42":43,"_41":155,"_153":59,"_39":183,"_13":63,"_34":20,"_36":184,"_24":25,"_158":-5,"_27":185},{"_44":173,"_32":174,"_30":175,"_151":-5,"_42":43,"_41":176,"_153":59,"_39":177,"_13":63,"_34":20,"_36":178,"_24":25,"_158":-5,"_27":179},{"_44":166,"_32":167,"_30":168,"_151":-5,"_42":43,"_41":169,"_153":59,"_39":170,"_13":155,"_34":20,"_36":171,"_24":25,"_158":-5,"_27":172},{"_44":160,"_32":161,"_30":162,"_151":-5,"_42":43,"_41":155,"_153":59,"_39":163,"_13":63,"_34":20,"_36":164,"_24":25,"_158":-5,"_27":165},{"_44":149,"_32":150,"_30":57,"_151":-5,"_42":43,"_41":152,"_153":25,"_39":154,"_13":155,"_34":156,"_36":157,"_24":25,"_158":-5,"_27":159},"1a3cd812-8830-11f0-8b63-0224a6c84d84","main.tsx","parentId",3,"latest",["D",1756840062830],1,"http","import { Hono } from \"npm:hono\";\nimport { parseVal, serveFile } from \"https://esm.town/v/std/utils/index.ts\";\nimport rtc from \"./routes/rtc.ts\";\nimport observer from \"./routes/observer.ts\";\n\nconst SOURCE_URL = parseVal(Deno.env.get(\"VALTOWN_ENTRYPOINT\"))?.links?.self?.val || \"#\";\n\nconst app = new Hono();\napp.route(\"/rtc\", rtc);\napp.route(\"/observer\", observer);\napp.get(\"/source\", (c) => c.redirect(SOURCE_URL));\n\napp.get(\"/\", () => serveFile(\"/frontend/index.html\"));\n\nexport default app.fetch;","label",[150],"50a3ea9a-882f-11f0-9954-0224a6c84d84","deno.json","01990bce-c638-714e-97a4-1fa9e6167c76",["D",1756839724543],"{\n \"$schema\": \"https://raw.githubusercontent.com/denoland/deno/348900b8b79f4a434cab4c74b3bc8d4d2fa8ee74/cli/schemas/config-file.v1.json\",\n \"lock\": false,\n \"compilerOptions\": {\n \"noImplicitAny\": false,\n \"strict\": false,\n \"types\": [\n \"https://www.val.town/types/valtown.d.ts\"\n ],\n \"lib\": [\n \"dom\",\n \"dom.iterable\",\n \"dom.asynciterable\",\n \"deno.ns\",\n \"deno.unstable\"\n ]\n },\n \"lint\": {\n \"files\": {\n \"include\": [\n \"deno:/https/esm.town/**/*\"\n ]\n },\n \"rules\": {\n \"exclude\": [\n \"no-explicit-any\"\n ]\n }\n },\n \"node_modules_dir\": false,\n \"experimental\": {\n \"unstable-node-globals\": true,\n \"unstable-temporal\": true,\n \"unstable-worker-options\": true,\n \"unstable-sloppy-imports\": true\n }\n}",[161],"1a739988-8830-11f0-a821-0224a6c84d84","README.md","01990bce-c638-714e-97a4-1a67f3821e53",10,["D",1756840062842],"# hello-transcription\n\nReal-time speech transcription using OpenAI's Realtime API - a demonstration of transcription-only mode without AI responses.\n\n## Features\n\n- **Real-time Transcription**: Live speech-to-text conversion as you speak\n- **Multiple Models**: Choose between GPT-4o Transcribe, GPT-4o Mini Transcribe, or Whisper-1\n- **Language Support**: Transcribe in multiple languages (English, Spanish, French, German, Italian, Portuguese, Chinese, Japanese, Korean)\n- **Voice Activity Detection (VAD)**: Automatic detection of speech segments\n- **Logprobs Support**: Optional confidence scores for transcriptions\n- **Split View**: See transcriptions and event logs side-by-side\n\n## How It Works\n\nThis app uses OpenAI's Realtime API in transcription-only mode:\n1. Your voice is captured via WebRTC\n2. Audio is streamed to OpenAI's transcription service\n3. Transcriptions are returned in real-time\n4. No AI responses are generated (transcription only)\n\n## Model Differences\n\n- **GPT-4o Transcribe**: Streaming transcription with incremental updates\n- **GPT-4o Mini Transcribe**: Smaller model, streaming transcription \n- **Whisper-1**: Complete transcription after each speech segment (no streaming)\n\n## Configuration Options\n\n- **Model**: Select the transcription model\n- **Language**: Choose the primary language for better accuracy\n- **VAD**: Enable/disable automatic voice activity detection\n- **Logprobs**: Include confidence scores (for advanced use)\n\n## Usage\n\n1. Click \"Start\" to begin transcription\n2. Allow microphone access when prompted\n3. Start speaking - transcriptions appear in real-time\n4. Partial transcriptions update as you speak\n5. Final transcriptions are marked in green\n6. Click \"Stop\" to end the session\n\n## API Endpoints\n\n- `GET /` - Serves the transcription interface\n- `POST /rtc` - Creates a WebRTC transcription session\n- `POST /observer/:callId` - WebSocket observer for transcription events\n\n## Environment Variables\n\nSet in your Val Town environment:\n- `OPENAI_API_KEY` - Your OpenAI API key (required)\n\n## Local Development\n\n```bash\n# Install Deno\ncurl -fsSL https://deno.land/install.sh | sh\n\n# Run locally\ndeno run --allow-all main.tsx\n```\n\n## Val Town Deployment\n\n1. Fork/remix this val on Val Town\n2. Add your `OPENAI_API_KEY` to Val Town secrets\n3. Your app will be available at `https://[your-val-name].val.run`\n\n## Technical Details\n\nThe app uses OpenAI's Realtime API in transcription mode:\n- Session type: `transcription` (not `realtime`)\n- Audio format: PCM16\n- Noise reduction: Near-field (optimized for close microphones)\n- WebRTC data channel for receiving transcription events\n\n## Event Flow\n\n1. `input_audio_buffer.committed` - Audio chunk received\n2. `conversation.item.input_audio_transcription.delta` - Partial transcription\n3. `conversation.item.input_audio_transcription.completed` - Final transcription\n\n## Credits\n\nBuilt with OpenAI's Realtime API for transcription-only use cases.",[167],"1a3463bc-8830-11f0-a44c-0224a6c84d84","CLAUDE.md","1a34648e-8830-11f0-a44c-0224a6c84d84",2,["D",1756840062786],"# Hello-Transcription - OpenAI Realtime API Transcription Demo\n\n## 🎯 Project Overview\n\nHello-Transcription demonstrates the transcription-only mode of OpenAI's Realtime API. Unlike the conversational mode, this implementation focuses purely on speech-to-text conversion without generating AI responses, making it ideal for subtitles, live captions, meeting transcriptions, and other transcription-focused use cases.\n\n**Created:** September 2, 2025 \n**Platform:** Val Town \n**API:** OpenAI Realtime API (Transcription Mode) \n**Key Feature:** Real-time streaming transcription with multiple model support\n\n## 🏗️ Technical Stack\n\n- **Runtime:** Deno (Val Town platform)\n- **Framework:** Hono (lightweight web framework)\n- **Transcription:** OpenAI Realtime API in transcription mode\n- **Connection:** WebRTC with data channel for events\n- **Frontend:** Vanilla JavaScript with split-view interface\n- **Models:** GPT-4o Transcribe, GPT-4o Mini Transcribe, Whisper-1\n\n## 📁 Project Structure\n\n```\nhello-transcription/\n├── frontend/\n│ └── index.html # Split-view transcription interface\n├── routes/\n│ ├── rtc.ts # WebRTC session setup for transcription\n│ ├── observer.ts # WebSocket observer for transcription events\n│ └── utils.ts # Transcription session configuration\n├── main.tsx # Main entry point\n├── deno.json # Deno configuration\n├── README.md # User documentation\n└── CLAUDE.md # This file - technical documentation\n```\n\n## 🔑 Core Concepts\n\n### Transcription vs Conversation Mode\n\nThe Realtime API supports two distinct modes:\n\n1. **Conversation Mode** (`type: \"realtime\"`):\n - Two-way interaction with AI\n - User speaks → AI responds\n - Used in hello-realtime and hello-mcp\n\n2. **Transcription Mode** (`type: \"transcription\"`):\n - One-way speech-to-text only\n - User speaks → Text output\n - No AI responses generated\n - Lower latency, lower cost\n - This demo uses transcription mode\n\n### Transcription Session Object\n\n```typescript\n{\n type: \"transcription\",\n input_audio_format: \"pcm16\",\n input_audio_transcription: {\n model: \"gpt-4o-transcribe\", // or \"gpt-4o-mini-transcribe\", \"whisper-1\"\n prompt: \"\", // Optional context hint\n language: \"en\" // ISO-639-1 language code\n },\n turn_detection: {\n type: \"server_vad\",\n threshold: 0.5,\n prefix_padding_ms: 300,\n silence_duration_ms: 500\n },\n input_audio_noise_reduction: {\n type: \"near_field\" // or \"far_field\", null\n },\n include: [\"item.input_audio_transcription.logprobs\"] // Optional\n}\n```\n\n## 🛠️ Key Components\n\n### 1. Utils Configuration (`/routes/utils.ts`)\n\nHandles transcription session configuration with sensible defaults:\n\n```typescript\nexport function makeTranscriptionSession(config: TranscriptionConfig = {}) {\n const {\n model = \"gpt-4o-transcribe\",\n language = \"en\",\n prompt = \"\",\n enableVAD = true,\n noiseReduction = \"near_field\",\n includeLogprobs = false\n } = config;\n \n // Build session object...\n}\n```\n\n**Key Configuration Options:**\n- **model**: Transcription model selection\n- **language**: Primary language for better accuracy\n- **prompt**: Context hints (e.g., \"Expect medical terminology\")\n- **enableVAD**: Automatic voice activity detection\n- **noiseReduction**: Audio preprocessing type\n- **includeLogprobs**: Confidence scores for words\n\n### 2. RTC Route Handler (`/routes/rtc.ts`)\n\nCreates WebRTC sessions specifically for transcription:\n\n```typescript\n// Get config from query params\nconst model = c.req.query(\"model\") || \"gpt-4o-transcribe\";\nconst language = c.req.query(\"language\") || \"en\";\nconst vad = c.req.query(\"vad\") !== \"false\";\nconst logprobs = c.req.query(\"logprobs\") === \"true\";\n\n// Create transcription session\nconst sessionConfig = makeTranscriptionSession({\n model,\n language,\n enableVAD: vad,\n includeLogprobs: logprobs\n});\n```\n\n**Important:** Uses `type: \"transcription\"` not `type: \"realtime\"`\n\n### 3. Observer WebSocket (`/routes/observer.ts`)\n\nMonitors transcription events via server-side WebSocket:\n\n```typescript\nws.onmessage = (event) => {\n const data = JSON.parse(event.data);\n \n if (data.type === \"conversation.item.input_audio_transcription.delta\") {\n // Streaming partial transcription\n console.log(`📝 Transcription delta: \"${data.delta}\"`);\n } else if (data.type === \"conversation.item.input_audio_transcription.completed\") {\n // Final transcription for segment\n console.log(`✅ Transcription completed: \"${data.transcript}\"`);\n }\n};\n```\n\n### 4. Frontend Interface (`/frontend/index.html`)\n\nSplit-view interface with real-time transcription display:\n\n#### Left Panel - Transcriptions\n- Shows live transcription stream\n- Partial transcriptions update in real-time\n- Final transcriptions marked with green border\n- Each segment timestamped\n\n#### Right Panel - Event Logs\n- Technical event stream\n- Debug information\n- Connection status\n\n#### Data Channel Handling\n```javascript\ndataChannel.onmessage = (event) => {\n const data = JSON.parse(event.data);\n \n if (data.type === \"conversation.item.input_audio_transcription.delta\") {\n // Update partial transcription\n addTranscript(data.item_id, data.delta, false);\n } else if (data.type === \"conversation.item.input_audio_transcription.completed\") {\n // Mark transcription as final\n addTranscript(data.item_id, data.transcript, true);\n }\n};\n```\n\n## 📊 Model Comparison\n\n### GPT-4o Transcribe\n- **Streaming:** Yes - incremental updates via delta events\n- **Latency:** Low\n- **Accuracy:** High\n- **Use Case:** Live subtitles, real-time captions\n\n### GPT-4o Mini Transcribe\n- **Streaming:** Yes - incremental updates\n- **Latency:** Very low\n- **Accuracy:** Good\n- **Use Case:** Quick transcriptions, lower cost\n\n### Whisper-1\n- **Streaming:** No - complete segments only\n- **Latency:** Higher (waits for complete utterance)\n- **Accuracy:** Very high\n- **Use Case:** High-accuracy transcriptions, post-processing\n\n## 🔄 Event Flow\n\n### Transcription Event Sequence\n\n1. **Audio Input**\n ```\n User speaks → Microphone → WebRTC → OpenAI\n ```\n\n2. **VAD Processing** (if enabled)\n ```\n Voice detected → Buffer audio → Silence detected → Commit buffer\n ```\n\n3. **Transcription Events**\n ```\n input_audio_buffer.committed\n ↓\n conversation.item.input_audio_transcription.delta (streaming models)\n ↓\n conversation.item.input_audio_transcription.completed\n ```\n\n### Event Types\n\n#### Delta Event (Streaming)\n```json\n{\n \"type\": \"conversation.item.input_audio_transcription.delta\",\n \"item_id\": \"item_003\",\n \"content_index\": 0,\n \"delta\": \"Hello, how\"\n}\n```\n\n#### Completed Event\n```json\n{\n \"type\": \"conversation.item.input_audio_transcription.completed\",\n \"item_id\": \"item_003\",\n \"content_index\": 0,\n \"transcript\": \"Hello, how are you today?\"\n}\n```\n\n## ⚙️ Configuration Details\n\n### Voice Activity Detection (VAD)\n\nVAD automatically detects speech segments:\n\n```typescript\nturn_detection: {\n type: \"server_vad\",\n threshold: 0.5, // Sensitivity (0-1)\n prefix_padding_ms: 300, // Audio before speech\n silence_duration_ms: 500 // Silence to end segment\n}\n```\n\n**VAD Disabled:**\n```typescript\nturn_detection: null // Manual control required\n```\n\n### Noise Reduction\n\nThree noise reduction modes:\n\n1. **near_field**: Optimized for close microphones (default)\n2. **far_field**: For distant microphones/speakers\n3. **null**: No noise reduction\n\n### Language Configuration\n\nISO-639-1 language codes improve accuracy:\n- \"en\" - English\n- \"es\" - Spanish \n- \"fr\" - French\n- \"de\" - German\n- \"zh\" - Chinese\n- \"ja\" - Japanese\n\n### Logprobs (Confidence Scores)\n\nWhen enabled, provides word-level confidence:\n\n```typescript\ninclude: [\"item.input_audio_transcription.logprobs\"]\n```\n\nReturns probability scores for each transcribed word, useful for:\n- Highlighting uncertain words\n- Quality assessment\n- Post-processing decisions\n\n## 🧪 Testing Guide\n\n### Local Development\n\n1. **Setup Environment**\n ```bash\n # Create .env file\n echo \"OPENAI_API_KEY=sk-...\" > .env\n \n # Install Deno\n curl -fsSL https://deno.land/install.sh | sh\n ```\n\n2. **Run Development Server**\n ```bash\n # With auto-reload\n deno run --watch --allow-all main.tsx\n \n # Or standard\n deno run --allow-all main.tsx\n ```\n\n3. **Test Transcription**\n - Open http://localhost:8000\n - Select model and language\n - Click \"Start\"\n - Speak clearly\n - Watch transcriptions appear\n\n### Testing Different Models\n\n1. **Test Streaming (GPT-4o)**\n - Select \"GPT-4o Transcribe\"\n - Speak continuously\n - Notice incremental updates\n\n2. **Test Non-Streaming (Whisper-1)**\n - Select \"Whisper-1\"\n - Speak, then pause\n - Notice complete segments only\n\n3. **Test VAD**\n - Enable VAD\n - Speak with pauses\n - Notice automatic segmentation\n\n4. **Test Without VAD**\n - Disable VAD\n - Requires manual commit (not implemented in this demo)\n\n## 🐛 Common Issues & Solutions\n\n### Issue: No transcriptions appearing\n**Solutions:**\n- Check microphone permissions\n- Verify OPENAI_API_KEY is set\n- Check browser console for errors\n- Ensure WebRTC connection established\n\n### Issue: Transcriptions cut off mid-sentence\n**Solutions:**\n- Adjust VAD silence_duration_ms (longer = fewer cuts)\n- Try different VAD threshold\n- Consider using different model\n\n### Issue: Poor transcription accuracy\n**Solutions:**\n- Set correct language parameter\n- Use appropriate noise reduction setting\n- Provide context via prompt parameter\n- Try higher accuracy model (Whisper-1)\n\n### Issue: High latency\n**Solutions:**\n- Use GPT-4o Mini for lower latency\n- Check network connection\n- Consider disabling logprobs\n\n## 📈 Performance Characteristics\n\n### Latency Comparison\n- **GPT-4o Mini**: ~100-200ms first token\n- **GPT-4o**: ~150-250ms first token \n- **Whisper-1**: ~500-1000ms (full segment)\n\n### Throughput\n- Handles real-time audio (16kHz PCM16)\n- Multiple concurrent sessions supported\n- No buffering required for streaming models\n\n### Cost Optimization\n- Transcription-only mode is cheaper than conversation mode\n- No AI inference costs\n- GPT-4o Mini most cost-effective\n- Whisper-1 for batch/accuracy needs\n\n## 🔒 Security Considerations\n\n### Current Implementation\n- API key stored in environment variable\n- No authentication on endpoints\n- No rate limiting\n- Single-tenant design\n\n### Production Recommendations\n1. **Authentication**: Add user authentication\n2. **Rate Limiting**: Implement per-user limits\n3. **CORS**: Configure appropriate origins\n4. **Monitoring**: Track usage and errors\n5. **Encryption**: Ensure HTTPS only\n6. **Token Security**: Use short-lived tokens\n\n## 🚀 Deployment\n\n### Val Town Deployment\n\n1. **Create/Remix Val**\n ```bash\n vt remix emcho/hello-transcription my-transcription\n ```\n\n2. **Set Environment**\n - Add `OPENAI_API_KEY` in Val Town secrets\n\n3. **Deploy**\n ```bash\n vt push\n ```\n\n4. **Access**\n - URL: `https://[your-val-name].val.run`\n\n### Environment Variables\n- `OPENAI_API_KEY` - Required for OpenAI API access\n\n## 📝 Future Enhancements\n\n### Potential Features\n\n1. **Recording & Export**\n - Save transcriptions to file\n - Export as SRT/VTT subtitles\n - Download audio recordings\n\n2. **Advanced Controls**\n - Manual VAD control\n - Custom VAD parameters UI\n - Prompt templates\n\n3. **Multi-Stream**\n - Multiple speaker support\n - Speaker diarization\n - Parallel transcriptions\n\n4. **Post-Processing**\n - Punctuation enhancement\n - Grammar correction\n - Translation\n\n5. **Visualization**\n - Audio waveform display\n - VAD activity indicator\n - Confidence heat map\n\n6. **Integration**\n - Webhook support\n - Real-time API streaming\n - Database storage\n\n## 🔗 References\n\n### Documentation\n- [OpenAI Realtime Transcription Guide](https://platform.openai.com/docs/guides/realtime-transcription)\n- [Realtime API Reference](https://platform.openai.com/docs/api-reference/realtime)\n- [Voice Activity Detection Guide](https://platform.openai.com/docs/guides/realtime-vad)\n- [Val Town Documentation](https://docs.val.town)\n\n### Related Projects\n- [hello-realtime](https://hello-realtime.val.run) - Conversation mode demo\n- [hello-mcp](https://github.com/hello-mcp) - MCP tool execution demo\n\n### Key Differences from Conversation Mode\n1. Session type: `transcription` vs `realtime`\n2. No AI responses generated\n3. Different event types\n4. Lower latency and cost\n5. Focused on speech-to-text only\n\n## 💡 Implementation Notes\n\n### Critical Discoveries\n1. **Session Type**: Must use `type: \"transcription\"` not `type: \"realtime\"`\n2. **Event Names**: Different from conversation mode events\n3. **Model Behavior**: Whisper doesn't stream, GPT-4o models do\n4. **VAD Impact**: Significantly affects transcription segmentation\n5. **Language Setting**: Dramatically improves accuracy for non-English\n\n### Best Practices\n1. Always specify language for non-English content\n2. Use near_field noise reduction for headset mics\n3. Enable VAD for natural speech segmentation\n4. Choose model based on latency vs accuracy needs\n5. Monitor item_id for proper segment ordering\n\n## 🎯 Summary\n\nHello-Transcription successfully demonstrates the transcription-only capabilities of OpenAI's Realtime API. Key achievements:\n\n1. **Pure Transcription**: No AI responses, focused solely on speech-to-text\n2. **Model Flexibility**: Support for three different transcription models\n3. **Real-time Streaming**: Live transcription updates for supported models\n4. **Configuration Options**: VAD, noise reduction, language, logprobs\n5. **Clean Interface**: Split-view design for transcriptions and logs\n\nThis implementation serves as a foundation for building transcription-focused applications like live captioning, meeting transcription, subtitle generation, and accessibility tools.",[174],"50a3e2c0-882f-11f0-9954-0224a6c84d84",".vtignore","01990bce-c638-714e-97a4-157f5fb8f430",["D",1756839724543],".git\n.vscode\n.cursorrules\n.DS_Store\nnode_modules\nvendor",[181],"50a3e18a-882f-11f0-9954-0224a6c84d84","routes","01990bce-c638-714e-97a4-1048c460355e",["D",1756839724543],"directory",[187],"50a3d50a-882f-11f0-9954-0224a6c84d84","frontend","01990bce-c637-779b-aded-96ab9fc690df",["D",1756839724543],[193],"pulls","branches","forks","envVars",{"_204":155,"_206":63},"connections",{"_204":63,"_205":63},"count","unfilledCount","emptyCount",{"_44":192,"_32":193,"_30":194,"_151":-5,"_42":43,"_41":155,"_153":59,"_39":265,"_13":63,"_34":190,"_36":-5,"_24":25,"_158":-5,"_219":266,"_27":267},{"_44":257,"_32":258,"_30":259,"_151":194,"_42":43,"_41":260,"_153":25,"_39":261,"_13":152,"_34":20,"_36":262,"_24":25,"_158":-5,"_219":263,"_27":264},{"_44":186,"_32":187,"_30":188,"_151":-5,"_42":43,"_41":155,"_153":59,"_39":254,"_13":63,"_34":190,"_36":-5,"_24":25,"_158":-5,"_219":255,"_27":256},{"_44":246,"_32":247,"_30":248,"_151":188,"_42":43,"_41":249,"_153":25,"_39":250,"_13":155,"_34":35,"_36":251,"_24":25,"_158":-5,"_219":252,"_27":253},{"_44":239,"_32":240,"_30":241,"_151":188,"_42":43,"_41":236,"_153":25,"_39":242,"_13":155,"_34":35,"_36":243,"_24":25,"_158":-5,"_219":244,"_27":245},{"_44":234,"_32":33,"_30":31,"_151":188,"_42":43,"_41":14,"_153":25,"_39":235,"_13":236,"_34":35,"_36":38,"_24":25,"_158":-5,"_219":237,"_27":238},{"_44":180,"_32":181,"_30":182,"_151":-5,"_42":43,"_41":155,"_153":59,"_39":231,"_13":63,"_34":20,"_36":184,"_24":25,"_158":-5,"_219":232,"_27":233},{"_44":173,"_32":174,"_30":175,"_151":-5,"_42":43,"_41":176,"_153":59,"_39":228,"_13":63,"_34":20,"_36":178,"_24":25,"_158":-5,"_219":229,"_27":230},{"_44":166,"_32":167,"_30":168,"_151":-5,"_42":43,"_41":169,"_153":59,"_39":225,"_13":155,"_34":20,"_36":171,"_24":25,"_158":-5,"_219":226,"_27":227},{"_44":160,"_32":161,"_30":162,"_151":-5,"_42":43,"_41":155,"_153":59,"_39":222,"_13":63,"_34":20,"_36":164,"_24":25,"_158":-5,"_219":223,"_27":224},{"_44":149,"_32":150,"_30":57,"_151":-5,"_42":43,"_41":152,"_153":25,"_39":218,"_13":155,"_34":156,"_36":157,"_24":25,"_158":-5,"_219":220,"_27":221},["D",1756840062830],"children",[],[150],["D",1756839724543],[],[161],["D",1756840062842],[],[167],["D",1756840062786],[],[174],["D",1756839724543],[],[181],"0d23556e-8832-11f0-8b38-0224a6c84d84",["D",1756840899853],5,[],[187,33],"1a4d460c-8830-11f0-bfd2-0224a6c84d84","rtc.ts","01990bce-c638-714e-97a4-07196c7003e4",["D",1756840062844],"import { Hono } from \"npm:hono\";\nimport { makeHeaders, makeTranscriptionSession } from \"./utils.ts\";\n\nconst rtc = new Hono();\n\n// POST /rtc : create a new transcription call\nrtc.post(\"/\", async (c) => {\n // Get transcription configuration from query params\n const model = c.req.query(\"model\") as any || \"gpt-4o-transcribe\";\n const language = c.req.query(\"language\") || \"en\";\n const vad = c.req.query(\"vad\") !== \"false\"; // default true\n const logprobs = c.req.query(\"logprobs\") === \"true\"; // default false\n \n // Create the transcription session config\n const sessionConfig = makeTranscriptionSession({\n model,\n language,\n enableVAD: vad,\n includeLogprobs: logprobs\n });\n\n // Create the call with transcription session\n const url = \"https://api.openai.com/v1/realtime/calls\";\n const headers = makeHeaders();\n const fd = new FormData();\n fd.set(\"sdp\", await c.req.text());\n fd.set(\"session\", JSON.stringify(sessionConfig));\n \n const opts = { method: \"POST\", headers, body: fd };\n const resp = await fetch(url, opts);\n \n if (!resp.ok) {\n const errText = await resp.text().catch(() => \"\");\n console.error(`🔴 start transcription call failed: ${resp.status} ${errText}`);\n return c.text(\"Internal error\", 500);\n }\n\n const contentType = resp.headers.get(\"Content-Type\");\n const location = resp.headers.get(\"Location\");\n const callId = location?.split(\"/\").pop();\n console.log(\"✅ transcription call created:\", callId);\n\n // Kick off observer in the background to monitor transcription events\n const selfUrl = new URL(c.req.url);\n fetch(`${selfUrl.origin}/observer/${callId}`, { method: \"POST\" })\n .catch(() => {}); // catch to not crash the process\n\n return c.newResponse(resp.body, {\n headers: { \"Content-Type\": contentType },\n });\n});\n\nexport default rtc;",[],[187,240],"1a514ed2-8830-11f0-8241-0224a6c84d84","observer.ts","01990bce-c638-714e-97a4-029a75bebd75",6,["D",1756840062935],"import { Hono } from \"npm:hono\";\nimport { makeHeaders } from \"./utils.ts\";\n\nconst observer = new Hono();\n\n// POST /observer/:callId : establish websocket to observe transcription events\nobserver.post(\"/:callId\", async (c) => {\n const callId = c.req.param(\"callId\");\n console.log(`📡 Starting observer for transcription call: ${callId}`);\n \n // Connect to OpenAI WebSocket for this call\n const wsUrl = `wss://api.openai.com/v1/realtime?call_id=${callId}`;\n const ws = new WebSocket(wsUrl, {\n headers: makeHeaders(),\n });\n\n ws.onopen = () => {\n console.log(`✅ Observer connected for call: ${callId}`);\n };\n\n ws.onmessage = (event) => {\n try {\n const data = JSON.parse(event.data);\n \n // Log transcription-specific events\n if (data.type === \"conversation.item.input_audio_transcription.delta\") {\n console.log(`📝 Transcription delta [${data.item_id}]: \"${data.delta}\"`);\n } else if (data.type === \"conversation.item.input_audio_transcription.completed\") {\n console.log(`✅ Transcription completed [${data.item_id}]: \"${data.transcript}\"`);\n } else if (data.type === \"input_audio_buffer.committed\") {\n console.log(`🎤 Audio buffer committed [${data.item_id}]`);\n } else if (data.type === \"error\") {\n console.error(`❌ Error: ${data.error?.message || JSON.stringify(data.error)}`);\n }\n \n // Log other important events at debug level\n if (data.type?.startsWith(\"transcription_session\")) {\n console.log(`🔧 Session event: ${data.type}`);\n }\n } catch (e) {\n console.error(\"Failed to parse WebSocket message:\", e);\n }\n };\n\n ws.onerror = (error) => {\n console.error(`❌ WebSocket error for call ${callId}:`, error);\n };\n\n ws.onclose = () => {\n console.log(`👋 Observer disconnected for call: ${callId}`);\n };\n\n // Return immediately (observer runs in background)\n return c.text(\"Observer started\", 200);\n});\n\nexport default observer;",[],[187,247],["D",1756839724543],[210,211,212],[187],"24e371d0-8831-11f0-b1f7-0224a6c84d84","index.html","01990bce-c637-779b-aded-90c65c11e441",14,["D",1756840510205],"\n\n \n \n \n OpenAI Realtime Transcription\n \n \n \n \n\n

Transcription

\n Start speaking to see transcription here...\n

Event Logs

\n\n

\n\n \n \n",[],[193,258],["D",1756839724543],[208],[193],["D",1756839724543],{"_100":99,"_107":103},{"_100":99,"_101":-5,"_102":103,"_18":55,"_104":105,"_106":25},{"_13":86,"_32":87,"_44":43,"_88":45,"_89":90,"_91":92,"_93":51,"_94":95,"_96":65,"_97":46,"_98":99},"ctx",{"_79":-5,"_129":305,"_306":307,"_308":309,"_310":25,"_311":25,"_312":25,"_313":59,"_314":-5,"_315":-7,"_316":-7,"_317":63,"_318":25,"_319":25,"_320":-5,"_321":25,"_322":25,"_323":25,"_324":-5},"ENV",{"_276":277,"_278":279,"_280":87,"_281":282,"_283":284,"_285":286,"_287":288,"_289":290,"_291":292,"_293":294,"_295":296,"_297":298,"_299":300,"_301":302,"_303":304},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","811aa22498c9b0b5a47e8d778de579cdb990f332","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v16