Add realtime dictation providers (#1329)

* feat: add realtime dictation providers * fix: cancel realtime dictation startup * fix: refresh local dictation availability * fix: preserve dictation on disconnect * fix: normalize OpenAI language hints
2026-08-05 06:24:37 +00:00 · 2026-08-03 10:03:06 +08:00
parent 3c3bffdfbd
commit c3a5522207
21 changed files with 1378 additions and 46 deletions
@@ -190,7 +190,7 @@ On first run, HAPI:
 | `DB_PATH` | `~/.hapi/hapi.db` | - | Database file path |
 | `ELEVENLABS_API_KEY` | - | - | ElevenLabs API key for voice |
 | `ELEVENLABS_AGENT_ID` | Auto-created | - | Custom ElevenLabs agent ID |
-| `OPENAI_API_KEY` | - | - | OpenAI API key for dictation (`gpt-4o-transcribe`) |
+| `OPENAI_API_KEY` | - | - | OpenAI API key for dictation (`gpt-transcribe` / `gpt-live-transcribe`) |
 | `DEEPGRAM_API_KEY` | - | - | Deepgram API key for dictation (`nova-3`) |
 | `GROQ_API_KEY` | - | - | Groq API key for dictation (`whisper-large-v3`) |
 | `TRANSCRIPTION_BASE_URL` | - | - | OpenAI-compatible/local transcription base URL |
@@ -2,15 +2,15 @@

 Control your AI coding agent with voice using the built-in voice assistant powered by ElevenLabs Conversational AI.

-For speech-to-text without a spoken assistant, open **Settings → Voice**, choose **Dictation**, then select a configured provider. Dictation records until you tap the microphone again, inserts the transcript into the composer, and never sends it automatically. Standard mode is the default.
+For speech-to-text without a spoken assistant, open **Settings → Voice**, choose **Dictation**, then select a configured provider. Dictation records until you tap the microphone again, inserts the transcript into the composer, and never sends it automatically. Standard mode is the default. Realtime mode shows a live transcript while you speak and inserts the final result when you stop.

 Provider credentials are read only from the hub's startup environment:

 ```bash
 # Pick any providers you use
-export OPENAI_API_KEY="..."          # gpt-4o-transcribe
-export ELEVENLABS_API_KEY="..."      # scribe_v2
-export DEEPGRAM_API_KEY="..."        # nova-3
+export OPENAI_API_KEY="..."          # gpt-transcribe / gpt-live-transcribe
+export ELEVENLABS_API_KEY="..."      # scribe_v2 / scribe_v2_realtime
+export DEEPGRAM_API_KEY="..."        # nova-3 standard / realtime
 export GROQ_API_KEY="..."             # whisper-large-v3

 # Or an OpenAI-compatible local server such as Speaches
@@ -20,6 +20,7 @@ export TRANSCRIPTION_API_KEY="..."    # optional
 ```

 Restart the hub after changing credentials. API keys are not entered or stored in the web app.
+Realtime OpenAI, ElevenLabs, and Deepgram sessions receive only short-lived credentials minted by the hub. Browsers with an installed on-device `SpeechRecognition` language pack also expose **Browser on-device** as a realtime-only provider; HAPI never falls back from that option to browser-hosted recognition.

 ## Overview

@@ -28,8 +28,8 @@ See `src/configuration.ts` for all options.

 - `ELEVENLABS_API_KEY` - ElevenLabs API key for voice assistant.
 - `ELEVENLABS_AGENT_ID` - Custom ElevenLabs agent ID (auto-created if not set).
- `OPENAI_API_KEY` - OpenAI dictation (`gpt-4o-transcribe`).
- `DEEPGRAM_API_KEY` - Deepgram dictation (`nova-3`).
+- `OPENAI_API_KEY` - OpenAI dictation (`gpt-transcribe` / `gpt-live-transcribe`).
+- `DEEPGRAM_API_KEY` - Deepgram dictation (`nova-3`, standard and realtime).
 - `GROQ_API_KEY` - Groq dictation (`whisper-large-v3`).
 - `TRANSCRIPTION_BASE_URL` and `TRANSCRIPTION_MODEL` - OpenAI-compatible/local transcription endpoint and model.
 - `TRANSCRIPTION_API_KEY` - Optional bearer token for the OpenAI-compatible endpoint.
@@ -131,6 +131,9 @@ See `src/web/routes/` for all endpoints.
 ### Voice (`src/web/routes/voice.ts`)

 - `POST /api/voice/token` - Get ElevenLabs conversation token.
+- `GET /api/voice/transcription/providers` - List configured providers and supported modes.
+- `POST /api/voice/transcription` - Transcribe a bounded recording.
+- `POST /api/voice/transcription/realtime-token` - Mint a short-lived OpenAI, ElevenLabs, or Deepgram credential.

 ### Push Notifications (`src/web/routes/push.ts`)

@@ -100,7 +100,7 @@ describe('voice transcription routes', () => {
        const res = await app.request('/api/voice/transcription/providers', { headers })
        expect(res.status).toBe(200)
        expect(await res.json()).toEqual({ providers: [
-            { id: 'openai', label: 'OpenAI', modes: ['standard'] },
+            { id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] },
            { id: 'openai-compatible', label: 'OpenAI-compatible / local', modes: ['standard'] }
        ] })

@@ -144,8 +144,13 @@ describe('voice transcription routes', () => {
        expect(upstreamUrl).toBe('https://api.openai.com/v1/audio/transcriptions')
        expect(new Headers(upstreamInit?.headers).get('authorization')).toBe('Bearer server-only-key')
        expect(upstreamInit?.body).toBeInstanceOf(FormData)
-        expect((upstreamInit?.body as FormData).get('model')).toBe('gpt-4o-transcribe')
-        expect((upstreamInit?.body as FormData).get('language')).toBe('zh')
+        expect((upstreamInit?.body as FormData).get('model')).toBe('gpt-transcribe')
+        expect((upstreamInit?.body as FormData).get('languages[]')).toBe('zh-cn')
+
+        form.set('language', 'en-US')
+        const englishRes = await app.request('/api/voice/transcription', { method: 'POST', headers, body: form })
+        expect(englishRes.status).toBe(200)
+        expect((upstreamInit?.body as FormData).get('languages[]')).toBe('en')

        global.fetch = originalFetch
        if (previousKey === undefined) delete process.env.OPENAI_API_KEY
@@ -179,6 +184,78 @@ describe('voice transcription routes', () => {
        expect(res.status).toBe(413)
        expect(await res.json()).toEqual({ error: 'Audio file too large' })
    })
+
+    test('mints provider-specific short-lived realtime credentials without exposing API keys', async () => {
+        const app = createApp()
+        const headers = { ...(await authHeaders()), 'content-type': 'application/json' }
+        const previous = {
+            openai: process.env.OPENAI_API_KEY,
+            elevenlabs: process.env.ELEVENLABS_API_KEY,
+            deepgram: process.env.DEEPGRAM_API_KEY
+        }
+        process.env.OPENAI_API_KEY = 'openai-server-key'
+        process.env.ELEVENLABS_API_KEY = 'elevenlabs-server-key'
+        process.env.DEEPGRAM_API_KEY = 'deepgram-server-key'
+        const originalFetch = global.fetch
+        const requests: Array<{ url: string; init?: RequestInit }> = []
+        // @ts-expect-error test override
+        global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+            const url = String(input)
+            requests.push({ url, init })
+            if (url.endsWith('/realtime/client_secrets')) {
+                return new Response(JSON.stringify({ value: 'openai-client-token' }), { status: 200 })
+            }
+            if (url.endsWith('/single-use-token/realtime_scribe')) {
+                return new Response(JSON.stringify({ token: 'elevenlabs-client-token' }), { status: 200 })
+            }
+            return new Response(JSON.stringify({ access_token: 'deepgram-client-token' }), { status: 200 })
+        }) as typeof fetch
+
+        for (const provider of ['openai', 'elevenlabs', 'deepgram'] as const) {
+            const res = await app.request('/api/voice/transcription/realtime-token', {
+                method: 'POST',
+                headers,
+                body: JSON.stringify({ provider, language: 'zh-TW' })
+            })
+            expect(res.status).toBe(200)
+            expect(await res.json()).toEqual({ token: `${provider}-client-token` })
+        }
+        const englishOpenAI = await app.request('/api/voice/transcription/realtime-token', {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({ provider: 'openai', language: 'en-US' })
+        })
+        expect(englishOpenAI.status).toBe(200)
+
+        expect(requests.map((request) => request.url)).toEqual([
+            'https://api.openai.com/v1/realtime/client_secrets',
+            'https://api.elevenlabs.io/v1/single-use-token/realtime_scribe',
+            'https://api.deepgram.com/v1/auth/grant',
+            'https://api.openai.com/v1/realtime/client_secrets'
+        ])
+        expect(new Headers(requests[0]?.init?.headers).get('authorization')).toBe('Bearer openai-server-key')
+        expect(JSON.parse(String(requests[0]?.init?.body))).toMatchObject({
+            session: {
+                type: 'transcription',
+                audio: { input: { transcription: { model: 'gpt-live-transcribe', languages: ['zh-tw'] } } }
+            }
+        })
+        expect(JSON.parse(String(requests[3]?.init?.body))).toMatchObject({
+            session: { audio: { input: { transcription: { languages: ['en'] } } } }
+        })
+        expect(new Headers(requests[1]?.init?.headers).get('xi-api-key')).toBe('elevenlabs-server-key')
+        expect(new Headers(requests[2]?.init?.headers).get('authorization')).toBe('Token deepgram-server-key')
+
+        global.fetch = originalFetch
+        for (const [key, value] of Object.entries({
+            OPENAI_API_KEY: previous.openai,
+            ELEVENLABS_API_KEY: previous.elevenlabs,
+            DEEPGRAM_API_KEY: previous.deepgram
+        })) {
+            if (value === undefined) delete process.env[key]
+            else process.env[key] = value
+        }
+    })
 })

 describe('POST /api/voice/token', () => {
@@ -3,7 +3,13 @@ import { bodyLimit } from 'hono/body-limit'
 import { z } from 'zod'
 import type { WebAppEnv } from '../middleware/auth'
 import {
+    DEEPGRAM_TRANSCRIPTION_MODEL,
+    ELEVENLABS_REALTIME_TRANSCRIPTION_MODEL,
+    ELEVENLABS_TRANSCRIPTION_MODEL,
    ELEVENLABS_API_BASE,
+    GROQ_TRANSCRIPTION_MODEL,
+    OPENAI_REALTIME_TRANSCRIPTION_MODEL,
+    OPENAI_TRANSCRIPTION_MODEL,
    VOICE_AGENT_NAME,
    buildVoiceAgentConfig,
    listConfiguredTranscriptionProviders,
@@ -43,6 +49,7 @@ const transcriptionProviderSchema = z.enum([
    'groq',
    'openai-compatible'
 ])
+const realtimeTranscriptionProviderSchema = z.enum(['openai', 'elevenlabs', 'deepgram'])
 const transcriptionLanguageSchema = z.string()
    .trim()
    .max(35)
@@ -52,11 +59,19 @@ const transcriptionLanguageSchema = z.string()
 const MAX_TRANSCRIPTION_BYTES = 25 * 1024 * 1024
 const MAX_TRANSCRIPTION_BODY_BYTES = MAX_TRANSCRIPTION_BYTES + 1024 * 1024
 const TRANSCRIPTION_TIMEOUT_MS = 120_000
+const REALTIME_TOKEN_TIMEOUT_MS = 15_000
+const MAX_REALTIME_TOKEN_BODY_BYTES = 8 * 1024

 function trimTrailingSlash(value: string): string {
    return value.replace(/\/+$/, '')
 }

+function normalizeOpenAILanguage(language?: string): string | undefined {
+    const value = language?.toLowerCase()
+    if (!value) return undefined
+    return ['zh-cn', 'zh-tw', 'zh-hk'].includes(value) ? value : value.split('-')[0]
+}
+
 function getTranscriptionConfig(provider: TranscriptionProvider): {
    apiKey?: string
    baseUrl: string
@@ -65,13 +80,13 @@ function getTranscriptionConfig(provider: TranscriptionProvider): {
    if (!listConfiguredTranscriptionProviders(process.env).some((candidate) => candidate.id === provider)) return null
    switch (provider) {
        case 'openai':
-            return { apiKey: process.env.OPENAI_API_KEY!.trim(), baseUrl: 'https://api.openai.com/v1', model: 'gpt-4o-transcribe' }
+            return { apiKey: process.env.OPENAI_API_KEY!.trim(), baseUrl: 'https://api.openai.com/v1', model: OPENAI_TRANSCRIPTION_MODEL }
        case 'elevenlabs':
-            return { apiKey: process.env.ELEVENLABS_API_KEY!.trim(), baseUrl: ELEVENLABS_API_BASE, model: 'scribe_v2' }
+            return { apiKey: process.env.ELEVENLABS_API_KEY!.trim(), baseUrl: ELEVENLABS_API_BASE, model: ELEVENLABS_TRANSCRIPTION_MODEL }
        case 'deepgram':
-            return { apiKey: process.env.DEEPGRAM_API_KEY!.trim(), baseUrl: 'https://api.deepgram.com/v1', model: 'nova-3' }
+            return { apiKey: process.env.DEEPGRAM_API_KEY!.trim(), baseUrl: 'https://api.deepgram.com/v1', model: DEEPGRAM_TRANSCRIPTION_MODEL }
        case 'groq':
-            return { apiKey: process.env.GROQ_API_KEY!.trim(), baseUrl: 'https://api.groq.com/openai/v1', model: 'whisper-large-v3' }
+            return { apiKey: process.env.GROQ_API_KEY!.trim(), baseUrl: 'https://api.groq.com/openai/v1', model: GROQ_TRANSCRIPTION_MODEL }
        case 'openai-compatible': {
            const baseUrl = process.env.TRANSCRIPTION_BASE_URL?.trim()
            const model = process.env.TRANSCRIPTION_MODEL?.trim()
@@ -79,6 +94,8 @@ function getTranscriptionConfig(provider: TranscriptionProvider): {
                ? { apiKey: process.env.TRANSCRIPTION_API_KEY?.trim(), baseUrl: trimTrailingSlash(baseUrl), model }
                : null
        }
+        case 'browser-local':
+            return null
    }
 }

@@ -110,6 +127,7 @@ async function transcribeStandard(
        form.set(provider === 'elevenlabs' ? 'model_id' : 'model', config.model)
        if (baseLanguage) {
            if (provider === 'elevenlabs') form.set('language_code', baseLanguage)
+            else if (provider === 'openai') form.set('languages[]', normalizeOpenAILanguage(language) ?? baseLanguage)
            else form.set('language', baseLanguage)
        }
        url = provider === 'elevenlabs'
@@ -151,6 +169,76 @@ async function transcribeStandard(
    }
 }

+async function createRealtimeTranscriptionToken(
+    provider: z.infer<typeof realtimeTranscriptionProviderSchema>,
+    language?: string
+): Promise<Response> {
+    const config = getTranscriptionConfig(provider)
+    if (!config?.apiKey) return Response.json({ error: `${provider} transcription is not configured` }, { status: 400 })
+
+    let url: string
+    let headers: Record<string, string>
+    let body: string | undefined
+
+    switch (provider) {
+        case 'openai': {
+            const openAILanguage = normalizeOpenAILanguage(language)
+            const languages = openAILanguage ? [openAILanguage] : undefined
+            url = 'https://api.openai.com/v1/realtime/client_secrets'
+            headers = { Authorization: `Bearer ${config.apiKey}`, 'Content-Type': 'application/json' }
+            body = JSON.stringify({
+                expires_after: { anchor: 'created_at', seconds: 60 },
+                session: {
+                    type: 'transcription',
+                    audio: {
+                        input: {
+                            transcription: {
+                                model: OPENAI_REALTIME_TRANSCRIPTION_MODEL,
+                                delay: 'low',
+                                ...(languages ? { languages } : {})
+                            },
+                            turn_detection: null
+                        }
+                    }
+                }
+            })
+            break
+        }
+        case 'elevenlabs':
+            url = `${ELEVENLABS_API_BASE}/single-use-token/realtime_scribe`
+            headers = { 'xi-api-key': config.apiKey }
+            break
+        case 'deepgram':
+            url = 'https://api.deepgram.com/v1/auth/grant'
+            headers = { Authorization: `Token ${config.apiKey}`, 'Content-Type': 'application/json' }
+            body = '{}'
+            break
+    }
+
+    try {
+        const response = await fetch(url, {
+            method: 'POST',
+            headers,
+            body,
+            signal: AbortSignal.timeout(REALTIME_TOKEN_TIMEOUT_MS)
+        })
+        if (!response.ok) {
+            console.warn('[Voice][RealtimeTranscription] Token request failed', { provider, status: response.status })
+            return Response.json({ error: `${provider} realtime transcription token failed (HTTP ${response.status})` }, { status: 502 })
+        }
+        const data = await response.json() as { value?: string; token?: string; access_token?: string }
+        const token = data.value ?? data.token ?? data.access_token
+        if (!token) return Response.json({ error: `${provider} realtime transcription returned no token` }, { status: 502 })
+        return Response.json({ token })
+    } catch (error) {
+        console.warn('[Voice][RealtimeTranscription] Token request error', {
+            provider,
+            error: error instanceof Error ? error.message : String(error)
+        })
+        return Response.json({ error: `${provider} realtime transcription token request failed` }, { status: 502 })
+    }
+}
+
 // Cache for auto-created agent IDs (keyed by API key hash)
 const agentIdCache = new Map<string, string>()

@@ -404,6 +492,18 @@ export function createVoiceRoutes(): Hono<WebAppEnv> {
        return transcribeStandard(provider.data, file, language.data)
    })

+    app.post('/voice/transcription/realtime-token', bodyLimit({
+        maxSize: MAX_REALTIME_TOKEN_BODY_BYTES,
+        onError: (c) => c.json({ error: 'Realtime token request too large' }, 413)
+    }), async (c) => {
+        const json = await c.req.json().catch(() => null)
+        const provider = realtimeTranscriptionProviderSchema.safeParse(json?.provider)
+        const language = transcriptionLanguageSchema.safeParse(json?.language || undefined)
+        if (!provider.success) return c.json({ error: 'Invalid realtime transcription provider' }, 400)
+        if (!language.success) return c.json({ error: 'Invalid transcription language' }, 400)
+        return createRealtimeTranscriptionToken(provider.data, language.data)
+    })
+
    // Get Gemini API key for Gemini Live voice sessions
    // Gemini Live API does not support ephemeral tokens, so we proxy the key.
    // The key is short-lived in the browser session and never persisted client-side.
@@ -11,11 +11,13 @@ describe('listConfiguredTranscriptionProviders', () => {
        expect(listConfiguredTranscriptionProviders({
            OPENAI_API_KEY: 'openai',
            ELEVENLABS_API_KEY: 'elevenlabs',
+            DEEPGRAM_API_KEY: 'deepgram',
            TRANSCRIPTION_BASE_URL: 'http://localhost:8000/v1',
            TRANSCRIPTION_MODEL: 'whisper-large-v3'
        })).toEqual([
-            { id: 'openai', label: 'OpenAI', modes: ['standard'] },
-            { id: 'elevenlabs', label: 'ElevenLabs', modes: ['standard'] },
+            { id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] },
+            { id: 'elevenlabs', label: 'ElevenLabs', modes: ['standard', 'realtime'] },
+            { id: 'deepgram', label: 'Deepgram', modes: ['standard', 'realtime'] },
            { id: 'openai-compatible', label: 'OpenAI-compatible / local', modes: ['standard'] }
        ])
    })
@@ -237,7 +237,14 @@ export type VoiceBackendType = 'elevenlabs' | 'gemini-live' | 'qwen-realtime'

 export type VoiceMode = 'assistant' | 'dictation'
 export type TranscriptionMode = 'standard' | 'realtime'
-export type TranscriptionProvider = 'openai' | 'elevenlabs' | 'deepgram' | 'groq' | 'openai-compatible'
+export type TranscriptionProvider = 'openai' | 'elevenlabs' | 'deepgram' | 'groq' | 'openai-compatible' | 'browser-local'
+
+export const OPENAI_TRANSCRIPTION_MODEL = 'gpt-transcribe'
+export const OPENAI_REALTIME_TRANSCRIPTION_MODEL = 'gpt-live-transcribe'
+export const ELEVENLABS_TRANSCRIPTION_MODEL = 'scribe_v2'
+export const ELEVENLABS_REALTIME_TRANSCRIPTION_MODEL = 'scribe_v2_realtime'
+export const DEEPGRAM_TRANSCRIPTION_MODEL = 'nova-3'
+export const GROQ_TRANSCRIPTION_MODEL = 'whisper-large-v3'

 export interface TranscriptionProviderInfo {
    id: TranscriptionProvider
@@ -246,13 +253,16 @@ export interface TranscriptionProviderInfo {
 }

 const TRANSCRIPTION_PROVIDERS: Record<TranscriptionProvider, TranscriptionProviderInfo> = {
-    openai: { id: 'openai', label: 'OpenAI', modes: ['standard'] },
-    elevenlabs: { id: 'elevenlabs', label: 'ElevenLabs', modes: ['standard'] },
-    deepgram: { id: 'deepgram', label: 'Deepgram', modes: ['standard'] },
+    openai: { id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] },
+    elevenlabs: { id: 'elevenlabs', label: 'ElevenLabs', modes: ['standard', 'realtime'] },
+    deepgram: { id: 'deepgram', label: 'Deepgram', modes: ['standard', 'realtime'] },
    groq: { id: 'groq', label: 'Groq', modes: ['standard'] },
-    'openai-compatible': { id: 'openai-compatible', label: 'OpenAI-compatible / local', modes: ['standard'] }
+    'openai-compatible': { id: 'openai-compatible', label: 'OpenAI-compatible / local', modes: ['standard'] },
+    'browser-local': { id: 'browser-local', label: 'Browser on-device', modes: ['realtime'] }
 }

+export const BROWSER_LOCAL_TRANSCRIPTION_PROVIDER = TRANSCRIPTION_PROVIDERS['browser-local']
+
 /** Transcription providers whose startup environment is complete. */
 export function listConfiguredTranscriptionProviders(env: VoiceBackendEnv): TranscriptionProviderInfo[] {
    const providers: TranscriptionProvider[] = []
@@ -85,6 +85,7 @@ See `src/router.tsx` for route definitions.

 - ElevenLabs integration (@elevenlabs/react)
 - Real-time voice control
+- Standard and realtime composer dictation with provider capability selection

 ### New session (`src/components/NewSession/`)

@@ -972,6 +972,18 @@ export class ApiClient {
        return await this.request('/api/voice/transcription', { method: 'POST', body: form })
    }

+    async fetchRealtimeTranscriptionToken(
+        provider: 'openai' | 'elevenlabs' | 'deepgram',
+        language?: string,
+        signal?: AbortSignal
+    ): Promise<{ token: string }> {
+        return await this.request('/api/voice/transcription/realtime-token', {
+            method: 'POST',
+            signal,
+            body: JSON.stringify({ provider, language })
+        })
+    }
+
    async fetchQwenToken(): Promise<{
        allowed: boolean
        wsUrl?: string
@@ -1632,6 +1632,16 @@ export function HappyComposer(props: {
                        voiceStatus={effectiveVoiceStatus}
                    />

+                    {dictationActive && dictation.partialTranscript ? (
+                        <div
+                            role="status"
+                            aria-live="polite"
+                            className="mb-2 max-h-20 overflow-y-auto rounded-md bg-[var(--app-subtle-bg)] px-3 py-2 text-sm text-[var(--app-fg)]"
+                        >
+                            {dictation.partialTranscript}
+                        </div>
+                    ) : null}
+
                    {dictationActive && dictation.error ? (
                        <div role="alert" className="mb-2 rounded-md bg-[var(--app-subtle-bg)] px-3 py-2 text-sm text-red-600">
                            {dictation.error}
@@ -0,0 +1,205 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { startDeepgramRealtimeTranscription, startOpenAIRealtimeTranscription } from './realtimeTranscription'
+
+describe('OpenAI realtime transcription', () => {
+    afterEach(() => vi.unstubAllGlobals())
+
+    it('streams partial text and returns the committed final transcript', async () => {
+        const stopTrack = vi.fn()
+        Object.defineProperty(navigator, 'mediaDevices', {
+            configurable: true,
+            value: {
+                getUserMedia: vi.fn(async () => ({
+                    getTracks: () => [{ stop: stopTrack }],
+                    getAudioTracks: () => [{ stop: stopTrack }]
+                }))
+            }
+        })
+
+        class MockDataChannel extends EventTarget {
+            readyState = 'open'
+            send = vi.fn()
+            close = vi.fn()
+        }
+        const channel = new MockDataChannel()
+        class MockPeerConnection extends EventTarget {
+            connectionState = 'connected'
+            createDataChannel() { return channel }
+            addTrack() {}
+            async createOffer() { return { type: 'offer', sdp: 'offer-sdp' } }
+            async setLocalDescription() {}
+            async setRemoteDescription() {}
+            close() {}
+        }
+        vi.stubGlobal('RTCPeerConnection', MockPeerConnection)
+        vi.stubGlobal('fetch', vi.fn(async () => new Response('answer-sdp', { status: 200 })))
+
+        const callbacks = {
+            onConnected: vi.fn(),
+            onPartial: vi.fn(),
+            onFinal: vi.fn(),
+            onError: vi.fn()
+        }
+        const session = await startOpenAIRealtimeTranscription({
+            getToken: async () => 'ephemeral-token',
+            callbacks
+        })
+        channel.dispatchEvent(new MessageEvent('message', {
+            data: JSON.stringify({
+                type: 'conversation.item.input_audio_transcription.delta',
+                delta: 'live text'
+            })
+        }))
+        expect(callbacks.onPartial).toHaveBeenCalledWith('live text')
+
+        const stopping = session.stop()
+        setTimeout(() => channel.dispatchEvent(new MessageEvent('message', {
+            data: JSON.stringify({
+                type: 'conversation.item.input_audio_transcription.completed',
+                transcript: 'final text'
+            })
+        })), 0)
+        await stopping
+
+        expect(callbacks.onFinal).toHaveBeenCalledWith('final text')
+        expect(callbacks.onError).not.toHaveBeenCalled()
+        expect(stopTrack).toHaveBeenCalled()
+    })
+
+    it('releases the microphone when startup is aborted', async () => {
+        const stopTrack = vi.fn()
+        Object.defineProperty(navigator, 'mediaDevices', {
+            configurable: true,
+            value: {
+                getUserMedia: vi.fn(async () => ({
+                    getTracks: () => [{ stop: stopTrack }],
+                    getAudioTracks: () => [{ stop: stopTrack }]
+                }))
+            }
+        })
+        class MockDataChannel extends EventTarget {
+            readyState = 'connecting'
+            close() {}
+        }
+        class MockPeerConnection extends EventTarget {
+            connectionState = 'connecting'
+            createDataChannel() { return new MockDataChannel() }
+            addTrack() {}
+            async createOffer() { return { type: 'offer', sdp: 'offer-sdp' } }
+            async setLocalDescription() {}
+            close() {}
+        }
+        vi.stubGlobal('RTCPeerConnection', MockPeerConnection)
+        const fetchMock = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => new Promise<Response>((_resolve, reject) => {
+            init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true })
+        }))
+        vi.stubGlobal('fetch', fetchMock)
+        const controller = new AbortController()
+        const starting = startOpenAIRealtimeTranscription({
+            getToken: async () => 'ephemeral-token',
+            signal: controller.signal,
+            callbacks: {
+                onConnected: vi.fn(),
+                onPartial: vi.fn(),
+                onFinal: vi.fn(),
+                onError: vi.fn()
+            }
+        })
+        await vi.waitFor(() => expect(fetchMock).toHaveBeenCalled())
+
+        controller.abort()
+
+        await expect(starting).rejects.toBeDefined()
+        expect(stopTrack).toHaveBeenCalled()
+    })
+})
+
+describe('Deepgram realtime transcription', () => {
+    afterEach(() => vi.unstubAllGlobals())
+
+    it('streams recorder chunks and returns the finalized transcript', async () => {
+        const stopTrack = vi.fn()
+        Object.defineProperty(navigator, 'mediaDevices', {
+            configurable: true,
+            value: {
+                getUserMedia: vi.fn(async () => ({
+                    getTracks: () => [{ stop: stopTrack }]
+                }))
+            }
+        })
+
+        class MockSocket extends EventTarget {
+            static OPEN = 1
+            readyState = 0
+            sent: unknown[] = []
+            constructor(readonly url: string, readonly protocols: string[]) {
+                super()
+                queueMicrotask(() => {
+                    this.readyState = MockSocket.OPEN
+                    this.dispatchEvent(new Event('open'))
+                })
+            }
+            send(value: unknown) { this.sent.push(value) }
+            close() { this.readyState = 3 }
+            result(transcript: string, final: boolean, fromFinalize = false) {
+                this.dispatchEvent(new MessageEvent('message', {
+                    data: JSON.stringify({
+                        type: 'Results',
+                        is_final: final,
+                        from_finalize: fromFinalize,
+                        channel: { alternatives: [{ transcript }] }
+                    })
+                }))
+            }
+        }
+        const sockets: MockSocket[] = []
+        class MockWebSocket extends MockSocket {
+            constructor(url: string, protocols: string[]) {
+                super(url, protocols)
+                sockets.push(this)
+            }
+        }
+        Object.assign(MockWebSocket, { OPEN: MockSocket.OPEN })
+        vi.stubGlobal('WebSocket', MockWebSocket)
+
+        class MockRecorder {
+            static isTypeSupported() { return true }
+            state = 'inactive'
+            mimeType = 'audio/webm;codecs=opus'
+            ondataavailable: ((event: { data: Blob }) => void) | null = null
+            onerror: (() => void) | null = null
+            onstop: (() => void) | null = null
+            start() {
+                this.state = 'recording'
+                this.ondataavailable?.({ data: new Blob(['audio']) })
+            }
+            stop() {
+                this.state = 'inactive'
+                this.onstop?.()
+            }
+        }
+        vi.stubGlobal('MediaRecorder', MockRecorder)
+
+        const callbacks = {
+            onConnected: vi.fn(),
+            onPartial: vi.fn(),
+            onFinal: vi.fn(),
+            onError: vi.fn()
+        }
+        const session = await startDeepgramRealtimeTranscription({
+            getToken: async () => 'temporary-jwt',
+            callbacks
+        })
+        const socket = sockets[0]!
+        expect(socket.protocols).toEqual(['bearer', 'temporary-jwt'])
+        socket.result('live', false)
+        const stopping = session.stop()
+        socket.result('final text', true, true)
+        await stopping
+
+        expect(callbacks.onPartial).toHaveBeenCalledWith('live')
+        expect(callbacks.onFinal).toHaveBeenCalledWith('final text')
+        expect(callbacks.onError).not.toHaveBeenCalled()
+        expect(stopTrack).toHaveBeenCalled()
+    })
+})
@@ -0,0 +1,452 @@
+import { DEEPGRAM_TRANSCRIPTION_MODEL } from '@hapi/protocol/voice'
+
+export interface RealtimeTranscriptionCallbacks {
+    onConnected: () => void
+    onPartial: (text: string) => void
+    onFinal: (text: string) => void
+    onError: (error: Error) => void
+}
+
+export interface RealtimeTranscriptionSession {
+    stop: () => Promise<void>
+    cancel: () => void
+}
+
+type TokenFactory = () => Promise<string>
+
+function joinTranscriptParts(...parts: string[]): string {
+    return parts.map((part) => part.trim()).filter(Boolean).join(' ')
+}
+
+function errorMessage(value: unknown, fallback: string): Error {
+    return value instanceof Error ? value : new Error(fallback)
+}
+
+export async function startOpenAIRealtimeTranscription(options: {
+    getToken: TokenFactory
+    signal?: AbortSignal
+    callbacks: RealtimeTranscriptionCallbacks
+}): Promise<RealtimeTranscriptionSession> {
+    const stream = await navigator.mediaDevices.getUserMedia({
+        audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }
+    })
+    try {
+        options.signal?.throwIfAborted()
+    } catch (error) {
+        stream.getTracks().forEach((track) => track.stop())
+        throw error
+    }
+    let peer: RTCPeerConnection
+    let channel: RTCDataChannel
+    try {
+        peer = new RTCPeerConnection()
+        channel = peer.createDataChannel('oai-events')
+    } catch (error) {
+        stream.getTracks().forEach((track) => track.stop())
+        throw error
+    }
+    let partial = ''
+    let finished = false
+    let stopping = false
+    let stopTimer: ReturnType<typeof setTimeout> | null = null
+
+    const cleanup = () => {
+        if (stopTimer) clearTimeout(stopTimer)
+        stopTimer = null
+        stream.getTracks().forEach((track) => track.stop())
+        channel.close()
+        peer.close()
+    }
+    const finish = (text: string) => {
+        if (finished) return
+        finished = true
+        cleanup()
+        options.callbacks.onFinal(text)
+    }
+    const fail = (value: unknown) => {
+        if (finished) return
+        finished = true
+        cleanup()
+        options.callbacks.onError(errorMessage(value, 'OpenAI realtime transcription failed'))
+    }
+
+    try {
+        const track = stream.getAudioTracks()[0]
+        if (!track) throw new Error('No microphone audio track is available')
+        peer.addTrack(track, stream)
+        channel.addEventListener('message', (event) => {
+            let data: {
+                type?: string
+                delta?: string
+                transcript?: string
+                error?: { message?: string }
+            }
+            try {
+                data = JSON.parse(String(event.data)) as typeof data
+            } catch {
+                return
+            }
+            if (data.type === 'conversation.item.input_audio_transcription.delta' && data.delta) {
+                partial += data.delta
+                options.callbacks.onPartial(partial)
+            } else if (data.type === 'conversation.item.input_audio_transcription.completed') {
+                finish(data.transcript ?? partial)
+            } else if (data.type === 'error') {
+                fail(new Error(data.error?.message || 'OpenAI realtime transcription failed'))
+            }
+        })
+        peer.addEventListener('connectionstatechange', () => {
+            if (!finished && (peer.connectionState === 'failed' || peer.connectionState === 'closed')) {
+                fail(new Error('OpenAI realtime transcription disconnected'))
+            }
+        })
+
+        const offer = await peer.createOffer()
+        await peer.setLocalDescription(offer)
+        const token = await options.getToken()
+        const response = await fetch('https://api.openai.com/v1/realtime/calls', {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${token}`,
+                'Content-Type': 'application/sdp'
+            },
+            body: offer.sdp,
+            signal: options.signal
+                ? AbortSignal.any([options.signal, AbortSignal.timeout(15_000)])
+                : AbortSignal.timeout(15_000)
+        })
+        if (!response.ok) throw new Error(`OpenAI realtime connection failed (HTTP ${response.status})`)
+        await peer.setRemoteDescription({ type: 'answer', sdp: await response.text() })
+        await new Promise<void>((resolve, reject) => {
+            if (channel.readyState === 'open') return resolve()
+            const timeout = setTimeout(() => reject(new Error('OpenAI realtime connection timed out')), 10_000)
+            const onAbort = () => {
+                clearTimeout(timeout)
+                reject(options.signal?.reason)
+            }
+            options.signal?.addEventListener('abort', onAbort, { once: true })
+            channel.addEventListener('open', () => {
+                clearTimeout(timeout)
+                options.signal?.removeEventListener('abort', onAbort)
+                resolve()
+            }, { once: true })
+            channel.addEventListener('error', () => {
+                clearTimeout(timeout)
+                options.signal?.removeEventListener('abort', onAbort)
+                reject(new Error('OpenAI realtime connection failed'))
+            }, { once: true })
+        })
+        options.callbacks.onConnected()
+    } catch (error) {
+        fail(error)
+        throw error
+    }
+
+    return {
+        stop: async () => {
+            if (finished || stopping) return
+            stopping = true
+            stream.getTracks().forEach((track) => track.stop())
+            if (channel.readyState !== 'open') return finish(partial)
+            channel.send(JSON.stringify({ type: 'input_audio_buffer.commit' }))
+            await new Promise<void>((resolve) => {
+                stopTimer = setTimeout(() => {
+                    clearInterval(check)
+                    finish(partial)
+                    resolve()
+                }, 2_500)
+                const check = setInterval(() => {
+                    if (!finished) return
+                    clearInterval(check)
+                    resolve()
+                }, 25)
+            })
+        },
+        cancel: () => {
+            if (finished) return
+            finished = true
+            cleanup()
+        }
+    }
+}
+
+export async function startDeepgramRealtimeTranscription(options: {
+    getToken: TokenFactory
+    language?: string
+    signal?: AbortSignal
+    callbacks: RealtimeTranscriptionCallbacks
+}): Promise<RealtimeTranscriptionSession> {
+    const stream = await navigator.mediaDevices.getUserMedia({
+        audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }
+    })
+    try {
+        options.signal?.throwIfAborted()
+    } catch (error) {
+        stream.getTracks().forEach((track) => track.stop())
+        throw error
+    }
+    let token: string
+    try {
+        token = await options.getToken()
+    } catch (error) {
+        stream.getTracks().forEach((track) => track.stop())
+        throw error
+    }
+    const query = new URLSearchParams({
+        model: DEEPGRAM_TRANSCRIPTION_MODEL,
+        smart_format: 'true',
+        interim_results: 'true',
+        endpointing: 'false'
+    })
+    if (options.language) query.set('language', options.language)
+    const mimeType = [
+        'audio/webm;codecs=opus',
+        'audio/webm',
+        'audio/ogg;codecs=opus'
+    ].find((type) => typeof MediaRecorder.isTypeSupported !== 'function' || MediaRecorder.isTypeSupported(type))
+    let socket: WebSocket
+    let recorder: MediaRecorder
+    try {
+        socket = new WebSocket(`wss://api.deepgram.com/v1/listen?${query}`, ['bearer', token])
+        recorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream)
+    } catch (error) {
+        stream.getTracks().forEach((track) => track.stop())
+        throw error
+    }
+    let committed = ''
+    let interim = ''
+    let finished = false
+    let stopping = false
+    let stopTimer: ReturnType<typeof setTimeout> | null = null
+
+    const cleanup = () => {
+        if (stopTimer) clearTimeout(stopTimer)
+        stopTimer = null
+        recorder.ondataavailable = null
+        recorder.onstop = null
+        if (recorder.state !== 'inactive') recorder.stop()
+        stream.getTracks().forEach((track) => track.stop())
+        if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify({ type: 'CloseStream' }))
+        socket.close()
+    }
+    const finish = (text: string) => {
+        if (finished) return
+        finished = true
+        cleanup()
+        options.callbacks.onFinal(text)
+    }
+    const fail = (value: unknown) => {
+        if (finished) return
+        finished = true
+        cleanup()
+        options.callbacks.onError(errorMessage(value, 'Deepgram realtime transcription failed'))
+    }
+
+    recorder.ondataavailable = (event) => {
+        if (event.data.size > 0 && socket.readyState === WebSocket.OPEN) socket.send(event.data)
+    }
+    recorder.onerror = () => fail(new Error('Audio recording failed'))
+    recorder.onstop = () => {
+        if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify({ type: 'Finalize' }))
+    }
+    socket.addEventListener('message', (event) => {
+        let data: {
+            type?: string
+            is_final?: boolean
+            from_finalize?: boolean
+            channel?: { alternatives?: Array<{ transcript?: string }> }
+            description?: string
+        }
+        try {
+            data = JSON.parse(String(event.data)) as typeof data
+        } catch {
+            return
+        }
+        if (data.type === 'Error') return fail(new Error(data.description || 'Deepgram realtime transcription failed'))
+        if (data.type !== 'Results') return
+        const text = data.channel?.alternatives?.[0]?.transcript ?? ''
+        if (data.is_final) {
+            committed = joinTranscriptParts(committed, text)
+            interim = ''
+        } else {
+            interim = text
+        }
+        const current = joinTranscriptParts(committed, interim)
+        options.callbacks.onPartial(current)
+        if (stopping && data.is_final && data.from_finalize) finish(current)
+    })
+    socket.addEventListener('close', () => {
+        if (!finished && !stopping) fail(new Error('Deepgram realtime transcription disconnected'))
+    })
+
+    await new Promise<void>((resolve, reject) => {
+        const timeout = setTimeout(() => reject(new Error('Deepgram realtime connection timed out')), 10_000)
+        const onAbort = () => {
+            clearTimeout(timeout)
+            reject(options.signal?.reason)
+        }
+        options.signal?.addEventListener('abort', onAbort, { once: true })
+        socket.addEventListener('open', () => {
+            clearTimeout(timeout)
+            options.signal?.removeEventListener('abort', onAbort)
+            recorder.start(250)
+            options.callbacks.onConnected()
+            resolve()
+        }, { once: true })
+        socket.addEventListener('error', () => {
+            clearTimeout(timeout)
+            options.signal?.removeEventListener('abort', onAbort)
+            reject(new Error('Deepgram realtime connection failed'))
+        }, { once: true })
+    }).catch((error) => {
+        fail(error)
+        throw error
+    })
+
+    return {
+        stop: async () => {
+            if (finished || stopping) return
+            stopping = true
+            if (recorder.state !== 'inactive') recorder.stop()
+            else if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify({ type: 'Finalize' }))
+            await new Promise<void>((resolve) => {
+                stopTimer = setTimeout(() => {
+                    clearInterval(check)
+                    finish(joinTranscriptParts(committed, interim))
+                    resolve()
+                }, 2_500)
+                const check = setInterval(() => {
+                    if (!finished) return
+                    clearInterval(check)
+                    resolve()
+                }, 25)
+            })
+        },
+        cancel: () => {
+            if (finished) return
+            finished = true
+            cleanup()
+        }
+    }
+}
+
+interface LocalSpeechRecognitionResult {
+    readonly isFinal: boolean
+    readonly 0: { readonly transcript: string }
+}
+
+interface LocalSpeechRecognitionEvent extends Event {
+    readonly results: { readonly length: number; readonly [index: number]: LocalSpeechRecognitionResult }
+}
+
+interface LocalSpeechRecognition extends EventTarget {
+    continuous: boolean
+    interimResults: boolean
+    lang: string
+    processLocally: boolean
+    onresult: ((event: LocalSpeechRecognitionEvent) => void) | null
+    onerror: ((event: Event & { error?: string }) => void) | null
+    onend: (() => void) | null
+    start: () => void
+    stop: () => void
+    abort: () => void
+}
+
+interface LocalSpeechRecognitionConstructor {
+    new(): LocalSpeechRecognition
+    prototype: LocalSpeechRecognition
+    available: (options: { langs: string[]; processLocally: true }) => Promise<string>
+}
+
+function localSpeechRecognitionConstructor(): LocalSpeechRecognitionConstructor | null {
+    const constructor = (globalThis as typeof globalThis & {
+        SpeechRecognition?: LocalSpeechRecognitionConstructor
+    }).SpeechRecognition
+    return constructor
+        && typeof constructor.available === 'function'
+        && 'processLocally' in constructor.prototype
+        ? constructor
+        : null
+}
+
+export async function startBrowserLocalTranscription(options: {
+    language?: string
+    signal?: AbortSignal
+    callbacks: RealtimeTranscriptionCallbacks
+}): Promise<RealtimeTranscriptionSession> {
+    options.signal?.throwIfAborted()
+    const constructor = localSpeechRecognitionConstructor()
+    if (!constructor) throw new Error('On-device speech recognition is not supported by this browser')
+    const language = options.language || navigator.language
+    if (await constructor.available({ langs: [language], processLocally: true }) !== 'available') {
+        throw new Error(`On-device speech recognition is not installed for ${language}`)
+    }
+    options.signal?.throwIfAborted()
+
+    const recognition = new constructor()
+    recognition.continuous = true
+    recognition.interimResults = true
+    recognition.lang = language
+    recognition.processLocally = true
+    let current = ''
+    let finished = false
+    let stopping = false
+
+    const finish = () => {
+        if (finished) return
+        finished = true
+        options.callbacks.onFinal(current)
+    }
+    recognition.onresult = (event) => {
+        const finalParts: string[] = []
+        const interimParts: string[] = []
+        for (let index = 0; index < event.results.length; index += 1) {
+            const result = event.results[index]
+            const transcript = result?.[0]?.transcript ?? ''
+            if (result?.isFinal) finalParts.push(transcript)
+            else interimParts.push(transcript)
+        }
+        current = joinTranscriptParts(...finalParts, ...interimParts)
+        options.callbacks.onPartial(current)
+    }
+    recognition.onerror = (event) => {
+        if (event.error === 'aborted' && stopping) return
+        finished = true
+        options.callbacks.onError(new Error(event.error ? `On-device transcription failed: ${event.error}` : 'On-device transcription failed'))
+    }
+    recognition.onend = () => {
+        if (stopping) finish()
+        else if (!finished) {
+            finished = true
+            options.callbacks.onError(new Error('On-device transcription stopped'))
+        }
+    }
+    recognition.start()
+    options.callbacks.onConnected()
+
+    return {
+        stop: async () => {
+            if (finished || stopping) return
+            stopping = true
+            recognition.stop()
+            await new Promise<void>((resolve) => {
+                const timeout = setTimeout(() => {
+                    clearInterval(check)
+                    finish()
+                    resolve()
+                }, 2_500)
+                const check = setInterval(() => {
+                    if (!finished) return
+                    clearInterval(check)
+                    clearTimeout(timeout)
+                    resolve()
+                }, 25)
+            })
+        },
+        cancel: () => {
+            if (finished) return
+            finished = true
+            recognition.abort()
+        }
+    }
+}
@@ -60,4 +60,51 @@ describe('useDictation', () => {
        expect(api.transcribeVoice).toHaveBeenCalledOnce()
        expect(stopTrack).toHaveBeenCalled()
    })
+
+    it('shows on-device partial text and inserts only the final transcript', async () => {
+        let recognition: MockSpeechRecognition | null = null
+        class MockSpeechRecognition {
+            static async available() { return 'available' }
+            continuous = false
+            interimResults = false
+            lang = ''
+            processLocally = false
+            onresult: ((event: Event & { results: unknown }) => void) | null = null
+            onerror: ((event: Event) => void) | null = null
+            onend: (() => void) | null = null
+            constructor() { recognition = this }
+            start() {}
+            stop() { this.onend?.() }
+            abort() {}
+            emit(text: string, isFinal: boolean) {
+                const result = Object.assign([{ transcript: text }], { isFinal })
+                this.onresult?.({ results: [result] } as unknown as Event & { results: unknown })
+            }
+        }
+        Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', {
+            configurable: true,
+            writable: true,
+            value: false
+        })
+        vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
+
+        const onTextChange = vi.fn()
+        const { result } = renderHook(() => useDictation({
+            api: {} as ApiClient,
+            provider: 'browser-local',
+            mode: 'realtime',
+            getCurrentText: () => 'existing draft',
+            onTextChange
+        }))
+
+        await act(() => result.current.toggle())
+        act(() => recognition?.emit('live words', false))
+        expect(result.current.partialTranscript).toBe('live words')
+        expect(onTextChange).not.toHaveBeenCalled()
+        act(() => recognition?.emit('final words', true))
+        await act(() => result.current.toggle())
+
+        await waitFor(() => expect(onTextChange).toHaveBeenCalledWith('existing draft final words'))
+        expect(result.current.partialTranscript).toBe('')
+    })
 })
@@ -2,6 +2,7 @@ import { useCallback, useEffect, useRef, useState } from 'react'
 import type { ApiClient } from '@/api/client'
 import type { ConversationStatus } from '@/realtime/types'
 import type { TranscriptionMode, TranscriptionProvider } from '@hapi/protocol/voice'
+import { useRealtimeDictation } from './useRealtimeDictation'

 export function appendTranscript(text: string, transcript: string): string {
    const addition = transcript.trim()
@@ -33,10 +34,19 @@ export function useDictation(config: {
    getCurrentText: () => string
    onTextChange: (text: string) => void
 }) {
+    const onFinalTranscript = useCallback((transcript: string) => {
+        config.onTextChange(appendTranscript(config.getCurrentText(), transcript))
+    }, [config])
+    const realtime = useRealtimeDictation({
+        api: config.api,
+        provider: config.provider,
+        mode: config.mode,
+        onFinalTranscript
+    })
    const browserCanRecord = typeof navigator !== 'undefined'
        && typeof navigator.mediaDevices?.getUserMedia === 'function'
        && typeof MediaRecorder !== 'undefined'
-    const supported = config.api !== null
+    const standardSupported = config.api !== null
        && config.provider !== null
        && config.mode === 'standard'
        && browserCanRecord
@@ -55,7 +65,7 @@ export function useDictation(config: {
    }, [])

    const start = useCallback(async () => {
-        if (!supported || !config.provider || status === 'connecting' || status === 'connected') return
+        if (!standardSupported || !config.provider || status === 'connecting' || status === 'connected') return
        const operation = ++operationRef.current
        const provider = config.provider
        const language = localStorage.getItem('hapi-voice-lang') || undefined
@@ -122,7 +132,7 @@ export function useDictation(config: {
            setError(startError instanceof Error ? startError.message : 'Could not start transcription')
            setStatus('error')
        }
-    }, [config, status, stopTracks, supported])
+    }, [config, standardSupported, status, stopTracks])

    const stop = useCallback(async () => {
        if (transcribingRef.current) return
@@ -155,5 +165,7 @@ export function useDictation(config: {
        }
    }, [stopTracks])

-    return { supported, status, error, toggle }
+    return config.mode === 'realtime'
+        ? realtime
+        : { supported: standardSupported, status, error, partialTranscript: '', toggle }
 }
@@ -0,0 +1,61 @@
+import { act, renderHook, waitFor } from '@testing-library/react'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import type { ApiClient } from '@/api/client'
+import { useRealtimeDictation } from './useRealtimeDictation'
+
+const scribe = vi.hoisted(() => ({
+    options: null as unknown,
+    connect: vi.fn(async () => {}),
+    disconnect: vi.fn(),
+    commit: vi.fn()
+}))
+
+vi.mock('@elevenlabs/react', () => ({
+    CommitStrategy: { MANUAL: 'manual' },
+    useScribe: (options: unknown) => {
+        scribe.options = options
+        return {
+            connect: scribe.connect,
+            disconnect: scribe.disconnect,
+            commit: scribe.commit
+        }
+    }
+}))
+
+type ScribeCallbacks = {
+    onPartialTranscript: (event: { text: string }) => void
+    onDisconnect: () => void
+}
+
+describe('useRealtimeDictation', () => {
+    afterEach(() => vi.clearAllMocks())
+
+    it('preserves partial ElevenLabs text on an unexpected disconnect', async () => {
+        Object.defineProperty(navigator, 'mediaDevices', {
+            configurable: true,
+            value: { getUserMedia: vi.fn() }
+        })
+        const api = {
+            fetchRealtimeTranscriptionToken: vi.fn(async () => ({ token: 'single-use-token' }))
+        } as unknown as ApiClient
+        const onFinalTranscript = vi.fn()
+        const { result } = renderHook(() => useRealtimeDictation({
+            api,
+            provider: 'elevenlabs',
+            mode: 'realtime',
+            onFinalTranscript
+        }))
+
+        await act(() => result.current.toggle())
+        const callbacks = scribe.options as ScribeCallbacks
+        act(() => callbacks.onPartialTranscript({ text: 'spoken words' }))
+        expect(onFinalTranscript).not.toHaveBeenCalled()
+
+        act(() => callbacks.onDisconnect())
+
+        await waitFor(() => expect(result.current.status).toBe('error'))
+        expect(result.current.error).toBe('ElevenLabs realtime transcription disconnected')
+        expect(result.current.partialTranscript).toBe('')
+        expect(onFinalTranscript).toHaveBeenCalledWith('spoken words')
+    })
+})
@@ -0,0 +1,248 @@
+import { useCallback, useEffect, useRef, useState } from 'react'
+import { CommitStrategy, useScribe } from '@elevenlabs/react'
+import {
+    ELEVENLABS_REALTIME_TRANSCRIPTION_MODEL,
+    type TranscriptionMode,
+    type TranscriptionProvider
+} from '@hapi/protocol/voice'
+import type { ApiClient } from '@/api/client'
+import type { ConversationStatus } from '@/realtime/types'
+import {
+    startBrowserLocalTranscription,
+    startDeepgramRealtimeTranscription,
+    startOpenAIRealtimeTranscription,
+    type RealtimeTranscriptionCallbacks,
+    type RealtimeTranscriptionSession
+} from './realtimeTranscription'
+
+function realtimeBrowserSupport(provider: TranscriptionProvider | null): boolean {
+    if (typeof navigator === 'undefined') return false
+    if (provider === 'browser-local') return true
+    if (typeof navigator.mediaDevices?.getUserMedia !== 'function') return false
+    if (provider === 'openai') return typeof RTCPeerConnection !== 'undefined'
+    if (provider === 'deepgram') return typeof WebSocket !== 'undefined' && typeof MediaRecorder !== 'undefined'
+    return provider === 'elevenlabs'
+}
+
+export function useRealtimeDictation(config: {
+    api: ApiClient | null
+    provider: TranscriptionProvider | null
+    mode: TranscriptionMode
+    onFinalTranscript: (text: string) => void
+}) {
+    const supported = config.api !== null
+        && config.mode === 'realtime'
+        && ['openai', 'elevenlabs', 'deepgram', 'browser-local'].includes(config.provider ?? '')
+        && realtimeBrowserSupport(config.provider)
+    const [status, setStatus] = useState<ConversationStatus>('disconnected')
+    const [error, setError] = useState<string | null>(null)
+    const [partialTranscript, setPartialTranscript] = useState('')
+    const mountedRef = useRef(true)
+    const operationRef = useRef(0)
+    const startAbortRef = useRef<AbortController | null>(null)
+    const sessionRef = useRef<RealtimeTranscriptionSession | null>(null)
+    const partialRef = useRef('')
+    const onFinalTranscriptRef = useRef(config.onFinalTranscript)
+    const elevenLabsActiveRef = useRef(false)
+    const elevenLabsFinalizedRef = useRef(false)
+    const resolveElevenLabsCommitRef = useRef<(() => void) | null>(null)
+    onFinalTranscriptRef.current = config.onFinalTranscript
+
+    const updatePartial = useCallback((text: string) => {
+        partialRef.current = text
+        if (mountedRef.current) setPartialTranscript(text)
+    }, [])
+
+    const finish = useCallback((text: string) => {
+        if (!mountedRef.current) return
+        updatePartial('')
+        onFinalTranscriptRef.current(text)
+        setStatus('disconnected')
+    }, [updatePartial])
+
+    const fail = useCallback((value: unknown) => {
+        if (!mountedRef.current) return
+        elevenLabsActiveRef.current = false
+        resolveElevenLabsCommitRef.current?.()
+        resolveElevenLabsCommitRef.current = null
+        sessionRef.current?.cancel()
+        sessionRef.current = null
+        setError(value instanceof Error ? value.message : 'Realtime transcription failed')
+        setStatus('error')
+    }, [])
+
+    const elevenLabs = useScribe({
+        modelId: ELEVENLABS_REALTIME_TRANSCRIPTION_MODEL,
+        commitStrategy: CommitStrategy.MANUAL,
+        onSessionStarted: () => {
+            if (mountedRef.current && elevenLabsActiveRef.current) setStatus('connected')
+        },
+        onPartialTranscript: ({ text }) => {
+            if (elevenLabsActiveRef.current) updatePartial(text)
+        },
+        onCommittedTranscript: ({ text }) => {
+            if (!elevenLabsActiveRef.current) return
+            elevenLabsFinalizedRef.current = true
+            elevenLabsActiveRef.current = false
+            finish(text)
+            resolveElevenLabsCommitRef.current?.()
+            resolveElevenLabsCommitRef.current = null
+        },
+        onError: (scribeError) => {
+            if (elevenLabsActiveRef.current) fail(scribeError)
+        },
+        onDisconnect: () => {
+            if (!mountedRef.current || !elevenLabsActiveRef.current) return
+            const partial = partialRef.current
+            elevenLabsActiveRef.current = false
+            resolveElevenLabsCommitRef.current?.()
+            resolveElevenLabsCommitRef.current = null
+            sessionRef.current = null
+            finish(partial)
+            setError('ElevenLabs realtime transcription disconnected')
+            setStatus('error')
+        }
+    })
+    const elevenLabsRef = useRef(elevenLabs)
+    elevenLabsRef.current = elevenLabs
+
+    const start = useCallback(async () => {
+        if (!supported || !config.api || !config.provider || status === 'connecting' || status === 'connected') return
+        const operation = ++operationRef.current
+        const controller = new AbortController()
+        startAbortRef.current = controller
+        const provider = config.provider
+        const language = localStorage.getItem('hapi-voice-lang') || undefined
+        setError(null)
+        updatePartial('')
+        setStatus('connecting')
+
+        const callbacks: RealtimeTranscriptionCallbacks = {
+            onConnected: () => {
+                if (mountedRef.current && operationRef.current === operation) setStatus('connected')
+            },
+            onPartial: (text) => {
+                if (operationRef.current === operation) updatePartial(text)
+            },
+            onFinal: (text) => {
+                if (operationRef.current === operation) finish(text)
+            },
+            onError: (realtimeError) => {
+                if (operationRef.current === operation) fail(realtimeError)
+            }
+        }
+
+        try {
+            if (provider === 'elevenlabs') {
+                const { token } = await config.api.fetchRealtimeTranscriptionToken('elevenlabs', language, controller.signal)
+                if (operationRef.current !== operation) return
+                elevenLabsActiveRef.current = true
+                elevenLabsFinalizedRef.current = false
+                await elevenLabs.connect({
+                    token,
+                    modelId: ELEVENLABS_REALTIME_TRANSCRIPTION_MODEL,
+                    commitStrategy: CommitStrategy.MANUAL,
+                    languageCode: language?.split('-')[0]?.toLowerCase(),
+                    microphone: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }
+                })
+                if (operationRef.current !== operation) {
+                    elevenLabsActiveRef.current = false
+                    elevenLabs.disconnect()
+                    return
+                }
+                sessionRef.current = {
+                    stop: async () => {
+                        setStatus('connecting')
+                        const committed = new Promise<void>((resolve) => {
+                            resolveElevenLabsCommitRef.current = resolve
+                        })
+                        try {
+                            elevenLabs.commit()
+                            await Promise.race([
+                                committed,
+                                new Promise<void>((resolve) => setTimeout(resolve, 2_500))
+                            ])
+                        } finally {
+                            if (elevenLabsActiveRef.current && !elevenLabsFinalizedRef.current) finish(partialRef.current)
+                            elevenLabsActiveRef.current = false
+                            resolveElevenLabsCommitRef.current = null
+                            elevenLabs.disconnect()
+                        }
+                    },
+                    cancel: () => {
+                        elevenLabsActiveRef.current = false
+                        elevenLabs.disconnect()
+                    }
+                }
+                return
+            }
+
+            const getToken = async () => {
+                if (operationRef.current !== operation) throw new Error('Realtime transcription cancelled')
+                const result = await config.api!.fetchRealtimeTranscriptionToken(
+                    provider as 'openai' | 'deepgram',
+                    language,
+                    controller.signal
+                )
+                if (operationRef.current !== operation) throw new Error('Realtime transcription cancelled')
+                return result.token
+            }
+            const session = provider === 'openai'
+                ? await startOpenAIRealtimeTranscription({ getToken, signal: controller.signal, callbacks })
+                : provider === 'deepgram'
+                    ? await startDeepgramRealtimeTranscription({ getToken, language, signal: controller.signal, callbacks })
+                    : await startBrowserLocalTranscription({ language, signal: controller.signal, callbacks })
+            if (operationRef.current !== operation) session.cancel()
+            else sessionRef.current = session
+        } catch (startError) {
+            if (operationRef.current === operation) fail(startError)
+        } finally {
+            if (startAbortRef.current === controller) startAbortRef.current = null
+        }
+    }, [config.api, config.provider, elevenLabs, fail, finish, status, supported, updatePartial])
+
+    const stop = useCallback(async () => {
+        const session = sessionRef.current
+        if (!session) {
+            operationRef.current += 1
+            startAbortRef.current?.abort()
+            startAbortRef.current = null
+            if (elevenLabsActiveRef.current) {
+                elevenLabsActiveRef.current = false
+                elevenLabs.disconnect()
+            }
+            setStatus('disconnected')
+            return
+        }
+        setStatus('connecting')
+        try {
+            await session.stop()
+        } catch (stopError) {
+            fail(stopError)
+        } finally {
+            if (sessionRef.current === session) sessionRef.current = null
+        }
+    }, [elevenLabs, fail])
+
+    const toggle = useCallback(async () => {
+        if (status === 'connected' || status === 'connecting') await stop()
+        else await start()
+    }, [start, status, stop])
+
+    useEffect(() => {
+        mountedRef.current = true
+        return () => {
+            mountedRef.current = false
+            operationRef.current += 1
+            startAbortRef.current?.abort()
+            startAbortRef.current = null
+            elevenLabsActiveRef.current = false
+            elevenLabsRef.current.disconnect()
+            resolveElevenLabsCommitRef.current?.()
+            sessionRef.current?.cancel()
+            sessionRef.current = null
+        }
+    }, [])
+
+    return { supported, status, error, partialTranscript, toggle }
+}
@@ -1,21 +1,39 @@
 import { useCallback, useEffect, useState } from 'react'
 import type { ApiClient } from '@/api/client'
-import type {
-    TranscriptionMode,
-    TranscriptionProvider,
-    TranscriptionProviderInfo,
-    VoiceMode
+import {
+    BROWSER_LOCAL_TRANSCRIPTION_PROVIDER,
+    type TranscriptionMode,
+    type TranscriptionProvider,
+    type TranscriptionProviderInfo,
+    type VoiceMode
 } from '@hapi/protocol/voice'

 const VOICE_MODE_KEY = 'hapi-voice-mode'
 const TRANSCRIPTION_PROVIDER_KEY = 'hapi-transcription-provider'
 const TRANSCRIPTION_MODE_KEY = 'hapi-transcription-mode'
 const CHANGE_EVENT = 'hapi-voice-input-change'
+export const VOICE_LANGUAGE_CHANGE_EVENT = 'hapi-voice-language-change'

 function notifyChange(): void {
    window.dispatchEvent(new Event(CHANGE_EVENT))
 }

+async function browserLocalTranscriptionSupported(): Promise<boolean> {
+    const constructor = (globalThis as typeof globalThis & {
+        SpeechRecognition?: {
+            prototype: object
+            available?: (options: { langs: string[]; processLocally: true }) => Promise<string>
+        }
+    }).SpeechRecognition
+    if (!constructor || typeof constructor.available !== 'function' || !('processLocally' in constructor.prototype)) return false
+    const language = localStorage.getItem('hapi-voice-lang') || navigator.language
+    try {
+        return await constructor.available({ langs: [language], processLocally: true }) === 'available'
+    } catch {
+        return false
+    }
+}
+
 function readVoiceMode(): VoiceMode {
    return localStorage.getItem(VOICE_MODE_KEY) === 'dictation' ? 'dictation' : 'assistant'
 }
@@ -33,7 +51,8 @@ function resolveMode(
    stored: string | null
 ): TranscriptionMode {
    const modes = providers.find((candidate) => candidate.id === provider)?.modes ?? ['standard']
-    return stored === 'realtime' && modes.includes('realtime') ? 'realtime' : 'standard'
+    if ((stored === 'standard' || stored === 'realtime') && modes.includes(stored)) return stored
+    return modes[0] ?? 'standard'
 }

 export function useVoiceInputPreferences(api: ApiClient | null) {
@@ -45,16 +64,28 @@ export function useVoiceInputPreferences(api: ApiClient | null) {
    useEffect(() => {
        if (!api) return
        let cancelled = false
-        api.fetchTranscriptionProviders().then(({ providers: available }) => {
-            if (cancelled) return
-            setProviders(available)
-            const selectedProvider = resolveProvider(available, localStorage.getItem(TRANSCRIPTION_PROVIDER_KEY))
-            setProviderState(selectedProvider)
-            setTranscriptionModeState(resolveMode(available, selectedProvider, localStorage.getItem(TRANSCRIPTION_MODE_KEY)))
-        }).catch(() => {
-            if (!cancelled) setProviders([])
-        })
-        return () => { cancelled = true }
+        let request = 0
+        const refreshProviders = () => {
+            const current = ++request
+            Promise.all([api.fetchTranscriptionProviders(), browserLocalTranscriptionSupported()]).then(([{ providers: configured }, browserLocal]) => {
+                if (cancelled || current !== request) return
+                const available = browserLocal
+                    ? [...configured, BROWSER_LOCAL_TRANSCRIPTION_PROVIDER]
+                    : configured
+                setProviders(available)
+                const selectedProvider = resolveProvider(available, localStorage.getItem(TRANSCRIPTION_PROVIDER_KEY))
+                setProviderState(selectedProvider)
+                setTranscriptionModeState(resolveMode(available, selectedProvider, localStorage.getItem(TRANSCRIPTION_MODE_KEY)))
+            }).catch(() => {
+                if (!cancelled && current === request) setProviders([])
+            })
+        }
+        refreshProviders()
+        window.addEventListener(VOICE_LANGUAGE_CHANGE_EVENT, refreshProviders)
+        return () => {
+            cancelled = true
+            window.removeEventListener(VOICE_LANGUAGE_CHANGE_EVENT, refreshProviders)
+        }
    }, [api])

    useEffect(() => {
@@ -783,7 +783,7 @@ export default {
  'settings.voice.transcriptionMode.standard': 'Standard',
  'settings.voice.transcriptionMode.standard.hint': 'Record, then transcribe (default)',
  'settings.voice.transcriptionMode.realtime': 'Realtime',
-  'settings.voice.transcriptionMode.realtime.hint': 'Insert text while speaking',
+  'settings.voice.transcriptionMode.realtime.hint': 'Show live text while speaking; insert the final result when stopped',
  'settings.voice.voices.description': 'Choose the voice used by the selected backend.',
  'settings.voice.sounds.title': 'How It Sounds',
  'settings.voice.responds.title': 'How It Responds',
@@ -787,7 +787,7 @@ export default {
  'settings.voice.transcriptionMode.standard': '标准',
  'settings.voice.transcriptionMode.standard.hint': '录音结束后转录（默认）',
  'settings.voice.transcriptionMode.realtime': '实时',
-  'settings.voice.transcriptionMode.realtime.hint': '边说边插入文字',
+  'settings.voice.transcriptionMode.realtime.hint': '说话时显示实时文字，停止后插入最终结果',
  'settings.voice.voices.description': '选择当前后端使用的声音。',
  'settings.voice.sounds.title': '声音效果',
  'settings.voice.responds.title': '对话风格',
@@ -1,10 +1,11 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 import { act, renderHook, waitFor } from '@testing-library/react'
+import type { TranscriptionProviderInfo } from '@hapi/protocol/voice'
 import { I18nProvider } from '@/lib/i18n-context'
 import { useVoiceSettings } from './useVoiceSettings'

 const { fetchTranscriptionProviders, fetchVoiceBackend, fetchVoices, pause, play } = vi.hoisted(() => ({
-    fetchTranscriptionProviders: vi.fn(() => Promise.resolve({ providers: [] })),
+    fetchTranscriptionProviders: vi.fn((): Promise<{ providers: TranscriptionProviderInfo[] }> => Promise.resolve({ providers: [] })),
    fetchVoiceBackend: vi.fn(),
    fetchVoices: vi.fn(),
    pause: vi.fn(),
@@ -26,6 +27,8 @@ function Wrapper(props: { children: React.ReactNode }) {
 }

 describe('useVoiceSettings', () => {
+    afterEach(() => vi.unstubAllGlobals())
+
    beforeEach(() => {
        vi.clearAllMocks()
        localStorage.clear()
@@ -80,4 +83,60 @@ describe('useVoiceSettings', () => {
        unmount()
        expect(pause).toHaveBeenCalled()
    })
+
+    it('uses realtime for the realtime-only browser provider', async () => {
+        class MockSpeechRecognition {
+            static available() { return Promise.resolve('available') }
+            processLocally = false
+        }
+        Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
+        vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
+        localStorage.setItem('hapi-transcription-mode', 'standard')
+
+        const { result } = renderHook(() => useVoiceSettings(), { wrapper: Wrapper })
+
+        await waitFor(() => expect(result.current.provider).toBe('browser-local'))
+        expect(result.current.transcriptionMode).toBe('realtime')
+    })
+
+    it('does not expose browser dictation without the selected language pack', async () => {
+        fetchTranscriptionProviders.mockResolvedValueOnce({
+            providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }]
+        })
+        class MockSpeechRecognition {
+            static available() { return Promise.resolve('unavailable') }
+            processLocally = false
+        }
+        Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
+        vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
+
+        const { result } = renderHook(() => useVoiceSettings(), { wrapper: Wrapper })
+
+        await waitFor(() => expect(result.current.provider).toBe('openai'))
+        expect(result.current.providers).toHaveLength(1)
+    })
+
+    it('rechecks the on-device language pack when the language changes', async () => {
+        fetchTranscriptionProviders.mockResolvedValue({
+            providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }]
+        })
+        class MockSpeechRecognition {
+            static available({ langs }: { langs: string[] }) {
+                return Promise.resolve(langs[0] === 'en-US' ? 'available' : 'unavailable')
+            }
+            processLocally = false
+        }
+        Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
+        vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
+        localStorage.setItem('hapi-voice-lang', 'en-US')
+        localStorage.setItem('hapi-transcription-provider', 'browser-local')
+        const { result } = renderHook(() => useVoiceSettings(), { wrapper: Wrapper })
+        await waitFor(() => expect(result.current.provider).toBe('browser-local'))
+
+        act(() => result.current.setVoiceLanguage({ code: 'zh-CN', name: 'Chinese', nativeName: '中文' }))
+        await waitFor(() => expect(result.current.provider).toBe('openai'))
+
+        act(() => result.current.setVoiceLanguage({ code: 'en-US', name: 'English', nativeName: 'English' }))
+        await waitFor(() => expect(result.current.provider).toBe('browser-local'))
+    })
 })
@@ -12,7 +12,7 @@ import {
    writeStoredVoiceSelection,
 } from '@/lib/voicePickerPreferences'
 import type { VoiceBackendType } from '@hapi/protocol/voice'
-import { useVoiceInputPreferences } from '@/hooks/useVoiceInputPreferences'
+import { useVoiceInputPreferences, VOICE_LANGUAGE_CHANGE_EVENT } from '@/hooks/useVoiceInputPreferences'

 export function useVoiceSettings() {
    const { api } = useAppContext()
@@ -83,6 +83,7 @@ export function useVoiceSettings() {
        setVoiceLanguageState(language.code)
        if (language.code === null) localStorage.removeItem('hapi-voice-lang')
        else localStorage.setItem('hapi-voice-lang', language.code)
+        window.dispatchEvent(new Event(VOICE_LANGUAGE_CHANGE_EVENT))
    }, [])

    const stopPreview = useCallback(() => {