From c3a5522207d6e0d2ba2f44f8b39188116683aaf4 Mon Sep 17 00:00:00 2001 From: SSU-WEI HUANG Date: Mon, 3 Aug 2026 10:03:06 +0800 Subject: [PATCH] Add realtime dictation providers (#1329) * feat: add realtime dictation providers * fix: cancel realtime dictation startup * fix: refresh local dictation availability * fix: preserve dictation on disconnect * fix: normalize OpenAI language hints --- docs/guide/installation.md | 2 +- docs/guide/voice-assistant.md | 9 +- hub/README.md | 7 +- hub/src/web/routes/voice.test.ts | 83 +++- hub/src/web/routes/voice.ts | 108 ++++- shared/src/voice.backends.test.ts | 6 +- shared/src/voice.ts | 20 +- web/README.md | 1 + web/src/api/client.ts | 12 + .../AssistantChat/HappyComposer.tsx | 10 + web/src/hooks/realtimeTranscription.test.ts | 205 ++++++++ web/src/hooks/realtimeTranscription.ts | 452 ++++++++++++++++++ web/src/hooks/useDictation.test.ts | 47 ++ web/src/hooks/useDictation.ts | 20 +- web/src/hooks/useRealtimeDictation.test.ts | 61 +++ web/src/hooks/useRealtimeDictation.ts | 248 ++++++++++ web/src/hooks/useVoiceInputPreferences.ts | 63 ++- web/src/lib/locales/en.ts | 2 +- web/src/lib/locales/zh-CN.ts | 2 +- .../routes/settings/useVoiceSettings.test.tsx | 63 ++- web/src/routes/settings/useVoiceSettings.ts | 3 +- 21 files changed, 1378 insertions(+), 46 deletions(-) create mode 100644 web/src/hooks/realtimeTranscription.test.ts create mode 100644 web/src/hooks/realtimeTranscription.ts create mode 100644 web/src/hooks/useRealtimeDictation.test.ts create mode 100644 web/src/hooks/useRealtimeDictation.ts diff --git a/docs/guide/installation.md b/docs/guide/installation.md index b0df4e0c..64626e47 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -190,7 +190,7 @@ On first run, HAPI: | `DB_PATH` | `~/.hapi/hapi.db` | - | Database file path | | `ELEVENLABS_API_KEY` | - | - | ElevenLabs API key for voice | | `ELEVENLABS_AGENT_ID` | Auto-created | - | Custom ElevenLabs agent ID | -| `OPENAI_API_KEY` | - | - | OpenAI API key for dictation (`gpt-4o-transcribe`) | +| `OPENAI_API_KEY` | - | - | OpenAI API key for dictation (`gpt-transcribe` / `gpt-live-transcribe`) | | `DEEPGRAM_API_KEY` | - | - | Deepgram API key for dictation (`nova-3`) | | `GROQ_API_KEY` | - | - | Groq API key for dictation (`whisper-large-v3`) | | `TRANSCRIPTION_BASE_URL` | - | - | OpenAI-compatible/local transcription base URL | diff --git a/docs/guide/voice-assistant.md b/docs/guide/voice-assistant.md index 6d4e4da8..6897fff6 100644 --- a/docs/guide/voice-assistant.md +++ b/docs/guide/voice-assistant.md @@ -2,15 +2,15 @@ Control your AI coding agent with voice using the built-in voice assistant powered by ElevenLabs Conversational AI. -For speech-to-text without a spoken assistant, open **Settings → Voice**, choose **Dictation**, then select a configured provider. Dictation records until you tap the microphone again, inserts the transcript into the composer, and never sends it automatically. Standard mode is the default. +For speech-to-text without a spoken assistant, open **Settings → Voice**, choose **Dictation**, then select a configured provider. Dictation records until you tap the microphone again, inserts the transcript into the composer, and never sends it automatically. Standard mode is the default. Realtime mode shows a live transcript while you speak and inserts the final result when you stop. Provider credentials are read only from the hub's startup environment: ```bash # Pick any providers you use -export OPENAI_API_KEY="..." # gpt-4o-transcribe -export ELEVENLABS_API_KEY="..." # scribe_v2 -export DEEPGRAM_API_KEY="..." # nova-3 +export OPENAI_API_KEY="..." # gpt-transcribe / gpt-live-transcribe +export ELEVENLABS_API_KEY="..." # scribe_v2 / scribe_v2_realtime +export DEEPGRAM_API_KEY="..." # nova-3 standard / realtime export GROQ_API_KEY="..." # whisper-large-v3 # Or an OpenAI-compatible local server such as Speaches @@ -20,6 +20,7 @@ export TRANSCRIPTION_API_KEY="..." # optional ``` Restart the hub after changing credentials. API keys are not entered or stored in the web app. +Realtime OpenAI, ElevenLabs, and Deepgram sessions receive only short-lived credentials minted by the hub. Browsers with an installed on-device `SpeechRecognition` language pack also expose **Browser on-device** as a realtime-only provider; HAPI never falls back from that option to browser-hosted recognition. ## Overview diff --git a/hub/README.md b/hub/README.md index 08634c4c..c00655fa 100644 --- a/hub/README.md +++ b/hub/README.md @@ -28,8 +28,8 @@ See `src/configuration.ts` for all options. - `ELEVENLABS_API_KEY` - ElevenLabs API key for voice assistant. - `ELEVENLABS_AGENT_ID` - Custom ElevenLabs agent ID (auto-created if not set). -- `OPENAI_API_KEY` - OpenAI dictation (`gpt-4o-transcribe`). -- `DEEPGRAM_API_KEY` - Deepgram dictation (`nova-3`). +- `OPENAI_API_KEY` - OpenAI dictation (`gpt-transcribe` / `gpt-live-transcribe`). +- `DEEPGRAM_API_KEY` - Deepgram dictation (`nova-3`, standard and realtime). - `GROQ_API_KEY` - Groq dictation (`whisper-large-v3`). - `TRANSCRIPTION_BASE_URL` and `TRANSCRIPTION_MODEL` - OpenAI-compatible/local transcription endpoint and model. - `TRANSCRIPTION_API_KEY` - Optional bearer token for the OpenAI-compatible endpoint. @@ -131,6 +131,9 @@ See `src/web/routes/` for all endpoints. ### Voice (`src/web/routes/voice.ts`) - `POST /api/voice/token` - Get ElevenLabs conversation token. +- `GET /api/voice/transcription/providers` - List configured providers and supported modes. +- `POST /api/voice/transcription` - Transcribe a bounded recording. +- `POST /api/voice/transcription/realtime-token` - Mint a short-lived OpenAI, ElevenLabs, or Deepgram credential. ### Push Notifications (`src/web/routes/push.ts`) diff --git a/hub/src/web/routes/voice.test.ts b/hub/src/web/routes/voice.test.ts index 12c1e08f..88fd344d 100644 --- a/hub/src/web/routes/voice.test.ts +++ b/hub/src/web/routes/voice.test.ts @@ -100,7 +100,7 @@ describe('voice transcription routes', () => { const res = await app.request('/api/voice/transcription/providers', { headers }) expect(res.status).toBe(200) expect(await res.json()).toEqual({ providers: [ - { id: 'openai', label: 'OpenAI', modes: ['standard'] }, + { id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }, { id: 'openai-compatible', label: 'OpenAI-compatible / local', modes: ['standard'] } ] }) @@ -144,8 +144,13 @@ describe('voice transcription routes', () => { expect(upstreamUrl).toBe('https://api.openai.com/v1/audio/transcriptions') expect(new Headers(upstreamInit?.headers).get('authorization')).toBe('Bearer server-only-key') expect(upstreamInit?.body).toBeInstanceOf(FormData) - expect((upstreamInit?.body as FormData).get('model')).toBe('gpt-4o-transcribe') - expect((upstreamInit?.body as FormData).get('language')).toBe('zh') + expect((upstreamInit?.body as FormData).get('model')).toBe('gpt-transcribe') + expect((upstreamInit?.body as FormData).get('languages[]')).toBe('zh-cn') + + form.set('language', 'en-US') + const englishRes = await app.request('/api/voice/transcription', { method: 'POST', headers, body: form }) + expect(englishRes.status).toBe(200) + expect((upstreamInit?.body as FormData).get('languages[]')).toBe('en') global.fetch = originalFetch if (previousKey === undefined) delete process.env.OPENAI_API_KEY @@ -179,6 +184,78 @@ describe('voice transcription routes', () => { expect(res.status).toBe(413) expect(await res.json()).toEqual({ error: 'Audio file too large' }) }) + + test('mints provider-specific short-lived realtime credentials without exposing API keys', async () => { + const app = createApp() + const headers = { ...(await authHeaders()), 'content-type': 'application/json' } + const previous = { + openai: process.env.OPENAI_API_KEY, + elevenlabs: process.env.ELEVENLABS_API_KEY, + deepgram: process.env.DEEPGRAM_API_KEY + } + process.env.OPENAI_API_KEY = 'openai-server-key' + process.env.ELEVENLABS_API_KEY = 'elevenlabs-server-key' + process.env.DEEPGRAM_API_KEY = 'deepgram-server-key' + const originalFetch = global.fetch + const requests: Array<{ url: string; init?: RequestInit }> = [] + // @ts-expect-error test override + global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input) + requests.push({ url, init }) + if (url.endsWith('/realtime/client_secrets')) { + return new Response(JSON.stringify({ value: 'openai-client-token' }), { status: 200 }) + } + if (url.endsWith('/single-use-token/realtime_scribe')) { + return new Response(JSON.stringify({ token: 'elevenlabs-client-token' }), { status: 200 }) + } + return new Response(JSON.stringify({ access_token: 'deepgram-client-token' }), { status: 200 }) + }) as typeof fetch + + for (const provider of ['openai', 'elevenlabs', 'deepgram'] as const) { + const res = await app.request('/api/voice/transcription/realtime-token', { + method: 'POST', + headers, + body: JSON.stringify({ provider, language: 'zh-TW' }) + }) + expect(res.status).toBe(200) + expect(await res.json()).toEqual({ token: `${provider}-client-token` }) + } + const englishOpenAI = await app.request('/api/voice/transcription/realtime-token', { + method: 'POST', + headers, + body: JSON.stringify({ provider: 'openai', language: 'en-US' }) + }) + expect(englishOpenAI.status).toBe(200) + + expect(requests.map((request) => request.url)).toEqual([ + 'https://api.openai.com/v1/realtime/client_secrets', + 'https://api.elevenlabs.io/v1/single-use-token/realtime_scribe', + 'https://api.deepgram.com/v1/auth/grant', + 'https://api.openai.com/v1/realtime/client_secrets' + ]) + expect(new Headers(requests[0]?.init?.headers).get('authorization')).toBe('Bearer openai-server-key') + expect(JSON.parse(String(requests[0]?.init?.body))).toMatchObject({ + session: { + type: 'transcription', + audio: { input: { transcription: { model: 'gpt-live-transcribe', languages: ['zh-tw'] } } } + } + }) + expect(JSON.parse(String(requests[3]?.init?.body))).toMatchObject({ + session: { audio: { input: { transcription: { languages: ['en'] } } } } + }) + expect(new Headers(requests[1]?.init?.headers).get('xi-api-key')).toBe('elevenlabs-server-key') + expect(new Headers(requests[2]?.init?.headers).get('authorization')).toBe('Token deepgram-server-key') + + global.fetch = originalFetch + for (const [key, value] of Object.entries({ + OPENAI_API_KEY: previous.openai, + ELEVENLABS_API_KEY: previous.elevenlabs, + DEEPGRAM_API_KEY: previous.deepgram + })) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + }) }) describe('POST /api/voice/token', () => { diff --git a/hub/src/web/routes/voice.ts b/hub/src/web/routes/voice.ts index 03ea148a..bea263b9 100644 --- a/hub/src/web/routes/voice.ts +++ b/hub/src/web/routes/voice.ts @@ -3,7 +3,13 @@ import { bodyLimit } from 'hono/body-limit' import { z } from 'zod' import type { WebAppEnv } from '../middleware/auth' import { + DEEPGRAM_TRANSCRIPTION_MODEL, + ELEVENLABS_REALTIME_TRANSCRIPTION_MODEL, + ELEVENLABS_TRANSCRIPTION_MODEL, ELEVENLABS_API_BASE, + GROQ_TRANSCRIPTION_MODEL, + OPENAI_REALTIME_TRANSCRIPTION_MODEL, + OPENAI_TRANSCRIPTION_MODEL, VOICE_AGENT_NAME, buildVoiceAgentConfig, listConfiguredTranscriptionProviders, @@ -43,6 +49,7 @@ const transcriptionProviderSchema = z.enum([ 'groq', 'openai-compatible' ]) +const realtimeTranscriptionProviderSchema = z.enum(['openai', 'elevenlabs', 'deepgram']) const transcriptionLanguageSchema = z.string() .trim() .max(35) @@ -52,11 +59,19 @@ const transcriptionLanguageSchema = z.string() const MAX_TRANSCRIPTION_BYTES = 25 * 1024 * 1024 const MAX_TRANSCRIPTION_BODY_BYTES = MAX_TRANSCRIPTION_BYTES + 1024 * 1024 const TRANSCRIPTION_TIMEOUT_MS = 120_000 +const REALTIME_TOKEN_TIMEOUT_MS = 15_000 +const MAX_REALTIME_TOKEN_BODY_BYTES = 8 * 1024 function trimTrailingSlash(value: string): string { return value.replace(/\/+$/, '') } +function normalizeOpenAILanguage(language?: string): string | undefined { + const value = language?.toLowerCase() + if (!value) return undefined + return ['zh-cn', 'zh-tw', 'zh-hk'].includes(value) ? value : value.split('-')[0] +} + function getTranscriptionConfig(provider: TranscriptionProvider): { apiKey?: string baseUrl: string @@ -65,13 +80,13 @@ function getTranscriptionConfig(provider: TranscriptionProvider): { if (!listConfiguredTranscriptionProviders(process.env).some((candidate) => candidate.id === provider)) return null switch (provider) { case 'openai': - return { apiKey: process.env.OPENAI_API_KEY!.trim(), baseUrl: 'https://api.openai.com/v1', model: 'gpt-4o-transcribe' } + return { apiKey: process.env.OPENAI_API_KEY!.trim(), baseUrl: 'https://api.openai.com/v1', model: OPENAI_TRANSCRIPTION_MODEL } case 'elevenlabs': - return { apiKey: process.env.ELEVENLABS_API_KEY!.trim(), baseUrl: ELEVENLABS_API_BASE, model: 'scribe_v2' } + return { apiKey: process.env.ELEVENLABS_API_KEY!.trim(), baseUrl: ELEVENLABS_API_BASE, model: ELEVENLABS_TRANSCRIPTION_MODEL } case 'deepgram': - return { apiKey: process.env.DEEPGRAM_API_KEY!.trim(), baseUrl: 'https://api.deepgram.com/v1', model: 'nova-3' } + return { apiKey: process.env.DEEPGRAM_API_KEY!.trim(), baseUrl: 'https://api.deepgram.com/v1', model: DEEPGRAM_TRANSCRIPTION_MODEL } case 'groq': - return { apiKey: process.env.GROQ_API_KEY!.trim(), baseUrl: 'https://api.groq.com/openai/v1', model: 'whisper-large-v3' } + return { apiKey: process.env.GROQ_API_KEY!.trim(), baseUrl: 'https://api.groq.com/openai/v1', model: GROQ_TRANSCRIPTION_MODEL } case 'openai-compatible': { const baseUrl = process.env.TRANSCRIPTION_BASE_URL?.trim() const model = process.env.TRANSCRIPTION_MODEL?.trim() @@ -79,6 +94,8 @@ function getTranscriptionConfig(provider: TranscriptionProvider): { ? { apiKey: process.env.TRANSCRIPTION_API_KEY?.trim(), baseUrl: trimTrailingSlash(baseUrl), model } : null } + case 'browser-local': + return null } } @@ -110,6 +127,7 @@ async function transcribeStandard( form.set(provider === 'elevenlabs' ? 'model_id' : 'model', config.model) if (baseLanguage) { if (provider === 'elevenlabs') form.set('language_code', baseLanguage) + else if (provider === 'openai') form.set('languages[]', normalizeOpenAILanguage(language) ?? baseLanguage) else form.set('language', baseLanguage) } url = provider === 'elevenlabs' @@ -151,6 +169,76 @@ async function transcribeStandard( } } +async function createRealtimeTranscriptionToken( + provider: z.infer, + language?: string +): Promise { + const config = getTranscriptionConfig(provider) + if (!config?.apiKey) return Response.json({ error: `${provider} transcription is not configured` }, { status: 400 }) + + let url: string + let headers: Record + let body: string | undefined + + switch (provider) { + case 'openai': { + const openAILanguage = normalizeOpenAILanguage(language) + const languages = openAILanguage ? [openAILanguage] : undefined + url = 'https://api.openai.com/v1/realtime/client_secrets' + headers = { Authorization: `Bearer ${config.apiKey}`, 'Content-Type': 'application/json' } + body = JSON.stringify({ + expires_after: { anchor: 'created_at', seconds: 60 }, + session: { + type: 'transcription', + audio: { + input: { + transcription: { + model: OPENAI_REALTIME_TRANSCRIPTION_MODEL, + delay: 'low', + ...(languages ? { languages } : {}) + }, + turn_detection: null + } + } + } + }) + break + } + case 'elevenlabs': + url = `${ELEVENLABS_API_BASE}/single-use-token/realtime_scribe` + headers = { 'xi-api-key': config.apiKey } + break + case 'deepgram': + url = 'https://api.deepgram.com/v1/auth/grant' + headers = { Authorization: `Token ${config.apiKey}`, 'Content-Type': 'application/json' } + body = '{}' + break + } + + try { + const response = await fetch(url, { + method: 'POST', + headers, + body, + signal: AbortSignal.timeout(REALTIME_TOKEN_TIMEOUT_MS) + }) + if (!response.ok) { + console.warn('[Voice][RealtimeTranscription] Token request failed', { provider, status: response.status }) + return Response.json({ error: `${provider} realtime transcription token failed (HTTP ${response.status})` }, { status: 502 }) + } + const data = await response.json() as { value?: string; token?: string; access_token?: string } + const token = data.value ?? data.token ?? data.access_token + if (!token) return Response.json({ error: `${provider} realtime transcription returned no token` }, { status: 502 }) + return Response.json({ token }) + } catch (error) { + console.warn('[Voice][RealtimeTranscription] Token request error', { + provider, + error: error instanceof Error ? error.message : String(error) + }) + return Response.json({ error: `${provider} realtime transcription token request failed` }, { status: 502 }) + } +} + // Cache for auto-created agent IDs (keyed by API key hash) const agentIdCache = new Map() @@ -404,6 +492,18 @@ export function createVoiceRoutes(): Hono { return transcribeStandard(provider.data, file, language.data) }) + app.post('/voice/transcription/realtime-token', bodyLimit({ + maxSize: MAX_REALTIME_TOKEN_BODY_BYTES, + onError: (c) => c.json({ error: 'Realtime token request too large' }, 413) + }), async (c) => { + const json = await c.req.json().catch(() => null) + const provider = realtimeTranscriptionProviderSchema.safeParse(json?.provider) + const language = transcriptionLanguageSchema.safeParse(json?.language || undefined) + if (!provider.success) return c.json({ error: 'Invalid realtime transcription provider' }, 400) + if (!language.success) return c.json({ error: 'Invalid transcription language' }, 400) + return createRealtimeTranscriptionToken(provider.data, language.data) + }) + // Get Gemini API key for Gemini Live voice sessions // Gemini Live API does not support ephemeral tokens, so we proxy the key. // The key is short-lived in the browser session and never persisted client-side. diff --git a/shared/src/voice.backends.test.ts b/shared/src/voice.backends.test.ts index 6e25c751..bb3d3ceb 100644 --- a/shared/src/voice.backends.test.ts +++ b/shared/src/voice.backends.test.ts @@ -11,11 +11,13 @@ describe('listConfiguredTranscriptionProviders', () => { expect(listConfiguredTranscriptionProviders({ OPENAI_API_KEY: 'openai', ELEVENLABS_API_KEY: 'elevenlabs', + DEEPGRAM_API_KEY: 'deepgram', TRANSCRIPTION_BASE_URL: 'http://localhost:8000/v1', TRANSCRIPTION_MODEL: 'whisper-large-v3' })).toEqual([ - { id: 'openai', label: 'OpenAI', modes: ['standard'] }, - { id: 'elevenlabs', label: 'ElevenLabs', modes: ['standard'] }, + { id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }, + { id: 'elevenlabs', label: 'ElevenLabs', modes: ['standard', 'realtime'] }, + { id: 'deepgram', label: 'Deepgram', modes: ['standard', 'realtime'] }, { id: 'openai-compatible', label: 'OpenAI-compatible / local', modes: ['standard'] } ]) }) diff --git a/shared/src/voice.ts b/shared/src/voice.ts index e8ea1b5d..82141982 100644 --- a/shared/src/voice.ts +++ b/shared/src/voice.ts @@ -237,7 +237,14 @@ export type VoiceBackendType = 'elevenlabs' | 'gemini-live' | 'qwen-realtime' export type VoiceMode = 'assistant' | 'dictation' export type TranscriptionMode = 'standard' | 'realtime' -export type TranscriptionProvider = 'openai' | 'elevenlabs' | 'deepgram' | 'groq' | 'openai-compatible' +export type TranscriptionProvider = 'openai' | 'elevenlabs' | 'deepgram' | 'groq' | 'openai-compatible' | 'browser-local' + +export const OPENAI_TRANSCRIPTION_MODEL = 'gpt-transcribe' +export const OPENAI_REALTIME_TRANSCRIPTION_MODEL = 'gpt-live-transcribe' +export const ELEVENLABS_TRANSCRIPTION_MODEL = 'scribe_v2' +export const ELEVENLABS_REALTIME_TRANSCRIPTION_MODEL = 'scribe_v2_realtime' +export const DEEPGRAM_TRANSCRIPTION_MODEL = 'nova-3' +export const GROQ_TRANSCRIPTION_MODEL = 'whisper-large-v3' export interface TranscriptionProviderInfo { id: TranscriptionProvider @@ -246,13 +253,16 @@ export interface TranscriptionProviderInfo { } const TRANSCRIPTION_PROVIDERS: Record = { - openai: { id: 'openai', label: 'OpenAI', modes: ['standard'] }, - elevenlabs: { id: 'elevenlabs', label: 'ElevenLabs', modes: ['standard'] }, - deepgram: { id: 'deepgram', label: 'Deepgram', modes: ['standard'] }, + openai: { id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }, + elevenlabs: { id: 'elevenlabs', label: 'ElevenLabs', modes: ['standard', 'realtime'] }, + deepgram: { id: 'deepgram', label: 'Deepgram', modes: ['standard', 'realtime'] }, groq: { id: 'groq', label: 'Groq', modes: ['standard'] }, - 'openai-compatible': { id: 'openai-compatible', label: 'OpenAI-compatible / local', modes: ['standard'] } + 'openai-compatible': { id: 'openai-compatible', label: 'OpenAI-compatible / local', modes: ['standard'] }, + 'browser-local': { id: 'browser-local', label: 'Browser on-device', modes: ['realtime'] } } +export const BROWSER_LOCAL_TRANSCRIPTION_PROVIDER = TRANSCRIPTION_PROVIDERS['browser-local'] + /** Transcription providers whose startup environment is complete. */ export function listConfiguredTranscriptionProviders(env: VoiceBackendEnv): TranscriptionProviderInfo[] { const providers: TranscriptionProvider[] = [] diff --git a/web/README.md b/web/README.md index 3498263f..096c02d3 100644 --- a/web/README.md +++ b/web/README.md @@ -85,6 +85,7 @@ See `src/router.tsx` for route definitions. - ElevenLabs integration (@elevenlabs/react) - Real-time voice control +- Standard and realtime composer dictation with provider capability selection ### New session (`src/components/NewSession/`) diff --git a/web/src/api/client.ts b/web/src/api/client.ts index e36bdf37..6b09045b 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -972,6 +972,18 @@ export class ApiClient { return await this.request('/api/voice/transcription', { method: 'POST', body: form }) } + async fetchRealtimeTranscriptionToken( + provider: 'openai' | 'elevenlabs' | 'deepgram', + language?: string, + signal?: AbortSignal + ): Promise<{ token: string }> { + return await this.request('/api/voice/transcription/realtime-token', { + method: 'POST', + signal, + body: JSON.stringify({ provider, language }) + }) + } + async fetchQwenToken(): Promise<{ allowed: boolean wsUrl?: string diff --git a/web/src/components/AssistantChat/HappyComposer.tsx b/web/src/components/AssistantChat/HappyComposer.tsx index 186c281f..7bc60f42 100644 --- a/web/src/components/AssistantChat/HappyComposer.tsx +++ b/web/src/components/AssistantChat/HappyComposer.tsx @@ -1632,6 +1632,16 @@ export function HappyComposer(props: { voiceStatus={effectiveVoiceStatus} /> + {dictationActive && dictation.partialTranscript ? ( +
+ {dictation.partialTranscript} +
+ ) : null} + {dictationActive && dictation.error ? (
{dictation.error} diff --git a/web/src/hooks/realtimeTranscription.test.ts b/web/src/hooks/realtimeTranscription.test.ts new file mode 100644 index 00000000..9106384c --- /dev/null +++ b/web/src/hooks/realtimeTranscription.test.ts @@ -0,0 +1,205 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { startDeepgramRealtimeTranscription, startOpenAIRealtimeTranscription } from './realtimeTranscription' + +describe('OpenAI realtime transcription', () => { + afterEach(() => vi.unstubAllGlobals()) + + it('streams partial text and returns the committed final transcript', async () => { + const stopTrack = vi.fn() + Object.defineProperty(navigator, 'mediaDevices', { + configurable: true, + value: { + getUserMedia: vi.fn(async () => ({ + getTracks: () => [{ stop: stopTrack }], + getAudioTracks: () => [{ stop: stopTrack }] + })) + } + }) + + class MockDataChannel extends EventTarget { + readyState = 'open' + send = vi.fn() + close = vi.fn() + } + const channel = new MockDataChannel() + class MockPeerConnection extends EventTarget { + connectionState = 'connected' + createDataChannel() { return channel } + addTrack() {} + async createOffer() { return { type: 'offer', sdp: 'offer-sdp' } } + async setLocalDescription() {} + async setRemoteDescription() {} + close() {} + } + vi.stubGlobal('RTCPeerConnection', MockPeerConnection) + vi.stubGlobal('fetch', vi.fn(async () => new Response('answer-sdp', { status: 200 }))) + + const callbacks = { + onConnected: vi.fn(), + onPartial: vi.fn(), + onFinal: vi.fn(), + onError: vi.fn() + } + const session = await startOpenAIRealtimeTranscription({ + getToken: async () => 'ephemeral-token', + callbacks + }) + channel.dispatchEvent(new MessageEvent('message', { + data: JSON.stringify({ + type: 'conversation.item.input_audio_transcription.delta', + delta: 'live text' + }) + })) + expect(callbacks.onPartial).toHaveBeenCalledWith('live text') + + const stopping = session.stop() + setTimeout(() => channel.dispatchEvent(new MessageEvent('message', { + data: JSON.stringify({ + type: 'conversation.item.input_audio_transcription.completed', + transcript: 'final text' + }) + })), 0) + await stopping + + expect(callbacks.onFinal).toHaveBeenCalledWith('final text') + expect(callbacks.onError).not.toHaveBeenCalled() + expect(stopTrack).toHaveBeenCalled() + }) + + it('releases the microphone when startup is aborted', async () => { + const stopTrack = vi.fn() + Object.defineProperty(navigator, 'mediaDevices', { + configurable: true, + value: { + getUserMedia: vi.fn(async () => ({ + getTracks: () => [{ stop: stopTrack }], + getAudioTracks: () => [{ stop: stopTrack }] + })) + } + }) + class MockDataChannel extends EventTarget { + readyState = 'connecting' + close() {} + } + class MockPeerConnection extends EventTarget { + connectionState = 'connecting' + createDataChannel() { return new MockDataChannel() } + addTrack() {} + async createOffer() { return { type: 'offer', sdp: 'offer-sdp' } } + async setLocalDescription() {} + close() {} + } + vi.stubGlobal('RTCPeerConnection', MockPeerConnection) + const fetchMock = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true }) + })) + vi.stubGlobal('fetch', fetchMock) + const controller = new AbortController() + const starting = startOpenAIRealtimeTranscription({ + getToken: async () => 'ephemeral-token', + signal: controller.signal, + callbacks: { + onConnected: vi.fn(), + onPartial: vi.fn(), + onFinal: vi.fn(), + onError: vi.fn() + } + }) + await vi.waitFor(() => expect(fetchMock).toHaveBeenCalled()) + + controller.abort() + + await expect(starting).rejects.toBeDefined() + expect(stopTrack).toHaveBeenCalled() + }) +}) + +describe('Deepgram realtime transcription', () => { + afterEach(() => vi.unstubAllGlobals()) + + it('streams recorder chunks and returns the finalized transcript', async () => { + const stopTrack = vi.fn() + Object.defineProperty(navigator, 'mediaDevices', { + configurable: true, + value: { + getUserMedia: vi.fn(async () => ({ + getTracks: () => [{ stop: stopTrack }] + })) + } + }) + + class MockSocket extends EventTarget { + static OPEN = 1 + readyState = 0 + sent: unknown[] = [] + constructor(readonly url: string, readonly protocols: string[]) { + super() + queueMicrotask(() => { + this.readyState = MockSocket.OPEN + this.dispatchEvent(new Event('open')) + }) + } + send(value: unknown) { this.sent.push(value) } + close() { this.readyState = 3 } + result(transcript: string, final: boolean, fromFinalize = false) { + this.dispatchEvent(new MessageEvent('message', { + data: JSON.stringify({ + type: 'Results', + is_final: final, + from_finalize: fromFinalize, + channel: { alternatives: [{ transcript }] } + }) + })) + } + } + const sockets: MockSocket[] = [] + class MockWebSocket extends MockSocket { + constructor(url: string, protocols: string[]) { + super(url, protocols) + sockets.push(this) + } + } + Object.assign(MockWebSocket, { OPEN: MockSocket.OPEN }) + vi.stubGlobal('WebSocket', MockWebSocket) + + class MockRecorder { + static isTypeSupported() { return true } + state = 'inactive' + mimeType = 'audio/webm;codecs=opus' + ondataavailable: ((event: { data: Blob }) => void) | null = null + onerror: (() => void) | null = null + onstop: (() => void) | null = null + start() { + this.state = 'recording' + this.ondataavailable?.({ data: new Blob(['audio']) }) + } + stop() { + this.state = 'inactive' + this.onstop?.() + } + } + vi.stubGlobal('MediaRecorder', MockRecorder) + + const callbacks = { + onConnected: vi.fn(), + onPartial: vi.fn(), + onFinal: vi.fn(), + onError: vi.fn() + } + const session = await startDeepgramRealtimeTranscription({ + getToken: async () => 'temporary-jwt', + callbacks + }) + const socket = sockets[0]! + expect(socket.protocols).toEqual(['bearer', 'temporary-jwt']) + socket.result('live', false) + const stopping = session.stop() + socket.result('final text', true, true) + await stopping + + expect(callbacks.onPartial).toHaveBeenCalledWith('live') + expect(callbacks.onFinal).toHaveBeenCalledWith('final text') + expect(callbacks.onError).not.toHaveBeenCalled() + expect(stopTrack).toHaveBeenCalled() + }) +}) diff --git a/web/src/hooks/realtimeTranscription.ts b/web/src/hooks/realtimeTranscription.ts new file mode 100644 index 00000000..4dec17b7 --- /dev/null +++ b/web/src/hooks/realtimeTranscription.ts @@ -0,0 +1,452 @@ +import { DEEPGRAM_TRANSCRIPTION_MODEL } from '@hapi/protocol/voice' + +export interface RealtimeTranscriptionCallbacks { + onConnected: () => void + onPartial: (text: string) => void + onFinal: (text: string) => void + onError: (error: Error) => void +} + +export interface RealtimeTranscriptionSession { + stop: () => Promise + cancel: () => void +} + +type TokenFactory = () => Promise + +function joinTranscriptParts(...parts: string[]): string { + return parts.map((part) => part.trim()).filter(Boolean).join(' ') +} + +function errorMessage(value: unknown, fallback: string): Error { + return value instanceof Error ? value : new Error(fallback) +} + +export async function startOpenAIRealtimeTranscription(options: { + getToken: TokenFactory + signal?: AbortSignal + callbacks: RealtimeTranscriptionCallbacks +}): Promise { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } + }) + try { + options.signal?.throwIfAborted() + } catch (error) { + stream.getTracks().forEach((track) => track.stop()) + throw error + } + let peer: RTCPeerConnection + let channel: RTCDataChannel + try { + peer = new RTCPeerConnection() + channel = peer.createDataChannel('oai-events') + } catch (error) { + stream.getTracks().forEach((track) => track.stop()) + throw error + } + let partial = '' + let finished = false + let stopping = false + let stopTimer: ReturnType | null = null + + const cleanup = () => { + if (stopTimer) clearTimeout(stopTimer) + stopTimer = null + stream.getTracks().forEach((track) => track.stop()) + channel.close() + peer.close() + } + const finish = (text: string) => { + if (finished) return + finished = true + cleanup() + options.callbacks.onFinal(text) + } + const fail = (value: unknown) => { + if (finished) return + finished = true + cleanup() + options.callbacks.onError(errorMessage(value, 'OpenAI realtime transcription failed')) + } + + try { + const track = stream.getAudioTracks()[0] + if (!track) throw new Error('No microphone audio track is available') + peer.addTrack(track, stream) + channel.addEventListener('message', (event) => { + let data: { + type?: string + delta?: string + transcript?: string + error?: { message?: string } + } + try { + data = JSON.parse(String(event.data)) as typeof data + } catch { + return + } + if (data.type === 'conversation.item.input_audio_transcription.delta' && data.delta) { + partial += data.delta + options.callbacks.onPartial(partial) + } else if (data.type === 'conversation.item.input_audio_transcription.completed') { + finish(data.transcript ?? partial) + } else if (data.type === 'error') { + fail(new Error(data.error?.message || 'OpenAI realtime transcription failed')) + } + }) + peer.addEventListener('connectionstatechange', () => { + if (!finished && (peer.connectionState === 'failed' || peer.connectionState === 'closed')) { + fail(new Error('OpenAI realtime transcription disconnected')) + } + }) + + const offer = await peer.createOffer() + await peer.setLocalDescription(offer) + const token = await options.getToken() + const response = await fetch('https://api.openai.com/v1/realtime/calls', { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/sdp' + }, + body: offer.sdp, + signal: options.signal + ? AbortSignal.any([options.signal, AbortSignal.timeout(15_000)]) + : AbortSignal.timeout(15_000) + }) + if (!response.ok) throw new Error(`OpenAI realtime connection failed (HTTP ${response.status})`) + await peer.setRemoteDescription({ type: 'answer', sdp: await response.text() }) + await new Promise((resolve, reject) => { + if (channel.readyState === 'open') return resolve() + const timeout = setTimeout(() => reject(new Error('OpenAI realtime connection timed out')), 10_000) + const onAbort = () => { + clearTimeout(timeout) + reject(options.signal?.reason) + } + options.signal?.addEventListener('abort', onAbort, { once: true }) + channel.addEventListener('open', () => { + clearTimeout(timeout) + options.signal?.removeEventListener('abort', onAbort) + resolve() + }, { once: true }) + channel.addEventListener('error', () => { + clearTimeout(timeout) + options.signal?.removeEventListener('abort', onAbort) + reject(new Error('OpenAI realtime connection failed')) + }, { once: true }) + }) + options.callbacks.onConnected() + } catch (error) { + fail(error) + throw error + } + + return { + stop: async () => { + if (finished || stopping) return + stopping = true + stream.getTracks().forEach((track) => track.stop()) + if (channel.readyState !== 'open') return finish(partial) + channel.send(JSON.stringify({ type: 'input_audio_buffer.commit' })) + await new Promise((resolve) => { + stopTimer = setTimeout(() => { + clearInterval(check) + finish(partial) + resolve() + }, 2_500) + const check = setInterval(() => { + if (!finished) return + clearInterval(check) + resolve() + }, 25) + }) + }, + cancel: () => { + if (finished) return + finished = true + cleanup() + } + } +} + +export async function startDeepgramRealtimeTranscription(options: { + getToken: TokenFactory + language?: string + signal?: AbortSignal + callbacks: RealtimeTranscriptionCallbacks +}): Promise { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } + }) + try { + options.signal?.throwIfAborted() + } catch (error) { + stream.getTracks().forEach((track) => track.stop()) + throw error + } + let token: string + try { + token = await options.getToken() + } catch (error) { + stream.getTracks().forEach((track) => track.stop()) + throw error + } + const query = new URLSearchParams({ + model: DEEPGRAM_TRANSCRIPTION_MODEL, + smart_format: 'true', + interim_results: 'true', + endpointing: 'false' + }) + if (options.language) query.set('language', options.language) + const mimeType = [ + 'audio/webm;codecs=opus', + 'audio/webm', + 'audio/ogg;codecs=opus' + ].find((type) => typeof MediaRecorder.isTypeSupported !== 'function' || MediaRecorder.isTypeSupported(type)) + let socket: WebSocket + let recorder: MediaRecorder + try { + socket = new WebSocket(`wss://api.deepgram.com/v1/listen?${query}`, ['bearer', token]) + recorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream) + } catch (error) { + stream.getTracks().forEach((track) => track.stop()) + throw error + } + let committed = '' + let interim = '' + let finished = false + let stopping = false + let stopTimer: ReturnType | null = null + + const cleanup = () => { + if (stopTimer) clearTimeout(stopTimer) + stopTimer = null + recorder.ondataavailable = null + recorder.onstop = null + if (recorder.state !== 'inactive') recorder.stop() + stream.getTracks().forEach((track) => track.stop()) + if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify({ type: 'CloseStream' })) + socket.close() + } + const finish = (text: string) => { + if (finished) return + finished = true + cleanup() + options.callbacks.onFinal(text) + } + const fail = (value: unknown) => { + if (finished) return + finished = true + cleanup() + options.callbacks.onError(errorMessage(value, 'Deepgram realtime transcription failed')) + } + + recorder.ondataavailable = (event) => { + if (event.data.size > 0 && socket.readyState === WebSocket.OPEN) socket.send(event.data) + } + recorder.onerror = () => fail(new Error('Audio recording failed')) + recorder.onstop = () => { + if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify({ type: 'Finalize' })) + } + socket.addEventListener('message', (event) => { + let data: { + type?: string + is_final?: boolean + from_finalize?: boolean + channel?: { alternatives?: Array<{ transcript?: string }> } + description?: string + } + try { + data = JSON.parse(String(event.data)) as typeof data + } catch { + return + } + if (data.type === 'Error') return fail(new Error(data.description || 'Deepgram realtime transcription failed')) + if (data.type !== 'Results') return + const text = data.channel?.alternatives?.[0]?.transcript ?? '' + if (data.is_final) { + committed = joinTranscriptParts(committed, text) + interim = '' + } else { + interim = text + } + const current = joinTranscriptParts(committed, interim) + options.callbacks.onPartial(current) + if (stopping && data.is_final && data.from_finalize) finish(current) + }) + socket.addEventListener('close', () => { + if (!finished && !stopping) fail(new Error('Deepgram realtime transcription disconnected')) + }) + + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error('Deepgram realtime connection timed out')), 10_000) + const onAbort = () => { + clearTimeout(timeout) + reject(options.signal?.reason) + } + options.signal?.addEventListener('abort', onAbort, { once: true }) + socket.addEventListener('open', () => { + clearTimeout(timeout) + options.signal?.removeEventListener('abort', onAbort) + recorder.start(250) + options.callbacks.onConnected() + resolve() + }, { once: true }) + socket.addEventListener('error', () => { + clearTimeout(timeout) + options.signal?.removeEventListener('abort', onAbort) + reject(new Error('Deepgram realtime connection failed')) + }, { once: true }) + }).catch((error) => { + fail(error) + throw error + }) + + return { + stop: async () => { + if (finished || stopping) return + stopping = true + if (recorder.state !== 'inactive') recorder.stop() + else if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify({ type: 'Finalize' })) + await new Promise((resolve) => { + stopTimer = setTimeout(() => { + clearInterval(check) + finish(joinTranscriptParts(committed, interim)) + resolve() + }, 2_500) + const check = setInterval(() => { + if (!finished) return + clearInterval(check) + resolve() + }, 25) + }) + }, + cancel: () => { + if (finished) return + finished = true + cleanup() + } + } +} + +interface LocalSpeechRecognitionResult { + readonly isFinal: boolean + readonly 0: { readonly transcript: string } +} + +interface LocalSpeechRecognitionEvent extends Event { + readonly results: { readonly length: number; readonly [index: number]: LocalSpeechRecognitionResult } +} + +interface LocalSpeechRecognition extends EventTarget { + continuous: boolean + interimResults: boolean + lang: string + processLocally: boolean + onresult: ((event: LocalSpeechRecognitionEvent) => void) | null + onerror: ((event: Event & { error?: string }) => void) | null + onend: (() => void) | null + start: () => void + stop: () => void + abort: () => void +} + +interface LocalSpeechRecognitionConstructor { + new(): LocalSpeechRecognition + prototype: LocalSpeechRecognition + available: (options: { langs: string[]; processLocally: true }) => Promise +} + +function localSpeechRecognitionConstructor(): LocalSpeechRecognitionConstructor | null { + const constructor = (globalThis as typeof globalThis & { + SpeechRecognition?: LocalSpeechRecognitionConstructor + }).SpeechRecognition + return constructor + && typeof constructor.available === 'function' + && 'processLocally' in constructor.prototype + ? constructor + : null +} + +export async function startBrowserLocalTranscription(options: { + language?: string + signal?: AbortSignal + callbacks: RealtimeTranscriptionCallbacks +}): Promise { + options.signal?.throwIfAborted() + const constructor = localSpeechRecognitionConstructor() + if (!constructor) throw new Error('On-device speech recognition is not supported by this browser') + const language = options.language || navigator.language + if (await constructor.available({ langs: [language], processLocally: true }) !== 'available') { + throw new Error(`On-device speech recognition is not installed for ${language}`) + } + options.signal?.throwIfAborted() + + const recognition = new constructor() + recognition.continuous = true + recognition.interimResults = true + recognition.lang = language + recognition.processLocally = true + let current = '' + let finished = false + let stopping = false + + const finish = () => { + if (finished) return + finished = true + options.callbacks.onFinal(current) + } + recognition.onresult = (event) => { + const finalParts: string[] = [] + const interimParts: string[] = [] + for (let index = 0; index < event.results.length; index += 1) { + const result = event.results[index] + const transcript = result?.[0]?.transcript ?? '' + if (result?.isFinal) finalParts.push(transcript) + else interimParts.push(transcript) + } + current = joinTranscriptParts(...finalParts, ...interimParts) + options.callbacks.onPartial(current) + } + recognition.onerror = (event) => { + if (event.error === 'aborted' && stopping) return + finished = true + options.callbacks.onError(new Error(event.error ? `On-device transcription failed: ${event.error}` : 'On-device transcription failed')) + } + recognition.onend = () => { + if (stopping) finish() + else if (!finished) { + finished = true + options.callbacks.onError(new Error('On-device transcription stopped')) + } + } + recognition.start() + options.callbacks.onConnected() + + return { + stop: async () => { + if (finished || stopping) return + stopping = true + recognition.stop() + await new Promise((resolve) => { + const timeout = setTimeout(() => { + clearInterval(check) + finish() + resolve() + }, 2_500) + const check = setInterval(() => { + if (!finished) return + clearInterval(check) + clearTimeout(timeout) + resolve() + }, 25) + }) + }, + cancel: () => { + if (finished) return + finished = true + recognition.abort() + } + } +} diff --git a/web/src/hooks/useDictation.test.ts b/web/src/hooks/useDictation.test.ts index 19ab7703..e151f60d 100644 --- a/web/src/hooks/useDictation.test.ts +++ b/web/src/hooks/useDictation.test.ts @@ -60,4 +60,51 @@ describe('useDictation', () => { expect(api.transcribeVoice).toHaveBeenCalledOnce() expect(stopTrack).toHaveBeenCalled() }) + + it('shows on-device partial text and inserts only the final transcript', async () => { + let recognition: MockSpeechRecognition | null = null + class MockSpeechRecognition { + static async available() { return 'available' } + continuous = false + interimResults = false + lang = '' + processLocally = false + onresult: ((event: Event & { results: unknown }) => void) | null = null + onerror: ((event: Event) => void) | null = null + onend: (() => void) | null = null + constructor() { recognition = this } + start() {} + stop() { this.onend?.() } + abort() {} + emit(text: string, isFinal: boolean) { + const result = Object.assign([{ transcript: text }], { isFinal }) + this.onresult?.({ results: [result] } as unknown as Event & { results: unknown }) + } + } + Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { + configurable: true, + writable: true, + value: false + }) + vi.stubGlobal('SpeechRecognition', MockSpeechRecognition) + + const onTextChange = vi.fn() + const { result } = renderHook(() => useDictation({ + api: {} as ApiClient, + provider: 'browser-local', + mode: 'realtime', + getCurrentText: () => 'existing draft', + onTextChange + })) + + await act(() => result.current.toggle()) + act(() => recognition?.emit('live words', false)) + expect(result.current.partialTranscript).toBe('live words') + expect(onTextChange).not.toHaveBeenCalled() + act(() => recognition?.emit('final words', true)) + await act(() => result.current.toggle()) + + await waitFor(() => expect(onTextChange).toHaveBeenCalledWith('existing draft final words')) + expect(result.current.partialTranscript).toBe('') + }) }) diff --git a/web/src/hooks/useDictation.ts b/web/src/hooks/useDictation.ts index 22252ee7..9976b597 100644 --- a/web/src/hooks/useDictation.ts +++ b/web/src/hooks/useDictation.ts @@ -2,6 +2,7 @@ import { useCallback, useEffect, useRef, useState } from 'react' import type { ApiClient } from '@/api/client' import type { ConversationStatus } from '@/realtime/types' import type { TranscriptionMode, TranscriptionProvider } from '@hapi/protocol/voice' +import { useRealtimeDictation } from './useRealtimeDictation' export function appendTranscript(text: string, transcript: string): string { const addition = transcript.trim() @@ -33,10 +34,19 @@ export function useDictation(config: { getCurrentText: () => string onTextChange: (text: string) => void }) { + const onFinalTranscript = useCallback((transcript: string) => { + config.onTextChange(appendTranscript(config.getCurrentText(), transcript)) + }, [config]) + const realtime = useRealtimeDictation({ + api: config.api, + provider: config.provider, + mode: config.mode, + onFinalTranscript + }) const browserCanRecord = typeof navigator !== 'undefined' && typeof navigator.mediaDevices?.getUserMedia === 'function' && typeof MediaRecorder !== 'undefined' - const supported = config.api !== null + const standardSupported = config.api !== null && config.provider !== null && config.mode === 'standard' && browserCanRecord @@ -55,7 +65,7 @@ export function useDictation(config: { }, []) const start = useCallback(async () => { - if (!supported || !config.provider || status === 'connecting' || status === 'connected') return + if (!standardSupported || !config.provider || status === 'connecting' || status === 'connected') return const operation = ++operationRef.current const provider = config.provider const language = localStorage.getItem('hapi-voice-lang') || undefined @@ -122,7 +132,7 @@ export function useDictation(config: { setError(startError instanceof Error ? startError.message : 'Could not start transcription') setStatus('error') } - }, [config, status, stopTracks, supported]) + }, [config, standardSupported, status, stopTracks]) const stop = useCallback(async () => { if (transcribingRef.current) return @@ -155,5 +165,7 @@ export function useDictation(config: { } }, [stopTracks]) - return { supported, status, error, toggle } + return config.mode === 'realtime' + ? realtime + : { supported: standardSupported, status, error, partialTranscript: '', toggle } } diff --git a/web/src/hooks/useRealtimeDictation.test.ts b/web/src/hooks/useRealtimeDictation.test.ts new file mode 100644 index 00000000..5983a176 --- /dev/null +++ b/web/src/hooks/useRealtimeDictation.test.ts @@ -0,0 +1,61 @@ +import { act, renderHook, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { ApiClient } from '@/api/client' +import { useRealtimeDictation } from './useRealtimeDictation' + +const scribe = vi.hoisted(() => ({ + options: null as unknown, + connect: vi.fn(async () => {}), + disconnect: vi.fn(), + commit: vi.fn() +})) + +vi.mock('@elevenlabs/react', () => ({ + CommitStrategy: { MANUAL: 'manual' }, + useScribe: (options: unknown) => { + scribe.options = options + return { + connect: scribe.connect, + disconnect: scribe.disconnect, + commit: scribe.commit + } + } +})) + +type ScribeCallbacks = { + onPartialTranscript: (event: { text: string }) => void + onDisconnect: () => void +} + +describe('useRealtimeDictation', () => { + afterEach(() => vi.clearAllMocks()) + + it('preserves partial ElevenLabs text on an unexpected disconnect', async () => { + Object.defineProperty(navigator, 'mediaDevices', { + configurable: true, + value: { getUserMedia: vi.fn() } + }) + const api = { + fetchRealtimeTranscriptionToken: vi.fn(async () => ({ token: 'single-use-token' })) + } as unknown as ApiClient + const onFinalTranscript = vi.fn() + const { result } = renderHook(() => useRealtimeDictation({ + api, + provider: 'elevenlabs', + mode: 'realtime', + onFinalTranscript + })) + + await act(() => result.current.toggle()) + const callbacks = scribe.options as ScribeCallbacks + act(() => callbacks.onPartialTranscript({ text: 'spoken words' })) + expect(onFinalTranscript).not.toHaveBeenCalled() + + act(() => callbacks.onDisconnect()) + + await waitFor(() => expect(result.current.status).toBe('error')) + expect(result.current.error).toBe('ElevenLabs realtime transcription disconnected') + expect(result.current.partialTranscript).toBe('') + expect(onFinalTranscript).toHaveBeenCalledWith('spoken words') + }) +}) diff --git a/web/src/hooks/useRealtimeDictation.ts b/web/src/hooks/useRealtimeDictation.ts new file mode 100644 index 00000000..8ed207e0 --- /dev/null +++ b/web/src/hooks/useRealtimeDictation.ts @@ -0,0 +1,248 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { CommitStrategy, useScribe } from '@elevenlabs/react' +import { + ELEVENLABS_REALTIME_TRANSCRIPTION_MODEL, + type TranscriptionMode, + type TranscriptionProvider +} from '@hapi/protocol/voice' +import type { ApiClient } from '@/api/client' +import type { ConversationStatus } from '@/realtime/types' +import { + startBrowserLocalTranscription, + startDeepgramRealtimeTranscription, + startOpenAIRealtimeTranscription, + type RealtimeTranscriptionCallbacks, + type RealtimeTranscriptionSession +} from './realtimeTranscription' + +function realtimeBrowserSupport(provider: TranscriptionProvider | null): boolean { + if (typeof navigator === 'undefined') return false + if (provider === 'browser-local') return true + if (typeof navigator.mediaDevices?.getUserMedia !== 'function') return false + if (provider === 'openai') return typeof RTCPeerConnection !== 'undefined' + if (provider === 'deepgram') return typeof WebSocket !== 'undefined' && typeof MediaRecorder !== 'undefined' + return provider === 'elevenlabs' +} + +export function useRealtimeDictation(config: { + api: ApiClient | null + provider: TranscriptionProvider | null + mode: TranscriptionMode + onFinalTranscript: (text: string) => void +}) { + const supported = config.api !== null + && config.mode === 'realtime' + && ['openai', 'elevenlabs', 'deepgram', 'browser-local'].includes(config.provider ?? '') + && realtimeBrowserSupport(config.provider) + const [status, setStatus] = useState('disconnected') + const [error, setError] = useState(null) + const [partialTranscript, setPartialTranscript] = useState('') + const mountedRef = useRef(true) + const operationRef = useRef(0) + const startAbortRef = useRef(null) + const sessionRef = useRef(null) + const partialRef = useRef('') + const onFinalTranscriptRef = useRef(config.onFinalTranscript) + const elevenLabsActiveRef = useRef(false) + const elevenLabsFinalizedRef = useRef(false) + const resolveElevenLabsCommitRef = useRef<(() => void) | null>(null) + onFinalTranscriptRef.current = config.onFinalTranscript + + const updatePartial = useCallback((text: string) => { + partialRef.current = text + if (mountedRef.current) setPartialTranscript(text) + }, []) + + const finish = useCallback((text: string) => { + if (!mountedRef.current) return + updatePartial('') + onFinalTranscriptRef.current(text) + setStatus('disconnected') + }, [updatePartial]) + + const fail = useCallback((value: unknown) => { + if (!mountedRef.current) return + elevenLabsActiveRef.current = false + resolveElevenLabsCommitRef.current?.() + resolveElevenLabsCommitRef.current = null + sessionRef.current?.cancel() + sessionRef.current = null + setError(value instanceof Error ? value.message : 'Realtime transcription failed') + setStatus('error') + }, []) + + const elevenLabs = useScribe({ + modelId: ELEVENLABS_REALTIME_TRANSCRIPTION_MODEL, + commitStrategy: CommitStrategy.MANUAL, + onSessionStarted: () => { + if (mountedRef.current && elevenLabsActiveRef.current) setStatus('connected') + }, + onPartialTranscript: ({ text }) => { + if (elevenLabsActiveRef.current) updatePartial(text) + }, + onCommittedTranscript: ({ text }) => { + if (!elevenLabsActiveRef.current) return + elevenLabsFinalizedRef.current = true + elevenLabsActiveRef.current = false + finish(text) + resolveElevenLabsCommitRef.current?.() + resolveElevenLabsCommitRef.current = null + }, + onError: (scribeError) => { + if (elevenLabsActiveRef.current) fail(scribeError) + }, + onDisconnect: () => { + if (!mountedRef.current || !elevenLabsActiveRef.current) return + const partial = partialRef.current + elevenLabsActiveRef.current = false + resolveElevenLabsCommitRef.current?.() + resolveElevenLabsCommitRef.current = null + sessionRef.current = null + finish(partial) + setError('ElevenLabs realtime transcription disconnected') + setStatus('error') + } + }) + const elevenLabsRef = useRef(elevenLabs) + elevenLabsRef.current = elevenLabs + + const start = useCallback(async () => { + if (!supported || !config.api || !config.provider || status === 'connecting' || status === 'connected') return + const operation = ++operationRef.current + const controller = new AbortController() + startAbortRef.current = controller + const provider = config.provider + const language = localStorage.getItem('hapi-voice-lang') || undefined + setError(null) + updatePartial('') + setStatus('connecting') + + const callbacks: RealtimeTranscriptionCallbacks = { + onConnected: () => { + if (mountedRef.current && operationRef.current === operation) setStatus('connected') + }, + onPartial: (text) => { + if (operationRef.current === operation) updatePartial(text) + }, + onFinal: (text) => { + if (operationRef.current === operation) finish(text) + }, + onError: (realtimeError) => { + if (operationRef.current === operation) fail(realtimeError) + } + } + + try { + if (provider === 'elevenlabs') { + const { token } = await config.api.fetchRealtimeTranscriptionToken('elevenlabs', language, controller.signal) + if (operationRef.current !== operation) return + elevenLabsActiveRef.current = true + elevenLabsFinalizedRef.current = false + await elevenLabs.connect({ + token, + modelId: ELEVENLABS_REALTIME_TRANSCRIPTION_MODEL, + commitStrategy: CommitStrategy.MANUAL, + languageCode: language?.split('-')[0]?.toLowerCase(), + microphone: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } + }) + if (operationRef.current !== operation) { + elevenLabsActiveRef.current = false + elevenLabs.disconnect() + return + } + sessionRef.current = { + stop: async () => { + setStatus('connecting') + const committed = new Promise((resolve) => { + resolveElevenLabsCommitRef.current = resolve + }) + try { + elevenLabs.commit() + await Promise.race([ + committed, + new Promise((resolve) => setTimeout(resolve, 2_500)) + ]) + } finally { + if (elevenLabsActiveRef.current && !elevenLabsFinalizedRef.current) finish(partialRef.current) + elevenLabsActiveRef.current = false + resolveElevenLabsCommitRef.current = null + elevenLabs.disconnect() + } + }, + cancel: () => { + elevenLabsActiveRef.current = false + elevenLabs.disconnect() + } + } + return + } + + const getToken = async () => { + if (operationRef.current !== operation) throw new Error('Realtime transcription cancelled') + const result = await config.api!.fetchRealtimeTranscriptionToken( + provider as 'openai' | 'deepgram', + language, + controller.signal + ) + if (operationRef.current !== operation) throw new Error('Realtime transcription cancelled') + return result.token + } + const session = provider === 'openai' + ? await startOpenAIRealtimeTranscription({ getToken, signal: controller.signal, callbacks }) + : provider === 'deepgram' + ? await startDeepgramRealtimeTranscription({ getToken, language, signal: controller.signal, callbacks }) + : await startBrowserLocalTranscription({ language, signal: controller.signal, callbacks }) + if (operationRef.current !== operation) session.cancel() + else sessionRef.current = session + } catch (startError) { + if (operationRef.current === operation) fail(startError) + } finally { + if (startAbortRef.current === controller) startAbortRef.current = null + } + }, [config.api, config.provider, elevenLabs, fail, finish, status, supported, updatePartial]) + + const stop = useCallback(async () => { + const session = sessionRef.current + if (!session) { + operationRef.current += 1 + startAbortRef.current?.abort() + startAbortRef.current = null + if (elevenLabsActiveRef.current) { + elevenLabsActiveRef.current = false + elevenLabs.disconnect() + } + setStatus('disconnected') + return + } + setStatus('connecting') + try { + await session.stop() + } catch (stopError) { + fail(stopError) + } finally { + if (sessionRef.current === session) sessionRef.current = null + } + }, [elevenLabs, fail]) + + const toggle = useCallback(async () => { + if (status === 'connected' || status === 'connecting') await stop() + else await start() + }, [start, status, stop]) + + useEffect(() => { + mountedRef.current = true + return () => { + mountedRef.current = false + operationRef.current += 1 + startAbortRef.current?.abort() + startAbortRef.current = null + elevenLabsActiveRef.current = false + elevenLabsRef.current.disconnect() + resolveElevenLabsCommitRef.current?.() + sessionRef.current?.cancel() + sessionRef.current = null + } + }, []) + + return { supported, status, error, partialTranscript, toggle } +} diff --git a/web/src/hooks/useVoiceInputPreferences.ts b/web/src/hooks/useVoiceInputPreferences.ts index 398cfc9e..df107572 100644 --- a/web/src/hooks/useVoiceInputPreferences.ts +++ b/web/src/hooks/useVoiceInputPreferences.ts @@ -1,21 +1,39 @@ import { useCallback, useEffect, useState } from 'react' import type { ApiClient } from '@/api/client' -import type { - TranscriptionMode, - TranscriptionProvider, - TranscriptionProviderInfo, - VoiceMode +import { + BROWSER_LOCAL_TRANSCRIPTION_PROVIDER, + type TranscriptionMode, + type TranscriptionProvider, + type TranscriptionProviderInfo, + type VoiceMode } from '@hapi/protocol/voice' const VOICE_MODE_KEY = 'hapi-voice-mode' const TRANSCRIPTION_PROVIDER_KEY = 'hapi-transcription-provider' const TRANSCRIPTION_MODE_KEY = 'hapi-transcription-mode' const CHANGE_EVENT = 'hapi-voice-input-change' +export const VOICE_LANGUAGE_CHANGE_EVENT = 'hapi-voice-language-change' function notifyChange(): void { window.dispatchEvent(new Event(CHANGE_EVENT)) } +async function browserLocalTranscriptionSupported(): Promise { + const constructor = (globalThis as typeof globalThis & { + SpeechRecognition?: { + prototype: object + available?: (options: { langs: string[]; processLocally: true }) => Promise + } + }).SpeechRecognition + if (!constructor || typeof constructor.available !== 'function' || !('processLocally' in constructor.prototype)) return false + const language = localStorage.getItem('hapi-voice-lang') || navigator.language + try { + return await constructor.available({ langs: [language], processLocally: true }) === 'available' + } catch { + return false + } +} + function readVoiceMode(): VoiceMode { return localStorage.getItem(VOICE_MODE_KEY) === 'dictation' ? 'dictation' : 'assistant' } @@ -33,7 +51,8 @@ function resolveMode( stored: string | null ): TranscriptionMode { const modes = providers.find((candidate) => candidate.id === provider)?.modes ?? ['standard'] - return stored === 'realtime' && modes.includes('realtime') ? 'realtime' : 'standard' + if ((stored === 'standard' || stored === 'realtime') && modes.includes(stored)) return stored + return modes[0] ?? 'standard' } export function useVoiceInputPreferences(api: ApiClient | null) { @@ -45,16 +64,28 @@ export function useVoiceInputPreferences(api: ApiClient | null) { useEffect(() => { if (!api) return let cancelled = false - api.fetchTranscriptionProviders().then(({ providers: available }) => { - if (cancelled) return - setProviders(available) - const selectedProvider = resolveProvider(available, localStorage.getItem(TRANSCRIPTION_PROVIDER_KEY)) - setProviderState(selectedProvider) - setTranscriptionModeState(resolveMode(available, selectedProvider, localStorage.getItem(TRANSCRIPTION_MODE_KEY))) - }).catch(() => { - if (!cancelled) setProviders([]) - }) - return () => { cancelled = true } + let request = 0 + const refreshProviders = () => { + const current = ++request + Promise.all([api.fetchTranscriptionProviders(), browserLocalTranscriptionSupported()]).then(([{ providers: configured }, browserLocal]) => { + if (cancelled || current !== request) return + const available = browserLocal + ? [...configured, BROWSER_LOCAL_TRANSCRIPTION_PROVIDER] + : configured + setProviders(available) + const selectedProvider = resolveProvider(available, localStorage.getItem(TRANSCRIPTION_PROVIDER_KEY)) + setProviderState(selectedProvider) + setTranscriptionModeState(resolveMode(available, selectedProvider, localStorage.getItem(TRANSCRIPTION_MODE_KEY))) + }).catch(() => { + if (!cancelled && current === request) setProviders([]) + }) + } + refreshProviders() + window.addEventListener(VOICE_LANGUAGE_CHANGE_EVENT, refreshProviders) + return () => { + cancelled = true + window.removeEventListener(VOICE_LANGUAGE_CHANGE_EVENT, refreshProviders) + } }, [api]) useEffect(() => { diff --git a/web/src/lib/locales/en.ts b/web/src/lib/locales/en.ts index 9473f24a..cfbb9296 100644 --- a/web/src/lib/locales/en.ts +++ b/web/src/lib/locales/en.ts @@ -783,7 +783,7 @@ export default { 'settings.voice.transcriptionMode.standard': 'Standard', 'settings.voice.transcriptionMode.standard.hint': 'Record, then transcribe (default)', 'settings.voice.transcriptionMode.realtime': 'Realtime', - 'settings.voice.transcriptionMode.realtime.hint': 'Insert text while speaking', + 'settings.voice.transcriptionMode.realtime.hint': 'Show live text while speaking; insert the final result when stopped', 'settings.voice.voices.description': 'Choose the voice used by the selected backend.', 'settings.voice.sounds.title': 'How It Sounds', 'settings.voice.responds.title': 'How It Responds', diff --git a/web/src/lib/locales/zh-CN.ts b/web/src/lib/locales/zh-CN.ts index 6abe0977..83b837db 100644 --- a/web/src/lib/locales/zh-CN.ts +++ b/web/src/lib/locales/zh-CN.ts @@ -787,7 +787,7 @@ export default { 'settings.voice.transcriptionMode.standard': '标准', 'settings.voice.transcriptionMode.standard.hint': '录音结束后转录（默认）', 'settings.voice.transcriptionMode.realtime': '实时', - 'settings.voice.transcriptionMode.realtime.hint': '边说边插入文字', + 'settings.voice.transcriptionMode.realtime.hint': '说话时显示实时文字，停止后插入最终结果', 'settings.voice.voices.description': '选择当前后端使用的声音。', 'settings.voice.sounds.title': '声音效果', 'settings.voice.responds.title': '对话风格', diff --git a/web/src/routes/settings/useVoiceSettings.test.tsx b/web/src/routes/settings/useVoiceSettings.test.tsx index cbd0eca7..897f8fff 100644 --- a/web/src/routes/settings/useVoiceSettings.test.tsx +++ b/web/src/routes/settings/useVoiceSettings.test.tsx @@ -1,10 +1,11 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { act, renderHook, waitFor } from '@testing-library/react' +import type { TranscriptionProviderInfo } from '@hapi/protocol/voice' import { I18nProvider } from '@/lib/i18n-context' import { useVoiceSettings } from './useVoiceSettings' const { fetchTranscriptionProviders, fetchVoiceBackend, fetchVoices, pause, play } = vi.hoisted(() => ({ - fetchTranscriptionProviders: vi.fn(() => Promise.resolve({ providers: [] })), + fetchTranscriptionProviders: vi.fn((): Promise<{ providers: TranscriptionProviderInfo[] }> => Promise.resolve({ providers: [] })), fetchVoiceBackend: vi.fn(), fetchVoices: vi.fn(), pause: vi.fn(), @@ -26,6 +27,8 @@ function Wrapper(props: { children: React.ReactNode }) { } describe('useVoiceSettings', () => { + afterEach(() => vi.unstubAllGlobals()) + beforeEach(() => { vi.clearAllMocks() localStorage.clear() @@ -80,4 +83,60 @@ describe('useVoiceSettings', () => { unmount() expect(pause).toHaveBeenCalled() }) + + it('uses realtime for the realtime-only browser provider', async () => { + class MockSpeechRecognition { + static available() { return Promise.resolve('available') } + processLocally = false + } + Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false }) + vi.stubGlobal('SpeechRecognition', MockSpeechRecognition) + localStorage.setItem('hapi-transcription-mode', 'standard') + + const { result } = renderHook(() => useVoiceSettings(), { wrapper: Wrapper }) + + await waitFor(() => expect(result.current.provider).toBe('browser-local')) + expect(result.current.transcriptionMode).toBe('realtime') + }) + + it('does not expose browser dictation without the selected language pack', async () => { + fetchTranscriptionProviders.mockResolvedValueOnce({ + providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }] + }) + class MockSpeechRecognition { + static available() { return Promise.resolve('unavailable') } + processLocally = false + } + Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false }) + vi.stubGlobal('SpeechRecognition', MockSpeechRecognition) + + const { result } = renderHook(() => useVoiceSettings(), { wrapper: Wrapper }) + + await waitFor(() => expect(result.current.provider).toBe('openai')) + expect(result.current.providers).toHaveLength(1) + }) + + it('rechecks the on-device language pack when the language changes', async () => { + fetchTranscriptionProviders.mockResolvedValue({ + providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }] + }) + class MockSpeechRecognition { + static available({ langs }: { langs: string[] }) { + return Promise.resolve(langs[0] === 'en-US' ? 'available' : 'unavailable') + } + processLocally = false + } + Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false }) + vi.stubGlobal('SpeechRecognition', MockSpeechRecognition) + localStorage.setItem('hapi-voice-lang', 'en-US') + localStorage.setItem('hapi-transcription-provider', 'browser-local') + const { result } = renderHook(() => useVoiceSettings(), { wrapper: Wrapper }) + await waitFor(() => expect(result.current.provider).toBe('browser-local')) + + act(() => result.current.setVoiceLanguage({ code: 'zh-CN', name: 'Chinese', nativeName: '中文' })) + await waitFor(() => expect(result.current.provider).toBe('openai')) + + act(() => result.current.setVoiceLanguage({ code: 'en-US', name: 'English', nativeName: 'English' })) + await waitFor(() => expect(result.current.provider).toBe('browser-local')) + }) }) diff --git a/web/src/routes/settings/useVoiceSettings.ts b/web/src/routes/settings/useVoiceSettings.ts index 77d0f35c..0b02781c 100644 --- a/web/src/routes/settings/useVoiceSettings.ts +++ b/web/src/routes/settings/useVoiceSettings.ts @@ -12,7 +12,7 @@ import { writeStoredVoiceSelection, } from '@/lib/voicePickerPreferences' import type { VoiceBackendType } from '@hapi/protocol/voice' -import { useVoiceInputPreferences } from '@/hooks/useVoiceInputPreferences' +import { useVoiceInputPreferences, VOICE_LANGUAGE_CHANGE_EVENT } from '@/hooks/useVoiceInputPreferences' export function useVoiceSettings() { const { api } = useAppContext() @@ -83,6 +83,7 @@ export function useVoiceSettings() { setVoiceLanguageState(language.code) if (language.code === null) localStorage.removeItem('hapi-voice-lang') else localStorage.setItem('hapi-voice-lang', language.code) + window.dispatchEvent(new Event(VOICE_LANGUAGE_CHANGE_EVENT)) }, []) const stopPreview = useCallback(() => {