From 9d07857570bb689b4ae64dcd956afc6ddd7f60b1 Mon Sep 17 00:00:00 2001 From: SSU-WEI HUANG Date: Mon, 3 Aug 2026 06:05:58 +0800 Subject: [PATCH] Add provider-backed dictation mode (#1327) --- docs/guide/installation.md | 6 + docs/guide/voice-assistant.md | 24 ++- hub/README.md | 5 + hub/src/web/routes/voice.test.ts | 106 ++++++++++++ hub/src/web/routes/voice.ts | 152 ++++++++++++++++- shared/src/voice.backends.test.ts | 20 +++ shared/src/voice.ts | 29 ++++ web/src/api/client.test.ts | 12 ++ web/src/api/client.ts | 21 ++- .../AssistantChat/ComposerButtons.test.tsx | 27 ++- .../AssistantChat/ComposerButtons.tsx | 45 ++++- .../AssistantChat/HappyComposer.tsx | 57 ++++++- web/src/components/SessionChat.tsx | 1 + web/src/hooks/useDictation.test.ts | 63 +++++++ web/src/hooks/useDictation.ts | 159 ++++++++++++++++++ web/src/hooks/useVoiceInputPreferences.ts | 107 ++++++++++++ web/src/lib/locales/en.ts | 20 ++- web/src/lib/locales/zh-CN.ts | 20 ++- web/src/routes/settings/index.test.tsx | 8 + .../routes/settings/useVoiceSettings.test.tsx | 10 +- web/src/routes/settings/useVoiceSettings.ts | 3 + web/src/routes/settings/voice.tsx | 82 ++++++--- 22 files changed, 934 insertions(+), 43 deletions(-) create mode 100644 web/src/hooks/useDictation.test.ts create mode 100644 web/src/hooks/useDictation.ts create mode 100644 web/src/hooks/useVoiceInputPreferences.ts diff --git a/docs/guide/installation.md b/docs/guide/installation.md index bd295d2f..b0df4e0c 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -190,6 +190,12 @@ On first run, HAPI: | `DB_PATH` | `~/.hapi/hapi.db` | - | Database file path | | `ELEVENLABS_API_KEY` | - | - | ElevenLabs API key for voice | | `ELEVENLABS_AGENT_ID` | Auto-created | - | Custom ElevenLabs agent ID | +| `OPENAI_API_KEY` | - | - | OpenAI API key for dictation (`gpt-4o-transcribe`) | +| `DEEPGRAM_API_KEY` | - | - | Deepgram API key for dictation (`nova-3`) | +| `GROQ_API_KEY` | - | - | Groq API key for dictation (`whisper-large-v3`) | +| `TRANSCRIPTION_BASE_URL` | - | - | OpenAI-compatible/local transcription base URL | +| `TRANSCRIPTION_MODEL` | - | - | Model for the OpenAI-compatible transcription endpoint | +| `TRANSCRIPTION_API_KEY` | - | - | Optional bearer token for that endpoint |
diff --git a/docs/guide/voice-assistant.md b/docs/guide/voice-assistant.md index bea5595a..6d4e4da8 100644 --- a/docs/guide/voice-assistant.md +++ b/docs/guide/voice-assistant.md @@ -1,7 +1,26 @@ -# Voice Assistant +# Voice input and assistant Control your AI coding agent with voice using the built-in voice assistant powered by ElevenLabs Conversational AI. +For speech-to-text without a spoken assistant, open **Settings → Voice**, choose **Dictation**, then select a configured provider. Dictation records until you tap the microphone again, inserts the transcript into the composer, and never sends it automatically. Standard mode is the default. + +Provider credentials are read only from the hub's startup environment: + +```bash +# Pick any providers you use +export OPENAI_API_KEY="..." # gpt-4o-transcribe +export ELEVENLABS_API_KEY="..." # scribe_v2 +export DEEPGRAM_API_KEY="..." # nova-3 +export GROQ_API_KEY="..." # whisper-large-v3 + +# Or an OpenAI-compatible local server such as Speaches +export TRANSCRIPTION_BASE_URL="http://127.0.0.1:8000/v1" +export TRANSCRIPTION_MODEL="Systran/faster-whisper-large-v3" +export TRANSCRIPTION_API_KEY="..." # optional +``` + +Restart the hub after changing credentials. API keys are not entered or stored in the web app. + ## Overview The voice assistant lets you: @@ -14,7 +33,8 @@ The assistant bridges voice communication with your active coding agent (Claude ## Prerequisites -An [ElevenLabs](https://elevenlabs.io) account with API access +- Voice assistant: an [ElevenLabs](https://elevenlabs.io) account with API access +- Dictation: at least one configured provider above, or an OpenAI-compatible local server ## Setup diff --git a/hub/README.md b/hub/README.md index 7c502639..08634c4c 100644 --- a/hub/README.md +++ b/hub/README.md @@ -28,6 +28,11 @@ See `src/configuration.ts` for all options. - `ELEVENLABS_API_KEY` - ElevenLabs API key for voice assistant. - `ELEVENLABS_AGENT_ID` - Custom ElevenLabs agent ID (auto-created if not set). +- `OPENAI_API_KEY` - OpenAI dictation (`gpt-4o-transcribe`). +- `DEEPGRAM_API_KEY` - Deepgram dictation (`nova-3`). +- `GROQ_API_KEY` - Groq dictation (`whisper-large-v3`). +- `TRANSCRIPTION_BASE_URL` and `TRANSCRIPTION_MODEL` - OpenAI-compatible/local transcription endpoint and model. +- `TRANSCRIPTION_API_KEY` - Optional bearer token for the OpenAI-compatible endpoint. ### Optional diff --git a/hub/src/web/routes/voice.test.ts b/hub/src/web/routes/voice.test.ts index c7c49b0f..12c1e08f 100644 --- a/hub/src/web/routes/voice.test.ts +++ b/hub/src/web/routes/voice.test.ts @@ -75,6 +75,112 @@ describe('GET /api/voice/voices', () => { }) }) +describe('voice transcription routes', () => { + test('discovers only providers configured at hub startup', async () => { + const app = createApp() + const headers = await authHeaders() + const previous = { + openai: process.env.OPENAI_API_KEY, + elevenlabs: process.env.ELEVENLABS_API_KEY, + deepgram: process.env.DEEPGRAM_API_KEY, + groq: process.env.GROQ_API_KEY, + baseUrl: process.env.TRANSCRIPTION_BASE_URL, + model: process.env.TRANSCRIPTION_MODEL + } + delete process.env.OPENAI_API_KEY + delete process.env.ELEVENLABS_API_KEY + delete process.env.DEEPGRAM_API_KEY + delete process.env.GROQ_API_KEY + delete process.env.TRANSCRIPTION_BASE_URL + delete process.env.TRANSCRIPTION_MODEL + process.env.OPENAI_API_KEY = 'server-only-key' + process.env.TRANSCRIPTION_BASE_URL = 'http://localhost:8000/v1' + process.env.TRANSCRIPTION_MODEL = 'local-whisper' + + const res = await app.request('/api/voice/transcription/providers', { headers }) + expect(res.status).toBe(200) + expect(await res.json()).toEqual({ providers: [ + { id: 'openai', label: 'OpenAI', modes: ['standard'] }, + { id: 'openai-compatible', label: 'OpenAI-compatible / local', modes: ['standard'] } + ] }) + + for (const [key, value] of Object.entries({ + OPENAI_API_KEY: previous.openai, + ELEVENLABS_API_KEY: previous.elevenlabs, + DEEPGRAM_API_KEY: previous.deepgram, + GROQ_API_KEY: previous.groq, + TRANSCRIPTION_BASE_URL: previous.baseUrl, + TRANSCRIPTION_MODEL: previous.model + })) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + }) + + test('proxies a bounded recording to OpenAI with the default best model', async () => { + const app = createApp() + const headers = await authHeaders() + const previousKey = process.env.OPENAI_API_KEY + process.env.OPENAI_API_KEY = 'server-only-key' + const originalFetch = global.fetch + let upstreamUrl = '' + let upstreamInit: RequestInit | undefined + // @ts-expect-error test override + global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + upstreamUrl = String(input) + upstreamInit = init + return new Response(JSON.stringify({ text: 'transcribed text', language: 'en' }), { status: 200 }) + }) as typeof fetch + + const form = new FormData() + form.set('provider', 'openai') + form.set('mode', 'standard') + form.set('language', 'zh-CN') + form.set('file', new File(['audio bytes'], 'speech.webm', { type: 'audio/webm' })) + const res = await app.request('/api/voice/transcription', { method: 'POST', headers, body: form }) + + expect(res.status).toBe(200) + expect(await res.json()).toEqual({ text: 'transcribed text', language: 'en' }) + expect(upstreamUrl).toBe('https://api.openai.com/v1/audio/transcriptions') + expect(new Headers(upstreamInit?.headers).get('authorization')).toBe('Bearer server-only-key') + expect(upstreamInit?.body).toBeInstanceOf(FormData) + expect((upstreamInit?.body as FormData).get('model')).toBe('gpt-4o-transcribe') + expect((upstreamInit?.body as FormData).get('language')).toBe('zh') + + global.fetch = originalFetch + if (previousKey === undefined) delete process.env.OPENAI_API_KEY + else process.env.OPENAI_API_KEY = previousKey + }) + + test('rejects unsupported files before calling a provider', async () => { + const app = createApp() + const headers = await authHeaders() + const form = new FormData() + form.set('provider', 'openai') + form.set('file', new File(['not audio'], 'notes.txt', { type: 'text/plain' })) + + const res = await app.request('/api/voice/transcription', { method: 'POST', headers, body: form }) + expect(res.status).toBe(400) + expect(await res.json()).toEqual({ error: 'Unsupported audio file type' }) + }) + + test('rejects oversized request bodies before multipart parsing', async () => { + const app = createApp() + const res = await app.request('/api/voice/transcription', { + method: 'POST', + headers: { + ...(await authHeaders()), + 'content-length': String(27 * 1024 * 1024), + 'content-type': 'multipart/form-data; boundary=test' + }, + body: '--test--' + }) + + expect(res.status).toBe(413) + expect(await res.json()).toEqual({ error: 'Audio file too large' }) + }) +}) + describe('POST /api/voice/token', () => { it('creates/selects voice-specific agent when voiceId is provided', async () => { const app = createApp() diff --git a/hub/src/web/routes/voice.ts b/hub/src/web/routes/voice.ts index 6e8e9efa..03ea148a 100644 --- a/hub/src/web/routes/voice.ts +++ b/hub/src/web/routes/voice.ts @@ -1,14 +1,16 @@ import { Hono } from 'hono' +import { bodyLimit } from 'hono/body-limit' import { z } from 'zod' import type { WebAppEnv } from '../middleware/auth' import { ELEVENLABS_API_BASE, VOICE_AGENT_NAME, buildVoiceAgentConfig, + listConfiguredTranscriptionProviders, listConfiguredVoiceBackends, resolveHubVoiceBackend } from '@hapi/protocol/voice' -import type { VoiceBackendType } from '@hapi/protocol/voice' +import type { TranscriptionProvider, VoiceBackendType } from '@hapi/protocol/voice' function buildVoiceWsUrl(base: string, pathname: string): string { const url = new URL(base) @@ -34,6 +36,121 @@ const telemetryEventSchema = z.object({ details: z.record(z.string(), z.unknown()).optional() }) +const transcriptionProviderSchema = z.enum([ + 'openai', + 'elevenlabs', + 'deepgram', + 'groq', + 'openai-compatible' +]) +const transcriptionLanguageSchema = z.string() + .trim() + .max(35) + .regex(/^[a-z]{2,3}(?:-[a-z0-9]{2,8})*$/i) + .optional() + +const MAX_TRANSCRIPTION_BYTES = 25 * 1024 * 1024 +const MAX_TRANSCRIPTION_BODY_BYTES = MAX_TRANSCRIPTION_BYTES + 1024 * 1024 +const TRANSCRIPTION_TIMEOUT_MS = 120_000 + +function trimTrailingSlash(value: string): string { + return value.replace(/\/+$/, '') +} + +function getTranscriptionConfig(provider: TranscriptionProvider): { + apiKey?: string + baseUrl: string + model: string +} | null { + if (!listConfiguredTranscriptionProviders(process.env).some((candidate) => candidate.id === provider)) return null + switch (provider) { + case 'openai': + return { apiKey: process.env.OPENAI_API_KEY!.trim(), baseUrl: 'https://api.openai.com/v1', model: 'gpt-4o-transcribe' } + case 'elevenlabs': + return { apiKey: process.env.ELEVENLABS_API_KEY!.trim(), baseUrl: ELEVENLABS_API_BASE, model: 'scribe_v2' } + case 'deepgram': + return { apiKey: process.env.DEEPGRAM_API_KEY!.trim(), baseUrl: 'https://api.deepgram.com/v1', model: 'nova-3' } + case 'groq': + return { apiKey: process.env.GROQ_API_KEY!.trim(), baseUrl: 'https://api.groq.com/openai/v1', model: 'whisper-large-v3' } + case 'openai-compatible': { + const baseUrl = process.env.TRANSCRIPTION_BASE_URL?.trim() + const model = process.env.TRANSCRIPTION_MODEL?.trim() + return baseUrl && model + ? { apiKey: process.env.TRANSCRIPTION_API_KEY?.trim(), baseUrl: trimTrailingSlash(baseUrl), model } + : null + } + } +} + +async function transcribeStandard( + provider: TranscriptionProvider, + file: File, + language?: string +): Promise { + const config = getTranscriptionConfig(provider) + if (!config) return Response.json({ error: `${provider} transcription is not configured` }, { status: 400 }) + + let url: string + let headers: Record + let body: File | FormData + + if (provider === 'deepgram') { + const query = new URLSearchParams({ model: config.model, smart_format: 'true' }) + if (language) query.set('language', language) + url = `${config.baseUrl}/listen?${query}` + headers = { + Authorization: `Token ${config.apiKey}`, + 'Content-Type': file.type || 'application/octet-stream' + } + body = file + } else { + const form = new FormData() + const baseLanguage = language?.split('-')[0]?.toLowerCase() + form.set('file', file, file.name || 'speech.webm') + form.set(provider === 'elevenlabs' ? 'model_id' : 'model', config.model) + if (baseLanguage) { + if (provider === 'elevenlabs') form.set('language_code', baseLanguage) + else form.set('language', baseLanguage) + } + url = provider === 'elevenlabs' + ? `${config.baseUrl}/speech-to-text` + : `${trimTrailingSlash(config.baseUrl)}/audio/transcriptions` + headers = provider === 'elevenlabs' + ? { 'xi-api-key': config.apiKey ?? '' } + : (config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}) + body = form + } + + try { + const response = await fetch(url, { + method: 'POST', + headers, + body, + signal: AbortSignal.timeout(TRANSCRIPTION_TIMEOUT_MS) + }) + if (!response.ok) { + console.warn('[Voice][Transcription] Upstream request failed', { provider, status: response.status }) + return Response.json({ error: `${provider} transcription failed (HTTP ${response.status})` }, { status: 502 }) + } + const data = await response.json() as { + text?: string + language?: string + language_code?: string + results?: { channels?: Array<{ alternatives?: Array<{ transcript?: string }> }> } + } + const text = provider === 'deepgram' + ? data.results?.channels?.[0]?.alternatives?.[0]?.transcript + : data.text + return Response.json({ text: text ?? '', language: data.language ?? data.language_code }) + } catch (error) { + console.warn('[Voice][Transcription] Upstream request error', { + provider, + error: error instanceof Error ? error.message : String(error) + }) + return Response.json({ error: `${provider} transcription request failed` }, { status: 502 }) + } +} + // Cache for auto-created agent IDs (keyed by API key hash) const agentIdCache = new Map() @@ -254,6 +371,39 @@ export function createVoiceRoutes(): Hono { return c.json({ backend, backends }) }) + app.get('/voice/transcription/providers', (c) => { + return c.json({ providers: listConfiguredTranscriptionProviders(process.env) }) + }) + + app.post('/voice/transcription', bodyLimit({ + maxSize: MAX_TRANSCRIPTION_BODY_BYTES, + onError: (c) => c.json({ error: 'Audio file too large' }, 413) + }), async (c) => { + const form = await c.req.formData().catch(() => null) + if (!form) return c.json({ error: 'Invalid form data' }, 400) + + const provider = transcriptionProviderSchema.safeParse(form.get('provider')) + const mode = form.get('mode') + const file = form.get('file') + const languageValue = form.get('language') + const language = transcriptionLanguageSchema.safeParse( + typeof languageValue === 'string' && languageValue.trim() ? languageValue : undefined + ) + + if (!provider.success) return c.json({ error: 'Invalid transcription provider' }, 400) + if (!language.success) return c.json({ error: 'Invalid transcription language' }, 400) + if (mode !== null && mode !== 'standard') return c.json({ error: 'This endpoint only accepts standard transcription' }, 400) + if (!(file instanceof File)) return c.json({ error: 'Missing audio file' }, 400) + if (file.size === 0 || file.size > MAX_TRANSCRIPTION_BYTES) { + return c.json({ error: `Audio file must be between 1 byte and ${MAX_TRANSCRIPTION_BYTES} bytes` }, 400) + } + if (file.type && !file.type.startsWith('audio/') && file.type !== 'video/webm' && file.type !== 'video/mp4') { + return c.json({ error: 'Unsupported audio file type' }, 400) + } + + return transcribeStandard(provider.data, file, language.data) + }) + // Get Gemini API key for Gemini Live voice sessions // Gemini Live API does not support ephemeral tokens, so we proxy the key. // The key is short-lived in the browser session and never persisted client-side. diff --git a/shared/src/voice.backends.test.ts b/shared/src/voice.backends.test.ts index f53f9940..6e25c751 100644 --- a/shared/src/voice.backends.test.ts +++ b/shared/src/voice.backends.test.ts @@ -1,10 +1,30 @@ import { describe, expect, test } from 'bun:test' import { + listConfiguredTranscriptionProviders, listConfiguredVoiceBackends, resolveEffectiveVoiceBackend, resolveHubVoiceBackend } from './voice' +describe('listConfiguredTranscriptionProviders', () => { + test('returns configured providers with honest mode capabilities', () => { + expect(listConfiguredTranscriptionProviders({ + OPENAI_API_KEY: 'openai', + ELEVENLABS_API_KEY: 'elevenlabs', + TRANSCRIPTION_BASE_URL: 'http://localhost:8000/v1', + TRANSCRIPTION_MODEL: 'whisper-large-v3' + })).toEqual([ + { id: 'openai', label: 'OpenAI', modes: ['standard'] }, + { id: 'elevenlabs', label: 'ElevenLabs', modes: ['standard'] }, + { id: 'openai-compatible', label: 'OpenAI-compatible / local', modes: ['standard'] } + ]) + }) + + test('does not advertise incomplete or missing configuration', () => { + expect(listConfiguredTranscriptionProviders({ TRANSCRIPTION_BASE_URL: 'http://localhost:8000/v1' })).toEqual([]) + }) +}) + describe('listConfiguredVoiceBackends', () => { test('returns only backends with API keys', () => { const backends = listConfiguredVoiceBackends({ diff --git a/shared/src/voice.ts b/shared/src/voice.ts index 7cc4738c..e8ea1b5d 100644 --- a/shared/src/voice.ts +++ b/shared/src/voice.ts @@ -235,6 +235,35 @@ export function buildVoiceAgentConfig(): VoiceAgentConfig { export type VoiceBackendType = 'elevenlabs' | 'gemini-live' | 'qwen-realtime' +export type VoiceMode = 'assistant' | 'dictation' +export type TranscriptionMode = 'standard' | 'realtime' +export type TranscriptionProvider = 'openai' | 'elevenlabs' | 'deepgram' | 'groq' | 'openai-compatible' + +export interface TranscriptionProviderInfo { + id: TranscriptionProvider + label: string + modes: TranscriptionMode[] +} + +const TRANSCRIPTION_PROVIDERS: Record = { + openai: { id: 'openai', label: 'OpenAI', modes: ['standard'] }, + elevenlabs: { id: 'elevenlabs', label: 'ElevenLabs', modes: ['standard'] }, + deepgram: { id: 'deepgram', label: 'Deepgram', modes: ['standard'] }, + groq: { id: 'groq', label: 'Groq', modes: ['standard'] }, + 'openai-compatible': { id: 'openai-compatible', label: 'OpenAI-compatible / local', modes: ['standard'] } +} + +/** Transcription providers whose startup environment is complete. */ +export function listConfiguredTranscriptionProviders(env: VoiceBackendEnv): TranscriptionProviderInfo[] { + const providers: TranscriptionProvider[] = [] + if (env.OPENAI_API_KEY?.trim()) providers.push('openai') + if (env.ELEVENLABS_API_KEY?.trim()) providers.push('elevenlabs') + if (env.DEEPGRAM_API_KEY?.trim()) providers.push('deepgram') + if (env.GROQ_API_KEY?.trim()) providers.push('groq') + if (env.TRANSCRIPTION_BASE_URL?.trim() && env.TRANSCRIPTION_MODEL?.trim()) providers.push('openai-compatible') + return providers.map((provider) => TRANSCRIPTION_PROVIDERS[provider]) +} + export const QWEN_REALTIME_MODEL = 'qwen3.5-omni-flash-realtime' export const QWEN_REALTIME_VOICE = 'Tina' diff --git a/web/src/api/client.test.ts b/web/src/api/client.test.ts index 134bde6d..2bfa8006 100644 --- a/web/src/api/client.test.ts +++ b/web/src/api/client.test.ts @@ -115,4 +115,16 @@ describe('ApiClient error mapping', () => { }) expect(new Headers(init?.headers).get('content-type')).toBe('application/json') }) + + it('lets fetch set the multipart boundary for transcription uploads', async () => { + fetchMock.mockResolvedValueOnce(new Response(JSON.stringify({ text: 'hello' }), { status: 200 })) + + const api = new ApiClient('test-token') + const file = new File(['audio'], 'speech.webm', { type: 'audio/webm' }) + await api.transcribeVoice({ file, provider: 'openai', mode: 'standard' }) + + const [, init] = fetchMock.mock.calls[0] ?? [] + expect(init?.body).toBeInstanceOf(FormData) + expect(new Headers(init?.headers).has('content-type')).toBe(false) + }) }) diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 17c1b9a3..cca70656 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -47,6 +47,7 @@ import type { } from '@hapi/protocol/apiTypes' import type { AgentFlavor } from '@hapi/protocol' import type { CancelMessageResponse } from '@hapi/protocol/schemas' +import type { TranscriptionMode, TranscriptionProvider, TranscriptionProviderInfo } from '@hapi/protocol/voice' type ApiClientOptions = { baseUrl?: string @@ -122,7 +123,7 @@ export class ApiClient { if (authToken) { headers.set('authorization', `Bearer ${authToken}`) } - if (init?.body !== undefined && !headers.has('content-type')) { + if (init?.body !== undefined && !(init.body instanceof FormData) && !headers.has('content-type')) { headers.set('content-type', 'application/json') } @@ -933,6 +934,24 @@ export class ApiClient { return await this.request('/api/voice/backend') } + async fetchTranscriptionProviders(): Promise<{ providers: TranscriptionProviderInfo[] }> { + return await this.request('/api/voice/transcription/providers') + } + + async transcribeVoice(options: { + file: File + provider: TranscriptionProvider + mode: TranscriptionMode + language?: string + }): Promise<{ text: string; language?: string }> { + const form = new FormData() + form.set('file', options.file) + form.set('provider', options.provider) + form.set('mode', options.mode) + if (options.language) form.set('language', options.language) + return await this.request('/api/voice/transcription', { method: 'POST', body: form }) + } + async fetchQwenToken(): Promise<{ allowed: boolean wsUrl?: string diff --git a/web/src/components/AssistantChat/ComposerButtons.test.tsx b/web/src/components/AssistantChat/ComposerButtons.test.tsx index c8fe33ca..183b970f 100644 --- a/web/src/components/AssistantChat/ComposerButtons.test.tsx +++ b/web/src/components/AssistantChat/ComposerButtons.test.tsx @@ -1,8 +1,8 @@ import type { ReactElement } from 'react' -import { cleanup, render, screen } from '@testing-library/react' -import { afterEach, describe, expect, it } from 'vitest' +import { cleanup, fireEvent, render, screen } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' import { I18nProvider } from '@/lib/i18n-context' -import { UnifiedButton } from './ComposerButtons' +import { DictationButton, UnifiedButton } from './ComposerButtons' function renderInProviders(ui: ReactElement) { return render({ui}) @@ -82,3 +82,24 @@ describe('UnifiedButton — routesToScratchlist visual state', () => { expect(btn.className).not.toContain('bg-amber-500') }) }) + +describe('DictationButton', () => { + afterEach(cleanup) + + it('keeps dictation available when an existing draft makes the main button a send button', () => { + const onVoiceToggle = vi.fn() + renderInProviders( + , + ) + + fireEvent.click(getButton('Dictate')) + expect(onVoiceToggle).toHaveBeenCalledOnce() + }) +}) diff --git a/web/src/components/AssistantChat/ComposerButtons.tsx b/web/src/components/AssistantChat/ComposerButtons.tsx index 89b1d96c..2701502f 100644 --- a/web/src/components/AssistantChat/ComposerButtons.tsx +++ b/web/src/components/AssistantChat/ComposerButtons.tsx @@ -389,6 +389,7 @@ export function UnifiedButton(props: { controlsDisabled: boolean onSend: () => void onVoiceToggle: () => void + voiceLabel?: string /** * When true, the send button repaints amber and the aria-label * announces "Send to scratchlist" instead of "Send message". The @@ -446,7 +447,7 @@ export function UnifiedButton(props: { } else if (props.voiceEnabled) { icon = className = 'bg-black text-white' - ariaLabel = t('composer.voice') + ariaLabel = props.voiceLabel ?? t('composer.voice') } else { icon = className = 'bg-[#C0C0C0] text-white' @@ -478,6 +479,37 @@ export function UnifiedButton(props: { ) } +export function DictationButton(props: { + enabled: boolean + canSend: boolean + voiceEnabled: boolean + voiceStatus: ConversationStatus + controlsDisabled: boolean + onVoiceToggle: () => void +}) { + const { t } = useTranslation() + if ( + !props.enabled + || !props.canSend + || !props.voiceEnabled + || props.voiceStatus === 'connecting' + || props.voiceStatus === 'connected' + ) return null + + return ( + + ) +} + export function ComposerButtons(props: { canSend: boolean controlsDisabled: boolean @@ -496,6 +528,7 @@ export function ComposerButtons(props: { isSwitching: boolean onSwitch: () => void voiceEnabled: boolean + dictationEnabled?: boolean voiceStatus: ConversationStatus voiceMicMuted?: boolean onVoiceToggle: () => void @@ -739,6 +772,15 @@ export function ComposerButtons(props: { + + void onVoiceMicToggle?: () => void + voiceTranscriptionApi?: ApiClient // Schedule props (lifted from internal state when provided) pendingSchedule?: PendingSchedule | null onSchedule?: (pending: PendingSchedule) => void @@ -315,6 +319,40 @@ export function HappyComposer(props: { const attachments = useAuiState((s) => s.composer.attachments) const threadIsRunning = useAuiState((s) => s.thread.isRunning) const threadIsDisabled = useAuiState((s) => s.thread.isDisabled) + const composerTextRef = useRef(composerText) + composerTextRef.current = composerText + const getCurrentComposerText = useCallback(() => composerTextRef.current, []) + const setComposerText = useCallback((text: string) => api.composer().setText(text), [api]) + const voiceInput = useVoiceInputPreferences(props.voiceTranscriptionApi ?? null) + const dictationConfig = useMemo(() => ({ + api: props.voiceTranscriptionApi ?? null, + provider: voiceInput.provider, + mode: voiceInput.transcriptionMode, + getCurrentText: getCurrentComposerText, + onTextChange: setComposerText + }), [ + props.voiceTranscriptionApi, + voiceInput.provider, + voiceInput.transcriptionMode, + getCurrentComposerText, + setComposerText + ]) + const dictation = useDictation(dictationConfig) + const dictationActive = voiceInput.voiceMode === 'dictation' + const effectiveVoiceStatus = dictationActive ? dictation.status : voiceStatus + const effectiveVoiceToggle = dictationActive + ? (dictation.supported ? dictation.toggle : undefined) + : onVoiceToggle + const previousVoiceModeRef = useRef(voiceInput.voiceMode) + useEffect(() => { + if (previousVoiceModeRef.current === voiceInput.voiceMode) return + previousVoiceModeRef.current = voiceInput.voiceMode + if (dictationActive && (voiceStatus === 'connected' || voiceStatus === 'connecting')) { + onVoiceToggle?.() + } else if (!dictationActive && (dictation.status === 'connected' || dictation.status === 'connecting')) { + void dictation.toggle() + } + }, [dictationActive, voiceInput.voiceMode, voiceStatus, onVoiceToggle, dictation.status, dictation.toggle]) const controlsDisabled = disabled || (!active && !allowSendWhenInactive) || threadIsDisabled const trimmed = composerText.trim() @@ -889,7 +927,7 @@ export function HappyComposer(props: { || showFastModeSettings ) const showAbortButton = true - const voiceEnabled = Boolean(onVoiceToggle) + const voiceEnabled = Boolean(effectiveVoiceToggle) const handleSend = useCallback(() => { flushAndSend() @@ -1380,9 +1418,15 @@ export function HappyComposer(props: { permissionMode={permissionMode} collaborationMode={collaborationMode} agentFlavor={agentFlavor} - voiceStatus={voiceStatus} + voiceStatus={effectiveVoiceStatus} /> + {dictationActive && dictation.error ? ( +
+ {dictation.error} +
+ ) : null} + {sendError ? (
{})} - onVoiceMicToggle={onVoiceMicToggle} + dictationEnabled={dictationActive} + voiceStatus={effectiveVoiceStatus} + voiceMicMuted={dictationActive ? false : voiceMicMuted} + onVoiceToggle={effectiveVoiceToggle ?? (() => {})} + onVoiceMicToggle={dictationActive ? undefined : onVoiceMicToggle} onSend={handleSend} pendingSchedule={pendingSchedule} onSchedule={setPendingSchedule} diff --git a/web/src/components/SessionChat.tsx b/web/src/components/SessionChat.tsx index 4c30da81..d0e3bb87 100644 --- a/web/src/components/SessionChat.tsx +++ b/web/src/components/SessionChat.tsx @@ -1573,6 +1573,7 @@ function SessionChatInner(props: SessionChatProps) { voiceMicMuted={voice?.micMuted} onVoiceToggle={voice && voiceBackendReady ? handleVoiceToggle : undefined} onVoiceMicToggle={voice && voiceBackendReady ? handleVoiceMicToggle : undefined} + voiceTranscriptionApi={props.api} scratchlistMode={scratchlistMode} scratchlistCount={scratchlist.entries.length} onScratchlistToggle={handleScratchlistToggle} diff --git a/web/src/hooks/useDictation.test.ts b/web/src/hooks/useDictation.test.ts new file mode 100644 index 00000000..19ab7703 --- /dev/null +++ b/web/src/hooks/useDictation.test.ts @@ -0,0 +1,63 @@ +import { StrictMode } from 'react' +import { act, renderHook, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { ApiClient } from '@/api/client' +import { appendTranscript, useDictation } from './useDictation' + +describe('appendTranscript', () => { + it('preserves the draft and adds one separator', () => { + expect(appendTranscript('existing draft ', ' dictated words ')).toBe('existing draft dictated words') + expect(appendTranscript('existing draft\n', 'dictated words')).toBe('existing draft\ndictated words') + expect(appendTranscript('', ' dictated words ')).toBe('dictated words') + expect(appendTranscript('existing draft', ' ')).toBe('existing draft') + expect(appendTranscript('請更新 API', 'and run tests')).toBe('請更新 API and run tests') + }) +}) + +describe('useDictation', () => { + afterEach(() => vi.unstubAllGlobals()) + + it('records and inserts a final transcript under React StrictMode', async () => { + const stopTrack = vi.fn() + Object.defineProperty(navigator, 'mediaDevices', { + configurable: true, + value: { getUserMedia: vi.fn(async () => ({ getTracks: () => [{ stop: stopTrack }] })) } + }) + + class MockMediaRecorder { + static isTypeSupported() { return true } + state: RecordingState = 'inactive' + mimeType = 'audio/webm' + ondataavailable: ((event: BlobEvent) => void) | null = null + onerror: (() => void) | null = null + onstop: (() => void) | null = null + start() { this.state = 'recording' } + stop() { + this.state = 'inactive' + this.ondataavailable?.({ data: new Blob(['audio'], { type: this.mimeType }) } as BlobEvent) + this.onstop?.() + } + } + vi.stubGlobal('MediaRecorder', MockMediaRecorder) + + const onTextChange = vi.fn() + const api = { + transcribeVoice: vi.fn(async () => ({ text: 'dictated words' })) + } + const { result } = renderHook(() => useDictation({ + api: api as unknown as ApiClient, + provider: 'openai', + mode: 'standard', + getCurrentText: () => 'existing draft', + onTextChange + }), { wrapper: StrictMode }) + + await act(() => result.current.toggle()) + expect(result.current.status).toBe('connected') + await act(() => result.current.toggle()) + + await waitFor(() => expect(onTextChange).toHaveBeenCalledWith('existing draft dictated words')) + expect(api.transcribeVoice).toHaveBeenCalledOnce() + expect(stopTrack).toHaveBeenCalled() + }) +}) diff --git a/web/src/hooks/useDictation.ts b/web/src/hooks/useDictation.ts new file mode 100644 index 00000000..22252ee7 --- /dev/null +++ b/web/src/hooks/useDictation.ts @@ -0,0 +1,159 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import type { ApiClient } from '@/api/client' +import type { ConversationStatus } from '@/realtime/types' +import type { TranscriptionMode, TranscriptionProvider } from '@hapi/protocol/voice' + +export function appendTranscript(text: string, transcript: string): string { + const addition = transcript.trim() + if (!addition) return text + if (!text) return addition + return `${text}${/\s$/.test(text) ? '' : ' '}${addition}` +} + +function recordingExtension(mimeType: string): string { + if (mimeType.includes('mp4')) return 'm4a' + if (mimeType.includes('ogg')) return 'ogg' + return 'webm' +} + +function preferredMimeType(): string | undefined { + if (typeof MediaRecorder.isTypeSupported !== 'function') return undefined + return [ + 'audio/webm;codecs=opus', + 'audio/mp4', + 'audio/webm', + 'audio/ogg;codecs=opus' + ].find((type) => MediaRecorder.isTypeSupported(type)) +} + +export function useDictation(config: { + api: ApiClient | null + provider: TranscriptionProvider | null + mode: TranscriptionMode + getCurrentText: () => string + onTextChange: (text: string) => void +}) { + const browserCanRecord = typeof navigator !== 'undefined' + && typeof navigator.mediaDevices?.getUserMedia === 'function' + && typeof MediaRecorder !== 'undefined' + const supported = config.api !== null + && config.provider !== null + && config.mode === 'standard' + && browserCanRecord + const [status, setStatus] = useState('disconnected') + const [error, setError] = useState(null) + const recorderRef = useRef(null) + const streamRef = useRef(null) + const chunksRef = useRef([]) + const mountedRef = useRef(true) + const operationRef = useRef(0) + const transcribingRef = useRef(false) + + const stopTracks = useCallback(() => { + streamRef.current?.getTracks().forEach((track) => track.stop()) + streamRef.current = null + }, []) + + const start = useCallback(async () => { + if (!supported || !config.provider || status === 'connecting' || status === 'connected') return + const operation = ++operationRef.current + const provider = config.provider + const language = localStorage.getItem('hapi-voice-lang') || undefined + setError(null) + setStatus('connecting') + try { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } + }) + if (operationRef.current !== operation) { + stream.getTracks().forEach((track) => track.stop()) + return + } + const mimeType = preferredMimeType() + const recorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream) + streamRef.current = stream + recorderRef.current = recorder + chunksRef.current = [] + recorder.ondataavailable = (event) => { + if (event.data.size > 0) chunksRef.current.push(event.data) + } + recorder.onerror = () => { + stopTracks() + setError('Audio recording failed') + setStatus('error') + } + recorder.onstop = async () => { + stopTracks() + const type = recorder.mimeType || mimeType || 'audio/webm' + const blob = new Blob(chunksRef.current, { type }) + recorderRef.current = null + chunksRef.current = [] + if (!mountedRef.current) return + if (!blob.size) { + transcribingRef.current = false + setError('No audio was recorded') + setStatus('error') + return + } + transcribingRef.current = true + try { + const result = await config.api!.transcribeVoice({ + file: new File([blob], `speech.${recordingExtension(type)}`, { type }), + provider, + mode: 'standard', + language + }) + if (!mountedRef.current) return + config.onTextChange(appendTranscript(config.getCurrentText(), result.text)) + setStatus('disconnected') + } catch (transcriptionError) { + if (!mountedRef.current) return + setError(transcriptionError instanceof Error ? transcriptionError.message : 'Transcription failed') + setStatus('error') + } finally { + transcribingRef.current = false + } + } + recorder.start() + setStatus('connected') + } catch (startError) { + if (operationRef.current !== operation) return + stopTracks() + setError(startError instanceof Error ? startError.message : 'Could not start transcription') + setStatus('error') + } + }, [config, status, stopTracks, supported]) + + const stop = useCallback(async () => { + if (transcribingRef.current) return + operationRef.current += 1 + const recorder = recorderRef.current + if (recorder && recorder.state !== 'inactive') { + transcribingRef.current = true + setStatus('connecting') + recorder.stop() + } else { + setStatus('disconnected') + stopTracks() + } + }, [stopTracks]) + + const toggle = useCallback(async () => { + if (status === 'connected' || status === 'connecting') await stop() + else await start() + }, [start, status, stop]) + + useEffect(() => { + mountedRef.current = true + return () => { + mountedRef.current = false + operationRef.current += 1 + transcribingRef.current = false + const recorder = recorderRef.current + if (recorder && recorder.state !== 'inactive') recorder.stop() + stopTracks() + } + }, [stopTracks]) + + return { supported, status, error, toggle } +} diff --git a/web/src/hooks/useVoiceInputPreferences.ts b/web/src/hooks/useVoiceInputPreferences.ts new file mode 100644 index 00000000..398cfc9e --- /dev/null +++ b/web/src/hooks/useVoiceInputPreferences.ts @@ -0,0 +1,107 @@ +import { useCallback, useEffect, useState } from 'react' +import type { ApiClient } from '@/api/client' +import type { + TranscriptionMode, + TranscriptionProvider, + TranscriptionProviderInfo, + VoiceMode +} from '@hapi/protocol/voice' + +const VOICE_MODE_KEY = 'hapi-voice-mode' +const TRANSCRIPTION_PROVIDER_KEY = 'hapi-transcription-provider' +const TRANSCRIPTION_MODE_KEY = 'hapi-transcription-mode' +const CHANGE_EVENT = 'hapi-voice-input-change' + +function notifyChange(): void { + window.dispatchEvent(new Event(CHANGE_EVENT)) +} + +function readVoiceMode(): VoiceMode { + return localStorage.getItem(VOICE_MODE_KEY) === 'dictation' ? 'dictation' : 'assistant' +} + +function resolveProvider( + providers: readonly TranscriptionProviderInfo[], + stored: string | null +): TranscriptionProvider | null { + return providers.find((provider) => provider.id === stored)?.id ?? providers[0]?.id ?? null +} + +function resolveMode( + providers: readonly TranscriptionProviderInfo[], + provider: TranscriptionProvider | null, + stored: string | null +): TranscriptionMode { + const modes = providers.find((candidate) => candidate.id === provider)?.modes ?? ['standard'] + return stored === 'realtime' && modes.includes('realtime') ? 'realtime' : 'standard' +} + +export function useVoiceInputPreferences(api: ApiClient | null) { + const [voiceMode, setVoiceModeState] = useState(readVoiceMode) + const [providers, setProviders] = useState([]) + const [provider, setProviderState] = useState(null) + const [transcriptionMode, setTranscriptionModeState] = useState('standard') + + useEffect(() => { + if (!api) return + let cancelled = false + api.fetchTranscriptionProviders().then(({ providers: available }) => { + if (cancelled) return + setProviders(available) + const selectedProvider = resolveProvider(available, localStorage.getItem(TRANSCRIPTION_PROVIDER_KEY)) + setProviderState(selectedProvider) + setTranscriptionModeState(resolveMode(available, selectedProvider, localStorage.getItem(TRANSCRIPTION_MODE_KEY))) + }).catch(() => { + if (!cancelled) setProviders([]) + }) + return () => { cancelled = true } + }, [api]) + + useEffect(() => { + const sync = () => { + setVoiceModeState(readVoiceMode()) + const selectedProvider = resolveProvider(providers, localStorage.getItem(TRANSCRIPTION_PROVIDER_KEY)) + setProviderState(selectedProvider) + setTranscriptionModeState(resolveMode(providers, selectedProvider, localStorage.getItem(TRANSCRIPTION_MODE_KEY))) + } + window.addEventListener('storage', sync) + window.addEventListener(CHANGE_EVENT, sync) + return () => { + window.removeEventListener('storage', sync) + window.removeEventListener(CHANGE_EVENT, sync) + } + }, [providers]) + + const setVoiceMode = useCallback((value: VoiceMode) => { + localStorage.setItem(VOICE_MODE_KEY, value) + setVoiceModeState(value) + notifyChange() + }, []) + + const setProvider = useCallback((value: TranscriptionProvider) => { + const nextMode = resolveMode(providers, value, localStorage.getItem(TRANSCRIPTION_MODE_KEY)) + localStorage.setItem(TRANSCRIPTION_PROVIDER_KEY, value) + localStorage.setItem(TRANSCRIPTION_MODE_KEY, nextMode) + setProviderState(value) + setTranscriptionModeState(nextMode) + notifyChange() + }, [providers]) + + const setTranscriptionMode = useCallback((value: TranscriptionMode) => { + const nextMode = resolveMode(providers, provider, value) + localStorage.setItem(TRANSCRIPTION_MODE_KEY, nextMode) + setTranscriptionModeState(nextMode) + notifyChange() + }, [provider, providers]) + + return { + voiceMode, + setVoiceMode, + providers, + provider, + setProvider, + transcriptionMode, + setTranscriptionMode, + modes: providers.find((candidate) => candidate.id === provider)?.modes ?? ['standard'] + } +} diff --git a/web/src/lib/locales/en.ts b/web/src/lib/locales/en.ts index 72fe5adf..8c193620 100644 --- a/web/src/lib/locales/en.ts +++ b/web/src/lib/locales/en.ts @@ -518,6 +518,7 @@ export default { 'composer.send': 'Send', 'composer.stop': 'Stop', 'composer.voice': 'Voice assistant', + 'composer.dictate': 'Dictate', 'composer.scheduleSend': 'Schedule send', 'composer.scheduleRelativeTab': 'Relative', 'composer.scheduleSpecificTab': 'Specific', @@ -753,8 +754,21 @@ export default { 'settings.chat.composerToolbar.item.voiceMic': 'Voice microphone', 'settings.chat.composerToolbar.item.scratchlist': 'Scratchlist', 'settings.chat.composerToolbar.item.schedule': 'Schedule send', - 'settings.voice.title': 'Voice Assistant', - 'settings.voice.description': 'Voice connection, language, and everyday behavior.', + 'settings.voice.title': 'Voice', + 'settings.voice.description': 'Choose dictation or the full voice assistant.', + 'settings.voice.inputMode.title': 'Voice mode', + 'settings.voice.inputMode.hint': 'Dictation only inserts text into the composer; it never sends automatically.', + 'settings.voice.inputMode.assistant': 'Voice assistant', + 'settings.voice.inputMode.assistant.hint': 'Two-way spoken conversation', + 'settings.voice.inputMode.dictation': 'Dictation', + 'settings.voice.inputMode.dictation.hint': 'Speech-to-text input only', + 'settings.voice.transcriptionProvider': 'Transcription provider', + 'settings.voice.noTranscriptionProvider': 'No transcription provider is configured on the hub.', + 'settings.voice.transcriptionMode': 'Transcription mode', + 'settings.voice.transcriptionMode.standard': 'Standard', + 'settings.voice.transcriptionMode.standard.hint': 'Record, then transcribe (default)', + 'settings.voice.transcriptionMode.realtime': 'Realtime', + 'settings.voice.transcriptionMode.realtime.hint': 'Insert text while speaking', 'settings.voice.voices.description': 'Choose the voice used by the selected backend.', 'settings.voice.sounds.title': 'How It Sounds', 'settings.voice.responds.title': 'How It Responds', @@ -775,7 +789,7 @@ export default { 'settings.voice.opening.greet.hint': 'Assistant says hello and waits for you to speak.', 'settings.voice.opening.brief.hint': 'Assistant summarises agent activity when you connect.', 'settings.voice.group.label': 'Voice settings', - 'settings.voice.group.hint': 'Language and voice options apply to the selected backend.', + 'settings.voice.group.hint': 'Options apply to the selected voice or transcription provider.', 'settings.voice.backend': 'Voice backend', 'settings.voice.language': 'Voice Language', 'settings.voice.autoDetect': 'Auto-detect', diff --git a/web/src/lib/locales/zh-CN.ts b/web/src/lib/locales/zh-CN.ts index 61471d14..6d8141c4 100644 --- a/web/src/lib/locales/zh-CN.ts +++ b/web/src/lib/locales/zh-CN.ts @@ -522,6 +522,7 @@ export default { 'composer.send': '发送', 'composer.stop': '停止', 'composer.voice': '语音助手', + 'composer.dictate': '语音输入', 'composer.scheduleSend': '定时发送', 'composer.scheduleRelativeTab': '相对时间', 'composer.scheduleSpecificTab': '指定时间', @@ -757,8 +758,21 @@ export default { 'settings.chat.composerToolbar.item.voiceMic': '语音麦克风', 'settings.chat.composerToolbar.item.scratchlist': '暂存清单', 'settings.chat.composerToolbar.item.schedule': '定时发送', - 'settings.voice.title': '语音助手', - 'settings.voice.description': '语音连接、语言和常用行为。', + 'settings.voice.title': '语音', + 'settings.voice.description': '选择语音输入或完整语音助手。', + 'settings.voice.inputMode.title': '语音模式', + 'settings.voice.inputMode.hint': '语音输入只会把文字插入编辑框，不会自动发送。', + 'settings.voice.inputMode.assistant': '语音助手', + 'settings.voice.inputMode.assistant.hint': '双向语音对话', + 'settings.voice.inputMode.dictation': '语音输入', + 'settings.voice.inputMode.dictation.hint': '只做语音转文字', + 'settings.voice.transcriptionProvider': '转录提供商', + 'settings.voice.noTranscriptionProvider': 'Hub 尚未配置转录提供商。', + 'settings.voice.transcriptionMode': '转录模式', + 'settings.voice.transcriptionMode.standard': '标准', + 'settings.voice.transcriptionMode.standard.hint': '录音结束后转录（默认）', + 'settings.voice.transcriptionMode.realtime': '实时', + 'settings.voice.transcriptionMode.realtime.hint': '边说边插入文字', 'settings.voice.voices.description': '选择当前后端使用的声音。', 'settings.voice.sounds.title': '声音效果', 'settings.voice.responds.title': '对话风格', @@ -779,7 +793,7 @@ export default { 'settings.voice.opening.greet.hint': '助手先问候您，然后等待您开口。', 'settings.voice.opening.brief.hint': '连接时助手汇报当前代理活动。', 'settings.voice.group.label': '语音设置', - 'settings.voice.group.hint': '语言和声音选项取决于所选后端。', + 'settings.voice.group.hint': '选项取决于所选语音或转录提供商。', 'settings.voice.backend': '语音后端', 'settings.voice.language': '语音语言', 'settings.voice.autoDetect': '自动检测', diff --git a/web/src/routes/settings/index.test.tsx b/web/src/routes/settings/index.test.tsx index 9f867fed..e380bc75 100644 --- a/web/src/routes/settings/index.test.tsx +++ b/web/src/routes/settings/index.test.tsx @@ -166,6 +166,14 @@ vi.mock('@/components/settings/VoiceAdvancedControls', () => ({ vi.mock('./useVoiceSettings', () => ({ useVoiceSettings: () => ({ + voiceMode: 'assistant', + setVoiceMode: vi.fn(), + providers: [], + provider: null, + setProvider: vi.fn(), + transcriptionMode: 'standard', + setTranscriptionMode: vi.fn(), + modes: ['standard'], configuredBackends: ['elevenlabs'], backend: 'elevenlabs', setBackend: vi.fn(), diff --git a/web/src/routes/settings/useVoiceSettings.test.tsx b/web/src/routes/settings/useVoiceSettings.test.tsx index a7ab41b6..cbd0eca7 100644 --- a/web/src/routes/settings/useVoiceSettings.test.tsx +++ b/web/src/routes/settings/useVoiceSettings.test.tsx @@ -3,16 +3,18 @@ import { act, renderHook, waitFor } from '@testing-library/react' import { I18nProvider } from '@/lib/i18n-context' import { useVoiceSettings } from './useVoiceSettings' -const { fetchVoiceBackend, fetchVoices, pause, play } = vi.hoisted(() => ({ +const { fetchTranscriptionProviders, fetchVoiceBackend, fetchVoices, pause, play } = vi.hoisted(() => ({ + fetchTranscriptionProviders: vi.fn(() => Promise.resolve({ providers: [] })), fetchVoiceBackend: vi.fn(), fetchVoices: vi.fn(), pause: vi.fn(), play: vi.fn(() => Promise.resolve()), })) -vi.mock('@/lib/app-context', () => ({ - useAppContext: () => ({ api: {} }), -})) +vi.mock('@/lib/app-context', () => { + const api = { fetchTranscriptionProviders } + return { useAppContext: () => ({ api }) } +}) vi.mock('@/api/voice', () => ({ fetchVoiceBackend, diff --git a/web/src/routes/settings/useVoiceSettings.ts b/web/src/routes/settings/useVoiceSettings.ts index 1ebcd261..77d0f35c 100644 --- a/web/src/routes/settings/useVoiceSettings.ts +++ b/web/src/routes/settings/useVoiceSettings.ts @@ -12,9 +12,11 @@ import { writeStoredVoiceSelection, } from '@/lib/voicePickerPreferences' import type { VoiceBackendType } from '@hapi/protocol/voice' +import { useVoiceInputPreferences } from '@/hooks/useVoiceInputPreferences' export function useVoiceSettings() { const { api } = useAppContext() + const input = useVoiceInputPreferences(api) const { locale } = useTranslation() const [configuredBackends, setConfiguredBackends] = useState([]) const [backend, setBackendState] = useState(null) @@ -106,6 +108,7 @@ export function useVoiceSettings() { useEffect(() => stopPreview, [stopPreview]) return { + ...input, configuredBackends, backend, setBackend, diff --git a/web/src/routes/settings/voice.tsx b/web/src/routes/settings/voice.tsx index 336f0c7d..ccb6c443 100644 --- a/web/src/routes/settings/voice.tsx +++ b/web/src/routes/settings/voice.tsx @@ -24,8 +24,21 @@ export default function SettingsVoicePage() { return ( + + + + - {voice.configuredBackends.length > 1 && voice.backend ? ( + {voice.voiceMode === 'assistant' && voice.configuredBackends.length > 1 && voice.backend ? ( ) : null} + {voice.voiceMode === 'dictation' && voice.provider ? ( + ({ value: provider.id, label: provider.label }))} + onChange={voice.setProvider} + /> + ) : null} + {voice.voiceMode === 'dictation' && voice.providers.length === 0 ? ( +
+ {t('settings.voice.noTranscriptionProvider')} +
+ ) : null} + {voice.voiceMode === 'dictation' && voice.provider && voice.modes.length > 1 ? ( + ({ + value: mode, + label: t(`settings.voice.transcriptionMode.${mode}`), + description: t(`settings.voice.transcriptionMode.${mode}.hint`) + }))} + onChange={voice.setTranscriptionMode} + /> + ) : null} - navigate({ to: '/settings/voice/voices' })} - /> + {voice.voiceMode === 'assistant' ? ( + navigate({ to: '/settings/voice/voices' })} + /> + ) : null} - - ({ value, label: t(`settings.voice.opening.${value}`), description: t(`settings.voice.opening.${value}.hint`) }))} - onChange={setVoiceOpening} - /> - - - - navigate({ to: '/settings/voice/advanced' })} /> - + {voice.voiceMode === 'assistant' ? ( + <> + + ({ value, label: t(`settings.voice.opening.${value}`), description: t(`settings.voice.opening.${value}.hint`) }))} + onChange={setVoiceOpening} + /> + + + + navigate({ to: '/settings/voice/advanced' })} /> + + + ) : null} ) }