diff --git a/docs/guide/voice-assistant.md b/docs/guide/voice-assistant.md index 6897fff6..58afcf6c 100644 --- a/docs/guide/voice-assistant.md +++ b/docs/guide/voice-assistant.md @@ -20,7 +20,7 @@ export TRANSCRIPTION_API_KEY="..." # optional ``` Restart the hub after changing credentials. API keys are not entered or stored in the web app. -Realtime OpenAI, ElevenLabs, and Deepgram sessions receive only short-lived credentials minted by the hub. Browsers with an installed on-device `SpeechRecognition` language pack also expose **Browser on-device** as a realtime-only provider; HAPI never falls back from that option to browser-hosted recognition. +Realtime OpenAI, ElevenLabs, and Deepgram sessions receive only short-lived credentials minted by the hub. Eligible desktop browsers with the on-device `SpeechRecognition` API expose **Browser on-device** as a realtime-only provider. HAPI checks the selected language pack when dictation starts and never falls back from that option to browser-hosted recognition. Mobile and unknown browser environments fail closed because this API is experimental and some Android WebViews expose unsafe partial implementations. ## Overview diff --git a/web/src/hooks/browserLocalSpeech.test.ts b/web/src/hooks/browserLocalSpeech.test.ts new file mode 100644 index 00000000..2cdc3d4c --- /dev/null +++ b/web/src/hooks/browserLocalSpeech.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it, vi } from 'vitest' +import { getBrowserLocalSpeechSupport } from './browserLocalSpeech' + +const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36' + +function speechRecognitionShape(availableDescriptor?: PropertyDescriptor) { + class MockSpeechRecognition { + processLocally = false + } + Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false }) + if (availableDescriptor) Object.defineProperty(MockSpeechRecognition, 'available', availableDescriptor) + return MockSpeechRecognition +} + +describe('browser-local speech capability detection', () => { + it('uses the static available data property as a no-probe capability requirement', () => { + const available = vi.fn(() => Promise.resolve('available')) + const SpeechRecognition = speechRecognitionShape({ value: available }) + + const support = getBrowserLocalSpeechSupport({ + userAgent: DESKTOP_USER_AGENT, + userAgentData: { platform: 'macOS', mobile: false }, + speechRecognition: SpeechRecognition + }) + + expect(support?.constructor).toBe(SpeechRecognition) + expect(support?.available).toBe(available) + expect(available).not.toHaveBeenCalled() + }) + + it('rejects missing, non-function, and getter available properties without reading a getter', () => { + const getter = vi.fn(() => vi.fn(() => Promise.resolve('available'))) + const cases = [ + speechRecognitionShape(), + speechRecognitionShape({ value: 'available' }), + speechRecognitionShape({ get: getter }) + ] + + for (const SpeechRecognition of cases) { + expect(getBrowserLocalSpeechSupport({ + userAgent: DESKTOP_USER_AGENT, + userAgentData: { platform: 'macOS', mobile: false }, + speechRecognition: SpeechRecognition + })).toBeNull() + } + + expect(getter).not.toHaveBeenCalled() + }) + + it('fails closed when a desktop-looking UA has mobile Android UA-CH signals', () => { + const available = vi.fn(() => Promise.resolve('available')) + const SpeechRecognition = speechRecognitionShape({ value: available }) + + expect(getBrowserLocalSpeechSupport({ + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/140.0 Safari/537.36', + userAgentData: { platform: 'Android', mobile: true }, + speechRecognition: SpeechRecognition + })).toBeNull() + expect(available).not.toHaveBeenCalled() + }) + + it('rejects desktop-looking UAs without trusted desktop UA-CH', () => { + const available = vi.fn(() => Promise.resolve('available')) + const SpeechRecognition = speechRecognitionShape({ value: available }) + + expect(getBrowserLocalSpeechSupport({ + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/140.0 Safari/537.36', + speechRecognition: SpeechRecognition + })).toBeNull() + expect(getBrowserLocalSpeechSupport({ + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Version/18.0 Mobile/15E148 Safari/604.1', + speechRecognition: SpeechRecognition + })).toBeNull() + expect(available).not.toHaveBeenCalled() + }) +}) diff --git a/web/src/hooks/browserLocalSpeech.ts b/web/src/hooks/browserLocalSpeech.ts new file mode 100644 index 00000000..98aeffbf --- /dev/null +++ b/web/src/hooks/browserLocalSpeech.ts @@ -0,0 +1,107 @@ +export interface LocalSpeechRecognitionResult { + readonly isFinal: boolean + readonly 0: { readonly transcript: string } +} + +export interface LocalSpeechRecognitionEvent extends Event { + readonly results: { readonly length: number; readonly [index: number]: LocalSpeechRecognitionResult } +} + +export interface LocalSpeechRecognition extends EventTarget { + continuous: boolean + interimResults: boolean + lang: string + processLocally: boolean + onresult: ((event: LocalSpeechRecognitionEvent) => void) | null + onerror: ((event: Event & { error?: string }) => void) | null + onend: (() => void) | null + start: () => void + stop: () => void + abort: () => void +} + +export interface LocalSpeechRecognitionConstructor { + new(): LocalSpeechRecognition + prototype: LocalSpeechRecognition +} + +export type LocalSpeechRecognitionAvailability = ( + options: { langs: string[]; processLocally: true } +) => Promise | string + +export interface BrowserLocalSpeechEnvironment { + userAgent?: string + userAgentData?: { platform?: string; mobile?: boolean } + speechRecognition?: unknown +} + +export interface BrowserLocalSpeechSupport { + constructor: LocalSpeechRecognitionConstructor + available: LocalSpeechRecognitionAvailability +} + +function currentUserAgent(): string { + return typeof navigator === 'undefined' ? '' : navigator.userAgent +} + +function currentSpeechRecognition(): unknown { + return (globalThis as typeof globalThis & { + SpeechRecognition?: unknown + }).SpeechRecognition +} + +function currentUserAgentData(): BrowserLocalSpeechEnvironment['userAgentData'] { + return (typeof navigator === 'undefined' + ? undefined + : (navigator as Navigator & { userAgentData?: BrowserLocalSpeechEnvironment['userAgentData'] }).userAgentData) +} + +const SAFE_DESKTOP_PLATFORMS = new Set(['Windows', 'macOS', 'Linux', 'Chrome OS']) + +/** + * The experimental on-device speech API is eligible only with explicit, + * trustworthy User-Agent Client Hints that identify a desktop platform. Android + * WebViews can expose a partial shape whose native `available()` call crashes + * the renderer, so missing, unknown, and mobile environments fail closed. + */ +export function isConfirmedDesktopSpeechEnvironment( + _userAgent: string, + userAgentData?: BrowserLocalSpeechEnvironment['userAgentData'] +): boolean { + return userAgentData?.mobile === false + && typeof userAgentData.platform === 'string' + && SAFE_DESKTOP_PLATFORMS.has(userAgentData.platform) +} + +function staticAvailabilityMethod(candidate: Function): LocalSpeechRecognitionAvailability | null { + for (let target: object | null = candidate; target && target !== Function.prototype; target = Object.getPrototypeOf(target)) { + const descriptor = Object.getOwnPropertyDescriptor(target, 'available') + if (descriptor) return typeof descriptor.value === 'function' + ? descriptor.value as LocalSpeechRecognitionAvailability + : null + } + return null +} + +/** + * Checks only the browser API shape. It intentionally does not instantiate + * recognition or call the experimental `SpeechRecognition.available()` method. + */ +export function getBrowserLocalSpeechSupport( + environment: BrowserLocalSpeechEnvironment = {} +): BrowserLocalSpeechSupport | null { + const userAgent = environment.userAgent ?? currentUserAgent() + const userAgentData = environment.userAgentData ?? currentUserAgentData() + if (!isConfirmedDesktopSpeechEnvironment(userAgent, userAgentData)) return null + + const candidate = environment.speechRecognition ?? currentSpeechRecognition() + if (typeof candidate !== 'function') return null + const constructor = candidate as LocalSpeechRecognitionConstructor + if (!constructor.prototype || !('processLocally' in constructor.prototype)) return null + const available = staticAvailabilityMethod(candidate) + return available ? { constructor, available } : null +} + +export function hasBrowserLocalSpeechSupport(): boolean { + return getBrowserLocalSpeechSupport() !== null +} diff --git a/web/src/hooks/realtimeTranscription.test.ts b/web/src/hooks/realtimeTranscription.test.ts index 9106384c..10cc6812 100644 --- a/web/src/hooks/realtimeTranscription.test.ts +++ b/web/src/hooks/realtimeTranscription.test.ts @@ -1,5 +1,191 @@ import { afterEach, describe, expect, it, vi } from 'vitest' -import { startDeepgramRealtimeTranscription, startOpenAIRealtimeTranscription } from './realtimeTranscription' +import { + BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS, + getBrowserLocalAvailabilityProbeSubscriberCountForTesting, + startBrowserLocalTranscription, + startDeepgramRealtimeTranscription, + startOpenAIRealtimeTranscription +} from './realtimeTranscription' + +function browserLocalCallbacks() { + return { + onConnected: vi.fn(), + onPartial: vi.fn(), + onFinal: vi.fn(), + onError: vi.fn() + } +} + +function installBrowserLocalSpeechRecognition(available: () => Promise | string, onConstruct = vi.fn()) { + class MockSpeechRecognition extends EventTarget { + static available = available + continuous = false + interimResults = false + lang = '' + processLocally = false + onresult: ((event: Event) => void) | null = null + onerror: ((event: Event) => void) | null = null + onend: (() => void) | null = null + start = vi.fn() + stop = vi.fn() + abort = vi.fn() + constructor() { + super() + onConstruct() + } + } + Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false, writable: true }) + vi.stubGlobal('SpeechRecognition', MockSpeechRecognition) + vi.stubGlobal('navigator', { + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36', + userAgentData: { platform: 'macOS', mobile: false }, + language: 'en-US' + }) + return { available, onConstruct, constructor: MockSpeechRecognition } +} + +describe('browser-local realtime transcription', () => { + afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() + }) + + it('calls available only after an explicit browser-local dictation start', async () => { + const available = vi.fn(() => Promise.resolve('available')) + installBrowserLocalSpeechRecognition(available) + const callbacks = browserLocalCallbacks() + + const session = await startBrowserLocalTranscription({ language: 'en-US', callbacks }) + + expect(available).toHaveBeenCalledOnce() + expect(available).toHaveBeenCalledWith({ langs: ['en-US'], processLocally: true }) + expect(callbacks.onConnected).toHaveBeenCalledOnce() + session.cancel() + }) + + it('shares one successful native probe across concurrent browser-local starts', async () => { + let resolveAvailable!: (status: string) => void + const available = vi.fn(() => new Promise((resolve) => { resolveAvailable = resolve })) + const onConstruct = vi.fn() + installBrowserLocalSpeechRecognition(available, onConstruct) + const firstCallbacks = browserLocalCallbacks() + const secondCallbacks = browserLocalCallbacks() + + const first = startBrowserLocalTranscription({ language: 'en-US', callbacks: firstCallbacks }) + const second = startBrowserLocalTranscription({ language: 'en-US', callbacks: secondCallbacks }) + await vi.waitFor(() => expect(available).toHaveBeenCalledOnce()) + + resolveAvailable('available') + const sessions = await Promise.all([first, second]) + + expect(onConstruct).toHaveBeenCalledTimes(2) + expect(firstCallbacks.onConnected).toHaveBeenCalledOnce() + expect(secondCallbacks.onConnected).toHaveBeenCalledOnce() + sessions.forEach((session) => session.cancel()) + }) + + it('surfaces a synchronous available failure', async () => { + const available = vi.fn(() => { throw new Error('native failure') }) + installBrowserLocalSpeechRecognition(available) + + await expect(startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() })) + .rejects.toThrow('native failure') + expect(available).toHaveBeenCalledOnce() + }) + + it('surfaces a rejected available probe', async () => { + const available = vi.fn(() => Promise.reject(new Error('probe rejected'))) + installBrowserLocalSpeechRecognition(available) + + await expect(startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() })) + .rejects.toThrow('probe rejected') + expect(available).toHaveBeenCalledOnce() + }) + + it('times out a stalled available probe', async () => { + vi.useFakeTimers() + const available = vi.fn(() => new Promise(() => {})) + installBrowserLocalSpeechRecognition(available) + const starting = startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() }) + const expectation = expect(starting).rejects.toThrow('availability check timed out') + + await vi.advanceTimersByTimeAsync(BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS) + + await expectation + expect(available).toHaveBeenCalledOnce() + }) + + it('does not invoke available when its start signal aborts before the native microtask', async () => { + const available = vi.fn(() => Promise.resolve('available')) + installBrowserLocalSpeechRecognition(available) + const controller = new AbortController() + const starting = startBrowserLocalTranscription({ + language: 'en-US', + signal: controller.signal, + callbacks: browserLocalCallbacks() + }) + const expectation = expect(starting).rejects.toBeDefined() + + controller.abort() + await expectation + await Promise.resolve() + + expect(available).not.toHaveBeenCalled() + }) + + it('shares a stalled native probe across timeout retries and ignores its late result', async () => { + vi.useFakeTimers() + let resolveAvailable!: (status: string) => void + const available = vi.fn(() => new Promise((resolve) => { resolveAvailable = resolve })) + const onConstruct = vi.fn() + const speechRecognition = installBrowserLocalSpeechRecognition(available, onConstruct) + const firstCallbacks = browserLocalCallbacks() + const first = startBrowserLocalTranscription({ language: 'en-US', callbacks: firstCallbacks }) + const firstExpectation = expect(first).rejects.toThrow('availability check timed out') + + await vi.advanceTimersByTimeAsync(BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS) + await firstExpectation + expect(getBrowserLocalAvailabilityProbeSubscriberCountForTesting(speechRecognition.constructor, 'en-US')).toBe(0) + + const second = startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() }) + const secondExpectation = expect(second).rejects.toThrow('availability check timed out') + await vi.advanceTimersByTimeAsync(BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS) + await secondExpectation + expect(getBrowserLocalAvailabilityProbeSubscriberCountForTesting(speechRecognition.constructor, 'en-US')).toBe(0) + + const controller = new AbortController() + const aborted = startBrowserLocalTranscription({ + language: 'en-US', + signal: controller.signal, + callbacks: browserLocalCallbacks() + }) + const abortedExpectation = expect(aborted).rejects.toBeDefined() + controller.abort() + await abortedExpectation + expect(getBrowserLocalAvailabilityProbeSubscriberCountForTesting(speechRecognition.constructor, 'en-US')).toBe(0) + + expect(available).toHaveBeenCalledOnce() + resolveAvailable('available') + await Promise.resolve() + await Promise.resolve() + expect(onConstruct).not.toHaveBeenCalled() + expect(firstCallbacks.onConnected).not.toHaveBeenCalled() + }) + + it('rejects Android browser-local startup without calling a partial native available API', async () => { + const available = vi.fn(() => Promise.resolve('available')) + installBrowserLocalSpeechRecognition(available) + vi.stubGlobal('navigator', { + userAgent: 'Mozilla/5.0 (Linux; Android 15; WebView)', + userAgentData: { platform: 'Android', mobile: true }, + language: 'en-US' + }) + + await expect(startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() })) + .rejects.toThrow('not supported by this browser') + expect(available).not.toHaveBeenCalled() + }) +}) describe('OpenAI realtime transcription', () => { afterEach(() => vi.unstubAllGlobals()) diff --git a/web/src/hooks/realtimeTranscription.ts b/web/src/hooks/realtimeTranscription.ts index 4dec17b7..4737beea 100644 --- a/web/src/hooks/realtimeTranscription.ts +++ b/web/src/hooks/realtimeTranscription.ts @@ -1,4 +1,11 @@ import { DEEPGRAM_TRANSCRIPTION_MODEL } from '@hapi/protocol/voice' +import { + getBrowserLocalSpeechSupport, + type LocalSpeechRecognition, + type LocalSpeechRecognitionAvailability, + type LocalSpeechRecognitionConstructor, + type LocalSpeechRecognitionEvent +} from './browserLocalSpeech' export interface RealtimeTranscriptionCallbacks { onConnected: () => void @@ -330,43 +337,120 @@ export async function startDeepgramRealtimeTranscription(options: { } } -interface LocalSpeechRecognitionResult { - readonly isFinal: boolean - readonly 0: { readonly transcript: string } +export const BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS = 10_000 + +interface BrowserLocalAvailabilityProbe { + readonly available: LocalSpeechRecognitionAvailability + readonly constructor: LocalSpeechRecognitionConstructor + readonly language: string + readonly subscribers: Set } -interface LocalSpeechRecognitionEvent extends Event { - readonly results: { readonly length: number; readonly [index: number]: LocalSpeechRecognitionResult } +interface BrowserLocalAvailabilitySubscriber { + resolve: (status: string) => void + reject: (error: unknown) => void } -interface LocalSpeechRecognition extends EventTarget { - continuous: boolean - interimResults: boolean - lang: string - processLocally: boolean - onresult: ((event: LocalSpeechRecognitionEvent) => void) | null - onerror: ((event: Event & { error?: string }) => void) | null - onend: (() => void) | null - start: () => void - stop: () => void - abort: () => void +const browserLocalAvailabilityProbes = new WeakMap>() + +function browserLocalProbeMap(constructor: LocalSpeechRecognitionConstructor): Map { + let probes = browserLocalAvailabilityProbes.get(constructor) + if (!probes) { + probes = new Map() + browserLocalAvailabilityProbes.set(constructor, probes) + } + return probes } -interface LocalSpeechRecognitionConstructor { - new(): LocalSpeechRecognition - prototype: LocalSpeechRecognition - available: (options: { langs: string[]; processLocally: true }) => Promise +function getBrowserLocalAvailabilityProbe(options: { + available: LocalSpeechRecognitionAvailability + constructor: LocalSpeechRecognitionConstructor + language: string +}): BrowserLocalAvailabilityProbe { + const probes = browserLocalProbeMap(options.constructor) + const existing = probes.get(options.language) + if (existing) return existing + + const probe: BrowserLocalAvailabilityProbe = { + ...options, + subscribers: new Set() + } + probes.set(options.language, probe) + queueMicrotask(() => { + if (probe.subscribers.size === 0) { + probes.delete(options.language) + return + } + Promise.resolve() + .then(() => probe.available.call(probe.constructor, { langs: [probe.language], processLocally: true })) + .then( + (status) => settleBrowserLocalAvailabilityProbe(probes, probe, (subscriber) => subscriber.resolve(status)), + (error) => settleBrowserLocalAvailabilityProbe(probes, probe, (subscriber) => subscriber.reject(error)) + ) + }) + return probe } -function localSpeechRecognitionConstructor(): LocalSpeechRecognitionConstructor | null { - const constructor = (globalThis as typeof globalThis & { - SpeechRecognition?: LocalSpeechRecognitionConstructor - }).SpeechRecognition - return constructor - && typeof constructor.available === 'function' - && 'processLocally' in constructor.prototype - ? constructor - : null +function settleBrowserLocalAvailabilityProbe( + probes: Map, + probe: BrowserLocalAvailabilityProbe, + notify: (subscriber: BrowserLocalAvailabilitySubscriber) => void +): void { + if (probes.get(probe.language) !== probe) return + probes.delete(probe.language) + const subscribers = Array.from(probe.subscribers) + probe.subscribers.clear() + subscribers.forEach(notify) +} + +export function getBrowserLocalAvailabilityProbeSubscriberCountForTesting( + constructor: object, + language: string +): number { + return browserLocalAvailabilityProbes + .get(constructor as LocalSpeechRecognitionConstructor) + ?.get(language) + ?.subscribers.size ?? 0 +} + +function abortError(signal: AbortSignal): unknown { + return signal.reason ?? new Error('On-device speech recognition availability check was aborted') +} + +async function checkBrowserLocalSpeechAvailability(options: { + available: LocalSpeechRecognitionAvailability + constructor: LocalSpeechRecognitionConstructor + language: string + signal?: AbortSignal +}): Promise { + options.signal?.throwIfAborted() + const probe = getBrowserLocalAvailabilityProbe(options) + + return await new Promise((resolve, reject) => { + let settled = false + const subscriber: BrowserLocalAvailabilitySubscriber = { resolve, reject } + const detach = () => probe.subscribers.delete(subscriber) + const finish = (callback: () => void) => { + if (settled) return + settled = true + clearTimeout(timeout) + options.signal?.removeEventListener('abort', onAbort) + detach() + callback() + } + const timeout = setTimeout(() => { + finish(() => reject(new Error('On-device speech recognition availability check timed out'))) + }, BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS) + const onAbort = () => finish(() => reject(abortError(options.signal!))) + if (options.signal?.aborted) { + onAbort() + return + } + probe.subscribers.add(subscriber) + options.signal?.addEventListener('abort', onAbort, { once: true }) + subscriber.resolve = (status) => finish(() => resolve(status)) + subscriber.reject = (error) => finish(() => reject(error)) + }) } export async function startBrowserLocalTranscription(options: { @@ -375,15 +459,18 @@ export async function startBrowserLocalTranscription(options: { callbacks: RealtimeTranscriptionCallbacks }): Promise { options.signal?.throwIfAborted() - const constructor = localSpeechRecognitionConstructor() - if (!constructor) throw new Error('On-device speech recognition is not supported by this browser') + const support = getBrowserLocalSpeechSupport() + if (!support) throw new Error('On-device speech recognition is not supported by this browser') const language = options.language || navigator.language - if (await constructor.available({ langs: [language], processLocally: true }) !== 'available') { + // `available()` is deferred to a microtask so an abort between start and + // native invocation detaches its only consumer before touching the API. + options.signal?.throwIfAborted() + if (await checkBrowserLocalSpeechAvailability({ ...support, language, signal: options.signal }) !== 'available') { throw new Error(`On-device speech recognition is not installed for ${language}`) } options.signal?.throwIfAborted() - const recognition = new constructor() + const recognition = new support.constructor() recognition.continuous = true recognition.interimResults = true recognition.lang = language diff --git a/web/src/hooks/useDictation.test.ts b/web/src/hooks/useDictation.test.ts index e151f60d..77d3870c 100644 --- a/web/src/hooks/useDictation.test.ts +++ b/web/src/hooks/useDictation.test.ts @@ -62,6 +62,11 @@ describe('useDictation', () => { }) it('shows on-device partial text and inserts only the final transcript', async () => { + vi.stubGlobal('navigator', { + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36', + userAgentData: { platform: 'macOS', mobile: false }, + language: 'en-US' + }) let recognition: MockSpeechRecognition | null = null class MockSpeechRecognition { static async available() { return 'available' } diff --git a/web/src/hooks/useVoiceInputPreferences.test.tsx b/web/src/hooks/useVoiceInputPreferences.test.tsx new file mode 100644 index 00000000..e41141cf --- /dev/null +++ b/web/src/hooks/useVoiceInputPreferences.test.tsx @@ -0,0 +1,51 @@ +import { renderHook, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { ApiClient } from '@/api/client' +import { useVoiceInputPreferences } from './useVoiceInputPreferences' + +function installPartialSpeechRecognition(available = vi.fn(() => Promise.resolve('available'))) { + class MockSpeechRecognition { + static available = available + processLocally = false + } + Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false }) + vi.stubGlobal('SpeechRecognition', MockSpeechRecognition) + return available +} + +describe('useVoiceInputPreferences', () => { + afterEach(() => vi.unstubAllGlobals()) + + it('discovers browser-local support from its shape without probing available on mount', async () => { + const available = installPartialSpeechRecognition() + vi.stubGlobal('navigator', { + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36', + userAgentData: { platform: 'macOS', mobile: false }, + language: 'en-US' + }) + const api = { + fetchTranscriptionProviders: vi.fn(async () => ({ providers: [] })) + } + + const { result } = renderHook(() => useVoiceInputPreferences(api as unknown as ApiClient)) + + await waitFor(() => expect(result.current.provider).toBe('browser-local')) + expect(available).not.toHaveBeenCalled() + }) + + it('does not expose or probe partial browser-local speech APIs on Android', async () => { + const available = installPartialSpeechRecognition() + vi.stubGlobal('navigator', { userAgent: 'Mozilla/5.0 (Linux; Android 15; WebView)', language: 'en-US' }) + const api = { + fetchTranscriptionProviders: vi.fn(async () => ({ + providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }] + })) + } + + const { result } = renderHook(() => useVoiceInputPreferences(api as unknown as ApiClient)) + + await waitFor(() => expect(result.current.provider).toBe('openai')) + expect(result.current.providers).toHaveLength(1) + expect(available).not.toHaveBeenCalled() + }) +}) diff --git a/web/src/hooks/useVoiceInputPreferences.ts b/web/src/hooks/useVoiceInputPreferences.ts index df107572..c655522b 100644 --- a/web/src/hooks/useVoiceInputPreferences.ts +++ b/web/src/hooks/useVoiceInputPreferences.ts @@ -7,33 +7,17 @@ import { type TranscriptionProviderInfo, type VoiceMode } from '@hapi/protocol/voice' +import { hasBrowserLocalSpeechSupport } from './browserLocalSpeech' const VOICE_MODE_KEY = 'hapi-voice-mode' const TRANSCRIPTION_PROVIDER_KEY = 'hapi-transcription-provider' const TRANSCRIPTION_MODE_KEY = 'hapi-transcription-mode' const CHANGE_EVENT = 'hapi-voice-input-change' -export const VOICE_LANGUAGE_CHANGE_EVENT = 'hapi-voice-language-change' function notifyChange(): void { window.dispatchEvent(new Event(CHANGE_EVENT)) } -async function browserLocalTranscriptionSupported(): Promise { - const constructor = (globalThis as typeof globalThis & { - SpeechRecognition?: { - prototype: object - available?: (options: { langs: string[]; processLocally: true }) => Promise - } - }).SpeechRecognition - if (!constructor || typeof constructor.available !== 'function' || !('processLocally' in constructor.prototype)) return false - const language = localStorage.getItem('hapi-voice-lang') || navigator.language - try { - return await constructor.available({ langs: [language], processLocally: true }) === 'available' - } catch { - return false - } -} - function readVoiceMode(): VoiceMode { return localStorage.getItem(VOICE_MODE_KEY) === 'dictation' ? 'dictation' : 'assistant' } @@ -64,27 +48,21 @@ export function useVoiceInputPreferences(api: ApiClient | null) { useEffect(() => { if (!api) return let cancelled = false - let request = 0 - const refreshProviders = () => { - const current = ++request - Promise.all([api.fetchTranscriptionProviders(), browserLocalTranscriptionSupported()]).then(([{ providers: configured }, browserLocal]) => { - if (cancelled || current !== request) return - const available = browserLocal - ? [...configured, BROWSER_LOCAL_TRANSCRIPTION_PROVIDER] - : configured - setProviders(available) - const selectedProvider = resolveProvider(available, localStorage.getItem(TRANSCRIPTION_PROVIDER_KEY)) - setProviderState(selectedProvider) - setTranscriptionModeState(resolveMode(available, selectedProvider, localStorage.getItem(TRANSCRIPTION_MODE_KEY))) - }).catch(() => { - if (!cancelled && current === request) setProviders([]) - }) - } - refreshProviders() - window.addEventListener(VOICE_LANGUAGE_CHANGE_EVENT, refreshProviders) + const browserLocal = hasBrowserLocalSpeechSupport() + api.fetchTranscriptionProviders().then(({ providers: configured }) => { + if (cancelled) return + const available = browserLocal + ? [...configured, BROWSER_LOCAL_TRANSCRIPTION_PROVIDER] + : configured + setProviders(available) + const selectedProvider = resolveProvider(available, localStorage.getItem(TRANSCRIPTION_PROVIDER_KEY)) + setProviderState(selectedProvider) + setTranscriptionModeState(resolveMode(available, selectedProvider, localStorage.getItem(TRANSCRIPTION_MODE_KEY))) + }).catch(() => { + if (!cancelled) setProviders([]) + }) return () => { cancelled = true - window.removeEventListener(VOICE_LANGUAGE_CHANGE_EVENT, refreshProviders) } }, [api]) diff --git a/web/src/routes/settings/useVoiceSettings.test.tsx b/web/src/routes/settings/useVoiceSettings.test.tsx index 897f8fff..c2d263e6 100644 --- a/web/src/routes/settings/useVoiceSettings.test.tsx +++ b/web/src/routes/settings/useVoiceSettings.test.tsx @@ -32,6 +32,11 @@ describe('useVoiceSettings', () => { beforeEach(() => { vi.clearAllMocks() localStorage.clear() + vi.stubGlobal('navigator', { + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36', + userAgentData: { platform: 'macOS', mobile: false }, + language: 'en-US' + }) fetchVoiceBackend.mockResolvedValue({ backend: 'elevenlabs', backends: ['elevenlabs'] }) fetchVoices.mockResolvedValue([]) class MockAudio { @@ -85,8 +90,9 @@ describe('useVoiceSettings', () => { }) it('uses realtime for the realtime-only browser provider', async () => { + const available = vi.fn(() => Promise.resolve('available')) class MockSpeechRecognition { - static available() { return Promise.resolve('available') } + static available = available processLocally = false } Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false }) @@ -97,46 +103,28 @@ describe('useVoiceSettings', () => { await waitFor(() => expect(result.current.provider).toBe('browser-local')) expect(result.current.transcriptionMode).toBe('realtime') + expect(available).not.toHaveBeenCalled() }) - it('does not expose browser dictation without the selected language pack', async () => { + it('does not probe browser-local speech availability when the language changes', async () => { fetchTranscriptionProviders.mockResolvedValueOnce({ providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }] }) + const available = vi.fn(() => Promise.resolve('available')) class MockSpeechRecognition { - static available() { return Promise.resolve('unavailable') } + static available = available processLocally = false } Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false }) vi.stubGlobal('SpeechRecognition', MockSpeechRecognition) - - const { result } = renderHook(() => useVoiceSettings(), { wrapper: Wrapper }) - - await waitFor(() => expect(result.current.provider).toBe('openai')) - expect(result.current.providers).toHaveLength(1) - }) - - it('rechecks the on-device language pack when the language changes', async () => { - fetchTranscriptionProviders.mockResolvedValue({ - providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }] - }) - class MockSpeechRecognition { - static available({ langs }: { langs: string[] }) { - return Promise.resolve(langs[0] === 'en-US' ? 'available' : 'unavailable') - } - processLocally = false - } - Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false }) - vi.stubGlobal('SpeechRecognition', MockSpeechRecognition) - localStorage.setItem('hapi-voice-lang', 'en-US') localStorage.setItem('hapi-transcription-provider', 'browser-local') + const { result } = renderHook(() => useVoiceSettings(), { wrapper: Wrapper }) + await waitFor(() => expect(result.current.provider).toBe('browser-local')) act(() => result.current.setVoiceLanguage({ code: 'zh-CN', name: 'Chinese', nativeName: '中文' })) - await waitFor(() => expect(result.current.provider).toBe('openai')) - - act(() => result.current.setVoiceLanguage({ code: 'en-US', name: 'English', nativeName: 'English' })) - await waitFor(() => expect(result.current.provider).toBe('browser-local')) + expect(result.current.provider).toBe('browser-local') + expect(available).not.toHaveBeenCalled() }) }) diff --git a/web/src/routes/settings/useVoiceSettings.ts b/web/src/routes/settings/useVoiceSettings.ts index 5a9481cf..9fe048d8 100644 --- a/web/src/routes/settings/useVoiceSettings.ts +++ b/web/src/routes/settings/useVoiceSettings.ts @@ -12,7 +12,7 @@ import { writeStoredVoiceSelection, } from '@/lib/voicePickerPreferences' import type { VoiceBackendType } from '@hapi/protocol/voice' -import { useVoiceInputPreferences, VOICE_LANGUAGE_CHANGE_EVENT } from '@/hooks/useVoiceInputPreferences' +import { useVoiceInputPreferences } from '@/hooks/useVoiceInputPreferences' export function useVoiceSettings() { const { api } = useAppContext() @@ -83,7 +83,6 @@ export function useVoiceSettings() { setVoiceLanguageState(language.code) if (language.code === null) localStorage.removeItem('hapi-voice-lang') else localStorage.setItem('hapi-voice-lang', language.code) - window.dispatchEvent(new Event(VOICE_LANGUAGE_CHANGE_EVENT)) }, []) const stopPreview = useCallback(() => {