fix(web): make browser-local speech probing Android-safe (#1349)

* fix(web): guard browser-local speech probes

* test(web): cover concurrent speech probes

* docs: clarify browser-local speech probing
This commit is contained in:
KorenKrita
2026-08-04 08:19:56 +08:00
committed by GitHub
parent f10fbc7496
commit c1b32b51fe
10 changed files with 577 additions and 100 deletions
+1 -1
View File
@@ -20,7 +20,7 @@ export TRANSCRIPTION_API_KEY="..." # optional
``` ```
Restart the hub after changing credentials. API keys are not entered or stored in the web app. Restart the hub after changing credentials. API keys are not entered or stored in the web app.
Realtime OpenAI, ElevenLabs, and Deepgram sessions receive only short-lived credentials minted by the hub. Browsers with an installed on-device `SpeechRecognition` language pack also expose **Browser on-device** as a realtime-only provider; HAPI never falls back from that option to browser-hosted recognition. Realtime OpenAI, ElevenLabs, and Deepgram sessions receive only short-lived credentials minted by the hub. Eligible desktop browsers with the on-device `SpeechRecognition` API expose **Browser on-device** as a realtime-only provider. HAPI checks the selected language pack when dictation starts and never falls back from that option to browser-hosted recognition. Mobile and unknown browser environments fail closed because this API is experimental and some Android WebViews expose unsafe partial implementations.
## Overview ## Overview
+76
View File
@@ -0,0 +1,76 @@
import { describe, expect, it, vi } from 'vitest'
import { getBrowserLocalSpeechSupport } from './browserLocalSpeech'
const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36'
function speechRecognitionShape(availableDescriptor?: PropertyDescriptor) {
class MockSpeechRecognition {
processLocally = false
}
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
if (availableDescriptor) Object.defineProperty(MockSpeechRecognition, 'available', availableDescriptor)
return MockSpeechRecognition
}
describe('browser-local speech capability detection', () => {
it('uses the static available data property as a no-probe capability requirement', () => {
const available = vi.fn(() => Promise.resolve('available'))
const SpeechRecognition = speechRecognitionShape({ value: available })
const support = getBrowserLocalSpeechSupport({
userAgent: DESKTOP_USER_AGENT,
userAgentData: { platform: 'macOS', mobile: false },
speechRecognition: SpeechRecognition
})
expect(support?.constructor).toBe(SpeechRecognition)
expect(support?.available).toBe(available)
expect(available).not.toHaveBeenCalled()
})
it('rejects missing, non-function, and getter available properties without reading a getter', () => {
const getter = vi.fn(() => vi.fn(() => Promise.resolve('available')))
const cases = [
speechRecognitionShape(),
speechRecognitionShape({ value: 'available' }),
speechRecognitionShape({ get: getter })
]
for (const SpeechRecognition of cases) {
expect(getBrowserLocalSpeechSupport({
userAgent: DESKTOP_USER_AGENT,
userAgentData: { platform: 'macOS', mobile: false },
speechRecognition: SpeechRecognition
})).toBeNull()
}
expect(getter).not.toHaveBeenCalled()
})
it('fails closed when a desktop-looking UA has mobile Android UA-CH signals', () => {
const available = vi.fn(() => Promise.resolve('available'))
const SpeechRecognition = speechRecognitionShape({ value: available })
expect(getBrowserLocalSpeechSupport({
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
userAgentData: { platform: 'Android', mobile: true },
speechRecognition: SpeechRecognition
})).toBeNull()
expect(available).not.toHaveBeenCalled()
})
it('rejects desktop-looking UAs without trusted desktop UA-CH', () => {
const available = vi.fn(() => Promise.resolve('available'))
const SpeechRecognition = speechRecognitionShape({ value: available })
expect(getBrowserLocalSpeechSupport({
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
speechRecognition: SpeechRecognition
})).toBeNull()
expect(getBrowserLocalSpeechSupport({
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Version/18.0 Mobile/15E148 Safari/604.1',
speechRecognition: SpeechRecognition
})).toBeNull()
expect(available).not.toHaveBeenCalled()
})
})
+107
View File
@@ -0,0 +1,107 @@
export interface LocalSpeechRecognitionResult {
readonly isFinal: boolean
readonly 0: { readonly transcript: string }
}
export interface LocalSpeechRecognitionEvent extends Event {
readonly results: { readonly length: number; readonly [index: number]: LocalSpeechRecognitionResult }
}
export interface LocalSpeechRecognition extends EventTarget {
continuous: boolean
interimResults: boolean
lang: string
processLocally: boolean
onresult: ((event: LocalSpeechRecognitionEvent) => void) | null
onerror: ((event: Event & { error?: string }) => void) | null
onend: (() => void) | null
start: () => void
stop: () => void
abort: () => void
}
export interface LocalSpeechRecognitionConstructor {
new(): LocalSpeechRecognition
prototype: LocalSpeechRecognition
}
export type LocalSpeechRecognitionAvailability = (
options: { langs: string[]; processLocally: true }
) => Promise<string> | string
export interface BrowserLocalSpeechEnvironment {
userAgent?: string
userAgentData?: { platform?: string; mobile?: boolean }
speechRecognition?: unknown
}
export interface BrowserLocalSpeechSupport {
constructor: LocalSpeechRecognitionConstructor
available: LocalSpeechRecognitionAvailability
}
function currentUserAgent(): string {
return typeof navigator === 'undefined' ? '' : navigator.userAgent
}
function currentSpeechRecognition(): unknown {
return (globalThis as typeof globalThis & {
SpeechRecognition?: unknown
}).SpeechRecognition
}
function currentUserAgentData(): BrowserLocalSpeechEnvironment['userAgentData'] {
return (typeof navigator === 'undefined'
? undefined
: (navigator as Navigator & { userAgentData?: BrowserLocalSpeechEnvironment['userAgentData'] }).userAgentData)
}
const SAFE_DESKTOP_PLATFORMS = new Set(['Windows', 'macOS', 'Linux', 'Chrome OS'])
/**
* The experimental on-device speech API is eligible only with explicit,
* trustworthy User-Agent Client Hints that identify a desktop platform. Android
* WebViews can expose a partial shape whose native `available()` call crashes
* the renderer, so missing, unknown, and mobile environments fail closed.
*/
export function isConfirmedDesktopSpeechEnvironment(
_userAgent: string,
userAgentData?: BrowserLocalSpeechEnvironment['userAgentData']
): boolean {
return userAgentData?.mobile === false
&& typeof userAgentData.platform === 'string'
&& SAFE_DESKTOP_PLATFORMS.has(userAgentData.platform)
}
function staticAvailabilityMethod(candidate: Function): LocalSpeechRecognitionAvailability | null {
for (let target: object | null = candidate; target && target !== Function.prototype; target = Object.getPrototypeOf(target)) {
const descriptor = Object.getOwnPropertyDescriptor(target, 'available')
if (descriptor) return typeof descriptor.value === 'function'
? descriptor.value as LocalSpeechRecognitionAvailability
: null
}
return null
}
/**
* Checks only the browser API shape. It intentionally does not instantiate
* recognition or call the experimental `SpeechRecognition.available()` method.
*/
export function getBrowserLocalSpeechSupport(
environment: BrowserLocalSpeechEnvironment = {}
): BrowserLocalSpeechSupport | null {
const userAgent = environment.userAgent ?? currentUserAgent()
const userAgentData = environment.userAgentData ?? currentUserAgentData()
if (!isConfirmedDesktopSpeechEnvironment(userAgent, userAgentData)) return null
const candidate = environment.speechRecognition ?? currentSpeechRecognition()
if (typeof candidate !== 'function') return null
const constructor = candidate as LocalSpeechRecognitionConstructor
if (!constructor.prototype || !('processLocally' in constructor.prototype)) return null
const available = staticAvailabilityMethod(candidate)
return available ? { constructor, available } : null
}
export function hasBrowserLocalSpeechSupport(): boolean {
return getBrowserLocalSpeechSupport() !== null
}
+187 -1
View File
@@ -1,5 +1,191 @@
import { afterEach, describe, expect, it, vi } from 'vitest' import { afterEach, describe, expect, it, vi } from 'vitest'
import { startDeepgramRealtimeTranscription, startOpenAIRealtimeTranscription } from './realtimeTranscription' import {
BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS,
getBrowserLocalAvailabilityProbeSubscriberCountForTesting,
startBrowserLocalTranscription,
startDeepgramRealtimeTranscription,
startOpenAIRealtimeTranscription
} from './realtimeTranscription'
function browserLocalCallbacks() {
return {
onConnected: vi.fn(),
onPartial: vi.fn(),
onFinal: vi.fn(),
onError: vi.fn()
}
}
function installBrowserLocalSpeechRecognition(available: () => Promise<string> | string, onConstruct = vi.fn()) {
class MockSpeechRecognition extends EventTarget {
static available = available
continuous = false
interimResults = false
lang = ''
processLocally = false
onresult: ((event: Event) => void) | null = null
onerror: ((event: Event) => void) | null = null
onend: (() => void) | null = null
start = vi.fn()
stop = vi.fn()
abort = vi.fn()
constructor() {
super()
onConstruct()
}
}
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false, writable: true })
vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
vi.stubGlobal('navigator', {
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
userAgentData: { platform: 'macOS', mobile: false },
language: 'en-US'
})
return { available, onConstruct, constructor: MockSpeechRecognition }
}
describe('browser-local realtime transcription', () => {
afterEach(() => {
vi.useRealTimers()
vi.unstubAllGlobals()
})
it('calls available only after an explicit browser-local dictation start', async () => {
const available = vi.fn(() => Promise.resolve('available'))
installBrowserLocalSpeechRecognition(available)
const callbacks = browserLocalCallbacks()
const session = await startBrowserLocalTranscription({ language: 'en-US', callbacks })
expect(available).toHaveBeenCalledOnce()
expect(available).toHaveBeenCalledWith({ langs: ['en-US'], processLocally: true })
expect(callbacks.onConnected).toHaveBeenCalledOnce()
session.cancel()
})
it('shares one successful native probe across concurrent browser-local starts', async () => {
let resolveAvailable!: (status: string) => void
const available = vi.fn(() => new Promise<string>((resolve) => { resolveAvailable = resolve }))
const onConstruct = vi.fn()
installBrowserLocalSpeechRecognition(available, onConstruct)
const firstCallbacks = browserLocalCallbacks()
const secondCallbacks = browserLocalCallbacks()
const first = startBrowserLocalTranscription({ language: 'en-US', callbacks: firstCallbacks })
const second = startBrowserLocalTranscription({ language: 'en-US', callbacks: secondCallbacks })
await vi.waitFor(() => expect(available).toHaveBeenCalledOnce())
resolveAvailable('available')
const sessions = await Promise.all([first, second])
expect(onConstruct).toHaveBeenCalledTimes(2)
expect(firstCallbacks.onConnected).toHaveBeenCalledOnce()
expect(secondCallbacks.onConnected).toHaveBeenCalledOnce()
sessions.forEach((session) => session.cancel())
})
it('surfaces a synchronous available failure', async () => {
const available = vi.fn(() => { throw new Error('native failure') })
installBrowserLocalSpeechRecognition(available)
await expect(startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() }))
.rejects.toThrow('native failure')
expect(available).toHaveBeenCalledOnce()
})
it('surfaces a rejected available probe', async () => {
const available = vi.fn(() => Promise.reject(new Error('probe rejected')))
installBrowserLocalSpeechRecognition(available)
await expect(startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() }))
.rejects.toThrow('probe rejected')
expect(available).toHaveBeenCalledOnce()
})
it('times out a stalled available probe', async () => {
vi.useFakeTimers()
const available = vi.fn(() => new Promise<string>(() => {}))
installBrowserLocalSpeechRecognition(available)
const starting = startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() })
const expectation = expect(starting).rejects.toThrow('availability check timed out')
await vi.advanceTimersByTimeAsync(BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS)
await expectation
expect(available).toHaveBeenCalledOnce()
})
it('does not invoke available when its start signal aborts before the native microtask', async () => {
const available = vi.fn(() => Promise.resolve('available'))
installBrowserLocalSpeechRecognition(available)
const controller = new AbortController()
const starting = startBrowserLocalTranscription({
language: 'en-US',
signal: controller.signal,
callbacks: browserLocalCallbacks()
})
const expectation = expect(starting).rejects.toBeDefined()
controller.abort()
await expectation
await Promise.resolve()
expect(available).not.toHaveBeenCalled()
})
it('shares a stalled native probe across timeout retries and ignores its late result', async () => {
vi.useFakeTimers()
let resolveAvailable!: (status: string) => void
const available = vi.fn(() => new Promise<string>((resolve) => { resolveAvailable = resolve }))
const onConstruct = vi.fn()
const speechRecognition = installBrowserLocalSpeechRecognition(available, onConstruct)
const firstCallbacks = browserLocalCallbacks()
const first = startBrowserLocalTranscription({ language: 'en-US', callbacks: firstCallbacks })
const firstExpectation = expect(first).rejects.toThrow('availability check timed out')
await vi.advanceTimersByTimeAsync(BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS)
await firstExpectation
expect(getBrowserLocalAvailabilityProbeSubscriberCountForTesting(speechRecognition.constructor, 'en-US')).toBe(0)
const second = startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() })
const secondExpectation = expect(second).rejects.toThrow('availability check timed out')
await vi.advanceTimersByTimeAsync(BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS)
await secondExpectation
expect(getBrowserLocalAvailabilityProbeSubscriberCountForTesting(speechRecognition.constructor, 'en-US')).toBe(0)
const controller = new AbortController()
const aborted = startBrowserLocalTranscription({
language: 'en-US',
signal: controller.signal,
callbacks: browserLocalCallbacks()
})
const abortedExpectation = expect(aborted).rejects.toBeDefined()
controller.abort()
await abortedExpectation
expect(getBrowserLocalAvailabilityProbeSubscriberCountForTesting(speechRecognition.constructor, 'en-US')).toBe(0)
expect(available).toHaveBeenCalledOnce()
resolveAvailable('available')
await Promise.resolve()
await Promise.resolve()
expect(onConstruct).not.toHaveBeenCalled()
expect(firstCallbacks.onConnected).not.toHaveBeenCalled()
})
it('rejects Android browser-local startup without calling a partial native available API', async () => {
const available = vi.fn(() => Promise.resolve('available'))
installBrowserLocalSpeechRecognition(available)
vi.stubGlobal('navigator', {
userAgent: 'Mozilla/5.0 (Linux; Android 15; WebView)',
userAgentData: { platform: 'Android', mobile: true },
language: 'en-US'
})
await expect(startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() }))
.rejects.toThrow('not supported by this browser')
expect(available).not.toHaveBeenCalled()
})
})
describe('OpenAI realtime transcription', () => { describe('OpenAI realtime transcription', () => {
afterEach(() => vi.unstubAllGlobals()) afterEach(() => vi.unstubAllGlobals())
+120 -33
View File
@@ -1,4 +1,11 @@
import { DEEPGRAM_TRANSCRIPTION_MODEL } from '@hapi/protocol/voice' import { DEEPGRAM_TRANSCRIPTION_MODEL } from '@hapi/protocol/voice'
import {
getBrowserLocalSpeechSupport,
type LocalSpeechRecognition,
type LocalSpeechRecognitionAvailability,
type LocalSpeechRecognitionConstructor,
type LocalSpeechRecognitionEvent
} from './browserLocalSpeech'
export interface RealtimeTranscriptionCallbacks { export interface RealtimeTranscriptionCallbacks {
onConnected: () => void onConnected: () => void
@@ -330,43 +337,120 @@ export async function startDeepgramRealtimeTranscription(options: {
} }
} }
interface LocalSpeechRecognitionResult { export const BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS = 10_000
readonly isFinal: boolean
readonly 0: { readonly transcript: string } interface BrowserLocalAvailabilityProbe {
readonly available: LocalSpeechRecognitionAvailability
readonly constructor: LocalSpeechRecognitionConstructor
readonly language: string
readonly subscribers: Set<BrowserLocalAvailabilitySubscriber>
} }
interface LocalSpeechRecognitionEvent extends Event { interface BrowserLocalAvailabilitySubscriber {
readonly results: { readonly length: number; readonly [index: number]: LocalSpeechRecognitionResult } resolve: (status: string) => void
reject: (error: unknown) => void
} }
interface LocalSpeechRecognition extends EventTarget { const browserLocalAvailabilityProbes = new WeakMap<LocalSpeechRecognitionConstructor, Map<string, BrowserLocalAvailabilityProbe>>()
continuous: boolean
interimResults: boolean function browserLocalProbeMap(constructor: LocalSpeechRecognitionConstructor): Map<string, BrowserLocalAvailabilityProbe> {
lang: string let probes = browserLocalAvailabilityProbes.get(constructor)
processLocally: boolean if (!probes) {
onresult: ((event: LocalSpeechRecognitionEvent) => void) | null probes = new Map()
onerror: ((event: Event & { error?: string }) => void) | null browserLocalAvailabilityProbes.set(constructor, probes)
onend: (() => void) | null }
start: () => void return probes
stop: () => void
abort: () => void
} }
interface LocalSpeechRecognitionConstructor { function getBrowserLocalAvailabilityProbe(options: {
new(): LocalSpeechRecognition available: LocalSpeechRecognitionAvailability
prototype: LocalSpeechRecognition constructor: LocalSpeechRecognitionConstructor
available: (options: { langs: string[]; processLocally: true }) => Promise<string> language: string
}): BrowserLocalAvailabilityProbe {
const probes = browserLocalProbeMap(options.constructor)
const existing = probes.get(options.language)
if (existing) return existing
const probe: BrowserLocalAvailabilityProbe = {
...options,
subscribers: new Set()
}
probes.set(options.language, probe)
queueMicrotask(() => {
if (probe.subscribers.size === 0) {
probes.delete(options.language)
return
}
Promise.resolve()
.then(() => probe.available.call(probe.constructor, { langs: [probe.language], processLocally: true }))
.then(
(status) => settleBrowserLocalAvailabilityProbe(probes, probe, (subscriber) => subscriber.resolve(status)),
(error) => settleBrowserLocalAvailabilityProbe(probes, probe, (subscriber) => subscriber.reject(error))
)
})
return probe
} }
function localSpeechRecognitionConstructor(): LocalSpeechRecognitionConstructor | null { function settleBrowserLocalAvailabilityProbe(
const constructor = (globalThis as typeof globalThis & { probes: Map<string, BrowserLocalAvailabilityProbe>,
SpeechRecognition?: LocalSpeechRecognitionConstructor probe: BrowserLocalAvailabilityProbe,
}).SpeechRecognition notify: (subscriber: BrowserLocalAvailabilitySubscriber) => void
return constructor ): void {
&& typeof constructor.available === 'function' if (probes.get(probe.language) !== probe) return
&& 'processLocally' in constructor.prototype probes.delete(probe.language)
? constructor const subscribers = Array.from(probe.subscribers)
: null probe.subscribers.clear()
subscribers.forEach(notify)
}
export function getBrowserLocalAvailabilityProbeSubscriberCountForTesting(
constructor: object,
language: string
): number {
return browserLocalAvailabilityProbes
.get(constructor as LocalSpeechRecognitionConstructor)
?.get(language)
?.subscribers.size ?? 0
}
function abortError(signal: AbortSignal): unknown {
return signal.reason ?? new Error('On-device speech recognition availability check was aborted')
}
async function checkBrowserLocalSpeechAvailability(options: {
available: LocalSpeechRecognitionAvailability
constructor: LocalSpeechRecognitionConstructor
language: string
signal?: AbortSignal
}): Promise<string> {
options.signal?.throwIfAborted()
const probe = getBrowserLocalAvailabilityProbe(options)
return await new Promise<string>((resolve, reject) => {
let settled = false
const subscriber: BrowserLocalAvailabilitySubscriber = { resolve, reject }
const detach = () => probe.subscribers.delete(subscriber)
const finish = (callback: () => void) => {
if (settled) return
settled = true
clearTimeout(timeout)
options.signal?.removeEventListener('abort', onAbort)
detach()
callback()
}
const timeout = setTimeout(() => {
finish(() => reject(new Error('On-device speech recognition availability check timed out')))
}, BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS)
const onAbort = () => finish(() => reject(abortError(options.signal!)))
if (options.signal?.aborted) {
onAbort()
return
}
probe.subscribers.add(subscriber)
options.signal?.addEventListener('abort', onAbort, { once: true })
subscriber.resolve = (status) => finish(() => resolve(status))
subscriber.reject = (error) => finish(() => reject(error))
})
} }
export async function startBrowserLocalTranscription(options: { export async function startBrowserLocalTranscription(options: {
@@ -375,15 +459,18 @@ export async function startBrowserLocalTranscription(options: {
callbacks: RealtimeTranscriptionCallbacks callbacks: RealtimeTranscriptionCallbacks
}): Promise<RealtimeTranscriptionSession> { }): Promise<RealtimeTranscriptionSession> {
options.signal?.throwIfAborted() options.signal?.throwIfAborted()
const constructor = localSpeechRecognitionConstructor() const support = getBrowserLocalSpeechSupport()
if (!constructor) throw new Error('On-device speech recognition is not supported by this browser') if (!support) throw new Error('On-device speech recognition is not supported by this browser')
const language = options.language || navigator.language const language = options.language || navigator.language
if (await constructor.available({ langs: [language], processLocally: true }) !== 'available') { // `available()` is deferred to a microtask so an abort between start and
// native invocation detaches its only consumer before touching the API.
options.signal?.throwIfAborted()
if (await checkBrowserLocalSpeechAvailability({ ...support, language, signal: options.signal }) !== 'available') {
throw new Error(`On-device speech recognition is not installed for ${language}`) throw new Error(`On-device speech recognition is not installed for ${language}`)
} }
options.signal?.throwIfAborted() options.signal?.throwIfAborted()
const recognition = new constructor() const recognition = new support.constructor()
recognition.continuous = true recognition.continuous = true
recognition.interimResults = true recognition.interimResults = true
recognition.lang = language recognition.lang = language
+5
View File
@@ -62,6 +62,11 @@ describe('useDictation', () => {
}) })
it('shows on-device partial text and inserts only the final transcript', async () => { it('shows on-device partial text and inserts only the final transcript', async () => {
vi.stubGlobal('navigator', {
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
userAgentData: { platform: 'macOS', mobile: false },
language: 'en-US'
})
let recognition: MockSpeechRecognition | null = null let recognition: MockSpeechRecognition | null = null
class MockSpeechRecognition { class MockSpeechRecognition {
static async available() { return 'available' } static async available() { return 'available' }
@@ -0,0 +1,51 @@
import { renderHook, waitFor } from '@testing-library/react'
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { ApiClient } from '@/api/client'
import { useVoiceInputPreferences } from './useVoiceInputPreferences'
function installPartialSpeechRecognition(available = vi.fn(() => Promise.resolve('available'))) {
class MockSpeechRecognition {
static available = available
processLocally = false
}
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
return available
}
describe('useVoiceInputPreferences', () => {
afterEach(() => vi.unstubAllGlobals())
it('discovers browser-local support from its shape without probing available on mount', async () => {
const available = installPartialSpeechRecognition()
vi.stubGlobal('navigator', {
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
userAgentData: { platform: 'macOS', mobile: false },
language: 'en-US'
})
const api = {
fetchTranscriptionProviders: vi.fn(async () => ({ providers: [] }))
}
const { result } = renderHook(() => useVoiceInputPreferences(api as unknown as ApiClient))
await waitFor(() => expect(result.current.provider).toBe('browser-local'))
expect(available).not.toHaveBeenCalled()
})
it('does not expose or probe partial browser-local speech APIs on Android', async () => {
const available = installPartialSpeechRecognition()
vi.stubGlobal('navigator', { userAgent: 'Mozilla/5.0 (Linux; Android 15; WebView)', language: 'en-US' })
const api = {
fetchTranscriptionProviders: vi.fn(async () => ({
providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }]
}))
}
const { result } = renderHook(() => useVoiceInputPreferences(api as unknown as ApiClient))
await waitFor(() => expect(result.current.provider).toBe('openai'))
expect(result.current.providers).toHaveLength(1)
expect(available).not.toHaveBeenCalled()
})
})
+14 -36
View File
@@ -7,33 +7,17 @@ import {
type TranscriptionProviderInfo, type TranscriptionProviderInfo,
type VoiceMode type VoiceMode
} from '@hapi/protocol/voice' } from '@hapi/protocol/voice'
import { hasBrowserLocalSpeechSupport } from './browserLocalSpeech'
const VOICE_MODE_KEY = 'hapi-voice-mode' const VOICE_MODE_KEY = 'hapi-voice-mode'
const TRANSCRIPTION_PROVIDER_KEY = 'hapi-transcription-provider' const TRANSCRIPTION_PROVIDER_KEY = 'hapi-transcription-provider'
const TRANSCRIPTION_MODE_KEY = 'hapi-transcription-mode' const TRANSCRIPTION_MODE_KEY = 'hapi-transcription-mode'
const CHANGE_EVENT = 'hapi-voice-input-change' const CHANGE_EVENT = 'hapi-voice-input-change'
export const VOICE_LANGUAGE_CHANGE_EVENT = 'hapi-voice-language-change'
function notifyChange(): void { function notifyChange(): void {
window.dispatchEvent(new Event(CHANGE_EVENT)) window.dispatchEvent(new Event(CHANGE_EVENT))
} }
async function browserLocalTranscriptionSupported(): Promise<boolean> {
const constructor = (globalThis as typeof globalThis & {
SpeechRecognition?: {
prototype: object
available?: (options: { langs: string[]; processLocally: true }) => Promise<string>
}
}).SpeechRecognition
if (!constructor || typeof constructor.available !== 'function' || !('processLocally' in constructor.prototype)) return false
const language = localStorage.getItem('hapi-voice-lang') || navigator.language
try {
return await constructor.available({ langs: [language], processLocally: true }) === 'available'
} catch {
return false
}
}
function readVoiceMode(): VoiceMode { function readVoiceMode(): VoiceMode {
return localStorage.getItem(VOICE_MODE_KEY) === 'dictation' ? 'dictation' : 'assistant' return localStorage.getItem(VOICE_MODE_KEY) === 'dictation' ? 'dictation' : 'assistant'
} }
@@ -64,27 +48,21 @@ export function useVoiceInputPreferences(api: ApiClient | null) {
useEffect(() => { useEffect(() => {
if (!api) return if (!api) return
let cancelled = false let cancelled = false
let request = 0 const browserLocal = hasBrowserLocalSpeechSupport()
const refreshProviders = () => { api.fetchTranscriptionProviders().then(({ providers: configured }) => {
const current = ++request if (cancelled) return
Promise.all([api.fetchTranscriptionProviders(), browserLocalTranscriptionSupported()]).then(([{ providers: configured }, browserLocal]) => { const available = browserLocal
if (cancelled || current !== request) return ? [...configured, BROWSER_LOCAL_TRANSCRIPTION_PROVIDER]
const available = browserLocal : configured
? [...configured, BROWSER_LOCAL_TRANSCRIPTION_PROVIDER] setProviders(available)
: configured const selectedProvider = resolveProvider(available, localStorage.getItem(TRANSCRIPTION_PROVIDER_KEY))
setProviders(available) setProviderState(selectedProvider)
const selectedProvider = resolveProvider(available, localStorage.getItem(TRANSCRIPTION_PROVIDER_KEY)) setTranscriptionModeState(resolveMode(available, selectedProvider, localStorage.getItem(TRANSCRIPTION_MODE_KEY)))
setProviderState(selectedProvider) }).catch(() => {
setTranscriptionModeState(resolveMode(available, selectedProvider, localStorage.getItem(TRANSCRIPTION_MODE_KEY))) if (!cancelled) setProviders([])
}).catch(() => { })
if (!cancelled && current === request) setProviders([])
})
}
refreshProviders()
window.addEventListener(VOICE_LANGUAGE_CHANGE_EVENT, refreshProviders)
return () => { return () => {
cancelled = true cancelled = true
window.removeEventListener(VOICE_LANGUAGE_CHANGE_EVENT, refreshProviders)
} }
}, [api]) }, [api])
@@ -32,6 +32,11 @@ describe('useVoiceSettings', () => {
beforeEach(() => { beforeEach(() => {
vi.clearAllMocks() vi.clearAllMocks()
localStorage.clear() localStorage.clear()
vi.stubGlobal('navigator', {
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
userAgentData: { platform: 'macOS', mobile: false },
language: 'en-US'
})
fetchVoiceBackend.mockResolvedValue({ backend: 'elevenlabs', backends: ['elevenlabs'] }) fetchVoiceBackend.mockResolvedValue({ backend: 'elevenlabs', backends: ['elevenlabs'] })
fetchVoices.mockResolvedValue([]) fetchVoices.mockResolvedValue([])
class MockAudio { class MockAudio {
@@ -85,8 +90,9 @@ describe('useVoiceSettings', () => {
}) })
it('uses realtime for the realtime-only browser provider', async () => { it('uses realtime for the realtime-only browser provider', async () => {
const available = vi.fn(() => Promise.resolve('available'))
class MockSpeechRecognition { class MockSpeechRecognition {
static available() { return Promise.resolve('available') } static available = available
processLocally = false processLocally = false
} }
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false }) Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
@@ -97,46 +103,28 @@ describe('useVoiceSettings', () => {
await waitFor(() => expect(result.current.provider).toBe('browser-local')) await waitFor(() => expect(result.current.provider).toBe('browser-local'))
expect(result.current.transcriptionMode).toBe('realtime') expect(result.current.transcriptionMode).toBe('realtime')
expect(available).not.toHaveBeenCalled()
}) })
it('does not expose browser dictation without the selected language pack', async () => { it('does not probe browser-local speech availability when the language changes', async () => {
fetchTranscriptionProviders.mockResolvedValueOnce({ fetchTranscriptionProviders.mockResolvedValueOnce({
providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }] providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }]
}) })
const available = vi.fn(() => Promise.resolve('available'))
class MockSpeechRecognition { class MockSpeechRecognition {
static available() { return Promise.resolve('unavailable') } static available = available
processLocally = false processLocally = false
} }
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false }) Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
vi.stubGlobal('SpeechRecognition', MockSpeechRecognition) vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
const { result } = renderHook(() => useVoiceSettings(), { wrapper: Wrapper })
await waitFor(() => expect(result.current.provider).toBe('openai'))
expect(result.current.providers).toHaveLength(1)
})
it('rechecks the on-device language pack when the language changes', async () => {
fetchTranscriptionProviders.mockResolvedValue({
providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }]
})
class MockSpeechRecognition {
static available({ langs }: { langs: string[] }) {
return Promise.resolve(langs[0] === 'en-US' ? 'available' : 'unavailable')
}
processLocally = false
}
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
localStorage.setItem('hapi-voice-lang', 'en-US')
localStorage.setItem('hapi-transcription-provider', 'browser-local') localStorage.setItem('hapi-transcription-provider', 'browser-local')
const { result } = renderHook(() => useVoiceSettings(), { wrapper: Wrapper }) const { result } = renderHook(() => useVoiceSettings(), { wrapper: Wrapper })
await waitFor(() => expect(result.current.provider).toBe('browser-local')) await waitFor(() => expect(result.current.provider).toBe('browser-local'))
act(() => result.current.setVoiceLanguage({ code: 'zh-CN', name: 'Chinese', nativeName: '中文' })) act(() => result.current.setVoiceLanguage({ code: 'zh-CN', name: 'Chinese', nativeName: '中文' }))
await waitFor(() => expect(result.current.provider).toBe('openai')) expect(result.current.provider).toBe('browser-local')
expect(available).not.toHaveBeenCalled()
act(() => result.current.setVoiceLanguage({ code: 'en-US', name: 'English', nativeName: 'English' }))
await waitFor(() => expect(result.current.provider).toBe('browser-local'))
}) })
}) })
+1 -2
View File
@@ -12,7 +12,7 @@ import {
writeStoredVoiceSelection, writeStoredVoiceSelection,
} from '@/lib/voicePickerPreferences' } from '@/lib/voicePickerPreferences'
import type { VoiceBackendType } from '@hapi/protocol/voice' import type { VoiceBackendType } from '@hapi/protocol/voice'
import { useVoiceInputPreferences, VOICE_LANGUAGE_CHANGE_EVENT } from '@/hooks/useVoiceInputPreferences' import { useVoiceInputPreferences } from '@/hooks/useVoiceInputPreferences'
export function useVoiceSettings() { export function useVoiceSettings() {
const { api } = useAppContext() const { api } = useAppContext()
@@ -83,7 +83,6 @@ export function useVoiceSettings() {
setVoiceLanguageState(language.code) setVoiceLanguageState(language.code)
if (language.code === null) localStorage.removeItem('hapi-voice-lang') if (language.code === null) localStorage.removeItem('hapi-voice-lang')
else localStorage.setItem('hapi-voice-lang', language.code) else localStorage.setItem('hapi-voice-lang', language.code)
window.dispatchEvent(new Event(VOICE_LANGUAGE_CHANGE_EVENT))
}, []) }, [])
const stopPreview = useCallback(() => { const stopPreview = useCallback(() => {