fix(web): make browser-local speech probing Android-safe (#1349)

* fix(web): guard browser-local speech probes

* test(web): cover concurrent speech probes

* docs: clarify browser-local speech probing
This commit is contained in:
KorenKrita
2026-08-04 08:19:56 +08:00
committed by GitHub
parent f10fbc7496
commit c1b32b51fe
10 changed files with 577 additions and 100 deletions
+1 -1
View File
@@ -20,7 +20,7 @@ export TRANSCRIPTION_API_KEY="..." # optional
```
Restart the hub after changing credentials. API keys are not entered or stored in the web app.
Realtime OpenAI, ElevenLabs, and Deepgram sessions receive only short-lived credentials minted by the hub. Browsers with an installed on-device `SpeechRecognition` language pack also expose **Browser on-device** as a realtime-only provider; HAPI never falls back from that option to browser-hosted recognition.
Realtime OpenAI, ElevenLabs, and Deepgram sessions receive only short-lived credentials minted by the hub. Eligible desktop browsers with the on-device `SpeechRecognition` API expose **Browser on-device** as a realtime-only provider. HAPI checks the selected language pack when dictation starts and never falls back from that option to browser-hosted recognition. Mobile and unknown browser environments fail closed because this API is experimental and some Android WebViews expose unsafe partial implementations.
## Overview
+76
View File
@@ -0,0 +1,76 @@
import { describe, expect, it, vi } from 'vitest'
import { getBrowserLocalSpeechSupport } from './browserLocalSpeech'
const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36'
function speechRecognitionShape(availableDescriptor?: PropertyDescriptor) {
class MockSpeechRecognition {
processLocally = false
}
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
if (availableDescriptor) Object.defineProperty(MockSpeechRecognition, 'available', availableDescriptor)
return MockSpeechRecognition
}
describe('browser-local speech capability detection', () => {
it('uses the static available data property as a no-probe capability requirement', () => {
const available = vi.fn(() => Promise.resolve('available'))
const SpeechRecognition = speechRecognitionShape({ value: available })
const support = getBrowserLocalSpeechSupport({
userAgent: DESKTOP_USER_AGENT,
userAgentData: { platform: 'macOS', mobile: false },
speechRecognition: SpeechRecognition
})
expect(support?.constructor).toBe(SpeechRecognition)
expect(support?.available).toBe(available)
expect(available).not.toHaveBeenCalled()
})
it('rejects missing, non-function, and getter available properties without reading a getter', () => {
const getter = vi.fn(() => vi.fn(() => Promise.resolve('available')))
const cases = [
speechRecognitionShape(),
speechRecognitionShape({ value: 'available' }),
speechRecognitionShape({ get: getter })
]
for (const SpeechRecognition of cases) {
expect(getBrowserLocalSpeechSupport({
userAgent: DESKTOP_USER_AGENT,
userAgentData: { platform: 'macOS', mobile: false },
speechRecognition: SpeechRecognition
})).toBeNull()
}
expect(getter).not.toHaveBeenCalled()
})
it('fails closed when a desktop-looking UA has mobile Android UA-CH signals', () => {
const available = vi.fn(() => Promise.resolve('available'))
const SpeechRecognition = speechRecognitionShape({ value: available })
expect(getBrowserLocalSpeechSupport({
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
userAgentData: { platform: 'Android', mobile: true },
speechRecognition: SpeechRecognition
})).toBeNull()
expect(available).not.toHaveBeenCalled()
})
it('rejects desktop-looking UAs without trusted desktop UA-CH', () => {
const available = vi.fn(() => Promise.resolve('available'))
const SpeechRecognition = speechRecognitionShape({ value: available })
expect(getBrowserLocalSpeechSupport({
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
speechRecognition: SpeechRecognition
})).toBeNull()
expect(getBrowserLocalSpeechSupport({
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Version/18.0 Mobile/15E148 Safari/604.1',
speechRecognition: SpeechRecognition
})).toBeNull()
expect(available).not.toHaveBeenCalled()
})
})
+107
View File
@@ -0,0 +1,107 @@
export interface LocalSpeechRecognitionResult {
readonly isFinal: boolean
readonly 0: { readonly transcript: string }
}
export interface LocalSpeechRecognitionEvent extends Event {
readonly results: { readonly length: number; readonly [index: number]: LocalSpeechRecognitionResult }
}
export interface LocalSpeechRecognition extends EventTarget {
continuous: boolean
interimResults: boolean
lang: string
processLocally: boolean
onresult: ((event: LocalSpeechRecognitionEvent) => void) | null
onerror: ((event: Event & { error?: string }) => void) | null
onend: (() => void) | null
start: () => void
stop: () => void
abort: () => void
}
export interface LocalSpeechRecognitionConstructor {
new(): LocalSpeechRecognition
prototype: LocalSpeechRecognition
}
export type LocalSpeechRecognitionAvailability = (
options: { langs: string[]; processLocally: true }
) => Promise<string> | string
export interface BrowserLocalSpeechEnvironment {
userAgent?: string
userAgentData?: { platform?: string; mobile?: boolean }
speechRecognition?: unknown
}
export interface BrowserLocalSpeechSupport {
constructor: LocalSpeechRecognitionConstructor
available: LocalSpeechRecognitionAvailability
}
function currentUserAgent(): string {
return typeof navigator === 'undefined' ? '' : navigator.userAgent
}
function currentSpeechRecognition(): unknown {
return (globalThis as typeof globalThis & {
SpeechRecognition?: unknown
}).SpeechRecognition
}
function currentUserAgentData(): BrowserLocalSpeechEnvironment['userAgentData'] {
return (typeof navigator === 'undefined'
? undefined
: (navigator as Navigator & { userAgentData?: BrowserLocalSpeechEnvironment['userAgentData'] }).userAgentData)
}
const SAFE_DESKTOP_PLATFORMS = new Set(['Windows', 'macOS', 'Linux', 'Chrome OS'])
/**
* The experimental on-device speech API is eligible only with explicit,
* trustworthy User-Agent Client Hints that identify a desktop platform. Android
* WebViews can expose a partial shape whose native `available()` call crashes
* the renderer, so missing, unknown, and mobile environments fail closed.
*/
export function isConfirmedDesktopSpeechEnvironment(
_userAgent: string,
userAgentData?: BrowserLocalSpeechEnvironment['userAgentData']
): boolean {
return userAgentData?.mobile === false
&& typeof userAgentData.platform === 'string'
&& SAFE_DESKTOP_PLATFORMS.has(userAgentData.platform)
}
function staticAvailabilityMethod(candidate: Function): LocalSpeechRecognitionAvailability | null {
for (let target: object | null = candidate; target && target !== Function.prototype; target = Object.getPrototypeOf(target)) {
const descriptor = Object.getOwnPropertyDescriptor(target, 'available')
if (descriptor) return typeof descriptor.value === 'function'
? descriptor.value as LocalSpeechRecognitionAvailability
: null
}
return null
}
/**
* Checks only the browser API shape. It intentionally does not instantiate
* recognition or call the experimental `SpeechRecognition.available()` method.
*/
export function getBrowserLocalSpeechSupport(
environment: BrowserLocalSpeechEnvironment = {}
): BrowserLocalSpeechSupport | null {
const userAgent = environment.userAgent ?? currentUserAgent()
const userAgentData = environment.userAgentData ?? currentUserAgentData()
if (!isConfirmedDesktopSpeechEnvironment(userAgent, userAgentData)) return null
const candidate = environment.speechRecognition ?? currentSpeechRecognition()
if (typeof candidate !== 'function') return null
const constructor = candidate as LocalSpeechRecognitionConstructor
if (!constructor.prototype || !('processLocally' in constructor.prototype)) return null
const available = staticAvailabilityMethod(candidate)
return available ? { constructor, available } : null
}
export function hasBrowserLocalSpeechSupport(): boolean {
return getBrowserLocalSpeechSupport() !== null
}
+187 -1
View File
@@ -1,5 +1,191 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { startDeepgramRealtimeTranscription, startOpenAIRealtimeTranscription } from './realtimeTranscription'
import {
BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS,
getBrowserLocalAvailabilityProbeSubscriberCountForTesting,
startBrowserLocalTranscription,
startDeepgramRealtimeTranscription,
startOpenAIRealtimeTranscription
} from './realtimeTranscription'
function browserLocalCallbacks() {
return {
onConnected: vi.fn(),
onPartial: vi.fn(),
onFinal: vi.fn(),
onError: vi.fn()
}
}
function installBrowserLocalSpeechRecognition(available: () => Promise<string> | string, onConstruct = vi.fn()) {
class MockSpeechRecognition extends EventTarget {
static available = available
continuous = false
interimResults = false
lang = ''
processLocally = false
onresult: ((event: Event) => void) | null = null
onerror: ((event: Event) => void) | null = null
onend: (() => void) | null = null
start = vi.fn()
stop = vi.fn()
abort = vi.fn()
constructor() {
super()
onConstruct()
}
}
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false, writable: true })
vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
vi.stubGlobal('navigator', {
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
userAgentData: { platform: 'macOS', mobile: false },
language: 'en-US'
})
return { available, onConstruct, constructor: MockSpeechRecognition }
}
describe('browser-local realtime transcription', () => {
afterEach(() => {
vi.useRealTimers()
vi.unstubAllGlobals()
})
it('calls available only after an explicit browser-local dictation start', async () => {
const available = vi.fn(() => Promise.resolve('available'))
installBrowserLocalSpeechRecognition(available)
const callbacks = browserLocalCallbacks()
const session = await startBrowserLocalTranscription({ language: 'en-US', callbacks })
expect(available).toHaveBeenCalledOnce()
expect(available).toHaveBeenCalledWith({ langs: ['en-US'], processLocally: true })
expect(callbacks.onConnected).toHaveBeenCalledOnce()
session.cancel()
})
it('shares one successful native probe across concurrent browser-local starts', async () => {
let resolveAvailable!: (status: string) => void
const available = vi.fn(() => new Promise<string>((resolve) => { resolveAvailable = resolve }))
const onConstruct = vi.fn()
installBrowserLocalSpeechRecognition(available, onConstruct)
const firstCallbacks = browserLocalCallbacks()
const secondCallbacks = browserLocalCallbacks()
const first = startBrowserLocalTranscription({ language: 'en-US', callbacks: firstCallbacks })
const second = startBrowserLocalTranscription({ language: 'en-US', callbacks: secondCallbacks })
await vi.waitFor(() => expect(available).toHaveBeenCalledOnce())
resolveAvailable('available')
const sessions = await Promise.all([first, second])
expect(onConstruct).toHaveBeenCalledTimes(2)
expect(firstCallbacks.onConnected).toHaveBeenCalledOnce()
expect(secondCallbacks.onConnected).toHaveBeenCalledOnce()
sessions.forEach((session) => session.cancel())
})
it('surfaces a synchronous available failure', async () => {
const available = vi.fn(() => { throw new Error('native failure') })
installBrowserLocalSpeechRecognition(available)
await expect(startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() }))
.rejects.toThrow('native failure')
expect(available).toHaveBeenCalledOnce()
})
it('surfaces a rejected available probe', async () => {
const available = vi.fn(() => Promise.reject(new Error('probe rejected')))
installBrowserLocalSpeechRecognition(available)
await expect(startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() }))
.rejects.toThrow('probe rejected')
expect(available).toHaveBeenCalledOnce()
})
it('times out a stalled available probe', async () => {
vi.useFakeTimers()
const available = vi.fn(() => new Promise<string>(() => {}))
installBrowserLocalSpeechRecognition(available)
const starting = startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() })
const expectation = expect(starting).rejects.toThrow('availability check timed out')
await vi.advanceTimersByTimeAsync(BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS)
await expectation
expect(available).toHaveBeenCalledOnce()
})
it('does not invoke available when its start signal aborts before the native microtask', async () => {
const available = vi.fn(() => Promise.resolve('available'))
installBrowserLocalSpeechRecognition(available)
const controller = new AbortController()
const starting = startBrowserLocalTranscription({
language: 'en-US',
signal: controller.signal,
callbacks: browserLocalCallbacks()
})
const expectation = expect(starting).rejects.toBeDefined()
controller.abort()
await expectation
await Promise.resolve()
expect(available).not.toHaveBeenCalled()
})
it('shares a stalled native probe across timeout retries and ignores its late result', async () => {
vi.useFakeTimers()
let resolveAvailable!: (status: string) => void
const available = vi.fn(() => new Promise<string>((resolve) => { resolveAvailable = resolve }))
const onConstruct = vi.fn()
const speechRecognition = installBrowserLocalSpeechRecognition(available, onConstruct)
const firstCallbacks = browserLocalCallbacks()
const first = startBrowserLocalTranscription({ language: 'en-US', callbacks: firstCallbacks })
const firstExpectation = expect(first).rejects.toThrow('availability check timed out')
await vi.advanceTimersByTimeAsync(BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS)
await firstExpectation
expect(getBrowserLocalAvailabilityProbeSubscriberCountForTesting(speechRecognition.constructor, 'en-US')).toBe(0)
const second = startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() })
const secondExpectation = expect(second).rejects.toThrow('availability check timed out')
await vi.advanceTimersByTimeAsync(BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS)
await secondExpectation
expect(getBrowserLocalAvailabilityProbeSubscriberCountForTesting(speechRecognition.constructor, 'en-US')).toBe(0)
const controller = new AbortController()
const aborted = startBrowserLocalTranscription({
language: 'en-US',
signal: controller.signal,
callbacks: browserLocalCallbacks()
})
const abortedExpectation = expect(aborted).rejects.toBeDefined()
controller.abort()
await abortedExpectation
expect(getBrowserLocalAvailabilityProbeSubscriberCountForTesting(speechRecognition.constructor, 'en-US')).toBe(0)
expect(available).toHaveBeenCalledOnce()
resolveAvailable('available')
await Promise.resolve()
await Promise.resolve()
expect(onConstruct).not.toHaveBeenCalled()
expect(firstCallbacks.onConnected).not.toHaveBeenCalled()
})
it('rejects Android browser-local startup without calling a partial native available API', async () => {
const available = vi.fn(() => Promise.resolve('available'))
installBrowserLocalSpeechRecognition(available)
vi.stubGlobal('navigator', {
userAgent: 'Mozilla/5.0 (Linux; Android 15; WebView)',
userAgentData: { platform: 'Android', mobile: true },
language: 'en-US'
})
await expect(startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() }))
.rejects.toThrow('not supported by this browser')
expect(available).not.toHaveBeenCalled()
})
})
describe('OpenAI realtime transcription', () => {
afterEach(() => vi.unstubAllGlobals())
+120 -33
View File
@@ -1,4 +1,11 @@
import { DEEPGRAM_TRANSCRIPTION_MODEL } from '@hapi/protocol/voice'
import {
getBrowserLocalSpeechSupport,
type LocalSpeechRecognition,
type LocalSpeechRecognitionAvailability,
type LocalSpeechRecognitionConstructor,
type LocalSpeechRecognitionEvent
} from './browserLocalSpeech'
export interface RealtimeTranscriptionCallbacks {
onConnected: () => void
@@ -330,43 +337,120 @@ export async function startDeepgramRealtimeTranscription(options: {
}
}
interface LocalSpeechRecognitionResult {
readonly isFinal: boolean
readonly 0: { readonly transcript: string }
export const BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS = 10_000
interface BrowserLocalAvailabilityProbe {
readonly available: LocalSpeechRecognitionAvailability
readonly constructor: LocalSpeechRecognitionConstructor
readonly language: string
readonly subscribers: Set<BrowserLocalAvailabilitySubscriber>
}
interface LocalSpeechRecognitionEvent extends Event {
readonly results: { readonly length: number; readonly [index: number]: LocalSpeechRecognitionResult }
interface BrowserLocalAvailabilitySubscriber {
resolve: (status: string) => void
reject: (error: unknown) => void
}
interface LocalSpeechRecognition extends EventTarget {
continuous: boolean
interimResults: boolean
lang: string
processLocally: boolean
onresult: ((event: LocalSpeechRecognitionEvent) => void) | null
onerror: ((event: Event & { error?: string }) => void) | null
onend: (() => void) | null
start: () => void
stop: () => void
abort: () => void
const browserLocalAvailabilityProbes = new WeakMap<LocalSpeechRecognitionConstructor, Map<string, BrowserLocalAvailabilityProbe>>()
function browserLocalProbeMap(constructor: LocalSpeechRecognitionConstructor): Map<string, BrowserLocalAvailabilityProbe> {
let probes = browserLocalAvailabilityProbes.get(constructor)
if (!probes) {
probes = new Map()
browserLocalAvailabilityProbes.set(constructor, probes)
}
return probes
}
interface LocalSpeechRecognitionConstructor {
new(): LocalSpeechRecognition
prototype: LocalSpeechRecognition
available: (options: { langs: string[]; processLocally: true }) => Promise<string>
function getBrowserLocalAvailabilityProbe(options: {
available: LocalSpeechRecognitionAvailability
constructor: LocalSpeechRecognitionConstructor
language: string
}): BrowserLocalAvailabilityProbe {
const probes = browserLocalProbeMap(options.constructor)
const existing = probes.get(options.language)
if (existing) return existing
const probe: BrowserLocalAvailabilityProbe = {
...options,
subscribers: new Set()
}
probes.set(options.language, probe)
queueMicrotask(() => {
if (probe.subscribers.size === 0) {
probes.delete(options.language)
return
}
Promise.resolve()
.then(() => probe.available.call(probe.constructor, { langs: [probe.language], processLocally: true }))
.then(
(status) => settleBrowserLocalAvailabilityProbe(probes, probe, (subscriber) => subscriber.resolve(status)),
(error) => settleBrowserLocalAvailabilityProbe(probes, probe, (subscriber) => subscriber.reject(error))
)
})
return probe
}
function localSpeechRecognitionConstructor(): LocalSpeechRecognitionConstructor | null {
const constructor = (globalThis as typeof globalThis & {
SpeechRecognition?: LocalSpeechRecognitionConstructor
}).SpeechRecognition
return constructor
&& typeof constructor.available === 'function'
&& 'processLocally' in constructor.prototype
? constructor
: null
function settleBrowserLocalAvailabilityProbe(
probes: Map<string, BrowserLocalAvailabilityProbe>,
probe: BrowserLocalAvailabilityProbe,
notify: (subscriber: BrowserLocalAvailabilitySubscriber) => void
): void {
if (probes.get(probe.language) !== probe) return
probes.delete(probe.language)
const subscribers = Array.from(probe.subscribers)
probe.subscribers.clear()
subscribers.forEach(notify)
}
export function getBrowserLocalAvailabilityProbeSubscriberCountForTesting(
constructor: object,
language: string
): number {
return browserLocalAvailabilityProbes
.get(constructor as LocalSpeechRecognitionConstructor)
?.get(language)
?.subscribers.size ?? 0
}
function abortError(signal: AbortSignal): unknown {
return signal.reason ?? new Error('On-device speech recognition availability check was aborted')
}
async function checkBrowserLocalSpeechAvailability(options: {
available: LocalSpeechRecognitionAvailability
constructor: LocalSpeechRecognitionConstructor
language: string
signal?: AbortSignal
}): Promise<string> {
options.signal?.throwIfAborted()
const probe = getBrowserLocalAvailabilityProbe(options)
return await new Promise<string>((resolve, reject) => {
let settled = false
const subscriber: BrowserLocalAvailabilitySubscriber = { resolve, reject }
const detach = () => probe.subscribers.delete(subscriber)
const finish = (callback: () => void) => {
if (settled) return
settled = true
clearTimeout(timeout)
options.signal?.removeEventListener('abort', onAbort)
detach()
callback()
}
const timeout = setTimeout(() => {
finish(() => reject(new Error('On-device speech recognition availability check timed out')))
}, BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS)
const onAbort = () => finish(() => reject(abortError(options.signal!)))
if (options.signal?.aborted) {
onAbort()
return
}
probe.subscribers.add(subscriber)
options.signal?.addEventListener('abort', onAbort, { once: true })
subscriber.resolve = (status) => finish(() => resolve(status))
subscriber.reject = (error) => finish(() => reject(error))
})
}
export async function startBrowserLocalTranscription(options: {
@@ -375,15 +459,18 @@ export async function startBrowserLocalTranscription(options: {
callbacks: RealtimeTranscriptionCallbacks
}): Promise<RealtimeTranscriptionSession> {
options.signal?.throwIfAborted()
const constructor = localSpeechRecognitionConstructor()
if (!constructor) throw new Error('On-device speech recognition is not supported by this browser')
const support = getBrowserLocalSpeechSupport()
if (!support) throw new Error('On-device speech recognition is not supported by this browser')
const language = options.language || navigator.language
if (await constructor.available({ langs: [language], processLocally: true }) !== 'available') {
// `available()` is deferred to a microtask so an abort between start and
// native invocation detaches its only consumer before touching the API.
options.signal?.throwIfAborted()
if (await checkBrowserLocalSpeechAvailability({ ...support, language, signal: options.signal }) !== 'available') {
throw new Error(`On-device speech recognition is not installed for ${language}`)
}
options.signal?.throwIfAborted()
const recognition = new constructor()
const recognition = new support.constructor()
recognition.continuous = true
recognition.interimResults = true
recognition.lang = language
+5
View File
@@ -62,6 +62,11 @@ describe('useDictation', () => {
})
it('shows on-device partial text and inserts only the final transcript', async () => {
vi.stubGlobal('navigator', {
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
userAgentData: { platform: 'macOS', mobile: false },
language: 'en-US'
})
let recognition: MockSpeechRecognition | null = null
class MockSpeechRecognition {
static async available() { return 'available' }
@@ -0,0 +1,51 @@
import { renderHook, waitFor } from '@testing-library/react'
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { ApiClient } from '@/api/client'
import { useVoiceInputPreferences } from './useVoiceInputPreferences'
function installPartialSpeechRecognition(available = vi.fn(() => Promise.resolve('available'))) {
class MockSpeechRecognition {
static available = available
processLocally = false
}
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
return available
}
describe('useVoiceInputPreferences', () => {
afterEach(() => vi.unstubAllGlobals())
it('discovers browser-local support from its shape without probing available on mount', async () => {
const available = installPartialSpeechRecognition()
vi.stubGlobal('navigator', {
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
userAgentData: { platform: 'macOS', mobile: false },
language: 'en-US'
})
const api = {
fetchTranscriptionProviders: vi.fn(async () => ({ providers: [] }))
}
const { result } = renderHook(() => useVoiceInputPreferences(api as unknown as ApiClient))
await waitFor(() => expect(result.current.provider).toBe('browser-local'))
expect(available).not.toHaveBeenCalled()
})
it('does not expose or probe partial browser-local speech APIs on Android', async () => {
const available = installPartialSpeechRecognition()
vi.stubGlobal('navigator', { userAgent: 'Mozilla/5.0 (Linux; Android 15; WebView)', language: 'en-US' })
const api = {
fetchTranscriptionProviders: vi.fn(async () => ({
providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }]
}))
}
const { result } = renderHook(() => useVoiceInputPreferences(api as unknown as ApiClient))
await waitFor(() => expect(result.current.provider).toBe('openai'))
expect(result.current.providers).toHaveLength(1)
expect(available).not.toHaveBeenCalled()
})
})
+5 -27
View File
@@ -7,33 +7,17 @@ import {
type TranscriptionProviderInfo,
type VoiceMode
} from '@hapi/protocol/voice'
import { hasBrowserLocalSpeechSupport } from './browserLocalSpeech'
const VOICE_MODE_KEY = 'hapi-voice-mode'
const TRANSCRIPTION_PROVIDER_KEY = 'hapi-transcription-provider'
const TRANSCRIPTION_MODE_KEY = 'hapi-transcription-mode'
const CHANGE_EVENT = 'hapi-voice-input-change'
export const VOICE_LANGUAGE_CHANGE_EVENT = 'hapi-voice-language-change'
function notifyChange(): void {
window.dispatchEvent(new Event(CHANGE_EVENT))
}
async function browserLocalTranscriptionSupported(): Promise<boolean> {
const constructor = (globalThis as typeof globalThis & {
SpeechRecognition?: {
prototype: object
available?: (options: { langs: string[]; processLocally: true }) => Promise<string>
}
}).SpeechRecognition
if (!constructor || typeof constructor.available !== 'function' || !('processLocally' in constructor.prototype)) return false
const language = localStorage.getItem('hapi-voice-lang') || navigator.language
try {
return await constructor.available({ langs: [language], processLocally: true }) === 'available'
} catch {
return false
}
}
function readVoiceMode(): VoiceMode {
return localStorage.getItem(VOICE_MODE_KEY) === 'dictation' ? 'dictation' : 'assistant'
}
@@ -64,11 +48,9 @@ export function useVoiceInputPreferences(api: ApiClient | null) {
useEffect(() => {
if (!api) return
let cancelled = false
let request = 0
const refreshProviders = () => {
const current = ++request
Promise.all([api.fetchTranscriptionProviders(), browserLocalTranscriptionSupported()]).then(([{ providers: configured }, browserLocal]) => {
if (cancelled || current !== request) return
const browserLocal = hasBrowserLocalSpeechSupport()
api.fetchTranscriptionProviders().then(({ providers: configured }) => {
if (cancelled) return
const available = browserLocal
? [...configured, BROWSER_LOCAL_TRANSCRIPTION_PROVIDER]
: configured
@@ -77,14 +59,10 @@ export function useVoiceInputPreferences(api: ApiClient | null) {
setProviderState(selectedProvider)
setTranscriptionModeState(resolveMode(available, selectedProvider, localStorage.getItem(TRANSCRIPTION_MODE_KEY)))
}).catch(() => {
if (!cancelled && current === request) setProviders([])
if (!cancelled) setProviders([])
})
}
refreshProviders()
window.addEventListener(VOICE_LANGUAGE_CHANGE_EVENT, refreshProviders)
return () => {
cancelled = true
window.removeEventListener(VOICE_LANGUAGE_CHANGE_EVENT, refreshProviders)
}
}, [api])
@@ -32,6 +32,11 @@ describe('useVoiceSettings', () => {
beforeEach(() => {
vi.clearAllMocks()
localStorage.clear()
vi.stubGlobal('navigator', {
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
userAgentData: { platform: 'macOS', mobile: false },
language: 'en-US'
})
fetchVoiceBackend.mockResolvedValue({ backend: 'elevenlabs', backends: ['elevenlabs'] })
fetchVoices.mockResolvedValue([])
class MockAudio {
@@ -85,8 +90,9 @@ describe('useVoiceSettings', () => {
})
it('uses realtime for the realtime-only browser provider', async () => {
const available = vi.fn(() => Promise.resolve('available'))
class MockSpeechRecognition {
static available() { return Promise.resolve('available') }
static available = available
processLocally = false
}
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
@@ -97,46 +103,28 @@ describe('useVoiceSettings', () => {
await waitFor(() => expect(result.current.provider).toBe('browser-local'))
expect(result.current.transcriptionMode).toBe('realtime')
expect(available).not.toHaveBeenCalled()
})
it('does not expose browser dictation without the selected language pack', async () => {
it('does not probe browser-local speech availability when the language changes', async () => {
fetchTranscriptionProviders.mockResolvedValueOnce({
providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }]
})
const available = vi.fn(() => Promise.resolve('available'))
class MockSpeechRecognition {
static available() { return Promise.resolve('unavailable') }
static available = available
processLocally = false
}
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
const { result } = renderHook(() => useVoiceSettings(), { wrapper: Wrapper })
await waitFor(() => expect(result.current.provider).toBe('openai'))
expect(result.current.providers).toHaveLength(1)
})
it('rechecks the on-device language pack when the language changes', async () => {
fetchTranscriptionProviders.mockResolvedValue({
providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }]
})
class MockSpeechRecognition {
static available({ langs }: { langs: string[] }) {
return Promise.resolve(langs[0] === 'en-US' ? 'available' : 'unavailable')
}
processLocally = false
}
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
localStorage.setItem('hapi-voice-lang', 'en-US')
localStorage.setItem('hapi-transcription-provider', 'browser-local')
const { result } = renderHook(() => useVoiceSettings(), { wrapper: Wrapper })
await waitFor(() => expect(result.current.provider).toBe('browser-local'))
act(() => result.current.setVoiceLanguage({ code: 'zh-CN', name: 'Chinese', nativeName: '中文' }))
await waitFor(() => expect(result.current.provider).toBe('openai'))
act(() => result.current.setVoiceLanguage({ code: 'en-US', name: 'English', nativeName: 'English' }))
await waitFor(() => expect(result.current.provider).toBe('browser-local'))
expect(result.current.provider).toBe('browser-local')
expect(available).not.toHaveBeenCalled()
})
})
+1 -2
View File
@@ -12,7 +12,7 @@ import {
writeStoredVoiceSelection,
} from '@/lib/voicePickerPreferences'
import type { VoiceBackendType } from '@hapi/protocol/voice'
import { useVoiceInputPreferences, VOICE_LANGUAGE_CHANGE_EVENT } from '@/hooks/useVoiceInputPreferences'
import { useVoiceInputPreferences } from '@/hooks/useVoiceInputPreferences'
export function useVoiceSettings() {
const { api } = useAppContext()
@@ -83,7 +83,6 @@ export function useVoiceSettings() {
setVoiceLanguageState(language.code)
if (language.code === null) localStorage.removeItem('hapi-voice-lang')
else localStorage.setItem('hapi-voice-lang', language.code)
window.dispatchEvent(new Event(VOICE_LANGUAGE_CHANGE_EVENT))
}, [])
const stopPreview = useCallback(() => {