mirror of
https://github.com/wu736139669/hapi.git
synced 2026-08-05 06:24:37 +00:00
fix(web): make browser-local speech probing Android-safe (#1349)
* fix(web): guard browser-local speech probes * test(web): cover concurrent speech probes * docs: clarify browser-local speech probing
This commit is contained in:
@@ -20,7 +20,7 @@ export TRANSCRIPTION_API_KEY="..." # optional
|
||||
```
|
||||
|
||||
Restart the hub after changing credentials. API keys are not entered or stored in the web app.
|
||||
Realtime OpenAI, ElevenLabs, and Deepgram sessions receive only short-lived credentials minted by the hub. Browsers with an installed on-device `SpeechRecognition` language pack also expose **Browser on-device** as a realtime-only provider; HAPI never falls back from that option to browser-hosted recognition.
|
||||
Realtime OpenAI, ElevenLabs, and Deepgram sessions receive only short-lived credentials minted by the hub. Eligible desktop browsers with the on-device `SpeechRecognition` API expose **Browser on-device** as a realtime-only provider. HAPI checks the selected language pack when dictation starts and never falls back from that option to browser-hosted recognition. Mobile and unknown browser environments fail closed because this API is experimental and some Android WebViews expose unsafe partial implementations.
|
||||
|
||||
## Overview
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { getBrowserLocalSpeechSupport } from './browserLocalSpeech'
|
||||
|
||||
const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36'
|
||||
|
||||
function speechRecognitionShape(availableDescriptor?: PropertyDescriptor) {
|
||||
class MockSpeechRecognition {
|
||||
processLocally = false
|
||||
}
|
||||
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
|
||||
if (availableDescriptor) Object.defineProperty(MockSpeechRecognition, 'available', availableDescriptor)
|
||||
return MockSpeechRecognition
|
||||
}
|
||||
|
||||
describe('browser-local speech capability detection', () => {
|
||||
it('uses the static available data property as a no-probe capability requirement', () => {
|
||||
const available = vi.fn(() => Promise.resolve('available'))
|
||||
const SpeechRecognition = speechRecognitionShape({ value: available })
|
||||
|
||||
const support = getBrowserLocalSpeechSupport({
|
||||
userAgent: DESKTOP_USER_AGENT,
|
||||
userAgentData: { platform: 'macOS', mobile: false },
|
||||
speechRecognition: SpeechRecognition
|
||||
})
|
||||
|
||||
expect(support?.constructor).toBe(SpeechRecognition)
|
||||
expect(support?.available).toBe(available)
|
||||
expect(available).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects missing, non-function, and getter available properties without reading a getter', () => {
|
||||
const getter = vi.fn(() => vi.fn(() => Promise.resolve('available')))
|
||||
const cases = [
|
||||
speechRecognitionShape(),
|
||||
speechRecognitionShape({ value: 'available' }),
|
||||
speechRecognitionShape({ get: getter })
|
||||
]
|
||||
|
||||
for (const SpeechRecognition of cases) {
|
||||
expect(getBrowserLocalSpeechSupport({
|
||||
userAgent: DESKTOP_USER_AGENT,
|
||||
userAgentData: { platform: 'macOS', mobile: false },
|
||||
speechRecognition: SpeechRecognition
|
||||
})).toBeNull()
|
||||
}
|
||||
|
||||
expect(getter).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('fails closed when a desktop-looking UA has mobile Android UA-CH signals', () => {
|
||||
const available = vi.fn(() => Promise.resolve('available'))
|
||||
const SpeechRecognition = speechRecognitionShape({ value: available })
|
||||
|
||||
expect(getBrowserLocalSpeechSupport({
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
|
||||
userAgentData: { platform: 'Android', mobile: true },
|
||||
speechRecognition: SpeechRecognition
|
||||
})).toBeNull()
|
||||
expect(available).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects desktop-looking UAs without trusted desktop UA-CH', () => {
|
||||
const available = vi.fn(() => Promise.resolve('available'))
|
||||
const SpeechRecognition = speechRecognitionShape({ value: available })
|
||||
|
||||
expect(getBrowserLocalSpeechSupport({
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
|
||||
speechRecognition: SpeechRecognition
|
||||
})).toBeNull()
|
||||
expect(getBrowserLocalSpeechSupport({
|
||||
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Version/18.0 Mobile/15E148 Safari/604.1',
|
||||
speechRecognition: SpeechRecognition
|
||||
})).toBeNull()
|
||||
expect(available).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,107 @@
|
||||
export interface LocalSpeechRecognitionResult {
|
||||
readonly isFinal: boolean
|
||||
readonly 0: { readonly transcript: string }
|
||||
}
|
||||
|
||||
export interface LocalSpeechRecognitionEvent extends Event {
|
||||
readonly results: { readonly length: number; readonly [index: number]: LocalSpeechRecognitionResult }
|
||||
}
|
||||
|
||||
export interface LocalSpeechRecognition extends EventTarget {
|
||||
continuous: boolean
|
||||
interimResults: boolean
|
||||
lang: string
|
||||
processLocally: boolean
|
||||
onresult: ((event: LocalSpeechRecognitionEvent) => void) | null
|
||||
onerror: ((event: Event & { error?: string }) => void) | null
|
||||
onend: (() => void) | null
|
||||
start: () => void
|
||||
stop: () => void
|
||||
abort: () => void
|
||||
}
|
||||
|
||||
export interface LocalSpeechRecognitionConstructor {
|
||||
new(): LocalSpeechRecognition
|
||||
prototype: LocalSpeechRecognition
|
||||
}
|
||||
|
||||
export type LocalSpeechRecognitionAvailability = (
|
||||
options: { langs: string[]; processLocally: true }
|
||||
) => Promise<string> | string
|
||||
|
||||
export interface BrowserLocalSpeechEnvironment {
|
||||
userAgent?: string
|
||||
userAgentData?: { platform?: string; mobile?: boolean }
|
||||
speechRecognition?: unknown
|
||||
}
|
||||
|
||||
export interface BrowserLocalSpeechSupport {
|
||||
constructor: LocalSpeechRecognitionConstructor
|
||||
available: LocalSpeechRecognitionAvailability
|
||||
}
|
||||
|
||||
function currentUserAgent(): string {
|
||||
return typeof navigator === 'undefined' ? '' : navigator.userAgent
|
||||
}
|
||||
|
||||
function currentSpeechRecognition(): unknown {
|
||||
return (globalThis as typeof globalThis & {
|
||||
SpeechRecognition?: unknown
|
||||
}).SpeechRecognition
|
||||
}
|
||||
|
||||
function currentUserAgentData(): BrowserLocalSpeechEnvironment['userAgentData'] {
|
||||
return (typeof navigator === 'undefined'
|
||||
? undefined
|
||||
: (navigator as Navigator & { userAgentData?: BrowserLocalSpeechEnvironment['userAgentData'] }).userAgentData)
|
||||
}
|
||||
|
||||
const SAFE_DESKTOP_PLATFORMS = new Set(['Windows', 'macOS', 'Linux', 'Chrome OS'])
|
||||
|
||||
/**
|
||||
* The experimental on-device speech API is eligible only with explicit,
|
||||
* trustworthy User-Agent Client Hints that identify a desktop platform. Android
|
||||
* WebViews can expose a partial shape whose native `available()` call crashes
|
||||
* the renderer, so missing, unknown, and mobile environments fail closed.
|
||||
*/
|
||||
export function isConfirmedDesktopSpeechEnvironment(
|
||||
_userAgent: string,
|
||||
userAgentData?: BrowserLocalSpeechEnvironment['userAgentData']
|
||||
): boolean {
|
||||
return userAgentData?.mobile === false
|
||||
&& typeof userAgentData.platform === 'string'
|
||||
&& SAFE_DESKTOP_PLATFORMS.has(userAgentData.platform)
|
||||
}
|
||||
|
||||
function staticAvailabilityMethod(candidate: Function): LocalSpeechRecognitionAvailability | null {
|
||||
for (let target: object | null = candidate; target && target !== Function.prototype; target = Object.getPrototypeOf(target)) {
|
||||
const descriptor = Object.getOwnPropertyDescriptor(target, 'available')
|
||||
if (descriptor) return typeof descriptor.value === 'function'
|
||||
? descriptor.value as LocalSpeechRecognitionAvailability
|
||||
: null
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks only the browser API shape. It intentionally does not instantiate
|
||||
* recognition or call the experimental `SpeechRecognition.available()` method.
|
||||
*/
|
||||
export function getBrowserLocalSpeechSupport(
|
||||
environment: BrowserLocalSpeechEnvironment = {}
|
||||
): BrowserLocalSpeechSupport | null {
|
||||
const userAgent = environment.userAgent ?? currentUserAgent()
|
||||
const userAgentData = environment.userAgentData ?? currentUserAgentData()
|
||||
if (!isConfirmedDesktopSpeechEnvironment(userAgent, userAgentData)) return null
|
||||
|
||||
const candidate = environment.speechRecognition ?? currentSpeechRecognition()
|
||||
if (typeof candidate !== 'function') return null
|
||||
const constructor = candidate as LocalSpeechRecognitionConstructor
|
||||
if (!constructor.prototype || !('processLocally' in constructor.prototype)) return null
|
||||
const available = staticAvailabilityMethod(candidate)
|
||||
return available ? { constructor, available } : null
|
||||
}
|
||||
|
||||
export function hasBrowserLocalSpeechSupport(): boolean {
|
||||
return getBrowserLocalSpeechSupport() !== null
|
||||
}
|
||||
@@ -1,5 +1,191 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { startDeepgramRealtimeTranscription, startOpenAIRealtimeTranscription } from './realtimeTranscription'
|
||||
import {
|
||||
BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS,
|
||||
getBrowserLocalAvailabilityProbeSubscriberCountForTesting,
|
||||
startBrowserLocalTranscription,
|
||||
startDeepgramRealtimeTranscription,
|
||||
startOpenAIRealtimeTranscription
|
||||
} from './realtimeTranscription'
|
||||
|
||||
function browserLocalCallbacks() {
|
||||
return {
|
||||
onConnected: vi.fn(),
|
||||
onPartial: vi.fn(),
|
||||
onFinal: vi.fn(),
|
||||
onError: vi.fn()
|
||||
}
|
||||
}
|
||||
|
||||
function installBrowserLocalSpeechRecognition(available: () => Promise<string> | string, onConstruct = vi.fn()) {
|
||||
class MockSpeechRecognition extends EventTarget {
|
||||
static available = available
|
||||
continuous = false
|
||||
interimResults = false
|
||||
lang = ''
|
||||
processLocally = false
|
||||
onresult: ((event: Event) => void) | null = null
|
||||
onerror: ((event: Event) => void) | null = null
|
||||
onend: (() => void) | null = null
|
||||
start = vi.fn()
|
||||
stop = vi.fn()
|
||||
abort = vi.fn()
|
||||
constructor() {
|
||||
super()
|
||||
onConstruct()
|
||||
}
|
||||
}
|
||||
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false, writable: true })
|
||||
vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
|
||||
vi.stubGlobal('navigator', {
|
||||
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
|
||||
userAgentData: { platform: 'macOS', mobile: false },
|
||||
language: 'en-US'
|
||||
})
|
||||
return { available, onConstruct, constructor: MockSpeechRecognition }
|
||||
}
|
||||
|
||||
describe('browser-local realtime transcription', () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
it('calls available only after an explicit browser-local dictation start', async () => {
|
||||
const available = vi.fn(() => Promise.resolve('available'))
|
||||
installBrowserLocalSpeechRecognition(available)
|
||||
const callbacks = browserLocalCallbacks()
|
||||
|
||||
const session = await startBrowserLocalTranscription({ language: 'en-US', callbacks })
|
||||
|
||||
expect(available).toHaveBeenCalledOnce()
|
||||
expect(available).toHaveBeenCalledWith({ langs: ['en-US'], processLocally: true })
|
||||
expect(callbacks.onConnected).toHaveBeenCalledOnce()
|
||||
session.cancel()
|
||||
})
|
||||
|
||||
it('shares one successful native probe across concurrent browser-local starts', async () => {
|
||||
let resolveAvailable!: (status: string) => void
|
||||
const available = vi.fn(() => new Promise<string>((resolve) => { resolveAvailable = resolve }))
|
||||
const onConstruct = vi.fn()
|
||||
installBrowserLocalSpeechRecognition(available, onConstruct)
|
||||
const firstCallbacks = browserLocalCallbacks()
|
||||
const secondCallbacks = browserLocalCallbacks()
|
||||
|
||||
const first = startBrowserLocalTranscription({ language: 'en-US', callbacks: firstCallbacks })
|
||||
const second = startBrowserLocalTranscription({ language: 'en-US', callbacks: secondCallbacks })
|
||||
await vi.waitFor(() => expect(available).toHaveBeenCalledOnce())
|
||||
|
||||
resolveAvailable('available')
|
||||
const sessions = await Promise.all([first, second])
|
||||
|
||||
expect(onConstruct).toHaveBeenCalledTimes(2)
|
||||
expect(firstCallbacks.onConnected).toHaveBeenCalledOnce()
|
||||
expect(secondCallbacks.onConnected).toHaveBeenCalledOnce()
|
||||
sessions.forEach((session) => session.cancel())
|
||||
})
|
||||
|
||||
it('surfaces a synchronous available failure', async () => {
|
||||
const available = vi.fn(() => { throw new Error('native failure') })
|
||||
installBrowserLocalSpeechRecognition(available)
|
||||
|
||||
await expect(startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() }))
|
||||
.rejects.toThrow('native failure')
|
||||
expect(available).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('surfaces a rejected available probe', async () => {
|
||||
const available = vi.fn(() => Promise.reject(new Error('probe rejected')))
|
||||
installBrowserLocalSpeechRecognition(available)
|
||||
|
||||
await expect(startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() }))
|
||||
.rejects.toThrow('probe rejected')
|
||||
expect(available).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('times out a stalled available probe', async () => {
|
||||
vi.useFakeTimers()
|
||||
const available = vi.fn(() => new Promise<string>(() => {}))
|
||||
installBrowserLocalSpeechRecognition(available)
|
||||
const starting = startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() })
|
||||
const expectation = expect(starting).rejects.toThrow('availability check timed out')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS)
|
||||
|
||||
await expectation
|
||||
expect(available).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('does not invoke available when its start signal aborts before the native microtask', async () => {
|
||||
const available = vi.fn(() => Promise.resolve('available'))
|
||||
installBrowserLocalSpeechRecognition(available)
|
||||
const controller = new AbortController()
|
||||
const starting = startBrowserLocalTranscription({
|
||||
language: 'en-US',
|
||||
signal: controller.signal,
|
||||
callbacks: browserLocalCallbacks()
|
||||
})
|
||||
const expectation = expect(starting).rejects.toBeDefined()
|
||||
|
||||
controller.abort()
|
||||
await expectation
|
||||
await Promise.resolve()
|
||||
|
||||
expect(available).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('shares a stalled native probe across timeout retries and ignores its late result', async () => {
|
||||
vi.useFakeTimers()
|
||||
let resolveAvailable!: (status: string) => void
|
||||
const available = vi.fn(() => new Promise<string>((resolve) => { resolveAvailable = resolve }))
|
||||
const onConstruct = vi.fn()
|
||||
const speechRecognition = installBrowserLocalSpeechRecognition(available, onConstruct)
|
||||
const firstCallbacks = browserLocalCallbacks()
|
||||
const first = startBrowserLocalTranscription({ language: 'en-US', callbacks: firstCallbacks })
|
||||
const firstExpectation = expect(first).rejects.toThrow('availability check timed out')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS)
|
||||
await firstExpectation
|
||||
expect(getBrowserLocalAvailabilityProbeSubscriberCountForTesting(speechRecognition.constructor, 'en-US')).toBe(0)
|
||||
|
||||
const second = startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() })
|
||||
const secondExpectation = expect(second).rejects.toThrow('availability check timed out')
|
||||
await vi.advanceTimersByTimeAsync(BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS)
|
||||
await secondExpectation
|
||||
expect(getBrowserLocalAvailabilityProbeSubscriberCountForTesting(speechRecognition.constructor, 'en-US')).toBe(0)
|
||||
|
||||
const controller = new AbortController()
|
||||
const aborted = startBrowserLocalTranscription({
|
||||
language: 'en-US',
|
||||
signal: controller.signal,
|
||||
callbacks: browserLocalCallbacks()
|
||||
})
|
||||
const abortedExpectation = expect(aborted).rejects.toBeDefined()
|
||||
controller.abort()
|
||||
await abortedExpectation
|
||||
expect(getBrowserLocalAvailabilityProbeSubscriberCountForTesting(speechRecognition.constructor, 'en-US')).toBe(0)
|
||||
|
||||
expect(available).toHaveBeenCalledOnce()
|
||||
resolveAvailable('available')
|
||||
await Promise.resolve()
|
||||
await Promise.resolve()
|
||||
expect(onConstruct).not.toHaveBeenCalled()
|
||||
expect(firstCallbacks.onConnected).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects Android browser-local startup without calling a partial native available API', async () => {
|
||||
const available = vi.fn(() => Promise.resolve('available'))
|
||||
installBrowserLocalSpeechRecognition(available)
|
||||
vi.stubGlobal('navigator', {
|
||||
userAgent: 'Mozilla/5.0 (Linux; Android 15; WebView)',
|
||||
userAgentData: { platform: 'Android', mobile: true },
|
||||
language: 'en-US'
|
||||
})
|
||||
|
||||
await expect(startBrowserLocalTranscription({ language: 'en-US', callbacks: browserLocalCallbacks() }))
|
||||
.rejects.toThrow('not supported by this browser')
|
||||
expect(available).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('OpenAI realtime transcription', () => {
|
||||
afterEach(() => vi.unstubAllGlobals())
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
import { DEEPGRAM_TRANSCRIPTION_MODEL } from '@hapi/protocol/voice'
|
||||
import {
|
||||
getBrowserLocalSpeechSupport,
|
||||
type LocalSpeechRecognition,
|
||||
type LocalSpeechRecognitionAvailability,
|
||||
type LocalSpeechRecognitionConstructor,
|
||||
type LocalSpeechRecognitionEvent
|
||||
} from './browserLocalSpeech'
|
||||
|
||||
export interface RealtimeTranscriptionCallbacks {
|
||||
onConnected: () => void
|
||||
@@ -330,43 +337,120 @@ export async function startDeepgramRealtimeTranscription(options: {
|
||||
}
|
||||
}
|
||||
|
||||
interface LocalSpeechRecognitionResult {
|
||||
readonly isFinal: boolean
|
||||
readonly 0: { readonly transcript: string }
|
||||
export const BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS = 10_000
|
||||
|
||||
interface BrowserLocalAvailabilityProbe {
|
||||
readonly available: LocalSpeechRecognitionAvailability
|
||||
readonly constructor: LocalSpeechRecognitionConstructor
|
||||
readonly language: string
|
||||
readonly subscribers: Set<BrowserLocalAvailabilitySubscriber>
|
||||
}
|
||||
|
||||
interface LocalSpeechRecognitionEvent extends Event {
|
||||
readonly results: { readonly length: number; readonly [index: number]: LocalSpeechRecognitionResult }
|
||||
interface BrowserLocalAvailabilitySubscriber {
|
||||
resolve: (status: string) => void
|
||||
reject: (error: unknown) => void
|
||||
}
|
||||
|
||||
interface LocalSpeechRecognition extends EventTarget {
|
||||
continuous: boolean
|
||||
interimResults: boolean
|
||||
lang: string
|
||||
processLocally: boolean
|
||||
onresult: ((event: LocalSpeechRecognitionEvent) => void) | null
|
||||
onerror: ((event: Event & { error?: string }) => void) | null
|
||||
onend: (() => void) | null
|
||||
start: () => void
|
||||
stop: () => void
|
||||
abort: () => void
|
||||
const browserLocalAvailabilityProbes = new WeakMap<LocalSpeechRecognitionConstructor, Map<string, BrowserLocalAvailabilityProbe>>()
|
||||
|
||||
function browserLocalProbeMap(constructor: LocalSpeechRecognitionConstructor): Map<string, BrowserLocalAvailabilityProbe> {
|
||||
let probes = browserLocalAvailabilityProbes.get(constructor)
|
||||
if (!probes) {
|
||||
probes = new Map()
|
||||
browserLocalAvailabilityProbes.set(constructor, probes)
|
||||
}
|
||||
return probes
|
||||
}
|
||||
|
||||
interface LocalSpeechRecognitionConstructor {
|
||||
new(): LocalSpeechRecognition
|
||||
prototype: LocalSpeechRecognition
|
||||
available: (options: { langs: string[]; processLocally: true }) => Promise<string>
|
||||
function getBrowserLocalAvailabilityProbe(options: {
|
||||
available: LocalSpeechRecognitionAvailability
|
||||
constructor: LocalSpeechRecognitionConstructor
|
||||
language: string
|
||||
}): BrowserLocalAvailabilityProbe {
|
||||
const probes = browserLocalProbeMap(options.constructor)
|
||||
const existing = probes.get(options.language)
|
||||
if (existing) return existing
|
||||
|
||||
const probe: BrowserLocalAvailabilityProbe = {
|
||||
...options,
|
||||
subscribers: new Set()
|
||||
}
|
||||
probes.set(options.language, probe)
|
||||
queueMicrotask(() => {
|
||||
if (probe.subscribers.size === 0) {
|
||||
probes.delete(options.language)
|
||||
return
|
||||
}
|
||||
Promise.resolve()
|
||||
.then(() => probe.available.call(probe.constructor, { langs: [probe.language], processLocally: true }))
|
||||
.then(
|
||||
(status) => settleBrowserLocalAvailabilityProbe(probes, probe, (subscriber) => subscriber.resolve(status)),
|
||||
(error) => settleBrowserLocalAvailabilityProbe(probes, probe, (subscriber) => subscriber.reject(error))
|
||||
)
|
||||
})
|
||||
return probe
|
||||
}
|
||||
|
||||
function localSpeechRecognitionConstructor(): LocalSpeechRecognitionConstructor | null {
|
||||
const constructor = (globalThis as typeof globalThis & {
|
||||
SpeechRecognition?: LocalSpeechRecognitionConstructor
|
||||
}).SpeechRecognition
|
||||
return constructor
|
||||
&& typeof constructor.available === 'function'
|
||||
&& 'processLocally' in constructor.prototype
|
||||
? constructor
|
||||
: null
|
||||
function settleBrowserLocalAvailabilityProbe(
|
||||
probes: Map<string, BrowserLocalAvailabilityProbe>,
|
||||
probe: BrowserLocalAvailabilityProbe,
|
||||
notify: (subscriber: BrowserLocalAvailabilitySubscriber) => void
|
||||
): void {
|
||||
if (probes.get(probe.language) !== probe) return
|
||||
probes.delete(probe.language)
|
||||
const subscribers = Array.from(probe.subscribers)
|
||||
probe.subscribers.clear()
|
||||
subscribers.forEach(notify)
|
||||
}
|
||||
|
||||
export function getBrowserLocalAvailabilityProbeSubscriberCountForTesting(
|
||||
constructor: object,
|
||||
language: string
|
||||
): number {
|
||||
return browserLocalAvailabilityProbes
|
||||
.get(constructor as LocalSpeechRecognitionConstructor)
|
||||
?.get(language)
|
||||
?.subscribers.size ?? 0
|
||||
}
|
||||
|
||||
function abortError(signal: AbortSignal): unknown {
|
||||
return signal.reason ?? new Error('On-device speech recognition availability check was aborted')
|
||||
}
|
||||
|
||||
async function checkBrowserLocalSpeechAvailability(options: {
|
||||
available: LocalSpeechRecognitionAvailability
|
||||
constructor: LocalSpeechRecognitionConstructor
|
||||
language: string
|
||||
signal?: AbortSignal
|
||||
}): Promise<string> {
|
||||
options.signal?.throwIfAborted()
|
||||
const probe = getBrowserLocalAvailabilityProbe(options)
|
||||
|
||||
return await new Promise<string>((resolve, reject) => {
|
||||
let settled = false
|
||||
const subscriber: BrowserLocalAvailabilitySubscriber = { resolve, reject }
|
||||
const detach = () => probe.subscribers.delete(subscriber)
|
||||
const finish = (callback: () => void) => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
clearTimeout(timeout)
|
||||
options.signal?.removeEventListener('abort', onAbort)
|
||||
detach()
|
||||
callback()
|
||||
}
|
||||
const timeout = setTimeout(() => {
|
||||
finish(() => reject(new Error('On-device speech recognition availability check timed out')))
|
||||
}, BROWSER_LOCAL_AVAILABILITY_TIMEOUT_MS)
|
||||
const onAbort = () => finish(() => reject(abortError(options.signal!)))
|
||||
if (options.signal?.aborted) {
|
||||
onAbort()
|
||||
return
|
||||
}
|
||||
probe.subscribers.add(subscriber)
|
||||
options.signal?.addEventListener('abort', onAbort, { once: true })
|
||||
subscriber.resolve = (status) => finish(() => resolve(status))
|
||||
subscriber.reject = (error) => finish(() => reject(error))
|
||||
})
|
||||
}
|
||||
|
||||
export async function startBrowserLocalTranscription(options: {
|
||||
@@ -375,15 +459,18 @@ export async function startBrowserLocalTranscription(options: {
|
||||
callbacks: RealtimeTranscriptionCallbacks
|
||||
}): Promise<RealtimeTranscriptionSession> {
|
||||
options.signal?.throwIfAborted()
|
||||
const constructor = localSpeechRecognitionConstructor()
|
||||
if (!constructor) throw new Error('On-device speech recognition is not supported by this browser')
|
||||
const support = getBrowserLocalSpeechSupport()
|
||||
if (!support) throw new Error('On-device speech recognition is not supported by this browser')
|
||||
const language = options.language || navigator.language
|
||||
if (await constructor.available({ langs: [language], processLocally: true }) !== 'available') {
|
||||
// `available()` is deferred to a microtask so an abort between start and
|
||||
// native invocation detaches its only consumer before touching the API.
|
||||
options.signal?.throwIfAborted()
|
||||
if (await checkBrowserLocalSpeechAvailability({ ...support, language, signal: options.signal }) !== 'available') {
|
||||
throw new Error(`On-device speech recognition is not installed for ${language}`)
|
||||
}
|
||||
options.signal?.throwIfAborted()
|
||||
|
||||
const recognition = new constructor()
|
||||
const recognition = new support.constructor()
|
||||
recognition.continuous = true
|
||||
recognition.interimResults = true
|
||||
recognition.lang = language
|
||||
|
||||
@@ -62,6 +62,11 @@ describe('useDictation', () => {
|
||||
})
|
||||
|
||||
it('shows on-device partial text and inserts only the final transcript', async () => {
|
||||
vi.stubGlobal('navigator', {
|
||||
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
|
||||
userAgentData: { platform: 'macOS', mobile: false },
|
||||
language: 'en-US'
|
||||
})
|
||||
let recognition: MockSpeechRecognition | null = null
|
||||
class MockSpeechRecognition {
|
||||
static async available() { return 'available' }
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import { renderHook, waitFor } from '@testing-library/react'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { ApiClient } from '@/api/client'
|
||||
import { useVoiceInputPreferences } from './useVoiceInputPreferences'
|
||||
|
||||
function installPartialSpeechRecognition(available = vi.fn(() => Promise.resolve('available'))) {
|
||||
class MockSpeechRecognition {
|
||||
static available = available
|
||||
processLocally = false
|
||||
}
|
||||
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
|
||||
vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
|
||||
return available
|
||||
}
|
||||
|
||||
describe('useVoiceInputPreferences', () => {
|
||||
afterEach(() => vi.unstubAllGlobals())
|
||||
|
||||
it('discovers browser-local support from its shape without probing available on mount', async () => {
|
||||
const available = installPartialSpeechRecognition()
|
||||
vi.stubGlobal('navigator', {
|
||||
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
|
||||
userAgentData: { platform: 'macOS', mobile: false },
|
||||
language: 'en-US'
|
||||
})
|
||||
const api = {
|
||||
fetchTranscriptionProviders: vi.fn(async () => ({ providers: [] }))
|
||||
}
|
||||
|
||||
const { result } = renderHook(() => useVoiceInputPreferences(api as unknown as ApiClient))
|
||||
|
||||
await waitFor(() => expect(result.current.provider).toBe('browser-local'))
|
||||
expect(available).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not expose or probe partial browser-local speech APIs on Android', async () => {
|
||||
const available = installPartialSpeechRecognition()
|
||||
vi.stubGlobal('navigator', { userAgent: 'Mozilla/5.0 (Linux; Android 15; WebView)', language: 'en-US' })
|
||||
const api = {
|
||||
fetchTranscriptionProviders: vi.fn(async () => ({
|
||||
providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }]
|
||||
}))
|
||||
}
|
||||
|
||||
const { result } = renderHook(() => useVoiceInputPreferences(api as unknown as ApiClient))
|
||||
|
||||
await waitFor(() => expect(result.current.provider).toBe('openai'))
|
||||
expect(result.current.providers).toHaveLength(1)
|
||||
expect(available).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -7,33 +7,17 @@ import {
|
||||
type TranscriptionProviderInfo,
|
||||
type VoiceMode
|
||||
} from '@hapi/protocol/voice'
|
||||
import { hasBrowserLocalSpeechSupport } from './browserLocalSpeech'
|
||||
|
||||
const VOICE_MODE_KEY = 'hapi-voice-mode'
|
||||
const TRANSCRIPTION_PROVIDER_KEY = 'hapi-transcription-provider'
|
||||
const TRANSCRIPTION_MODE_KEY = 'hapi-transcription-mode'
|
||||
const CHANGE_EVENT = 'hapi-voice-input-change'
|
||||
export const VOICE_LANGUAGE_CHANGE_EVENT = 'hapi-voice-language-change'
|
||||
|
||||
function notifyChange(): void {
|
||||
window.dispatchEvent(new Event(CHANGE_EVENT))
|
||||
}
|
||||
|
||||
async function browserLocalTranscriptionSupported(): Promise<boolean> {
|
||||
const constructor = (globalThis as typeof globalThis & {
|
||||
SpeechRecognition?: {
|
||||
prototype: object
|
||||
available?: (options: { langs: string[]; processLocally: true }) => Promise<string>
|
||||
}
|
||||
}).SpeechRecognition
|
||||
if (!constructor || typeof constructor.available !== 'function' || !('processLocally' in constructor.prototype)) return false
|
||||
const language = localStorage.getItem('hapi-voice-lang') || navigator.language
|
||||
try {
|
||||
return await constructor.available({ langs: [language], processLocally: true }) === 'available'
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function readVoiceMode(): VoiceMode {
|
||||
return localStorage.getItem(VOICE_MODE_KEY) === 'dictation' ? 'dictation' : 'assistant'
|
||||
}
|
||||
@@ -64,27 +48,21 @@ export function useVoiceInputPreferences(api: ApiClient | null) {
|
||||
useEffect(() => {
|
||||
if (!api) return
|
||||
let cancelled = false
|
||||
let request = 0
|
||||
const refreshProviders = () => {
|
||||
const current = ++request
|
||||
Promise.all([api.fetchTranscriptionProviders(), browserLocalTranscriptionSupported()]).then(([{ providers: configured }, browserLocal]) => {
|
||||
if (cancelled || current !== request) return
|
||||
const available = browserLocal
|
||||
? [...configured, BROWSER_LOCAL_TRANSCRIPTION_PROVIDER]
|
||||
: configured
|
||||
setProviders(available)
|
||||
const selectedProvider = resolveProvider(available, localStorage.getItem(TRANSCRIPTION_PROVIDER_KEY))
|
||||
setProviderState(selectedProvider)
|
||||
setTranscriptionModeState(resolveMode(available, selectedProvider, localStorage.getItem(TRANSCRIPTION_MODE_KEY)))
|
||||
}).catch(() => {
|
||||
if (!cancelled && current === request) setProviders([])
|
||||
})
|
||||
}
|
||||
refreshProviders()
|
||||
window.addEventListener(VOICE_LANGUAGE_CHANGE_EVENT, refreshProviders)
|
||||
const browserLocal = hasBrowserLocalSpeechSupport()
|
||||
api.fetchTranscriptionProviders().then(({ providers: configured }) => {
|
||||
if (cancelled) return
|
||||
const available = browserLocal
|
||||
? [...configured, BROWSER_LOCAL_TRANSCRIPTION_PROVIDER]
|
||||
: configured
|
||||
setProviders(available)
|
||||
const selectedProvider = resolveProvider(available, localStorage.getItem(TRANSCRIPTION_PROVIDER_KEY))
|
||||
setProviderState(selectedProvider)
|
||||
setTranscriptionModeState(resolveMode(available, selectedProvider, localStorage.getItem(TRANSCRIPTION_MODE_KEY)))
|
||||
}).catch(() => {
|
||||
if (!cancelled) setProviders([])
|
||||
})
|
||||
return () => {
|
||||
cancelled = true
|
||||
window.removeEventListener(VOICE_LANGUAGE_CHANGE_EVENT, refreshProviders)
|
||||
}
|
||||
}, [api])
|
||||
|
||||
|
||||
@@ -32,6 +32,11 @@ describe('useVoiceSettings', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
localStorage.clear()
|
||||
vi.stubGlobal('navigator', {
|
||||
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/140.0 Safari/537.36',
|
||||
userAgentData: { platform: 'macOS', mobile: false },
|
||||
language: 'en-US'
|
||||
})
|
||||
fetchVoiceBackend.mockResolvedValue({ backend: 'elevenlabs', backends: ['elevenlabs'] })
|
||||
fetchVoices.mockResolvedValue([])
|
||||
class MockAudio {
|
||||
@@ -85,8 +90,9 @@ describe('useVoiceSettings', () => {
|
||||
})
|
||||
|
||||
it('uses realtime for the realtime-only browser provider', async () => {
|
||||
const available = vi.fn(() => Promise.resolve('available'))
|
||||
class MockSpeechRecognition {
|
||||
static available() { return Promise.resolve('available') }
|
||||
static available = available
|
||||
processLocally = false
|
||||
}
|
||||
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
|
||||
@@ -97,46 +103,28 @@ describe('useVoiceSettings', () => {
|
||||
|
||||
await waitFor(() => expect(result.current.provider).toBe('browser-local'))
|
||||
expect(result.current.transcriptionMode).toBe('realtime')
|
||||
expect(available).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not expose browser dictation without the selected language pack', async () => {
|
||||
it('does not probe browser-local speech availability when the language changes', async () => {
|
||||
fetchTranscriptionProviders.mockResolvedValueOnce({
|
||||
providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }]
|
||||
})
|
||||
const available = vi.fn(() => Promise.resolve('available'))
|
||||
class MockSpeechRecognition {
|
||||
static available() { return Promise.resolve('unavailable') }
|
||||
static available = available
|
||||
processLocally = false
|
||||
}
|
||||
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
|
||||
vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
|
||||
|
||||
const { result } = renderHook(() => useVoiceSettings(), { wrapper: Wrapper })
|
||||
|
||||
await waitFor(() => expect(result.current.provider).toBe('openai'))
|
||||
expect(result.current.providers).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('rechecks the on-device language pack when the language changes', async () => {
|
||||
fetchTranscriptionProviders.mockResolvedValue({
|
||||
providers: [{ id: 'openai', label: 'OpenAI', modes: ['standard', 'realtime'] }]
|
||||
})
|
||||
class MockSpeechRecognition {
|
||||
static available({ langs }: { langs: string[] }) {
|
||||
return Promise.resolve(langs[0] === 'en-US' ? 'available' : 'unavailable')
|
||||
}
|
||||
processLocally = false
|
||||
}
|
||||
Object.defineProperty(MockSpeechRecognition.prototype, 'processLocally', { value: false })
|
||||
vi.stubGlobal('SpeechRecognition', MockSpeechRecognition)
|
||||
localStorage.setItem('hapi-voice-lang', 'en-US')
|
||||
localStorage.setItem('hapi-transcription-provider', 'browser-local')
|
||||
|
||||
const { result } = renderHook(() => useVoiceSettings(), { wrapper: Wrapper })
|
||||
|
||||
await waitFor(() => expect(result.current.provider).toBe('browser-local'))
|
||||
|
||||
act(() => result.current.setVoiceLanguage({ code: 'zh-CN', name: 'Chinese', nativeName: '中文' }))
|
||||
await waitFor(() => expect(result.current.provider).toBe('openai'))
|
||||
|
||||
act(() => result.current.setVoiceLanguage({ code: 'en-US', name: 'English', nativeName: 'English' }))
|
||||
await waitFor(() => expect(result.current.provider).toBe('browser-local'))
|
||||
expect(result.current.provider).toBe('browser-local')
|
||||
expect(available).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
writeStoredVoiceSelection,
|
||||
} from '@/lib/voicePickerPreferences'
|
||||
import type { VoiceBackendType } from '@hapi/protocol/voice'
|
||||
import { useVoiceInputPreferences, VOICE_LANGUAGE_CHANGE_EVENT } from '@/hooks/useVoiceInputPreferences'
|
||||
import { useVoiceInputPreferences } from '@/hooks/useVoiceInputPreferences'
|
||||
|
||||
export function useVoiceSettings() {
|
||||
const { api } = useAppContext()
|
||||
@@ -83,7 +83,6 @@ export function useVoiceSettings() {
|
||||
setVoiceLanguageState(language.code)
|
||||
if (language.code === null) localStorage.removeItem('hapi-voice-lang')
|
||||
else localStorage.setItem('hapi-voice-lang', language.code)
|
||||
window.dispatchEvent(new Event(VOICE_LANGUAGE_CHANGE_EVENT))
|
||||
}, [])
|
||||
|
||||
const stopPreview = useCallback(() => {
|
||||
|
||||
Reference in New Issue
Block a user