From a812a51dd7b8cd4f87c03395ebd9b6093df4b695 Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:43:04 +0100 Subject: [PATCH] feat(voice): backend voice picker + advanced controls behind disclosure (#742) (#743) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(voice): voice personality, picker catalog, and prompt layer foundation - voicePickerCatalog.ts: per-backend voice lists for Gemini and Qwen with resolve helpers (resolveGeminiLiveVoice, resolveQwenRealtimeVoice) - voicePersonality.ts: VoicePersonalityPreferences schema, presets, composed system prompt with identity/character/response-length layers - voicePromptLayers.ts: buildResolvedVoiceSystemPrompt, preset delivery snippets - voiceSystemPromptParam.ts: hub-side base64url decode for ?systemPrompt= - voicePickerPreferences.ts, voicePersonalitySession.ts: browser-side encode, decode, and storage helpers - useVoicePersonality: React hook for preferences persistence via [HAPI](https://hapi.run) Co-Authored-By: HAPI * fix(voice): preset delivery included when non-balanced preset selected; restore test typecheck - isDefaultVoicePersonality: add preset check so warm/calm/direct presets trigger the delivery snippet instead of being treated as default - web/tsconfig.json: remove test file exclusion from typecheck (restoring strict coverage of test code); fix resulting type error in mock declaration via [HAPI](https://hapi.run) Co-Authored-By: HAPI * fix(voice): include use_speaker_boost in ElevenLabs TTS override payload The checkbox persisted the pref but ttsDiffersFromDefault and buildElevenLabsTtsOverride both omitted it, so the setting was never sent to the agent. via [HAPI](https://hapi.run) Co-Authored-By: HAPI * test(voice): update speaker_boost test to assert it IS included in override The previous test asserted use_speaker_boost was omitted; now it's correctly included in the TTS payload. via [HAPI](https://hapi.run) Co-Authored-By: HAPI * fix(voice): authorize use_speaker_boost in ElevenLabs override schema Add use_speaker_boost to both the VoiceAgentConfig tts override type and the buildVoiceAgentConfig() platform_settings so the field is accepted by the ElevenLabs agent runtime. via [HAPI](https://hapi.run) Co-Authored-By: HAPI * fix(voice): propagate full language code through composed prompt, not just zh getDefaultVoiceSystemPrompt and resolveComposedVoiceSystemPrompt were filtering language to zh-only before passing to composeVoiceAgentPrompt. Now append buildVoiceLanguageBlock(language) after composition so French, Spanish, Japanese etc. reach Gemini/Qwen sessions correctly. via [HAPI](https://hapi.run) Co-Authored-By: HAPI * fix(voice): only append language block when language explicitly set Building language block unconditionally when no language is given caused getDefaultVoiceSystemPrompt() to diverge from VOICE_SYSTEM_PROMPT. Only append the block when a code is explicitly provided. via [HAPI](https://hapi.run) Co-Authored-By: HAPI * fix(voice): always include language block for Gemini/Qwen in composed prompt When auto-detect is on (language=undefined), the composed prompt sent via hub proxy was losing the language auto-detect instruction because the block was only added when language was explicitly set. Now: ElevenLabs skips the block (has its own language field); Gemini/Qwen always include it — undefined produces the auto-detect block, an explicit code produces the appropriate language instruction. via [HAPI](https://hapi.run) Co-Authored-By: HAPI --------- Co-authored-by: HAPI --- hub/src/voiceSystemPromptParam.ts | 13 + hub/src/web/qwenProxyHandler.test.ts | 150 ++++++ hub/src/web/qwenProxyHandler.ts | 165 +++++++ hub/src/web/routes/voice.test.ts | 155 +++++- hub/src/web/routes/voice.ts | 88 +++- hub/src/web/server.ts | 110 ++--- shared/package.json | 4 +- shared/src/voice.backends.test.ts | 53 ++ shared/src/voice.gemini.test.ts | 47 +- shared/src/voice.ts | 281 +++++------ shared/src/voicePersonality.test.ts | 81 ++++ shared/src/voicePersonality.ts | 456 ++++++++++++++++++ shared/src/voicePickerCatalog.ts | 68 +++ shared/src/voicePromptLayers.ts | 213 ++++++++ web/src/api/client.ts | 2 +- web/src/api/voice.ts | 20 +- .../settings/VoiceAdvancedControls.tsx | 374 ++++++++++++++ web/src/hooks/useVoicePersonality.ts | 131 +++++ web/src/lib/locales/en.ts | 81 +++- web/src/lib/locales/zh-CN.ts | 81 +++- web/src/lib/voice-context.tsx | 14 +- web/src/lib/voiceContextStream.ts | 37 ++ web/src/lib/voicePersonalitySession.test.ts | 94 ++++ web/src/lib/voicePersonalitySession.ts | 154 ++++++ web/src/lib/voicePickerPreferences.ts | 78 +++ web/src/realtime/GeminiLiveVoiceSession.tsx | 55 ++- web/src/realtime/QwenVoiceSession.tsx | 52 +- web/src/realtime/RealtimeSession.ts | 21 +- web/src/realtime/RealtimeVoiceSession.tsx | 44 +- web/src/realtime/VoiceBackendSession.tsx | 7 +- .../realtime/hooks/voiceContextPlan.test.ts | 42 ++ web/src/realtime/hooks/voiceContextPlan.ts | 132 +++++ web/src/realtime/hooks/voiceHooks.ts | 14 +- web/src/realtime/types.ts | 7 + web/src/routes/settings/index.test.tsx | 100 +++- web/src/routes/settings/index.tsx | 445 +++++++++-------- web/tsconfig.json | 3 +- 37 files changed, 3346 insertions(+), 526 deletions(-) create mode 100644 hub/src/voiceSystemPromptParam.ts create mode 100644 hub/src/web/qwenProxyHandler.test.ts create mode 100644 hub/src/web/qwenProxyHandler.ts create mode 100644 shared/src/voice.backends.test.ts create mode 100644 shared/src/voicePersonality.test.ts create mode 100644 shared/src/voicePersonality.ts create mode 100644 shared/src/voicePickerCatalog.ts create mode 100644 shared/src/voicePromptLayers.ts create mode 100644 web/src/components/settings/VoiceAdvancedControls.tsx create mode 100644 web/src/hooks/useVoicePersonality.ts create mode 100644 web/src/lib/voiceContextStream.ts create mode 100644 web/src/lib/voicePersonalitySession.test.ts create mode 100644 web/src/lib/voicePersonalitySession.ts create mode 100644 web/src/lib/voicePickerPreferences.ts create mode 100644 web/src/realtime/hooks/voiceContextPlan.test.ts create mode 100644 web/src/realtime/hooks/voiceContextPlan.ts diff --git a/hub/src/voiceSystemPromptParam.ts b/hub/src/voiceSystemPromptParam.ts new file mode 100644 index 00000000..49644330 --- /dev/null +++ b/hub/src/voiceSystemPromptParam.ts @@ -0,0 +1,13 @@ +/** Decode ?systemPrompt= from Gemini hub proxy (base64url, UTF-8). */ +export function decodeVoiceSystemPromptParam(param: string | null | undefined): string | undefined { + if (!param?.trim()) return undefined + try { + const normalized = param.replace(/-/g, '+').replace(/_/g, '/') + const pad = '='.repeat((4 - (normalized.length % 4)) % 4) + const decoded = Buffer.from(normalized + pad, 'base64').toString('utf8') + if (!decoded.trim() || decoded.length > 48_000) return undefined + return decoded + } catch { + return undefined + } +} diff --git a/hub/src/web/qwenProxyHandler.test.ts b/hub/src/web/qwenProxyHandler.test.ts new file mode 100644 index 00000000..337bd069 --- /dev/null +++ b/hub/src/web/qwenProxyHandler.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, test, beforeEach, afterEach } from 'bun:test' +import type { ServerWebSocket } from 'bun' +import { createQwenProxyWebSocketHandler, type WebSocketLike } from './qwenProxyHandler' + +const WS_OPEN = 1 + +class FakeUpstream implements WebSocketLike { + readyState = WS_OPEN + sent: Array = [] + closed = false + onmessage: ((event: { data: string | ArrayBuffer | Uint8Array }) => void) | null = null + onerror: ((event: unknown) => void) | null = null + onclose: ((event: { code: number; reason: string }) => void) | null = null + + constructor(public url: string, public opts?: unknown) {} + + send(data: string | ArrayBuffer | Uint8Array) { + this.sent.push(data) + } + + close(code = 1000, reason = '') { + this.closed = true + this.readyState = 3 + this.onclose?.({ code, reason }) + } + + deliver(payload: object | string) { + const text = typeof payload === 'string' ? payload : JSON.stringify(payload) + this.onmessage?.({ data: text }) + } +} + +class FakeClient { + readyState = WS_OPEN + sent: string[] = [] + closeCode: number | null = null + closeReason: string | null = null + data: object + + constructor(data: object) { + this.data = data + } + + send(payload: string | ArrayBuffer | Uint8Array) { + this.sent.push(typeof payload === 'string' ? payload : new TextDecoder().decode(payload as Uint8Array)) + } + + close(code?: number, reason?: string) { + this.closeCode = code ?? null + this.closeReason = reason ?? null + this.readyState = 3 + } +} + +let lastUpstream: FakeUpstream | null = null +const FakeWebSocket = function FakeWebSocket(url: string, opts?: unknown) { + const u = new FakeUpstream(url, opts) + lastUpstream = u + return u +} as unknown as new (url: string, opts?: unknown) => WebSocketLike + +beforeEach(() => { + lastUpstream = null +}) + +afterEach(() => { + lastUpstream = null +}) + +function newClient() { + return new FakeClient({ apiKey: 'k', model: 'qwen3-omni-flash-realtime', language: 'en', voiceName: 'Cherry' }) as unknown as ServerWebSocket & FakeClient +} + +describe('createQwenProxyWebSocketHandler ack-gate', () => { + test('queues client frames until upstream acks hub-owned session.update with session.updated', () => { + const handler = createQwenProxyWebSocketHandler(FakeWebSocket) + const client = newClient() + + handler.open(client) + const upstream = lastUpstream! + expect(upstream.sent).toHaveLength(0) + + upstream.deliver({ type: 'session.created' }) + expect(upstream.sent).toHaveLength(1) + const hubSetup = JSON.parse(upstream.sent[0] as string) as { type: string; session: { instructions: string } } + expect(hubSetup.type).toBe('session.update') + expect(typeof hubSetup.session.instructions).toBe('string') + + handler.message(client, JSON.stringify({ type: 'response.create' })) + handler.message(client, JSON.stringify({ type: 'conversation.item.create', item: { type: 'message' } })) + handler.message(client, JSON.stringify({ type: 'session.update', session: { instructions: 'updated' } })) + + expect(upstream.sent).toHaveLength(1) + + upstream.deliver({ type: 'session.updated', session: { instructions: hubSetup.session.instructions } }) + + expect(upstream.sent.length).toBeGreaterThanOrEqual(4) + const flushedTypes = upstream.sent.slice(1).map(raw => { + try { return (JSON.parse(raw as string) as { type?: string }).type } + catch { return undefined } + }) + expect(flushedTypes).toEqual(['response.create', 'conversation.item.create', 'session.update']) + }) + + test('forwards client frames immediately after the gate has flipped', () => { + const handler = createQwenProxyWebSocketHandler(FakeWebSocket) + const client = newClient() + + handler.open(client) + const upstream = lastUpstream! + + upstream.deliver({ type: 'session.created' }) + upstream.deliver({ type: 'session.updated', session: {} }) + const sentBefore = upstream.sent.length + + handler.message(client, JSON.stringify({ type: 'input_audio_buffer.append', audio: 'abc' })) + expect(upstream.sent).toHaveLength(sentBefore + 1) + }) + + test('rejects client frames that fail the safe-frame allowlist with 1008', () => { + const handler = createQwenProxyWebSocketHandler(FakeWebSocket) + const client = newClient() + + handler.open(client) + const upstream = lastUpstream! + upstream.deliver({ type: 'session.created' }) + upstream.deliver({ type: 'session.updated', session: {} }) + + handler.message(client, JSON.stringify({ + type: 'session.update', + session: { tools: [{ type: 'function', name: 'evil' }] } + })) + + expect(client.closeCode).toBe(1008) + expect(client.closeReason).toContain('instructions') + }) + + test('clears queued frames on close so no stale data leaks if a new client reuses memory', () => { + const handler = createQwenProxyWebSocketHandler(FakeWebSocket) + const client = newClient() + + handler.open(client) + const upstream = lastUpstream! + upstream.deliver({ type: 'session.created' }) + handler.message(client, JSON.stringify({ type: 'response.create' })) + + handler.close(client, 1000, 'bye') + expect(upstream.closed).toBe(true) + }) +}) diff --git a/hub/src/web/qwenProxyHandler.ts b/hub/src/web/qwenProxyHandler.ts new file mode 100644 index 00000000..8c1cd18b --- /dev/null +++ b/hub/src/web/qwenProxyHandler.ts @@ -0,0 +1,165 @@ +// Qwen Realtime WebSocket proxy factory — extracted from server.ts so the ack-gate +// behaviour (queue client frames until DashScope acknowledges the hub-owned session.update) +// can be unit-tested without spinning up Hono + Bun.serve. +// +// The factory accepts an optional WebSocket constructor injection so tests can substitute +// a deterministic fake for the upstream connection. + +import type { ServerWebSocket } from 'bun' +import { buildQwenSessionUpdateMessage, isQwenSafeClientFrame } from '@hapi/protocol/voice' + +type WebSocketCtor = new (url: string, opts?: unknown) => WebSocketLike + +export interface WebSocketLike { + readyState: number + send(data: string | ArrayBuffer | Uint8Array): void + close(code?: number, reason?: string): void + onmessage: ((event: { data: string | ArrayBuffer | Uint8Array }) => void) | null + onerror: ((event: unknown) => void) | null + onclose: ((event: { code: number; reason: string }) => void) | null +} + +export interface QwenProxyHandler { + open(clientWs: ServerWebSocket): void + message(clientWs: ServerWebSocket, message: string | ArrayBuffer | Uint8Array): void + close(clientWs: ServerWebSocket, code: number, reason: string): void +} + +const QWEN_WS_BASE = 'wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime' +const WS_OPEN = 1 + +function toClientCloseCode(code: number): number { + return code >= 1000 && code <= 4999 && code !== 1005 && code !== 1006 && code !== 1015 + ? code + : 1011 +} + +export function createQwenProxyWebSocketHandler( + WebSocketImpl: WebSocketCtor = WebSocket as unknown as WebSocketCtor +): QwenProxyHandler { + const MAX_PENDING_BYTES = 1024 * 1024 // 1 MiB — rejects setup-gate floods + const upstreamMap = new WeakMap, WebSocketLike>() + // Holds the hub-owned session.update payload until session.created arrives from DashScope. + // Sending session.update before session.created violates the Qwen Realtime protocol ordering. + const pendingSetupMap = new WeakMap, string>() + // Tracks whether DashScope has acknowledged the hub-owned session.update with a session.updated + // frame. Until the ack arrives, client frames are queued, never forwarded - otherwise an + // authenticated client could push response.create / conversation.item.create / instruction-only + // session.update before HAPI's tools/voice/instructions are locked into the upstream session. + const setupAckedMap = new WeakMap, boolean>() + const pendingClientFrames = new WeakMap, Array>() + const pendingClientBytes = new WeakMap, number>() + + return { + open(clientWs) { + const data = clientWs.data as { apiKey: string; model: string; language?: string; voiceName?: string; systemInstruction?: string } + const upstreamUrl = `${process.env.QWEN_REALTIME_WS_URL || QWEN_WS_BASE}?model=${encodeURIComponent(data.model)}` + + const upstream = new WebSocketImpl(upstreamUrl, { + headers: { 'Authorization': `Bearer ${data.apiKey}` } + }) + + upstreamMap.set(clientWs, upstream) + pendingSetupMap.set(clientWs, JSON.stringify(buildQwenSessionUpdateMessage(data.language, data.voiceName, data.systemInstruction))) + setupAckedMap.set(clientWs, false) + pendingClientBytes.set(clientWs, 0) + + upstream.onmessage = (event) => { + const raw = event.data + const text = typeof raw === 'string' + ? raw + : new TextDecoder().decode(raw instanceof Uint8Array ? raw : new Uint8Array(raw as ArrayBuffer)) + + const pendingSetup = pendingSetupMap.get(clientWs) + if (pendingSetup) { + try { + const parsed = JSON.parse(text) as { type?: string } + if (parsed.type === 'session.created') { + pendingSetupMap.delete(clientWs) + try { if (clientWs.readyState === 1) clientWs.send(text) } catch { /* client gone */ } + upstream.send(pendingSetup) + return + } + } catch { /* not JSON */ } + } + + // Once the upstream acks the hub-owned setup with session.updated, flip the gate + // and flush any client frames the proxy held while tools/voice/instructions were + // still being installed. + if (setupAckedMap.get(clientWs) === false) { + try { + const parsed = JSON.parse(text) as { type?: string } + if (parsed.type === 'session.updated') { + setupAckedMap.set(clientWs, true) + const queued = pendingClientFrames.get(clientWs) ?? [] + pendingClientFrames.delete(clientWs) + for (const frame of queued) { + try { upstream.send(frame) } catch { /* upstream gone */ } + } + } + } catch { /* not JSON */ } + } + + try { + if (clientWs.readyState === 1) { + clientWs.send(typeof raw === 'string' ? raw : new Uint8Array(raw as ArrayBuffer)) + } + } catch { /* client gone */ } + } + upstream.onerror = () => { + pendingSetupMap.delete(clientWs) + setupAckedMap.delete(clientWs) + pendingClientFrames.delete(clientWs) + pendingClientBytes.delete(clientWs) + upstreamMap.delete(clientWs) + try { clientWs.close(1011, 'Upstream error') } catch { /* */ } + } + upstream.onclose = (event) => { + pendingSetupMap.delete(clientWs) + setupAckedMap.delete(clientWs) + pendingClientFrames.delete(clientWs) + pendingClientBytes.delete(clientWs) + try { clientWs.close(toClientCloseCode(event.code), event.reason || 'Upstream closed') } catch { /* */ } + upstreamMap.delete(clientWs) + } + }, + message(clientWs, message) { + if (!isQwenSafeClientFrame(message)) { + try { clientWs.close(1008, 'Client session.update may only modify instructions') } catch { /* */ } + return + } + const upstream = upstreamMap.get(clientWs) + if (upstream?.readyState !== WS_OPEN) return + + // Hold client frames until DashScope has acknowledged the hub-owned session.update. + // Without this gate, the client could race response.create / conversation.item.create / + // instruction-only session.update past the lockdown of tools/voice/instructions and run + // the upstream session under the provider default config or partially-applied state. + if (setupAckedMap.get(clientWs) !== true) { + const frameSize = typeof message === 'string' ? message.length : (message as ArrayBuffer | Uint8Array).byteLength + const total = (pendingClientBytes.get(clientWs) ?? 0) + frameSize + if (total > MAX_PENDING_BYTES) { + try { clientWs.close(1009, 'Setup-gate frame budget exceeded') } catch { /* */ } + return + } + pendingClientBytes.set(clientWs, total) + const queue = pendingClientFrames.get(clientWs) ?? [] + queue.push(message) + pendingClientFrames.set(clientWs, queue) + return + } + upstream.send(message) + }, + close(clientWs, code, reason) { + pendingSetupMap.delete(clientWs) + setupAckedMap.delete(clientWs) + pendingClientFrames.delete(clientWs) + pendingClientBytes.delete(clientWs) + const upstream = upstreamMap.get(clientWs) + if (upstream) { + try { upstream.close(toClientCloseCode(code), (reason || 'Client closed').slice(0, 123)) } catch { /* */ } + upstreamMap.delete(clientWs) + } + } + } +} diff --git a/hub/src/web/routes/voice.test.ts b/hub/src/web/routes/voice.test.ts index 9cc3f423..c7c49b0f 100644 --- a/hub/src/web/routes/voice.test.ts +++ b/hub/src/web/routes/voice.test.ts @@ -133,6 +133,102 @@ describe('POST /api/voice/token', () => { else delete process.env.ELEVENLABS_AGENT_ID }) + it('reconciles platform_settings.overrides on existing agents (one PATCH per process)', async () => { + const app = createApp() + const headers = { + ...(await authHeaders()), + 'content-type': 'application/json' + } + + const prevKey = process.env.ELEVENLABS_API_KEY + const prevAgent = process.env.ELEVENLABS_AGENT_ID + process.env.ELEVENLABS_API_KEY = 'test-key-ensure' + delete process.env.ELEVENLABS_AGENT_ID + + const existingAgentId = `agent_ensure_${Math.random().toString(36).slice(2, 10)}` + const existingAgentName = `Hapi Voice Assistant [voice:ensure-voice-${Math.random().toString(36).slice(2, 6)}]` + const voiceId = existingAgentName.match(/\[voice:([^\]]+)\]/)?.[1] ?? '' + + const patchCalls: Array<{ url: string; body: unknown }> = [] + const originalFetch = global.fetch + // @ts-expect-error test override + global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input) + + if (url.endsWith('/convai/agents') && init?.method === 'GET') { + return new Response(JSON.stringify({ + agents: [{ agent_id: existingAgentId, name: existingAgentName }] + }), { status: 200 }) + } + if ( + url.includes(`/convai/agents/${existingAgentId}`) + && init?.method === 'PATCH' + ) { + const body = init?.body ? JSON.parse(String(init.body)) : null + patchCalls.push({ url, body }) + return new Response(JSON.stringify({ agent_id: existingAgentId }), { status: 200 }) + } + if (url.includes('/convai/conversation/token?agent_id=')) { + return new Response(JSON.stringify({ token: 'tok_ensure' }), { status: 200 }) + } + return new Response('not found', { status: 404 }) + }) as typeof fetch + + const first = await app.request('/api/voice/token', { + method: 'POST', + headers, + body: JSON.stringify({ voiceId }) + }) + expect(first.status).toBe(200) + expect(await first.json()).toMatchObject({ + allowed: true, + agentId: existingAgentId, + token: 'tok_ensure' + }) + + expect(patchCalls.length).toBeGreaterThanOrEqual(1) + const patchedBody = patchCalls[0]?.body as { + platform_settings?: { + overrides?: { + conversation_config_override?: { + agent?: { language?: boolean; prompt?: { prompt?: boolean } } + tts?: { + voice_id?: boolean + stability?: boolean + similarity_boost?: boolean + style?: boolean + speed?: boolean + } + } + } + } + } + const overrides = patchedBody.platform_settings?.overrides?.conversation_config_override + expect(overrides?.agent?.language).toBe(true) + expect(overrides?.agent?.prompt?.prompt).toBe(true) + expect(overrides?.tts?.voice_id).toBe(true) + expect(overrides?.tts?.stability).toBe(true) + expect(overrides?.tts?.similarity_boost).toBe(true) + expect(overrides?.tts?.style).toBe(true) + expect(overrides?.tts?.speed).toBe(true) + + // Second call within the same process must NOT re-issue the PATCH. + const before = patchCalls.length + const second = await app.request('/api/voice/token', { + method: 'POST', + headers, + body: JSON.stringify({ voiceId }) + }) + expect(second.status).toBe(200) + expect(patchCalls.length).toBe(before) + + global.fetch = originalFetch + if (prevKey) process.env.ELEVENLABS_API_KEY = prevKey + else delete process.env.ELEVENLABS_API_KEY + if (prevAgent) process.env.ELEVENLABS_AGENT_ID = prevAgent + else delete process.env.ELEVENLABS_AGENT_ID + }) + it('prefers voice-specific agent over ELEVENLABS_AGENT_ID when voiceId is provided', async () => { const app = createApp() const headers = { @@ -190,54 +286,83 @@ describe('POST /api/voice/token', () => { }) describe('GET /api/voice/backend', () => { - const originalEnv = process.env.VOICE_BACKEND + const originalEnv = { + VOICE_BACKEND: process.env.VOICE_BACKEND, + ELEVENLABS_API_KEY: process.env.ELEVENLABS_API_KEY, + GEMINI_API_KEY: process.env.GEMINI_API_KEY, + GOOGLE_API_KEY: process.env.GOOGLE_API_KEY, + DASHSCOPE_API_KEY: process.env.DASHSCOPE_API_KEY, + QWEN_API_KEY: process.env.QWEN_API_KEY + } afterEach(() => { - if (originalEnv === undefined) { - delete process.env.VOICE_BACKEND - } else { - process.env.VOICE_BACKEND = originalEnv + for (const [key, value] of Object.entries(originalEnv)) { + if (value === undefined) { + delete process.env[key] + } else { + process.env[key] = value + } } }) - test('returns elevenlabs by default', async () => { + test('returns elevenlabs by default with backends list', async () => { delete process.env.VOICE_BACKEND + delete process.env.GEMINI_API_KEY + delete process.env.GOOGLE_API_KEY + delete process.env.DASHSCOPE_API_KEY + delete process.env.QWEN_API_KEY + process.env.ELEVENLABS_API_KEY = 'test-el' const app = createApp() const headers = await authHeaders() const res = await app.request('/api/voice/backend', { headers }) expect(res.status).toBe(200) - const body = await res.json() as { backend: string } + const body = await res.json() as { backend: string; backends: string[] } expect(body.backend).toBe('elevenlabs') + expect(body.backends).toEqual(['elevenlabs']) }) - test('returns gemini-live when configured', async () => { + test('returns gemini-live when configured and key present', async () => { process.env.VOICE_BACKEND = 'gemini-live' + process.env.GEMINI_API_KEY = 'test-gm' + delete process.env.ELEVENLABS_API_KEY + delete process.env.DASHSCOPE_API_KEY const app = createApp() const headers = await authHeaders() const res = await app.request('/api/voice/backend', { headers }) expect(res.status).toBe(200) - const body = await res.json() as { backend: string } + const body = await res.json() as { backend: string; backends: string[] } expect(body.backend).toBe('gemini-live') + expect(body.backends).toEqual(['gemini-live']) }) - test('returns qwen-realtime when configured', async () => { - process.env.VOICE_BACKEND = 'qwen-realtime' + test('lists every backend with credentials', async () => { + process.env.VOICE_BACKEND = 'gemini-live' + process.env.ELEVENLABS_API_KEY = 'test-el' + process.env.GEMINI_API_KEY = 'test-gm' + process.env.DASHSCOPE_API_KEY = 'test-qw' const app = createApp() const headers = await authHeaders() const res = await app.request('/api/voice/backend', { headers }) expect(res.status).toBe(200) - const body = await res.json() as { backend: string } - expect(body.backend).toBe('qwen-realtime') + const body = await res.json() as { backend: string; backends: string[] } + expect(body.backend).toBe('gemini-live') + expect(body.backends).toEqual(['elevenlabs', 'gemini-live', 'qwen-realtime']) }) - test('falls back to elevenlabs for unknown values', async () => { + test('falls back to elevenlabs for unknown VOICE_BACKEND values', async () => { process.env.VOICE_BACKEND = 'unknown-backend' + delete process.env.GEMINI_API_KEY + delete process.env.GOOGLE_API_KEY + delete process.env.DASHSCOPE_API_KEY + delete process.env.QWEN_API_KEY + process.env.ELEVENLABS_API_KEY = 'test-el' const app = createApp() const headers = await authHeaders() const res = await app.request('/api/voice/backend', { headers }) expect(res.status).toBe(200) - const body = await res.json() as { backend: string } + const body = await res.json() as { backend: string; backends: string[] } expect(body.backend).toBe('elevenlabs') + expect(body.backends).toEqual(['elevenlabs']) }) }) diff --git a/hub/src/web/routes/voice.ts b/hub/src/web/routes/voice.ts index 3ccc4815..6e8e9efa 100644 --- a/hub/src/web/routes/voice.ts +++ b/hub/src/web/routes/voice.ts @@ -5,7 +5,8 @@ import { ELEVENLABS_API_BASE, VOICE_AGENT_NAME, buildVoiceAgentConfig, - DEFAULT_VOICE_BACKEND + listConfiguredVoiceBackends, + resolveHubVoiceBackend } from '@hapi/protocol/voice' import type { VoiceBackendType } from '@hapi/protocol/voice' @@ -36,11 +37,74 @@ const telemetryEventSchema = z.object({ // Cache for auto-created agent IDs (keyed by API key hash) const agentIdCache = new Map() +// Per-process set of agent IDs that already had their platform_settings.overrides +// reconciled with the canonical buildVoiceAgentConfig() shape. Reset on process restart. +const overridesEnsuredAgents = new Set() + interface ElevenLabsAgent { agent_id: string name: string } +/** + * Ensure an existing ElevenLabs ConvAI agent has the platform_settings.overrides + * declared by buildVoiceAgentConfig(). Without these overrides, the client-side + * SDK crashes with `Cannot read properties of undefined (reading 'error_type')` + * when a session sends agent.prompt / tts.* override fields the server rejects. + * + * Idempotent and best-effort: PATCH failures are logged and swallowed so token + * issuance is never blocked by reconciliation. Cached per agent_id per process. + * + * See: https://elevenlabs.io/docs/agents-platform/customization/personalization/overrides + */ +async function ensureAgentOverrides(apiKey: string, agentId: string): Promise { + if (overridesEnsuredAgents.has(agentId)) return + overridesEnsuredAgents.add(agentId) + + const canonical = buildVoiceAgentConfig().platform_settings + if (!canonical) return + + try { + const response = await fetch( + `${ELEVENLABS_API_BASE}/convai/agents/${encodeURIComponent(agentId)}`, + { + method: 'PATCH', + headers: { + 'xi-api-key': apiKey, + 'Content-Type': 'application/json', + 'Accept': 'application/json' + }, + body: JSON.stringify({ platform_settings: canonical }) + } + ) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) as { + detail?: { message?: string } | string + } + const errorMessage = typeof errorData.detail === 'string' + ? errorData.detail + : (errorData.detail as { message?: string })?.message + || `API error: ${response.status}` + console.warn('[Voice] Failed to ensure agent overrides (non-fatal)', { + agentId, + status: response.status, + errorMessage + }) + overridesEnsuredAgents.delete(agentId) + return + } + + console.log('[Voice] Reconciled platform_settings.overrides on agent', { agentId }) + } catch (error) { + console.warn('[Voice] Error reconciling agent overrides (non-fatal)', { + agentId, + error: error instanceof Error ? error.message : String(error) + }) + overridesEnsuredAgents.delete(agentId) + } +} + function parseVoiceAgentMap(): Record { const raw = process.env.ELEVENLABS_VOICE_AGENT_MAP if (!raw) return {} @@ -157,12 +221,18 @@ async function getOrCreateAgentIdForVoice(apiKey: string, voiceId?: string): Pro if (agentId) { console.log('[Voice] Found existing agent:', agentId) + // Existing agents may predate the platform_settings.overrides we declare + // in buildVoiceAgentConfig() — reconcile so client overrides are accepted. + await ensureAgentOverrides(apiKey, agentId) } else { // Create new agent console.log('[Voice] No existing agent found, creating new one...') agentId = await createNamedHapiAgent(apiKey, agentName, voiceId) if (agentId) { console.log('[Voice] Created new agent:', agentId) + // Newly-created agents already carry the canonical overrides, but + // mark as ensured so we don't re-PATCH on every token request. + overridesEnsuredAgents.add(agentId) } } @@ -177,14 +247,11 @@ async function getOrCreateAgentIdForVoice(apiKey: string, voiceId?: string): Pro export function createVoiceRoutes(): Hono { const app = new Hono() - // Return the configured voice backend type + // Hub default backend + all backends with credentials configured app.get('/voice/backend', (c) => { - const raw = process.env.VOICE_BACKEND - const backend: VoiceBackendType = - raw === 'gemini-live' ? 'gemini-live' - : raw === 'qwen-realtime' ? 'qwen-realtime' - : DEFAULT_VOICE_BACKEND - return c.json({ backend }) + const backends = listConfiguredVoiceBackends(process.env) + const backend = resolveHubVoiceBackend(process.env) + return c.json({ backend, backends }) }) // Get Gemini API key for Gemini Live voice sessions @@ -286,6 +353,11 @@ export function createVoiceRoutes(): Hono { }, 500) } + // Operator-supplied agent ids (env ELEVENLABS_AGENT_ID, ELEVENLABS_VOICE_AGENT_MAP, + // customAgentId) won't have been routed through ensureAgentOverrides above — + // reconcile here so the client overrides payload is always accepted. + await ensureAgentOverrides(apiKey, agentId) + try { console.log('[Voice][Token] Requesting ElevenLabs conversation token', { requestId, diff --git a/hub/src/web/server.ts b/hub/src/web/server.ts index 32eaeac0..b0cf0592 100644 --- a/hub/src/web/server.ts +++ b/hub/src/web/server.ts @@ -6,7 +6,9 @@ import { existsSync } from 'node:fs' import { serveStatic } from 'hono/bun' import { getConfiguration } from '../configuration' import { PROTOCOL_VERSION } from '@hapi/protocol' -import { buildGeminiLiveSetupMessage, buildQwenSessionUpdateMessage, isQwenSafeClientFrame, QWEN_REALTIME_MODEL } from '@hapi/protocol/voice' +import { buildGeminiLiveSetupMessage, QWEN_REALTIME_MODEL } from '@hapi/protocol/voice' +import { createQwenProxyWebSocketHandler } from './qwenProxyHandler' +import { decodeVoiceSystemPromptParam } from '../voiceSystemPromptParam' import type { SyncEngine } from '../sync/syncEngine' import { createAuthMiddleware, type WebAppEnv } from './middleware/auth' import { createAuthRoutes } from './routes/auth' @@ -80,7 +82,14 @@ function createGeminiProxyWebSocketHandler() { return { open(clientWs: ServerWebSocket) { - const data = clientWs.data as { _geminiProxy: boolean; apiKey: string; language?: string } + const data = clientWs.data as { + _geminiProxy: boolean + apiKey: string + language?: string + voiceName?: string + systemInstruction?: string + affectiveDialog?: boolean + } const upstreamUrl = `${process.env.GEMINI_LIVE_WS_URL || GEMINI_WS_BASE}?key=${encodeURIComponent(data.apiKey)}` const pending: Array = [] pendingMap.set(clientWs, pending) @@ -92,7 +101,12 @@ function createGeminiProxyWebSocketHandler() { upstream.onopen = () => { // Hub-owned setup only — never forward client setup (prevents generic Gemini proxy abuse). // Do NOT flush pending here: wait for Google's setupComplete before forwarding client frames. - upstream.send(JSON.stringify(buildGeminiLiveSetupMessage(data.language))) + upstream.send(JSON.stringify(buildGeminiLiveSetupMessage( + data.language, + data.voiceName, + data.systemInstruction, + { affectiveDialog: data.affectiveDialog } + ))) } upstream.onmessage = (event) => { try { @@ -154,84 +168,9 @@ function createGeminiProxyWebSocketHandler() { } } -// Qwen Realtime WebSocket proxy — bridges browser (no custom headers) to DashScope (requires Authorization header) -function createQwenProxyWebSocketHandler() { - const QWEN_WS_BASE = 'wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime' - const upstreamMap = new WeakMap, WebSocket>() - // Holds the hub-owned session.update payload until session.created arrives from DashScope. - // Sending session.update before session.created violates the Qwen Realtime protocol ordering. - const pendingSetupMap = new WeakMap, string>() - - return { - open(clientWs: ServerWebSocket) { - const data = clientWs.data as { apiKey: string; model: string; language?: string } - const upstreamUrl = `${process.env.QWEN_REALTIME_WS_URL || QWEN_WS_BASE}?model=${encodeURIComponent(data.model)}` - - const upstream = new WebSocket(upstreamUrl, { - headers: { 'Authorization': `Bearer ${data.apiKey}` } - } as unknown as string[]) - - upstreamMap.set(clientWs, upstream) - pendingSetupMap.set(clientWs, JSON.stringify(buildQwenSessionUpdateMessage(data.language))) - - upstream.onmessage = (event) => { - const raw = event.data - const text = typeof raw === 'string' - ? raw - : new TextDecoder().decode(raw instanceof Uint8Array ? raw : new Uint8Array(raw as ArrayBuffer)) - - // Respect Qwen protocol ordering: relay session.created first, then send hub-owned - // session.update. DashScope must receive session.update after session.created. - const pendingSetup = pendingSetupMap.get(clientWs) - if (pendingSetup) { - try { - const parsed = JSON.parse(text) as { type?: string } - if (parsed.type === 'session.created') { - pendingSetupMap.delete(clientWs) - try { if (clientWs.readyState === 1) clientWs.send(text) } catch { /* client gone */ } - upstream.send(pendingSetup) - return - } - } catch { /* not JSON — relay as-is below */ } - } - - try { - if (clientWs.readyState === 1) { - clientWs.send(typeof raw === 'string' ? raw : new Uint8Array(raw as ArrayBuffer)) - } - } catch { /* client gone */ } - } - upstream.onerror = () => { - pendingSetupMap.delete(clientWs) - upstreamMap.delete(clientWs) - try { clientWs.close(1011, 'Upstream error') } catch { /* */ } - } - upstream.onclose = (event) => { - pendingSetupMap.delete(clientWs) - try { clientWs.close(toClientCloseCode(event.code), event.reason || 'Upstream closed') } catch { /* client gone */ } - upstreamMap.delete(clientWs) - } - }, - message(clientWs: ServerWebSocket, message: string | ArrayBuffer | Uint8Array) { - if (!isQwenSafeClientFrame(message)) { - try { clientWs.close(1008, 'Client session.update may only modify instructions') } catch { /* */ } - return - } - const upstream = upstreamMap.get(clientWs) - if (upstream?.readyState === WebSocket.OPEN) { - upstream.send(message) - } - }, - close(clientWs: ServerWebSocket, code: number, reason: string) { - pendingSetupMap.delete(clientWs) - const upstream = upstreamMap.get(clientWs) - if (upstream) { - try { upstream.close(toClientCloseCode(code), (reason || 'Client closed').slice(0, 123)) } catch { /* */ } - upstreamMap.delete(clientWs) - } - } - } -} +// Qwen Realtime WebSocket proxy — bridges browser (no custom headers) to DashScope +// (requires Authorization header). Implementation extracted to `./qwenProxyHandler` so +// the ack-gating behaviour is unit-testable; `createQwenProxyWebSocketHandler` is imported above. function findWebappDistDir(): { distDir: string; indexHtmlPath: string } { const candidates = [ @@ -513,8 +452,11 @@ export async function startWebServer(options: { return new Response('Gemini API key not configured', { status: 400 }) } const language = url.searchParams.get('language') ?? undefined + const voiceParam = url.searchParams.get('voice')?.trim() || undefined + const systemInstruction = decodeVoiceSystemPromptParam(url.searchParams.get('systemPrompt')) + const affectiveDialog = url.searchParams.get('affectiveDialog') === '1' const upgraded = (server as unknown as { upgrade: (req: Request, opts: unknown) => boolean }).upgrade(req, { - data: { _geminiProxy: true, apiKey, language } + data: { _geminiProxy: true, apiKey, language, voiceName: voiceParam, systemInstruction, affectiveDialog } }) if (!upgraded) { return new Response('WebSocket upgrade failed', { status: 500 }) @@ -526,11 +468,13 @@ export async function startWebServer(options: { const apiKey = process.env.DASHSCOPE_API_KEY || process.env.QWEN_API_KEY const model = QWEN_REALTIME_MODEL const language = url.searchParams.get('language') ?? undefined + const voiceParam = url.searchParams.get('voice')?.trim() || undefined + const systemInstruction = decodeVoiceSystemPromptParam(url.searchParams.get('systemPrompt')) if (!apiKey) { return new Response('DashScope API key not configured', { status: 400 }) } const upgraded = (server as unknown as { upgrade: (req: Request, opts: unknown) => boolean }).upgrade(req, { - data: { _qwenProxy: true, apiKey, model, language } + data: { _qwenProxy: true, apiKey, model, language, voiceName: voiceParam, systemInstruction } }) if (!upgraded) { return new Response('WebSocket upgrade failed', { status: 500 }) diff --git a/shared/package.json b/shared/package.json index e05df72b..1513cb3c 100644 --- a/shared/package.json +++ b/shared/package.json @@ -16,7 +16,9 @@ "./schemas": "./src/schemas.ts", "./sessionExport": "./src/sessionExport.ts", "./types": "./src/types.ts", - "./voice": "./src/voice.ts" + "./voice": "./src/voice.ts", + "./voicePickerCatalog": "./src/voicePickerCatalog.ts", + "./voice-personality": "./src/voicePersonality.ts" }, "sideEffects": false, "scripts": { diff --git a/shared/src/voice.backends.test.ts b/shared/src/voice.backends.test.ts new file mode 100644 index 00000000..f53f9940 --- /dev/null +++ b/shared/src/voice.backends.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, test } from 'bun:test' +import { + listConfiguredVoiceBackends, + resolveEffectiveVoiceBackend, + resolveHubVoiceBackend +} from './voice' + +describe('listConfiguredVoiceBackends', () => { + test('returns only backends with API keys', () => { + const backends = listConfiguredVoiceBackends({ + ELEVENLABS_API_KEY: 'el', + GEMINI_API_KEY: 'gm', + DASHSCOPE_API_KEY: 'qw' + }) + expect(backends).toEqual(['elevenlabs', 'gemini-live', 'qwen-realtime']) + }) + + test('falls back to elevenlabs when no keys configured', () => { + expect(listConfiguredVoiceBackends({})).toEqual(['elevenlabs']) + }) +}) + +describe('resolveHubVoiceBackend', () => { + test('uses VOICE_BACKEND when that backend is configured', () => { + const backend = resolveHubVoiceBackend({ + VOICE_BACKEND: 'gemini-live', + GEMINI_API_KEY: 'gm', + ELEVENLABS_API_KEY: 'el' + }) + expect(backend).toBe('gemini-live') + }) + + test('falls back to first configured when VOICE_BACKEND unavailable', () => { + const backend = resolveHubVoiceBackend({ + VOICE_BACKEND: 'qwen-realtime', + ELEVENLABS_API_KEY: 'el' + }) + expect(backend).toBe('elevenlabs') + }) +}) + +describe('resolveEffectiveVoiceBackend', () => { + const configured = ['elevenlabs', 'gemini-live'] as const + + test('prefers stored preference when configured', () => { + expect(resolveEffectiveVoiceBackend(configured, 'gemini-live', 'elevenlabs')).toBe('elevenlabs') + }) + + test('uses hub default when preference missing or invalid', () => { + expect(resolveEffectiveVoiceBackend(configured, 'gemini-live', null)).toBe('gemini-live') + expect(resolveEffectiveVoiceBackend(configured, 'gemini-live', 'qwen-realtime')).toBe('gemini-live') + }) +}) diff --git a/shared/src/voice.gemini.test.ts b/shared/src/voice.gemini.test.ts index b9602811..67b6fcb5 100644 --- a/shared/src/voice.gemini.test.ts +++ b/shared/src/voice.gemini.test.ts @@ -1,5 +1,13 @@ import { describe, expect, test } from 'bun:test' -import { buildGeminiLiveSetupMessage, buildQwenSessionUpdateMessage, isQwenSafeClientFrame, GEMINI_LIVE_MODEL, GEMINI_LIVE_VOICE, QWEN_REALTIME_VOICE } from './voice' +import { + buildGeminiLiveSetupMessage, + buildQwenSessionUpdateMessage, + isQwenSafeClientFrame, + GEMINI_LIVE_MODEL, + GEMINI_LIVE_VOICE, + QWEN_REALTIME_VOICE +} from './voice' +import { resolveGeminiLiveVoice, resolveQwenRealtimeVoice } from './voicePickerCatalog' describe('buildGeminiLiveSetupMessage', () => { test('locks model and voice to HAPI defaults', () => { @@ -18,15 +26,50 @@ describe('buildGeminiLiveSetupMessage', () => { const zhText = (zh.setup.systemInstruction as { parts: Array<{ text: string }> }).parts[0].text expect(zhText.length).toBeGreaterThan(enText.length) }) + + test('uses selected prebuilt voice when valid', () => { + const msg = buildGeminiLiveSetupMessage(undefined, 'Puck') + const speech = msg.setup.generationConfig as { + speechConfig?: { voiceConfig?: { prebuiltVoiceConfig?: { voiceName?: string } } } + } + expect(speech.speechConfig?.voiceConfig?.prebuiltVoiceConfig?.voiceName).toBe('Puck') + }) + + test('honors custom system instruction override', () => { + const custom = 'Speak only in haiku.' + const msg = buildGeminiLiveSetupMessage(undefined, undefined, custom) + const text = (msg.setup.systemInstruction as { parts: Array<{ text: string }> }).parts[0].text + expect(text).toBe(custom) + }) + + test('falls back to default for unknown voice names', () => { + const msg = buildGeminiLiveSetupMessage(undefined, 'NotARealVoice') + const speech = msg.setup.generationConfig as { + speechConfig?: { voiceConfig?: { prebuiltVoiceConfig?: { voiceName?: string } } } + } + expect(speech.speechConfig?.voiceConfig?.prebuiltVoiceConfig?.voiceName).toBe(resolveGeminiLiveVoice()) + }) }) describe('buildQwenSessionUpdateMessage', () => { - test('locks voice to HAPI default', () => { + test('locks voice to HAPI default when no voice name supplied', () => { const msg = buildQwenSessionUpdateMessage() const session = msg.session as { voice: string } expect(session.voice).toBe(QWEN_REALTIME_VOICE) }) + test('uses selected prebuilt voice when valid', () => { + const msg = buildQwenSessionUpdateMessage(undefined, 'Ethan') + const session = msg.session as { voice: string } + expect(session.voice).toBe('Ethan') + }) + + test('falls back to catalog default for unknown voice names', () => { + const msg = buildQwenSessionUpdateMessage(undefined, 'NotARealVoice') + const session = msg.session as { voice: string } + expect(session.voice).toBe(resolveQwenRealtimeVoice()) + }) + test('includes both tools', () => { const msg = buildQwenSessionUpdateMessage() // Realtime shape: flat {type, name, description, parameters} — NOT chat-completions {function:{...}} diff --git a/shared/src/voice.ts b/shared/src/voice.ts index 3099a905..0f3ac56c 100644 --- a/shared/src/voice.ts +++ b/shared/src/voice.ts @@ -5,165 +5,27 @@ * ensuring consistency between server-side auto-creation and client-side usage. */ +import { resolveGeminiLiveVoice, resolveQwenRealtimeVoice } from './voicePickerCatalog' +import { + VOICE_CHINESE_LANGUAGE_BLOCK, + composeVoiceAgentPrompt, + type VoicePromptLayerInput +} from './voicePromptLayers' + +export { VOICE_CHINESE_LANGUAGE_BLOCK } from './voicePromptLayers' + export const ELEVENLABS_API_BASE = 'https://api.elevenlabs.io/v1' export const VOICE_AGENT_NAME = 'Hapi Voice Assistant' -export const VOICE_SYSTEM_PROMPT = `# CRITICAL RULE - Tool Usage +const DEFAULT_COMPOSED_LAYERS: VoicePromptLayerInput = { + identity: '', + character: '', + legacySystemPrompt: '', + presetDeliverySnippet: '' +} -You MUST call the messageCodingAgent tool for ANY request related to coding, files, development, debugging, or tasks for the agent. Do NOT respond verbally to these requests — call the tool FIRST, then briefly confirm. This is your most important behavior. - -# Identity - -You are Hapi Voice Assistant. You bridge voice communication between users and their AI coding agents in the Hapi ecosystem. - -IMPORTANT: Never refer to yourself as Gemini, Google, or any underlying model or provider name. You are HAPI — always. - -You are friendly, proactive, and highly intelligent with a world-class engineering background. Your approach is warm, witty, and relaxed, balancing professionalism with an approachable vibe. - -# Environment Overview - -Hapi is a multi-agent development platform supporting: -- **Claude Code** - Anthropic's coding assistant (primary) -- **Codex** - OpenAI's coding agent -- **Gemini** - Google's coding agent - -Users control these agents through the Hapi web interface or Telegram Mini App. You serve as the voice interface to whichever agent is currently active. - -# How Context Updates Work - -You receive automatic context updates when: -- A session becomes focused (you see the full session history) -- The agent sends messages or uses tools -- Permission requests arrive -- The agent finishes working (ready event) - -These updates appear as system messages. You do NOT need to poll or ask for updates. Simply wait for them and summarize when relevant. - -# Tools - -## messageCodingAgent -Send user requests to the active coding agent. - -When to use: -- User says "ask Claude to..." or "have it..." -- Any coding, file, or development request -- User wants to continue a task - -Example: User says "refactor the auth module" -> call messageCodingAgent with the full request. - -## processPermissionRequest -Approve or deny pending permission requests. - -When to use: -- User says "yes", "allow", "go ahead", "approve" -- User says "no", "deny", "cancel", "stop" - -The decision parameter must be exactly "allow" or "deny". - -# Voice Output Guidelines - -## Summarization (Critical) -- NEVER read hashes, IDs, or paths character-by-character -- Say "session ending in ZAJ" not "c-m-i-a-b-c-1-2-3..." -- Say "file in the src folder" not the full path -- Summarize code changes at a high level -- Skip tool arguments unless specifically asked - -## TTS Formatting -- Use ellipses "..." for pauses -- Say "dot" for periods in URLs/paths -- Spell out acronyms: "API" becomes "A P I" -- Use normalized spoken language - -## Conversation Style -- Keep responses to 1-3 sentences typically -- Use brief affirmations: "got it", "sure thing" -- Occasional natural fillers: "so", "actually" -- Mirror user energy: terse replies for terse questions -- Lead with empathy for frustrated users - -# Behavioral Guidelines - -## Patience -After sending a message to the agent, WAIT SILENTLY. The agent may take 30+ seconds for complex tasks. Do NOT: -- Ask "are you still there?" -- Repeat the request -- Fill silence with chatter - -You will receive a context update when the agent responds or finishes. - -## Request Routing -- Direct address ("Assistant, explain...") -> Answer yourself -- Explicit delegation ("Have Claude...") -> Use messageCodingAgent -- Coding/file tasks -> Use messageCodingAgent -- General questions you can answer -> Answer yourself - -Do NOT second-guess what the agent can do. If in doubt, pass it through. - -## Proactive Updates -Speak proactively when: -- Permission is requested (inform user and ask for decision) -- Agent finishes a task (summarize results) -- Error occurs (explain clearly) -- Session status changes significantly - -Stay silent when: -- Agent is actively working -- No meaningful update to share - -# Common Scenarios - -## Permission Requests -When you see a permission request, immediately inform the user: -"Claude wants to run a bash command. Should I allow it?" -Then wait for their response and use processPermissionRequest. - -## Errors -If the agent reports an error: -- Summarize the error type -- Suggest what the user might do -- Do NOT read stack traces verbatim - -## Session Issues -If there is no active session: -- Tell the user to select or start a session in the app -- You cannot start sessions yourself - -## Long Operations -For builds, tests, or large file operations: -- Acknowledge the task was sent -- Wait silently for completion -- Summarize results when ready - -# Guardrails - -- Never read code line-by-line or provide inline code samples -- Never repeat the same information multiple ways in one response -- Treat garbled input as phonetic hints and ask for clarification -- Correct yourself immediately if you realize you made an error -- Keep conversations forward-moving with fresh insights -- Assume a technical software developer audience - -# First Interaction - -When the user speaks to you for the first time, begin your response with a brief greeting before addressing their request. If their first message is a coding request, greet briefly AND call the tool — do both.` - -/** - * Language blocks appended to VOICE_SYSTEM_PROMPT for Gemini/Qwen backends - * (ElevenLabs has its own language field). - * - * Always append one of these — silence causes models to drift to their training - * language (Chinese for Qwen, mixed for Gemini). - */ -export const VOICE_CHINESE_LANGUAGE_BLOCK = ` - -# Language - -IMPORTANT: Always respond in Chinese (Mandarin). Use natural spoken Chinese. -- Greet users in Chinese -- Summarize technical content in Chinese -- Use English only for proper nouns, tool names, and code identifiers -- Keep the same warm, concise conversational style in Chinese` +/** Bundled default composed prompt (fixtures + default identity/character). */ +export const VOICE_SYSTEM_PROMPT = composeVoiceAgentPrompt(DEFAULT_COMPOSED_LAYERS) /** When no language is selected: mirror the user's detected speech language. */ const VOICE_LANGUAGE_BLOCK_AUTO = ` @@ -300,9 +162,17 @@ export interface VoiceAgentConfig { agent?: { language?: boolean first_message?: boolean + prompt?: { + prompt?: boolean + } } tts?: { voice_id?: boolean + stability?: boolean + similarity_boost?: boolean + style?: boolean + speed?: boolean + use_speaker_boost?: boolean } } } @@ -344,10 +214,18 @@ export function buildVoiceAgentConfig(): VoiceAgentConfig { overrides: { conversation_config_override: { agent: { - language: true + language: true, + prompt: { + prompt: true + } }, tts: { - voice_id: true + voice_id: true, + stability: true, + similarity_boost: true, + style: true, + speed: true, + use_speaker_boost: true } } } @@ -362,6 +240,58 @@ export const QWEN_REALTIME_VOICE = 'Tina' export const DEFAULT_VOICE_BACKEND: VoiceBackendType = 'elevenlabs' +const VOICE_BACKEND_VALUES: readonly VoiceBackendType[] = [ + 'elevenlabs', + 'gemini-live', + 'qwen-realtime' +] as const + +export type VoiceBackendEnv = Record + +/** Backends whose API keys are present on the hub. */ +export function listConfiguredVoiceBackends(env: VoiceBackendEnv): VoiceBackendType[] { + const backends: VoiceBackendType[] = [] + if (env.ELEVENLABS_API_KEY?.trim()) { + backends.push('elevenlabs') + } + if (env.GEMINI_API_KEY?.trim() || env.GOOGLE_API_KEY?.trim()) { + backends.push('gemini-live') + } + if (env.DASHSCOPE_API_KEY?.trim() || env.QWEN_API_KEY?.trim()) { + backends.push('qwen-realtime') + } + return backends.length > 0 ? backends : [DEFAULT_VOICE_BACKEND] +} + +/** Hub default from VOICE_BACKEND when configured, else first available backend. */ +export function resolveHubVoiceBackend(env: VoiceBackendEnv): VoiceBackendType { + const configured = listConfiguredVoiceBackends(env) + const raw = env.VOICE_BACKEND + const fromEnv = VOICE_BACKEND_VALUES.includes(raw as VoiceBackendType) + ? (raw as VoiceBackendType) + : DEFAULT_VOICE_BACKEND + return configured.includes(fromEnv) ? fromEnv : (configured[0] ?? DEFAULT_VOICE_BACKEND) +} + +/** User preference wins when valid; otherwise hub default. */ +export function resolveEffectiveVoiceBackend( + configured: readonly VoiceBackendType[], + hubDefault: VoiceBackendType, + storedPreference: string | null | undefined +): VoiceBackendType { + if ( + storedPreference + && VOICE_BACKEND_VALUES.includes(storedPreference as VoiceBackendType) + && configured.includes(storedPreference as VoiceBackendType) + ) { + return storedPreference as VoiceBackendType + } + if (configured.includes(hubDefault)) { + return hubDefault + } + return configured[0] ?? hubDefault +} + export const GEMINI_LIVE_MODEL = 'gemini-2.5-flash-native-audio-latest' export const GEMINI_LIVE_VOICE = 'Aoede' @@ -418,11 +348,17 @@ export function buildGeminiLiveFunctionDeclarations(): GeminiLiveFunctionDeclara return VOICE_TOOLS.map(cloneVoiceToolDefinition) } -export function buildGeminiLiveConfig(language?: string): GeminiLiveConfig { - const systemInstruction = `${VOICE_SYSTEM_PROMPT}${buildVoiceLanguageBlock(language)}` +export function buildGeminiLiveConfig( + language?: string, + voiceName?: string, + systemInstruction?: string +): GeminiLiveConfig { + const systemInstructionText = systemInstruction?.trim() + ? systemInstruction + : `${VOICE_SYSTEM_PROMPT}${buildVoiceLanguageBlock(language)}` return { model: GEMINI_LIVE_MODEL, - systemInstruction, + systemInstruction: systemInstructionText, tools: [ { functionDeclarations: buildGeminiLiveFunctionDeclarations() @@ -433,8 +369,14 @@ export function buildGeminiLiveConfig(language?: string): GeminiLiveConfig { } /** Hub-owned initial session.update for Qwen Realtime (hub proxy). */ -export function buildQwenSessionUpdateMessage(language?: string): Record { - const instructions = `${VOICE_SYSTEM_PROMPT}${buildVoiceLanguageBlock(language)}` +export function buildQwenSessionUpdateMessage( + language?: string, + voiceName?: string, + systemInstruction?: string +): Record { + const instructions = systemInstruction?.trim() + ? systemInstruction + : `${VOICE_SYSTEM_PROMPT}${buildVoiceLanguageBlock(language)}` // Qwen Realtime uses the flat Realtime shape, not the chat-completions nested {function:{...}} shape. const tools = VOICE_TOOL_DEFINITIONS.map((td) => ({ type: 'function' as const, @@ -446,7 +388,7 @@ export function buildQwenSessionUpdateMessage(language?: string): Record } { - const liveConfig = buildGeminiLiveConfig(language) +export function buildGeminiLiveSetupMessage( + language?: string, + voiceName?: string, + systemInstruction?: string, + options?: { affectiveDialog?: boolean } +): { setup: Record } { + const liveConfig = buildGeminiLiveConfig(language, voiceName, systemInstruction) + const resolvedVoice = resolveGeminiLiveVoice(voiceName) return { setup: { model: `models/${liveConfig.model}`, generationConfig: { responseModalities: ['AUDIO'], + ...(options?.affectiveDialog ? { enableAffectiveDialog: true } : {}), speechConfig: { voiceConfig: { - prebuiltVoiceConfig: { voiceName: GEMINI_LIVE_VOICE } + prebuiltVoiceConfig: { voiceName: resolvedVoice } } } }, diff --git a/shared/src/voicePersonality.test.ts b/shared/src/voicePersonality.test.ts new file mode 100644 index 00000000..f66a22d1 --- /dev/null +++ b/shared/src/voicePersonality.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, test } from 'bun:test' +import { VOICE_SYSTEM_PROMPT } from './voice' +import { + ELEVENLABS_WEBRTC_CONTEXT_MAX_BYTES, + getDefaultVoiceSystemPrompt, + getVoicePersonalityPreset, + isDefaultVoicePersonality, + parseVoicePersonalityPreferences, + resolveComposedVoiceSystemPrompt, + resolveElevenLabsVoiceSettings, + truncateUtf8ByteLength, + utf8ByteLength +} from './voicePersonality' +import { VOICE_PLATFORM_FIXTURES } from './voicePromptLayers' + +describe('voicePersonality', () => { + test('parseVoicePersonalityPreferences returns defaults for invalid input', () => { + const prefs = parseVoicePersonalityPreferences(null) + expect(prefs.preset).toBe('balanced') + expect(prefs.identity).toBe('') + expect(prefs.character).toBe('') + }) + + test('migrates legacy customPrompt into character', () => { + const prefs = parseVoicePersonalityPreferences({ + customPrompt: 'Call me G.' + }) + expect(prefs.character).toBe('Call me G.') + }) + + test('migrates non-monolith systemPrompt into character', () => { + const prefs = parseVoicePersonalityPreferences({ + systemPrompt: 'You are a pirate. Arr.' + }) + expect(prefs.character).toBe('You are a pirate. Arr.') + expect(prefs.systemPrompt).toBe('') + }) + + test('composed prompt always includes platform fixtures', () => { + const prefs = parseVoicePersonalityPreferences({ + character: 'You are a pirate. Arr.' + }) + const { prompt } = resolveComposedVoiceSystemPrompt(prefs) + expect(prompt).toContain('messageCodingAgent') + expect(prompt).toContain(VOICE_PLATFORM_FIXTURES.slice(0, 40)) + expect(prompt).toContain('You are a pirate') + expect(prompt).toContain('Never refer to yourself as Gemini') + }) + + test('resolveComposedVoiceSystemPrompt does not embed session context', () => { + const prefs = parseVoicePersonalityPreferences({ character: 'Base.' }) + const { prompt } = resolveComposedVoiceSystemPrompt(prefs) + expect(prompt).not.toContain('[Current Context]') + expect(prompt).not.toContain('Working on auth.') + }) + + test('getDefaultVoiceSystemPrompt matches bundled VOICE_SYSTEM_PROMPT', () => { + const prefs = parseVoicePersonalityPreferences({}) + expect(getDefaultVoiceSystemPrompt()).toBe(VOICE_SYSTEM_PROMPT) + expect(isDefaultVoicePersonality(prefs)).toBe(true) + }) + + test('resolveElevenLabsVoiceSettings uses preset sliders unless custom', () => { + const prefs = parseVoicePersonalityPreferences({ + preset: 'custom', + elevenLabs: { stability: 0.42, similarity_boost: 0.8, style: 0.2, speed: 1.05, use_speaker_boost: true } + }) + expect(resolveElevenLabsVoiceSettings(prefs).stability).toBe(0.42) + }) + + test('defines all preset ids', () => { + expect(getVoicePersonalityPreset('calm').elevenLabs.speed).toBeLessThan(1) + }) + + test('truncateUtf8ByteLength respects byte budget', () => { + const text = truncateUtf8ByteLength('hello 🎙️ world', ELEVENLABS_WEBRTC_CONTEXT_MAX_BYTES) + expect(text).toBe('hello 🎙️ world') + const huge = truncateUtf8ByteLength('a'.repeat(50_000), 100) + expect(utf8ByteLength(huge)).toBeLessThanOrEqual(100) + }) +}) diff --git a/shared/src/voicePersonality.ts b/shared/src/voicePersonality.ts new file mode 100644 index 00000000..d479acb1 --- /dev/null +++ b/shared/src/voicePersonality.ts @@ -0,0 +1,456 @@ +/** + * Voice personality / delivery controls shared by Settings UI and realtime sessions. + * @see docs/plans/voice-personality-config.md + */ + +import { VOICE_SYSTEM_PROMPT, buildVoiceLanguageBlock } from './voice' +import { + composeVoiceAgentPrompt, + getVoicePlatformFixturesPreview, + type VoicePromptLayerInput +} from './voicePromptLayers' + +export type VoicePersonalityPresetId = + | 'balanced' + | 'warm' + | 'calm' + | 'direct' + | 'custom' + +export type VoiceBackendKind = 'elevenlabs' | 'gemini-live' | 'qwen-realtime' + +/** ElevenLabs ConvAI runtime TTS overrides (snake_case for API). */ +export interface ElevenLabsVoiceSettings { + stability: number + similarity_boost: number + style: number + speed: number + use_speaker_boost: boolean +} + +export interface GeminiVoiceOptions { + affective_dialog: boolean +} + +export type ResponseLengthOption = 'brief' | 'balanced' | 'detailed' + +export const RESPONSE_LENGTH_OPTIONS: readonly ResponseLengthOption[] = ['brief', 'balanced', 'detailed'] + +const RESPONSE_LENGTH_INSTRUCTIONS: Record = { + brief: '\n\n# Response length\n\nKeep all voice responses to 1–2 sentences. Be concise and direct.', + balanced: '', + detailed: '\n\n# Response length\n\nGive thorough responses when the topic warrants depth. Do not truncate if completeness is needed.', +} + +export function getResponseLengthInstruction(length: ResponseLengthOption): string { + return RESPONSE_LENGTH_INSTRUCTIONS[length] +} + +export interface VoicePersonalityPreferences { + preset: VoicePersonalityPresetId + /** Who the assistant is (rebrand / overseer). Empty = bundled default identity. */ + identity: string + /** Delivery, tone, preset overlays. Empty = default character + preset snippet. */ + character: string + /** + * @deprecated Migrated to identity/character on parse. Full monolith kept only when it + * still contains platform fixtures (legacy override). + */ + systemPrompt: string + /** @deprecated Migrated into character on parse. */ + customPrompt: string + /** How long responses should be. 'balanced' is the default (no extra instruction). */ + responseLength: ResponseLengthOption + elevenLabs: ElevenLabsVoiceSettings + gemini: GeminiVoiceOptions +} + +export const VOICE_IDENTITY_MAX_LENGTH = 8_000 +export const VOICE_CHARACTER_MAX_LENGTH = 16_000 + +/** Max stored prompt size (localStorage + Gemini WS query param budget). */ +export const VOICE_SYSTEM_PROMPT_MAX_LENGTH = 48_000 + +/** ElevenLabs ConvAI WebRTC data channel limit per message (bytes). */ +export const ELEVENLABS_WEBRTC_MAX_MESSAGE_BYTES = 65_535 + +/** Budget for session bootstrap in startSession dynamicVariables. */ +export const ELEVENLABS_WEBRTC_CONTEXT_MAX_BYTES = 4_000 + +/** Budget per deferred context chunk (streamed after connect). */ +export const VOICE_CONTEXT_STREAM_CHUNK_MAX_BYTES = 8_000 + +/** Budget for composed system prompt in agent overrides (+ tools JSON uses the rest). */ +export const ELEVENLABS_WEBRTC_PROMPT_MAX_BYTES = 12_000 + +/** Trim UTF-8 text to a byte budget (for WebRTC / query-param limits). */ +export function truncateUtf8ByteLength(text: string, maxBytes: number): string { + if (maxBytes <= 0) return '' + const encoder = new TextEncoder() + if (encoder.encode(text).length <= maxBytes) return text + + const suffix = '\n\n[…truncated for voice transport…]' + const suffixBytes = encoder.encode(suffix).length + const budget = Math.max(0, maxBytes - suffixBytes) + + let lo = 0 + let hi = text.length + while (lo < hi) { + const mid = Math.ceil((lo + hi) / 2) + if (encoder.encode(text.slice(0, mid)).length <= budget) lo = mid + else hi = mid - 1 + } + return text.slice(0, lo) + suffix +} + +export function utf8ByteLength(text: string): number { + return new TextEncoder().encode(text).length +} + +export const VOICE_PERSONALITY_STORAGE_KEY = 'hapi-voice-personality' +export const VOICE_CONTEXT_NOTICE_STORAGE_KEY = 'hapi-voice-context-notice' + +export const DEFAULT_ELEVENLABS_VOICE_SETTINGS: ElevenLabsVoiceSettings = { + stability: 0.5, + similarity_boost: 0.75, + style: 0.1, + speed: 1.0, + use_speaker_boost: false +} + +export const DEFAULT_GEMINI_VOICE_OPTIONS: GeminiVoiceOptions = { + affective_dialog: true +} + +export const DEFAULT_VOICE_PERSONALITY: VoicePersonalityPreferences = { + preset: 'balanced', + identity: '', + character: '', + systemPrompt: '', + customPrompt: '', + responseLength: 'balanced', + elevenLabs: { ...DEFAULT_ELEVENLABS_VOICE_SETTINGS }, + gemini: { ...DEFAULT_GEMINI_VOICE_OPTIONS } +} + +export function voicePromptLayersFromPrefs(prefs: VoicePersonalityPreferences): VoicePromptLayerInput { + return { + identity: prefs.identity, + character: prefs.character, + legacySystemPrompt: prefs.systemPrompt, + presetDeliverySnippet: getPresetDeliverySnippet(prefs.preset) + } +} + +/** Bundled composed prompt for the session language (editable copy baseline in Settings). */ +export function getDefaultVoiceSystemPrompt(language?: string): string { + return composeVoiceAgentPrompt(voicePromptLayersFromPrefs(DEFAULT_VOICE_PERSONALITY)) + + (language ? buildVoiceLanguageBlock(language) : '') +} + +export interface VoicePromptComposeResult { + prompt: string + truncated: boolean + wireBytes: number +} + +/** Effective composed system prompt for a voice session (all backends). Session context is never embedded here. */ +export function resolveComposedVoiceSystemPrompt( + prefs: VoicePersonalityPreferences, + options?: { + language?: string + backend?: VoiceBackendKind + maxWireBytes?: number + } +): VoicePromptComposeResult { + const lang = options?.language + const isElevenLabs = options?.backend === 'elevenlabs' + // ElevenLabs has its own language field; Gemini/Qwen need the block in the prompt. + // For Gemini/Qwen, always include it: undefined → auto-detect block, code → explicit block. + let prompt = composeVoiceAgentPrompt(voicePromptLayersFromPrefs(prefs)) + + (isElevenLabs ? (lang ? buildVoiceLanguageBlock(lang) : '') : buildVoiceLanguageBlock(lang)) + prompt += getResponseLengthInstruction(prefs.responseLength ?? 'balanced') + + const maxBytes = options?.maxWireBytes + ?? (options?.backend === 'elevenlabs' ? ELEVENLABS_WEBRTC_PROMPT_MAX_BYTES : undefined) + + let truncated = false + if (maxBytes && utf8ByteLength(prompt) > maxBytes) { + prompt = truncateUtf8ByteLength(prompt, maxBytes) + truncated = true + } + + return { prompt, truncated, wireBytes: utf8ByteLength(prompt) } +} + +/** @deprecated Use resolveComposedVoiceSystemPrompt().prompt */ +export function resolveVoiceSystemPrompt( + prefs: VoicePersonalityPreferences, + options?: { language?: string; initialContext?: string } +): string { + void options?.initialContext + return resolveComposedVoiceSystemPrompt(prefs, { language: options?.language }).prompt +} + +export function isDefaultVoicePersonality(prefs: VoicePersonalityPreferences): boolean { + return (prefs.preset ?? 'balanced') === 'balanced' + && !prefs.identity.trim() + && !prefs.character.trim() + && !prefs.systemPrompt.trim() + && (prefs.responseLength ?? 'balanced') === 'balanced' +} + +/** @deprecated Use isDefaultVoicePersonality */ +export function isDefaultVoiceSystemPrompt(prefs: VoicePersonalityPreferences, _language?: string): boolean { + return isDefaultVoicePersonality(prefs) +} + +export { + DEFAULT_VOICE_CHARACTER, + DEFAULT_VOICE_IDENTITY, + getVoicePlatformFixturesPreview +} from './voicePromptLayers' + +export interface VoicePersonalityPresetDefinition { + id: VoicePersonalityPresetId + labelKey: string + descriptionKey: string + promptAddition: string + elevenLabs: ElevenLabsVoiceSettings +} + +export const VOICE_PERSONALITY_PRESETS: readonly VoicePersonalityPresetDefinition[] = [ + { + id: 'balanced', + labelKey: 'settings.voice.character.preset.balanced', + descriptionKey: 'settings.voice.character.preset.balancedHint', + promptAddition: '', + elevenLabs: { + stability: 0.5, + similarity_boost: 0.75, + style: 0.1, + speed: 1.0, + use_speaker_boost: false + } + }, + { + id: 'warm', + labelKey: 'settings.voice.character.preset.warm', + descriptionKey: 'settings.voice.character.preset.warmHint', + promptAddition: `Speak with natural warmth and personality. Use audio cues where they fit: +[chuckles] or [laughs] when something is genuinely funny, +[excited] when sharing something interesting, +[sighs] for wistful or empathetic moments, +[warm tone] as your default register. +One or two tags per response maximum — never performed, always earned. +Speak at a relaxed pace. Pause before considered answers.`, + elevenLabs: { + stability: 0.35, + similarity_boost: 0.75, + style: 0.3, + speed: 0.97, + use_speaker_boost: true + } + }, + { + id: 'calm', + labelKey: 'settings.voice.character.preset.calm', + descriptionKey: 'settings.voice.character.preset.calmHint', + promptAddition: `Speak slowly and deliberately. [pauses] before important points. +Use [sighs] and [hesitates] naturally. Never rush. +Keep energy low and steady.`, + elevenLabs: { + stability: 0.75, + similarity_boost: 0.75, + style: 0.0, + speed: 0.93, + use_speaker_boost: false + } + }, + { + id: 'direct', + labelKey: 'settings.voice.character.preset.direct', + descriptionKey: 'settings.voice.character.preset.directHint', + promptAddition: `Be concise. Skip pleasantries unless asked. No filler phrases. +Short answers. Confirm before elaborating.`, + elevenLabs: { + stability: 0.65, + similarity_boost: 0.75, + style: 0.05, + speed: 1.08, + use_speaker_boost: false + } + }, + { + id: 'custom', + labelKey: 'settings.voice.character.preset.custom', + descriptionKey: 'settings.voice.character.preset.customHint', + promptAddition: '', + elevenLabs: { ...DEFAULT_ELEVENLABS_VOICE_SETTINGS } + } +] as const + +export function getVoicePersonalityPreset( + id: VoicePersonalityPresetId +): VoicePersonalityPresetDefinition { + return VOICE_PERSONALITY_PRESETS.find((p) => p.id === id) ?? VOICE_PERSONALITY_PRESETS[0] +} + +export function clamp01(value: number): number { + if (Number.isNaN(value)) return 0 + return Math.min(1, Math.max(0, value)) +} + +export function clampSpeed(value: number): number { + if (Number.isNaN(value)) return 1 + return Math.min(1.2, Math.max(0.7, value)) +} + +function isBundledMonolith(systemPrompt: string): boolean { + const trimmed = systemPrompt.trim() + if (!trimmed) return false + return trimmed === VOICE_SYSTEM_PROMPT || trimmed.startsWith('# CRITICAL RULE') +} + +export function parseVoicePersonalityPreferences(raw: unknown): VoicePersonalityPreferences { + if (!raw || typeof raw !== 'object') { + return structuredClone(DEFAULT_VOICE_PERSONALITY) + } + const record = raw as Record + const presetRaw = record.preset + const preset = VOICE_PERSONALITY_PRESETS.some((p) => p.id === presetRaw) + ? (presetRaw as VoicePersonalityPresetId) + : 'balanced' + + const elRaw = record.elevenLabs + const el = elRaw && typeof elRaw === 'object' + ? (elRaw as Record) + : {} + + const geminiRaw = record.gemini + const gemini = geminiRaw && typeof geminiRaw === 'object' + ? (geminiRaw as Record) + : {} + + let identity = typeof record.identity === 'string' ? record.identity : '' + let character = typeof record.character === 'string' ? record.character : '' + let systemPrompt = typeof record.systemPrompt === 'string' ? record.systemPrompt : '' + + const legacyNotes = typeof record.customPrompt === 'string' ? record.customPrompt.trim() : '' + if (legacyNotes && !character.trim()) { + character = legacyNotes + } + + if (!identity.trim() && !character.trim() && systemPrompt.trim()) { + if (isBundledMonolith(systemPrompt)) { + systemPrompt = '' + } else if (systemPrompt.includes('# CRITICAL RULE')) { + // Legacy full override — keep in systemPrompt for composeVoiceAgentPrompt + } else { + character = systemPrompt + systemPrompt = '' + } + } + + const responseLengthRaw = record.responseLength + const responseLength: ResponseLengthOption = RESPONSE_LENGTH_OPTIONS.includes(responseLengthRaw as ResponseLengthOption) + ? (responseLengthRaw as ResponseLengthOption) + : 'balanced' + + return { + preset, + identity: identity.slice(0, VOICE_IDENTITY_MAX_LENGTH), + character: character.slice(0, VOICE_CHARACTER_MAX_LENGTH), + systemPrompt: systemPrompt.slice(0, VOICE_SYSTEM_PROMPT_MAX_LENGTH), + customPrompt: '', + responseLength, + elevenLabs: { + stability: clamp01(Number(el.stability ?? DEFAULT_ELEVENLABS_VOICE_SETTINGS.stability)), + similarity_boost: clamp01(Number(el.similarity_boost ?? el.similarityBoost ?? DEFAULT_ELEVENLABS_VOICE_SETTINGS.similarity_boost)), + style: clamp01(Number(el.style ?? DEFAULT_ELEVENLABS_VOICE_SETTINGS.style)), + speed: clampSpeed(Number(el.speed ?? DEFAULT_ELEVENLABS_VOICE_SETTINGS.speed)), + use_speaker_boost: Boolean(el.use_speaker_boost ?? el.useSpeakerBoost ?? DEFAULT_ELEVENLABS_VOICE_SETTINGS.use_speaker_boost) + }, + gemini: { + affective_dialog: gemini.affective_dialog !== false && gemini.affectiveDialog !== false + } + } +} + +/** Optional delivery snippet from the selected preset (merged into character layer when character empty). */ +export function getPresetDeliverySnippet(presetId: VoicePersonalityPresetId): string { + return getVoicePersonalityPreset(presetId).promptAddition.trim() +} + +/** + * @deprecated Use resolveComposedVoiceSystemPrompt. Preset text is merged in the character layer. + */ +export function buildVoicePersonalityPromptAddition(prefs: VoicePersonalityPreferences): string { + return getPresetDeliverySnippet(prefs.preset) +} + +/** Effective ElevenLabs sliders — custom preset uses stored values; others use preset defaults unless preset is custom. */ +export function resolveElevenLabsVoiceSettings(prefs: VoicePersonalityPreferences): ElevenLabsVoiceSettings { + if (prefs.preset === 'custom') { + return { ...prefs.elevenLabs } + } + return { ...getVoicePersonalityPreset(prefs.preset).elevenLabs } +} + +export type VoiceControlLeverId = + | 'character_preset' + | 'voice_identity' + | 'voice_character' + | 'voice_fixtures' + | 'speaking_rate' + | 'expressiveness' + | 'stability' + | 'similarity_boost' + | 'speaker_boost' + | 'affective_dialog' + +export interface VoiceControlLever { + id: VoiceControlLeverId + backends: VoiceBackendKind[] +} + +/** Levers exposed in Settings → Advanced voice (common vs backend-specific). */ +export const VOICE_CONTROL_LEVERS: readonly VoiceControlLever[] = [ + { id: 'character_preset', backends: ['elevenlabs', 'gemini-live', 'qwen-realtime'] }, + { id: 'voice_identity', backends: ['elevenlabs', 'gemini-live', 'qwen-realtime'] }, + { id: 'voice_character', backends: ['elevenlabs', 'gemini-live', 'qwen-realtime'] }, + { id: 'voice_fixtures', backends: ['elevenlabs', 'gemini-live', 'qwen-realtime'] }, + { id: 'speaking_rate', backends: ['elevenlabs', 'gemini-live', 'qwen-realtime'] }, + { id: 'expressiveness', backends: ['elevenlabs', 'gemini-live', 'qwen-realtime'] }, + { id: 'stability', backends: ['elevenlabs'] }, + { id: 'similarity_boost', backends: ['elevenlabs'] }, + { id: 'speaker_boost', backends: ['elevenlabs'] }, + { id: 'affective_dialog', backends: ['gemini-live'] } +] + +export function leverAppliesToBackend(lever: VoiceControlLever, backend: VoiceBackendKind): boolean { + return lever.backends.includes(backend) +} + +export function getVoiceWireBudgetHint(backend: VoiceBackendKind): { + storageMaxChars: number + wireNoteKey: string +} { + switch (backend) { + case 'elevenlabs': + return { + storageMaxChars: VOICE_SYSTEM_PROMPT_MAX_LENGTH, + wireNoteKey: 'settings.voice.wireBudget.elevenlabs' + } + case 'gemini-live': + return { + storageMaxChars: VOICE_SYSTEM_PROMPT_MAX_LENGTH, + wireNoteKey: 'settings.voice.wireBudget.gemini' + } + case 'qwen-realtime': + return { + storageMaxChars: VOICE_SYSTEM_PROMPT_MAX_LENGTH, + wireNoteKey: 'settings.voice.wireBudget.qwen' + } + } +} diff --git a/shared/src/voicePickerCatalog.ts b/shared/src/voicePickerCatalog.ts new file mode 100644 index 00000000..7a273373 --- /dev/null +++ b/shared/src/voicePickerCatalog.ts @@ -0,0 +1,68 @@ +/** + * Static voice catalogs for Settings picker (Gemini Live, Qwen Realtime). + * ElevenLabs voices remain dynamic via GET /api/voice/voices (#690). + * + * @see https://github.com/tiann/hapi/issues/742 + */ + +export type VoicePickerOption = { + id: string + label: string + description?: string +} + +/** Prebuilt voices documented for Gemini Live BidiGenerateContent. */ +export const GEMINI_LIVE_VOICE_OPTIONS: readonly VoicePickerOption[] = [ + { id: 'Puck', label: 'Puck', description: 'Conversational, friendly' }, + { id: 'Charon', label: 'Charon', description: 'Deep, authoritative' }, + { id: 'Kore', label: 'Kore', description: 'Neutral, professional' }, + { id: 'Fenrir', label: 'Fenrir', description: 'Warm, approachable' }, + { id: 'Aoede', label: 'Aoede', description: 'Default' } +] as const + +/** English-accessible Qwen Realtime voices (expand after DashScope verification). */ +export const QWEN_REALTIME_VOICE_OPTIONS: readonly VoicePickerOption[] = [ + { id: 'Tina', label: 'Tina', description: 'Default' }, + { id: 'Cherry', label: 'Cherry' }, + { id: 'Mia', label: 'Mia' }, + { id: 'Chelsie', label: 'Chelsie' }, + { id: 'Serena', label: 'Serena' }, + { id: 'Ethan', label: 'Ethan' } +] as const + +export const VOICE_PICKER_STORAGE_KEYS = { + elevenlabs: 'hapi-voice-elevenlabs', + 'gemini-live': 'hapi-voice-gemini', + 'qwen-realtime': 'hapi-voice-qwen' +} as const + +/** Legacy ElevenLabs key from #690 — read for migration. */ +export const LEGACY_ELEVENLABS_VOICE_STORAGE_KEY = 'hapi-voice-id' + +/** User-selected voice backend when hub has more than one configured. */ +export const VOICE_BACKEND_PREFERENCE_STORAGE_KEY = 'hapi-voice-backend' + +export const VOICE_BACKEND_LABELS = { + elevenlabs: 'ElevenLabs', + 'gemini-live': 'Gemini Live', + 'qwen-realtime': 'Qwen Realtime' +} as const + +const geminiVoiceIds = new Set(GEMINI_LIVE_VOICE_OPTIONS.map((v) => v.id)) +const qwenVoiceIds = new Set(QWEN_REALTIME_VOICE_OPTIONS.map((v) => v.id)) + +/** Valid Gemini Live prebuilt voice name, or default (Aoede). */ +export function resolveGeminiLiveVoice(voiceName?: string | null): string { + if (voiceName && geminiVoiceIds.has(voiceName)) { + return voiceName + } + return GEMINI_LIVE_VOICE_OPTIONS.find((v) => v.id === 'Aoede')?.id ?? GEMINI_LIVE_VOICE_OPTIONS[0].id +} + +/** Valid Qwen Realtime voice id, or hub default (Tina — matches QWEN_REALTIME_VOICE on qwen3.5-omni-flash-realtime). */ +export function resolveQwenRealtimeVoice(voiceName?: string | null): string { + if (voiceName && qwenVoiceIds.has(voiceName)) { + return voiceName + } + return QWEN_REALTIME_VOICE_OPTIONS.find((v) => v.id === 'Tina')?.id ?? QWEN_REALTIME_VOICE_OPTIONS[0].id +} diff --git a/shared/src/voicePromptLayers.ts b/shared/src/voicePromptLayers.ts new file mode 100644 index 00000000..a21b9435 --- /dev/null +++ b/shared/src/voicePromptLayers.ts @@ -0,0 +1,213 @@ +/** + * Layered voice system prompt: platform fixtures (ship with repo) + editable identity/character. + * All voice backends compose the same layers at runtime. + */ + +/** Appended for zh sessions on Gemini/Qwen (no separate language field). */ +export const VOICE_CHINESE_LANGUAGE_BLOCK = ` + +# Language + +IMPORTANT: Always respond in Chinese (Mandarin). Use natural spoken Chinese. +- Greet users in Chinese +- Summarize technical content in Chinese +- Use English only for proper nouns, tool names, and code identifiers +- Keep the same warm, concise conversational style in Chinese` + +/** Tool contracts, context plumbing, output rules, routing — not user-editable in Settings. */ +export const VOICE_PLATFORM_FIXTURES = `# CRITICAL RULE - Tool Usage + +You MUST call the messageCodingAgent tool for ANY request related to coding, files, development, debugging, or tasks for the agent. Do NOT respond verbally to these requests — call the tool FIRST, then briefly confirm. This is your most important behavior. + +# Environment Overview + +Hapi is a multi-agent development platform supporting: +- **Claude Code** - Anthropic's coding assistant (primary) +- **Codex** - OpenAI's coding agent +- **Gemini** - Google's coding agent + +Users control these agents through the Hapi web interface or Telegram Mini App. You serve as the voice interface to whichever agent is currently active in the current session. + +# How Context Updates Work + +You receive automatic context updates when: +- A session becomes focused (you see the full session history) +- The agent sends messages or uses tools +- Permission requests arrive +- The agent finishes working (ready event) + +These updates appear as system messages. You do NOT need to poll or ask for updates. Simply wait for them and summarize when relevant. + +# Tools + +## messageCodingAgent +Send user requests to the active coding agent. + +When to use: +- User says "ask Claude to..." or "have it..." +- Any coding, file, or development request +- User wants to continue a task + +Example: User says "refactor the auth module" -> call messageCodingAgent with the full request. + +## processPermissionRequest +Approve or deny pending permission requests. + +When to use: +- User says "yes", "allow", "go ahead", "approve" +- User says "no", "deny", "cancel", "stop" + +The decision parameter must be exactly "allow" or "deny". + +# Voice Output Guidelines + +## Summarization (Critical) +- NEVER read hashes, IDs, or paths character-by-character +- Say "session ending in ZAJ" not "c-m-i-a-b-c-1-2-3..." +- Say "file in the src folder" not the full path +- Summarize code changes at a high level +- Skip tool arguments unless specifically asked + +## TTS Formatting +- Use ellipses "..." for pauses +- Say "dot" for periods in URLs/paths +- Spell out acronyms: "API" becomes "A P I" +- Use normalized spoken language + +## Conversation Style +- Keep responses to 1-3 sentences typically +- Use brief affirmations: "got it", "sure thing" +- Occasional natural fillers: "so", "actually" +- Mirror user energy: terse replies for terse questions +- Lead with empathy for frustrated users + +# Behavioral Guidelines + +## Patience +After sending a message to the agent, WAIT SILENTLY. The agent may take 30+ seconds for complex tasks. Do NOT: +- Ask "are you still there?" +- Repeat the request +- Fill silence with chatter + +You will receive a context update when the agent responds or finishes. + +## Request Routing +- Direct address ("Assistant, explain...") -> Answer yourself +- Explicit delegation ("Have Claude...") -> Use messageCodingAgent +- Coding/file tasks -> Use messageCodingAgent +- General questions you can answer -> Answer yourself + +Do NOT second-guess what the agent can do. If in doubt, pass it through. + +## Proactive Updates +Speak proactively when: +- Permission is requested (inform user and ask for decision) +- Agent finishes a task (summarize results) +- Error occurs (explain clearly) +- Session status changes significantly + +Stay silent when: +- Agent is actively working +- No meaningful update to share + +# Common Scenarios + +## Permission Requests +When you see a permission request, immediately inform the user: +"Claude wants to run a bash command. Should I allow it?" +Then wait for their response and use processPermissionRequest. + +## Errors +If the agent reports an error: +- Summarize the error type +- Suggest what the user might do +- Do NOT read stack traces verbatim + +## Session Issues +If there is no active session: +- Tell the user to select or start a session in the app +- You cannot start sessions yourself + +## Long Operations +For builds, tests, or large file operations: +- Acknowledge the task was sent +- Wait silently for completion +- Summarize results when ready + +# Guardrails + +- Never read code line-by-line or provide inline code samples +- Never repeat the same information multiple ways in one response +- Treat garbled input as phonetic hints and ask for clarification +- Correct yourself immediately if you realize you made an error +- Keep conversations forward-moving with fresh insights +- Assume a technical software developer audience + +# First Interaction + +When the user speaks to you for the first time, begin your response with a brief greeting before addressing their request. If their first message is a coding request, greet briefly AND call the tool — do both.` + +/** Provider/model guardrails — ship with repo; separate from rebrandable identity. */ +export const VOICE_PROVIDER_GUARDRAILS = `# Provider guardrails + +IMPORTANT: Never refer to yourself as Gemini, Google, Claude, OpenAI, Qwen, ElevenLabs, or any underlying model or provider name. You are the user's voice assistant for this workspace — always.` + +/** Default persona when the operator has not set a custom identity. */ +export const DEFAULT_VOICE_IDENTITY = `# Identity + +You are the voice assistant for this workspace. HAPI is the application the user employs to manage coding agents and sessions — it is not your name unless they configure one below. + +You bridge voice between the user and whichever coding agent is active in the current session.` + +/** Default delivery / tone when character layer is empty. */ +export const DEFAULT_VOICE_CHARACTER = `You are friendly, proactive, and highly intelligent with a world-class engineering background. Your approach is warm, witty, and relaxed, balancing professionalism with an approachable vibe.` + +export interface VoicePromptLayerInput { + identity: string + character: string + /** @deprecated Legacy full prompt override when it still contains platform fixtures. */ + legacySystemPrompt: string + presetDeliverySnippet: string +} + +export function composeVoiceAgentPrompt( + layers: VoicePromptLayerInput, + options?: { language?: 'zh' | undefined } +): string { + const legacy = layers.legacySystemPrompt.trim() + if (legacy && legacy.includes('# CRITICAL RULE') && !layers.identity.trim() && !layers.character.trim()) { + return options?.language === 'zh' + ? `${legacy}${VOICE_CHINESE_LANGUAGE_BLOCK}` + : legacy + } + + const parts: string[] = [ + VOICE_PLATFORM_FIXTURES, + VOICE_PROVIDER_GUARDRAILS + ] + + const identity = layers.identity.trim() || DEFAULT_VOICE_IDENTITY + parts.push(identity) + + let character = layers.character.trim() + if (!character) { + const snippet = layers.presetDeliverySnippet.trim() + character = snippet + ? `${DEFAULT_VOICE_CHARACTER}\n\n${snippet}` + : DEFAULT_VOICE_CHARACTER + } + parts.push(character) + + let prompt = parts.join('\n\n') + if (options?.language === 'zh') { + prompt += VOICE_CHINESE_LANGUAGE_BLOCK + } + return prompt +} + +/** Short preview for Settings (fixtures are read-only). */ +export function getVoicePlatformFixturesPreview(maxChars = 600): string { + const text = VOICE_PLATFORM_FIXTURES + if (text.length <= maxChars) return text + return `${text.slice(0, maxChars)}\n\n[…]` +} diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 7b0131e0..a29678a1 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -625,7 +625,7 @@ export class ApiClient { return this.getToken ? this.getToken() : this.token } - async fetchVoiceBackend(): Promise<{ backend: string }> { + async fetchVoiceBackend(): Promise<{ backend: string; backends: string[] }> { return await this.request('/api/voice/backend') } diff --git a/web/src/api/voice.ts b/web/src/api/voice.ts index df682812..9df55841 100644 --- a/web/src/api/voice.ts +++ b/web/src/api/voice.ts @@ -57,6 +57,8 @@ export interface VoiceInfo { name: string previewUrl: string category: string + /** Static-catalog hint (Gemini/Qwen); ElevenLabs uses API name only. */ + description?: string } export async function fetchVoices(api: ApiClient): Promise { @@ -202,7 +204,10 @@ export async function fetchQwenToken(api: ApiClient): Promise } export interface VoiceBackendResponse { + /** Hub default (VOICE_BACKEND env, validated against configured backends). */ backend: VoiceBackendType + /** Backends with API keys configured on the hub. */ + backends: VoiceBackendType[] } export interface GeminiTokenResponse { @@ -217,13 +222,22 @@ export interface GeminiTokenResponse { * Discover which voice backend the hub is configured to use. * Throws on network/server error or unrecognised backend value — callers must handle failures explicitly. */ +function isVoiceBackendType(value: string): value is VoiceBackendType { + return value === 'elevenlabs' || value === 'gemini-live' || value === 'qwen-realtime' +} + export async function fetchVoiceBackend(api: ApiClient): Promise { const result = await api.fetchVoiceBackend() const { backend } = result - if (backend === 'elevenlabs' || backend === 'gemini-live' || backend === 'qwen-realtime') { - return { backend } + if (!isVoiceBackendType(backend)) { + throw new Error(`Unrecognised voice backend: ${backend}`) } - throw new Error(`Unrecognised voice backend: ${backend}`) + const rawBackends = Array.isArray(result.backends) ? result.backends : [backend] + const backends = rawBackends.filter(isVoiceBackendType) + if (backends.length === 0) { + backends.push(backend) + } + return { backend, backends } } /** diff --git a/web/src/components/settings/VoiceAdvancedControls.tsx b/web/src/components/settings/VoiceAdvancedControls.tsx new file mode 100644 index 00000000..4f67aa2c --- /dev/null +++ b/web/src/components/settings/VoiceAdvancedControls.tsx @@ -0,0 +1,374 @@ +import { useState } from 'react' +import { + DEFAULT_VOICE_CHARACTER, + DEFAULT_VOICE_IDENTITY, + ELEVENLABS_WEBRTC_MAX_MESSAGE_BYTES, + ELEVENLABS_WEBRTC_PROMPT_MAX_BYTES, + VOICE_CHARACTER_MAX_LENGTH, + VOICE_IDENTITY_MAX_LENGTH, + VOICE_PERSONALITY_PRESETS, + RESPONSE_LENGTH_OPTIONS, + getPresetDeliverySnippet, + getVoicePlatformFixturesPreview, + getVoicePersonalityPreset, + getVoiceWireBudgetHint, + isDefaultVoicePersonality, + resolveComposedVoiceSystemPrompt, + type VoiceBackendKind, + type VoicePersonalityPresetId, + type ResponseLengthOption +} from '@hapi/protocol/voice-personality' +import { readVoiceContextNotice } from '@/lib/voiceContextStream' +import { useVoicePersonality } from '@/hooks/useVoicePersonality' +import { useMemo } from 'react' + +type Translate = (key: string) => string + +function ChevronDownIcon(props: { className?: string }) { + return ( + + + + ) +} + +function VoiceSlider(props: { + label: string + hint?: string + value: number + min: number + max: number + step: number + onChange: (value: number) => void + formatValue?: (value: number) => string +}) { + const display = props.formatValue ? props.formatValue(props.value) : props.value.toFixed(2) + return ( + + ) +} + +/** "How it behaves" — response length selector. */ +export function VoiceRespondsControls(props: { + t: Translate + voiceBackend?: VoiceBackendKind | null +}) { + const { prefs, setResponseLength } = useVoicePersonality() + const currentResponseLength: ResponseLengthOption = prefs.responseLength ?? 'balanced' + + return ( +
+

{props.t('settings.voice.responseLength.label')}

+
+ {RESPONSE_LENGTH_OPTIONS.map((opt) => ( + + ))} +
+

+ {props.t(`settings.voice.responseLength.${currentResponseLength}.hint`)} +

+
+ ) +} + +/** "Persona & instructions" — identity, character, and speaking style preset. */ +export function VoicePersonaControls(props: { + t: Translate + voiceBackend?: VoiceBackendKind | null +}) { + const { + prefs, + setPreset, + setIdentity, + setCharacter, + resetIdentity, + resetCharacter, + resetVoicePersonalityLayers, + appendPresetDeliveryToCharacter, + } = useVoicePersonality() + const [identityOpen, setIdentityOpen] = useState(false) + const [characterOpen, setCharacterOpen] = useState(false) + const [deliveryOpen, setDeliveryOpen] = useState(false) + + const identityEditor = prefs.identity.trim() || DEFAULT_VOICE_IDENTITY + const characterEditor = prefs.character.trim() || DEFAULT_VOICE_CHARACTER + const usingDefaults = isDefaultVoicePersonality(prefs) + const presetSnippet = getPresetDeliverySnippet(prefs.preset) + + return ( +
+ {/* Identity */} + + {identityOpen && ( +
+

{props.t('settings.voice.identity.hint')}

+