Files
hapi/shared/src/voice.gemini.test.ts
T
a812a51dd7 feat(voice): backend voice picker + advanced controls behind disclosure (#742) (#743)
* feat(voice): voice personality, picker catalog, and prompt layer foundation

- voicePickerCatalog.ts: per-backend voice lists for Gemini and Qwen with
  resolve helpers (resolveGeminiLiveVoice, resolveQwenRealtimeVoice)
- voicePersonality.ts: VoicePersonalityPreferences schema, presets, composed
  system prompt with identity/character/response-length layers
- voicePromptLayers.ts: buildResolvedVoiceSystemPrompt, preset delivery snippets
- voiceSystemPromptParam.ts: hub-side base64url decode for ?systemPrompt=
- voicePickerPreferences.ts, voicePersonalitySession.ts: browser-side encode,
  decode, and storage helpers
- useVoicePersonality: React hook for preferences persistence

via [HAPI](https://hapi.run)

Co-Authored-By: HAPI <noreply@hapi.run>

* fix(voice): preset delivery included when non-balanced preset selected; restore test typecheck

- isDefaultVoicePersonality: add preset check so warm/calm/direct presets
  trigger the delivery snippet instead of being treated as default
- web/tsconfig.json: remove test file exclusion from typecheck (restoring
  strict coverage of test code); fix resulting type error in mock declaration

via [HAPI](https://hapi.run)

Co-Authored-By: HAPI <noreply@hapi.run>

* fix(voice): include use_speaker_boost in ElevenLabs TTS override payload

The checkbox persisted the pref but ttsDiffersFromDefault and
buildElevenLabsTtsOverride both omitted it, so the setting was never
sent to the agent.

via [HAPI](https://hapi.run)

Co-Authored-By: HAPI <noreply@hapi.run>

* test(voice): update speaker_boost test to assert it IS included in override

The previous test asserted use_speaker_boost was omitted; now it's
correctly included in the TTS payload.

via [HAPI](https://hapi.run)

Co-Authored-By: HAPI <noreply@hapi.run>

* fix(voice): authorize use_speaker_boost in ElevenLabs override schema

Add use_speaker_boost to both the VoiceAgentConfig tts override type
and the buildVoiceAgentConfig() platform_settings so the field is
accepted by the ElevenLabs agent runtime.

via [HAPI](https://hapi.run)

Co-Authored-By: HAPI <noreply@hapi.run>

* fix(voice): propagate full language code through composed prompt, not just zh

getDefaultVoiceSystemPrompt and resolveComposedVoiceSystemPrompt were
filtering language to zh-only before passing to composeVoiceAgentPrompt.
Now append buildVoiceLanguageBlock(language) after composition so French,
Spanish, Japanese etc. reach Gemini/Qwen sessions correctly.

via [HAPI](https://hapi.run)

Co-Authored-By: HAPI <noreply@hapi.run>

* fix(voice): only append language block when language explicitly set

Building language block unconditionally when no language is given
caused getDefaultVoiceSystemPrompt() to diverge from VOICE_SYSTEM_PROMPT.
Only append the block when a code is explicitly provided.

via [HAPI](https://hapi.run)

Co-Authored-By: HAPI <noreply@hapi.run>

* fix(voice): always include language block for Gemini/Qwen in composed prompt

When auto-detect is on (language=undefined), the composed prompt sent
via hub proxy was losing the language auto-detect instruction because
the block was only added when language was explicitly set.

Now: ElevenLabs skips the block (has its own language field); Gemini/Qwen
always include it — undefined produces the auto-detect block, an explicit
code produces the appropriate language instruction.

via [HAPI](https://hapi.run)

Co-Authored-By: HAPI <noreply@hapi.run>

---------

Co-authored-by: HAPI <noreply@hapi.run>
2026-06-05 21:43:04 +08:00

132 lines
5.7 KiB
TypeScript

import { describe, expect, test } from 'bun:test'
import {
buildGeminiLiveSetupMessage,
buildQwenSessionUpdateMessage,
isQwenSafeClientFrame,
GEMINI_LIVE_MODEL,
GEMINI_LIVE_VOICE,
QWEN_REALTIME_VOICE
} from './voice'
import { resolveGeminiLiveVoice, resolveQwenRealtimeVoice } from './voicePickerCatalog'
describe('buildGeminiLiveSetupMessage', () => {
test('locks model and voice to HAPI defaults', () => {
const msg = buildGeminiLiveSetupMessage()
expect(msg.setup.model).toBe(`models/${GEMINI_LIVE_MODEL}`)
const speech = msg.setup.generationConfig as {
speechConfig?: { voiceConfig?: { prebuiltVoiceConfig?: { voiceName?: string } } }
}
expect(speech.speechConfig?.voiceConfig?.prebuiltVoiceConfig?.voiceName).toBe(GEMINI_LIVE_VOICE)
})
test('appends Chinese block when language is zh', () => {
const en = buildGeminiLiveSetupMessage()
const zh = buildGeminiLiveSetupMessage('zh')
const enText = (en.setup.systemInstruction as { parts: Array<{ text: string }> }).parts[0].text
const zhText = (zh.setup.systemInstruction as { parts: Array<{ text: string }> }).parts[0].text
expect(zhText.length).toBeGreaterThan(enText.length)
})
test('uses selected prebuilt voice when valid', () => {
const msg = buildGeminiLiveSetupMessage(undefined, 'Puck')
const speech = msg.setup.generationConfig as {
speechConfig?: { voiceConfig?: { prebuiltVoiceConfig?: { voiceName?: string } } }
}
expect(speech.speechConfig?.voiceConfig?.prebuiltVoiceConfig?.voiceName).toBe('Puck')
})
test('honors custom system instruction override', () => {
const custom = 'Speak only in haiku.'
const msg = buildGeminiLiveSetupMessage(undefined, undefined, custom)
const text = (msg.setup.systemInstruction as { parts: Array<{ text: string }> }).parts[0].text
expect(text).toBe(custom)
})
test('falls back to default for unknown voice names', () => {
const msg = buildGeminiLiveSetupMessage(undefined, 'NotARealVoice')
const speech = msg.setup.generationConfig as {
speechConfig?: { voiceConfig?: { prebuiltVoiceConfig?: { voiceName?: string } } }
}
expect(speech.speechConfig?.voiceConfig?.prebuiltVoiceConfig?.voiceName).toBe(resolveGeminiLiveVoice())
})
})
describe('buildQwenSessionUpdateMessage', () => {
test('locks voice to HAPI default when no voice name supplied', () => {
const msg = buildQwenSessionUpdateMessage()
const session = msg.session as { voice: string }
expect(session.voice).toBe(QWEN_REALTIME_VOICE)
})
test('uses selected prebuilt voice when valid', () => {
const msg = buildQwenSessionUpdateMessage(undefined, 'Ethan')
const session = msg.session as { voice: string }
expect(session.voice).toBe('Ethan')
})
test('falls back to catalog default for unknown voice names', () => {
const msg = buildQwenSessionUpdateMessage(undefined, 'NotARealVoice')
const session = msg.session as { voice: string }
expect(session.voice).toBe(resolveQwenRealtimeVoice())
})
test('includes both tools', () => {
const msg = buildQwenSessionUpdateMessage()
// Realtime shape: flat {type, name, description, parameters} — NOT chat-completions {function:{...}}
const session = msg.session as { tools: Array<{ type: string; name: string }> }
const names = session.tools.map(t => t.name)
expect(names).toContain('messageCodingAgent')
expect(names).toContain('processPermissionRequest')
// Ensure no nested function key (would be wrong chat-completions shape)
session.tools.forEach(t => expect((t as Record<string, unknown>).function).toBeUndefined())
})
test('appends Chinese block when language is zh', () => {
const en = buildQwenSessionUpdateMessage()
const zh = buildQwenSessionUpdateMessage('zh')
const enInstr = (en.session as { instructions: string }).instructions
const zhInstr = (zh.session as { instructions: string }).instructions
expect(zhInstr.length).toBeGreaterThan(enInstr.length)
})
})
describe('isQwenSafeClientFrame', () => {
test('allows non-session.update frames', () => {
expect(isQwenSafeClientFrame(JSON.stringify({ type: 'input_audio_buffer.append', audio: 'abc' }))).toBe(true)
expect(isQwenSafeClientFrame(JSON.stringify({ type: 'response.create' }))).toBe(true)
expect(isQwenSafeClientFrame(JSON.stringify({ type: 'conversation.item.create', item: {} }))).toBe(true)
})
test('allows session.update with only instructions', () => {
expect(isQwenSafeClientFrame(JSON.stringify({
type: 'session.update',
session: { instructions: 'updated prompt' }
}))).toBe(true)
})
test('blocks session.update that includes tools', () => {
expect(isQwenSafeClientFrame(JSON.stringify({
type: 'session.update',
session: { instructions: 'x', tools: [] }
}))).toBe(false)
})
test('blocks session.update that includes voice', () => {
expect(isQwenSafeClientFrame(JSON.stringify({
type: 'session.update',
session: { voice: 'Cherry' }
}))).toBe(false)
})
test('blocks full config session.update', () => {
expect(isQwenSafeClientFrame(JSON.stringify({
type: 'session.update',
session: { modalities: ['text', 'audio'], voice: 'Cherry', instructions: 'x', tools: [], tool_choice: 'auto' }
}))).toBe(false)
})
test('allows non-JSON (binary audio frames pass through)', () => {
expect(isQwenSafeClientFrame('not json {')).toBe(true)
})
})