mirror of
https://github.com/wu736139669/hapi.git
synced 2026-08-07 06:52:28 +00:00
* feat(voice): voice personality, picker catalog, and prompt layer foundation - voicePickerCatalog.ts: per-backend voice lists for Gemini and Qwen with resolve helpers (resolveGeminiLiveVoice, resolveQwenRealtimeVoice) - voicePersonality.ts: VoicePersonalityPreferences schema, presets, composed system prompt with identity/character/response-length layers - voicePromptLayers.ts: buildResolvedVoiceSystemPrompt, preset delivery snippets - voiceSystemPromptParam.ts: hub-side base64url decode for ?systemPrompt= - voicePickerPreferences.ts, voicePersonalitySession.ts: browser-side encode, decode, and storage helpers - useVoicePersonality: React hook for preferences persistence via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix(voice): preset delivery included when non-balanced preset selected; restore test typecheck - isDefaultVoicePersonality: add preset check so warm/calm/direct presets trigger the delivery snippet instead of being treated as default - web/tsconfig.json: remove test file exclusion from typecheck (restoring strict coverage of test code); fix resulting type error in mock declaration via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix(voice): include use_speaker_boost in ElevenLabs TTS override payload The checkbox persisted the pref but ttsDiffersFromDefault and buildElevenLabsTtsOverride both omitted it, so the setting was never sent to the agent. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * test(voice): update speaker_boost test to assert it IS included in override The previous test asserted use_speaker_boost was omitted; now it's correctly included in the TTS payload. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix(voice): authorize use_speaker_boost in ElevenLabs override schema Add use_speaker_boost to both the VoiceAgentConfig tts override type and the buildVoiceAgentConfig() platform_settings so the field is accepted by the ElevenLabs agent runtime. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix(voice): propagate full language code through composed prompt, not just zh getDefaultVoiceSystemPrompt and resolveComposedVoiceSystemPrompt were filtering language to zh-only before passing to composeVoiceAgentPrompt. Now append buildVoiceLanguageBlock(language) after composition so French, Spanish, Japanese etc. reach Gemini/Qwen sessions correctly. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix(voice): only append language block when language explicitly set Building language block unconditionally when no language is given caused getDefaultVoiceSystemPrompt() to diverge from VOICE_SYSTEM_PROMPT. Only append the block when a code is explicitly provided. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix(voice): always include language block for Gemini/Qwen in composed prompt When auto-detect is on (language=undefined), the composed prompt sent via hub proxy was losing the language auto-detect instruction because the block was only added when language was explicitly set. Now: ElevenLabs skips the block (has its own language field); Gemini/Qwen always include it — undefined produces the auto-detect block, an explicit code produces the appropriate language instruction. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> --------- Co-authored-by: HAPI <noreply@hapi.run>
This commit is contained in:
+3
-1
@@ -16,7 +16,9 @@
|
||||
"./schemas": "./src/schemas.ts",
|
||||
"./sessionExport": "./src/sessionExport.ts",
|
||||
"./types": "./src/types.ts",
|
||||
"./voice": "./src/voice.ts"
|
||||
"./voice": "./src/voice.ts",
|
||||
"./voicePickerCatalog": "./src/voicePickerCatalog.ts",
|
||||
"./voice-personality": "./src/voicePersonality.ts"
|
||||
},
|
||||
"sideEffects": false,
|
||||
"scripts": {
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import {
|
||||
listConfiguredVoiceBackends,
|
||||
resolveEffectiveVoiceBackend,
|
||||
resolveHubVoiceBackend
|
||||
} from './voice'
|
||||
|
||||
describe('listConfiguredVoiceBackends', () => {
|
||||
test('returns only backends with API keys', () => {
|
||||
const backends = listConfiguredVoiceBackends({
|
||||
ELEVENLABS_API_KEY: 'el',
|
||||
GEMINI_API_KEY: 'gm',
|
||||
DASHSCOPE_API_KEY: 'qw'
|
||||
})
|
||||
expect(backends).toEqual(['elevenlabs', 'gemini-live', 'qwen-realtime'])
|
||||
})
|
||||
|
||||
test('falls back to elevenlabs when no keys configured', () => {
|
||||
expect(listConfiguredVoiceBackends({})).toEqual(['elevenlabs'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveHubVoiceBackend', () => {
|
||||
test('uses VOICE_BACKEND when that backend is configured', () => {
|
||||
const backend = resolveHubVoiceBackend({
|
||||
VOICE_BACKEND: 'gemini-live',
|
||||
GEMINI_API_KEY: 'gm',
|
||||
ELEVENLABS_API_KEY: 'el'
|
||||
})
|
||||
expect(backend).toBe('gemini-live')
|
||||
})
|
||||
|
||||
test('falls back to first configured when VOICE_BACKEND unavailable', () => {
|
||||
const backend = resolveHubVoiceBackend({
|
||||
VOICE_BACKEND: 'qwen-realtime',
|
||||
ELEVENLABS_API_KEY: 'el'
|
||||
})
|
||||
expect(backend).toBe('elevenlabs')
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveEffectiveVoiceBackend', () => {
|
||||
const configured = ['elevenlabs', 'gemini-live'] as const
|
||||
|
||||
test('prefers stored preference when configured', () => {
|
||||
expect(resolveEffectiveVoiceBackend(configured, 'gemini-live', 'elevenlabs')).toBe('elevenlabs')
|
||||
})
|
||||
|
||||
test('uses hub default when preference missing or invalid', () => {
|
||||
expect(resolveEffectiveVoiceBackend(configured, 'gemini-live', null)).toBe('gemini-live')
|
||||
expect(resolveEffectiveVoiceBackend(configured, 'gemini-live', 'qwen-realtime')).toBe('gemini-live')
|
||||
})
|
||||
})
|
||||
@@ -1,5 +1,13 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { buildGeminiLiveSetupMessage, buildQwenSessionUpdateMessage, isQwenSafeClientFrame, GEMINI_LIVE_MODEL, GEMINI_LIVE_VOICE, QWEN_REALTIME_VOICE } from './voice'
|
||||
import {
|
||||
buildGeminiLiveSetupMessage,
|
||||
buildQwenSessionUpdateMessage,
|
||||
isQwenSafeClientFrame,
|
||||
GEMINI_LIVE_MODEL,
|
||||
GEMINI_LIVE_VOICE,
|
||||
QWEN_REALTIME_VOICE
|
||||
} from './voice'
|
||||
import { resolveGeminiLiveVoice, resolveQwenRealtimeVoice } from './voicePickerCatalog'
|
||||
|
||||
describe('buildGeminiLiveSetupMessage', () => {
|
||||
test('locks model and voice to HAPI defaults', () => {
|
||||
@@ -18,15 +26,50 @@ describe('buildGeminiLiveSetupMessage', () => {
|
||||
const zhText = (zh.setup.systemInstruction as { parts: Array<{ text: string }> }).parts[0].text
|
||||
expect(zhText.length).toBeGreaterThan(enText.length)
|
||||
})
|
||||
|
||||
test('uses selected prebuilt voice when valid', () => {
|
||||
const msg = buildGeminiLiveSetupMessage(undefined, 'Puck')
|
||||
const speech = msg.setup.generationConfig as {
|
||||
speechConfig?: { voiceConfig?: { prebuiltVoiceConfig?: { voiceName?: string } } }
|
||||
}
|
||||
expect(speech.speechConfig?.voiceConfig?.prebuiltVoiceConfig?.voiceName).toBe('Puck')
|
||||
})
|
||||
|
||||
test('honors custom system instruction override', () => {
|
||||
const custom = 'Speak only in haiku.'
|
||||
const msg = buildGeminiLiveSetupMessage(undefined, undefined, custom)
|
||||
const text = (msg.setup.systemInstruction as { parts: Array<{ text: string }> }).parts[0].text
|
||||
expect(text).toBe(custom)
|
||||
})
|
||||
|
||||
test('falls back to default for unknown voice names', () => {
|
||||
const msg = buildGeminiLiveSetupMessage(undefined, 'NotARealVoice')
|
||||
const speech = msg.setup.generationConfig as {
|
||||
speechConfig?: { voiceConfig?: { prebuiltVoiceConfig?: { voiceName?: string } } }
|
||||
}
|
||||
expect(speech.speechConfig?.voiceConfig?.prebuiltVoiceConfig?.voiceName).toBe(resolveGeminiLiveVoice())
|
||||
})
|
||||
})
|
||||
|
||||
describe('buildQwenSessionUpdateMessage', () => {
|
||||
test('locks voice to HAPI default', () => {
|
||||
test('locks voice to HAPI default when no voice name supplied', () => {
|
||||
const msg = buildQwenSessionUpdateMessage()
|
||||
const session = msg.session as { voice: string }
|
||||
expect(session.voice).toBe(QWEN_REALTIME_VOICE)
|
||||
})
|
||||
|
||||
test('uses selected prebuilt voice when valid', () => {
|
||||
const msg = buildQwenSessionUpdateMessage(undefined, 'Ethan')
|
||||
const session = msg.session as { voice: string }
|
||||
expect(session.voice).toBe('Ethan')
|
||||
})
|
||||
|
||||
test('falls back to catalog default for unknown voice names', () => {
|
||||
const msg = buildQwenSessionUpdateMessage(undefined, 'NotARealVoice')
|
||||
const session = msg.session as { voice: string }
|
||||
expect(session.voice).toBe(resolveQwenRealtimeVoice())
|
||||
})
|
||||
|
||||
test('includes both tools', () => {
|
||||
const msg = buildQwenSessionUpdateMessage()
|
||||
// Realtime shape: flat {type, name, description, parameters} — NOT chat-completions {function:{...}}
|
||||
|
||||
+115
-166
@@ -5,165 +5,27 @@
|
||||
* ensuring consistency between server-side auto-creation and client-side usage.
|
||||
*/
|
||||
|
||||
import { resolveGeminiLiveVoice, resolveQwenRealtimeVoice } from './voicePickerCatalog'
|
||||
import {
|
||||
VOICE_CHINESE_LANGUAGE_BLOCK,
|
||||
composeVoiceAgentPrompt,
|
||||
type VoicePromptLayerInput
|
||||
} from './voicePromptLayers'
|
||||
|
||||
export { VOICE_CHINESE_LANGUAGE_BLOCK } from './voicePromptLayers'
|
||||
|
||||
export const ELEVENLABS_API_BASE = 'https://api.elevenlabs.io/v1'
|
||||
export const VOICE_AGENT_NAME = 'Hapi Voice Assistant'
|
||||
|
||||
export const VOICE_SYSTEM_PROMPT = `# CRITICAL RULE - Tool Usage
|
||||
const DEFAULT_COMPOSED_LAYERS: VoicePromptLayerInput = {
|
||||
identity: '',
|
||||
character: '',
|
||||
legacySystemPrompt: '',
|
||||
presetDeliverySnippet: ''
|
||||
}
|
||||
|
||||
You MUST call the messageCodingAgent tool for ANY request related to coding, files, development, debugging, or tasks for the agent. Do NOT respond verbally to these requests — call the tool FIRST, then briefly confirm. This is your most important behavior.
|
||||
|
||||
# Identity
|
||||
|
||||
You are Hapi Voice Assistant. You bridge voice communication between users and their AI coding agents in the Hapi ecosystem.
|
||||
|
||||
IMPORTANT: Never refer to yourself as Gemini, Google, or any underlying model or provider name. You are HAPI — always.
|
||||
|
||||
You are friendly, proactive, and highly intelligent with a world-class engineering background. Your approach is warm, witty, and relaxed, balancing professionalism with an approachable vibe.
|
||||
|
||||
# Environment Overview
|
||||
|
||||
Hapi is a multi-agent development platform supporting:
|
||||
- **Claude Code** - Anthropic's coding assistant (primary)
|
||||
- **Codex** - OpenAI's coding agent
|
||||
- **Gemini** - Google's coding agent
|
||||
|
||||
Users control these agents through the Hapi web interface or Telegram Mini App. You serve as the voice interface to whichever agent is currently active.
|
||||
|
||||
# How Context Updates Work
|
||||
|
||||
You receive automatic context updates when:
|
||||
- A session becomes focused (you see the full session history)
|
||||
- The agent sends messages or uses tools
|
||||
- Permission requests arrive
|
||||
- The agent finishes working (ready event)
|
||||
|
||||
These updates appear as system messages. You do NOT need to poll or ask for updates. Simply wait for them and summarize when relevant.
|
||||
|
||||
# Tools
|
||||
|
||||
## messageCodingAgent
|
||||
Send user requests to the active coding agent.
|
||||
|
||||
When to use:
|
||||
- User says "ask Claude to..." or "have it..."
|
||||
- Any coding, file, or development request
|
||||
- User wants to continue a task
|
||||
|
||||
Example: User says "refactor the auth module" -> call messageCodingAgent with the full request.
|
||||
|
||||
## processPermissionRequest
|
||||
Approve or deny pending permission requests.
|
||||
|
||||
When to use:
|
||||
- User says "yes", "allow", "go ahead", "approve"
|
||||
- User says "no", "deny", "cancel", "stop"
|
||||
|
||||
The decision parameter must be exactly "allow" or "deny".
|
||||
|
||||
# Voice Output Guidelines
|
||||
|
||||
## Summarization (Critical)
|
||||
- NEVER read hashes, IDs, or paths character-by-character
|
||||
- Say "session ending in ZAJ" not "c-m-i-a-b-c-1-2-3..."
|
||||
- Say "file in the src folder" not the full path
|
||||
- Summarize code changes at a high level
|
||||
- Skip tool arguments unless specifically asked
|
||||
|
||||
## TTS Formatting
|
||||
- Use ellipses "..." for pauses
|
||||
- Say "dot" for periods in URLs/paths
|
||||
- Spell out acronyms: "API" becomes "A P I"
|
||||
- Use normalized spoken language
|
||||
|
||||
## Conversation Style
|
||||
- Keep responses to 1-3 sentences typically
|
||||
- Use brief affirmations: "got it", "sure thing"
|
||||
- Occasional natural fillers: "so", "actually"
|
||||
- Mirror user energy: terse replies for terse questions
|
||||
- Lead with empathy for frustrated users
|
||||
|
||||
# Behavioral Guidelines
|
||||
|
||||
## Patience
|
||||
After sending a message to the agent, WAIT SILENTLY. The agent may take 30+ seconds for complex tasks. Do NOT:
|
||||
- Ask "are you still there?"
|
||||
- Repeat the request
|
||||
- Fill silence with chatter
|
||||
|
||||
You will receive a context update when the agent responds or finishes.
|
||||
|
||||
## Request Routing
|
||||
- Direct address ("Assistant, explain...") -> Answer yourself
|
||||
- Explicit delegation ("Have Claude...") -> Use messageCodingAgent
|
||||
- Coding/file tasks -> Use messageCodingAgent
|
||||
- General questions you can answer -> Answer yourself
|
||||
|
||||
Do NOT second-guess what the agent can do. If in doubt, pass it through.
|
||||
|
||||
## Proactive Updates
|
||||
Speak proactively when:
|
||||
- Permission is requested (inform user and ask for decision)
|
||||
- Agent finishes a task (summarize results)
|
||||
- Error occurs (explain clearly)
|
||||
- Session status changes significantly
|
||||
|
||||
Stay silent when:
|
||||
- Agent is actively working
|
||||
- No meaningful update to share
|
||||
|
||||
# Common Scenarios
|
||||
|
||||
## Permission Requests
|
||||
When you see a permission request, immediately inform the user:
|
||||
"Claude wants to run a bash command. Should I allow it?"
|
||||
Then wait for their response and use processPermissionRequest.
|
||||
|
||||
## Errors
|
||||
If the agent reports an error:
|
||||
- Summarize the error type
|
||||
- Suggest what the user might do
|
||||
- Do NOT read stack traces verbatim
|
||||
|
||||
## Session Issues
|
||||
If there is no active session:
|
||||
- Tell the user to select or start a session in the app
|
||||
- You cannot start sessions yourself
|
||||
|
||||
## Long Operations
|
||||
For builds, tests, or large file operations:
|
||||
- Acknowledge the task was sent
|
||||
- Wait silently for completion
|
||||
- Summarize results when ready
|
||||
|
||||
# Guardrails
|
||||
|
||||
- Never read code line-by-line or provide inline code samples
|
||||
- Never repeat the same information multiple ways in one response
|
||||
- Treat garbled input as phonetic hints and ask for clarification
|
||||
- Correct yourself immediately if you realize you made an error
|
||||
- Keep conversations forward-moving with fresh insights
|
||||
- Assume a technical software developer audience
|
||||
|
||||
# First Interaction
|
||||
|
||||
When the user speaks to you for the first time, begin your response with a brief greeting before addressing their request. If their first message is a coding request, greet briefly AND call the tool — do both.`
|
||||
|
||||
/**
|
||||
* Language blocks appended to VOICE_SYSTEM_PROMPT for Gemini/Qwen backends
|
||||
* (ElevenLabs has its own language field).
|
||||
*
|
||||
* Always append one of these — silence causes models to drift to their training
|
||||
* language (Chinese for Qwen, mixed for Gemini).
|
||||
*/
|
||||
export const VOICE_CHINESE_LANGUAGE_BLOCK = `
|
||||
|
||||
# Language
|
||||
|
||||
IMPORTANT: Always respond in Chinese (Mandarin). Use natural spoken Chinese.
|
||||
- Greet users in Chinese
|
||||
- Summarize technical content in Chinese
|
||||
- Use English only for proper nouns, tool names, and code identifiers
|
||||
- Keep the same warm, concise conversational style in Chinese`
|
||||
/** Bundled default composed prompt (fixtures + default identity/character). */
|
||||
export const VOICE_SYSTEM_PROMPT = composeVoiceAgentPrompt(DEFAULT_COMPOSED_LAYERS)
|
||||
|
||||
/** When no language is selected: mirror the user's detected speech language. */
|
||||
const VOICE_LANGUAGE_BLOCK_AUTO = `
|
||||
@@ -300,9 +162,17 @@ export interface VoiceAgentConfig {
|
||||
agent?: {
|
||||
language?: boolean
|
||||
first_message?: boolean
|
||||
prompt?: {
|
||||
prompt?: boolean
|
||||
}
|
||||
}
|
||||
tts?: {
|
||||
voice_id?: boolean
|
||||
stability?: boolean
|
||||
similarity_boost?: boolean
|
||||
style?: boolean
|
||||
speed?: boolean
|
||||
use_speaker_boost?: boolean
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -344,10 +214,18 @@ export function buildVoiceAgentConfig(): VoiceAgentConfig {
|
||||
overrides: {
|
||||
conversation_config_override: {
|
||||
agent: {
|
||||
language: true
|
||||
language: true,
|
||||
prompt: {
|
||||
prompt: true
|
||||
}
|
||||
},
|
||||
tts: {
|
||||
voice_id: true
|
||||
voice_id: true,
|
||||
stability: true,
|
||||
similarity_boost: true,
|
||||
style: true,
|
||||
speed: true,
|
||||
use_speaker_boost: true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -362,6 +240,58 @@ export const QWEN_REALTIME_VOICE = 'Tina'
|
||||
|
||||
export const DEFAULT_VOICE_BACKEND: VoiceBackendType = 'elevenlabs'
|
||||
|
||||
const VOICE_BACKEND_VALUES: readonly VoiceBackendType[] = [
|
||||
'elevenlabs',
|
||||
'gemini-live',
|
||||
'qwen-realtime'
|
||||
] as const
|
||||
|
||||
export type VoiceBackendEnv = Record<string, string | undefined>
|
||||
|
||||
/** Backends whose API keys are present on the hub. */
|
||||
export function listConfiguredVoiceBackends(env: VoiceBackendEnv): VoiceBackendType[] {
|
||||
const backends: VoiceBackendType[] = []
|
||||
if (env.ELEVENLABS_API_KEY?.trim()) {
|
||||
backends.push('elevenlabs')
|
||||
}
|
||||
if (env.GEMINI_API_KEY?.trim() || env.GOOGLE_API_KEY?.trim()) {
|
||||
backends.push('gemini-live')
|
||||
}
|
||||
if (env.DASHSCOPE_API_KEY?.trim() || env.QWEN_API_KEY?.trim()) {
|
||||
backends.push('qwen-realtime')
|
||||
}
|
||||
return backends.length > 0 ? backends : [DEFAULT_VOICE_BACKEND]
|
||||
}
|
||||
|
||||
/** Hub default from VOICE_BACKEND when configured, else first available backend. */
|
||||
export function resolveHubVoiceBackend(env: VoiceBackendEnv): VoiceBackendType {
|
||||
const configured = listConfiguredVoiceBackends(env)
|
||||
const raw = env.VOICE_BACKEND
|
||||
const fromEnv = VOICE_BACKEND_VALUES.includes(raw as VoiceBackendType)
|
||||
? (raw as VoiceBackendType)
|
||||
: DEFAULT_VOICE_BACKEND
|
||||
return configured.includes(fromEnv) ? fromEnv : (configured[0] ?? DEFAULT_VOICE_BACKEND)
|
||||
}
|
||||
|
||||
/** User preference wins when valid; otherwise hub default. */
|
||||
export function resolveEffectiveVoiceBackend(
|
||||
configured: readonly VoiceBackendType[],
|
||||
hubDefault: VoiceBackendType,
|
||||
storedPreference: string | null | undefined
|
||||
): VoiceBackendType {
|
||||
if (
|
||||
storedPreference
|
||||
&& VOICE_BACKEND_VALUES.includes(storedPreference as VoiceBackendType)
|
||||
&& configured.includes(storedPreference as VoiceBackendType)
|
||||
) {
|
||||
return storedPreference as VoiceBackendType
|
||||
}
|
||||
if (configured.includes(hubDefault)) {
|
||||
return hubDefault
|
||||
}
|
||||
return configured[0] ?? hubDefault
|
||||
}
|
||||
|
||||
export const GEMINI_LIVE_MODEL = 'gemini-2.5-flash-native-audio-latest'
|
||||
export const GEMINI_LIVE_VOICE = 'Aoede'
|
||||
|
||||
@@ -418,11 +348,17 @@ export function buildGeminiLiveFunctionDeclarations(): GeminiLiveFunctionDeclara
|
||||
return VOICE_TOOLS.map(cloneVoiceToolDefinition)
|
||||
}
|
||||
|
||||
export function buildGeminiLiveConfig(language?: string): GeminiLiveConfig {
|
||||
const systemInstruction = `${VOICE_SYSTEM_PROMPT}${buildVoiceLanguageBlock(language)}`
|
||||
export function buildGeminiLiveConfig(
|
||||
language?: string,
|
||||
voiceName?: string,
|
||||
systemInstruction?: string
|
||||
): GeminiLiveConfig {
|
||||
const systemInstructionText = systemInstruction?.trim()
|
||||
? systemInstruction
|
||||
: `${VOICE_SYSTEM_PROMPT}${buildVoiceLanguageBlock(language)}`
|
||||
return {
|
||||
model: GEMINI_LIVE_MODEL,
|
||||
systemInstruction,
|
||||
systemInstruction: systemInstructionText,
|
||||
tools: [
|
||||
{
|
||||
functionDeclarations: buildGeminiLiveFunctionDeclarations()
|
||||
@@ -433,8 +369,14 @@ export function buildGeminiLiveConfig(language?: string): GeminiLiveConfig {
|
||||
}
|
||||
|
||||
/** Hub-owned initial session.update for Qwen Realtime (hub proxy). */
|
||||
export function buildQwenSessionUpdateMessage(language?: string): Record<string, unknown> {
|
||||
const instructions = `${VOICE_SYSTEM_PROMPT}${buildVoiceLanguageBlock(language)}`
|
||||
export function buildQwenSessionUpdateMessage(
|
||||
language?: string,
|
||||
voiceName?: string,
|
||||
systemInstruction?: string
|
||||
): Record<string, unknown> {
|
||||
const instructions = systemInstruction?.trim()
|
||||
? systemInstruction
|
||||
: `${VOICE_SYSTEM_PROMPT}${buildVoiceLanguageBlock(language)}`
|
||||
// Qwen Realtime uses the flat Realtime shape, not the chat-completions nested {function:{...}} shape.
|
||||
const tools = VOICE_TOOL_DEFINITIONS.map((td) => ({
|
||||
type: 'function' as const,
|
||||
@@ -446,7 +388,7 @@ export function buildQwenSessionUpdateMessage(language?: string): Record<string,
|
||||
type: 'session.update',
|
||||
session: {
|
||||
modalities: ['text', 'audio'],
|
||||
voice: QWEN_REALTIME_VOICE,
|
||||
voice: resolveQwenRealtimeVoice(voiceName),
|
||||
input_audio_format: 'pcm',
|
||||
output_audio_format: 'pcm',
|
||||
instructions,
|
||||
@@ -487,16 +429,23 @@ export function isQwenSafeClientFrame(message: string | ArrayBuffer | Uint8Array
|
||||
}
|
||||
|
||||
/** Wire-format setup frame for Gemini Live BidiGenerateContent (hub proxy + web client). */
|
||||
export function buildGeminiLiveSetupMessage(language?: string): { setup: Record<string, unknown> } {
|
||||
const liveConfig = buildGeminiLiveConfig(language)
|
||||
export function buildGeminiLiveSetupMessage(
|
||||
language?: string,
|
||||
voiceName?: string,
|
||||
systemInstruction?: string,
|
||||
options?: { affectiveDialog?: boolean }
|
||||
): { setup: Record<string, unknown> } {
|
||||
const liveConfig = buildGeminiLiveConfig(language, voiceName, systemInstruction)
|
||||
const resolvedVoice = resolveGeminiLiveVoice(voiceName)
|
||||
return {
|
||||
setup: {
|
||||
model: `models/${liveConfig.model}`,
|
||||
generationConfig: {
|
||||
responseModalities: ['AUDIO'],
|
||||
...(options?.affectiveDialog ? { enableAffectiveDialog: true } : {}),
|
||||
speechConfig: {
|
||||
voiceConfig: {
|
||||
prebuiltVoiceConfig: { voiceName: GEMINI_LIVE_VOICE }
|
||||
prebuiltVoiceConfig: { voiceName: resolvedVoice }
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { VOICE_SYSTEM_PROMPT } from './voice'
|
||||
import {
|
||||
ELEVENLABS_WEBRTC_CONTEXT_MAX_BYTES,
|
||||
getDefaultVoiceSystemPrompt,
|
||||
getVoicePersonalityPreset,
|
||||
isDefaultVoicePersonality,
|
||||
parseVoicePersonalityPreferences,
|
||||
resolveComposedVoiceSystemPrompt,
|
||||
resolveElevenLabsVoiceSettings,
|
||||
truncateUtf8ByteLength,
|
||||
utf8ByteLength
|
||||
} from './voicePersonality'
|
||||
import { VOICE_PLATFORM_FIXTURES } from './voicePromptLayers'
|
||||
|
||||
describe('voicePersonality', () => {
|
||||
test('parseVoicePersonalityPreferences returns defaults for invalid input', () => {
|
||||
const prefs = parseVoicePersonalityPreferences(null)
|
||||
expect(prefs.preset).toBe('balanced')
|
||||
expect(prefs.identity).toBe('')
|
||||
expect(prefs.character).toBe('')
|
||||
})
|
||||
|
||||
test('migrates legacy customPrompt into character', () => {
|
||||
const prefs = parseVoicePersonalityPreferences({
|
||||
customPrompt: 'Call me G.'
|
||||
})
|
||||
expect(prefs.character).toBe('Call me G.')
|
||||
})
|
||||
|
||||
test('migrates non-monolith systemPrompt into character', () => {
|
||||
const prefs = parseVoicePersonalityPreferences({
|
||||
systemPrompt: 'You are a pirate. Arr.'
|
||||
})
|
||||
expect(prefs.character).toBe('You are a pirate. Arr.')
|
||||
expect(prefs.systemPrompt).toBe('')
|
||||
})
|
||||
|
||||
test('composed prompt always includes platform fixtures', () => {
|
||||
const prefs = parseVoicePersonalityPreferences({
|
||||
character: 'You are a pirate. Arr.'
|
||||
})
|
||||
const { prompt } = resolveComposedVoiceSystemPrompt(prefs)
|
||||
expect(prompt).toContain('messageCodingAgent')
|
||||
expect(prompt).toContain(VOICE_PLATFORM_FIXTURES.slice(0, 40))
|
||||
expect(prompt).toContain('You are a pirate')
|
||||
expect(prompt).toContain('Never refer to yourself as Gemini')
|
||||
})
|
||||
|
||||
test('resolveComposedVoiceSystemPrompt does not embed session context', () => {
|
||||
const prefs = parseVoicePersonalityPreferences({ character: 'Base.' })
|
||||
const { prompt } = resolveComposedVoiceSystemPrompt(prefs)
|
||||
expect(prompt).not.toContain('[Current Context]')
|
||||
expect(prompt).not.toContain('Working on auth.')
|
||||
})
|
||||
|
||||
test('getDefaultVoiceSystemPrompt matches bundled VOICE_SYSTEM_PROMPT', () => {
|
||||
const prefs = parseVoicePersonalityPreferences({})
|
||||
expect(getDefaultVoiceSystemPrompt()).toBe(VOICE_SYSTEM_PROMPT)
|
||||
expect(isDefaultVoicePersonality(prefs)).toBe(true)
|
||||
})
|
||||
|
||||
test('resolveElevenLabsVoiceSettings uses preset sliders unless custom', () => {
|
||||
const prefs = parseVoicePersonalityPreferences({
|
||||
preset: 'custom',
|
||||
elevenLabs: { stability: 0.42, similarity_boost: 0.8, style: 0.2, speed: 1.05, use_speaker_boost: true }
|
||||
})
|
||||
expect(resolveElevenLabsVoiceSettings(prefs).stability).toBe(0.42)
|
||||
})
|
||||
|
||||
test('defines all preset ids', () => {
|
||||
expect(getVoicePersonalityPreset('calm').elevenLabs.speed).toBeLessThan(1)
|
||||
})
|
||||
|
||||
test('truncateUtf8ByteLength respects byte budget', () => {
|
||||
const text = truncateUtf8ByteLength('hello 🎙️ world', ELEVENLABS_WEBRTC_CONTEXT_MAX_BYTES)
|
||||
expect(text).toBe('hello 🎙️ world')
|
||||
const huge = truncateUtf8ByteLength('a'.repeat(50_000), 100)
|
||||
expect(utf8ByteLength(huge)).toBeLessThanOrEqual(100)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,456 @@
|
||||
/**
|
||||
* Voice personality / delivery controls shared by Settings UI and realtime sessions.
|
||||
* @see docs/plans/voice-personality-config.md
|
||||
*/
|
||||
|
||||
import { VOICE_SYSTEM_PROMPT, buildVoiceLanguageBlock } from './voice'
|
||||
import {
|
||||
composeVoiceAgentPrompt,
|
||||
getVoicePlatformFixturesPreview,
|
||||
type VoicePromptLayerInput
|
||||
} from './voicePromptLayers'
|
||||
|
||||
export type VoicePersonalityPresetId =
|
||||
| 'balanced'
|
||||
| 'warm'
|
||||
| 'calm'
|
||||
| 'direct'
|
||||
| 'custom'
|
||||
|
||||
export type VoiceBackendKind = 'elevenlabs' | 'gemini-live' | 'qwen-realtime'
|
||||
|
||||
/** ElevenLabs ConvAI runtime TTS overrides (snake_case for API). */
|
||||
export interface ElevenLabsVoiceSettings {
|
||||
stability: number
|
||||
similarity_boost: number
|
||||
style: number
|
||||
speed: number
|
||||
use_speaker_boost: boolean
|
||||
}
|
||||
|
||||
export interface GeminiVoiceOptions {
|
||||
affective_dialog: boolean
|
||||
}
|
||||
|
||||
export type ResponseLengthOption = 'brief' | 'balanced' | 'detailed'
|
||||
|
||||
export const RESPONSE_LENGTH_OPTIONS: readonly ResponseLengthOption[] = ['brief', 'balanced', 'detailed']
|
||||
|
||||
const RESPONSE_LENGTH_INSTRUCTIONS: Record<ResponseLengthOption, string> = {
|
||||
brief: '\n\n# Response length\n\nKeep all voice responses to 1–2 sentences. Be concise and direct.',
|
||||
balanced: '',
|
||||
detailed: '\n\n# Response length\n\nGive thorough responses when the topic warrants depth. Do not truncate if completeness is needed.',
|
||||
}
|
||||
|
||||
export function getResponseLengthInstruction(length: ResponseLengthOption): string {
|
||||
return RESPONSE_LENGTH_INSTRUCTIONS[length]
|
||||
}
|
||||
|
||||
export interface VoicePersonalityPreferences {
|
||||
preset: VoicePersonalityPresetId
|
||||
/** Who the assistant is (rebrand / overseer). Empty = bundled default identity. */
|
||||
identity: string
|
||||
/** Delivery, tone, preset overlays. Empty = default character + preset snippet. */
|
||||
character: string
|
||||
/**
|
||||
* @deprecated Migrated to identity/character on parse. Full monolith kept only when it
|
||||
* still contains platform fixtures (legacy override).
|
||||
*/
|
||||
systemPrompt: string
|
||||
/** @deprecated Migrated into character on parse. */
|
||||
customPrompt: string
|
||||
/** How long responses should be. 'balanced' is the default (no extra instruction). */
|
||||
responseLength: ResponseLengthOption
|
||||
elevenLabs: ElevenLabsVoiceSettings
|
||||
gemini: GeminiVoiceOptions
|
||||
}
|
||||
|
||||
export const VOICE_IDENTITY_MAX_LENGTH = 8_000
|
||||
export const VOICE_CHARACTER_MAX_LENGTH = 16_000
|
||||
|
||||
/** Max stored prompt size (localStorage + Gemini WS query param budget). */
|
||||
export const VOICE_SYSTEM_PROMPT_MAX_LENGTH = 48_000
|
||||
|
||||
/** ElevenLabs ConvAI WebRTC data channel limit per message (bytes). */
|
||||
export const ELEVENLABS_WEBRTC_MAX_MESSAGE_BYTES = 65_535
|
||||
|
||||
/** Budget for session bootstrap in startSession dynamicVariables. */
|
||||
export const ELEVENLABS_WEBRTC_CONTEXT_MAX_BYTES = 4_000
|
||||
|
||||
/** Budget per deferred context chunk (streamed after connect). */
|
||||
export const VOICE_CONTEXT_STREAM_CHUNK_MAX_BYTES = 8_000
|
||||
|
||||
/** Budget for composed system prompt in agent overrides (+ tools JSON uses the rest). */
|
||||
export const ELEVENLABS_WEBRTC_PROMPT_MAX_BYTES = 12_000
|
||||
|
||||
/** Trim UTF-8 text to a byte budget (for WebRTC / query-param limits). */
|
||||
export function truncateUtf8ByteLength(text: string, maxBytes: number): string {
|
||||
if (maxBytes <= 0) return ''
|
||||
const encoder = new TextEncoder()
|
||||
if (encoder.encode(text).length <= maxBytes) return text
|
||||
|
||||
const suffix = '\n\n[…truncated for voice transport…]'
|
||||
const suffixBytes = encoder.encode(suffix).length
|
||||
const budget = Math.max(0, maxBytes - suffixBytes)
|
||||
|
||||
let lo = 0
|
||||
let hi = text.length
|
||||
while (lo < hi) {
|
||||
const mid = Math.ceil((lo + hi) / 2)
|
||||
if (encoder.encode(text.slice(0, mid)).length <= budget) lo = mid
|
||||
else hi = mid - 1
|
||||
}
|
||||
return text.slice(0, lo) + suffix
|
||||
}
|
||||
|
||||
export function utf8ByteLength(text: string): number {
|
||||
return new TextEncoder().encode(text).length
|
||||
}
|
||||
|
||||
export const VOICE_PERSONALITY_STORAGE_KEY = 'hapi-voice-personality'
|
||||
export const VOICE_CONTEXT_NOTICE_STORAGE_KEY = 'hapi-voice-context-notice'
|
||||
|
||||
export const DEFAULT_ELEVENLABS_VOICE_SETTINGS: ElevenLabsVoiceSettings = {
|
||||
stability: 0.5,
|
||||
similarity_boost: 0.75,
|
||||
style: 0.1,
|
||||
speed: 1.0,
|
||||
use_speaker_boost: false
|
||||
}
|
||||
|
||||
export const DEFAULT_GEMINI_VOICE_OPTIONS: GeminiVoiceOptions = {
|
||||
affective_dialog: true
|
||||
}
|
||||
|
||||
export const DEFAULT_VOICE_PERSONALITY: VoicePersonalityPreferences = {
|
||||
preset: 'balanced',
|
||||
identity: '',
|
||||
character: '',
|
||||
systemPrompt: '',
|
||||
customPrompt: '',
|
||||
responseLength: 'balanced',
|
||||
elevenLabs: { ...DEFAULT_ELEVENLABS_VOICE_SETTINGS },
|
||||
gemini: { ...DEFAULT_GEMINI_VOICE_OPTIONS }
|
||||
}
|
||||
|
||||
export function voicePromptLayersFromPrefs(prefs: VoicePersonalityPreferences): VoicePromptLayerInput {
|
||||
return {
|
||||
identity: prefs.identity,
|
||||
character: prefs.character,
|
||||
legacySystemPrompt: prefs.systemPrompt,
|
||||
presetDeliverySnippet: getPresetDeliverySnippet(prefs.preset)
|
||||
}
|
||||
}
|
||||
|
||||
/** Bundled composed prompt for the session language (editable copy baseline in Settings). */
|
||||
export function getDefaultVoiceSystemPrompt(language?: string): string {
|
||||
return composeVoiceAgentPrompt(voicePromptLayersFromPrefs(DEFAULT_VOICE_PERSONALITY))
|
||||
+ (language ? buildVoiceLanguageBlock(language) : '')
|
||||
}
|
||||
|
||||
export interface VoicePromptComposeResult {
|
||||
prompt: string
|
||||
truncated: boolean
|
||||
wireBytes: number
|
||||
}
|
||||
|
||||
/** Effective composed system prompt for a voice session (all backends). Session context is never embedded here. */
|
||||
export function resolveComposedVoiceSystemPrompt(
|
||||
prefs: VoicePersonalityPreferences,
|
||||
options?: {
|
||||
language?: string
|
||||
backend?: VoiceBackendKind
|
||||
maxWireBytes?: number
|
||||
}
|
||||
): VoicePromptComposeResult {
|
||||
const lang = options?.language
|
||||
const isElevenLabs = options?.backend === 'elevenlabs'
|
||||
// ElevenLabs has its own language field; Gemini/Qwen need the block in the prompt.
|
||||
// For Gemini/Qwen, always include it: undefined → auto-detect block, code → explicit block.
|
||||
let prompt = composeVoiceAgentPrompt(voicePromptLayersFromPrefs(prefs))
|
||||
+ (isElevenLabs ? (lang ? buildVoiceLanguageBlock(lang) : '') : buildVoiceLanguageBlock(lang))
|
||||
prompt += getResponseLengthInstruction(prefs.responseLength ?? 'balanced')
|
||||
|
||||
const maxBytes = options?.maxWireBytes
|
||||
?? (options?.backend === 'elevenlabs' ? ELEVENLABS_WEBRTC_PROMPT_MAX_BYTES : undefined)
|
||||
|
||||
let truncated = false
|
||||
if (maxBytes && utf8ByteLength(prompt) > maxBytes) {
|
||||
prompt = truncateUtf8ByteLength(prompt, maxBytes)
|
||||
truncated = true
|
||||
}
|
||||
|
||||
return { prompt, truncated, wireBytes: utf8ByteLength(prompt) }
|
||||
}
|
||||
|
||||
/** @deprecated Use resolveComposedVoiceSystemPrompt().prompt */
|
||||
export function resolveVoiceSystemPrompt(
|
||||
prefs: VoicePersonalityPreferences,
|
||||
options?: { language?: string; initialContext?: string }
|
||||
): string {
|
||||
void options?.initialContext
|
||||
return resolveComposedVoiceSystemPrompt(prefs, { language: options?.language }).prompt
|
||||
}
|
||||
|
||||
export function isDefaultVoicePersonality(prefs: VoicePersonalityPreferences): boolean {
|
||||
return (prefs.preset ?? 'balanced') === 'balanced'
|
||||
&& !prefs.identity.trim()
|
||||
&& !prefs.character.trim()
|
||||
&& !prefs.systemPrompt.trim()
|
||||
&& (prefs.responseLength ?? 'balanced') === 'balanced'
|
||||
}
|
||||
|
||||
/** @deprecated Use isDefaultVoicePersonality */
|
||||
export function isDefaultVoiceSystemPrompt(prefs: VoicePersonalityPreferences, _language?: string): boolean {
|
||||
return isDefaultVoicePersonality(prefs)
|
||||
}
|
||||
|
||||
export {
|
||||
DEFAULT_VOICE_CHARACTER,
|
||||
DEFAULT_VOICE_IDENTITY,
|
||||
getVoicePlatformFixturesPreview
|
||||
} from './voicePromptLayers'
|
||||
|
||||
export interface VoicePersonalityPresetDefinition {
|
||||
id: VoicePersonalityPresetId
|
||||
labelKey: string
|
||||
descriptionKey: string
|
||||
promptAddition: string
|
||||
elevenLabs: ElevenLabsVoiceSettings
|
||||
}
|
||||
|
||||
export const VOICE_PERSONALITY_PRESETS: readonly VoicePersonalityPresetDefinition[] = [
|
||||
{
|
||||
id: 'balanced',
|
||||
labelKey: 'settings.voice.character.preset.balanced',
|
||||
descriptionKey: 'settings.voice.character.preset.balancedHint',
|
||||
promptAddition: '',
|
||||
elevenLabs: {
|
||||
stability: 0.5,
|
||||
similarity_boost: 0.75,
|
||||
style: 0.1,
|
||||
speed: 1.0,
|
||||
use_speaker_boost: false
|
||||
}
|
||||
},
|
||||
{
|
||||
id: 'warm',
|
||||
labelKey: 'settings.voice.character.preset.warm',
|
||||
descriptionKey: 'settings.voice.character.preset.warmHint',
|
||||
promptAddition: `Speak with natural warmth and personality. Use audio cues where they fit:
|
||||
[chuckles] or [laughs] when something is genuinely funny,
|
||||
[excited] when sharing something interesting,
|
||||
[sighs] for wistful or empathetic moments,
|
||||
[warm tone] as your default register.
|
||||
One or two tags per response maximum — never performed, always earned.
|
||||
Speak at a relaxed pace. Pause before considered answers.`,
|
||||
elevenLabs: {
|
||||
stability: 0.35,
|
||||
similarity_boost: 0.75,
|
||||
style: 0.3,
|
||||
speed: 0.97,
|
||||
use_speaker_boost: true
|
||||
}
|
||||
},
|
||||
{
|
||||
id: 'calm',
|
||||
labelKey: 'settings.voice.character.preset.calm',
|
||||
descriptionKey: 'settings.voice.character.preset.calmHint',
|
||||
promptAddition: `Speak slowly and deliberately. [pauses] before important points.
|
||||
Use [sighs] and [hesitates] naturally. Never rush.
|
||||
Keep energy low and steady.`,
|
||||
elevenLabs: {
|
||||
stability: 0.75,
|
||||
similarity_boost: 0.75,
|
||||
style: 0.0,
|
||||
speed: 0.93,
|
||||
use_speaker_boost: false
|
||||
}
|
||||
},
|
||||
{
|
||||
id: 'direct',
|
||||
labelKey: 'settings.voice.character.preset.direct',
|
||||
descriptionKey: 'settings.voice.character.preset.directHint',
|
||||
promptAddition: `Be concise. Skip pleasantries unless asked. No filler phrases.
|
||||
Short answers. Confirm before elaborating.`,
|
||||
elevenLabs: {
|
||||
stability: 0.65,
|
||||
similarity_boost: 0.75,
|
||||
style: 0.05,
|
||||
speed: 1.08,
|
||||
use_speaker_boost: false
|
||||
}
|
||||
},
|
||||
{
|
||||
id: 'custom',
|
||||
labelKey: 'settings.voice.character.preset.custom',
|
||||
descriptionKey: 'settings.voice.character.preset.customHint',
|
||||
promptAddition: '',
|
||||
elevenLabs: { ...DEFAULT_ELEVENLABS_VOICE_SETTINGS }
|
||||
}
|
||||
] as const
|
||||
|
||||
export function getVoicePersonalityPreset(
|
||||
id: VoicePersonalityPresetId
|
||||
): VoicePersonalityPresetDefinition {
|
||||
return VOICE_PERSONALITY_PRESETS.find((p) => p.id === id) ?? VOICE_PERSONALITY_PRESETS[0]
|
||||
}
|
||||
|
||||
export function clamp01(value: number): number {
|
||||
if (Number.isNaN(value)) return 0
|
||||
return Math.min(1, Math.max(0, value))
|
||||
}
|
||||
|
||||
export function clampSpeed(value: number): number {
|
||||
if (Number.isNaN(value)) return 1
|
||||
return Math.min(1.2, Math.max(0.7, value))
|
||||
}
|
||||
|
||||
function isBundledMonolith(systemPrompt: string): boolean {
|
||||
const trimmed = systemPrompt.trim()
|
||||
if (!trimmed) return false
|
||||
return trimmed === VOICE_SYSTEM_PROMPT || trimmed.startsWith('# CRITICAL RULE')
|
||||
}
|
||||
|
||||
export function parseVoicePersonalityPreferences(raw: unknown): VoicePersonalityPreferences {
|
||||
if (!raw || typeof raw !== 'object') {
|
||||
return structuredClone(DEFAULT_VOICE_PERSONALITY)
|
||||
}
|
||||
const record = raw as Record<string, unknown>
|
||||
const presetRaw = record.preset
|
||||
const preset = VOICE_PERSONALITY_PRESETS.some((p) => p.id === presetRaw)
|
||||
? (presetRaw as VoicePersonalityPresetId)
|
||||
: 'balanced'
|
||||
|
||||
const elRaw = record.elevenLabs
|
||||
const el = elRaw && typeof elRaw === 'object'
|
||||
? (elRaw as Record<string, unknown>)
|
||||
: {}
|
||||
|
||||
const geminiRaw = record.gemini
|
||||
const gemini = geminiRaw && typeof geminiRaw === 'object'
|
||||
? (geminiRaw as Record<string, unknown>)
|
||||
: {}
|
||||
|
||||
let identity = typeof record.identity === 'string' ? record.identity : ''
|
||||
let character = typeof record.character === 'string' ? record.character : ''
|
||||
let systemPrompt = typeof record.systemPrompt === 'string' ? record.systemPrompt : ''
|
||||
|
||||
const legacyNotes = typeof record.customPrompt === 'string' ? record.customPrompt.trim() : ''
|
||||
if (legacyNotes && !character.trim()) {
|
||||
character = legacyNotes
|
||||
}
|
||||
|
||||
if (!identity.trim() && !character.trim() && systemPrompt.trim()) {
|
||||
if (isBundledMonolith(systemPrompt)) {
|
||||
systemPrompt = ''
|
||||
} else if (systemPrompt.includes('# CRITICAL RULE')) {
|
||||
// Legacy full override — keep in systemPrompt for composeVoiceAgentPrompt
|
||||
} else {
|
||||
character = systemPrompt
|
||||
systemPrompt = ''
|
||||
}
|
||||
}
|
||||
|
||||
const responseLengthRaw = record.responseLength
|
||||
const responseLength: ResponseLengthOption = RESPONSE_LENGTH_OPTIONS.includes(responseLengthRaw as ResponseLengthOption)
|
||||
? (responseLengthRaw as ResponseLengthOption)
|
||||
: 'balanced'
|
||||
|
||||
return {
|
||||
preset,
|
||||
identity: identity.slice(0, VOICE_IDENTITY_MAX_LENGTH),
|
||||
character: character.slice(0, VOICE_CHARACTER_MAX_LENGTH),
|
||||
systemPrompt: systemPrompt.slice(0, VOICE_SYSTEM_PROMPT_MAX_LENGTH),
|
||||
customPrompt: '',
|
||||
responseLength,
|
||||
elevenLabs: {
|
||||
stability: clamp01(Number(el.stability ?? DEFAULT_ELEVENLABS_VOICE_SETTINGS.stability)),
|
||||
similarity_boost: clamp01(Number(el.similarity_boost ?? el.similarityBoost ?? DEFAULT_ELEVENLABS_VOICE_SETTINGS.similarity_boost)),
|
||||
style: clamp01(Number(el.style ?? DEFAULT_ELEVENLABS_VOICE_SETTINGS.style)),
|
||||
speed: clampSpeed(Number(el.speed ?? DEFAULT_ELEVENLABS_VOICE_SETTINGS.speed)),
|
||||
use_speaker_boost: Boolean(el.use_speaker_boost ?? el.useSpeakerBoost ?? DEFAULT_ELEVENLABS_VOICE_SETTINGS.use_speaker_boost)
|
||||
},
|
||||
gemini: {
|
||||
affective_dialog: gemini.affective_dialog !== false && gemini.affectiveDialog !== false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Optional delivery snippet from the selected preset (merged into character layer when character empty). */
|
||||
export function getPresetDeliverySnippet(presetId: VoicePersonalityPresetId): string {
|
||||
return getVoicePersonalityPreset(presetId).promptAddition.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Use resolveComposedVoiceSystemPrompt. Preset text is merged in the character layer.
|
||||
*/
|
||||
export function buildVoicePersonalityPromptAddition(prefs: VoicePersonalityPreferences): string {
|
||||
return getPresetDeliverySnippet(prefs.preset)
|
||||
}
|
||||
|
||||
/** Effective ElevenLabs sliders — custom preset uses stored values; others use preset defaults unless preset is custom. */
|
||||
export function resolveElevenLabsVoiceSettings(prefs: VoicePersonalityPreferences): ElevenLabsVoiceSettings {
|
||||
if (prefs.preset === 'custom') {
|
||||
return { ...prefs.elevenLabs }
|
||||
}
|
||||
return { ...getVoicePersonalityPreset(prefs.preset).elevenLabs }
|
||||
}
|
||||
|
||||
export type VoiceControlLeverId =
|
||||
| 'character_preset'
|
||||
| 'voice_identity'
|
||||
| 'voice_character'
|
||||
| 'voice_fixtures'
|
||||
| 'speaking_rate'
|
||||
| 'expressiveness'
|
||||
| 'stability'
|
||||
| 'similarity_boost'
|
||||
| 'speaker_boost'
|
||||
| 'affective_dialog'
|
||||
|
||||
export interface VoiceControlLever {
|
||||
id: VoiceControlLeverId
|
||||
backends: VoiceBackendKind[]
|
||||
}
|
||||
|
||||
/** Levers exposed in Settings → Advanced voice (common vs backend-specific). */
|
||||
export const VOICE_CONTROL_LEVERS: readonly VoiceControlLever[] = [
|
||||
{ id: 'character_preset', backends: ['elevenlabs', 'gemini-live', 'qwen-realtime'] },
|
||||
{ id: 'voice_identity', backends: ['elevenlabs', 'gemini-live', 'qwen-realtime'] },
|
||||
{ id: 'voice_character', backends: ['elevenlabs', 'gemini-live', 'qwen-realtime'] },
|
||||
{ id: 'voice_fixtures', backends: ['elevenlabs', 'gemini-live', 'qwen-realtime'] },
|
||||
{ id: 'speaking_rate', backends: ['elevenlabs', 'gemini-live', 'qwen-realtime'] },
|
||||
{ id: 'expressiveness', backends: ['elevenlabs', 'gemini-live', 'qwen-realtime'] },
|
||||
{ id: 'stability', backends: ['elevenlabs'] },
|
||||
{ id: 'similarity_boost', backends: ['elevenlabs'] },
|
||||
{ id: 'speaker_boost', backends: ['elevenlabs'] },
|
||||
{ id: 'affective_dialog', backends: ['gemini-live'] }
|
||||
]
|
||||
|
||||
export function leverAppliesToBackend(lever: VoiceControlLever, backend: VoiceBackendKind): boolean {
|
||||
return lever.backends.includes(backend)
|
||||
}
|
||||
|
||||
export function getVoiceWireBudgetHint(backend: VoiceBackendKind): {
|
||||
storageMaxChars: number
|
||||
wireNoteKey: string
|
||||
} {
|
||||
switch (backend) {
|
||||
case 'elevenlabs':
|
||||
return {
|
||||
storageMaxChars: VOICE_SYSTEM_PROMPT_MAX_LENGTH,
|
||||
wireNoteKey: 'settings.voice.wireBudget.elevenlabs'
|
||||
}
|
||||
case 'gemini-live':
|
||||
return {
|
||||
storageMaxChars: VOICE_SYSTEM_PROMPT_MAX_LENGTH,
|
||||
wireNoteKey: 'settings.voice.wireBudget.gemini'
|
||||
}
|
||||
case 'qwen-realtime':
|
||||
return {
|
||||
storageMaxChars: VOICE_SYSTEM_PROMPT_MAX_LENGTH,
|
||||
wireNoteKey: 'settings.voice.wireBudget.qwen'
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Static voice catalogs for Settings picker (Gemini Live, Qwen Realtime).
|
||||
* ElevenLabs voices remain dynamic via GET /api/voice/voices (#690).
|
||||
*
|
||||
* @see https://github.com/tiann/hapi/issues/742
|
||||
*/
|
||||
|
||||
export type VoicePickerOption = {
|
||||
id: string
|
||||
label: string
|
||||
description?: string
|
||||
}
|
||||
|
||||
/** Prebuilt voices documented for Gemini Live BidiGenerateContent. */
|
||||
export const GEMINI_LIVE_VOICE_OPTIONS: readonly VoicePickerOption[] = [
|
||||
{ id: 'Puck', label: 'Puck', description: 'Conversational, friendly' },
|
||||
{ id: 'Charon', label: 'Charon', description: 'Deep, authoritative' },
|
||||
{ id: 'Kore', label: 'Kore', description: 'Neutral, professional' },
|
||||
{ id: 'Fenrir', label: 'Fenrir', description: 'Warm, approachable' },
|
||||
{ id: 'Aoede', label: 'Aoede', description: 'Default' }
|
||||
] as const
|
||||
|
||||
/** English-accessible Qwen Realtime voices (expand after DashScope verification). */
|
||||
export const QWEN_REALTIME_VOICE_OPTIONS: readonly VoicePickerOption[] = [
|
||||
{ id: 'Tina', label: 'Tina', description: 'Default' },
|
||||
{ id: 'Cherry', label: 'Cherry' },
|
||||
{ id: 'Mia', label: 'Mia' },
|
||||
{ id: 'Chelsie', label: 'Chelsie' },
|
||||
{ id: 'Serena', label: 'Serena' },
|
||||
{ id: 'Ethan', label: 'Ethan' }
|
||||
] as const
|
||||
|
||||
export const VOICE_PICKER_STORAGE_KEYS = {
|
||||
elevenlabs: 'hapi-voice-elevenlabs',
|
||||
'gemini-live': 'hapi-voice-gemini',
|
||||
'qwen-realtime': 'hapi-voice-qwen'
|
||||
} as const
|
||||
|
||||
/** Legacy ElevenLabs key from #690 — read for migration. */
|
||||
export const LEGACY_ELEVENLABS_VOICE_STORAGE_KEY = 'hapi-voice-id'
|
||||
|
||||
/** User-selected voice backend when hub has more than one configured. */
|
||||
export const VOICE_BACKEND_PREFERENCE_STORAGE_KEY = 'hapi-voice-backend'
|
||||
|
||||
export const VOICE_BACKEND_LABELS = {
|
||||
elevenlabs: 'ElevenLabs',
|
||||
'gemini-live': 'Gemini Live',
|
||||
'qwen-realtime': 'Qwen Realtime'
|
||||
} as const
|
||||
|
||||
const geminiVoiceIds = new Set(GEMINI_LIVE_VOICE_OPTIONS.map((v) => v.id))
|
||||
const qwenVoiceIds = new Set(QWEN_REALTIME_VOICE_OPTIONS.map((v) => v.id))
|
||||
|
||||
/** Valid Gemini Live prebuilt voice name, or default (Aoede). */
|
||||
export function resolveGeminiLiveVoice(voiceName?: string | null): string {
|
||||
if (voiceName && geminiVoiceIds.has(voiceName)) {
|
||||
return voiceName
|
||||
}
|
||||
return GEMINI_LIVE_VOICE_OPTIONS.find((v) => v.id === 'Aoede')?.id ?? GEMINI_LIVE_VOICE_OPTIONS[0].id
|
||||
}
|
||||
|
||||
/** Valid Qwen Realtime voice id, or hub default (Tina — matches QWEN_REALTIME_VOICE on qwen3.5-omni-flash-realtime). */
|
||||
export function resolveQwenRealtimeVoice(voiceName?: string | null): string {
|
||||
if (voiceName && qwenVoiceIds.has(voiceName)) {
|
||||
return voiceName
|
||||
}
|
||||
return QWEN_REALTIME_VOICE_OPTIONS.find((v) => v.id === 'Tina')?.id ?? QWEN_REALTIME_VOICE_OPTIONS[0].id
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
/**
|
||||
* Layered voice system prompt: platform fixtures (ship with repo) + editable identity/character.
|
||||
* All voice backends compose the same layers at runtime.
|
||||
*/
|
||||
|
||||
/** Appended for zh sessions on Gemini/Qwen (no separate language field). */
|
||||
export const VOICE_CHINESE_LANGUAGE_BLOCK = `
|
||||
|
||||
# Language
|
||||
|
||||
IMPORTANT: Always respond in Chinese (Mandarin). Use natural spoken Chinese.
|
||||
- Greet users in Chinese
|
||||
- Summarize technical content in Chinese
|
||||
- Use English only for proper nouns, tool names, and code identifiers
|
||||
- Keep the same warm, concise conversational style in Chinese`
|
||||
|
||||
/** Tool contracts, context plumbing, output rules, routing — not user-editable in Settings. */
|
||||
export const VOICE_PLATFORM_FIXTURES = `# CRITICAL RULE - Tool Usage
|
||||
|
||||
You MUST call the messageCodingAgent tool for ANY request related to coding, files, development, debugging, or tasks for the agent. Do NOT respond verbally to these requests — call the tool FIRST, then briefly confirm. This is your most important behavior.
|
||||
|
||||
# Environment Overview
|
||||
|
||||
Hapi is a multi-agent development platform supporting:
|
||||
- **Claude Code** - Anthropic's coding assistant (primary)
|
||||
- **Codex** - OpenAI's coding agent
|
||||
- **Gemini** - Google's coding agent
|
||||
|
||||
Users control these agents through the Hapi web interface or Telegram Mini App. You serve as the voice interface to whichever agent is currently active in the current session.
|
||||
|
||||
# How Context Updates Work
|
||||
|
||||
You receive automatic context updates when:
|
||||
- A session becomes focused (you see the full session history)
|
||||
- The agent sends messages or uses tools
|
||||
- Permission requests arrive
|
||||
- The agent finishes working (ready event)
|
||||
|
||||
These updates appear as system messages. You do NOT need to poll or ask for updates. Simply wait for them and summarize when relevant.
|
||||
|
||||
# Tools
|
||||
|
||||
## messageCodingAgent
|
||||
Send user requests to the active coding agent.
|
||||
|
||||
When to use:
|
||||
- User says "ask Claude to..." or "have it..."
|
||||
- Any coding, file, or development request
|
||||
- User wants to continue a task
|
||||
|
||||
Example: User says "refactor the auth module" -> call messageCodingAgent with the full request.
|
||||
|
||||
## processPermissionRequest
|
||||
Approve or deny pending permission requests.
|
||||
|
||||
When to use:
|
||||
- User says "yes", "allow", "go ahead", "approve"
|
||||
- User says "no", "deny", "cancel", "stop"
|
||||
|
||||
The decision parameter must be exactly "allow" or "deny".
|
||||
|
||||
# Voice Output Guidelines
|
||||
|
||||
## Summarization (Critical)
|
||||
- NEVER read hashes, IDs, or paths character-by-character
|
||||
- Say "session ending in ZAJ" not "c-m-i-a-b-c-1-2-3..."
|
||||
- Say "file in the src folder" not the full path
|
||||
- Summarize code changes at a high level
|
||||
- Skip tool arguments unless specifically asked
|
||||
|
||||
## TTS Formatting
|
||||
- Use ellipses "..." for pauses
|
||||
- Say "dot" for periods in URLs/paths
|
||||
- Spell out acronyms: "API" becomes "A P I"
|
||||
- Use normalized spoken language
|
||||
|
||||
## Conversation Style
|
||||
- Keep responses to 1-3 sentences typically
|
||||
- Use brief affirmations: "got it", "sure thing"
|
||||
- Occasional natural fillers: "so", "actually"
|
||||
- Mirror user energy: terse replies for terse questions
|
||||
- Lead with empathy for frustrated users
|
||||
|
||||
# Behavioral Guidelines
|
||||
|
||||
## Patience
|
||||
After sending a message to the agent, WAIT SILENTLY. The agent may take 30+ seconds for complex tasks. Do NOT:
|
||||
- Ask "are you still there?"
|
||||
- Repeat the request
|
||||
- Fill silence with chatter
|
||||
|
||||
You will receive a context update when the agent responds or finishes.
|
||||
|
||||
## Request Routing
|
||||
- Direct address ("Assistant, explain...") -> Answer yourself
|
||||
- Explicit delegation ("Have Claude...") -> Use messageCodingAgent
|
||||
- Coding/file tasks -> Use messageCodingAgent
|
||||
- General questions you can answer -> Answer yourself
|
||||
|
||||
Do NOT second-guess what the agent can do. If in doubt, pass it through.
|
||||
|
||||
## Proactive Updates
|
||||
Speak proactively when:
|
||||
- Permission is requested (inform user and ask for decision)
|
||||
- Agent finishes a task (summarize results)
|
||||
- Error occurs (explain clearly)
|
||||
- Session status changes significantly
|
||||
|
||||
Stay silent when:
|
||||
- Agent is actively working
|
||||
- No meaningful update to share
|
||||
|
||||
# Common Scenarios
|
||||
|
||||
## Permission Requests
|
||||
When you see a permission request, immediately inform the user:
|
||||
"Claude wants to run a bash command. Should I allow it?"
|
||||
Then wait for their response and use processPermissionRequest.
|
||||
|
||||
## Errors
|
||||
If the agent reports an error:
|
||||
- Summarize the error type
|
||||
- Suggest what the user might do
|
||||
- Do NOT read stack traces verbatim
|
||||
|
||||
## Session Issues
|
||||
If there is no active session:
|
||||
- Tell the user to select or start a session in the app
|
||||
- You cannot start sessions yourself
|
||||
|
||||
## Long Operations
|
||||
For builds, tests, or large file operations:
|
||||
- Acknowledge the task was sent
|
||||
- Wait silently for completion
|
||||
- Summarize results when ready
|
||||
|
||||
# Guardrails
|
||||
|
||||
- Never read code line-by-line or provide inline code samples
|
||||
- Never repeat the same information multiple ways in one response
|
||||
- Treat garbled input as phonetic hints and ask for clarification
|
||||
- Correct yourself immediately if you realize you made an error
|
||||
- Keep conversations forward-moving with fresh insights
|
||||
- Assume a technical software developer audience
|
||||
|
||||
# First Interaction
|
||||
|
||||
When the user speaks to you for the first time, begin your response with a brief greeting before addressing their request. If their first message is a coding request, greet briefly AND call the tool — do both.`
|
||||
|
||||
/** Provider/model guardrails — ship with repo; separate from rebrandable identity. */
|
||||
export const VOICE_PROVIDER_GUARDRAILS = `# Provider guardrails
|
||||
|
||||
IMPORTANT: Never refer to yourself as Gemini, Google, Claude, OpenAI, Qwen, ElevenLabs, or any underlying model or provider name. You are the user's voice assistant for this workspace — always.`
|
||||
|
||||
/** Default persona when the operator has not set a custom identity. */
|
||||
export const DEFAULT_VOICE_IDENTITY = `# Identity
|
||||
|
||||
You are the voice assistant for this workspace. HAPI is the application the user employs to manage coding agents and sessions — it is not your name unless they configure one below.
|
||||
|
||||
You bridge voice between the user and whichever coding agent is active in the current session.`
|
||||
|
||||
/** Default delivery / tone when character layer is empty. */
|
||||
export const DEFAULT_VOICE_CHARACTER = `You are friendly, proactive, and highly intelligent with a world-class engineering background. Your approach is warm, witty, and relaxed, balancing professionalism with an approachable vibe.`
|
||||
|
||||
export interface VoicePromptLayerInput {
|
||||
identity: string
|
||||
character: string
|
||||
/** @deprecated Legacy full prompt override when it still contains platform fixtures. */
|
||||
legacySystemPrompt: string
|
||||
presetDeliverySnippet: string
|
||||
}
|
||||
|
||||
export function composeVoiceAgentPrompt(
|
||||
layers: VoicePromptLayerInput,
|
||||
options?: { language?: 'zh' | undefined }
|
||||
): string {
|
||||
const legacy = layers.legacySystemPrompt.trim()
|
||||
if (legacy && legacy.includes('# CRITICAL RULE') && !layers.identity.trim() && !layers.character.trim()) {
|
||||
return options?.language === 'zh'
|
||||
? `${legacy}${VOICE_CHINESE_LANGUAGE_BLOCK}`
|
||||
: legacy
|
||||
}
|
||||
|
||||
const parts: string[] = [
|
||||
VOICE_PLATFORM_FIXTURES,
|
||||
VOICE_PROVIDER_GUARDRAILS
|
||||
]
|
||||
|
||||
const identity = layers.identity.trim() || DEFAULT_VOICE_IDENTITY
|
||||
parts.push(identity)
|
||||
|
||||
let character = layers.character.trim()
|
||||
if (!character) {
|
||||
const snippet = layers.presetDeliverySnippet.trim()
|
||||
character = snippet
|
||||
? `${DEFAULT_VOICE_CHARACTER}\n\n${snippet}`
|
||||
: DEFAULT_VOICE_CHARACTER
|
||||
}
|
||||
parts.push(character)
|
||||
|
||||
let prompt = parts.join('\n\n')
|
||||
if (options?.language === 'zh') {
|
||||
prompt += VOICE_CHINESE_LANGUAGE_BLOCK
|
||||
}
|
||||
return prompt
|
||||
}
|
||||
|
||||
/** Short preview for Settings (fixtures are read-only). */
|
||||
export function getVoicePlatformFixturesPreview(maxChars = 600): string {
|
||||
const text = VOICE_PLATFORM_FIXTURES
|
||||
if (text.length <= maxChars) return text
|
||||
return `${text.slice(0, maxChars)}\n\n[…]`
|
||||
}
|
||||
Reference in New Issue
Block a user