From ae671c123bb2acc94a9226556716e58e93ce338a Mon Sep 17 00:00:00 2001 From: HeavyGee <133152184+heavygee@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:01:32 +0100 Subject: [PATCH] fix(web): deliver voice session bootstrap via contextual updates (#1344) ElevenLabs only passed bootstrap context through dynamicVariables without a matching {{initialConversationContext}} prompt placeholder, so Brief me connected with no session history. Stream deferred chunks then push bootstrap on all backends; add the placeholder for newly created ConvAI agents. Co-authored-by: Cursor --- shared/src/voice.backends.test.ts | 8 ++++ shared/src/voice.ts | 13 ++++++- web/src/lib/voiceContextStream.test.ts | 42 +++++++++++++++++++++ web/src/lib/voiceContextStream.ts | 23 +++++++++++ web/src/realtime/GeminiLiveVoiceSession.tsx | 17 +++------ web/src/realtime/QwenVoiceSession.tsx | 17 +++------ web/src/realtime/RealtimeVoiceSession.tsx | 13 +++---- 7 files changed, 103 insertions(+), 30 deletions(-) create mode 100644 web/src/lib/voiceContextStream.test.ts diff --git a/shared/src/voice.backends.test.ts b/shared/src/voice.backends.test.ts index bb3d3ceb..9de033c2 100644 --- a/shared/src/voice.backends.test.ts +++ b/shared/src/voice.backends.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from 'bun:test' import { + buildVoiceAgentConfig, listConfiguredTranscriptionProviders, listConfiguredVoiceBackends, resolveEffectiveVoiceBackend, @@ -42,6 +43,13 @@ describe('listConfiguredVoiceBackends', () => { }) }) +describe('buildVoiceAgentConfig', () => { + test('includes ElevenLabs session context placeholder for dynamicVariables', () => { + const prompt = buildVoiceAgentConfig().conversation_config.agent.prompt.prompt + expect(prompt).toContain('{{initialConversationContext}}') + }) +}) + describe('resolveHubVoiceBackend', () => { test('uses VOICE_BACKEND when that backend is configured', () => { const backend = resolveHubVoiceBackend({ diff --git a/shared/src/voice.ts b/shared/src/voice.ts index 82141982..5f32bf39 100644 --- a/shared/src/voice.ts +++ b/shared/src/voice.ts @@ -95,6 +95,17 @@ no ${name} equivalent.` /** ElevenLabs first message — language controlled by ElevenLabs language field */ export const VOICE_FIRST_MESSAGE = "Hey! Hapi here — what can I help you with?" +/** Appended to ElevenLabs ConvAI agent prompt; filled via startSession dynamicVariables. */ +export const VOICE_ELEVENLABS_SESSION_CONTEXT_BLOCK = ` + +# Active session at connect + +The user connected from a live coding session. Snapshot at connect (may be empty if unavailable): + +{{initialConversationContext}} + +Use this plus any later context updates when briefing or routing.` + export const VOICE_TOOLS = [ { type: 'client' as const, @@ -191,7 +202,7 @@ export function buildVoiceAgentConfig(): VoiceAgentConfig { first_message: VOICE_FIRST_MESSAGE, language: 'en', prompt: { - prompt: VOICE_SYSTEM_PROMPT, + prompt: VOICE_SYSTEM_PROMPT + VOICE_ELEVENLABS_SESSION_CONTEXT_BLOCK, llm: 'gemini-2.5-flash', temperature: 0.7, max_tokens: 1024, diff --git a/web/src/lib/voiceContextStream.test.ts b/web/src/lib/voiceContextStream.test.ts new file mode 100644 index 00000000..b8efa271 --- /dev/null +++ b/web/src/lib/voiceContextStream.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test, vi } from 'vitest' +import { deliverVoiceSessionContextAfterConnect } from './voiceContextStream' + +describe('deliverVoiceSessionContextAfterConnect', () => { + test('streams older chunks then bootstrap context', async () => { + const sent: string[] = [] + const sendChunk = vi.fn((chunk: string) => { + sent.push(chunk) + }) + + await deliverVoiceSessionContextAfterConnect({ + streamContextChunks: ['older-a', 'older-b'], + initialContext: 'bootstrap recent', + sendChunk + }) + + expect(sent).toEqual(['older-a', 'older-b', 'bootstrap recent']) + }) + + test('sends bootstrap even when there are no stream chunks', async () => { + const sent: string[] = [] + + await deliverVoiceSessionContextAfterConnect({ + initialContext: ' session header only ', + sendChunk: (chunk) => sent.push(chunk) + }) + + expect(sent).toEqual(['session header only']) + }) + + test('skips empty bootstrap', async () => { + const sent: string[] = [] + + await deliverVoiceSessionContextAfterConnect({ + streamContextChunks: ['chunk'], + initialContext: ' ', + sendChunk: (chunk) => sent.push(chunk) + }) + + expect(sent).toEqual(['chunk']) + }) +}) diff --git a/web/src/lib/voiceContextStream.ts b/web/src/lib/voiceContextStream.ts index d1012248..55f05fae 100644 --- a/web/src/lib/voiceContextStream.ts +++ b/web/src/lib/voiceContextStream.ts @@ -32,6 +32,29 @@ export async function streamDeferredVoiceContext( } } +/** + * Stream older history, then deliver bootstrap context — same order on every backend. + * ElevenLabs dynamicVariables alone are insufficient unless the agent prompt references + * {{initialConversationContext}}; contextual updates are the reliable delivery path. + */ +export async function deliverVoiceSessionContextAfterConnect(options: { + streamContextChunks?: string[] + initialContext?: string + sendChunk: (chunk: string) => void + streamDelayMs?: number +}): Promise { + const streamChunks = options.streamContextChunks ?? [] + if (streamChunks.length > 0) { + await streamDeferredVoiceContext(options.sendChunk, streamChunks, { + delayMs: options.streamDelayMs + }) + } + const bootstrap = options.initialContext?.trim() + if (bootstrap) { + options.sendChunk(bootstrap) + } +} + export function isVoiceProactiveSummaryEnabled(): boolean { return localStorage.getItem('hapi-voice-proactive') === 'true' } diff --git a/web/src/realtime/GeminiLiveVoiceSession.tsx b/web/src/realtime/GeminiLiveVoiceSession.tsx index cc50a732..fcd84a49 100644 --- a/web/src/realtime/GeminiLiveVoiceSession.tsx +++ b/web/src/realtime/GeminiLiveVoiceSession.tsx @@ -13,7 +13,7 @@ import { truncatePromptForProxy } from '@/lib/voicePersonalitySession' import { loadVoicePersonalityFromStorage } from '@/hooks/useVoicePersonality' -import { isVoiceProactiveSummaryEnabled, streamDeferredVoiceContext } from '@/lib/voiceContextStream' +import { deliverVoiceSessionContextAfterConnect, isVoiceProactiveSummaryEnabled } from '@/lib/voiceContextStream' import { readStoredVoiceSelection } from '@/lib/voicePickerPreferences' import type { VoiceSession, VoiceSessionConfig, StatusCallback } from './types' import type { ApiClient } from '@/api/client' @@ -200,24 +200,19 @@ class GeminiLiveVoiceSessionImpl implements VoiceSession { } state.statusCallback?.('connected') - await streamDeferredVoiceContext( - (chunk) => sendClientContent(`[Context] ${chunk}`, false), - config.streamContextChunks ?? [] - ) + await deliverVoiceSessionContextAfterConnect({ + streamContextChunks: config.streamContextChunks, + initialContext: config.initialContext, + sendChunk: (chunk) => sendClientContent(`[Context] ${chunk}`, false) + }) const proactive = isVoiceProactiveSummaryEnabled() if (proactive) { - if (config.initialContext?.trim()) { - sendClientContent(`[Context] ${config.initialContext}`, false) - } sendClientContent( '[Summarize] Based on all session context above, give the user a brief spoken summary of what the coding agent has been doing, then wait.', true ) } else { - if (config.initialContext?.trim()) { - sendClientContent(`[Context] ${config.initialContext}`, false) - } sendClientContent( '[Greet the user. Say a brief hello and invite them to speak. Do not mention Gemini or any model name.]', true diff --git a/web/src/realtime/QwenVoiceSession.tsx b/web/src/realtime/QwenVoiceSession.tsx index 31ffad3f..fbe8d2b9 100644 --- a/web/src/realtime/QwenVoiceSession.tsx +++ b/web/src/realtime/QwenVoiceSession.tsx @@ -11,7 +11,7 @@ import { encodeVoiceSystemPromptForProxy, truncatePromptForProxy } from '@/lib/voicePersonalitySession' -import { isVoiceProactiveSummaryEnabled, streamDeferredVoiceContext } from '@/lib/voiceContextStream' +import { deliverVoiceSessionContextAfterConnect, isVoiceProactiveSummaryEnabled } from '@/lib/voiceContextStream' import { readStoredVoiceSelection } from '@/lib/voicePickerPreferences' import type { VoiceSession, VoiceSessionConfig, StatusCallback } from './types' import type { ApiClient } from '@/api/client' @@ -197,23 +197,18 @@ class QwenVoiceSessionImpl implements VoiceSession { } state.statusCallback?.('connected') - await streamDeferredVoiceContext( - (chunk) => this.sendContextualUpdate(chunk), - config.streamContextChunks ?? [] - ) + await deliverVoiceSessionContextAfterConnect({ + streamContextChunks: config.streamContextChunks, + initialContext: config.initialContext, + sendChunk: (chunk) => this.sendContextualUpdate(chunk) + }) const proactive = isVoiceProactiveSummaryEnabled() if (proactive) { - if (config.initialContext?.trim()) { - this.sendContextualUpdate(config.initialContext) - } this.sendTextMessage( 'Based on all session context above, give me a brief spoken summary of what the coding agent has been doing, then wait.' ) } else { - if (config.initialContext?.trim()) { - this.sendContextualUpdate(config.initialContext) - } this.sendTextMessage( '[Greet the user. Say a brief hello and invite them to speak. Do not mention Qwen or any model name.]' ) diff --git a/web/src/realtime/RealtimeVoiceSession.tsx b/web/src/realtime/RealtimeVoiceSession.tsx index 415b1786..c3375a42 100644 --- a/web/src/realtime/RealtimeVoiceSession.tsx +++ b/web/src/realtime/RealtimeVoiceSession.tsx @@ -4,7 +4,7 @@ import { registerVoiceSession, resetRealtimeSessionState } from './RealtimeSessi import { realtimeClientTools, registerSessionStore } from './realtimeClientTools' import { fetchVoiceToken } from '@/api/voice' import { buildElevenLabsSessionOverrides, capElevenLabsInitialContext } from '@/lib/voicePersonalitySession' -import { isVoiceProactiveSummaryEnabled, streamDeferredVoiceContext } from '@/lib/voiceContextStream' +import { deliverVoiceSessionContextAfterConnect, isVoiceProactiveSummaryEnabled } from '@/lib/voiceContextStream' import { readStoredVoiceSelection } from '@/lib/voicePickerPreferences' import type { VoiceSession, VoiceSessionConfig, ConversationStatus, StatusCallback } from './types' import type { ApiClient } from '@/api/client' @@ -101,12 +101,11 @@ class RealtimeVoiceSessionImpl implements VoiceSession { console.log('[Voice] Started conversation with ID:', conversationId) } - if (config.streamContextChunks?.length) { - await streamDeferredVoiceContext( - (chunk) => conversationInstance?.sendContextualUpdate(chunk), - config.streamContextChunks - ) - } + await deliverVoiceSessionContextAfterConnect({ + streamContextChunks: config.streamContextChunks, + initialContext: config.initialContext, + sendChunk: (chunk) => conversationInstance?.sendContextualUpdate(chunk) + }) if (isVoiceProactiveSummaryEnabled()) { this.sendTextMessage(