mirror of
https://github.com/wu736139669/hapi.git
synced 2026-08-05 06:24:37 +00:00
* feat(voice): voice personality, picker catalog, and prompt layer foundation - voicePickerCatalog.ts: per-backend voice lists for Gemini and Qwen with resolve helpers (resolveGeminiLiveVoice, resolveQwenRealtimeVoice) - voicePersonality.ts: VoicePersonalityPreferences schema, presets, composed system prompt with identity/character/response-length layers - voicePromptLayers.ts: buildResolvedVoiceSystemPrompt, preset delivery snippets - voiceSystemPromptParam.ts: hub-side base64url decode for ?systemPrompt= - voicePickerPreferences.ts, voicePersonalitySession.ts: browser-side encode, decode, and storage helpers - useVoicePersonality: React hook for preferences persistence via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix(voice): preset delivery included when non-balanced preset selected; restore test typecheck - isDefaultVoicePersonality: add preset check so warm/calm/direct presets trigger the delivery snippet instead of being treated as default - web/tsconfig.json: remove test file exclusion from typecheck (restoring strict coverage of test code); fix resulting type error in mock declaration via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix(voice): include use_speaker_boost in ElevenLabs TTS override payload The checkbox persisted the pref but ttsDiffersFromDefault and buildElevenLabsTtsOverride both omitted it, so the setting was never sent to the agent. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * test(voice): update speaker_boost test to assert it IS included in override The previous test asserted use_speaker_boost was omitted; now it's correctly included in the TTS payload. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix(voice): authorize use_speaker_boost in ElevenLabs override schema Add use_speaker_boost to both the VoiceAgentConfig tts override type and the buildVoiceAgentConfig() platform_settings so the field is accepted by the ElevenLabs agent runtime. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix(voice): propagate full language code through composed prompt, not just zh getDefaultVoiceSystemPrompt and resolveComposedVoiceSystemPrompt were filtering language to zh-only before passing to composeVoiceAgentPrompt. Now append buildVoiceLanguageBlock(language) after composition so French, Spanish, Japanese etc. reach Gemini/Qwen sessions correctly. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix(voice): only append language block when language explicitly set Building language block unconditionally when no language is given caused getDefaultVoiceSystemPrompt() to diverge from VOICE_SYSTEM_PROMPT. Only append the block when a code is explicitly provided. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix(voice): always include language block for Gemini/Qwen in composed prompt When auto-detect is on (language=undefined), the composed prompt sent via hub proxy was losing the language auto-detect instruction because the block was only added when language was explicitly set. Now: ElevenLabs skips the block (has its own language field); Gemini/Qwen always include it — undefined produces the auto-detect block, an explicit code produces the appropriate language instruction. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> --------- Co-authored-by: HAPI <noreply@hapi.run>
This commit is contained in:
HeavyGee
+27
-83
@@ -6,7 +6,9 @@ import { existsSync } from 'node:fs'
|
||||
import { serveStatic } from 'hono/bun'
|
||||
import { getConfiguration } from '../configuration'
|
||||
import { PROTOCOL_VERSION } from '@hapi/protocol'
|
||||
import { buildGeminiLiveSetupMessage, buildQwenSessionUpdateMessage, isQwenSafeClientFrame, QWEN_REALTIME_MODEL } from '@hapi/protocol/voice'
|
||||
import { buildGeminiLiveSetupMessage, QWEN_REALTIME_MODEL } from '@hapi/protocol/voice'
|
||||
import { createQwenProxyWebSocketHandler } from './qwenProxyHandler'
|
||||
import { decodeVoiceSystemPromptParam } from '../voiceSystemPromptParam'
|
||||
import type { SyncEngine } from '../sync/syncEngine'
|
||||
import { createAuthMiddleware, type WebAppEnv } from './middleware/auth'
|
||||
import { createAuthRoutes } from './routes/auth'
|
||||
@@ -80,7 +82,14 @@ function createGeminiProxyWebSocketHandler() {
|
||||
|
||||
return {
|
||||
open(clientWs: ServerWebSocket<unknown>) {
|
||||
const data = clientWs.data as { _geminiProxy: boolean; apiKey: string; language?: string }
|
||||
const data = clientWs.data as {
|
||||
_geminiProxy: boolean
|
||||
apiKey: string
|
||||
language?: string
|
||||
voiceName?: string
|
||||
systemInstruction?: string
|
||||
affectiveDialog?: boolean
|
||||
}
|
||||
const upstreamUrl = `${process.env.GEMINI_LIVE_WS_URL || GEMINI_WS_BASE}?key=${encodeURIComponent(data.apiKey)}`
|
||||
const pending: Array<string | ArrayBuffer | Uint8Array> = []
|
||||
pendingMap.set(clientWs, pending)
|
||||
@@ -92,7 +101,12 @@ function createGeminiProxyWebSocketHandler() {
|
||||
upstream.onopen = () => {
|
||||
// Hub-owned setup only — never forward client setup (prevents generic Gemini proxy abuse).
|
||||
// Do NOT flush pending here: wait for Google's setupComplete before forwarding client frames.
|
||||
upstream.send(JSON.stringify(buildGeminiLiveSetupMessage(data.language)))
|
||||
upstream.send(JSON.stringify(buildGeminiLiveSetupMessage(
|
||||
data.language,
|
||||
data.voiceName,
|
||||
data.systemInstruction,
|
||||
{ affectiveDialog: data.affectiveDialog }
|
||||
)))
|
||||
}
|
||||
upstream.onmessage = (event) => {
|
||||
try {
|
||||
@@ -154,84 +168,9 @@ function createGeminiProxyWebSocketHandler() {
|
||||
}
|
||||
}
|
||||
|
||||
// Qwen Realtime WebSocket proxy — bridges browser (no custom headers) to DashScope (requires Authorization header)
|
||||
function createQwenProxyWebSocketHandler() {
|
||||
const QWEN_WS_BASE = 'wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime'
|
||||
const upstreamMap = new WeakMap<ServerWebSocket<unknown>, WebSocket>()
|
||||
// Holds the hub-owned session.update payload until session.created arrives from DashScope.
|
||||
// Sending session.update before session.created violates the Qwen Realtime protocol ordering.
|
||||
const pendingSetupMap = new WeakMap<ServerWebSocket<unknown>, string>()
|
||||
|
||||
return {
|
||||
open(clientWs: ServerWebSocket<unknown>) {
|
||||
const data = clientWs.data as { apiKey: string; model: string; language?: string }
|
||||
const upstreamUrl = `${process.env.QWEN_REALTIME_WS_URL || QWEN_WS_BASE}?model=${encodeURIComponent(data.model)}`
|
||||
|
||||
const upstream = new WebSocket(upstreamUrl, {
|
||||
headers: { 'Authorization': `Bearer ${data.apiKey}` }
|
||||
} as unknown as string[])
|
||||
|
||||
upstreamMap.set(clientWs, upstream)
|
||||
pendingSetupMap.set(clientWs, JSON.stringify(buildQwenSessionUpdateMessage(data.language)))
|
||||
|
||||
upstream.onmessage = (event) => {
|
||||
const raw = event.data
|
||||
const text = typeof raw === 'string'
|
||||
? raw
|
||||
: new TextDecoder().decode(raw instanceof Uint8Array ? raw : new Uint8Array(raw as ArrayBuffer))
|
||||
|
||||
// Respect Qwen protocol ordering: relay session.created first, then send hub-owned
|
||||
// session.update. DashScope must receive session.update after session.created.
|
||||
const pendingSetup = pendingSetupMap.get(clientWs)
|
||||
if (pendingSetup) {
|
||||
try {
|
||||
const parsed = JSON.parse(text) as { type?: string }
|
||||
if (parsed.type === 'session.created') {
|
||||
pendingSetupMap.delete(clientWs)
|
||||
try { if (clientWs.readyState === 1) clientWs.send(text) } catch { /* client gone */ }
|
||||
upstream.send(pendingSetup)
|
||||
return
|
||||
}
|
||||
} catch { /* not JSON — relay as-is below */ }
|
||||
}
|
||||
|
||||
try {
|
||||
if (clientWs.readyState === 1) {
|
||||
clientWs.send(typeof raw === 'string' ? raw : new Uint8Array(raw as ArrayBuffer))
|
||||
}
|
||||
} catch { /* client gone */ }
|
||||
}
|
||||
upstream.onerror = () => {
|
||||
pendingSetupMap.delete(clientWs)
|
||||
upstreamMap.delete(clientWs)
|
||||
try { clientWs.close(1011, 'Upstream error') } catch { /* */ }
|
||||
}
|
||||
upstream.onclose = (event) => {
|
||||
pendingSetupMap.delete(clientWs)
|
||||
try { clientWs.close(toClientCloseCode(event.code), event.reason || 'Upstream closed') } catch { /* client gone */ }
|
||||
upstreamMap.delete(clientWs)
|
||||
}
|
||||
},
|
||||
message(clientWs: ServerWebSocket<unknown>, message: string | ArrayBuffer | Uint8Array) {
|
||||
if (!isQwenSafeClientFrame(message)) {
|
||||
try { clientWs.close(1008, 'Client session.update may only modify instructions') } catch { /* */ }
|
||||
return
|
||||
}
|
||||
const upstream = upstreamMap.get(clientWs)
|
||||
if (upstream?.readyState === WebSocket.OPEN) {
|
||||
upstream.send(message)
|
||||
}
|
||||
},
|
||||
close(clientWs: ServerWebSocket<unknown>, code: number, reason: string) {
|
||||
pendingSetupMap.delete(clientWs)
|
||||
const upstream = upstreamMap.get(clientWs)
|
||||
if (upstream) {
|
||||
try { upstream.close(toClientCloseCode(code), (reason || 'Client closed').slice(0, 123)) } catch { /* */ }
|
||||
upstreamMap.delete(clientWs)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Qwen Realtime WebSocket proxy — bridges browser (no custom headers) to DashScope
|
||||
// (requires Authorization header). Implementation extracted to `./qwenProxyHandler` so
|
||||
// the ack-gating behaviour is unit-testable; `createQwenProxyWebSocketHandler` is imported above.
|
||||
|
||||
function findWebappDistDir(): { distDir: string; indexHtmlPath: string } {
|
||||
const candidates = [
|
||||
@@ -513,8 +452,11 @@ export async function startWebServer(options: {
|
||||
return new Response('Gemini API key not configured', { status: 400 })
|
||||
}
|
||||
const language = url.searchParams.get('language') ?? undefined
|
||||
const voiceParam = url.searchParams.get('voice')?.trim() || undefined
|
||||
const systemInstruction = decodeVoiceSystemPromptParam(url.searchParams.get('systemPrompt'))
|
||||
const affectiveDialog = url.searchParams.get('affectiveDialog') === '1'
|
||||
const upgraded = (server as unknown as { upgrade: (req: Request, opts: unknown) => boolean }).upgrade(req, {
|
||||
data: { _geminiProxy: true, apiKey, language }
|
||||
data: { _geminiProxy: true, apiKey, language, voiceName: voiceParam, systemInstruction, affectiveDialog }
|
||||
})
|
||||
if (!upgraded) {
|
||||
return new Response('WebSocket upgrade failed', { status: 500 })
|
||||
@@ -526,11 +468,13 @@ export async function startWebServer(options: {
|
||||
const apiKey = process.env.DASHSCOPE_API_KEY || process.env.QWEN_API_KEY
|
||||
const model = QWEN_REALTIME_MODEL
|
||||
const language = url.searchParams.get('language') ?? undefined
|
||||
const voiceParam = url.searchParams.get('voice')?.trim() || undefined
|
||||
const systemInstruction = decodeVoiceSystemPromptParam(url.searchParams.get('systemPrompt'))
|
||||
if (!apiKey) {
|
||||
return new Response('DashScope API key not configured', { status: 400 })
|
||||
}
|
||||
const upgraded = (server as unknown as { upgrade: (req: Request, opts: unknown) => boolean }).upgrade(req, {
|
||||
data: { _qwenProxy: true, apiKey, model, language }
|
||||
data: { _qwenProxy: true, apiKey, model, language, voiceName: voiceParam, systemInstruction }
|
||||
})
|
||||
if (!upgraded) {
|
||||
return new Response('WebSocket upgrade failed', { status: 500 })
|
||||
|
||||
Reference in New Issue
Block a user