From 15d550d39a535b6bcf515777278849fe3789f13f Mon Sep 17 00:00:00 2001 From: weishu Date: Tue, 20 Jan 2026 00:57:01 +0800 Subject: [PATCH] feat: add voice language selection in settings Add support for users to select voice language preference, which is passed through to ElevenLabs agents via platform settings overrides. Includes new language mapping utilities, UI controls in settings page, and updated voice session initialization. --- shared/src/voice.ts | 21 ++++ web/src/lib/languages.ts | 117 ++++++++++++++++++++++ web/src/lib/locales/en.ts | 3 + web/src/lib/locales/zh-CN.ts | 3 + web/src/lib/voice-context.tsx | 8 +- web/src/realtime/RealtimeSession.ts | 10 +- web/src/realtime/RealtimeVoiceSession.tsx | 7 ++ web/src/realtime/types.ts | 3 + web/src/routes/settings/index.tsx | 99 +++++++++++++++++- 9 files changed, 263 insertions(+), 8 deletions(-) create mode 100644 web/src/lib/languages.ts diff --git a/shared/src/voice.ts b/shared/src/voice.ts index b2d69157..6751f0eb 100644 --- a/shared/src/voice.ts +++ b/shared/src/voice.ts @@ -201,6 +201,16 @@ export interface VoiceAgentConfig { speed: number } } + platform_settings?: { + overrides?: { + conversation_config_override?: { + agent?: { + language?: boolean + first_message?: boolean + } + } + } + } } /** @@ -231,6 +241,17 @@ export function buildVoiceAgentConfig(): VoiceAgentConfig { model_id: 'eleven_flash_v2', speed: 1.1 } + }, + // Enable runtime overrides for language selection + // See: https://elevenlabs.io/docs/agents-platform/customization/personalization/overrides + platform_settings: { + overrides: { + conversation_config_override: { + agent: { + language: true + } + } + } } } } diff --git a/web/src/lib/languages.ts b/web/src/lib/languages.ts new file mode 100644 index 00000000..0d98fb9c --- /dev/null +++ b/web/src/lib/languages.ts @@ -0,0 +1,117 @@ +// ElevenLabs supported language codes +export type ElevenLabsLanguage = "en" | "ja" | "zh" | "de" | "hi" | "fr" | "ko" | + "pt" | "pt-br" | "it" | "es" | "id" | "nl" | "tr" | "pl" | "sv" | "bg" | + "ro" | "ar" | "cs" | "el" | "fi" | "ms" | "da" | "ta" | "uk" | "ru" | + "hu" | "hr" | "sk" | "no" | "vi" | "tl"; + +// Language type definition +export interface Language { + code: string | null; // null for autodetect + name: string; + nativeName: string; + region?: string; + elevenLabsCode?: ElevenLabsLanguage; // ElevenLabs language code mapping +} + +// Comprehensive language list with locale codes, names, and regions +// First option is autodetect (null value) +export const LANGUAGES: Language[] = [ + { code: null, name: 'Auto-detect', nativeName: 'Auto-detect' }, + { code: 'en-US', name: 'English', nativeName: 'English', region: 'United States', elevenLabsCode: 'en' }, + { code: 'en-GB', name: 'English', nativeName: 'English', region: 'United Kingdom', elevenLabsCode: 'en' }, + { code: 'en-AU', name: 'English', nativeName: 'English', region: 'Australia', elevenLabsCode: 'en' }, + { code: 'en-CA', name: 'English', nativeName: 'English', region: 'Canada', elevenLabsCode: 'en' }, + { code: 'es-ES', name: 'Spanish', nativeName: 'Español', region: 'Spain', elevenLabsCode: 'es' }, + { code: 'es-MX', name: 'Spanish', nativeName: 'Español', region: 'Mexico', elevenLabsCode: 'es' }, + { code: 'es-AR', name: 'Spanish', nativeName: 'Español', region: 'Argentina', elevenLabsCode: 'es' }, + { code: 'fr-FR', name: 'French', nativeName: 'Français', region: 'France', elevenLabsCode: 'fr' }, + { code: 'fr-CA', name: 'French', nativeName: 'Français', region: 'Canada', elevenLabsCode: 'fr' }, + { code: 'de-DE', name: 'German', nativeName: 'Deutsch', region: 'Germany', elevenLabsCode: 'de' }, + { code: 'de-AT', name: 'German', nativeName: 'Deutsch', region: 'Austria', elevenLabsCode: 'de' }, + { code: 'it-IT', name: 'Italian', nativeName: 'Italiano', elevenLabsCode: 'it' }, + { code: 'pt-BR', name: 'Portuguese', nativeName: 'Português', region: 'Brazil', elevenLabsCode: 'pt-br' }, + { code: 'pt-PT', name: 'Portuguese', nativeName: 'Português', region: 'Portugal', elevenLabsCode: 'pt' }, + { code: 'ru-RU', name: 'Russian', nativeName: 'Русский', elevenLabsCode: 'ru' }, + { code: 'zh-CN', name: 'Chinese', nativeName: '中文', region: 'Simplified', elevenLabsCode: 'zh' }, + { code: 'zh-TW', name: 'Chinese', nativeName: '中文', region: 'Traditional', elevenLabsCode: 'zh' }, + { code: 'ja-JP', name: 'Japanese', nativeName: '日本語', elevenLabsCode: 'ja' }, + { code: 'ko-KR', name: 'Korean', nativeName: '한국어', elevenLabsCode: 'ko' }, + { code: 'ar-SA', name: 'Arabic', nativeName: 'العربية', elevenLabsCode: 'ar' }, + { code: 'hi-IN', name: 'Hindi', nativeName: 'हिन्दी', elevenLabsCode: 'hi' }, + { code: 'nl-NL', name: 'Dutch', nativeName: 'Nederlands', elevenLabsCode: 'nl' }, + { code: 'sv-SE', name: 'Swedish', nativeName: 'Svenska', elevenLabsCode: 'sv' }, + { code: 'no-NO', name: 'Norwegian', nativeName: 'Norsk', elevenLabsCode: 'no' }, + { code: 'da-DK', name: 'Danish', nativeName: 'Dansk', elevenLabsCode: 'da' }, + { code: 'fi-FI', name: 'Finnish', nativeName: 'Suomi', elevenLabsCode: 'fi' }, + { code: 'pl-PL', name: 'Polish', nativeName: 'Polski', elevenLabsCode: 'pl' }, + { code: 'tr-TR', name: 'Turkish', nativeName: 'Türkçe', elevenLabsCode: 'tr' }, + { code: 'he-IL', name: 'Hebrew', nativeName: 'עברית' }, // Not supported by ElevenLabs + { code: 'th-TH', name: 'Thai', nativeName: 'ไทย' }, // Not supported by ElevenLabs + { code: 'vi-VN', name: 'Vietnamese', nativeName: 'Tiếng Việt', elevenLabsCode: 'vi' }, + { code: 'id-ID', name: 'Indonesian', nativeName: 'Bahasa Indonesia', elevenLabsCode: 'id' }, + { code: 'ms-MY', name: 'Malay', nativeName: 'Bahasa Melayu', elevenLabsCode: 'ms' }, + { code: 'tl-PH', name: 'Tagalog', nativeName: 'Tagalog', elevenLabsCode: 'tl' }, + { code: 'uk-UA', name: 'Ukrainian', nativeName: 'Українська', elevenLabsCode: 'uk' }, + { code: 'cs-CZ', name: 'Czech', nativeName: 'Čeština', elevenLabsCode: 'cs' }, + { code: 'hu-HU', name: 'Hungarian', nativeName: 'Magyar', elevenLabsCode: 'hu' }, + { code: 'ro-RO', name: 'Romanian', nativeName: 'Română', elevenLabsCode: 'ro' }, + { code: 'bg-BG', name: 'Bulgarian', nativeName: 'Български', elevenLabsCode: 'bg' }, + { code: 'el-GR', name: 'Greek', nativeName: 'Ελληνικά', elevenLabsCode: 'el' }, + { code: 'hr-HR', name: 'Croatian', nativeName: 'Hrvatski', elevenLabsCode: 'hr' }, + { code: 'sk-SK', name: 'Slovak', nativeName: 'Slovenčina', elevenLabsCode: 'sk' }, + { code: 'sl-SI', name: 'Slovenian', nativeName: 'Slovenščina' }, // Not supported by ElevenLabs + { code: 'et-EE', name: 'Estonian', nativeName: 'Eesti' }, // Not supported by ElevenLabs + { code: 'lv-LV', name: 'Latvian', nativeName: 'Latviešu' }, // Not supported by ElevenLabs + { code: 'lt-LT', name: 'Lithuanian', nativeName: 'Lietuvių' }, // Not supported by ElevenLabs +]; + +/** + * Format display name for a language + */ +export const getLanguageDisplayName = (language: Language) => { + const parts = []; + + if (language.name !== language.nativeName) { + parts.push(`${language.name} (${language.nativeName})`); + } else { + parts.push(language.name); + } + + if (language.region) { + parts.push(language.region); + } + + return parts.join(' - '); +}; + +/** + * Find a language by its code (including null for autodetect) + */ +export const findLanguageByCode = (code: string | null): Language | undefined => { + return LANGUAGES.find(lang => lang.code === code); +}; + +/** + * Get the ElevenLabs language code for a given language + */ +export const getElevenLabsCode = (language: Language): ElevenLabsLanguage | undefined => { + return language.elevenLabsCode; +}; + +/** + * Get ElevenLabs code from user's language preference (handles null/autodetect) + */ +export const getElevenLabsCodeFromPreference = ( + languageCode: string | null +): ElevenLabsLanguage | undefined => { + if (!languageCode) return undefined; // Auto-detect case + const language = findLanguageByCode(languageCode); + return language?.elevenLabsCode; +}; + +/** + * Get all languages that support ElevenLabs (including auto-detect) + */ +export const getElevenLabsSupportedLanguages = (): Language[] => { + return LANGUAGES.filter(lang => lang.code === null || lang.elevenLabsCode !== undefined); +}; diff --git a/web/src/lib/locales/en.ts b/web/src/lib/locales/en.ts index 408d780c..d68f5562 100644 --- a/web/src/lib/locales/en.ts +++ b/web/src/lib/locales/en.ts @@ -229,6 +229,9 @@ export default { 'settings.title': 'Settings', 'settings.language.title': 'Language', 'settings.language.label': 'Language', + 'settings.voice.title': 'Voice Assistant', + 'settings.voice.language': 'Voice Language', + 'settings.voice.autoDetect': 'Auto-detect', // Misc 'misc.noMachines': 'No machines available', diff --git a/web/src/lib/locales/zh-CN.ts b/web/src/lib/locales/zh-CN.ts index 569939b1..ae75adea 100644 --- a/web/src/lib/locales/zh-CN.ts +++ b/web/src/lib/locales/zh-CN.ts @@ -231,6 +231,9 @@ export default { 'settings.title': '设置', 'settings.language.title': '语言', 'settings.language.label': '语言', + 'settings.voice.title': '语音助手', + 'settings.voice.language': '语音语言', + 'settings.voice.autoDetect': '自动检测', // Misc 'misc.noMachines': '无可用机器', diff --git a/web/src/lib/voice-context.tsx b/web/src/lib/voice-context.tsx index 473ce195..71424a43 100644 --- a/web/src/lib/voice-context.tsx +++ b/web/src/lib/voice-context.tsx @@ -1,6 +1,7 @@ import { createContext, useCallback, useContext, useState, type ReactNode } from 'react' import type { ConversationStatus, StatusCallback } from '@/realtime/types' import { startRealtimeSession, stopRealtimeSession, voiceHooks } from '@/realtime' +import { getElevenLabsCodeFromPreference } from '@/lib/languages' interface VoiceContextValue { status: ConversationStatus @@ -38,7 +39,12 @@ export function VoiceProvider({ children }: { children: ReactNode }) { const startVoice = useCallback(async (sessionId: string) => { setCurrentSessionId(sessionId) const initialContext = voiceHooks.onVoiceStarted(sessionId) - await startRealtimeSession(sessionId, initialContext) + + // Read voice language preference from localStorage + const voiceLang = localStorage.getItem('hapi-voice-lang') + const elevenLabsLang = getElevenLabsCodeFromPreference(voiceLang) + + await startRealtimeSession(sessionId, initialContext, elevenLabsLang) }, []) const stopVoice = useCallback(async () => { diff --git a/web/src/realtime/RealtimeSession.ts b/web/src/realtime/RealtimeSession.ts index fffc5ed7..132304f8 100644 --- a/web/src/realtime/RealtimeSession.ts +++ b/web/src/realtime/RealtimeSession.ts @@ -1,10 +1,15 @@ import type { VoiceSession } from './types' +import type { ElevenLabsLanguage } from '@/lib/languages' let voiceSession: VoiceSession | null = null let voiceSessionStarted = false let currentSessionId: string | null = null -export async function startRealtimeSession(sessionId: string, initialContext?: string) { +export async function startRealtimeSession( + sessionId: string, + initialContext?: string, + language?: ElevenLabsLanguage +) { if (!voiceSession) { console.warn('[Voice] No voice session registered') return @@ -14,7 +19,8 @@ export async function startRealtimeSession(sessionId: string, initialContext?: s currentSessionId = sessionId await voiceSession.startSession({ sessionId, - initialContext + initialContext, + language }) voiceSessionStarted = true } catch (error) { diff --git a/web/src/realtime/RealtimeVoiceSession.tsx b/web/src/realtime/RealtimeVoiceSession.tsx index 71e00b71..fff9b7b4 100644 --- a/web/src/realtime/RealtimeVoiceSession.tsx +++ b/web/src/realtime/RealtimeVoiceSession.tsx @@ -70,6 +70,13 @@ class RealtimeVoiceSessionImpl implements VoiceSession { dynamicVariables: { sessionId: config.sessionId, initialConversationContext: config.initialContext || '' + }, + // Language override - requires agent to have platform_settings.overrides enabled + // See: https://elevenlabs.io/docs/agents-platform/customization/personalization/overrides + overrides: { + agent: { + language: config.language + } } }) diff --git a/web/src/realtime/types.ts b/web/src/realtime/types.ts index 3daefa38..47753c52 100644 --- a/web/src/realtime/types.ts +++ b/web/src/realtime/types.ts @@ -1,6 +1,9 @@ +import type { ElevenLabsLanguage } from '@/lib/languages' + export interface VoiceSessionConfig { sessionId: string initialContext?: string + language?: ElevenLabsLanguage } export interface VoiceSession { diff --git a/web/src/routes/settings/index.tsx b/web/src/routes/settings/index.tsx index 9ac85c2a..4730d022 100644 --- a/web/src/routes/settings/index.tsx +++ b/web/src/routes/settings/index.tsx @@ -1,12 +1,15 @@ import { useState, useRef, useEffect } from 'react' import { useTranslation, type Locale } from '@/lib/use-translation' import { useAppGoBack } from '@/hooks/useAppGoBack' +import { getElevenLabsSupportedLanguages, getLanguageDisplayName, type Language } from '@/lib/languages' const locales: { value: Locale; nativeLabel: string }[] = [ { value: 'en', nativeLabel: 'English' }, { value: 'zh-CN', nativeLabel: '简体中文' }, ] +const voiceLanguages = getElevenLabsSupportedLanguages() + function BackIcon(props: { className?: string }) { return ( (null) + const voiceContainerRef = useRef(null) + + // Voice language state - read from localStorage + const [voiceLanguage, setVoiceLanguage] = useState(() => { + return localStorage.getItem('hapi-voice-lang') + }) const currentLocale = locales.find((loc) => loc.value === locale) + const currentVoiceLanguage = voiceLanguages.find((lang) => lang.code === voiceLanguage) const handleLocaleChange = (newLocale: Locale) => { setLocale(newLocale) setIsOpen(false) } + const handleVoiceLanguageChange = (language: Language) => { + setVoiceLanguage(language.code) + if (language.code === null) { + localStorage.removeItem('hapi-voice-lang') + } else { + localStorage.setItem('hapi-voice-lang', language.code) + } + setIsVoiceOpen(false) + } + // Close dropdown when clicking outside useEffect(() => { - if (!isOpen) return + if (!isOpen && !isVoiceOpen) return const handleClickOutside = (event: MouseEvent) => { - if (containerRef.current && !containerRef.current.contains(event.target as Node)) { + if (isOpen && containerRef.current && !containerRef.current.contains(event.target as Node)) { setIsOpen(false) } + if (isVoiceOpen && voiceContainerRef.current && !voiceContainerRef.current.contains(event.target as Node)) { + setIsVoiceOpen(false) + } } document.addEventListener('mousedown', handleClickOutside) return () => document.removeEventListener('mousedown', handleClickOutside) - }, [isOpen]) + }, [isOpen, isVoiceOpen]) // Close on escape key useEffect(() => { - if (!isOpen) return + if (!isOpen && !isVoiceOpen) return const handleEscape = (event: KeyboardEvent) => { if (event.key === 'Escape') { setIsOpen(false) + setIsVoiceOpen(false) } } document.addEventListener('keydown', handleEscape) return () => document.removeEventListener('keydown', handleEscape) - }, [isOpen]) + }, [isOpen, isVoiceOpen]) return (
@@ -176,6 +201,70 @@ export default function SettingsPage() { )}
+ + {/* Voice Assistant section */} +
+
+ {t('settings.voice.title')} +
+
+ + + {isVoiceOpen && ( +
+ {voiceLanguages.map((lang) => { + const isSelected = voiceLanguage === lang.code + const displayName = lang.code === null + ? t('settings.voice.autoDetect') + : getLanguageDisplayName(lang) + return ( + + ) + })} +
+ )} +
+