diff --git a/bun.lock b/bun.lock index 47b2ada9..947bb5cc 100644 --- a/bun.lock +++ b/bun.lock @@ -92,6 +92,7 @@ "dependencies": { "@assistant-ui/react": "^0.11.53", "@assistant-ui/react-markdown": "^0.11.9", + "@elevenlabs/react": "^0.13.0", "@hapi/protocol": "workspace:*", "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-slot": "^1.2.4", @@ -455,6 +456,8 @@ "@braintree/sanitize-url": ["@braintree/sanitize-url@7.1.1", "", {}, "sha512-i1L7noDNxtFyL5DmZafWy1wRVhGehQmzZaz1HiN5e7iylJMSZR7ekOV7NsIqa5qBldlLrsKv4HbgFUVlQrz8Mw=="], + "@bufbuild/protobuf": ["@bufbuild/protobuf@1.10.1", "", {}, "sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ=="], + "@builder.io/jsx-loc-internals": ["@builder.io/jsx-loc-internals@0.0.1", "", { "dependencies": { "@babel/parser": "^7.24.0", "estree-walker": "2.0.2", "magic-string": "^0.30.8" } }, "sha512-cSADapVCi07DDhcuDmcAVItqSVmji7DNyD3xxYTHyNCwhWMNnTpZjyvDIWwYFJLleyDCJ9VUtbaXtUjjqBiRqw=="], "@builder.io/vite-plugin-jsx-loc": ["@builder.io/vite-plugin-jsx-loc@0.1.1", "", { "dependencies": { "@builder.io/jsx-loc-internals": "0.0.1" }, "peerDependencies": { "vite": "^4.0.0 || ^5.0.0" } }, "sha512-iAHFkaLBDJBC+EkGO1hF7hnIW2+oKKYVOl8NFAQH//3xeNEzvGdS9tOALRPR+JjR/M5NLyj+FG0VV7WFb1aJmw=="], @@ -477,6 +480,12 @@ "@docsearch/react": ["@docsearch/react@3.8.2", "", { "dependencies": { "@algolia/autocomplete-core": "1.17.7", "@algolia/autocomplete-preset-algolia": "1.17.7", "@docsearch/css": "3.8.2", "algoliasearch": "^5.14.2" }, "peerDependencies": { "@types/react": ">= 16.8.0 < 19.0.0", "react": ">= 16.8.0 < 19.0.0", "react-dom": ">= 16.8.0 < 19.0.0", "search-insights": ">= 1 < 3" }, "optionalPeers": ["@types/react", "react", "react-dom", "search-insights"] }, "sha512-xCRrJQlTt8N9GU0DG4ptwHRkfnSnD/YpdeaXe02iKfqs97TkZJv60yE+1eq/tjPcVnTW8dP5qLP7itifFVV5eg=="], + "@elevenlabs/client": ["@elevenlabs/client@0.13.0", "", { "dependencies": { "@elevenlabs/types": "0.4.0", "livekit-client": "^2.11.4" } }, "sha512-0UjouSOVK+mECJuYg/DqM9V1NIsabIrlRmw/RjEMS5UmOpsAga3KxMC3et8fKh8G/MI2cZtLr2764wIOZgyrQg=="], + + "@elevenlabs/react": ["@elevenlabs/react@0.13.0", "", { "dependencies": { "@elevenlabs/client": "0.13.0" }, "peerDependencies": { "react": ">=16.8.0" } }, "sha512-IOaRHDCA/x8H+KNYE7r6IOU4mp+PLTwv68Bx3z36B0GzP8Ba8hYL8BDhCtftpUA6gcSJifQefNrUvVcKt5JQ8Q=="], + + "@elevenlabs/types": ["@elevenlabs/types@0.4.0", "", {}, "sha512-6CQb4r9DjepILhLybDxW0h8rvbsR8jPAp74HvFGYuvYdXcQRTkYTD9BQ4PV6VcuC1iqx0UljzJj8js3ADQ8EXw=="], + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="], "@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="], @@ -585,6 +594,10 @@ "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], + "@livekit/mutex": ["@livekit/mutex@1.1.1", "", {}, "sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw=="], + + "@livekit/protocol": ["@livekit/protocol@1.42.2", "", { "dependencies": { "@bufbuild/protobuf": "^1.10.0" } }, "sha512-0jeCwoMJKcwsZICg5S6RZM4xhJoF78qMvQELjACJQn6/VB+jmiySQKOSELTXvPBVafHfEbMlqxUw2UR1jTXs2g=="], + "@medv/finder": ["@medv/finder@4.0.2", "", {}, "sha512-RraNY9SCcx4KZV0Dh6BEW6XEW2swkqYca74pkFFRw6hHItSHiy+O/xMnpbofjYbzXj0tSpBGthUF1hHTsr3vIQ=="], "@mermaid-js/parser": ["@mermaid-js/parser@0.6.3", "", { "dependencies": { "langium": "3.3.1" } }, "sha512-lnjOhe7zyHjc+If7yT4zoedx2vo4sHaTmtkl1+or8BRTnCtDmcTpAjpzDSfCZrshM5bCoz0GyidzadJAH1xobA=="], @@ -845,6 +858,8 @@ "@twsxtd/hapi-linux-x64": ["@twsxtd/hapi-linux-x64@0.10.0", "", { "os": "linux", "cpu": "x64", "bin": { "hapi": "bin/hapi" } }, "sha512-TU6pB/XMz4xVW9LwZqA8sRANd8RK8qrm7jFp7cANuS+L/66J4VGuGmi1cziFhVLleyqqur7fh2J/xJtkY5Cb5w=="], + "@twsxtd/hapi-win32-x64": ["@twsxtd/hapi-win32-x64@0.10.0", "", { "os": "win32", "cpu": "x64", "bin": { "hapi": "bin/hapi.exe" } }, "sha512-n10LemuRIWp/WnZ0WoBm2f2BU4EAdGRw74T3lR5kpFgOPm3QDSbRf+B2kDJqEmZmeCLf6V0hBMU2LullYu4LYw=="], + "@types/babel__core": ["@types/babel__core@7.20.5", "", { "dependencies": { "@babel/parser": "^7.20.7", "@babel/types": "^7.20.7", "@types/babel__generator": "*", "@types/babel__template": "*", "@types/babel__traverse": "*" } }, "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA=="], "@types/babel__generator": ["@types/babel__generator@7.27.0", "", { "dependencies": { "@babel/types": "^7.0.0" } }, "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg=="], @@ -929,6 +944,8 @@ "@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="], + "@types/dom-mediacapture-record": ["@types/dom-mediacapture-record@1.0.22", "", {}, "sha512-mUMZLK3NvwRLcAAT9qmcK+9p7tpU2FHdDsntR3YI4+GY88XrgG4XiE7u1Q2LAN2/FZOz/tdMDC3GQCR4T8nFuw=="], + "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], "@types/estree-jsx": ["@types/estree-jsx@1.0.5", "", { "dependencies": { "@types/estree": "*" } }, "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg=="], @@ -1433,6 +1450,8 @@ "eventemitter3": ["eventemitter3@4.0.7", "", {}, "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw=="], + "events": ["events@3.3.0", "", {}, "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="], + "eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="], "eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="], @@ -1773,6 +1792,8 @@ "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.30.2", "", { "os": "win32", "cpu": "x64" }, "sha512-5g1yc73p+iAkid5phb4oVFMB45417DkRevRbt/El/gKXJk4jid+vPFF/AXbxn05Aky8PapwzZrdJShv5C0avjw=="], + "livekit-client": ["livekit-client@2.17.0", "", { "dependencies": { "@livekit/mutex": "1.1.1", "@livekit/protocol": "1.42.2", "events": "^3.3.0", "jose": "^6.1.0", "loglevel": "^1.9.2", "sdp-transform": "^2.15.0", "ts-debounce": "^4.0.0", "tslib": "2.8.1", "typed-emitter": "^2.1.0", "webrtc-adapter": "^9.0.1" }, "peerDependencies": { "@types/dom-mediacapture-record": "^1" } }, "sha512-BD1QUS44ancVTBdnAher0aO7DV5holFYH2lYradYT/HgXtn6R8xPyvtDAH3UH40jGcesDo9fEopCFwEdOgrIhg=="], + "locate-path": ["locate-path@5.0.0", "", { "dependencies": { "p-locate": "^4.1.0" } }, "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g=="], "lodash": ["lodash@4.17.21", "", {}, "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="], @@ -1783,6 +1804,8 @@ "lodash.sortby": ["lodash.sortby@4.7.0", "", {}, "sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA=="], + "loglevel": ["loglevel@1.9.2", "", {}, "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg=="], + "longest-streak": ["longest-streak@3.1.0", "", {}, "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g=="], "loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="], @@ -2207,6 +2230,10 @@ "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], + "sdp": ["sdp@3.2.1", "", {}, "sha512-lwsAIzOPlH8/7IIjjz3K0zYBk7aBVVcvjMwt3M4fLxpjMYyy7i3I97SLHebgn4YBjirkzfp3RvRDWSKsh/+WFw=="], + + "sdp-transform": ["sdp-transform@2.15.0", "", { "bin": { "sdp-verify": "checker.js" } }, "sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw=="], + "search-insights": ["search-insights@2.17.3", "", {}, "sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ=="], "secure-json-parse": ["secure-json-parse@4.1.0", "", {}, "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA=="], @@ -2381,6 +2408,8 @@ "trough": ["trough@2.2.0", "", {}, "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw=="], + "ts-debounce": ["ts-debounce@4.0.0", "", {}, "sha512-+1iDGY6NmOGidq7i7xZGA4cm8DAa6fqdYcvO5Z6yBevH++Bdo9Qt/mN0TzHUgcCcKv1gmh9+W5dHqz8pMWbCbg=="], + "ts-dedent": ["ts-dedent@2.2.0", "", {}, "sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ=="], "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], @@ -2401,6 +2430,8 @@ "typed-array-length": ["typed-array-length@1.0.7", "", { "dependencies": { "call-bind": "^1.0.7", "for-each": "^0.3.3", "gopd": "^1.0.1", "is-typed-array": "^1.1.13", "possible-typed-array-names": "^1.0.0", "reflect.getprototypeof": "^1.0.6" } }, "sha512-3KS2b+kL7fsuk/eJZ7EQdnEmQoaho/r6KUef7hxvltNA5DR8NAUM+8wJMbJyZ4G9/7i3v5zPBIMN5aybAh2/Jg=="], + "typed-emitter": ["typed-emitter@2.1.0", "", { "optionalDependencies": { "rxjs": "*" } }, "sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA=="], + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], "ufo": ["ufo@1.6.1", "", {}, "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA=="], @@ -2507,6 +2538,8 @@ "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], + "webrtc-adapter": ["webrtc-adapter@9.0.3", "", { "dependencies": { "sdp": "^3.2.0" } }, "sha512-5fALBcroIl31OeXAdd1YUntxiZl1eHlZZWzNg3U4Fn+J9/cGL3eT80YlrsWGvj2ojuz1rZr2OXkgCzIxAZ7vRQ=="], + "whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], diff --git a/server/src/web/routes/voice.ts b/server/src/web/routes/voice.ts new file mode 100644 index 00000000..1a55f836 --- /dev/null +++ b/server/src/web/routes/voice.ts @@ -0,0 +1,197 @@ +import { Hono } from 'hono' +import { z } from 'zod' +import type { WebAppEnv } from '../middleware/auth' +import { + ELEVENLABS_API_BASE, + VOICE_AGENT_NAME, + buildVoiceAgentConfig +} from '@hapi/protocol/voice' + +const tokenRequestSchema = z.object({ + customAgentId: z.string().optional(), + customApiKey: z.string().optional() +}) + +// Cache for auto-created agent IDs (keyed by API key hash) +const agentIdCache = new Map() + +interface ElevenLabsAgent { + agent_id: string + name: string +} + +/** + * Find an existing "Hapi Voice Assistant" agent + */ +async function findHapiAgent(apiKey: string): Promise { + try { + const response = await fetch(`${ELEVENLABS_API_BASE}/convai/agents`, { + method: 'GET', + headers: { + 'xi-api-key': apiKey, + 'Accept': 'application/json' + } + }) + + if (!response.ok) { + return null + } + + const data = await response.json() as { agents?: ElevenLabsAgent[] } + const agents: ElevenLabsAgent[] = data.agents || [] + const hapiAgent = agents.find(agent => agent.name === VOICE_AGENT_NAME) + + return hapiAgent?.agent_id || null + } catch { + return null + } +} + +/** + * Create a new "Hapi Voice Assistant" agent + */ +async function createHapiAgent(apiKey: string): Promise { + try { + const response = await fetch(`${ELEVENLABS_API_BASE}/convai/agents/create`, { + method: 'POST', + headers: { + 'xi-api-key': apiKey, + 'Content-Type': 'application/json', + 'Accept': 'application/json' + }, + body: JSON.stringify(buildVoiceAgentConfig()) + }) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) as { detail?: { message?: string } | string } + const errorMessage = typeof errorData.detail === 'string' + ? errorData.detail + : (errorData.detail as { message?: string })?.message || `API error: ${response.status}` + console.error('[Voice] Failed to create agent:', errorMessage) + return null + } + + const data = await response.json() as { agent_id?: string } + return data.agent_id || null + } catch (error) { + console.error('[Voice] Error creating agent:', error) + return null + } +} + +/** + * Get or create agent ID - finds existing or creates new "Hapi Voice Assistant" agent + */ +async function getOrCreateAgentId(apiKey: string): Promise { + // Check cache first (simple hash of first/last chars of API key) + const cacheKey = `${apiKey.slice(0, 4)}...${apiKey.slice(-4)}` + const cached = agentIdCache.get(cacheKey) + if (cached) { + return cached + } + + // Try to find existing agent + console.log('[Voice] No agent ID configured, searching for existing agent...') + let agentId = await findHapiAgent(apiKey) + + if (agentId) { + console.log('[Voice] Found existing agent:', agentId) + } else { + // Create new agent + console.log('[Voice] No existing agent found, creating new one...') + agentId = await createHapiAgent(apiKey) + if (agentId) { + console.log('[Voice] Created new agent:', agentId) + } + } + + // Cache the result + if (agentId) { + agentIdCache.set(cacheKey, agentId) + } + + return agentId +} + +export function createVoiceRoutes(): Hono { + const app = new Hono() + + // Get ElevenLabs ConvAI conversation token + app.post('/voice/token', async (c) => { + const json = await c.req.json().catch(() => null) + const parsed = tokenRequestSchema.safeParse(json ?? {}) + if (!parsed.success) { + return c.json({ allowed: false, error: 'Invalid request body' }, 400) + } + + const { customAgentId, customApiKey } = parsed.data + + // Use custom credentials if provided, otherwise fall back to env vars + const apiKey = customApiKey || process.env.ELEVENLABS_API_KEY + let agentId = customAgentId || process.env.ELEVENLABS_AGENT_ID + + if (!apiKey) { + return c.json({ + allowed: false, + error: 'ElevenLabs API key not configured' + }, 400) + } + + // Auto-create agent if not configured + if (!agentId) { + agentId = await getOrCreateAgentId(apiKey) ?? undefined + if (!agentId) { + return c.json({ + allowed: false, + error: 'Failed to create ElevenLabs agent automatically' + }, 500) + } + } + + try { + // Fetch conversation token from ElevenLabs + const response = await fetch( + `https://api.elevenlabs.io/v1/convai/conversation/token?agent_id=${encodeURIComponent(agentId)}`, + { + method: 'GET', + headers: { + 'xi-api-key': apiKey, + 'Accept': 'application/json' + } + } + ) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) as { detail?: { message?: string }; error?: string } + const errorMessage = errorData.detail?.message || errorData.error || `ElevenLabs API error: ${response.status}` + console.error('[Voice] Failed to get token from ElevenLabs:', errorMessage) + return c.json({ + allowed: false, + error: errorMessage + }, 500) + } + + const data = await response.json() as { token?: string } + if (!data.token) { + return c.json({ + allowed: false, + error: 'No token in ElevenLabs response' + }, 500) + } + + return c.json({ + allowed: true, + token: data.token, + agentId + }) + } catch (error) { + console.error('[Voice] Error fetching token:', error) + return c.json({ + allowed: false, + error: error instanceof Error ? error.message : 'Network error' + }, 500) + } + }) + + return app +} diff --git a/server/src/web/server.ts b/server/src/web/server.ts index c5136d73..51bfb3f0 100644 --- a/server/src/web/server.ts +++ b/server/src/web/server.ts @@ -17,6 +17,7 @@ import { createMachinesRoutes } from './routes/machines' import { createGitRoutes } from './routes/git' import { createCliRoutes } from './routes/cli' import { createPushRoutes } from './routes/push' +import { createVoiceRoutes } from './routes/voice' import type { SSEManager } from '../sse/sseManager' import type { VisibilityTracker } from '../visibility/visibilityTracker' import type { Server as BunServer } from 'bun' @@ -94,6 +95,7 @@ function createWebApp(options: { app.route('/api', createMachinesRoutes(options.getSyncEngine)) app.route('/api', createGitRoutes(options.getSyncEngine)) app.route('/api', createPushRoutes(options.store, options.vapidPublicKey)) + app.route('/api', createVoiceRoutes()) // Skip static serving in relay mode, show helpful message on root if (options.relayMode) { diff --git a/shared/package.json b/shared/package.json index a13c4bfe..ca7ba820 100644 --- a/shared/package.json +++ b/shared/package.json @@ -10,7 +10,8 @@ "./messages": "./src/messages.ts", "./modes": "./src/modes.ts", "./schemas": "./src/schemas.ts", - "./types": "./src/types.ts" + "./types": "./src/types.ts", + "./voice": "./src/voice.ts" }, "sideEffects": false, "dependencies": { diff --git a/shared/src/voice.ts b/shared/src/voice.ts new file mode 100644 index 00000000..b2d69157 --- /dev/null +++ b/shared/src/voice.ts @@ -0,0 +1,236 @@ +/** + * Shared voice assistant configuration for ElevenLabs ConvAI. + * + * This module provides the unified configuration for the Hapi Voice Assistant, + * ensuring consistency between server-side auto-creation and client-side usage. + */ + +export const ELEVENLABS_API_BASE = 'https://api.elevenlabs.io/v1' +export const VOICE_AGENT_NAME = 'Hapi Voice Assistant' + +export const VOICE_SYSTEM_PROMPT = `# Identity + +You are Hapi Voice Assistant. You bridge voice communication between users and their AI coding agents in the Hapi ecosystem. + +You are friendly, proactive, and highly intelligent with a world-class engineering background. Your approach is warm, witty, and relaxed, balancing professionalism with an approachable vibe. + +# Environment Overview + +Hapi is a multi-agent development platform supporting: +- **Claude Code** - Anthropic's coding assistant (primary) +- **Codex** - OpenAI's coding agent +- **Gemini** - Google's coding agent + +Users control these agents through the Hapi web interface or Telegram Mini App. You serve as the voice interface to whichever agent is currently active. + +# How Context Updates Work + +You receive automatic context updates when: +- A session becomes focused (you see the full session history) +- The agent sends messages or uses tools +- Permission requests arrive +- The agent finishes working (ready event) + +These updates appear as system messages. You do NOT need to poll or ask for updates. Simply wait for them and summarize when relevant. + +# Tools + +## messageCodingAgent +Send user requests to the active coding agent. + +When to use: +- User says "ask Claude to..." or "have it..." +- Any coding, file, or development request +- User wants to continue a task + +Example: User says "refactor the auth module" -> call messageCodingAgent with the full request. + +## processPermissionRequest +Approve or deny pending permission requests. + +When to use: +- User says "yes", "allow", "go ahead", "approve" +- User says "no", "deny", "cancel", "stop" + +The decision parameter must be exactly "allow" or "deny". + +# Voice Output Guidelines + +## Summarization (Critical) +- NEVER read hashes, IDs, or paths character-by-character +- Say "session ending in ZAJ" not "c-m-i-a-b-c-1-2-3..." +- Say "file in the src folder" not the full path +- Summarize code changes at a high level +- Skip tool arguments unless specifically asked + +## TTS Formatting +- Use ellipses "..." for pauses +- Say "dot" for periods in URLs/paths +- Spell out acronyms: "API" becomes "A P I" +- Use normalized spoken language + +## Conversation Style +- Keep responses to 1-3 sentences typically +- Use brief affirmations: "got it", "sure thing" +- Occasional natural fillers: "so", "actually" +- Mirror user energy: terse replies for terse questions +- Lead with empathy for frustrated users + +# Behavioral Guidelines + +## Patience +After sending a message to the agent, WAIT SILENTLY. The agent may take 30+ seconds for complex tasks. Do NOT: +- Ask "are you still there?" +- Repeat the request +- Fill silence with chatter + +You will receive a context update when the agent responds or finishes. + +## Request Routing +- Direct address ("Assistant, explain...") -> Answer yourself +- Explicit delegation ("Have Claude...") -> Use messageCodingAgent +- Coding/file tasks -> Use messageCodingAgent +- General questions you can answer -> Answer yourself + +Do NOT second-guess what the agent can do. If in doubt, pass it through. + +## Proactive Updates +Speak proactively when: +- Permission is requested (inform user and ask for decision) +- Agent finishes a task (summarize results) +- Error occurs (explain clearly) +- Session status changes significantly + +Stay silent when: +- Agent is actively working +- No meaningful update to share + +# Common Scenarios + +## Permission Requests +When you see a permission request, immediately inform the user: +"Claude wants to run a bash command. Should I allow it?" +Then wait for their response and use processPermissionRequest. + +## Errors +If the agent reports an error: +- Summarize the error type +- Suggest what the user might do +- Do NOT read stack traces verbatim + +## Session Issues +If there is no active session: +- Tell the user to select or start a session in the app +- You cannot start sessions yourself + +## Long Operations +For builds, tests, or large file operations: +- Acknowledge the task was sent +- Wait silently for completion +- Summarize results when ready + +# Guardrails + +- Never read code line-by-line or provide inline code samples +- Never repeat the same information multiple ways in one response +- Treat garbled input as phonetic hints and ask for clarification +- Correct yourself immediately if you realize you made an error +- Keep conversations forward-moving with fresh insights +- Assume a technical software developer audience` + +export const VOICE_FIRST_MESSAGE = "Hey! Hapi here." + +export const VOICE_TOOLS = [ + { + type: 'client' as const, + name: 'messageCodingAgent', + description: 'Send a message to the active coding agent. Use this tool to relay the user\'s coding requests, questions, or instructions to the agent. The message should be clear and complete.', + expects_response: true, + response_timeout_secs: 120, + parameters: { + type: 'object', + required: ['message'], + properties: { + message: { + type: 'string', + description: 'The message to send to the coding agent. Should contain the user\'s complete request or instruction.' + } + } + } + }, + { + type: 'client' as const, + name: 'processPermissionRequest', + description: 'Process a permission request from the coding agent. Use this when the user wants to allow or deny a pending permission request.', + expects_response: true, + response_timeout_secs: 30, + parameters: { + type: 'object', + required: ['decision'], + properties: { + decision: { + type: 'string', + description: "The user's decision: must be either 'allow' or 'deny'" + } + } + } + } +] + +export interface VoiceAgentConfig { + name: string + conversation_config: { + agent: { + first_message: string + language: string + prompt: { + prompt: string + llm: string + temperature: number + max_tokens: number + tools: typeof VOICE_TOOLS + } + } + turn: { + turn_timeout: number + silence_end_call_timeout: number + } + tts: { + voice_id: string + model_id: string + speed: number + } + } +} + +/** + * Build the agent configuration for Hapi Voice Assistant. + * Used by both server-side auto-creation and client-side configuration. + */ +export function buildVoiceAgentConfig(): VoiceAgentConfig { + return { + name: VOICE_AGENT_NAME, + conversation_config: { + agent: { + first_message: VOICE_FIRST_MESSAGE, + language: 'en', + prompt: { + prompt: VOICE_SYSTEM_PROMPT, + llm: 'gemini-2.5-flash', + temperature: 0.7, + max_tokens: 1024, + tools: VOICE_TOOLS + } + }, + turn: { + turn_timeout: 30.0, + silence_end_call_timeout: 600.0 + }, + tts: { + voice_id: 'cgSgspJ2msm6clMCkdW9', // Jessica + model_id: 'eleven_flash_v2', + speed: 1.1 + } + } + } +} diff --git a/web/package.json b/web/package.json index a87c6eb0..5d3b327f 100644 --- a/web/package.json +++ b/web/package.json @@ -11,9 +11,10 @@ "preview": "vite preview" }, "dependencies": { - "@hapi/protocol": "workspace:*", "@assistant-ui/react": "^0.11.53", "@assistant-ui/react-markdown": "^0.11.9", + "@elevenlabs/react": "^0.13.0", + "@hapi/protocol": "workspace:*", "@radix-ui/react-dialog": "^1.1.15", "@radix-ui/react-slot": "^1.2.4", "@shikijs/langs": "^3.20.0", diff --git a/web/src/App.tsx b/web/src/App.tsx index 377af3b1..3199c40a 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -15,6 +15,7 @@ import { AppContextProvider } from '@/lib/app-context' import { fetchLatestMessages } from '@/lib/message-window-store' import { useAppGoBack } from '@/hooks/useAppGoBack' import { useTranslation } from '@/lib/use-translation' +import { VoiceProvider } from '@/lib/voice-context' import { LoginPrompt } from '@/components/LoginPrompt' import { InstallPrompt } from '@/components/InstallPrompt' import { OfflineBanner } from '@/components/OfflineBanner' @@ -314,13 +315,15 @@ function AppInner() { return ( - - -
- -
- - + + + +
+ +
+ + + ) } diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 752bb19d..79f20b56 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -379,4 +379,16 @@ export class ApiClient { method: 'DELETE' }) } + + async fetchVoiceToken(options?: { customAgentId?: string; customApiKey?: string }): Promise<{ + allowed: boolean + token?: string + agentId?: string + error?: string + }> { + return await this.request('/api/voice/token', { + method: 'POST', + body: JSON.stringify(options || {}) + }) + } } diff --git a/web/src/api/voice.ts b/web/src/api/voice.ts new file mode 100644 index 00000000..5a6da937 --- /dev/null +++ b/web/src/api/voice.ts @@ -0,0 +1,162 @@ +/** + * API functions for voice assistant integration. + * + * Fetches conversation tokens from the server for ElevenLabs integration. + * The server handles authentication with ElevenLabs API, keeping credentials secure. + * + * Supports two modes: + * 1. Default: Server uses its own ElevenLabs credentials (production) + * 2. Custom: Client provides their own ElevenLabs agent ID and API key + */ + +import type { ApiClient } from './client' +import { + ELEVENLABS_API_BASE, + VOICE_AGENT_NAME, + buildVoiceAgentConfig +} from '@hapi/protocol/voice' + +export interface VoiceTokenResponse { + allowed: boolean + token?: string + agentId?: string + error?: string +} + +export interface VoiceTokenRequest { + customAgentId?: string + customApiKey?: string +} + +/** + * Fetch a conversation token from the server for ElevenLabs voice sessions. + * + * This uses the private agent flow where: + * 1. Server holds the ELEVENLABS_API_KEY and ELEVENLABS_AGENT_ID (or uses user-provided ones) + * 2. Server fetches a short-lived conversation token from ElevenLabs + * 3. Client uses this token to establish WebRTC connection + */ +export async function fetchVoiceToken( + api: ApiClient, + options?: VoiceTokenRequest +): Promise { + try { + return await api.fetchVoiceToken(options) + } catch (error) { + return { + allowed: false, + error: error instanceof Error ? error.message : 'Network error' + } + } +} + +export interface ElevenLabsAgent { + agent_id: string + name: string +} + +export interface FindAgentResult { + success: boolean + agentId?: string + error?: string +} + +export interface CreateAgentResult { + success: boolean + agentId?: string + error?: string + created?: boolean +} + +/** + * Find an existing "Hapi Voice Assistant" agent using the provided API key. + */ +export async function findHapiAgent(apiKey: string): Promise { + try { + const response = await fetch(`${ELEVENLABS_API_BASE}/convai/agents`, { + method: 'GET', + headers: { + 'xi-api-key': apiKey, + 'Accept': 'application/json' + } + }) + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) as { detail?: { message?: string } | string } + const errorMessage = typeof errorData.detail === 'string' + ? errorData.detail + : errorData.detail?.message || `API error: ${response.status}` + return { success: false, error: errorMessage } + } + + const data = await response.json() as { agents?: ElevenLabsAgent[] } + const agents: ElevenLabsAgent[] = data.agents || [] + + const hapiAgent = agents.find(agent => agent.name === VOICE_AGENT_NAME) + + if (hapiAgent) { + return { success: true, agentId: hapiAgent.agent_id } + } else { + return { success: false, error: `No agent named "${VOICE_AGENT_NAME}" found` } + } + } catch (e) { + return { success: false, error: e instanceof Error ? e.message : 'Network error' } + } +} + +/** + * Create or update the "Hapi Voice Assistant" agent with our default configuration. + */ +export async function createOrUpdateHapiAgent(apiKey: string): Promise { + try { + const findResult = await findHapiAgent(apiKey) + const existingAgentId = findResult.success ? findResult.agentId : null + + const agentConfig = buildVoiceAgentConfig() + + let response: Response + let created = false + + if (existingAgentId) { + response = await fetch(`${ELEVENLABS_API_BASE}/convai/agents/${existingAgentId}`, { + method: 'PATCH', + headers: { + 'xi-api-key': apiKey, + 'Content-Type': 'application/json', + 'Accept': 'application/json' + }, + body: JSON.stringify(agentConfig) + }) + } else { + response = await fetch(`${ELEVENLABS_API_BASE}/convai/agents/create`, { + method: 'POST', + headers: { + 'xi-api-key': apiKey, + 'Content-Type': 'application/json', + 'Accept': 'application/json' + }, + body: JSON.stringify(agentConfig) + }) + created = true + } + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})) as { detail?: { message?: string } | string } + const errorMessage = typeof errorData.detail === 'string' + ? errorData.detail + : errorData.detail?.message || `API error: ${response.status}` + return { success: false, error: errorMessage } + } + + const data = await response.json() as { agent_id?: string } + const agentId = existingAgentId || data.agent_id + + if (!agentId) { + return { success: false, error: 'Failed to get agent ID from response' } + } + + return { success: true, agentId, created } + } catch (e) { + return { success: false, error: e instanceof Error ? e.message : 'Network error' } + } +} diff --git a/web/src/components/AssistantChat/ComposerButtons.tsx b/web/src/components/AssistantChat/ComposerButtons.tsx index 3c983a58..3ba63a20 100644 --- a/web/src/components/AssistantChat/ComposerButtons.tsx +++ b/web/src/components/AssistantChat/ComposerButtons.tsx @@ -1,6 +1,69 @@ import { ComposerPrimitive } from '@assistant-ui/react' +import type { ConversationStatus } from '@/realtime/types' import { useTranslation } from '@/lib/use-translation' +function MicrophoneIcon(props: { active?: boolean }) { + return ( + + + + + + ) +} + +function SpeakerIcon(props: { muted?: boolean }) { + if (props.muted) { + // Speaker with X (muted) + return ( + + + + + + ) + } + + // Speaker with sound waves + return ( + + + + + + ) +} + function SettingsIcon() { return ( + + + ) +} + +function LoadingIcon() { + return ( + + + + + ) +} + +function UnifiedButton(props: { + canSend: boolean + voiceStatus: ConversationStatus + voiceEnabled: boolean + controlsDisabled: boolean + onSend: () => void + onVoiceToggle: () => void +}) { + const { t } = useTranslation() + + // Determine button state + const isConnecting = props.voiceStatus === 'connecting' + const isConnected = props.voiceStatus === 'connected' + const isVoiceActive = isConnecting || isConnected + const hasText = props.canSend + + // Determine button behavior + const handleClick = () => { + if (isVoiceActive) { + props.onVoiceToggle() // Stop voice + } else if (hasText) { + props.onSend() // Send message + } else if (props.voiceEnabled) { + props.onVoiceToggle() // Start voice + } + } + + // Determine button style and icon + let icon: React.ReactNode + let className: string + let ariaLabel: string + + if (isConnecting) { + icon = + className = 'bg-black text-white' + ariaLabel = t('voice.connecting') + } else if (isConnected) { + icon = + className = 'bg-black text-white' + ariaLabel = t('composer.stop') + } else if (hasText) { + icon = + className = 'bg-black text-white' + ariaLabel = t('composer.send') + } else if (props.voiceEnabled) { + icon = + className = 'bg-black text-white' + ariaLabel = t('composer.voice') + } else { + icon = + className = 'bg-[#C0C0C0] text-white' + ariaLabel = t('composer.send') + } + + const isDisabled = props.controlsDisabled || (!hasText && !props.voiceEnabled && !isVoiceActive) + + return ( + + ) +} + export function ComposerButtons(props: { canSend: boolean controlsDisabled: boolean @@ -144,8 +309,15 @@ export function ComposerButtons(props: { switchDisabled: boolean isSwitching: boolean onSwitch: () => void + voiceEnabled: boolean + voiceStatus: ConversationStatus + voiceMicMuted?: boolean + onVoiceToggle: () => void + onVoiceMicToggle?: () => void + onSend: () => void }) { const { t } = useTranslation() + const isVoiceConnected = props.voiceStatus === 'connected' return (
@@ -210,20 +382,32 @@ export function ComposerButtons(props: { ) : null} + + {isVoiceConnected && props.onVoiceMicToggle ? ( + + ) : null}
- - - + ) } diff --git a/web/src/components/AssistantChat/HappyComposer.tsx b/web/src/components/AssistantChat/HappyComposer.tsx index 5880f714..7eb0ddbd 100644 --- a/web/src/components/AssistantChat/HappyComposer.tsx +++ b/web/src/components/AssistantChat/HappyComposer.tsx @@ -13,6 +13,7 @@ import { } from 'react' import type { AgentState, ModelMode, PermissionMode } from '@/types/api' import type { Suggestion } from '@/hooks/useActiveSuggestions' +import type { ConversationStatus } from '@/realtime/types' import { useActiveWord } from '@/hooks/useActiveWord' import { useActiveSuggestions } from '@/hooks/useActiveSuggestions' import { applySuggestion } from '@/utils/applySuggestion' @@ -48,6 +49,11 @@ export function HappyComposer(props: { onTerminal?: () => void autocompletePrefixes?: string[] autocompleteSuggestions?: (query: string) => Promise + // Voice assistant props + voiceStatus?: ConversationStatus + voiceMicMuted?: boolean + onVoiceToggle?: () => void + onVoiceMicToggle?: () => void }) { const { t } = useTranslation() const { @@ -65,7 +71,11 @@ export function HappyComposer(props: { onSwitchToRemote, onTerminal, autocompletePrefixes = ['@', '/'], - autocompleteSuggestions = defaultSuggestionHandler + autocompleteSuggestions = defaultSuggestionHandler, + voiceStatus = 'disconnected', + voiceMicMuted = false, + onVoiceToggle, + onVoiceMicToggle } = props // Use ?? so missing values fall back to default (destructuring defaults only handle undefined) @@ -355,6 +365,11 @@ export function HappyComposer(props: { const showModelSettings = Boolean(onModelModeChange && agentFlavor !== 'codex' && agentFlavor !== 'gemini') const showSettingsButton = Boolean(showPermissionSettings || showModelSettings) const showAbortButton = true + const voiceEnabled = Boolean(onVoiceToggle) + + const handleSend = useCallback(() => { + api.composer().send() + }, [api]) const overlays = useMemo(() => { if (showSettings && (showPermissionSettings || showModelSettings)) { @@ -487,6 +502,7 @@ export function HappyComposer(props: { modelMode={modelMode} permissionMode={permissionMode} agentFlavor={agentFlavor} + voiceStatus={voiceStatus} />
@@ -528,6 +544,12 @@ export function HappyComposer(props: { switchDisabled={switchDisabled} isSwitching={isSwitching} onSwitch={handleSwitch} + voiceEnabled={voiceEnabled} + voiceStatus={voiceStatus} + voiceMicMuted={voiceMicMuted} + onVoiceToggle={onVoiceToggle ?? (() => {})} + onVoiceMicToggle={onVoiceMicToggle} + onSend={handleSend} />
diff --git a/web/src/components/AssistantChat/StatusBar.tsx b/web/src/components/AssistantChat/StatusBar.tsx index 584b565c..948c89c2 100644 --- a/web/src/components/AssistantChat/StatusBar.tsx +++ b/web/src/components/AssistantChat/StatusBar.tsx @@ -2,6 +2,7 @@ import { getPermissionModeLabel, getPermissionModeTone, isPermissionModeAllowedF import type { PermissionModeTone } from '@hapi/protocol' import { useMemo } from 'react' import type { AgentState, ModelMode, PermissionMode } from '@/types/api' +import type { ConversationStatus } from '@/realtime/types' import { getContextBudgetTokens } from '@/chat/modelConfig' import { useTranslation } from '@/lib/use-translation' @@ -35,10 +36,21 @@ function getConnectionStatus( active: boolean, thinking: boolean, agentState: AgentState | null | undefined, + voiceStatus: ConversationStatus | undefined, t: (key: string) => string ): { text: string; color: string; dotColor: string; isPulsing: boolean } { const hasPermissions = agentState?.requests && Object.keys(agentState.requests).length > 0 + // Voice connecting takes priority + if (voiceStatus === 'connecting') { + return { + text: t('voice.connecting'), + color: 'text-[#007AFF]', + dotColor: 'bg-[#007AFF]', + isPulsing: true + } + } + if (!active) { return { text: t('misc.offline'), @@ -97,11 +109,12 @@ export function StatusBar(props: { modelMode?: ModelMode permissionMode?: PermissionMode agentFlavor?: string | null + voiceStatus?: ConversationStatus }) { const { t } = useTranslation() const connectionStatus = useMemo( - () => getConnectionStatus(props.active, props.thinking, props.agentState, t), - [props.active, props.thinking, props.agentState, t] + () => getConnectionStatus(props.active, props.thinking, props.agentState, props.voiceStatus, t), + [props.active, props.thinking, props.agentState, props.voiceStatus, t] ) const contextWarning = useMemo( diff --git a/web/src/components/SessionChat.tsx b/web/src/components/SessionChat.tsx index 1707b9cd..9c14889b 100644 --- a/web/src/components/SessionChat.tsx +++ b/web/src/components/SessionChat.tsx @@ -15,6 +15,8 @@ import { createAttachmentAdapter } from '@/lib/attachmentAdapter' import { SessionHeader } from '@/components/SessionHeader' import { usePlatform } from '@/hooks/usePlatform' import { useSessionActions } from '@/hooks/mutations/useSessionActions' +import { useVoiceOptional } from '@/lib/voice-context' +import { RealtimeVoiceSession, registerSessionStore, registerVoiceHooksStore, voiceHooks } from '@/realtime' export function SessionChat(props: { api: ApiClient @@ -49,6 +51,96 @@ export function SessionChat(props: { agentFlavor ) + // Voice assistant integration + const voice = useVoiceOptional() + + // Register session store for voice client tools + useEffect(() => { + registerSessionStore({ + getSession: () => props.session as { agentState?: { requests?: Record } } | null, + sendMessage: (_sessionId: string, message: string) => props.onSend(message), + approvePermission: async (_sessionId: string, requestId: string) => { + await props.api.approvePermission(props.session.id, requestId) + props.onRefresh() + }, + denyPermission: async (_sessionId: string, requestId: string) => { + await props.api.denyPermission(props.session.id, requestId) + props.onRefresh() + } + }) + }, [props.session, props.api, props.onSend, props.onRefresh]) + + useEffect(() => { + registerVoiceHooksStore( + (sessionId) => (sessionId === props.session.id ? props.session : null), + (sessionId) => (sessionId === props.session.id ? props.messages : []) + ) + }, [props.session, props.messages]) + + // Track and report new messages to voice assistant + // Note: voiceHooks internally checks isVoiceSessionStarted() so we don't need to check voice.status here + const prevMessagesRef = useRef([]) + + useEffect(() => { + const prevIds = new Set(prevMessagesRef.current.map(m => m.id)) + const newMessages = props.messages.filter(m => !prevIds.has(m.id)) + + if (newMessages.length > 0) { + voiceHooks.onMessages(props.session.id, newMessages) + } + + prevMessagesRef.current = props.messages + }, [props.messages, props.session.id]) + + // Report ready event when thinking stops + // Note: voiceHooks internally checks isVoiceSessionStarted() so we don't need to check voice.status here + const prevThinkingRef = useRef(props.session.thinking) + + useEffect(() => { + // Detect transition: thinking → not thinking + if (prevThinkingRef.current && !props.session.thinking) { + voiceHooks.onReady(props.session.id) + } + + prevThinkingRef.current = props.session.thinking + }, [props.session.thinking, props.session.id]) + + // Report permission requests to voice assistant + // Note: voiceHooks internally checks isVoiceSessionStarted() so we don't need to check voice.status here + const prevRequestIdsRef = useRef>(new Set()) + + useEffect(() => { + const requests = props.session.agentState?.requests ?? {} + const currentIds = new Set(Object.keys(requests)) + + for (const [requestId, request] of Object.entries(requests)) { + if (!prevRequestIdsRef.current.has(requestId)) { + voiceHooks.onPermissionRequested( + props.session.id, + requestId, + (request as { tool?: string }).tool ?? 'unknown', + (request as { arguments?: unknown }).arguments + ) + } + } + + prevRequestIdsRef.current = currentIds + }, [props.session.agentState?.requests, props.session.id]) + + const handleVoiceToggle = useCallback(async () => { + if (!voice) return + if (voice.status === 'connected' || voice.status === 'connecting') { + await voice.stopVoice() + } else { + await voice.startVoice(props.session.id) + } + }, [voice, props.session.id]) + + const handleVoiceMicToggle = useCallback(() => { + if (!voice) return + voice.toggleMic() + }, [voice]) + useEffect(() => { normalizedCacheRef.current.clear() blocksByIdRef.current.clear() @@ -218,9 +310,22 @@ export function SessionChat(props: { onSwitchToRemote={handleSwitchToRemote} onTerminal={props.session.active ? handleViewTerminal : undefined} autocompleteSuggestions={props.autocompleteSuggestions} + voiceStatus={voice?.status} + voiceMicMuted={voice?.micMuted} + onVoiceToggle={voice ? handleVoiceToggle : undefined} + onVoiceMicToggle={voice ? handleVoiceMicToggle : undefined} /> + + {/* Voice session component - renders nothing but initializes ElevenLabs */} + {voice && ( + + )} ) } diff --git a/web/src/components/VoiceStatusBar.tsx b/web/src/components/VoiceStatusBar.tsx new file mode 100644 index 00000000..9e6b7d75 --- /dev/null +++ b/web/src/components/VoiceStatusBar.tsx @@ -0,0 +1,146 @@ +import { useTranslation } from '@/lib/use-translation' +import type { ConversationStatus } from '@/realtime/types' + +interface VoiceStatusBarProps { + status: ConversationStatus + onStop: () => void + micMuted?: boolean + onMicToggle?: () => void +} + +function MicrophoneIcon(props: { muted?: boolean }) { + if (props.muted) { + return ( + + + + + + + + + ) + } + + return ( + + + + + + ) +} + +function StatusDot(props: { color: 'yellow' | 'green' | 'gray' | 'red'; pulsing?: boolean }) { + const colorClasses = { + yellow: 'bg-yellow-500', + green: 'bg-emerald-500', + gray: 'bg-gray-400', + red: 'bg-red-500' + } + + const pulsingColorClasses = { + yellow: 'bg-yellow-400', + green: 'bg-emerald-400', + gray: 'bg-gray-300', + red: 'bg-red-400' + } + + if (props.pulsing) { + return ( + + + + + ) + } + + return +} + +export function VoiceStatusBar({ status, onStop, micMuted, onMicToggle }: VoiceStatusBarProps) { + const { t } = useTranslation() + + if (status === 'disconnected') { + return null + } + + // Determine status dot appearance + let dotColor: 'yellow' | 'green' | 'gray' | 'red' = 'green' + let dotPulsing = false + + if (status === 'connecting') { + dotColor = 'yellow' + dotPulsing = true + } else if (status === 'error') { + dotColor = 'red' + } else if (micMuted) { + dotColor = 'gray' + } + + // Determine status text + const statusText = status === 'connecting' + ? t('voice.connecting') + : status === 'error' + ? t('voice.error') + : micMuted + ? t('voice.muted') + : t('voice.active') + + return ( +
+ {/* Left section: status dot + mic icon + status text */} + + + {/* Right section: mute button + end button */} +
+ {status === 'connected' && onMicToggle && ( + + )} + + +
+
+ ) +} diff --git a/web/src/lib/locales/en.ts b/web/src/lib/locales/en.ts index b15507a9..3a5698ee 100644 --- a/web/src/lib/locales/en.ts +++ b/web/src/lib/locales/en.ts @@ -190,6 +190,17 @@ export default { 'composer.switchRemote': 'Switch to remote mode', 'composer.attach': 'Attach file', 'composer.send': 'Send', + 'composer.stop': 'Stop', + 'composer.voice': 'Voice assistant', + + // Voice assistant + 'voice.connecting': 'Connecting...', + 'voice.active': 'Active', + 'voice.muted': 'Muted', + 'voice.error': 'Error', + 'voice.mute': 'Mute', + 'voice.unmute': 'Unmute', + 'voice.end': 'End', // Banners 'offline.title': 'Offline', diff --git a/web/src/lib/locales/zh-CN.ts b/web/src/lib/locales/zh-CN.ts index 7d0e9827..cf8019f3 100644 --- a/web/src/lib/locales/zh-CN.ts +++ b/web/src/lib/locales/zh-CN.ts @@ -192,6 +192,17 @@ export default { 'composer.switchRemote': '切换到远程模式', 'composer.attach': '添加文件', 'composer.send': '发送', + 'composer.stop': '停止', + 'composer.voice': '语音助手', + + // Voice assistant + 'voice.connecting': '连接中...', + 'voice.active': '活跃', + 'voice.muted': '已静音', + 'voice.error': '错误', + 'voice.mute': '静音', + 'voice.unmute': '取消静音', + 'voice.end': '结束', // Banners 'offline.title': '离线', diff --git a/web/src/lib/voice-context.tsx b/web/src/lib/voice-context.tsx new file mode 100644 index 00000000..9754c008 --- /dev/null +++ b/web/src/lib/voice-context.tsx @@ -0,0 +1,68 @@ +import { createContext, useCallback, useContext, useState, type ReactNode } from 'react' +import type { ConversationStatus } from '@/realtime/types' +import { startRealtimeSession, stopRealtimeSession, voiceHooks } from '@/realtime' + +interface VoiceContextValue { + status: ConversationStatus + micMuted: boolean + currentSessionId: string | null + setStatus: (status: ConversationStatus) => void + setMicMuted: (muted: boolean) => void + toggleMic: () => void + startVoice: (sessionId: string) => Promise + stopVoice: () => Promise +} + +const VoiceContext = createContext(null) + +export function VoiceProvider({ children }: { children: ReactNode }) { + const [status, setStatus] = useState('disconnected') + const [micMuted, setMicMuted] = useState(false) + const [currentSessionId, setCurrentSessionId] = useState(null) + + const toggleMic = useCallback(() => { + setMicMuted((prev) => !prev) + }, []) + + const startVoice = useCallback(async (sessionId: string) => { + setCurrentSessionId(sessionId) + const initialContext = voiceHooks.onVoiceStarted(sessionId) + await startRealtimeSession(sessionId, initialContext) + }, []) + + const stopVoice = useCallback(async () => { + voiceHooks.onVoiceStopped() + await stopRealtimeSession() + setCurrentSessionId(null) + setStatus('disconnected') + }, []) + + return ( + + {children} + + ) +} + +export function useVoice(): VoiceContextValue { + const context = useContext(VoiceContext) + if (!context) { + throw new Error('useVoice must be used within a VoiceProvider') + } + return context +} + +export function useVoiceOptional(): VoiceContextValue | null { + return useContext(VoiceContext) +} diff --git a/web/src/realtime/RealtimeSession.ts b/web/src/realtime/RealtimeSession.ts new file mode 100644 index 00000000..5d947bba --- /dev/null +++ b/web/src/realtime/RealtimeSession.ts @@ -0,0 +1,63 @@ +import type { VoiceSession } from './types' + +let voiceSession: VoiceSession | null = null +let voiceSessionStarted = false +let currentSessionId: string | null = null + +export async function startRealtimeSession(sessionId: string, initialContext?: string) { + if (!voiceSession) { + console.warn('[Voice] No voice session registered') + return + } + + try { + currentSessionId = sessionId + await voiceSession.startSession({ + sessionId, + initialContext + }) + voiceSessionStarted = true + } catch (error) { + console.error('[Voice] Failed to start realtime session:', error) + currentSessionId = null + voiceSessionStarted = false + } +} + +export async function stopRealtimeSession() { + if (!voiceSession) { + return + } + + try { + await voiceSession.endSession() + currentSessionId = null + voiceSessionStarted = false + } catch (error) { + console.error('[Voice] Failed to stop realtime session:', error) + } +} + +export function registerVoiceSession(session: VoiceSession) { + if (voiceSession) { + console.warn('[Voice] Voice session already registered, replacing with new one') + } + voiceSession = session +} + +export function isVoiceSessionStarted(): boolean { + return voiceSessionStarted +} + +export function getVoiceSession(): VoiceSession | null { + return voiceSession +} + +export function getCurrentRealtimeSessionId(): string | null { + return currentSessionId +} + +export function updateCurrentSessionId(sessionId: string | null) { + console.log(`[Voice] Realtime session ID updated: ${sessionId}`) + currentSessionId = sessionId +} diff --git a/web/src/realtime/RealtimeVoiceSession.tsx b/web/src/realtime/RealtimeVoiceSession.tsx new file mode 100644 index 00000000..3730f083 --- /dev/null +++ b/web/src/realtime/RealtimeVoiceSession.tsx @@ -0,0 +1,235 @@ +import { useEffect, useRef, useCallback, useState } from 'react' +import { useConversation } from '@elevenlabs/react' +import { registerVoiceSession } from './RealtimeSession' +import { realtimeClientTools, registerSessionStore } from './realtimeClientTools' +import { fetchVoiceToken } from '@/api/voice' +import type { VoiceSession, VoiceSessionConfig, ConversationStatus } from './types' +import type { ApiClient } from '@/api/client' +import type { Session } from '@/types/api' + +// Debug logging +const DEBUG = import.meta.env.DEV + +// Static reference to the conversation hook instance +let conversationInstance: ReturnType | null = null + +// Store reference for status updates +let statusCallback: ((status: ConversationStatus) => void) | null = null + +// Global voice session implementation +class RealtimeVoiceSessionImpl implements VoiceSession { + private api: ApiClient + + constructor(api: ApiClient) { + this.api = api + } + + async startSession(config: VoiceSessionConfig): Promise { + if (!conversationInstance) { + const error = new Error('Realtime voice session not initialized') + console.warn('[Voice] Realtime voice session not initialized') + statusCallback?.('error') + throw error + } + + statusCallback?.('connecting') + + // Request microphone permission first + try { + await navigator.mediaDevices.getUserMedia({ audio: true }) + } catch (error) { + console.error('[Voice] Failed to get microphone permission:', error) + statusCallback?.('error') + throw error + } + + // Fetch conversation token from server + let tokenResponse: Awaited> + try { + tokenResponse = await fetchVoiceToken(this.api) + } catch (error) { + console.error('[Voice] Failed to fetch voice token:', error) + statusCallback?.('error') + throw error + } + if (!tokenResponse.allowed || !tokenResponse.token) { + const error = new Error(tokenResponse.error ?? 'Voice not allowed or no token') + console.error('[Voice] Voice not allowed or no token:', tokenResponse.error) + statusCallback?.('error') + throw error + } + + // Use conversation token from server (private agent flow) + try { + const conversationId = await conversationInstance.startSession({ + conversationToken: tokenResponse.token, + connectionType: 'webrtc', + dynamicVariables: { + sessionId: config.sessionId, + initialConversationContext: config.initialContext || '' + } + }) + + if (DEBUG) { + console.log('[Voice] Started conversation with ID:', conversationId) + } + } catch (error) { + console.error('[Voice] Failed to start realtime session:', error) + statusCallback?.('error') + throw error + } + } + + async endSession(): Promise { + if (!conversationInstance) { + return + } + + try { + await conversationInstance.endSession() + statusCallback?.('disconnected') + } catch (error) { + console.error('[Voice] Failed to end realtime session:', error) + } + } + + sendTextMessage(message: string): void { + if (!conversationInstance) { + console.warn('[Voice] Realtime voice session not initialized') + return + } + + conversationInstance.sendUserMessage(message) + } + + sendContextualUpdate(update: string): void { + if (!conversationInstance) { + console.warn('[Voice] Realtime voice session not initialized') + return + } + + conversationInstance.sendContextualUpdate(update) + } +} + +export interface RealtimeVoiceSessionProps { + api: ApiClient + micMuted?: boolean + onStatusChange?: (status: ConversationStatus) => void + getSession?: (sessionId: string) => Session | null + sendMessage?: (sessionId: string, message: string) => void + approvePermission?: (sessionId: string, requestId: string) => Promise + denyPermission?: (sessionId: string, requestId: string) => Promise +} + +export function RealtimeVoiceSession({ + api, + micMuted: micMutedProp = false, + onStatusChange, + getSession, + sendMessage, + approvePermission, + denyPermission +}: RealtimeVoiceSessionProps) { + const hasRegistered = useRef(false) + + // Use local state for micMuted that syncs with prop + // This is recommended by ElevenLabs SDK docs + const [micMuted, setMicMuted] = useState(micMutedProp) + + // Sync local state with prop changes + useEffect(() => { + setMicMuted(micMutedProp) + }, [micMutedProp]) + + // Store status callback + useEffect(() => { + statusCallback = onStatusChange || null + return () => { + statusCallback = null + } + }, [onStatusChange]) + + // Register session store for client tools + useEffect(() => { + if (getSession && sendMessage && approvePermission && denyPermission) { + registerSessionStore({ + getSession: (sessionId: string) => getSession(sessionId) as { agentState?: { requests?: Record } } | null, + sendMessage, + approvePermission, + denyPermission + }) + } + }, [getSession, sendMessage, approvePermission, denyPermission]) + + const handleConnect = useCallback(() => { + if (DEBUG) console.log('[Voice] Realtime session connected') + onStatusChange?.('connected') + }, [onStatusChange]) + + const handleDisconnect = useCallback(() => { + if (DEBUG) console.log('[Voice] Realtime session disconnected') + onStatusChange?.('disconnected') + }, [onStatusChange]) + + const handleError = useCallback((error: unknown) => { + if (DEBUG) console.error('[Voice] Realtime error:', error) + onStatusChange?.('error') + }, [onStatusChange]) + + const handleMessage = useCallback((data: unknown) => { + if (DEBUG) console.log('[Voice] Realtime message:', data) + }, []) + + const handleStatusChange = useCallback((data: unknown) => { + if (DEBUG) console.log('[Voice] Realtime status change:', data) + }, []) + + const handleModeChange = useCallback((data: unknown) => { + if (DEBUG) console.log('[Voice] Realtime mode change:', data) + }, []) + + const handleDebug = useCallback((message: unknown) => { + if (DEBUG) console.debug('[Voice] Realtime debug:', message) + }, []) + + // Debug: log when micMuted changes + useEffect(() => { + if (DEBUG) console.log('[Voice] micMuted changed to:', micMuted) + }, [micMuted]) + + const conversation = useConversation({ + clientTools: realtimeClientTools, + micMuted, + onConnect: handleConnect, + onDisconnect: handleDisconnect, + onMessage: handleMessage, + onError: handleError, + onStatusChange: handleStatusChange, + onModeChange: handleModeChange, + onDebug: handleDebug + }) + + useEffect(() => { + // Store the conversation instance globally + conversationInstance = conversation + + // Register the voice session once + if (!hasRegistered.current) { + try { + registerVoiceSession(new RealtimeVoiceSessionImpl(api)) + hasRegistered.current = true + } catch (error) { + console.error('[Voice] Failed to register voice session:', error) + } + } + + return () => { + // Clean up on unmount + conversationInstance = null + } + }, [conversation, api]) + + // This component doesn't render anything visible + return null +} diff --git a/web/src/realtime/hooks/contextFormatters.ts b/web/src/realtime/hooks/contextFormatters.ts new file mode 100644 index 00000000..fd20f1e2 --- /dev/null +++ b/web/src/realtime/hooks/contextFormatters.ts @@ -0,0 +1,207 @@ +import { unwrapRoleWrappedRecordEnvelope } from '@hapi/protocol/messages' +import type { DecryptedMessage, Session } from '@/types/api' +import { VOICE_CONFIG } from '../voiceConfig' + +interface SessionMetadata { + summary?: { text?: string } + path?: string + machineId?: string + homeDir?: string +} + +interface ContentItem { + type: string + text?: string + name?: string + input?: unknown +} + +type NormalizedRole = 'assistant' | 'user' + +function isContentArray(content: unknown): content is ContentItem[] { + return Array.isArray(content) +} + +function isObject(value: unknown): value is Record { + return Boolean(value) && typeof value === 'object' +} + +function normalizeRole(role: string | null | undefined): NormalizedRole | null { + if (role === 'agent' || role === 'assistant') return 'assistant' + if (role === 'user') return 'user' + return null +} + +function unwrapRoleWrappedContent(message: DecryptedMessage): { role: NormalizedRole | null; content: unknown } { + const record = unwrapRoleWrappedRecordEnvelope(message.content) + if (!record) { + return { role: null, content: message.content } + } + return { role: normalizeRole(record.role), content: record.content } +} + +function unwrapOutputContent(content: unknown): { roleOverride: NormalizedRole | null; content: unknown } { + if (!isObject(content) || content.type !== 'output') { + return { roleOverride: null, content } + } + + const data = isObject(content.data) ? content.data : null + if (!data || typeof data.type !== 'string') { + return { roleOverride: null, content } + } + + const message = isObject(data.message) ? data.message : null + if (!message) { + return { roleOverride: null, content } + } + + const messageContent = (message as { content?: unknown }).content + if (typeof messageContent === 'undefined') { + return { roleOverride: null, content } + } + + const roleOverride = data.type === 'assistant' + ? 'assistant' + : data.type === 'user' + ? 'user' + : null + + return { roleOverride, content: messageContent } +} + +function formatPlainText(role: NormalizedRole | null, text: string): string { + if (role === 'assistant') { + return `Claude Code: \n${text}` + } + return `User sent message: \n${text}` +} + +/** + * Format a permission request for natural language context + */ +export function formatPermissionRequest( + sessionId: string, + requestId: string, + toolName: string, + toolArgs: unknown +): string { + return `Claude Code is requesting permission to use ${toolName} (session ${sessionId}): +${requestId} +${toolName} +${JSON.stringify(toolArgs)}` +} + +/** + * Format a single message for voice context + */ +export function formatMessage(message: DecryptedMessage): string | null { + const lines: string[] = [] + const { role, content: wrappedContent } = unwrapRoleWrappedContent(message) + const { roleOverride, content } = unwrapOutputContent(wrappedContent) + const normalizedRole = roleOverride ?? role + + if (!isContentArray(content)) { + if (typeof content === 'string') { + return formatPlainText(normalizedRole, content) + } + if (isObject(content) && content.type === 'text' && typeof content.text === 'string') { + return formatPlainText(normalizedRole, content.text) + } + return null + } + + // Determine message type by checking for tool_use (assistant) vs user content + const hasToolUse = content.some(item => item.type === 'tool_use') + const isAssistant = normalizedRole === 'assistant' + ? true + : normalizedRole === 'user' + ? false + : hasToolUse || content.some(item => item.type === 'text' && content.length === 1 === false) + + for (const item of content) { + if (item.type === 'text' && item.text) { + lines.push(formatPlainText(isAssistant ? 'assistant' : 'user', item.text)) + } else if (item.type === 'tool_use' && !VOICE_CONFIG.DISABLE_TOOL_CALLS) { + const name = item.name || 'unknown' + if (VOICE_CONFIG.LIMITED_TOOL_CALLS) { + lines.push(`Claude Code is using ${name}`) + } else { + lines.push(`Claude Code is using ${name} with arguments: ${JSON.stringify(item.input)}`) + } + } + } + + if (lines.length === 0) { + return null + } + return lines.join('\n\n') +} + +export function formatNewSingleMessage(sessionId: string, message: DecryptedMessage): string | null { + const formatted = formatMessage(message) + if (!formatted) { + return null + } + return 'New message in session: ' + sessionId + '\n\n' + formatted +} + +export function formatNewMessages(sessionId: string, messages: DecryptedMessage[]): string | null { + const formatted = [...messages] + .sort((a, b) => (a.seq ?? 0) - (b.seq ?? 0)) + .map(formatMessage) + .filter(Boolean) + if (formatted.length === 0) { + return null + } + return 'New messages in session: ' + sessionId + '\n\n' + formatted.join('\n\n') +} + +export function formatHistory(sessionId: string, messages: DecryptedMessage[]): string { + const messagesToFormat = VOICE_CONFIG.MAX_HISTORY_MESSAGES > 0 + ? messages.slice(-VOICE_CONFIG.MAX_HISTORY_MESSAGES) + : messages + const formatted = messagesToFormat.map(formatMessage).filter(Boolean) + return 'History of messages in session: ' + sessionId + '\n\n' + formatted.join('\n\n') +} + +export function formatSessionFull(session: Session | null, messages: DecryptedMessage[]): string { + if (!session) { + return 'Session not available' + } + + const sessionName = session.metadata?.summary?.text + const sessionPath = session.metadata?.path + const lines: string[] = [] + + lines.push(`# Session ID: ${session.id}`) + lines.push(`# Project path: ${sessionPath}`) + lines.push(`# Session summary:\n${sessionName}`) + + if (session.metadata?.summary?.text) { + lines.push('## Session Summary') + lines.push(session.metadata.summary.text) + lines.push('') + } + + lines.push('## Our interaction history so far') + lines.push('') + lines.push(formatHistory(session.id, messages)) + + return lines.join('\n\n') +} + +export function formatSessionOffline(sessionId: string, _metadata?: SessionMetadata): string { + return `Session went offline: ${sessionId}` +} + +export function formatSessionOnline(sessionId: string, _metadata?: SessionMetadata): string { + return `Session came online: ${sessionId}` +} + +export function formatSessionFocus(sessionId: string, _metadata?: SessionMetadata): string { + return `Session became focused: ${sessionId}` +} + +export function formatReadyEvent(sessionId: string): string { + return `Claude Code done working in session: ${sessionId}. The previous message(s) are the summary of the work done. Report this to the human immediately.` +} diff --git a/web/src/realtime/hooks/voiceHooks.ts b/web/src/realtime/hooks/voiceHooks.ts new file mode 100644 index 00000000..5f321937 --- /dev/null +++ b/web/src/realtime/hooks/voiceHooks.ts @@ -0,0 +1,163 @@ +import { getCurrentRealtimeSessionId, getVoiceSession, isVoiceSessionStarted } from '../RealtimeSession' +import { + formatNewMessages, + formatPermissionRequest, + formatReadyEvent, + formatSessionFocus, + formatSessionFull, + formatSessionOffline, + formatSessionOnline +} from './contextFormatters' +import { VOICE_CONFIG } from '../voiceConfig' +import type { DecryptedMessage, Session } from '@/types/api' + +interface SessionMetadata { + summary?: { text?: string } + path?: string + machineId?: string +} + +// Track which sessions have been reported +const shownSessions = new Set() +let lastFocusSession: string | null = null + +// Session and message store references +let sessionGetter: ((sessionId: string) => Session | null) | null = null +let messagesGetter: ((sessionId: string) => DecryptedMessage[]) | null = null + +/** + * Register the session and message getters for voice hooks + */ +export function registerVoiceHooksStore( + getSession: (sessionId: string) => Session | null, + getMessages: (sessionId: string) => DecryptedMessage[] +) { + sessionGetter = getSession + messagesGetter = getMessages +} + +function reportContextualUpdate(update: string | null | undefined) { + if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) { + console.log('[Voice] Reporting contextual update:', update) + } + if (!update) return + const voice = getVoiceSession() + if (!voice || !isVoiceSessionStarted()) return + voice.sendContextualUpdate(update) +} + +function reportTextUpdate(update: string | null | undefined) { + if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) { + console.log('[Voice] Reporting text update:', update) + } + if (!update) return + const voice = getVoiceSession() + if (!voice || !isVoiceSessionStarted()) return + voice.sendTextMessage(update) +} + +function reportSession(sessionId: string) { + if (shownSessions.has(sessionId)) return + shownSessions.add(sessionId) + + const session = sessionGetter?.(sessionId) ?? null + if (!session) return + + const messages = messagesGetter?.(sessionId) ?? [] + const contextUpdate = formatSessionFull(session, messages) + reportContextualUpdate(contextUpdate) +} + +export const voiceHooks = { + /** + * Called when a session comes online/connects + */ + onSessionOnline(sessionId: string, metadata?: SessionMetadata) { + if (VOICE_CONFIG.DISABLE_SESSION_STATUS) return + + reportSession(sessionId) + const contextUpdate = formatSessionOnline(sessionId, metadata) + reportContextualUpdate(contextUpdate) + }, + + /** + * Called when a session goes offline/disconnects + */ + onSessionOffline(sessionId: string, metadata?: SessionMetadata) { + if (VOICE_CONFIG.DISABLE_SESSION_STATUS) return + + reportSession(sessionId) + const contextUpdate = formatSessionOffline(sessionId, metadata) + reportContextualUpdate(contextUpdate) + }, + + /** + * Called when user navigates to/views a session + */ + onSessionFocus(sessionId: string, metadata?: SessionMetadata) { + if (VOICE_CONFIG.DISABLE_SESSION_FOCUS) return + if (lastFocusSession === sessionId) return + lastFocusSession = sessionId + reportSession(sessionId) + reportContextualUpdate(formatSessionFocus(sessionId, metadata)) + }, + + /** + * Called when Claude requests permission for a tool use + */ + onPermissionRequested(sessionId: string, requestId: string, toolName: string, toolArgs: unknown) { + if (VOICE_CONFIG.DISABLE_PERMISSION_REQUESTS) return + + reportSession(sessionId) + reportTextUpdate(formatPermissionRequest(sessionId, requestId, toolName, toolArgs)) + }, + + /** + * Called when agent sends messages + */ + onMessages(sessionId: string, messages: DecryptedMessage[]) { + if (VOICE_CONFIG.DISABLE_MESSAGES) return + + reportSession(sessionId) + reportContextualUpdate(formatNewMessages(sessionId, messages)) + }, + + /** + * Called when voice session starts - returns initial context + */ + onVoiceStarted(sessionId: string): string { + if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) { + console.log('[Voice] Voice session started for:', sessionId) + } + shownSessions.clear() + + const session = sessionGetter?.(sessionId) ?? null + const messages = messagesGetter?.(sessionId) ?? [] + + let prompt = 'THIS IS AN ACTIVE SESSION: \n\n' + formatSessionFull(session, messages) + shownSessions.add(sessionId) + + return prompt + }, + + /** + * Called when Claude Code finishes processing (ready event) + */ + onReady(sessionId: string) { + if (VOICE_CONFIG.DISABLE_READY_EVENTS) return + + reportSession(sessionId) + reportTextUpdate(formatReadyEvent(sessionId)) + }, + + /** + * Called when voice session stops + */ + onVoiceStopped() { + if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) { + console.log('[Voice] Voice session stopped') + } + shownSessions.clear() + lastFocusSession = null + } +} diff --git a/web/src/realtime/index.ts b/web/src/realtime/index.ts new file mode 100644 index 00000000..a7fa2fbe --- /dev/null +++ b/web/src/realtime/index.ts @@ -0,0 +1,39 @@ +// Types +export type { VoiceSession, VoiceSessionConfig, ConversationStatus, ConversationMode } from './types' + +// Session management +export { + startRealtimeSession, + stopRealtimeSession, + registerVoiceSession, + isVoiceSessionStarted, + getVoiceSession, + getCurrentRealtimeSessionId, + updateCurrentSessionId +} from './RealtimeSession' + +// Client tools +export { realtimeClientTools, registerSessionStore } from './realtimeClientTools' + +// Voice session component +export { RealtimeVoiceSession, type RealtimeVoiceSessionProps } from './RealtimeVoiceSession' + +// Voice hooks +export { voiceHooks, registerVoiceHooksStore } from './hooks/voiceHooks' + +// Context formatters +export { + formatMessage, + formatNewSingleMessage, + formatNewMessages, + formatHistory, + formatSessionFull, + formatSessionOnline, + formatSessionOffline, + formatSessionFocus, + formatPermissionRequest, + formatReadyEvent +} from './hooks/contextFormatters' + +// Config +export { VOICE_CONFIG } from './voiceConfig' diff --git a/web/src/realtime/realtimeClientTools.ts b/web/src/realtime/realtimeClientTools.ts new file mode 100644 index 00000000..a2490ac8 --- /dev/null +++ b/web/src/realtime/realtimeClientTools.ts @@ -0,0 +1,108 @@ +import { getCurrentRealtimeSessionId } from './RealtimeSession' +import { VOICE_CONFIG } from './voiceConfig' + +// Store for session state and API client +let sessionStore: { + getSession: (sessionId: string) => { agentState?: { requests?: Record } } | null + sendMessage: (sessionId: string, message: string) => void + approvePermission: (sessionId: string, requestId: string) => Promise + denyPermission: (sessionId: string, requestId: string) => Promise +} | null = null + +/** + * Register the session store for client tools to access + */ +export function registerSessionStore(store: typeof sessionStore) { + sessionStore = store +} + +/** + * Static client tools for the realtime voice interface. + * These tools allow the voice assistant to interact with Claude Code. + */ +export const realtimeClientTools = { + /** + * Send a message to the active coding agent + */ + messageCodingAgent: async (parameters: unknown) => { + const params = parameters as { message?: string } + + if (!params.message || typeof params.message !== 'string' || params.message.trim() === '') { + console.error('[Voice] Invalid message parameter:', parameters) + return 'error (invalid message parameter)' + } + + const message = params.message.trim() + const sessionId = getCurrentRealtimeSessionId() + + if (!sessionId) { + console.error('[Voice] No active session') + return 'error (no active session)' + } + + if (!sessionStore) { + console.error('[Voice] Session store not registered') + return 'error (session store not available)' + } + + if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) { + console.log('[Voice] messageCodingAgent called with:', message) + console.log('[Voice] Sending message to session:', sessionId) + } + + sessionStore.sendMessage(sessionId, message) + return "sent [DO NOT say anything else, simply say 'sent']" + }, + + /** + * Process a permission request from Claude Code + */ + processPermissionRequest: async (parameters: unknown) => { + const params = parameters as { decision?: string } + + if (!params.decision || (params.decision !== 'allow' && params.decision !== 'deny')) { + console.error('[Voice] Invalid decision parameter:', parameters) + return "error (invalid decision parameter, expected 'allow' or 'deny')" + } + + const decision = params.decision + const sessionId = getCurrentRealtimeSessionId() + + if (!sessionId) { + console.error('[Voice] No active session') + return 'error (no active session)' + } + + if (!sessionStore) { + console.error('[Voice] Session store not registered') + return 'error (session store not available)' + } + + if (VOICE_CONFIG.ENABLE_DEBUG_LOGGING) { + console.log('[Voice] processPermissionRequest called with:', decision) + } + + // Get the current session to check for permission requests + const session = sessionStore.getSession(sessionId) + const requests = session?.agentState?.requests + + if (!requests || Object.keys(requests).length === 0) { + console.error('[Voice] No active permission request') + return 'error (no active permission request)' + } + + const requestId = Object.keys(requests)[0] + + try { + if (decision === 'allow') { + await sessionStore.approvePermission(sessionId, requestId) + } else { + await sessionStore.denyPermission(sessionId, requestId) + } + return "done [DO NOT say anything else, simply say 'done']" + } catch (error) { + console.error('[Voice] Failed to process permission:', error) + return `error (failed to ${decision} permission)` + } + } +} diff --git a/web/src/realtime/types.ts b/web/src/realtime/types.ts new file mode 100644 index 00000000..814f6313 --- /dev/null +++ b/web/src/realtime/types.ts @@ -0,0 +1,14 @@ +export interface VoiceSessionConfig { + sessionId: string + initialContext?: string +} + +export interface VoiceSession { + startSession(config: VoiceSessionConfig): Promise + endSession(): Promise + sendTextMessage(message: string): void + sendContextualUpdate(update: string): void +} + +export type ConversationStatus = 'disconnected' | 'connecting' | 'connected' | 'error' +export type ConversationMode = 'speaking' | 'listening' diff --git a/web/src/realtime/voiceConfig.ts b/web/src/realtime/voiceConfig.ts new file mode 100644 index 00000000..c95b2bc1 --- /dev/null +++ b/web/src/realtime/voiceConfig.ts @@ -0,0 +1,31 @@ +/** + * Static voice context configuration + */ +export const VOICE_CONFIG = { + /** Disable all tool call information from being sent to voice context */ + DISABLE_TOOL_CALLS: false, + + /** Send only tool names and descriptions, exclude arguments */ + LIMITED_TOOL_CALLS: true, + + /** Disable permission request forwarding */ + DISABLE_PERMISSION_REQUESTS: false, + + /** Disable session online/offline notifications */ + DISABLE_SESSION_STATUS: true, + + /** Disable message forwarding */ + DISABLE_MESSAGES: false, + + /** Disable session focus notifications */ + DISABLE_SESSION_FOCUS: false, + + /** Disable ready event notifications */ + DISABLE_READY_EVENTS: false, + + /** Maximum number of messages to include in session history */ + MAX_HISTORY_MESSAGES: 50, + + /** Enable debug logging for voice context updates */ + ENABLE_DEBUG_LOGGING: import.meta.env.DEV, +} as const