From baeeb1135611d9cd3ad3d3406ec4e5468cdc2b09 Mon Sep 17 00:00:00 2001 From: wusumac <736139669@qq.com> Date: Thu, 8 Oct 2026 19:51:45 +0800 Subject: [PATCH] fix: preserve deployed session fixes Persist paged Codex history checkpoints and bounded, acknowledged delivery. Keep usage sources and accounting across deleted or merged sessions. Include native Goal controls, faster peer/session queries, and release guards. Track the complete deployed source so isolated builds cannot omit local fixes. Validation: all typechecks pass. The full suite reports 7,997 passing tests and 46 existing failures, each reproduced against the prior commit. --- bun.lock | 1 + cli/README.md | 8 + cli/package.json | 1 + cli/src/api/apiSession.test.ts | 63 +++- cli/src/api/apiSession.ts | 48 ++- ...confirmedMessageQueue.slow-network.test.ts | 111 ++++++ cli/src/api/confirmedMessageQueue.test.ts | 96 ++++++ cli/src/api/confirmedMessageQueue.ts | 154 +++++++++ cli/src/codex/codexRemoteLauncher.test.ts | 28 ++ cli/src/codex/codexRemoteLauncher.ts | 14 + .../codex/shared/frontend.integration.test.ts | 22 +- cli/src/codex/shared/gateway.test.ts | 23 ++ cli/src/codex/shared/gateway.ts | 3 + cli/src/codex/shared/history.test.ts | 152 +++++++++ cli/src/codex/shared/history.ts | 183 ++++++++++ cli/src/codex/shared/projection.test.ts | 3 +- cli/src/codex/shared/projection.ts | 115 ++++--- cli/src/codex/shared/queue.ts | 1 + cli/src/codex/shared/root.test.ts | 62 +++- cli/src/codex/shared/root.ts | 93 ++++- .../codex/shared/runtime.integration.test.ts | 68 ++++ cli/src/codex/shared/runtime.ts | 23 +- cli/src/codex/utils/manageThreadGoal.test.ts | 56 +++ cli/src/codex/utils/manageThreadGoal.ts | 39 +++ cli/src/modules/pingPeer/inspectPeer.test.ts | 1 + cli/src/modules/pingPeer/pingPeer.test.ts | 3 +- cli/src/modules/pingPeer/pingPeer.ts | 26 +- cli/src/runner/README.md | 2 +- cli/src/runner/opencodeUsageScanner.ts | 2 +- cli/src/runner/run.ts | 11 +- cli/src/test/auditTestProcesses.ts | 10 +- docs/local-deployment.md | 32 ++ hub/README.md | 2 + hub/scripts/restore-usage.ts | 176 ++++++++++ .../cli/machineHandlers.opencodeUsage.test.ts | 26 +- .../socket/handlers/cli/machineHandlers.ts | 18 +- .../handlers/cli/sessionHandlers.test.ts | 61 ++++ .../socket/handlers/cli/sessionHandlers.ts | 32 +- hub/src/store/contentCodec.ts | 61 +--- hub/src/store/historySyncStore.test.ts | 91 +++++ hub/src/store/historySyncStore.ts | 61 ++++ hub/src/store/index.ts | 24 +- hub/src/store/messageStore.ts | 12 +- hub/src/store/messages.test.ts | 65 +++- hub/src/store/messages.ts | 79 ++--- hub/src/store/sessionStore.ts | 26 +- hub/src/store/usage.ts | 266 ++++++++------- hub/src/store/usageParser.ts | 227 +++++++++++++ hub/src/store/usagePersistence.test.ts | 200 +++++++++++ hub/src/store/usageSources.ts | 53 +++ hub/src/store/usageStore.ts | 144 +++++++- hub/src/sync/rpcGateway.test.ts | 13 + hub/src/sync/rpcGateway.ts | 12 +- hub/src/sync/syncEngine.ts | 20 +- hub/src/sync/usageReconciliation.ts | 22 +- hub/src/sync/usageRecovery.test.ts | 179 ++++++++++ hub/src/sync/usageRecovery.ts | 77 +++++ hub/src/sync/usageService.test.ts | 19 +- hub/src/sync/usageService.ts | 319 +++--------------- hub/src/web/routes/sessionShares.test.ts | 9 +- hub/src/web/routes/sessions.test.ts | 62 +++- hub/src/web/routes/sessions.ts | 28 +- scripts/deploy-local.sh | 5 + scripts/deploy-remote.sh | 13 +- scripts/verify-deploy-build.sh | 27 ++ shared/src/apiTypes.ts | 19 +- shared/src/historySync.ts | 30 ++ shared/src/index.ts | 2 + shared/src/messageTruncation.ts | 36 ++ shared/src/resume.test.ts | 9 +- shared/src/resume.ts | 7 +- shared/src/rpcMethods.ts | 1 + shared/src/schemas.ts | 1 + shared/src/socket.ts | 8 +- web/README.md | 1 + web/src/api/client.ts | 12 + web/src/components/CodexGoalControls.test.tsx | 77 +++++ web/src/components/CodexGoalControls.tsx | 88 +++++ web/src/components/SessionChat.tsx | 18 +- web/src/components/SessionStatusPanel.tsx | 14 +- web/src/hooks/useCodexGoal.test.tsx | 62 ++++ web/src/hooks/useCodexGoal.ts | 39 +++ web/src/lib/locales/en.ts | 15 +- web/src/lib/locales/zh-CN.ts | 15 +- web/src/lib/query-keys.ts | 1 + 85 files changed, 3647 insertions(+), 691 deletions(-) create mode 100644 cli/src/api/confirmedMessageQueue.slow-network.test.ts create mode 100644 cli/src/api/confirmedMessageQueue.test.ts create mode 100644 cli/src/api/confirmedMessageQueue.ts create mode 100644 cli/src/codex/shared/history.test.ts create mode 100644 cli/src/codex/shared/history.ts create mode 100644 cli/src/codex/utils/manageThreadGoal.test.ts create mode 100644 cli/src/codex/utils/manageThreadGoal.ts create mode 100644 hub/scripts/restore-usage.ts create mode 100644 hub/src/store/historySyncStore.test.ts create mode 100644 hub/src/store/historySyncStore.ts create mode 100644 hub/src/store/usageParser.ts create mode 100644 hub/src/store/usagePersistence.test.ts create mode 100644 hub/src/store/usageSources.ts create mode 100644 hub/src/sync/usageRecovery.test.ts create mode 100644 hub/src/sync/usageRecovery.ts create mode 100644 scripts/verify-deploy-build.sh create mode 100644 shared/src/historySync.ts create mode 100644 shared/src/messageTruncation.ts create mode 100644 web/src/components/CodexGoalControls.test.tsx create mode 100644 web/src/components/CodexGoalControls.tsx create mode 100644 web/src/hooks/useCodexGoal.test.tsx create mode 100644 web/src/hooks/useCodexGoal.ts diff --git a/bun.lock b/bun.lock index ddfadeab..9d79c292 100644 --- a/bun.lock +++ b/bun.lock @@ -44,6 +44,7 @@ "@types/ws": "8.18.1", "bun-types": "1.4.0", "dotenv": "^17.2.3", + "socket.io": "^4.8.3", "typescript": "^5", "vitest": "^4.0.16", }, diff --git a/cli/README.md b/cli/README.md index 3d35e6eb..9c5a8f99 100644 --- a/cli/README.md +++ b/cli/README.md @@ -40,6 +40,14 @@ Run Claude Code, Codex, Cursor Agent, DeepSeek Harness, Grok Build, or OpenCode - `hapi ping-peer ` - Resume (if needed) and message another session. Prefer this or MCP `ping_peer` / `list_peers` over reinventing JWT+curl. Also `--message-file` / `--list`. - `hapi inspect-peer ` - Read-only peer metadata + recent message text (no resume). Prefer this or MCP `inspect_peer` when a user cites `[title](/sessions/)` or Copy-reference `See session "…" (/sessions/) for context`. `/sessions/` is a hub path, not a local file. Optional `--limit`. +Codex recovery reads current state and a small recent preview first. A background +worker pages native items and persists confirmed progress in the Hub database; +later restarts or reconnects continue from that position. Existing sessions need +one initial background reconciliation. A history rollback or replaced Hub +message epoch invalidates the checkpoint. Native versions without item paging +fall back to one turn per request, so a very large individual turn still needs +native item-pagination support. + ### Resume a remote session locally ```bash diff --git a/cli/package.json b/cli/package.json index e7f9a084..a6658138 100644 --- a/cli/package.json +++ b/cli/package.json @@ -79,6 +79,7 @@ "@types/ws": "8.18.1", "bun-types": "1.4.0", "dotenv": "^17.2.3", + "socket.io": "^4.8.3", "typescript": "^5", "vitest": "^4.0.16" }, diff --git a/cli/src/api/apiSession.test.ts b/cli/src/api/apiSession.test.ts index 9721b487..76595039 100644 --- a/cli/src/api/apiSession.test.ts +++ b/cli/src/api/apiSession.test.ts @@ -6,6 +6,7 @@ const socketHarness = vi.hoisted(() => ({ connected: boolean connectCalls: number connectImmediately: boolean + ack: (event: string, ...args: unknown[]) => Promise emitted: Array<{ event: string; args: unknown[] }> listeners: Map void>> trigger: (event: string, ...args: any[]) => void @@ -24,6 +25,7 @@ vi.mock('socket.io-client', () => ({ connected: false, connectCalls: 0, connectImmediately: true, + ack: async () => ({}), emitted: [] as Array<{ event: string; args: unknown[] }>, listeners: new Map void>>(), trigger: () => {}, @@ -62,8 +64,8 @@ vi.mock('socket.io-client', () => ({ state.emitted.push({ event, args }) return socket }, - emitWithAck: async () => ({}), - timeout: () => ({ emitWithAck: async () => ({}) }), + emitWithAck: (event: string, ...args: unknown[]) => state.ack(event, ...args), + timeout: () => ({ emitWithAck: (event: string, ...args: unknown[]) => state.ack(event, ...args) }), connect: () => { state.connectCalls += 1 if (state.connectImmediately) { @@ -96,6 +98,63 @@ vi.mock('axios', () => ({ import { ApiSessionClient, isExternalUserMessage, IncomingMessageFilter } from './apiSession' +describe('ApiSessionClient confirmed Codex transcripts', () => { + it('preserves native history times, validates durable progress, and trims agent output before upload', async () => { + const client = new ApiSessionClient('token', createSession({ + metadata: { path: '/repo', host: 'test', capabilities: { concurrentClients: true } } + })) + const socket = socketHarness.sockets.at(-1)! + const payloads: Array<{ message: { content: { data?: { message: string }; text?: string } }; createdAt?: number }> = [] + socket.ack = async (event, data) => { + if (event === 'codex-history-sync') return { ok: true, state: { epoch: 0, revision: 1, checkpoint: null } } + if (event === 'message') payloads.push(data as typeof payloads[number]) + return { ok: true } + } + client.sendAgentMessage({ type: 'message', message: 'x'.repeat(1024 * 1024) }, 'large', { replay: true, createdAt: 1000 }) + client.sendUserMessage('u'.repeat(100_000), undefined, 'user', { replay: true, createdAt: 2000 }) + expect(await client.waitForMessages(['large', 'user'])).toBe(true) + expect(payloads.map(payload => payload.createdAt)).toEqual([1000, 2000]) + expect(payloads[0].message.content.data?.message.length).toBeLessThan(64 * 1024) + expect(payloads[1].message.content.text?.length).toBe(100_000) + expect(await client.syncCodexHistory('thread')).toEqual({ epoch: 0, revision: 1, checkpoint: null }) + socket.ack = async () => ({ ok: false }) + await expect(client.syncCodexHistory('thread')).rejects.toThrow('did not confirm') + client.close() + }) + it('waits for persistence before consumption and skips confirmed history on reconnect', async () => { + const client = new ApiSessionClient('token', createSession({ + metadata: { path: '/repo', host: 'test', capabilities: { concurrentClients: true } } + })) + const socket = socketHarness.sockets.at(-1)! + const persisted = deferred() + const ack = vi.fn(() => persisted.promise) + socket.ack = ack + client.sendAgentMessage({ type: 'message', message: 'completed' }, 'stable', { replay: true }) + expect(ack).toHaveBeenCalledTimes(1) + expect(socket.emitted.filter(e => e.event === 'messages-consumed')).toHaveLength(0) + persisted.resolve({ ok: true }) + await vi.waitFor(() => expect(socket.emitted.filter(e => e.event === 'messages-consumed')).toHaveLength(1)) + socket.connected = false + socket.trigger('disconnect', 'transport close') + socket.triggerConnect() + client.sendAgentMessage({ type: 'message', message: 'completed' }, 'stable', { replay: true }) + expect(ack).toHaveBeenCalledTimes(1) + client.close() + }) + + it('keeps flush unconfirmed when the hub rejects transcript persistence', async () => { + const client = new ApiSessionClient('token', createSession({ + metadata: { path: '/repo', host: 'test', capabilities: { concurrentClients: true } } + })) + const socket = socketHarness.sockets.at(-1)! + socket.ack = async () => ({ ok: false }) + client.sendUserMessage('native history', undefined, 'user-id', { replay: true }) + expect(await client.flush({ timeoutMs: 20 })).toBe(false) + expect(socket.emitted.filter(e => e.event === 'messages-consumed')).toHaveLength(0) + client.close() + }) +}) + function createSession(overrides: Partial = {}): Session { return { id: '11111111-1111-4111-8111-111111111111', diff --git a/cli/src/api/apiSession.ts b/cli/src/api/apiSession.ts index 7cade4f6..df0279f8 100644 --- a/cli/src/api/apiSession.ts +++ b/cli/src/api/apiSession.ts @@ -41,6 +41,8 @@ import { cleanupUploadDir, preserveUploadDirOnExit } from '../modules/common/han import { TerminalManager } from '@/terminal/TerminalManager' import { applyVersionedAck } from './versionedUpdate' import { buildHubRequestHeaders, buildSocketIoExtraHeaderOptions } from './hubExtraHeaders' +import { ConfirmedMessageQueue } from './confirmedMessageQueue' +import { CodexHistoryStateSchema, truncateOversizedMessageContent, type CodexHistoryState, type CodexHistorySyncRequest } from '@hapi/protocol' /** * XML tags that Claude Code injects as `type:'user'` messages. @@ -234,6 +236,17 @@ function hasSameJsonValue(left: unknown, right: unknown): boolean { export class ApiSessionClient extends EventEmitter { private reconnectHandler: (() => void) | null = null onReconnect(handler: (() => void) | null): void { this.reconnectHandler = handler } + async syncCodexHistory(threadId: string, commit?: CodexHistorySyncRequest['commit']): Promise { + if (!this.socket.connected) throw new Error('Hub disconnected during history synchronization') + const response: unknown = await this.socket.timeout(15_000).emitWithAck('codex-history-sync', { sid: this.sessionId, threadId, ...(commit ? { commit } : {}) }) + if (typeof response !== 'object' || response === null || !('ok' in response) || response.ok !== true || !('state' in response)) { + throw new Error('Hub did not confirm history synchronization progress') + } + return CodexHistoryStateSchema.parse(response.state) + } + waitForMessages(localIds: string[], signal?: AbortSignal): Promise { + return this.confirmedMessages.waitFor(localIds, 30_000, signal) + } private readonly token: string readonly sessionId: string private metadata: Metadata | null @@ -281,6 +294,7 @@ export class ApiSessionClient extends EventEmitter { private agentStateChangedDuringAttempt = false private readonly pendingOutboundEvents: PendingOutboundEvent[] = [] private didWarnPendingQueueFull = false + private readonly confirmedMessages: ConfirmedMessageQueue constructor(token: string, session: Session, options: ApiSessionClientOptions = {}) { super() @@ -319,6 +333,17 @@ export class ApiSessionClient extends EventEmitter { ...buildSocketIoExtraHeaderOptions() }) + this.confirmedMessages = new ConfirmedMessageQueue( + async message => { + const response: unknown = await this.socket.emitWithAck('message', message) + if (typeof response !== 'object' || response === null || !('ok' in response) || response.ok !== true) { + throw new Error('Hub did not confirm transcript persistence') + } + }, + localId => this.socket.emit('messages-consumed', { sid: this.sessionId, localIds: [localId] }), + error => logger.debug('[API] Transcript delivery unconfirmed; retaining for reconnect', error) + ) + this.terminalManager = new TerminalManager({ sessionId: this.sessionId, getSessionPath: () => this.metadata?.path ?? null, @@ -332,6 +357,7 @@ export class ApiSessionClient extends EventEmitter { logger.debug('Socket connected successfully') this.awaitingMaterializedConnection = false this.rpcHandlerManager.onSocketConnect(this.socket) + this.confirmedMessages.setConnected(true) if (this.hasConnectedOnce) { this.needsBackfill = true } @@ -351,6 +377,7 @@ export class ApiSessionClient extends EventEmitter { this.socket.on('disconnect', (reason) => { logger.debug('[API] Socket disconnected:', reason) + this.confirmedMessages.setConnected(false) this.rpcHandlerManager.onSocketDisconnect() this.terminalManager.closeAll() if (this.hasConnectedOnce) { @@ -1027,7 +1054,7 @@ export class ApiSessionClient extends EventEmitter { }) } - sendUserMessage(text: string, meta?: MessageMeta, localId?: string): void { + sendUserMessage(text: string, meta?: MessageMeta, localId?: string, options?: { replay?: boolean; verifyPersistence?: boolean; createdAt?: number }): void { if (!text) { return } @@ -1045,6 +1072,10 @@ export class ApiSessionClient extends EventEmitter { } this.emitOrQueue(() => { + if (localId && this.metadata?.capabilities?.concurrentClients) { + this.confirmedMessages.enqueue({ sid: this.sessionId, message: content, localId, ...(options?.createdAt !== undefined ? { createdAt: options.createdAt } : {}) }, options?.replay, options?.verifyPersistence) + return + } this.socket.emit('message', { sid: this.sessionId, message: content, @@ -1063,8 +1094,8 @@ export class ApiSessionClient extends EventEmitter { void this.materialize() } - sendAgentMessage(body: unknown, localId?: string): void { - const content = { + sendAgentMessage(body: unknown, localId?: string, options?: { replay?: boolean; verifyPersistence?: boolean; createdAt?: number }): void { + const content = truncateOversizedMessageContent({ role: 'agent', content: { type: AGENT_MESSAGE_PAYLOAD_TYPE, @@ -1073,8 +1104,12 @@ export class ApiSessionClient extends EventEmitter { meta: { sentFrom: 'cli' } - } + }) this.emitOrQueue(() => { + if (localId && this.metadata?.capabilities?.concurrentClients) { + this.confirmedMessages.enqueue({ sid: this.sessionId, message: content, localId, ...(options?.createdAt !== undefined ? { createdAt: options.createdAt } : {}) }, options?.replay, options?.verifyPersistence) + return + } this.socket.emit('message', { sid: this.sessionId, message: content, @@ -1520,6 +1555,10 @@ export class ApiSessionClient extends EventEmitter { } } + if (!await this.confirmedMessages.drain(remainingMs())) { + return false + } + if (!await this.drainLock(this.metadataLock, remainingMs())) { return false } @@ -1556,6 +1595,7 @@ export class ApiSessionClient extends EventEmitter { this.materializationRetryAbortController = null this.awaitingMaterializedConnection = false this.pendingOutboundEvents.length = 0 + this.confirmedMessages.close() this.rpcHandlerManager.onSocketDisconnect() this.terminalManager.closeAll() this.socket.disconnect() diff --git a/cli/src/api/confirmedMessageQueue.slow-network.test.ts b/cli/src/api/confirmedMessageQueue.slow-network.test.ts new file mode 100644 index 00000000..ab603438 --- /dev/null +++ b/cli/src/api/confirmedMessageQueue.slow-network.test.ts @@ -0,0 +1,111 @@ +import { createServer } from 'node:http' +import { Server, type Socket as ServerSocket } from 'socket.io' +import { io, type Socket } from 'socket.io-client' +import WebSocket, { WebSocketServer } from 'ws' +import { describe, expect, it } from 'vitest' +import { ConfirmedMessageQueue } from './confirmedMessageQueue' + +async function slowConnection() { + const http = createServer() + const hub = new Server(http, { path: '/socket.io/', ...{ pingInterval: 1000, pingTimeout: 3000 } }) + await new Promise(resolve => http.listen(0, '127.0.0.1', resolve)) + const address = http.address() + if (!address || typeof address === 'string') throw new Error('No hub port') + const proxy = new WebSocketServer({ port: 0, host: '127.0.0.1' }) + await new Promise(resolve => proxy.once('listening', resolve)) + const proxyAddress = proxy.address() + if (!proxyAddress || typeof proxyAddress === 'string') throw new Error('No proxy port') + const cleanups: Array<() => void> = [] + let backlogBytes = 0 + let peakBacklogBytes = 0 + proxy.on('connection', (downstream, request) => { + const upstream = new WebSocket(`ws://127.0.0.1:${address.port}${request.url}`) + const packets: Array<{ data: Buffer; binary: boolean }> = [] + let timer: ReturnType | undefined + let stopped = false + const pump = () => { + if (stopped || timer || upstream.readyState !== WebSocket.OPEN) return + const packet = packets.shift() + if (!packet) return + // Throttle the entire uplink, including pongs and RPC replies. + timer = setTimeout(() => { + timer = undefined + backlogBytes -= packet.data.length + if (upstream.readyState === WebSocket.OPEN) upstream.send(packet.data, { binary: packet.binary }) + pump() + }, Math.max(1, packet.data.length / (128 * 1024) * 1000)) + } + upstream.on('open', pump) + upstream.on('message', (data, binary) => { + if (downstream.readyState === WebSocket.OPEN) downstream.send(data, { binary }) + }) + downstream.on('message', (raw, binary) => { + const data = Buffer.isBuffer(raw) ? raw : Array.isArray(raw) ? Buffer.concat(raw) : Buffer.from(raw) + packets.push({ data, binary }) + backlogBytes += data.length + peakBacklogBytes = Math.max(peakBacklogBytes, backlogBytes) + pump() + }) + const stop = () => { + if (stopped) return + stopped = true + clearTimeout(timer) + downstream.terminate() + upstream.terminate() + } + upstream.on('error', stop) + downstream.on('error', stop) + upstream.on('close', stop) + downstream.on('close', stop) + cleanups.push(stop) + }) + const connection = new Promise(resolve => hub.once('connection', resolve)) + const socket: Socket = io(`http://127.0.0.1:${proxyAddress.port}`, { transports: ['websocket'], reconnection: false }) + await new Promise(resolve => socket.once('connect', resolve)) + const serverSocket = await connection + serverSocket.on('message', (_payload: unknown, ack?: (response: { ok: boolean }) => void) => ack?.({ ok: true })) + socket.on('rpc-request', (ack: (value: string) => void) => ack('ok')) + return { + socket, serverSocket, peakBacklog: () => peakBacklogBytes, + async close() { + socket.disconnect() + for (const cleanup of cleanups) cleanup() + await new Promise(resolve => proxy.close(() => resolve())) + await new Promise(resolve => hub.close(() => resolve())) + } + } +} + +describe('transcript delivery over a slow uplink', () => { + it('reproduces heartbeat failure when a long history floods the socket', async () => { + const f = await slowConnection() + try { + const disconnected = new Promise(resolve => f.socket.once('disconnect', () => resolve())) + for (let i = 0; i < 1000; i++) f.socket.emit('message', { localId: `${i}`, text: 'x'.repeat(32_000) }) + await disconnected + expect(f.socket.connected).toBe(false) + expect(f.peakBacklog()).toBeGreaterThan(1024 * 1024) + } finally { await f.close() } + }, 10_000) + + it('keeps heartbeats, RPCs and live content moving during the same history replay', async () => { + const f = await slowConnection() + const confirmed: string[] = [] + const queue = new ConfirmedMessageQueue(async payload => { + const ack: unknown = await f.socket.emitWithAck('message', payload) + if ((ack as { ok?: boolean })?.ok !== true) throw new Error('Not confirmed') + }, id => confirmed.push(id), () => {}) + f.socket.on('disconnect', () => queue.setConnected(false)) + queue.setConnected(true) + try { + for (let i = 0; i < 1000; i++) queue.enqueue({ sid: 'sid', localId: `history-${i}`, message: 'x'.repeat(32_000) }, true) + queue.enqueue({ sid: 'sid', localId: 'live', message: 'new response' }) + expect(await f.serverSocket.timeout(2800).emitWithAck('rpc-request')).toBe('ok') + await new Promise(resolve => setTimeout(resolve, 5000)) + expect(f.socket.connected).toBe(true) + expect(confirmed).toContain('live') + expect(confirmed.length).toBeLessThan(1000) + expect(f.peakBacklog()).toBeLessThan(270 * 1024) + } finally { queue.close(); await f.close() } + }, 12_000) +}) diff --git a/cli/src/api/confirmedMessageQueue.test.ts b/cli/src/api/confirmedMessageQueue.test.ts new file mode 100644 index 00000000..29bda80b --- /dev/null +++ b/cli/src/api/confirmedMessageQueue.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it, vi } from 'vitest' +import { ConfirmedMessageQueue } from './confirmedMessageQueue' + +function fixture() { + const attempts: Array<{ id: string; resolve(): void; reject(error: Error): void }> = [] + const confirmed = vi.fn() + const error = vi.fn() + const queue = new ConfirmedMessageQueue(message => new Promise((resolve, reject) => { + attempts.push({ id: message.localId, resolve, reject }) + }), confirmed, error) + const enqueue = (id: string, bytes = 80_000, replay = true) => queue.enqueue({ + sid: 'sid', localId: id, message: { text: 'x'.repeat(bytes) } + }, replay) + return { queue, attempts, confirmed, error, enqueue } +} + +describe('confirmed transcript delivery', () => { + it('confirms a history batch independently of continuous live traffic and allows restored-Hub verification', async () => { + const f = fixture(); f.queue.setConnected(true) + f.enqueue('history', 100); f.enqueue('live', 100, false) + const done = f.queue.waitFor(['history'], 1000) + f.attempts[0].resolve() + expect(await done).toBe(true) + expect(await f.queue.drain(0)).toBe(false) + f.queue.enqueue({ sid: 'sid', localId: 'history', message: {} }, true, true) + expect(f.attempts.map(attempt => attempt.id)).toEqual(['history', 'live', 'history']) + const abort = new AbortController() + const waiting = f.queue.waitFor(['history'], 1000, abort.signal) + abort.abort(); expect(await waiting).toBe(false) + f.queue.close() + }) + it('bounds a long history and puts live messages ahead of unsent history', async () => { + const f = fixture() + f.queue.setConnected(true) + for (let i = 0; i < 1000; i++) f.enqueue(`history-${i}`) + expect(f.attempts.map(a => a.id)).toEqual(['history-0', 'history-1', 'history-2']) + f.enqueue('live', 100, false) + const drained = f.queue.drain(1000) + f.attempts[0].resolve() + await vi.waitFor(() => expect(f.attempts[3]?.id).toBe('live')) + expect(f.confirmed).toHaveBeenCalledWith('history-0') + f.queue.close() + expect(await drained).toBe(false) + }) + + it('retains only unconfirmed messages across reconnect and ignores stale ACKs', async () => { + const f = fixture() + f.queue.setConnected(true) + f.enqueue('confirmed', 100) + f.enqueue('uncertain', 100) + f.attempts[0].resolve() + await vi.waitFor(() => expect(f.confirmed).toHaveBeenCalledWith('confirmed')) + f.queue.setConnected(false) + f.attempts[1].resolve() + await Promise.resolve() + f.enqueue('confirmed', 100) + f.enqueue('uncertain', 100) + f.queue.setConnected(true) + expect(f.attempts.map(a => a.id)).toEqual(['confirmed', 'uncertain', 'uncertain']) + expect(f.confirmed).not.toHaveBeenCalledWith('uncertain') + f.attempts[2].resolve() + expect(await f.queue.drain(1000)).toBe(true) + expect(f.confirmed).toHaveBeenCalledWith('uncertain') + f.queue.close() + }) + + it('does not turn a rejected persistence ACK into confirmation or a retry storm', async () => { + const f = fixture() + f.queue.setConnected(true) + f.enqueue('failed', 100) + f.attempts[0].reject(new Error('not persisted')) + await vi.waitFor(() => expect(f.error).toHaveBeenCalledTimes(1)) + expect(f.confirmed).not.toHaveBeenCalled() + expect(f.attempts).toHaveLength(1) + expect(await f.queue.drain(0)).toBe(false) + f.queue.setConnected(false) + f.queue.setConnected(true) + expect(f.attempts).toHaveLength(2) + f.attempts[1].resolve() + expect(await f.queue.drain(1000)).toBe(true) + f.queue.close() + }) + + it('sends one oversized result alone and counts UTF-8 bytes rather than characters', async () => { + const f = fixture() + f.queue.setConnected(true) + f.queue.enqueue({ sid: 'sid', localId: 'large', message: '中'.repeat(100_000) }, true) + f.enqueue('next', 100) + expect(f.attempts).toHaveLength(1) + f.attempts[0].resolve() + await vi.waitFor(() => expect(f.attempts[1]?.id).toBe('next')) + f.attempts[1].resolve() + expect(await f.queue.drain(1000)).toBe(true) + f.queue.close() + }) +}) diff --git a/cli/src/api/confirmedMessageQueue.ts b/cli/src/api/confirmedMessageQueue.ts new file mode 100644 index 00000000..bbc45330 --- /dev/null +++ b/cli/src/api/confirmedMessageQueue.ts @@ -0,0 +1,154 @@ +type Message = { + sid: string + message: unknown + localId: string + createdAt?: number +} + +type Entry = { + message: Message + bytes: number + replay: boolean + inFlight: boolean + failedGeneration?: number +} + +const MAX_IN_FLIGHT_BYTES = 256 * 1024 +const MAX_IN_FLIGHT_MESSAGES = 16 + +/** Transcript delivery only; native user submissions remain in the native queue. + * Bound the socket backlog so pongs, keep-alives and RPC acks can pass between + * history batches. Only a hub persistence ACK permits forgetting a message. */ +export class ConfirmedMessageQueue { + private readonly pending = new Map() + private readonly confirmed = new Set() + private readonly drainListeners = new Set<(drained: boolean) => void>() + private readonly progressListeners = new Set<() => void>() + private connected = false + private closed = false + private generation = 0 + private inFlightBytes = 0 + private inFlightMessages = 0 + + constructor( + private readonly send: (message: Message) => Promise, + private readonly onConfirmed: (localId: string) => void, + private readonly onError: (error: unknown) => void + ) {} + + enqueue(message: Message, replay = false, verifyPersistence = false): void { + // The Hub checkpoint is authoritative after restore/rollback. A + // process-local confirmation cannot prove that Hub still has a row. + if (verifyPersistence) this.confirmed.delete(message.localId) + if (this.closed || this.confirmed.has(message.localId)) return + const existing = this.pending.get(message.localId) + if (existing) { + if (!replay) existing.replay = false + } else { + this.pending.set(message.localId, { + message, replay, inFlight: false, + bytes: Buffer.byteLength(JSON.stringify(message)) + }) + } + this.pump() + } + + setConnected(connected: boolean): void { + if (this.closed || this.connected === connected) return + this.connected = connected + this.generation++ + this.inFlightBytes = 0 + this.inFlightMessages = 0 + for (const entry of this.pending.values()) entry.inFlight = false + if (connected) this.pump() + } + + private next(): Entry | undefined { + let history: Entry | undefined + for (const entry of this.pending.values()) { + if (entry.inFlight || entry.failedGeneration === this.generation) continue + if (!entry.replay) return entry + history ??= entry + } + return history + } + + private pump(): void { + while (this.connected && !this.closed && this.inFlightMessages < MAX_IN_FLIGHT_MESSAGES) { + const entry = this.next() + if (!entry) return + // An individual large result may exceed the window; send it alone. + if (this.inFlightMessages > 0 && this.inFlightBytes + entry.bytes > MAX_IN_FLIGHT_BYTES) return + const generation = this.generation + entry.inFlight = true + this.inFlightBytes += entry.bytes + this.inFlightMessages++ + void this.send(entry.message).then(() => { + if (this.closed || generation !== this.generation) return + this.pending.delete(entry.message.localId) + this.confirmed.add(entry.message.localId) + this.onConfirmed(entry.message.localId) + for (const listener of this.progressListeners) listener() + }).catch(error => { + if (this.closed || generation !== this.generation) return + // A failed ACK is uncertain. Retain it for the next connection, + // rather than adding retries to an already congested socket. + entry.failedGeneration = generation + this.onError(error) + }).finally(() => { + if (this.closed || generation !== this.generation) return + entry.inFlight = false + this.inFlightBytes -= entry.bytes + this.inFlightMessages-- + if (this.pending.size === 0) this.notifyDrained(true) + this.pump() + }) + } + } + + drain(timeoutMs: number): Promise { + if (this.closed) return Promise.resolve(false) + if (this.pending.size === 0) return Promise.resolve(true) + if (timeoutMs <= 0) return Promise.resolve(false) + return new Promise(resolve => { + const finish = (drained: boolean) => { + clearTimeout(timer) + this.drainListeners.delete(finish) + resolve(drained) + } + const timer = setTimeout(() => finish(false), timeoutMs) + this.drainListeners.add(finish) + }) + } + + /** Wait only for this history batch. Continuous live traffic must not + * prevent a completed batch from advancing its durable checkpoint. */ + waitFor(localIds: string[], timeoutMs: number, signal?: AbortSignal): Promise { + const confirmed = () => localIds.every(id => this.confirmed.has(id)) + if (this.closed || signal?.aborted) return Promise.resolve(false) + if (confirmed()) return Promise.resolve(true) + return new Promise(resolve => { + const finish = (result: boolean) => { + clearTimeout(timer); this.progressListeners.delete(check) + signal?.removeEventListener('abort', aborted); resolve(result) + } + const check = () => { if (this.closed) finish(false); else if (confirmed()) finish(true) } + const aborted = () => finish(false) + const timer = setTimeout(() => finish(false), timeoutMs) + this.progressListeners.add(check) + signal?.addEventListener('abort', aborted, { once: true }) + }) + } + + private notifyDrained(drained: boolean): void { + for (const listener of this.drainListeners) listener(drained) + } + + close(): void { + this.closed = true + this.pending.clear() + this.confirmed.clear() + for (const listener of this.progressListeners) listener() + this.notifyDrained(false) + } +} diff --git a/cli/src/codex/codexRemoteLauncher.test.ts b/cli/src/codex/codexRemoteLauncher.test.ts index b2511727..4b06a44e 100644 --- a/cli/src/codex/codexRemoteLauncher.test.ts +++ b/cli/src/codex/codexRemoteLauncher.test.ts @@ -225,6 +225,13 @@ vi.mock('./codexAppServerClient', () => { return { goal: harness.goal }; } + async request(method: string, params: Record) { + if (method === 'thread/goal/get') return this.getThreadGoal(params); + if (method === 'thread/goal/set') return this.setThreadGoal(params); + if (method === 'thread/goal/clear') return this.clearThreadGoal(params); + throw new Error(`Unexpected request: ${method}`); + } + async steerTurn(params?: { threadId?: string; input?: unknown[]; expectedTurnId?: string; clientUserMessageId?: string }): Promise<{ dispatched: Promise; completed: Promise; @@ -1958,6 +1965,27 @@ describe('codexRemoteLauncher', () => { ])); }); + it('manages a Goal directly through RPC while the remote turn remains busy', async () => { + harness.suppressTurnCompletion = true; + harness.suppressGoalNotifications = true; + harness.goal = { threadId: 'thread-1', objective: 'Existing Goal', status: 'active', tokenBudget: null, + tokensUsed: 0, timeUsedSeconds: 0, createdAt: 1, updatedAt: 1 }; + const { session, rpcHandlers, codexMessages } = createSessionStub(['first'], createMode(), false, false); + void codexRemoteLauncher(session as never); + await vi.waitFor(() => expect(harness.startTurnThreadIds).toHaveLength(1)); + const action = rpcHandlers.get(RPC_METHODS.CodexGoal)!; + expect(await action({ action: 'pause' })).toMatchObject({ goal: { status: 'paused' } }); + expect(await action({ action: 'resume' })).toMatchObject({ goal: { status: 'active' } }); + expect(await action({ action: 'clear' })).toEqual({ goal: null }); + expect(harness.startTurnThreadIds).toHaveLength(1); + expect(harness.interruptedTurns).toHaveLength(0); + expect(codexMessages).toEqual(expect.arrayContaining([ + expect.objectContaining({ type: 'thread_goal_updated', goal: expect.objectContaining({ status: 'paused' }) }), + expect.objectContaining({ type: 'thread_goal_cleared', thread_id: 'thread-1' }) + ])); + session.queue.close(); + }); + it('still attempts goal RPC when dynamic goals feature enablement is unsupported', async () => { harness.failSetFeatureEnablement = true; const { session, sessionEvents } = createSessionStub(['/goal improve benchmark coverage']); diff --git a/cli/src/codex/codexRemoteLauncher.ts b/cli/src/codex/codexRemoteLauncher.ts index fac12854..aae66f96 100644 --- a/cli/src/codex/codexRemoteLauncher.ts +++ b/cli/src/codex/codexRemoteLauncher.ts @@ -38,6 +38,7 @@ import { type RemoteLauncherExitReason } from '@/modules/common/remote/RemoteLauncherBase'; import { CodexConversationHistory } from './conversationHistory'; +import { manageThreadGoal } from './utils/manageThreadGoal'; @@ -4057,6 +4058,19 @@ class CodexRemoteLauncher extends RemoteLauncherBase { ?? 0 }); + session.client.rpcHandlerManager.registerHandler(RPC_METHODS.CodexGoal, async (raw: unknown) => { + const threadId = this.currentThreadId; + if (!supportsGoals || !threadId) throw new Error('Goal controls are unavailable for this Codex session'); + const result = await manageThreadGoal(threadId, raw, (method, params) => appServerClient.request(method, params)); + if (this.currentThreadId !== threadId) throw new Error('The Codex conversation changed. Refresh the session.'); + if ((raw as { action: string }).action !== 'get') { + sendGoalEvent({ type: result.goal ? 'thread_goal_updated' : 'thread_goal_cleared', + thread_id: threadId, ...(result.goal ? { goal: result.goal } : {}) }); + } + return result; + }); + session.client.updateMetadata(metadata => ({ ...metadata, capabilities: { ...metadata.capabilities, codexGoal: true } })); + const handleGoalCommand = async (message: QueuedMessage): Promise => { const command = parseGoalCommand(message.message); if (!command) { diff --git a/cli/src/codex/shared/frontend.integration.test.ts b/cli/src/codex/shared/frontend.integration.test.ts index 6e68e396..530b61b2 100644 --- a/cli/src/codex/shared/frontend.integration.test.ts +++ b/cli/src/codex/shared/frontend.integration.test.ts @@ -4,6 +4,8 @@ import { createServer } from 'node:http'; import { mkdtemp, mkdir, writeFile, readFile, readdir, rm } from 'node:fs/promises'; import { dirname, join, resolve } from 'node:path'; import { randomUUID } from 'node:crypto'; +import { io } from 'socket.io-client'; +import { CodexHistoryStateSchema } from '@hapi/protocol'; import { record } from './gateway'; import { CodexAppServerClient } from '../codexAppServerClient'; import { initializeSharedClient } from './launch'; @@ -107,6 +109,12 @@ describe.skipIf(process.env.HAPI_RUN_SHARED_CODEX_TESTS !== '1' || process.platf terminals.push(child); return child; }; const base = process.env.HAPI_API_URL!; + const historyObserver = io(`${base}/cli`, { auth: { token: env.CLI_API_TOKEN }, transports: ['websocket'], reconnection: false }); + const progress = async (sid: string, threadId: string) => { + const response = record(await historyObserver.timeout(5_000).emitWithAck('codex-history-sync', { sid, threadId })); + expect(response.ok).toBe(true); + return CodexHistoryStateSchema.parse(response.state); + }; const auth = await fetch(`${base}/api/auth`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ accessToken: env.CLI_API_TOKEN }) }); const jwt = String(record(await auth.json()).token); const api = async (path: string, body?: unknown) => { @@ -233,9 +241,15 @@ describe.skipIf(process.env.HAPI_RUN_SHARED_CODEX_TESTS !== '1' || process.platf await eventually(async () => requests, value => JSON.stringify(value).includes(`ROOT=${sessionId}`), 'per-root shell identity'); await eventually(async () => webEvents, value => value.includes('CodexBash') && value.includes('tool-call-result') && value.includes(`ROOT=${sessionId}`), 'tool call and result over Web SSE'); await eventually(() => api(`/sessions/${sessionId}/messages`), value => JSON.stringify(value).includes('CodexBash'), 'tool call persisted for Web reload'); + const latest = record(await client.request('thread/turns/list', { threadId: nativeThreadId, limit: 1, sortDirection: 'desc', itemsView: 'notLoaded' })); + const savedTurnId = record((latest.data as unknown[])[0]).id; + const savedProgress = await eventually(() => progress(sessionId, nativeThreadId), value => value.checkpoint?.turnComplete === true + && value.checkpoint.turnId === savedTurnId, 'native history checkpoint persisted at Hub'); const secondOutput = output.length; const second = openTerminal(['resume', sessionId]); - await eventually(async () => output.slice(secondOutput), text => text.includes('HELLO_SHARED'), 'secondary attach to terminal-owned engine'); + // Native pagination renders the recent viewport first. Earlier + // turns need not appear until the user scrolls back. + await eventually(async () => output.slice(secondOutput), text => text.includes('CHECK_ROOT_ENV'), 'secondary attach to terminal-owned engine'); second.stdin.write('/exit'); await new Promise(resolve => setTimeout(resolve, 150)); second.stdin.write('\r'); await eventually(async () => second.exitCode, value => value !== null, 'secondary detach'); expect(record((await detail()).session).active).toBe(true); expect(isProcessAlive(Number(runtime.serverPid))).toBe(true); @@ -281,6 +295,7 @@ describe.skipIf(process.env.HAPI_RUN_SHARED_CODEX_TESTS !== '1' || process.platf }; const runnerRuntime = await readRuntime(resumedPid); client = new CodexAppServerClient({ endpoint: String(runnerRuntime.endpoint) }); client.setServerRequestHandler(() => {}); await initializeSharedClient(client); + expect((await progress(sessionId, nativeThreadId)).checkpoint).toEqual(savedProgress.checkpoint); await client.request('hapi/stopSession', { sessionId: coldClear.sessionId }); await send('AFTER_WEB_RESUME'); await eventually(() => api(`/sessions/${sessionId}/messages`), value => hasReply(value, 'AFTER_WEB_RESUME'), 'Web reply after resume'); @@ -388,9 +403,14 @@ describe.skipIf(process.env.HAPI_RUN_SHARED_CODEX_TESTS !== '1' || process.platf await api(`/sessions/${webId}/archive`, {}); await eventually(webDetail, value => record(value.session).active === false, 'end reopened Web session'); } catch (error) { + const runtimeLogs = await Promise.all((await readdir(join(hh, 'logs')).catch(() => [] as string[])) + .filter(name => name.endsWith('.log')).map(async name => (await readFile(join(hh, 'logs', name), 'utf8')) + .split('\n').filter(line => line.includes('[Codex shared]')).join('\n'))); + await writeFile('/tmp/hapi-shared-e2e-runtime.log', runtimeLogs.join('\n')); await writeFile('/tmp/hapi-shared-e2e-terminal.log', output + '\nRUNNER:\n' + runnerOutput); throw error; } finally { + historyObserver.disconnect(); eventsAbort.abort(); await eventStream; await client?.disconnect(); for (const child of terminals) if (child.pid && child.exitCode === null) await killProcessTreeByPid(child.pid); diff --git a/cli/src/codex/shared/gateway.test.ts b/cli/src/codex/shared/gateway.test.ts index fbbf77cc..3f0c6f54 100644 --- a/cli/src/codex/shared/gateway.test.ts +++ b/cli/src/codex/shared/gateway.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; import { WebSocket, WebSocketServer } from 'ws'; import { startCodexGateway, type Envelope } from './gateway'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; async function fixture(after: (request: Envelope, response: Envelope, connection: string) => Promise) { const engine = new WebSocketServer({ port: 0, host: '127.0.0.1' }); @@ -31,6 +33,27 @@ async function fixture(after: (request: Envelope, response: Envelope, connection } }; } describe('native gateway barriers', () => { + it.skipIf(process.platform === 'win32')('releases an attached Unix frontend when the Bun owner stops', () => { + const gatewayPath = fileURLToPath(new URL('./gateway.ts', import.meta.url)); + const program = ` + import { createRequire } from 'node:module'; + import { startCodexGateway, socketUrl } from ${JSON.stringify(gatewayPath)}; + const require = createRequire(${JSON.stringify(fileURLToPath(new URL('../../../package.json', import.meta.url)))}); + const { default: WebSocket, WebSocketServer } = await import(require.resolve('ws')); + setTimeout(() => process.exit(1), 2000); + const engine = new WebSocketServer({ port: 0, host: '127.0.0.1' }); + await new Promise(resolve => engine.once('listening', resolve)); + const gateway = await startCodexGateway({ upstream: 'ws://127.0.0.1:' + engine.address().port, + path: '/tmp/hapi-gateway-close-' + process.pid + '.sock', token: 'test', + hooks: { before: async value => value, after: async () => {}, control: async () => {} } }); + const client = new WebSocket(socketUrl(gateway.endpoint)); + await new Promise((resolve, reject) => { client.once('open', resolve); client.once('error', reject); }); + await new Promise(resolve => setTimeout(resolve, 50)); + await gateway.close(); + process.exit(0); + `; + expect(() => execFileSync(process.env.HAPI_BUN_EXEC!, ['-e', program], { timeout: 5000, stdio: 'pipe' })).not.toThrow(); + }); it('namespaces colliding native request IDs by connection and withholds replies until binding', async () => { let release!: () => void; const barrier = new Promise(resolve => { release = resolve; }); const after = vi.fn(async () => { await barrier; }); const f = await fixture(after); diff --git a/cli/src/codex/shared/gateway.ts b/cli/src/codex/shared/gateway.ts index 4ad40acc..0bb58dd8 100644 --- a/cli/src/codex/shared/gateway.ts +++ b/cli/src/codex/shared/gateway.ts @@ -134,6 +134,9 @@ export async function startCodexGateway(options: { return { endpoint, async close() { for (const [client, upstream] of connections) { client.terminate(); upstream.terminate(); } websocket.close(); + // Bun can retain upgraded Unix connections after ws.terminate(), + // leaving server.close() and the execution owner waiting forever. + server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); if (options.path) await unlink(options.path).catch(() => {}); } }; diff --git a/cli/src/codex/shared/history.test.ts b/cli/src/codex/shared/history.test.ts new file mode 100644 index 00000000..fde8eca0 --- /dev/null +++ b/cli/src/codex/shared/history.test.ts @@ -0,0 +1,152 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import type { CodexHistoryCheckpoint, CodexHistoryState, CodexHistorySyncRequest } from '@hapi/protocol'; +import type { ApiSessionClient } from '@/api/apiSession'; +import type { CodexAppServerClient } from '../codexAppServerClient'; +import { CodexHistorySync, readCurrentThread } from './history'; +import { SharedCodexProjection } from './projection'; + +type Item = { id: string; type: string; text?: string; bytes?: number; completed?: boolean; forbidden?: boolean }; +type Turn = { id: string; status: string; items: Item[] }; +const workers: CodexHistorySync[] = []; +afterEach(() => { for (const worker of workers.splice(0)) worker.close(); }); + +function fixture(turns: Turn[], checkpoint: CodexHistoryCheckpoint | null = null) { + let state: CodexHistoryState = { epoch: 0, revision: 0, checkpoint }; + let acknowledge = true; + let metadata = {}; + const saved = new Set(); + const emitted: Array<{ id: string; text?: string }> = []; + const reads: string[] = []; + let maxFrame = 0; + const session = { + getMetadata: () => metadata, + updateMetadata: (fn: (m: typeof metadata) => typeof metadata) => { metadata = fn(metadata); }, + sendAgentMessage: (body: { message?: string }, id: string) => { + emitted.push({ id, ...(body.message && body.message.length < 100 ? { text: body.message } : {}) }); saved.add(id); + }, + sendUserMessage: (_text: string, _meta: unknown, id: string) => { emitted.push({ id }); saved.add(id); }, + async syncCodexHistory(_threadId: string, commit?: CodexHistorySyncRequest['commit']) { + if (commit) { + expect(commit.epoch).toBe(state.epoch); expect(commit.revision).toBe(state.revision); + expect(commit.localIds.every(id => saved.has(id))).toBe(true); + state = { epoch: state.epoch, revision: state.revision + 1, checkpoint: commit.checkpoint }; + } + return structuredClone(state); + }, + async waitForMessages(ids: string[]) { return acknowledge && ids.every(id => saved.has(id)); }, + async flushMetadata() { return true; } + } as unknown as ApiSessionClient; + const client = { async request(method: string, params: Record) { + if (method === 'thread/read') { + expect(params.includeTurns).toBe(false); + return { thread: { id: 'thread', historyMode: 'paginated' } }; + } + if (method === 'thread/turns/list') { + if (params.itemsView === 'full') throw new Error('Full turn would exceed the native frame limit'); + const rows = params.sortDirection === 'desc' ? [...turns].reverse() : turns; + const start = Number(params.cursor ?? 0); + const data = rows.slice(start, start + Number(params.limit)).map(turn => ({ id: turn.id, status: turn.status, items: [] })); + return { data, nextCursor: start + Number(params.limit) < rows.length ? String(start + Number(params.limit)) : null }; + } + if (method === 'thread/items/list') { + expect(params.limit).toBe(1); + const turn = turns.find(turn => turn.id === params.turnId)!; + const rows = params.sortDirection === 'desc' ? [...turn.items].reverse() : turn.items; + const anchor = params.cursor as { itemId: string }; + const found = typeof params.cursor === 'object' ? rows.findIndex(item => item.id === anchor.itemId) : -1; + if (params.cursor && typeof params.cursor === 'object' && found < 0) throw new Error('Item anchor not found'); + const start = typeof params.cursor === 'object' ? found + 1 : Number(params.cursor ?? 0); + const data = rows.slice(start, start + 1).map(item => { + if (item.forbidden) throw new Error(`Saved old body was reread: ${item.id}`); + reads.push(item.id); + return { turnId: turn.id, completedAtMs: item.completed || turn.status !== 'inProgress' ? 1 : null, + item: item.type === 'userMessage' ? { id: item.id, type: item.type, content: [{ type: 'text', text: item.text ?? 'input' }] } + : { id: item.id, type: item.type, text: item.bytes ? 'x'.repeat(item.bytes) : item.text ?? item.id } }; + }); + const result = { data, nextCursor: start + 1 < rows.length ? String(start + 1) : null }; + maxFrame = Math.max(maxFrame, Buffer.byteLength(JSON.stringify(result))); + return result; + } + throw new Error(`Unexpected native method ${method}`); + } } as unknown as Pick; + const create = () => { + const projection = new SharedCodexProjection(session, 'thread', async () => {}); + const worker = new CodexHistorySync(client, session, projection, () => {}, () => {}); + workers.push(worker); return { worker, projection }; + }; + return { client, session, turns, create, reads, emitted, state: () => state, maxFrame: () => maxFrame, + acknowledge: (value: boolean) => { acknowledge = value; } }; +} + +describe('incremental native history', () => { + it.each([10, 300])('skips an already saved %iMiB history before reading or sending bodies', async oldMiB => { + const f = fixture([ + { id: 'old', status: 'completed', items: [{ id: 'old-body', type: 'agentMessage', bytes: oldMiB * 1024 * 1024, forbidden: true }] }, + { id: 'new', status: 'completed', items: Array.from({ length: 20 }, (_, i) => ({ id: `new-${i}`, type: 'agentMessage' })) } + ], { version: 1, pageCursor: null, turnId: 'old', itemId: 'old-body', turnComplete: true }); + await f.create().worker.sync(); + expect(f.reads).toHaveLength(20); + expect(f.emitted).toHaveLength(20); + expect(f.state().checkpoint).toMatchObject({ turnId: 'new', itemId: 'new-19', turnComplete: true }); + // New process: no converter or confirmation cache survives. Only the + // persisted Hub checkpoint prevents the old bodies from being read. + for (const turn of f.turns) for (const item of turn.items) item.forbidden = true; + await f.create().worker.sync(); + expect(f.reads).toHaveLength(20); + expect(f.emitted).toHaveLength(20); + }); + + it('shows the latest result first and pages a 129MiB single turn without a full-turn frame', async () => { + const f = fixture([ + { id: 'huge', status: 'completed', items: Array.from({ length: 129 }, (_, i) => ({ id: `large-${i}`, type: 'agentMessage', bytes: 1024 * 1024 })) }, + { id: 'latest', status: 'completed', items: [{ id: 'final', type: 'agentMessage', text: 'Latest result' }] } + ]); + const { worker } = f.create(); + const current = await readCurrentThread(f.client, 'thread'); + const latest = (current.turns as Array>)[0]; + await worker.preview(latest); + expect(f.emitted[0].text).toBe('Latest result'); + expect(f.reads).toEqual(['final']); + await worker.sync(); + expect(f.maxFrame()).toBeLessThan(2 * 1024 * 1024); + expect(f.state().checkpoint?.turnId).toBe('latest'); + }, 15_000); + + it('does not skip an unfinished reply or move over it when newer items finish', async () => { + const f = fixture([{ id: 'active', status: 'inProgress', items: [ + { id: 'user', type: 'userMessage' }, + { id: 'reply', type: 'agentMessage', text: 'partial' }, + { id: 'later', type: 'agentMessage', text: 'Later completed item', completed: true } + ] }]); + await f.create().worker.sync(); + expect(f.state().checkpoint).toMatchObject({ itemId: 'user', turnComplete: false }); + expect(f.emitted.some(message => message.text === 'partial')).toBe(false); + f.turns[0].items[0].forbidden = true; + f.turns[0].items[1].text = 'Final complete reply'; f.turns[0].items[1].completed = true; + f.turns[0].status = 'completed'; + await f.create().worker.sync(); + expect(f.emitted.some(message => message.text === 'Final complete reply')).toBe(true); + expect(f.state().checkpoint).toMatchObject({ itemId: 'later', turnComplete: true }); + }); + + it('retries a small batch after an uncertain ACK instead of persisting false progress', async () => { + const f = fixture([{ id: 'turn', status: 'completed', items: [{ id: 'reply', type: 'agentMessage' }] }]); + f.acknowledge(false); + await expect(f.create().worker.sync()).rejects.toThrow('not confirmed'); + expect(f.state().checkpoint).toBeNull(); + f.acknowledge(true); + await f.create().worker.sync(); + expect(f.emitted[0].id).toBe(f.emitted[1].id); + expect(f.state().checkpoint?.turnComplete).toBe(true); + }); + + it('invalidates a removed native anchor before starting a new bounded reconciliation', async () => { + const f = fixture([{ id: 'survivor', status: 'completed', items: [{ id: 'reply', type: 'agentMessage' }] }], + { version: 1, pageCursor: null, turnId: 'removed', itemId: 'removed-item', turnComplete: true }); + const { worker } = f.create(); + await expect(worker.sync()).rejects.toThrow('anchor'); worker.close(); + expect(f.state().checkpoint).toBeNull(); + await f.create().worker.sync(); + expect(f.state().checkpoint?.turnId).toBe('survivor'); + }); +}); diff --git a/cli/src/codex/shared/history.ts b/cli/src/codex/shared/history.ts new file mode 100644 index 00000000..a660dc23 --- /dev/null +++ b/cli/src/codex/shared/history.ts @@ -0,0 +1,183 @@ +import { z } from 'zod'; +import type { CodexHistoryCheckpoint, CodexHistoryState } from '@hapi/protocol'; +import type { ApiSessionClient } from '@/api/apiSession'; +import type { CodexAppServerClient } from '../codexAppServerClient'; +import type { SharedCodexProjection } from './projection'; +import { record, string } from './gateway'; + +const TurnSchema = z.object({ id: z.string(), status: z.string(), items: z.array(z.unknown()).optional() }).passthrough(); +const TurnsSchema = z.object({ data: z.array(TurnSchema), nextCursor: z.string().nullish() }); +const ItemsSchema = z.object({ data: z.array(z.object({ turnId: z.string(), item: z.object({ id: z.string(), type: z.string() }).passthrough(), startedAtMs: z.number().nullish(), completedAtMs: z.number().nullish() })), nextCursor: z.string().nullish() }); +const EMPTY: CodexHistoryCheckpoint = { version: 1, pageCursor: null, turnId: null, itemId: null, turnComplete: false }; +const BATCH_ITEMS = 16; +const BATCH_BYTES = 256 * 1024; + +export async function readCurrentThread(client: Pick, threadId: string): Promise> { + const thread = record(record(await client.request('thread/read', { threadId, includeTurns: false })).thread); + const page = TurnsSchema.parse(await client.request('thread/turns/list', { threadId, limit: 1, sortDirection: 'desc', itemsView: 'summary' })); + return { ...thread, turns: page.data }; +} + +/** One native item body at a time: even a 129MiB turn never becomes one RPC + * frame or one accumulated history array. Checkpoints cover only final items + * whose complete projection and history metadata are durable at the Hub. */ +export class CodexHistorySync { + private running?: Promise; + private timer?: ReturnType; + private rerun = false; + private invalidated = false; + private readonly abort = new AbortController(); + private itemPagination = true; + + constructor( + private readonly client: Pick, + private readonly session: Pick, + private readonly projection: SharedCodexProjection, + private readonly onItem: (turnId: string, item: unknown) => void, + private readonly onError: (error: unknown) => void + ) {} + + schedule(delay = 1_000): void { + if (this.abort.signal.aborted) return; + if (this.running) { this.rerun = true; return; } + if (this.timer) return; + this.timer = setTimeout(() => { this.timer = undefined; void this.sync().catch(this.onError); }, delay); + } + invalidate(): void { this.invalidated = true; this.schedule(0); } + close(): void { this.abort.abort(); clearTimeout(this.timer); } + sync(): Promise { + if (this.abort.signal.aborted) return Promise.resolve(); + return this.running ??= this.syncNow().finally(() => { + this.running = undefined; + if (this.rerun) { this.rerun = false; this.schedule(0); } + }); + } + + /** Small tail preview for first paint. User history boundaries are rebuilt + * in ascending order below, not guessed from a recent steered input. */ + async preview(turn: Record): Promise { + if (!string(turn.id) || !this.itemPagination) return; + let cursor: string | undefined; let bytes = 0; + const revision = this.projection.historyRevision(); let title: string | undefined; + for (let i = 0; i < 12 && !this.abort.signal.aborted; i++) { + let page: z.infer; + try { page = ItemsSchema.parse(await this.client.request('thread/items/list', { threadId: this.projection.threadId, turnId: turn.id, cursor, limit: 1, sortDirection: 'desc' })); } + catch (error) { if (this.unsupported(error)) { this.itemPagination = false; return; } throw error; } + for (const entry of page.data) { + bytes += Buffer.byteLength(JSON.stringify(entry)); + if (entry.item.type === 'userMessage') continue; + this.onItem(entry.turnId, entry.item); + const result = await this.projection.historyItem(turn, entry.item, this.completed(turn, entry), true, this.createdAt(turn, entry)); + title ??= result.title; + if (!await this.session.waitForMessages(result.localIds, this.abort.signal)) throw new Error('Latest history messages were not confirmed'); + } + cursor = page.nextCursor ?? undefined; + if (!cursor || bytes >= BATCH_BYTES) break; + } + this.projection.finishHistory(revision, title); + } + + private unsupported(error: unknown): boolean { + return error instanceof Error && /unsupported|unknown method|method not found/i.test(error.message); + } + private completed(turn: Record, entry: z.infer['data'][number]): boolean { + return turn.status !== 'inProgress' || entry.completedAtMs != null || entry.item.type === 'userMessage' + || ['completed', 'failed', 'declined'].includes(String(entry.item.status)); + } + private createdAt(turn: Record, entry: z.infer['data'][number]): number | undefined { + return entry.completedAtMs ?? entry.startedAtMs ?? (typeof turn.startedAt === 'number' ? turn.startedAt * 1000 : undefined); + } + private async syncNow(): Promise { + let state = await this.session.syncCodexHistory(this.projection.threadId); + if (this.invalidated) { + state = await this.commit(state, null, []); this.invalidated = false; + } + try { await this.catchUp(state); } + catch (error) { + // A removed turn/item anchor is a history change, not an excuse to + // trust an old checkpoint. Persist the reset before retrying. + if (state.checkpoint && error instanceof Error && /cursor|anchor|item .*not found|turn .*not found/i.test(error.message)) { + const current = await this.session.syncCodexHistory(this.projection.threadId); + await this.commit(current, null, []); this.rerun = true; + } + throw error; + } + } + private async commit(state: CodexHistoryState, checkpoint: CodexHistoryCheckpoint | null, localIds: string[]): Promise { + return this.session.syncCodexHistory(this.projection.threadId, { epoch: state.epoch, revision: state.revision, checkpoint, localIds: [...new Set(localIds)] }); + } + private async catchUp(initial: CodexHistoryState): Promise { + let state = initial; + let position = state.checkpoint ?? { ...EMPTY }; + let cursor = position.pageCursor; + let anchor = position.turnId; + let batchIds: string[] = []; let batchItems = 0; let batchBytes = 0; + const revision = this.projection.historyRevision(); let title: string | undefined; + const flush = async () => { + if (!await this.session.waitForMessages(batchIds, this.abort.signal)) throw new Error('History batch was not confirmed'); + if (this.abort.signal.aborted) return; + if (JSON.stringify(position) !== JSON.stringify(state.checkpoint)) { + if (!await this.session.flushMetadata(15_000)) throw new Error('History metadata was not confirmed'); + state = await this.commit(state, position, batchIds); + } + batchIds = []; batchItems = 0; batchBytes = 0; + }; + do { + if (this.abort.signal.aborted) return; + const page = TurnsSchema.parse(await this.client.request('thread/turns/list', { threadId: this.projection.threadId, cursor, limit: 1, sortDirection: 'asc', itemsView: 'notLoaded' })); + if (anchor && page.data[0]?.id !== anchor) throw new Error('History turn anchor no longer exists'); + anchor = null; + for (const turn of page.data) { + const continuing = position.turnId === turn.id; + if (continuing && position.turnComplete && turn.status !== 'inProgress') continue; + let itemCursor: string | { type: 'item'; itemId: string } | undefined = continuing && position.itemId && !position.turnComplete + ? { type: 'item', itemId: position.itemId } : undefined; + let gap = false; + let legacy: z.infer | undefined; + do { + if (this.abort.signal.aborted) return; + let items: z.infer; + if (this.itemPagination) { + try { items = ItemsSchema.parse(await this.client.request('thread/items/list', { threadId: this.projection.threadId, turnId: turn.id, cursor: itemCursor, limit: 1, sortDirection: 'asc' })); } + catch (error) { if (!this.unsupported(error)) throw error; this.itemPagination = false; } + } + if (!this.itemPagination) { + // Older stores: one turn, never includeTurns:true or an + // unbounded full-history request. Large single legacy + // turns still require native item-pagination support. + legacy ??= await this.legacyItems(turn, cursor, itemCursor); + items = legacy; + } + for (const entry of items!.data) { + const completed = this.completed(turn, entry); + this.onItem(turn.id, entry.item); + const result = await this.projection.historyItem(turn, entry.item, completed, true, this.createdAt(turn, entry)); + title = result.title ?? title; + batchIds.push(...result.localIds); batchItems++; batchBytes += Buffer.byteLength(JSON.stringify(entry)); + gap ||= !completed; + if (!gap) position = { version: 1, pageCursor: cursor, turnId: turn.id, itemId: entry.item.id, turnComplete: false }; + if (batchItems >= BATCH_ITEMS || batchBytes >= BATCH_BYTES) await flush(); + } + itemCursor = items!.nextCursor ?? undefined; + } while (itemCursor); + if (!gap && turn.status !== 'inProgress') position = { ...position, pageCursor: cursor, turnId: turn.id, + itemId: position.turnId === turn.id ? position.itemId : null, turnComplete: true }; + await flush(); + // Keep a contiguous prefix: later live output stays visible, + // but cannot move the checkpoint over an unfinished item. + if (gap || turn.status === 'inProgress') { this.projection.finishHistory(revision, title); return; } + } + cursor = page.nextCursor ?? null; + } while (cursor); + this.projection.finishHistory(revision, title); + } + + private async legacyItems(turn: z.infer, cursor: string | null, anchor?: string | { type: 'item'; itemId: string }): Promise> { + const page = TurnsSchema.parse(await this.client.request('thread/turns/list', { threadId: this.projection.threadId, cursor, limit: 1, sortDirection: 'asc', itemsView: 'full' })); + if (page.data[0]?.id !== turn.id) throw new Error('History turn anchor no longer exists'); + const items = (page.data[0].items ?? []).map(record); + const offset = typeof anchor === 'object' ? items.findIndex(item => item.id === anchor.itemId) : -1; + if (anchor && offset < 0) throw new Error('History item anchor no longer exists'); + return ItemsSchema.parse({ data: items.slice(offset + 1).map(item => ({ turnId: turn.id, item })), nextCursor: null }); + } +} diff --git a/cli/src/codex/shared/projection.test.ts b/cli/src/codex/shared/projection.test.ts index 3d421dc9..7d4d65fb 100644 --- a/cli/src/codex/shared/projection.test.ts +++ b/cli/src/codex/shared/projection.test.ts @@ -28,7 +28,8 @@ describe('shared history projection', () => { const original = send.mock.calls.slice(1, 3); projection.reset(); send.mockClear(); await projection.history({ turns: [{ id: 'turn', status: 'completed', items: [item] }] }); - expect(send.mock.calls).toEqual(original); + expect(send.mock.calls.map(call => call.slice(0, 2))).toEqual(original); + expect(send.mock.calls.every(call => call[2]?.replay === true)).toBe(true); }); it('waits for final proposal content after an active snapshot', async () => { diff --git a/cli/src/codex/shared/projection.ts b/cli/src/codex/shared/projection.ts index c1e44e76..bd55d27b 100644 --- a/cli/src/codex/shared/projection.ts +++ b/cli/src/codex/shared/projection.ts @@ -44,14 +44,17 @@ export class SharedCodexProjection { turnFor(id: string): string | undefined { return this.turns.get(id); } reset(): void { this.converter = new AppServerEventConverter(); this.emitted.clear(); } - private send(body: Record, key: string): void { - if (this.emitted.has(key)) return; - this.emitted.add(key); + private send(body: Record, key: string, replay = false, verifyPersistence = false, createdAt?: number): string { const id = `codex:${this.threadId}:${key}`; - this.session.sendAgentMessage(this.parentThreadId && !String(body.type).startsWith('agent-run-') ? { + if (this.emitted.has(key) && !verifyPersistence) return id; + this.emitted.add(key); + const message = this.parentThreadId && !String(body.type).startsWith('agent-run-') ? { type: 'agent-run-trace', agentId: this.threadId, cardId: `codex-agent:${this.threadId}`, message: { ...body, id }, id, scope: { role: 'child', threadId: this.threadId, parentThreadId: this.parentThreadId }, scope_role: 'child' - } : { ...body, id }, id); + } : { ...body, id }; + if (replay) this.session.sendAgentMessage(message, id, { replay: true, ...(verifyPersistence ? { verifyPersistence: true } : {}), ...(createdAt !== undefined ? { createdAt } : {}) }); + else this.session.sendAgentMessage(message, id); + return id; } async notification(method: string, params: unknown, modelAtReceipt?: string): Promise { @@ -78,8 +81,11 @@ export class SharedCodexProjection { await this.project(method, params, modelAtReceipt); } - private async project(method: string, params: unknown, modelAtReceipt?: string): Promise { - if (method.startsWith('codex/event/')) return; + private async project(method: string, params: unknown, modelAtReceipt?: string, replay = false, verifyPersistence = false): Promise { + const localIds: string[] = []; + const createdAt = typeof record(params).historyCreatedAt === 'number' ? record(params).historyCreatedAt as number : undefined; + const send = (body: Record, key: string) => { localIds.push(this.send(body, key, replay, verifyPersistence, createdAt)); }; + if (method.startsWith('codex/event/')) return localIds; const p = record(params); const item = record(p.item); const turnId = string(p.turnId) ?? string(record(p.turn).id); @@ -96,17 +102,22 @@ export class SharedCodexProjection { const id = string(item.clientId ?? item.clientUserMessageId) ?? (itemId ? `codex:${this.threadId}:user:${itemId}` : undefined); if (id) { const firstInTurn = turnId ? [...this.turns].find(([, value]) => value === turnId)?.[0] : undefined; + const changed = turnId && this.turns.get(id) !== turnId; if (turnId) this.turns.set(id, turnId); await this.committed(id); const text = inputText(item.content); - if (text) this.session.sendUserMessage(text, undefined, id); - this.session.updateMetadata(metadata => ({ ...metadata, conversationHistoryTurns: Object.fromEntries(this.turns), + if (text) { + localIds.push(id); + if (replay) this.session.sendUserMessage(text, undefined, id, { replay: true, ...(verifyPersistence ? { verifyPersistence: true } : {}), ...(createdAt !== undefined ? { createdAt } : {}) }); + else this.session.sendUserMessage(text, undefined, id); + } + if (changed) this.session.updateMetadata(metadata => ({ ...metadata, conversationHistoryTurns: Object.fromEntries(this.turns), ...(turnId && (!firstInTurn || firstInTurn === id) ? { conversationHistoryPoints: { ...metadata.conversationHistoryPoints, [id]: true } } : {}) })); } } if (this.parentThreadId && (method === 'turn/started' || method === 'turn/completed')) { - this.send({ type: 'agent-run-update', agentId: this.threadId, cardId: `codex-agent:${this.threadId}`, + send({ type: 'agent-run-update', agentId: this.threadId, cardId: `codex-agent:${this.threadId}`, status: method === 'turn/started' ? 'running' : record(p.turn).status === 'completed' ? 'completed' : 'failed' }, `lifecycle:${turnId}:${method}`); } @@ -114,54 +125,75 @@ export class SharedCodexProjection { for (const event of events) { const callId = string(event.call_id); const key = `${turnId ?? 'thread'}:${itemId ?? callId ?? createHash('sha256').update(JSON.stringify(event)).digest('hex')}:${event.type}`; - if (event.type === 'agent_message') this.send({ type: 'message', message: event.message }, key); - else if (event.type === 'agent_reasoning') this.send({ type: 'reasoning', message: event.text }, key); + if (event.type === 'agent_message') send({ type: 'message', message: event.message }, key); + else if (event.type === 'agent_reasoning') send({ type: 'reasoning', message: event.text }, key); else if (event.type === 'exec_command_begin' && callId) { - this.send({ type: 'tool-call', name: 'CodexBash', callId, input: event }, key); + send({ type: 'tool-call', name: 'CodexBash', callId, input: event }, key); } else if (event.type === 'exec_command_end' && callId) { - this.send({ type: 'tool-call-result', callId, output: { ...event, stdout: event.output } }, key); + send({ type: 'tool-call-result', callId, output: { ...event, stdout: event.output } }, key); } else if (event.type === 'patch_apply_begin' && callId) { - this.send({ type: 'tool-call', name: 'CodexPatch', callId, input: { changes: event.changes, auto_approved: event.auto_approved } }, key); + send({ type: 'tool-call', name: 'CodexPatch', callId, input: { changes: event.changes, auto_approved: event.auto_approved } }, key); } else if (event.type === 'patch_apply_end' && callId) { - this.send({ type: 'tool-call-result', callId, output: { stdout: event.stdout, stderr: event.stderr, success: event.success } }, key); + send({ type: 'tool-call-result', callId, output: { stdout: event.stdout, stderr: event.stderr, success: event.success } }, key); } else if (event.type === 'mcp_tool_call_begin' && callId) { const invocation = record(event.invocation); - this.send({ type: 'tool-call', name: `mcp__${invocation.server}__${invocation.tool}`, callId, input: invocation.arguments ?? {} }, key); + send({ type: 'tool-call', name: `mcp__${invocation.server}__${invocation.tool}`, callId, input: invocation.arguments ?? {} }, key); } else if (event.type === 'mcp_tool_call_end' && callId) { const result = record(event.result); - this.send({ type: 'tool-call-result', callId, output: result.Ok ?? result.Err ?? event.result, is_error: 'Err' in result }, key); + send({ type: 'tool-call-result', callId, output: result.Ok ?? result.Err ?? event.result, is_error: 'Err' in result }, key); } else if (event.type === 'codex_tool_call_begin' && callId) { - this.send({ type: 'tool-call', name: event.name, callId, input: event.input ?? event.arguments }, key); + send({ type: 'tool-call', name: event.name, callId, input: event.input ?? event.arguments }, key); } else if (event.type === 'codex_tool_call_end' && callId) { - this.send({ type: 'tool-call-result', callId, output: event.output, is_error: event.is_error }, key); + send({ type: 'tool-call-result', callId, output: event.output, is_error: event.is_error }, key); } else if (event.type === 'token_count' || event.type === 'context_compacted' || event.type.startsWith('thread_goal_')) { const model = event.type === 'token_count' && turnId ? this.turnModels.get(turnId) : undefined; - this.send({ ...event, ...(model ? { model } : {}), flavor: 'codex', scope: { role: 'parent', threadId: this.threadId }, scope_role: 'parent', thread_id: this.threadId }, key); + send({ ...event, ...(model ? { model } : {}), flavor: 'codex', scope: { role: 'parent', threadId: this.threadId }, scope_role: 'parent', thread_id: this.threadId }, key); } else if (event.type === 'proposed_plan' && turnId && itemId) { const planId = codexPlanProposalId(this.threadId, turnId, itemId); - this.send({ type: 'tool-call', name: 'ExitPlanMode', callId: planId, input: { plan: event.plan } }, key); + send({ type: 'tool-call', name: 'ExitPlanMode', callId: planId, input: { plan: event.plan } }, key); // A proposal is durable content, not a native approval request. - this.send({ type: 'tool-call-result', callId: planId, output: null }, `${key}:result`); + send({ type: 'tool-call-result', callId: planId, output: null }, `${key}:result`); } else if (event.type === 'plan_update') { - this.send({ type: 'tool-call', name: 'update_plan', callId: 'codex-plan-state', input: { plan: event.plan, source: 'codex' } }, key); - this.send({ type: 'tool-call-result', callId: 'codex-plan-state', output: { plan: event.plan, source: 'codex', status: 'updated' } }, `${key}:result`); + send({ type: 'tool-call', name: 'update_plan', callId: 'codex-plan-state', input: { plan: event.plan, source: 'codex' } }, key); + send({ type: 'tool-call-result', callId: 'codex-plan-state', output: { plan: event.plan, source: 'codex', status: 'updated' } }, `${key}:result`); } else if (event.type === 'generated_image' && typeof event.saved_path === 'string') { const image = await registerGeneratedImageFromPath({ path: event.saved_path, id: createHash('sha256').update(`${this.threadId}:${key}`).digest('hex'), fileName: string(event.file_name) }); - if (image) this.send({ type: 'generated-image', imageId: image.id, fileName: image.fileName, mimeType: image.mimeType }, key); + if (image) send({ type: 'generated-image', imageId: image.id, fileName: image.fileName, mimeType: image.mimeType }, key); } else if (event.type === 'task_failed') { - this.send({ type: 'message', message: `Codex error: ${event.error ?? event.message ?? 'Turn failed'}` }, key); + send({ type: 'message', message: `Codex error: ${event.error ?? event.message ?? 'Turn failed'}` }, key); } } if (item.type === 'collabAgentToolCall') { const states = record(item.agentsStates); for (const [agentId, state] of Object.entries(states)) { const status = string(record(state).status) ?? 'running'; - this.send({ type: 'agent-run-update', agentId, cardId: `codex-agent:${agentId}`, + send({ type: 'agent-run-update', agentId, cardId: `codex-agent:${agentId}`, status: status === 'completed' ? 'completed' : status === 'errored' ? 'failed' : 'running', summary: record(state).message, input: item, scope: { role: 'child', threadId: agentId, parentThreadId: this.threadId }, scope_role: 'child', thread_id: agentId }, `agent:${agentId}:${itemId}:${method}:${JSON.stringify(state)}`); } } + return localIds; + } + + historyRevision(): number { return this.titleRevision; } + async historyItem(turn: Record, value: unknown, completed?: boolean, verifyPersistence = false, createdAt?: number): Promise<{ localIds: string[]; title?: string }> { + const item = record(value); + const params = { threadId: this.threadId, turnId: turn.id, item, historyCreatedAt: createdAt }; + const titleKey = `${string(turn.id) ?? 'thread'}:${item.id}`; + const pendingTitle = requestedTitle(item); + if (!this.parentThreadId && string(item.id) && pendingTitle && !this.completedTitles.has(titleKey)) this.pendingTitles.set(titleKey, pendingTitle); + const localIds = await this.project('item/started', params, undefined, true, verifyPersistence); + let title: string | undefined; + if (completed ?? (turn.status !== 'inProgress' || item.status === 'completed' || item.type === 'userMessage')) { + if (!this.parentThreadId) { + title = successfulTitle(item); + if (title && string(item.id)) this.completedTitles.add(titleKey); + this.pendingTitles.delete(titleKey); + } + localIds.push(...await this.project('item/completed', params, undefined, true, verifyPersistence)); + } + return { localIds, title }; } async history(thread: unknown): Promise { @@ -172,29 +204,12 @@ export class SharedCodexProjection { for (const value of turns) { const turn = record(value); if (!Array.isArray(turn.items)) continue; - for (const item of turn.items) { - const params = { threadId: this.threadId, turnId: turn.id, item }; - const titleKey = `${string(turn.id) ?? 'thread'}:${record(item).id}`; - const pendingTitle = requestedTitle(record(item)); - if (!this.parentThreadId && string(record(item).id) && pendingTitle && !this.completedTitles.has(titleKey)) { - this.pendingTitles.set(titleKey, pendingTitle); - } - await this.project('item/started', params); - // Active snapshots can contain partial assistant text. Do not - // settle it under the final stable id and suppress completion. - if (turn.status !== 'inProgress' || record(item).status === 'completed' || record(item).type === 'userMessage') { - if (!this.parentThreadId) { - const title = successfulTitle(record(item)); - if (title && string(record(item).id)) { - latestTitle = title; - this.completedTitles.add(titleKey); - } - this.pendingTitles.delete(titleKey); - } - await this.project('item/completed', params); - } - } + for (const item of turn.items) latestTitle = (await this.historyItem(turn, item)).title ?? latestTitle; } + this.finishHistory(titleRevision, latestTitle); + } + + finishHistory(titleRevision: number, latestTitle?: string): void { // Repair sessions created while remote title projection was missing. // Recheck inside the metadata lock: live updates may still be queued, // and a replay must never replace an existing or newer title. diff --git a/cli/src/codex/shared/queue.ts b/cli/src/codex/shared/queue.ts index 0ddb01bc..88804c3d 100644 --- a/cli/src/codex/shared/queue.ts +++ b/cli/src/codex/shared/queue.ts @@ -63,6 +63,7 @@ export class SharedCodexQueue { }); } async committed(id: string): Promise { + if (this.entries[id]?.state === 'consumed') { this.consumed([id]); return; } // History may prove acceptance after execution replacement, before hub // redelivery. Remember it even when this generation did not enqueue it. const entry = this.entries[id] ??= { state: 'consumed', input: [] }; diff --git a/cli/src/codex/shared/root.test.ts b/cli/src/codex/shared/root.test.ts index 392644ce..a6797f18 100644 --- a/cli/src/codex/shared/root.test.ts +++ b/cli/src/codex/shared/root.test.ts @@ -28,6 +28,22 @@ vi.mock('../codexAppServerClient', () => ({ async request(method: string, params: Record = {}) { this.requests.push(method); if (method === 'thread/read' || method === 'thread/resume') return { ...this.settings, thread: structuredClone(this.thread) }; + if (method === 'thread/turns/list') { + const turns = params.sortDirection === 'desc' ? [...this.thread.turns].reverse() : this.thread.turns; + const offset = Number(params.cursor ?? 0); const limit = Number(params.limit ?? 1); + const data = structuredClone(turns.slice(offset, offset + limit)); + if (params.itemsView === 'notLoaded') for (const turn of data) turn.items = []; + return { data, nextCursor: offset + limit < turns.length ? String(offset + limit) : null }; + } + if (method === 'thread/items/list') { + const turn = this.thread.turns.find(turn => turn.id === params.turnId); + const items = params.sortDirection === 'desc' ? [...turn?.items ?? []].reverse() : turn?.items ?? []; + const anchor = params.cursor as { itemId?: string } | undefined; + const offset = typeof params.cursor === 'object' ? items.findIndex(item => (item as { id: string }).id === anchor?.itemId) + 1 : Number(params.cursor ?? 0); + const data = items.slice(offset, offset + 1).map(item => ({ turnId: turn?.id, item, + completedAtMs: turn?.status !== 'inProgress' || (item as { status?: string }).status === 'completed' ? 1 : null })); + return { data: structuredClone(data), nextCursor: offset + 1 < items.length ? String(offset + 1) : null }; + } if (method === 'thread/list') return { data: [] }; if (method === 'thread/queue/list') return { data: this.queue }; if (method === 'thread/settings/update') { @@ -63,6 +79,7 @@ async function fixture(options: { deferHistory?: boolean } = {}) { const updateState = vi.fn((fn: (value: AgentState) => AgentState) => { state = fn(state); }); const rpc = new Map Promise>(); const send = vi.fn(); + let historyState = { epoch: 0, revision: 0, checkpoint: null as import('@hapi/protocol').CodexHistoryCheckpoint | null }; const session = { sessionId: 'sid', getMetadata: () => metadata, updateMetadata: (fn: (value: Metadata) => Metadata) => { metadata = fn(metadata); }, @@ -72,7 +89,12 @@ async function fixture(options: { deferHistory?: boolean } = {}) { rpcHandlerManager: { registerHandler: (name: string, handler: (raw: unknown) => Promise) => rpc.set(name, handler) }, sendSessionEvent() {}, sendAgentMessage: send, emitSessionReady() {}, sendUserMessage() {}, emitMessagesConsumed() {}, emitSteerIndeterminate() {}, syncNativeQueuedMessage() {}, - sendSessionDeath() {}, async flush() {}, close() {} + sendSessionDeath() {}, async flush() {}, close() {}, + async flushMetadata() { return true; }, async waitForMessages() { return true; }, + async syncCodexHistory(_threadId: string, commit?: import('@hapi/protocol').CodexHistorySyncRequest['commit']) { + if (commit) historyState = { epoch: commit.epoch, revision: historyState.revision + 1, checkpoint: commit.checkpoint }; + return historyState; + } } as unknown as ApiSessionClient; const root = new SharedCodexRoot({ session, workingDirectory: directory } as SessionBootstrapResult, { directory, generation: 'test', endpoint: 'mock', settingsFor: () => undefined, @@ -108,6 +130,30 @@ async function completePlan(f: Awaited>, status = 'co } describe('shared plan actions', () => { + it('offers immediate native Goal controls while a turn is running', async () => { + const f = await fixture(); + await f.root.activate(); + expect(f.metadata().capabilities?.codexGoal).toBe(true); + const goal = { threadId: 'thread', objective: 'Fix the build', status: 'active', tokenBudget: null, + tokensUsed: 100, timeUsedSeconds: 1, createdAt: 1, updatedAt: 2 }; + const request = vi.spyOn(f.root.client, 'request').mockImplementation(async (method, params) => { + if (method === 'thread/goal/get') return { goal }; + if (method === 'thread/goal/set') return { goal: { ...goal, ...params as Record } }; + if (method === 'thread/goal/clear') return { cleared: true }; + throw new Error(`Unexpected request: ${method}`); + }); + f.native.notify('turn/started', { threadId: 'thread', turn: { id: 'busy-turn' } }); + const action = f.rpc.get(RPC_METHODS.CodexGoal)!; + expect(await action({ action: 'pause' })).toMatchObject({ goal: { status: 'paused' } }); + expect(await action({ action: 'resume' })).toMatchObject({ goal: { status: 'active' } }); + expect(await action({ action: 'clear' })).toEqual({ goal: null }); + expect(request.mock.calls.every(([method, params]) => method.startsWith('thread/goal/') + && (params as { threadId: string }).threadId === 'thread')).toBe(true); + expect(f.send).toHaveBeenCalledWith(expect.objectContaining({ type: 'thread_goal_updated', + scope: { role: 'parent', threadId: 'thread' }, goal: expect.objectContaining({ status: 'paused' }) })); + await vi.waitFor(() => expect(f.send).toHaveBeenCalledWith(expect.objectContaining({ type: 'thread_goal_cleared', thread_id: 'thread' }))); + }); + it('keeps bind local and reconciles native history only in syncHistory', async () => { const f = await fixture({ deferHistory: true }); expect(f.native.requests).not.toContain('thread/read'); @@ -128,6 +174,17 @@ describe('shared plan actions', () => { expect(f.metadata().name).toBe('Terminal title'); }); + it('uses the authoritative settings snapshot for a no-op cold-resume mode', async () => { + const f = await fixture(); + f.native.notify('thread/settings/updated', { threadId: 'thread', threadSettings: { model: 'mock', + collaborationMode: { mode: 'plan', settings: { model: 'mock', reasoning_effort: null } } } }); + const before = f.native.requests.filter(method => method === 'thread/settings/update').length; + await f.root.initialSettings({ collaborationMode: 'plan' }); + expect(f.native.requests.filter(method => method === 'thread/settings/update')).toHaveLength(before); + await f.root.applySettings({ collaborationMode: 'default' }); + expect(f.native.requests.filter(method => method === 'thread/settings/update')).toHaveLength(before + 1); + }); + it('preserves content while native turns, mode changes and disconnects withdraw controls', async () => { const f = await fixture(); const id = await completePlan(f); @@ -194,7 +251,7 @@ describe('shared plan actions', () => { const started = new Promise(resolve => { reading = resolve; }); vi.spyOn(f.root.client, 'request').mockImplementation(async (method, params) => { const result = await request(method, params); - if (method === 'thread/read' && (params as { includeTurns?: boolean }).includeTurns) { + if (method === 'thread/turns/list' && (params as { itemsView?: string }).itemsView === 'summary') { reading(); await blocked; } return result; @@ -258,6 +315,7 @@ describe('shared plan actions', () => { describe('shared steering availability', () => { it('keeps idle sessions online without polling usage or publishing agent-state updates', async () => { const f = await fixture(); + await vi.waitFor(() => expect(f.native.requests.filter(method => method === 'thread/turns/list').length).toBeGreaterThan(1)); const requests = vi.spyOn(f.root.client, 'request'); const heartbeat = vi.spyOn(f.root.session, 'keepAlive'); const updates = f.updateState.mock.calls.length; diff --git a/cli/src/codex/shared/root.ts b/cli/src/codex/shared/root.ts index 8642fe42..dddf929f 100644 --- a/cli/src/codex/shared/root.ts +++ b/cli/src/codex/shared/root.ts @@ -15,11 +15,13 @@ import { buildUserInputFromMessage } from '../utils/appServerConfig'; import { resolveCodexPermissionModeConfig } from '../utils/permissionModeConfig'; import { resolveCodexSlashCommand } from '../utils/slashCommands'; import { parseReasoningEffortValue } from '../utils/reasoningEffort'; +import { manageThreadGoal } from '../utils/manageThreadGoal'; import { SharedCodexPermissions } from './permissions'; import { SharedCodexQueue } from './queue'; import { SharedCodexProjection, inputText } from './projection'; import { getCodexSystemPrompt } from '../utils/systemPrompt'; import { record, string } from './gateway'; +import { CodexHistorySync, readCurrentThread } from './history'; import { initializeSharedClient, type SharedLaunchOptions } from './launch'; import { inheritedSandbox, settingsMatch } from './settings'; import { planImplementationMessageId, planProposalForItem, planProposalForTurn } from './plan'; @@ -48,12 +50,15 @@ export class SharedCodexRoot { private permissions!: SharedCodexPermissions; private queue!: SharedCodexQueue; private projection!: SharedCodexProjection; + private history!: CodexHistorySync; + private readonly childHistories = new Map(); private readonly children = new Map(); private readonly ancestry = new Map(); private heartbeat?: ReturnType; private work: Promise = Promise.resolve(); private notifications = Promise.resolve(); private currentTurn: string | undefined; + private goalRevision = 0; private latestTurn?: { id: string; status: string; planId?: string }; private publishedPlanId: string | null | undefined; private submittingPlan = false; @@ -62,6 +67,7 @@ export class SharedCodexRoot { private settingsRevision = 0; private refreshing?: Promise; private interrupted = false; + private pristineThread?: Record; private closed = false; private stopping = false; private closing?: Promise; @@ -83,7 +89,9 @@ export class SharedCodexRoot { } if (method === 'thread/settings/updated' && record(params).threadId === this.threadId) this.acceptSettings(record(record(params).threadSettings)); if (record(params).threadId === this.threadId) { + if (method === 'thread/goal/updated' || method === 'thread/goal/cleared') this.goalRevision++; if (method === 'turn/started') { + this.pristineThread = undefined; this.turnRevision++; this.currentTurn = string(record(record(params).turn).id); this.interrupted = false; if (this.currentTurn) this.latestTurn = { id: this.currentTurn, status: 'inProgress' }; this.publishSteering(); @@ -148,6 +156,8 @@ export class SharedCodexRoot { this.bridge = await buildHapiMcpBridge(this.session, { exportSessionEnv: false, emitTitleSummary: false, skillLookup: { workingDirectory: this.bootstrap.workingDirectory, flavor: 'codex' } }); await initializeSharedClient(this.client); + await this.client.request('experimentalFeature/enablement/set', { enablement: { goals: true } }) + .catch(error => logger.debug('[Codex shared] Goal feature enablement', error)); this.permissions = new SharedCodexPermissions(this.session, this.client, this.host.generation); this.client.setServerRequestHandler(request => { void this.receiveRequest(request); }); } @@ -188,12 +198,18 @@ export class SharedCodexRoot { async bind(threadId: string, response: Record, subscribe: boolean): Promise> { if (this.threadId && this.threadId !== threadId) throw new Error('Cannot retarget a shared HAPI session'); this.threadId = threadId; + const initialThread = record(response.thread); + // Native 0.159 does not create a source rollout until the first input. + // A freshly created empty root already has an authoritative snapshot; + // asking persisted-history APIs here rejects an otherwise usable root. + if (subscribe && initialThread.preview === '' && Array.isArray(initialThread.turns) && !initialThread.turns.length) this.pristineThread = initialThread; this.queue = new SharedCodexQueue(this.client, threadId, join(this.host.directory, `${this.session.sessionId}.queue.json`), (ids, steered) => this.session.emitMessagesConsumed(ids, { steered }), ids => this.session.emitSteerIndeterminate(ids), (id, input) => this.session.syncNativeQueuedMessage(id, input === null ? null : inputText(input)), ids => this.session.setSteerDeliveryState(ids, 'queued')); await this.queue.load(); this.projection = new SharedCodexProjection(this.session, threadId, id => this.queue.committed(id)); + this.history = this.historyFor(this.projection); this.session.updateMetadata(metadata => ({ ...metadata, codexSessionId: threadId, capabilities: { ...metadata.capabilities, concurrentClients: true, terminal: true, // In-place rewind needs a native + hub commit barrier. Do not @@ -204,20 +220,30 @@ export class SharedCodexRoot { completedRequests: { ...state.completedRequests, ...Object.fromEntries(Object.entries(state.requests ?? {}).map(([id, request]) => [id, { ...request, completedAt: Date.now(), status: 'canceled' as const }])) } })); - if (subscribe) response = record(await this.client.request('thread/resume', { threadId })); + if (subscribe && !this.pristineThread) response = record(await this.client.request('thread/resume', { threadId, excludeTurns: true })); this.acceptSettings(response); this.acceptSettings(this.host.settingsFor(threadId) ?? {}); return response; } - /** Reconcile the full native history into the hub. Huge threads can take - * minutes; the HAPI session is already usable while this runs. */ - async syncHistory(response: Record): Promise { - await this.projection.history(response.thread); await this.refresh(); await this.refreshChildren(true); + /** Current state first; durable history catch-up runs independently. */ + async syncHistory(_response: Record): Promise { + await this.refresh(); await this.refreshChildren(true); } + hasNativeHistory(): boolean { return !this.pristineThread; } + private historyFor(projection: SharedCodexProjection): CodexHistorySync { + return new CodexHistorySync(this.client, this.session, projection, (turnId, item) => { + if (projection.threadId !== this.threadId || this.latestTurn?.id !== turnId) return; + const planId = planProposalForItem(this.threadId, turnId, item); + if (planId && this.latestTurn.planId !== planId) { this.turnRevision++; this.latestTurn.planId = planId; this.publishPlan(); } + }, error => { if (!this.closed && !this.stopping) logger.debug('[Codex shared] incremental history', + error instanceof Error ? { message: error.message, stack: error.stack } : error); }); + } + invalidateHistory(): void { this.history?.invalidate(); for (const history of this.childHistories.values()) history.invalidate(); } async activate(options: SharedLaunchOptions = {}): Promise { // Restored input cannot run before the cold-resume settings are applied. await this.initialSettings(options); if (this.stopping) throw new Error('Codex execution is stopping'); this.registerControls(); this.ready(); + this.session.updateMetadata(metadata => ({ ...metadata, capabilities: { ...metadata.capabilities, codexGoal: true } })); this.heartbeat = setInterval(() => this.alive(), 2_000); this.alive(); this.session.emitSessionReady(); } private publishSteering(): void { @@ -274,7 +300,13 @@ export class SharedCodexRoot { projection = new SharedCodexProjection(this.session, eventThread, async () => {}, this.threadId); this.children.set(eventThread, projection); } - await projection.notification(method, params); return; + await projection.notification(method, params); + if (method === 'item/completed' || method === 'turn/completed') { + let history = this.childHistories.get(eventThread); + if (!history) { history = this.historyFor(projection); this.childHistories.set(eventThread, history); } + history.schedule(); + } + return; } if (method === 'turn/completed') this.session.sendSessionEvent({ type: 'ready' }); if (method === 'thread/name/updated' && (typeof p.threadName === 'string' || p.threadName === null)) { @@ -283,6 +315,7 @@ export class SharedCodexRoot { if (method === 'thread/archived') { await this.host.end(this, false); return; } if (method === 'thread/queue/changed') await this.queue.reconcile(); await this.projection.notification(method, params, modelAtReceipt); this.alive(); + if (method === 'item/completed' || method === 'turn/completed') this.history?.schedule(); } async readThread(threadId = this.threadId): Promise> { let thread = record(record(await this.client.request('thread/read', { threadId, includeTurns: false })).thread); @@ -303,7 +336,7 @@ export class SharedCodexRoot { private async refreshNow(): Promise { if (!this.threadId || this.closed || !this.client.isInitialized()) return; const revision = this.turnRevision; - const thread = await this.readThread(); + const thread = this.pristineThread ? { ...this.pristineThread, turns: [] } : await readCurrentThread(this.client, this.threadId); const turns = Array.isArray(thread.turns) ? thread.turns.map(record) : []; if (revision === this.turnRevision) { this.currentTurn = string(turns.find(turn => turn.status === 'inProgress')?.id); @@ -312,13 +345,15 @@ export class SharedCodexRoot { const id = string(last?.id); this.latestTurn = id && last ? { id, status: string(last.status) ?? 'unknown', planId: planProposalForTurn(this.threadId, last) } : undefined; } - await this.projection.history(thread); await this.queue.reconcile(); this.alive(); + const last = turns.at(-1); + if (last) await this.history.preview(last); + await this.queue.reconcile(); this.alive(); if (!this.pristineThread) this.history.schedule(0); } private async refreshChildren(subscribe: boolean): Promise { let cursor: string | undefined; do { const page = record(await this.client.request('thread/list', { ancestorThreadId: this.threadId, cursor, - sourceKinds: ['subAgent', 'subAgentReview', 'subAgentCompact', 'subAgentThreadSpawn', 'subAgentOther'] })); + useStateDbOnly: true, sourceKinds: ['subAgent', 'subAgentReview', 'subAgentCompact', 'subAgentThreadSpawn', 'subAgentOther'] })); for (const value of Array.isArray(page.data) ? page.data : []) { const thread = record(value); const id = string(thread.id); if (id && id !== this.threadId) this.ancestry.set(id, string(thread.parentThreadId) ?? null); @@ -337,8 +372,13 @@ export class SharedCodexRoot { for (const [id, projection] of this.children) { // Replaying a completed/unloaded child must not start its engine. try { - if (subscribe && loaded.has(id)) await this.client.request('thread/resume', { threadId: id }); - projection.reset(); await projection.history(await this.readThread(id)); + if (subscribe && loaded.has(id)) await this.client.request('thread/resume', { threadId: id, excludeTurns: true }); + let history = this.childHistories.get(id); + if (!history) { history = this.historyFor(projection); this.childHistories.set(id, history); } + const thread = await readCurrentThread(this.client, id); + const last = Array.isArray(thread.turns) ? record(thread.turns.at(-1)) : undefined; + if (last) await history.preview(last); + history.schedule(0); } catch (error) { logger.debug('[Codex shared] child history unavailable', { id, error }); } } } @@ -359,7 +399,7 @@ export class SharedCodexRoot { this.permissions = new SharedCodexPermissions(this.session, this.client, `${this.host.generation}:${randomUUID()}`); this.client.setServerRequestHandler(request => { void this.receiveRequest(request); }); const settingsRevision = this.settingsRevision; - const response = record(await this.client.request('thread/resume', { threadId: this.threadId })); + const response = this.pristineThread ? {} : record(await this.client.request('thread/resume', { threadId: this.threadId, excludeTurns: true })); const observedSettings = this.settingsRevision !== settingsRevision; this.acceptSettings(response); if (!observedSettings) this.acceptSettings(this.host.settingsFor(this.threadId) ?? {}); @@ -402,6 +442,9 @@ export class SharedCodexRoot { developer_instructions: null } } } : {}) }; + // Native 0.159 omits a settings notification for a no-op update. + // An existing authoritative snapshot already confirms these values. + if (settingsMatch(this.settingsNative, params)) return { applied: this.settings }; let changed!: () => void; let timer: ReturnType | undefined; const accepted = new Promise((resolve, reject) => { @@ -415,6 +458,14 @@ export class SharedCodexRoot { // The RPC only acknowledges queueing the settings operation. The // native notification, not our submitted candidate, establishes state. await accepted; this.alive(); return { applied: this.settings }; + } catch (error) { + logger.debug('[Codex shared] settings confirmation failed', { expected: params, actual: { + model: this.settingsNative.model, effort: this.settingsNative.effort, + collaborationMode: { mode: record(this.settingsNative.collaborationMode).mode, + settings: { model: record(record(this.settingsNative.collaborationMode).settings).model, + reasoning_effort: record(record(this.settingsNative.collaborationMode).settings).reasoning_effort } } + } }); + throw error; } finally { clearTimeout(timer); this.settingsListeners.delete(changed); } } async initialSettings(options: SharedLaunchOptions): Promise { @@ -473,6 +524,23 @@ export class SharedCodexRoot { }); rpc.registerHandler(RPC_METHODS.KillSession, async () => { await this.host.end(this); return { success: true }; }); rpc.registerHandler(RPC_METHODS.SetSessionConfig, raw => this.applySettings(raw)); + rpc.registerHandler(RPC_METHODS.CodexGoal, async raw => { + const revision = this.goalRevision; + const result = await manageThreadGoal(this.threadId, raw, (method, params) => this.client.request(method, params)); + if (record(raw).action !== 'get' && this.goalRevision === revision) { + // Response fallback for servers that omit notifications. Keep + // older projections ahead of it and prefer newer native events. + this.notifications = this.notifications.then(() => { + if (this.closed || this.goalRevision !== revision) return; + this.session.sendAgentMessage({ + type: result.goal ? 'thread_goal_updated' : 'thread_goal_cleared', + thread_id: this.threadId, scope: { role: 'parent', threadId: this.threadId }, + ...(result.goal ? { goal: result.goal } : {}) + }); + }).catch(error => logger.debug('[Codex shared] Goal projection', error)); + } + return result; + }); rpc.registerHandler(RPC_METHODS.ImplementCodexPlan, raw => { const { planId } = ImplementCodexPlanRequestSchema.parse(raw); const work = this.work.catch(() => {}).then(() => this.implementPlan(planId)); @@ -552,6 +620,7 @@ export class SharedCodexRoot { } stopAccepting(): void { this.stopping = true; this.ready(); + this.history?.close(); for (const history of this.childHistories.values()) history.close(); this.publishSteering(); this.publishPlan(); this.session.onReconnect(null); this.client.setTransportAbandonedHandler(null); diff --git a/cli/src/codex/shared/runtime.integration.test.ts b/cli/src/codex/shared/runtime.integration.test.ts index 1b2a9583..9bfd836d 100644 --- a/cli/src/codex/shared/runtime.integration.test.ts +++ b/cli/src/codex/shared/runtime.integration.test.ts @@ -15,6 +15,13 @@ class MockSession { readonly sessionId = randomUUID(); state: AgentState = { requests: { 'old-worker': { tool: 'request_user_input', arguments: {}, createdAt: 0 } } }; metadata: Metadata; messages: unknown[] = []; consumed: string[] = []; dead = false; + histories = new Map(); + async syncCodexHistory(threadId: string, commit?: import('@hapi/protocol').CodexHistorySyncRequest['commit']) { + const prior = this.histories.get(threadId) ?? { epoch: 0, revision: 0, checkpoint: null }; + const state = commit ? { epoch: 0, revision: prior.revision + 1, checkpoint: commit.checkpoint } : prior; + this.histories.set(threadId, state); return state; + } + async waitForMessages() { return true; } async flushMetadata() { return true; } user?: (message: { content: { text: string } }, id?: string) => void; rpc = new Map Promise>(); rpcHandlerManager = { registerHandler: (name: string, fn: (params: unknown) => Promise) => { this.rpc.set(name, fn); } }; @@ -170,4 +177,65 @@ describe.skipIf(process.env.HAPI_RUN_SHARED_CODEX_TESTS !== '1')('installed Code await rm(home, { recursive: true, force: true, maxRetries: 3 }); } }, 60_000); + it('cold-resumes before history synchronization can block the runner webhook', async () => { + const home = await mkdtemp('/tmp/hapi-shared-resume-'); state.home = home; + const ch = join(home, 'codex'); await mkdir(ch); + await writeFile(join(ch, 'config.toml'), 'model = "mock-model"\nmodel_provider = "mock"\n[model_providers.mock]\nname = "No model calls"\nbase_url = "http://127.0.0.1:1/v1"\nwire_api = "responses"\nrequires_openai_auth = false\n[analytics]\nenabled = false\n[feedback]\nenabled = false\n'); + vi.stubEnv('CODEX_HOME', ch); vi.stubEnv('HOME', home); + const { runSharedRuntime } = await import('./runtime'); + const { SharedCodexRoot } = await import('./root'); + const { bootstrapExistingSession } = await import('@/agent/sessionFactory'); + const { notifyRunnerSessionStarted } = await import('@/runner/controlClient'); + const seedRequest = CodexAppServerClient.prototype.request; + let coldThreadId: string | undefined; + // Native 0.159 cannot hydrate a new empty paginated thread without a source rollout. + // Seed a persisted legacy thread; the cold-resume behavior is shared by both formats. + const seed = vi.spyOn(CodexAppServerClient.prototype, 'request').mockImplementation(function(this: CodexAppServerClient, method: string, params?: unknown): Promise { + if (method === 'thread/resume' && coldThreadId && record(params).threadId === coldThreadId && record(params).excludeTurns !== true) { + return Promise.reject(new Error('Full history hydration would block startup')); + } + return seedRequest.call(this, method, method === 'thread/start' ? { ...record(params), historyMode: 'legacy' } : params) as Promise; + }); + const stop = new AbortController(); + let initialReady!: (value: import('./runtime').RuntimeReady) => void; + const initial = new Promise(resolve => { initialReady = resolve; }); + const running = runSharedRuntime({ workingDirectory: home }, initialReady, stop.signal); + let resumed: Promise | undefined; + const coldStop = new AbortController(); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + let historyEntered!: () => void; + const history = new Promise(resolve => { historyEntered = resolve; }); + const spies: Array<{ mockRestore(): void }> = [seed]; + try { + const { sessionId, runtime } = await Promise.race([initial, running.then(() => { throw new Error('Runtime stopped before ready'); })]); + const threadId = runtime.sessions[sessionId].threadId; + stop.abort(); await running; + coldThreadId = threadId; + const session = state.sessions.get(sessionId)!; session.dead = false; + vi.mocked(bootstrapExistingSession).mockResolvedValueOnce({ + session: session as unknown as ApiSessionClient, sessionInfo: { id: sessionId, namespace: 'test' }, + metadata: session.metadata, workingDirectory: home, machineId: 'test', startedBy: 'terminal', api: {} + } as unknown as Awaited>); + const syncHistory = SharedCodexRoot.prototype.syncHistory; + spies.push(vi.spyOn(SharedCodexRoot.prototype, 'syncHistory').mockImplementation(async function(this: import('./root').SharedCodexRoot, response: Record) { + historyEntered(); await gate; await syncHistory.call(this, response); + })); + vi.mocked(notifyRunnerSessionStarted).mockClear(); + let coldReady!: (value: import('./runtime').RuntimeReady) => void; + const cold = new Promise(resolve => { coldReady = resolve; }); + resumed = runSharedRuntime({ workingDirectory: home, resumeSessionId: threadId, existingSessionId: sessionId }, coldReady, coldStop.signal); + await Promise.race([history, resumed]); + expect(notifyRunnerSessionStarted).toHaveBeenCalledWith(sessionId, expect.objectContaining({ codexSessionId: threadId })); + expect(session.dead).toBe(false); + expect(state.sessions.size).toBe(1); + release(); + expect((await Promise.race([cold, resumed.then(() => { throw new Error('Resume stopped before ready'); })])).sessionId).toBe(sessionId); + } finally { + release(); stop.abort(); coldStop.abort(); + await running.catch(() => {}); await resumed?.catch(() => {}); + for (const spy of spies) spy.mockRestore(); + await rm(home, { recursive: true, force: true }); + } + }, 30_000); }); diff --git a/cli/src/codex/shared/runtime.ts b/cli/src/codex/shared/runtime.ts index 987f1f24..bc217b83 100644 --- a/cli/src/codex/shared/runtime.ts +++ b/cli/src/codex/shared/runtime.ts @@ -104,7 +104,9 @@ export async function runSharedRuntime(options: SharedLaunchOptions, onReady?: ( try { await initializeSharedClient(control); // Roots remain loaded by their independent side-clients. - for (const threadId of roots.keys()) await control.request('thread/resume', { threadId }); + for (const [threadId, root] of roots) { + if (root.hasNativeHistory()) await control.request('thread/resume', { threadId, excludeTurns: true }); + } return; } catch (error) { logger.debug('[Codex shared] control reconnect', error); @@ -123,8 +125,10 @@ export async function runSharedRuntime(options: SharedLaunchOptions, onReady?: ( shutdownPromise = (async () => { // Close frontend admission first, then drain binding/startup work. // No root may materialize after cleanup took its snapshot. + logger.debug('[Codex shared] shutdown: closing gateway'); await gateway?.close().catch(() => {}); await startup.catch(() => {}); + logger.debug('[Codex shared] shutdown: draining lifecycle', { pending: operations.size }); while (operations.size) await Promise.allSettled([...operations]); let timer: ReturnType | undefined; try { @@ -133,6 +137,7 @@ export async function runSharedRuntime(options: SharedLaunchOptions, onReady?: ( new Promise(resolve => { timer = setTimeout(resolve, 5_000); }) ]); } finally { clearTimeout(timer); } + logger.debug('[Codex shared] shutdown: stopping engine'); await control.disconnect(); if (server && server.exitCode === null && server.signalCode === null) { if (!await killProcessByChildProcess(server)) throw new Error('Codex engine did not stop; ownership retained for orphan recovery'); @@ -146,9 +151,11 @@ export async function runSharedRuntime(options: SharedLaunchOptions, onReady?: ( failure ??= error; logger.debug('[Codex shared] shutdown', error); }).finally(async () => { + logger.debug('[Codex shared] shutdown: closing roots'); await Promise.allSettled([...prepared].map(root => root.close(ending.has(root)))); await rm(sockets, { recursive: true, force: true }).catch(error => logger.debug('[Codex shared] socket cleanup', error)); finish(); + logger.debug('[Codex shared] shutdown: finished'); }); return shutdownPromise; }; @@ -197,7 +204,7 @@ export async function runSharedRuntime(options: SharedLaunchOptions, onReady?: ( const reserved = runtime.sessions[root.session.sessionId]; if (reserved && reserved.threadId !== threadId) throw new Error('Codex retargeted a reserved thread'); if (!reserved) await reserve(root, threadId); - // No fake user turn/name. Metadata write materializes an empty legacy rollout for native resume. + // Persist native Git context without creating a synthetic user turn. await control.request('thread/metadata/update', { threadId, gitInfo: gitInfo(string(record(response.thread).cwd) ?? root.bootstrap.workingDirectory) }); assertRunning(); roots.set(threadId, root); @@ -212,7 +219,9 @@ export async function runSharedRuntime(options: SharedLaunchOptions, onReady?: ( await notifyRunnerSessionStarted(root.session.sessionId, root.session.getMetadata() ?? root.bootstrap.metadata); // Cold-resumed threads predate the control connection's automatic // new-thread subscription. Subscribe once without changing settings. - await control.request('thread/resume', { threadId }); + // New threads already subscribe all initialized connections. Native + // resume cannot reload an empty 0.159 thread before its first rollout. + if (root.hasNativeHistory()) await control.request('thread/resume', { threadId, excludeTurns: true }); await root.syncHistory(effective); }; const create = (method: 'thread/start' | 'thread/fork', params: Record, parent?: SharedCodexRoot, initialOptions?: SharedLaunchOptions): Promise => operation(async () => { @@ -289,6 +298,8 @@ export async function runSharedRuntime(options: SharedLaunchOptions, onReady?: ( const root = roots.get(string(record(request.params).threadId) ?? ''); const nativeId = string(record(request.params).queuedSubmissionId); if (root && nativeId) await root.nativeQueueDeleted(nativeId); + } else if (!response.error && ['thread/rollback', 'thread/revert'].includes(request.method ?? '')) { + roots.get(string(record(request.params).threadId) ?? '')?.invalidateHistory(); } if (!response.error && ['thread/start', 'thread/resume', 'thread/fork'].includes(request.method ?? '')) { const threadId = string(record(record(response.result).thread).id); @@ -343,7 +354,7 @@ export async function runSharedRuntime(options: SharedLaunchOptions, onReady?: ( let root: SharedCodexRoot; if (options.resumeLast) { const response = record(await control.request('thread/list', { limit: 1, sortKey: 'updated_at', sortDirection: 'desc', - ...(!options.resumeAll ? { cwd: launch.cwd } : {}) })); + useStateDbOnly: true, ...(!options.resumeAll ? { cwd: launch.cwd } : {}) })); options.resumeSessionId = string(record(Array.isArray(response.data) ? response.data[0] : undefined).id); if (!options.resumeSessionId) throw new Error('No Codex session to resume'); } @@ -360,7 +371,9 @@ export async function runSharedRuntime(options: SharedLaunchOptions, onReady?: ( const root = await prepare(launch.cwd, existing); await reserveRecord(root, threadId); return root; }); - const params = root.config({ ...launch.threadParams, threadId }); + // History is synchronized after the runner receives its startup webhook. + // Hydrating it inside native resume can exceed that deadline on large threads. + const params = root.config({ ...launch.threadParams, threadId, excludeTurns: true }); let response: Record; try { response = record(await root.client.request('thread/resume', params)); diff --git a/cli/src/codex/utils/manageThreadGoal.test.ts b/cli/src/codex/utils/manageThreadGoal.test.ts new file mode 100644 index 00000000..28d93ff7 --- /dev/null +++ b/cli/src/codex/utils/manageThreadGoal.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it, vi } from 'vitest'; +import { manageThreadGoal } from './manageThreadGoal'; + +const goal = { threadId: 'root', objective: 'Ship the fix', status: 'paused', tokenBudget: 10000, + tokensUsed: 350, timeUsedSeconds: 12, createdAt: 1, updatedAt: 2 }; + +describe('native Goal controls', () => { + it('edits objective and budget without resuming a paused Goal or starting a turn', async () => { + const request = vi.fn(async (method: string, params: Record) => ({ + goal: method === 'thread/goal/get' ? goal : { ...goal, ...params } + })); + const result = await manageThreadGoal('root', { action: 'update', objective: ' Ship with tests ', tokenBudget: null }, request); + expect(result.goal).toMatchObject({ objective: 'Ship with tests', status: 'paused', tokenBudget: null, tokensUsed: 350 }); + expect(request.mock.calls).toEqual([ + ['thread/goal/get', { threadId: 'root' }], + ['thread/goal/set', { threadId: 'root', origin: 'user', objective: 'Ship with tests', tokenBudget: null }] + ]); + }); + + it.each([['pause', 'paused'], ['resume', 'active']] as const)('uses native status for %s', async (action, status) => { + const request = vi.fn(async (method: string, params: Record) => ({ + goal: method === 'thread/goal/get' ? goal : { ...goal, ...params } + })); + expect((await manageThreadGoal('root', { action }, request)).goal?.status).toBe(status); + expect(request.mock.calls.at(-1)).toEqual(['thread/goal/set', { threadId: 'root', origin: 'user', status }]); + }); + + it('clears only the scoped native Goal and tolerates an already cleared Goal', async () => { + const request = vi.fn(async () => ({ cleared: false })); + expect(await manageThreadGoal('root', { action: 'clear' }, request)).toEqual({ goal: null }); + expect(request.mock.calls).toEqual([['thread/goal/clear', { threadId: 'root', origin: 'user' }]]); + }); + + it('does not create a Goal when editing one that another client removed', async () => { + const request = vi.fn(async () => ({ goal: null })); + await expect(manageThreadGoal('root', { action: 'update', objective: 'Stale edit' }, request)).rejects.toThrow('No Goal'); + expect(request).toHaveBeenCalledTimes(1); + }); + + it('validates actions before contacting Codex and propagates native failures', async () => { + const request = vi.fn(async () => { throw new Error('Native Goal failure'); }); + await expect(manageThreadGoal('root', { action: 'update', objective: ' ' }, request)).rejects.toThrow(); + await expect(manageThreadGoal('root', { action: 'update', objective: 'Fix', tokenBudget: -1 }, request)).rejects.toThrow(); + await expect(manageThreadGoal('root', { action: 'pause', threadId: 'other' }, request)).rejects.toThrow(); + expect(request).not.toHaveBeenCalled(); + await expect(manageThreadGoal('root', { action: 'clear' }, request)).rejects.toThrow('Native Goal failure'); + }); + + it('normalizes snake-case native usage and accepts the Unicode character limit', async () => { + const request = vi.fn(async () => ({ goal: { objective: '目标', status: 'paused', thread_id: 'root', + token_budget: 5000, tokens_used: 20, time_used_seconds: 5, created_at: 1, updated_at: 2 } })); + expect((await manageThreadGoal('root', { action: 'get' }, request)).goal).toMatchObject({ threadId: 'root', tokensUsed: 20, tokenBudget: 5000 }); + await manageThreadGoal('root', { action: 'update', objective: '😀'.repeat(4000) }, request); + await expect(manageThreadGoal('root', { action: 'update', objective: '😀'.repeat(4001) }, request)).rejects.toThrow(); + }); +}); diff --git a/cli/src/codex/utils/manageThreadGoal.ts b/cli/src/codex/utils/manageThreadGoal.ts new file mode 100644 index 00000000..628ab305 --- /dev/null +++ b/cli/src/codex/utils/manageThreadGoal.ts @@ -0,0 +1,39 @@ +import { CodexGoalRequestSchema, type CodexGoalResponse } from '@hapi/protocol/apiTypes'; +import { ThreadGoalSchema } from '@hapi/protocol/schemas'; + +type NativeRequest = (method: string, params: Record) => Promise; + +function readGoal(response: unknown): CodexGoalResponse { + const value = (response as { goal?: unknown } | null)?.goal; + if (value === null) return { goal: null }; + const goal = value as Record | undefined; + return { goal: ThreadGoalSchema.parse({ + ...goal, + threadId: goal?.threadId ?? goal?.thread_id, + tokenBudget: goal?.tokenBudget ?? goal?.token_budget ?? null, + tokensUsed: goal?.tokensUsed ?? goal?.tokens_used ?? 0, + timeUsedSeconds: goal?.timeUsedSeconds ?? goal?.time_used_seconds ?? 0, + createdAt: goal?.createdAt ?? goal?.created_at ?? 0, + updatedAt: goal?.updatedAt ?? goal?.updated_at ?? 0 + }) }; +} + +/** Native Goal controls bypass the chat queue and never interrupt a turn. */ +export async function manageThreadGoal(threadId: string, raw: unknown, request: NativeRequest): Promise { + const action = CodexGoalRequestSchema.parse(raw); + if (action.action === 'get') return readGoal(await request('thread/goal/get', { threadId })); + if (action.action === 'clear') { + await request('thread/goal/clear', { threadId, origin: 'user' }); + return { goal: null }; + } + // Check for a removed Goal before applying an edit. + const current = readGoal(await request('thread/goal/get', { threadId })); + if (!current.goal) throw new Error('No Goal to update. Refresh the session.'); + return readGoal(await request('thread/goal/set', { + threadId, + origin: 'user', + ...(action.action === 'update' + ? { objective: action.objective, ...(action.tokenBudget !== undefined ? { tokenBudget: action.tokenBudget } : {}) } + : { status: action.action === 'pause' ? 'paused' : 'active' }) + })); +} diff --git a/cli/src/modules/pingPeer/inspectPeer.test.ts b/cli/src/modules/pingPeer/inspectPeer.test.ts index 2745f14c..522e67a3 100644 --- a/cli/src/modules/pingPeer/inspectPeer.test.ts +++ b/cli/src/modules/pingPeer/inspectPeer.test.ts @@ -137,6 +137,7 @@ describe('inspectPeer', () => { text: 'status on runner versions?' }) expect(result.messages[1]?.text).toContain('Looking into it.') + expect(http.get.mock.calls.some(([url]) => String(url).endsWith('/api/sessions'))).toBe(false) // Read-only: never resume expect(http.post).toHaveBeenCalledTimes(1) diff --git a/cli/src/modules/pingPeer/pingPeer.test.ts b/cli/src/modules/pingPeer/pingPeer.test.ts index de317a71..ef13613b 100644 --- a/cli/src/modules/pingPeer/pingPeer.test.ts +++ b/cli/src/modules/pingPeer/pingPeer.test.ts @@ -116,7 +116,7 @@ describe('pingPeer', () => { }) const result = await pingPeer({ - sessionIdPrefix: '05d9f0f2', + sessionIdPrefix: sessionId, message: 'hello peer', accessToken: 'tok', apiUrl: 'http://127.0.0.1:3006', @@ -128,6 +128,7 @@ describe('pingPeer', () => { name: 'Orchestrator', resumed: false }) + expect(http.get.mock.calls.some(([url]) => String(url).endsWith('/api/sessions'))).toBe(false) expect(http.post).toHaveBeenCalledTimes(2) }) diff --git a/cli/src/modules/pingPeer/pingPeer.ts b/cli/src/modules/pingPeer/pingPeer.ts index 75ada864..c99aa1ea 100644 --- a/cli/src/modules/pingPeer/pingPeer.ts +++ b/cli/src/modules/pingPeer/pingPeer.ts @@ -80,11 +80,16 @@ export type ListPeerSessionsOptions = { const DEFAULT_WAIT_ACTIVE_SECS = 60 const POLL_ACTIVE_MS = 2_000 const POLL_PI_READY_MS = 1_000 +const FULL_SESSION_ID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i function defaultSleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)) } +function isFullSessionId(value: string): boolean { + return FULL_SESSION_ID_RE.test(value) +} + const AUTH_RECOVERY_HINT = 'On a remote runner, set HAPI_API_URL to the runner hub, and set CLI_API_TOKEN ' + 'or run `hapi auth login` to save the token. Inside a HAPI session prefer MCP ' + @@ -484,8 +489,12 @@ export async function pingPeer(options: PingPeerOptions): Promise 0 - ? envWebhookTimeout - : 15_000; // Session spawning awaiter system const pidToAwaiter = new Map void>(); @@ -500,6 +496,9 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): const { directory, sessionId, machineId, approvedNewDirectoryCreation = true } = options; const agent = options.agent ?? 'claude'; + const webhookTimeoutMs = Number.isFinite(envWebhookTimeout) && envWebhookTimeout > 0 + ? envWebhookTimeout + : sessionStartupTimeoutMs(agent, options.resumeSessionId); if (agent === 'gemini') { throw new Error('Gemini CLI is no longer supported and cannot be launched (Google sunset the consumer Gemini CLI on 2026-06-18). Existing Gemini sessions remain viewable in the web UI.'); } @@ -815,7 +814,7 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}): logger.debug(`[RUNNER RUN] Waiting for session webhook for PID ${pid}`); const spawnResult = await new Promise((resolve) => { - // Set timeout for webhook. Default is 15s but can be raised via + // Cold Codex resumes get 120s; other launches get 15s. Override via // HAPI_RUNNER_WEBHOOK_TIMEOUT_MS for users on slow models // (e.g. opus[1m] --resume). const timeout = setTimeout(() => { diff --git a/cli/src/test/auditTestProcesses.ts b/cli/src/test/auditTestProcesses.ts index a0aaa497..dbc7862a 100644 --- a/cli/src/test/auditTestProcesses.ts +++ b/cli/src/test/auditTestProcesses.ts @@ -38,11 +38,11 @@ export function findTestOwnedProcesses(marker: string): TestOwnedProcess[] { let output: string try { - // `-eo` (not `-axo`): procps-ng 4.x rejects `-x` with "must set - // personality" on some Linux builds. `e` shows the environment after - // the command; `ww` removes width truncation so the env dump is not - // cut off. - output = execFileSync('ps', ['eww', '-eo', 'pid=,ppid=,rss=,command='], { + // BSD ps needs -ax to include detached processes; `eww -eo` can exit + // silently on macOS. Linux procps-ng needs -e instead of -ax. + // `eww` includes the complete environment for marker matching only. + const selection = process.platform === 'darwin' ? '-axo' : '-eo' + output = execFileSync('ps', ['eww', selection, 'pid=,ppid=,rss=,command='], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'], diff --git a/docs/local-deployment.md b/docs/local-deployment.md index 512c46cb..2dbf92a5 100644 --- a/docs/local-deployment.md +++ b/docs/local-deployment.md @@ -52,6 +52,18 @@ Remote Macs with the same layout: scripts/deploy-remote.sh [tag] # e.g. scripts/deploy-remote.sh k2lab card-dedupe ``` +Both deploy scripts verify that the compiled artifact contains durable Codex +history synchronization with a separate invalidation epoch, confirmed transcript delivery, and persistent usage +storage before signing or installation. A matching version number is not +enough: local builds can all report the same version while containing different +fixes. Build from the complete deployment workspace. If using an isolated +checkout, include all local changes and untracked source files; a checkout of +tracked files plus one task's patch can silently remove another task's fixes. +Use the current deployment scripts, not an older copy from that checkout. +Remote transfer uses rsync with a staged copy of the installed binary when +both Macs have rsync; unchanged blocks are reused. Otherwise it uses scp. +This avoids a global `PATH` override for a temporary transfer wrapper. + ### Fixed install path (why) The binary always installs to the **fixed path** `~/.hapi/bin/hapi` (a real @@ -194,6 +206,26 @@ running; the CLI stops the stale runner and starts a fresh one with `HAPI_CLI_EXECUTABLE` pinned to the fixed path. Running sessions are detached and survive the restart. +The refreshed runner inherits the deployment shell's environment, including +`PATH`. Deploy from a normal login Terminal without replacing `PATH` with a +short list of system directories: agents installed under `~/.local/bin`, +`~/.opencode/bin`, `~/.grok/bin`, or nvm otherwise disappear from availability +checks and model discovery fails with `spawn codex ENOENT`. Temporary `scp` or +`rsync` wrappers must prepend to the existing `PATH` and apply only to the +remote transfer command, not the local deployment or runner refresh. + +After refreshing, verify the authenticated Hub APIs +`GET /api/machines//agent-availability` and +`GET /api/machines//codex-models`. Compare availability with the +installed agents before deployment, and require a successful Codex model +response where Codex is installed. `/health`, binary version, signature, and +runner PID alone do not validate the runner's launch environment. If it was +replaced, restore it from a normal login Terminal, then repeat the API checks: + +```bash +HAPI_CLI_EXECUTABLE="$HOME/.hapi/bin/hapi" "$HOME/.hapi/bin/hapi" runner start +``` + ### Manual sequence ```bash diff --git a/hub/README.md b/hub/README.md index 1a591166..d494470e 100644 --- a/hub/README.md +++ b/hub/README.md @@ -104,6 +104,8 @@ for request/response shapes and error semantics, and `src/web/routes/` for all e - `POST /api/sessions/:id/resume` - Resume inactive session. - `POST /api/sessions/:id/reopen` - Reopen a session; follow the returned session ID. - `POST /api/sessions/:id/clear` - Start a fresh conversation when supported. +- `GET /api/sessions/:id/codex/goal` - Read the native Codex Goal. Requires an active owner session with `capabilities.codexGoal`. +- `POST /api/sessions/:id/codex/goal` - Manage that Goal: `{ action: "update", objective, tokenBudget? }`, `{ action: "pause" }`, `{ action: "resume" }`, or `{ action: "clear" }`. Returns `{ goal }` (`null` after clear); bypasses the chat queue. A null token budget uses Codex's default limits. - `POST /api/sessions/:id/upload` - Upload file (base64, max 50MB). - `POST /api/sessions/:id/upload/delete` - Delete uploaded file. - `POST /api/sessions/:id/archive` - Archive active session. diff --git a/hub/scripts/restore-usage.ts b/hub/scripts/restore-usage.ts new file mode 100644 index 00000000..7f290783 --- /dev/null +++ b/hub/scripts/restore-usage.ts @@ -0,0 +1,176 @@ +import { Database } from 'bun:sqlite' +import { readFileSync, statSync, writeFileSync } from 'node:fs' +import { parseArgs } from 'node:util' +import { z } from 'zod' +import { Store } from '../src/store' +import { decodeMessageContent } from '../src/store/contentCodec' +import { getUsageEventsByNamespace } from '../src/store/usage' +import type { UsageEvent, ReconciledUsageRow } from '../src/store/usage' +import { parseUsageEvent } from '../src/store/usageParser' +import type { UsageSource } from '../src/store/usageSources' +import { mergeRecoveredUsage } from '../src/sync/usageRecovery' +import { getUsageSummary } from '../src/sync/usageService' + +// Operates only on accounting tables. Supply a SQLite backup for rehearsal; +// an explicit --apply is required for every run, including rehearsals. +const options = parseArgs({ options: { + db: { type: 'string' }, input: { type: 'string', multiple: true }, + backup: { type: 'string', multiple: true }, report: { type: 'string' }, + namespace: { type: 'string', default: 'default' }, apply: { type: 'boolean', default: false } +} }).values +if (!options.db || !options.report || !options.apply) { + throw new Error('Required: --db --input --report --apply; rehearse on a backup first') +} +const namespace = options.namespace +const base = { machineId: z.string().min(1), agent: z.enum(['codex', 'claude', 'dsh', 'opencode']), nativeSessionId: z.string().min(1) } +const count = z.number().int().nonnegative() +const schema = z.discriminatedUnion('type', [ + z.object({ type: z.literal('binding'), ...base, sessionId: z.string().min(1), namespace: z.string().min(1), + createdAt: count.nullish(), model: z.string().nullish(), nativeHapi: z.boolean().optional() }), + z.object({ type: z.literal('event'), ...base, createdAt: count, content: z.unknown(), nativeHapi: z.boolean().optional() }), + z.object({ type: z.literal('counter'), ...base, createdAt: count, sourceKey: z.string().min(1), model: z.string().nullish(), + inputTokens: count, outputTokens: count, cacheReadTokens: count, cacheCreationTokens: count }) +]) +type NativeRow = z.infer +type Binding = Extract +function sourceKey(row: { machineId: string; agent: string; nativeSessionId: string }): string { + return JSON.stringify([row.machineId, row.agent, row.nativeSessionId]) +} +const inputs: NativeRow[] = [] +const collectedAt = new Map() +for (const path of options.input ?? []) { + const capturedAt = Math.floor(statSync(path).mtimeMs) + for (const line of readFileSync(path, 'utf8').split('\n')) { + if (!line.trim()) continue + const row = schema.parse(JSON.parse(line)) + inputs.push(row) + collectedAt.set(sourceKey(row), Math.max(collectedAt.get(sourceKey(row)) ?? 0, capturedAt)) + } +} +const store = new Store(options.db) +const before = getUsageSummary(store, namespace, 'all', 'Asia/Shanghai') +const bindings = new Map() +for (const row of inputs) { + if (row.type !== 'binding' || row.namespace !== namespace) continue + const key = sourceKey(row) + const values = bindings.get(key) ?? [] + if (!values.some((value) => value.sessionId === row.sessionId)) values.push(row) + bindings.set(key, values) + store.usage.rememberSource({ ...row, namespace }) +} +let repaired = store.usage.repairTransferredEvents(namespace) +const stages: Array<{ source: string; inserted: number; matched: number; enriched: number }> = [] + +// Old snapshots did not necessarily index all messages. Re-parse retained +// messages so cache-exclusive provider formats use today's normalized input. +for (const path of options.backup ?? []) { + console.error(`Recovering backup: ${path}`) + const db = new Database(path, { readonly: true }) + const sessions = new Map() + for (const row of db.prepare('SELECT id, metadata, model FROM sessions WHERE namespace = ?').all(namespace) as Array<{ id: string; metadata: string | null; model: string | null }>) { + sessions.set(row.id, { ...row, metadata: row.metadata ? JSON.parse(row.metadata) as unknown : null }) + } + const events = new Map() + type BackupMessage = { id: string; session_id: string; seq: number; created_at: number; local_id: string | null; content: string | Uint8Array | null } + for (const row of db.prepare(`SELECT id, session_id, seq, created_at, local_id, content FROM messages + WHERE session_id IN (SELECT id FROM sessions WHERE namespace = ?) ORDER BY created_at, seq`).iterate(namespace) as Iterable) { + const session = sessions.get(row.session_id) + if (!session) continue + const event = parseUsageEvent(session, { id: row.id, seq: row.seq, createdAt: row.created_at, + localId: row.local_id, content: decodeMessageContent(row.content) }) + if (!event) continue + event.model ??= session.model + const key = `${event.agent}|${event.sourceKey}` + const prior = events.get(key) + if (!prior || event.kind === 'delta') events.set(key, event) + } + const hasNamespace = (db.prepare('PRAGMA table_info(usage_events)').all() as Array<{ name: string }>).some((row) => row.name === 'namespace') + // Namespace-less legacy ledgers were single-user. Read them via a TEMP + // view on this readonly connection, never migrate the backup itself. + if (!hasNamespace) { + db.exec(`CREATE TEMP VIEW usage_events_legacy AS SELECT '${namespace.replaceAll("'", "''")}' AS namespace, * FROM main.usage_events; + CREATE TEMP VIEW usage_events AS SELECT * FROM usage_events_legacy`) + } + for (const event of getUsageEventsByNamespace(db, namespace)) { + const key = `${event.agent}|${event.sourceKey}` + if (!events.has(key)) events.set(key, event) + } + stages.push({ source: path, ...mergeRecoveredUsage(store, namespace, [...events.values()]) }) + db.close() +} + +const existing = store.usage.getEventsByNamespace(namespace) +const streams = new Map() +for (const event of existing) { + if (event.agent !== 'codex' || event.kind !== 'cumulative') continue + const native = event.sourceKey.split('|')[1]! + const values = streams.get(native) ?? [] + values.push(event) + streams.set(native, values) +} +const nativeEvents = new Map() +const authoritativeModels = new Set() +const snapshots = new Map; seen: Set }>() +const date = new Intl.DateTimeFormat('en-CA', { timeZone: 'Asia/Shanghai', year: 'numeric', month: '2-digit', day: '2-digit' }) +let sequence = 0 +let unbound = 0 +for (const row of inputs) { + if (row.type === 'binding') continue + const aliases = bindings.get(sourceKey(row)) + if (!aliases?.length) { unbound += 1; continue } + const ordered = [...aliases].sort((a, b) => (a.createdAt ?? 0) - (b.createdAt ?? 0)) + const binding = ordered.findLast((value) => (value.createdAt ?? 0) <= row.createdAt) ?? ordered[0]! + let sessionId = binding.sessionId + if (row.agent === 'codex') { + const stream = streams.get(row.nativeSessionId) + // Preserve the ledger's session boundaries across resumes/merges. + const prior = stream?.findLast((event) => event.createdAt <= row.createdAt) + sessionId = prior?.sessionId ?? stream?.[0]?.sessionId ?? sessionId + } + if (row.type === 'counter' && row.agent === 'opencode') { + const key = sourceKey(row) + const value = snapshots.get(key) ?? { source: { ...binding, namespace }, rows: new Map(), seen: new Set() } + if (value.seen.has(row.sourceKey)) continue + value.seen.add(row.sourceKey) + const day = date.format(new Date(row.createdAt)) + const model = row.model ?? binding.model ?? 'unknown' + const bucketKey = `${day}|${model}` + const bucket = value.rows.get(bucketKey) ?? { sessionId, day, model, agent: 'opencode', inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0, requests: 0 } + bucket.inputTokens += row.inputTokens + bucket.outputTokens += row.outputTokens + bucket.cacheReadTokens += row.cacheReadTokens + bucket.cacheCreationTokens += row.cacheCreationTokens + bucket.requests += 1 + value.rows.set(bucketKey, bucket) + snapshots.set(key, value) + continue + } + const event: UsageEvent | null = row.type === 'event' + ? parseUsageEvent({ id: sessionId, metadata: { flavor: row.agent } }, { + id: `native-${++sequence}`, seq: sequence, createdAt: row.createdAt, localId: null, content: row.content }) + : { sessionId, sourceKey: `native|${row.machineId}|${row.agent}|${row.nativeSessionId}|${row.sourceKey}`, + sourceSeq: ++sequence, createdAt: row.createdAt, agent: row.agent, model: row.model ?? null, kind: 'delta', + inputTokens: row.inputTokens, outputTokens: row.outputTokens, cacheReadTokens: row.cacheReadTokens, + cacheCreationTokens: row.cacheCreationTokens, lastInputTokens: null, lastOutputTokens: null, + lastCacheReadTokens: null, lastCacheCreationTokens: null } + if (!event) continue + const hasNativeModel = event.model !== null + event.model ??= binding.model ?? null + const key = `${event.agent}|${event.sourceKey}` + if (hasNativeModel) authoritativeModels.add(key) + const prior = nativeEvents.get(key) + if (!prior || (event.kind === 'delta' && event.outputTokens >= prior.outputTokens)) nativeEvents.set(key, event) +} +stages.push({ source: 'native histories', ...mergeRecoveredUsage(store, namespace, [...nativeEvents.values()], authoritativeModels) }) +for (const [key, snapshot] of snapshots) store.usage.reconcileSource(snapshot.source, [...snapshot.rows.values()], collectedAt.get(key)!) +repaired += store.usage.repairTransferredEvents(namespace) +const after = getUsageSummary(store, namespace, 'all', 'Asia/Shanghai') +const report = { database: options.db, namespace, repairedTransferredEvents: repaired, stages, nativeSources: bindings.size, + nativeSnapshots: snapshots.size, unboundRows: unbound, before, after, + addedTokens: after.totals.totalTokens - before.totals.totalTokens, + addedRequests: after.totals.requests - before.totals.requests, + generatedAt: new Date().toISOString() } +writeFileSync(options.report, `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 }) +store.close() +console.log(JSON.stringify({ repaired, stages, nativeSources: bindings.size, nativeSnapshots: snapshots.size, unbound, + before: before.totals, after: after.totals, addedTokens: report.addedTokens })) diff --git a/hub/src/socket/handlers/cli/machineHandlers.opencodeUsage.test.ts b/hub/src/socket/handlers/cli/machineHandlers.opencodeUsage.test.ts index 93cccd9c..d0eb8684 100644 --- a/hub/src/socket/handlers/cli/machineHandlers.opencodeUsage.test.ts +++ b/hub/src/socket/handlers/cli/machineHandlers.opencodeUsage.test.ts @@ -3,6 +3,7 @@ import { EventEmitter } from 'node:events' import { Store, type StoredMachine } from '../../../store' import type { CliSocketWithData } from '../../socketTypes' import { registerMachineHandlers } from './machineHandlers' +import { getUsageSummary } from '../../../sync/usageService' function harness(options: { access: 'ok' | 'denied' }) { const socket = new EventEmitter() as unknown as CliSocketWithData @@ -48,8 +49,31 @@ describe('opencode-usage-report', () => { }) expect(store.usage.getReconciledByNamespace('default')).toEqual([ - expect.objectContaining({ sessionId: session.id, day: '2026-09-17', requests: 2 }) + expect.objectContaining({ day: '2026-09-17', requests: 2 }) ]) + expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(2) + expect(store.usage.getSources('default', 'machine-1').some((source) => source.sessionId === session.id)).toBe(true) + }) + + it('updates deleted sources and keeps different machines isolated', () => { + const { socket, store } = harness({ access: 'ok' }) + const session = store.sessions.getOrCreateSession('deleted-source', { + machineId: 'machine-1', flavor: 'opencode', opencodeSessionId: 'ses_deleted' + }, null, 'default') + store.sessions.deleteSession(session.id, 'default') + socket.emit('opencode-usage-report', { + machineId: 'machine-2', sessions: [{ opencodeSessionId: 'ses_deleted', rows: [row] }] + }) + expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(0) + socket.emit('opencode-usage-report', { + machineId: 'machine-1', sessions: [{ opencodeSessionId: 'ses_deleted', rows: [row] }] + }) + expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(2) + socket.emit('opencode-usage-report', { + machineId: 'machine-1', sessions: [{ opencodeSessionId: 'ses_deleted', rows: [{ ...row, requests: 3 }] }] + }) + expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(3) + store.close() }) it('ignores reports for machines without access', () => { diff --git a/hub/src/socket/handlers/cli/machineHandlers.ts b/hub/src/socket/handlers/cli/machineHandlers.ts index 8d8a38dc..e65525b7 100644 --- a/hub/src/socket/handlers/cli/machineHandlers.ts +++ b/hub/src/socket/handlers/cli/machineHandlers.ts @@ -205,7 +205,16 @@ export function registerMachineHandlers(socket: CliSocketWithData, deps: Machine const opencodeSessionId = metadata && typeof metadata.opencodeSessionId === 'string' ? metadata.opencodeSessionId : null - if (opencodeSessionId) sessionIdByOpencodeId.set(opencodeSessionId, session.id) + if (typeof metadata?.machineId === 'string' && metadata.machineId !== machineId) continue + if (opencodeSessionId) { + store.usage.rememberSource({ namespace, machineId, agent: 'opencode', nativeSessionId: opencodeSessionId, sessionId: session.id }) + sessionIdByOpencodeId.set(opencodeSessionId, session.id) + } + } + for (const source of store.usage.getSources(namespace, machineId)) { + if (source.agent === 'opencode' && !sessionIdByOpencodeId.has(source.nativeSessionId)) { + sessionIdByOpencodeId.set(source.nativeSessionId, source.sessionId) + } } const now = Date.now() @@ -213,10 +222,9 @@ export function registerMachineHandlers(socket: CliSocketWithData, deps: Machine if (entry.rows.length === 0) continue const sessionId = sessionIdByOpencodeId.get(entry.opencodeSessionId) if (!sessionId) continue - store.usage.replaceReconciled( - sessionId, - namespace, - entry.rows.map((row) => ({ ...row, sessionId, agent: 'opencode' as const })), + store.usage.reconcileSource( + { namespace, machineId, agent: 'opencode', nativeSessionId: entry.opencodeSessionId, sessionId }, + entry.rows.map((row) => ({ ...row, agent: 'opencode' as const })), now ) } diff --git a/hub/src/socket/handlers/cli/sessionHandlers.test.ts b/hub/src/socket/handlers/cli/sessionHandlers.test.ts index cec001db..50de01f3 100644 --- a/hub/src/socket/handlers/cli/sessionHandlers.test.ts +++ b/hub/src/socket/handlers/cli/sessionHandlers.test.ts @@ -53,6 +53,67 @@ function reasoningTextOf(message: { content: unknown }): string { } describe('cli session handlers', () => { + it('keeps tail previews newer than subsequently backfilled history after consumption ACKs', () => { + const store = new Store(':memory:') + const session = store.sessions.getOrCreateSession('history-order', {}, null, 'default') + const socket = new FakeSocket() + registerSessionHandlers(socket as unknown as CliSocketWithData, { + store, resolveSessionAccess: () => ({ ok: true, value: session }), emitAccessError() {} + }) + socket.trigger('message', { sid: session.id, localId: 'latest', createdAt: 3000, message: reasoningContent('latest', 'latest result', false) }) + socket.trigger('message', { sid: session.id, localId: 'older', createdAt: 2000, message: reasoningContent('older', 'old result', false) }) + store.messages.markMessagesInvoked(session.id, ['latest', 'older'], Date.now()) + const rows = store.messages.getMessagesByPosition(session.id, 20) + expect(rows.map(row => row.localId)).toEqual(['older', 'latest']) + expect(rows.map(row => row.invokedAt)).toEqual([2000, 3000]) + store.close() + }) + it('isolates history progress by session access and validates the native checkpoint', () => { + const store = new Store(':memory:') + const session = store.sessions.getOrCreateSession('allowed', {}, null, 'default') + const socket = new FakeSocket(); const ack = mock() + registerSessionHandlers(socket as unknown as CliSocketWithData, { + store, resolveSessionAccess: sid => sid === session.id ? { ok: true, value: session } : { ok: false, reason: 'access-denied' }, emitAccessError() {} + }) + socket.trigger('codex-history-sync', { sid: session.id, threadId: 'thread' }, ack) + expect(ack.mock.lastCall?.[0]).toEqual({ ok: true, state: { epoch: 0, revision: 0, checkpoint: null } }) + socket.trigger('codex-history-sync', { sid: 'other', threadId: 'thread' }, ack) + expect(ack.mock.lastCall?.[0]).toEqual({ ok: false }) + socket.trigger('codex-history-sync', { sid: session.id, threadId: 'thread', commit: { epoch: 0, revision: 0, checkpoint: { version: 99 }, localIds: [] } }, ack) + expect(ack.mock.lastCall?.[0]).toEqual({ ok: false }) + store.close() + }) + it('confirms persisted transcript messages and stable-ID retries without duplicating rows', () => { + const store = new Store(':memory:') + const session = store.sessions.getOrCreateSession('transcript-ack', {}, null, 'default') + const socket = new FakeSocket() + registerSessionHandlers(socket as unknown as CliSocketWithData, { + store, resolveSessionAccess: () => ({ ok: true, value: session }), emitAccessError() {} + }) + const payload = { sid: session.id, localId: 'stable', message: reasoningContent('item', 'settled', false) } + const ack = mock((response: unknown) => { + expect(response).toEqual({ ok: true }) + expect(store.messages.getAllMessages(session.id)).toHaveLength(1) + }) + socket.trigger('message', payload, ack) + socket.trigger('message', payload, ack) + expect(ack).toHaveBeenCalledTimes(2) + store.close() + }) + + it('rejects malformed or unauthorized transcript messages without a success ACK', () => { + const store = new Store(':memory:') + const socket = new FakeSocket() + registerSessionHandlers(socket as unknown as CliSocketWithData, { + store, resolveSessionAccess: () => ({ ok: false, reason: 'access-denied' }), emitAccessError() {} + }) + const ack = mock() + socket.trigger('message', {}, ack) + socket.trigger('message', { sid: 'other', message: {} }, ack) + expect(ack.mock.calls).toEqual([[{ ok: false }], [{ ok: false }]]) + store.close() + }) + it.each([undefined, 'terminated', 'error'] as const)('preserves shared Codex pending input on execution exit (%s)', reason => { const store = new Store(':memory:') const session = store.sessions.getOrCreateSession('shared-end', { flavor: 'codex', capabilities: { concurrentClients: true } }, null, 'default') diff --git a/hub/src/socket/handlers/cli/sessionHandlers.ts b/hub/src/socket/handlers/cli/sessionHandlers.ts index 59aa2ea4..f87324cb 100644 --- a/hub/src/socket/handlers/cli/sessionHandlers.ts +++ b/hub/src/socket/handlers/cli/sessionHandlers.ts @@ -1,4 +1,4 @@ -import type { ClientToServerEvents } from '@hapi/protocol' +import { CodexHistorySyncRequestSchema, type ClientToServerEvents } from '@hapi/protocol' import { z } from 'zod' import { randomUUID } from 'node:crypto' import type { CopilotAgentMode } from '@hapi/protocol' @@ -51,8 +51,7 @@ const messageSchema = z.object({ message: z.union([z.string(), z.unknown()]), localId: z.string().optional(), // Client-provided origin timestamp (epoch ms) — e.g. a Claude transcript - // entry's own `timestamp`. Only honored for agent messages (no localId); - // see addMessage in messages.ts. + // entry's own `timestamp`, or a Codex history item's origin time. createdAt: z.number().optional() }) @@ -103,6 +102,17 @@ export type SessionHandlersDeps = { export function registerSessionHandlers(socket: CliSocketWithData, deps: SessionHandlersDeps): void { const { store, resolveSessionAccess, emitAccessError, onSessionAlive, onSessionReady, onSessionEnd, onWebappEvent, onBackgroundTaskDelta, onSessionActivity, onSweepImmediateQueued, onMessagesConsumed } = deps + socket.on('codex-history-sync', (data, ack) => { + if (typeof ack !== 'function') return + const parsed = CodexHistorySyncRequestSchema.safeParse(data) + if (!parsed.success) { ack({ ok: false }); return } + const { sid, threadId, commit } = parsed.data + const access = resolveSessionAccess(sid) + if (!access.ok) { emitAccessError('session', sid, access.reason); ack({ ok: false }); return } + const state = commit ? store.historySync.commit(sid, threadId, commit) : store.historySync.get(sid, threadId) + ack(state ? { ok: true, state } : { ok: false }) + }) + socket.on('native-queue-message', data => { const parsed = z.object({ sid: z.string(), localId: z.string().min(1), text: z.string().nullable() }).safeParse(data) if (!parsed.success) return @@ -122,9 +132,10 @@ export function registerSessionHandlers(socket: CliSocketWithData, deps: Session } }) - socket.on('message', (data: unknown) => { + socket.on('message', (data: unknown, ack?: (response: { ok: boolean }) => void) => { const parsed = messageSchema.safeParse(data) if (!parsed.success) { + ack?.({ ok: false }) return } @@ -144,15 +155,24 @@ export function registerSessionHandlers(socket: CliSocketWithData, deps: Session const sessionAccess = resolveSessionAccess(sid) if (!sessionAccess.ok) { emitAccessError('session', sid, sessionAccess.reason) + ack?.({ ok: false }) return } const session = sessionAccess.value if (isRedundantGoalStatusEventContent(content)) { + ack?.({ ok: true }) return } - const msg = store.messages.addMessage(sid, content, localId, undefined, createdAt) + let msg = store.messages.addMessage(sid, content, localId, undefined, createdAt) + if (localId && createdAt !== undefined) { + // Native backfill may arrive after its latest tail preview. Keep + // historical positions at their origin time, including the later + // messages-consumed ACK (which is guarded by first-write-wins). + store.messages.markMessagesInvoked(sid, [localId], Math.min(createdAt, Date.now())) + msg = store.messages.addMessage(sid, content, localId) + } // A reasoning stream arrives as a series of growing snapshots under one // stable id, so a stream should cost one row rather than one per @@ -249,6 +269,8 @@ export function registerSessionHandlers(socket: CliSocketWithData, deps: Session invokedAt: msg.invokedAt } }) + // ACK only after persistence and publication, including stable-ID replays. + ack?.({ ok: true }) }) const handleUpdateMetadata: UpdateMetadataHandler = (data, cb) => { diff --git a/hub/src/store/contentCodec.ts b/hub/src/store/contentCodec.ts index cb6b2d8e..df6c6b90 100644 --- a/hub/src/store/contentCodec.ts +++ b/hub/src/store/contentCodec.ts @@ -5,65 +5,8 @@ import { zstdCompressSync, zstdDecompressSync } from 'node:zlib' * plaintext rows stay grep-able when inspecting the DB by hand. */ export const COMPRESS_MIN_CHARS = 256 -/** Per-string cap inside agent message content. Tool results above this are - * head+tail truncated at ingest — a single file read or build log can otherwise - * persist megabytes that no client ever renders in full. */ -export const TRUNCATE_STRING_LIMIT = 64 * 1024 -const TRUNCATE_HEAD = 48 * 1024 -const TRUNCATE_TAIL = 12 * 1024 - -function truncateString(value: string): string { - const removed = value.length - TRUNCATE_HEAD - TRUNCATE_TAIL - // head + tail + marker stays well under TRUNCATE_STRING_LIMIT so the - // function is idempotent — codexDesktop compares stored rows against - // freshly-normalized transcripts and must get identical strings on both sides. - return `${value.slice(0, TRUNCATE_HEAD)}\n…[hapi: truncated ${removed} chars]…\n${value.slice(value.length - TRUNCATE_TAIL)}` -} - -/** Returns the same reference when nothing was truncated so callers can use - * identity to detect change. */ -function truncateDeep(value: unknown): unknown { - if (typeof value === 'string') { - return value.length > TRUNCATE_STRING_LIMIT ? truncateString(value) : value - } - if (Array.isArray(value)) { - let copy: unknown[] | null = null - for (let i = 0; i < value.length; i++) { - const item = truncateDeep(value[i]) - if (item !== value[i]) { - copy ??= value.slice() - copy[i] = item - } - } - return copy ?? value - } - if (value !== null && typeof value === 'object') { - const record = value as Record - let copy: Record | null = null - for (const key of Object.keys(record)) { - const item = truncateDeep(record[key]) - if (item !== record[key]) { - copy ??= { ...record } - copy[key] = item - } - } - return copy ?? value - } - return value -} - -/** Truncate oversized strings inside agent-role message content. - * - * Only role === 'agent' envelopes are touched: user messages are queued in the - * DB and later *delivered* to the CLI verbatim (getMatureScheduledMessages / - * getDeliverableMessagesAfter), so truncating them would corrupt real prompts, - * not just display history. Idempotent: re-applying to already-truncated - * content returns the same reference. */ -export function truncateOversizedMessageContent(content: unknown): unknown { - if (content === null || typeof content !== 'object') return content - if ((content as Record).role !== 'agent') return content - return truncateDeep(content) -} +export { TRUNCATE_STRING_LIMIT, truncateOversizedMessageContent } from '@hapi/protocol' +import { truncateOversizedMessageContent } from '@hapi/protocol' /** Compress a JSON string for the messages.content column. * Storage contract: TEXT value = plaintext JSON, BLOB value = zstd(JSON). */ diff --git a/hub/src/store/historySyncStore.test.ts b/hub/src/store/historySyncStore.test.ts new file mode 100644 index 00000000..4e558be5 --- /dev/null +++ b/hub/src/store/historySyncStore.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'bun:test' +import { Database } from 'bun:sqlite' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { CodexHistoryCheckpoint } from '@hapi/protocol' +import { Store } from './index' + +const checkpoint: CodexHistoryCheckpoint = { version: 1, pageCursor: null, turnId: 'turn', itemId: 'item', turnComplete: true } + +describe('durable Codex history progress', () => { + it('survives Hub reopening and invalidates progress when messages are replaced', () => { + const dir = mkdtempSync(join(tmpdir(), 'hapi-history-store-')) + const path = join(dir, 'hub.db') + let store = new Store(path) + try { + const sid = store.sessions.getOrCreateSession('history', {}, null, 'default').id + store.messages.addMessage(sid, { text: 'saved' }, 'output') + expect(store.historySync.commit(sid, 'thread', { epoch: 0, revision: 0, checkpoint, localIds: ['output'] })) + .toEqual({ epoch: 0, revision: 1, checkpoint }) + store.close(); store = new Store(path) + expect(store.historySync.get(sid, 'thread')).toEqual({ epoch: 0, revision: 1, checkpoint }) + expect(store.historySync.get(sid, 'child').checkpoint).toBeNull() + store.messages.bumpMessageEpoch(sid) + expect(store.historySync.get(sid, 'thread')).toEqual({ epoch: 1, revision: 1, checkpoint: null }) + expect(store.historySync.commit(sid, 'thread', { epoch: 0, revision: 1, checkpoint, localIds: [] })).toBeNull() + } finally { store.close(); rmSync(dir, { recursive: true, force: true }) } + }) + it('refuses missing outputs and stale writers instead of advancing across a gap', () => { + const store = new Store(':memory:') + try { + const sid = store.sessions.getOrCreateSession('history', {}, null, 'default').id + store.messages.addMessage(sid, { text: 'first output' }, 'first') + const commit = { epoch: 0, revision: 0, checkpoint, localIds: ['first', 'second'] } + expect(store.historySync.commit(sid, 'thread', commit)).toBeNull() + expect(store.historySync.get(sid, 'thread').checkpoint).toBeNull() + store.messages.addMessage(sid, { text: 'second output' }, 'second') + expect(store.historySync.commit(sid, 'thread', commit)?.revision).toBe(1) + expect(store.historySync.commit(sid, 'thread', { ...commit, checkpoint: null })).toBeNull() + expect(store.historySync.get(sid, 'thread').checkpoint).toEqual(checkpoint) + } finally { store.close() } + }) + it('keeps the native checkpoint when tail preview and backfill change display pagination', () => { + const store = new Store(':memory:') + try { + const sid = store.sessions.getOrCreateSession('backfill', {}, null, 'default').id + store.messages.addMessage(sid, { text: 'latest preview' }, 'latest', null, 3_000) + store.historySync.commit(sid, 'thread', { epoch: 0, revision: 0, checkpoint, localIds: ['latest'] }) + const before = store.historySync.get(sid, 'thread') + store.messages.addMessage(sid, { text: 'earlier item' }, 'earlier', null, 1_000) + store.messages.addImportedMessage(sid, { text: 'earlier imported item' }, 'imported', 2_000) + expect(store.messages.getMessageEpoch(sid)).toBe(2) + expect(store.historySync.get(sid, 'thread')).toEqual(before) + expect(store.historySync.commit(sid, 'thread', { ...before, localIds: ['earlier', 'imported'] })?.revision).toBe(2) + store.messages.truncateMessagesFromLocalId(sid, 'earlier', []) + expect(store.historySync.get(sid, 'thread').checkpoint).toBeNull() + expect(store.historySync.commit(sid, 'thread', { ...before, localIds: [] })).toBeNull() + } finally { store.close() } + }) + it('does not adopt the display epoch again after a Hub restart', () => { + const dir = mkdtempSync(join(tmpdir(), 'hapi-history-backfill-')) + const path = join(dir, 'hub.db') + let store = new Store(path) + try { + const sid = store.sessions.getOrCreateSession('history', {}, null, 'default').id + store.messages.addMessage(sid, { text: 'latest' }, 'latest', null, 3_000) + store.messages.addMessage(sid, { text: 'earlier' }, 'earlier', null, 1_000) + const saved = store.historySync.commit(sid, 'thread', { epoch: 0, revision: 0, checkpoint, localIds: ['earlier'] }) + if (!saved) throw new Error('Expected a durable checkpoint') + store.close(); store = new Store(path) + expect(store.historySync.get(sid, 'thread')).toEqual(saved) + } finally { store.close(); rmSync(dir, { recursive: true, force: true }) } + }) + it('retains the old invalidation fence while upgrading an existing database', () => { + const dir = mkdtempSync(join(tmpdir(), 'hapi-history-upgrade-')) + const path = join(dir, 'hub.db') + let store = new Store(path) + try { + const sid = store.sessions.getOrCreateSession('legacy', {}, null, 'default').id + store.messages.addMessage(sid, { text: 'saved' }, 'saved') + store.historySync.commit(sid, 'thread', { epoch: 0, revision: 0, checkpoint, localIds: ['saved'] }) + store.messages.bumpMessageEpoch(sid) + store.close() + const legacy = new Database(path) + legacy.exec('DROP TABLE codex_history_epochs'); legacy.close() + store = new Store(path) + expect(store.historySync.get(sid, 'thread')).toEqual({ epoch: 1, revision: 1, checkpoint: null }) + expect(store.historySync.commit(sid, 'thread', { epoch: 0, revision: 1, checkpoint, localIds: ['saved'] })).toBeNull() + } finally { store.close(); rmSync(dir, { recursive: true, force: true }) } + }) +}) diff --git a/hub/src/store/historySyncStore.ts b/hub/src/store/historySyncStore.ts new file mode 100644 index 00000000..2e3556df --- /dev/null +++ b/hub/src/store/historySyncStore.ts @@ -0,0 +1,61 @@ +import type { Database } from 'bun:sqlite' +import { CodexHistoryCheckpointSchema, type CodexHistoryState, type CodexHistorySyncRequest } from '@hapi/protocol' + +export function bumpHistoryEpoch(db: Database, sid: string): void { + db.prepare(`INSERT INTO codex_history_epochs (session_id, epoch) VALUES (?, 1) + ON CONFLICT(session_id) DO UPDATE SET epoch = epoch + 1`).run(sid) +} + +/** Checkpoints are derived from durable messages. Keeping both in the Hub + * makes a restored database authoritative even when the CLI cache survives. */ +export class HistorySyncStore { + constructor(private readonly db: Database) { + db.exec(`CREATE TABLE IF NOT EXISTS codex_history_sync ( + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + thread_id TEXT NOT NULL, + epoch INTEGER NOT NULL, + revision INTEGER NOT NULL, + checkpoint TEXT, + PRIMARY KEY (session_id, thread_id) + )`) + // Display pagination changes when older messages arrive. Only removal + // or replacement invalidates the native history's durable prefix. + const existingEpochs = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'codex_history_epochs'").get() + db.exec(`CREATE TABLE IF NOT EXISTS codex_history_epochs ( + session_id TEXT PRIMARY KEY REFERENCES sessions(id) ON DELETE CASCADE, + epoch INTEGER NOT NULL + )`) + // Preserve the old invalidation fence during the one-time transition. + if (!existingEpochs) db.exec(`INSERT OR IGNORE INTO codex_history_epochs (session_id, epoch) + SELECT DISTINCT progress.session_id, COALESCE(display.epoch, 0) + FROM codex_history_sync progress + LEFT JOIN message_epochs display ON display.session_id = progress.session_id`) + } + + get(sid: string, threadId: string): CodexHistoryState { + const epoch = (this.db.prepare('SELECT epoch FROM codex_history_epochs WHERE session_id = ?') + .get(sid) as { epoch: number } | undefined)?.epoch ?? 0 + const row = this.db.prepare('SELECT epoch, revision, checkpoint FROM codex_history_sync WHERE session_id = ? AND thread_id = ?') + .get(sid, threadId) as { epoch: number; revision: number; checkpoint: string | null } | undefined + const parsed = row?.checkpoint ? CodexHistoryCheckpointSchema.safeParse(JSON.parse(row.checkpoint)) : null + return { epoch, revision: row?.revision ?? 0, + checkpoint: row?.epoch === epoch && parsed?.success ? parsed.data : null } + } + + commit(sid: string, threadId: string, commit: NonNullable): CodexHistoryState | null { + return this.db.transaction(() => { + const prior = this.get(sid, threadId) + if (prior.epoch !== commit.epoch || prior.revision !== commit.revision) return null + // Do not trust a client cursor whose corresponding outputs were + // never saved, even if a transport ACK was misinterpreted. + const exists = this.db.prepare('SELECT 1 FROM messages WHERE session_id = ? AND local_id = ?') + if (commit.localIds.some(id => !exists.get(sid, id))) return null + const revision = prior.revision + 1 + this.db.prepare(`INSERT INTO codex_history_sync (session_id, thread_id, epoch, revision, checkpoint) + VALUES (?, ?, ?, ?, ?) ON CONFLICT(session_id, thread_id) DO UPDATE SET + epoch = excluded.epoch, revision = excluded.revision, checkpoint = excluded.checkpoint`) + .run(sid, threadId, prior.epoch, revision, commit.checkpoint ? JSON.stringify(commit.checkpoint) : null) + return { epoch: prior.epoch, revision, checkpoint: commit.checkpoint } + })() + } +} diff --git a/hub/src/store/index.ts b/hub/src/store/index.ts index d1b957d8..14f4823b 100644 --- a/hub/src/store/index.ts +++ b/hub/src/store/index.ts @@ -16,6 +16,7 @@ import { UsageStore } from './usageStore' import { StudioStore } from './studioStore' import { WorkGraphStore } from './workGraphStore' import { SessionShareStore } from './sessionShareStore' +import { HistorySyncStore } from './historySyncStore' export type { NativeDevicePlatform, @@ -92,6 +93,7 @@ export class Store { readonly studios: StudioStore readonly workGraph: WorkGraphStore readonly sessionShares: SessionShareStore + readonly historySync: HistorySyncStore /** * Filesystem path of the underlying SQLite database, or ':memory:' for @@ -142,12 +144,14 @@ export class Store { this.sessions = new SessionStore(this.db) this.machines = new MachineStore(this.db) this.messages = new MessageStore(this.db) + this.historySync = new HistorySyncStore(this.db) this.users = new UserStore(this.db) this.push = new PushStore(this.db) this.fcm = new FcmStore(this.db) this.scratchlist = new ScratchlistStore(this.db) this.chatAttachments = new ChatAttachmentsStore(this.db) this.usage = new UsageStore(this.db) + for (const session of this.sessions.getSessions()) this.usage.rememberSessionSources(session) this.studios = new StudioStore(this.db) this.workGraph = new WorkGraphStore(this.db) this.sessionShares = new SessionShareStore(this.db) @@ -1303,6 +1307,14 @@ export class Store { */ private ensureUsageReconciliationSchema(): void { this.db.exec(` + CREATE TABLE IF NOT EXISTS usage_session_sources ( + namespace TEXT NOT NULL, + machine_id TEXT NOT NULL, + agent TEXT NOT NULL, + native_session_id TEXT NOT NULL, + session_id TEXT NOT NULL, + PRIMARY KEY (namespace, machine_id, agent, native_session_id, session_id) + ); CREATE TABLE IF NOT EXISTS usage_reconciliation ( namespace TEXT NOT NULL DEFAULT 'default', session_id TEXT NOT NULL, @@ -1315,11 +1327,21 @@ export class Store { cache_creation_tokens INTEGER NOT NULL DEFAULT 0, requests INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL, - PRIMARY KEY (session_id, day, model) + PRIMARY KEY (session_id, agent, day, model) ); CREATE INDEX IF NOT EXISTS idx_usage_reconciliation_namespace_day ON usage_reconciliation(namespace, day); `) + const columns = this.db.prepare('PRAGMA table_info(usage_reconciliation)').all() as Array<{ name: string; pk: number }> + if (columns.find((column) => column.name === 'agent')?.pk === 0) { + this.db.transaction(() => { + this.db.exec(`ALTER TABLE usage_reconciliation RENAME TO usage_reconciliation_legacy`) + this.ensureUsageReconciliationSchema() + this.db.exec(`INSERT INTO usage_reconciliation SELECT * FROM usage_reconciliation_legacy; + DROP TABLE usage_reconciliation_legacy; + CREATE INDEX IF NOT EXISTS idx_usage_reconciliation_namespace_day ON usage_reconciliation(namespace, day)`) + })() + } } private getSessionColumnNames(): Set { diff --git a/hub/src/store/messageStore.ts b/hub/src/store/messageStore.ts index d56d5224..b7591340 100644 --- a/hub/src/store/messageStore.ts +++ b/hub/src/store/messageStore.ts @@ -25,9 +25,8 @@ import { getUninvokedLocalMessages, getMatureScheduledMessages, getImmediateQueuedLocalMessages, - countFutureScheduledBySessionIds, + getFutureScheduledStatsBySessionIds, countFutureScheduledLocalMessages, - minFutureScheduledAtBySessionIds, countMessages, markMessagesInvoked, markMessagesIndeterminate, @@ -46,6 +45,7 @@ import { type LookupQueuedMessageResult, type LocalMessageState, type MessagePosition, + type FutureScheduledMessageStats, } from './messages' export class MessageStore { @@ -155,12 +155,8 @@ export class MessageStore { return countFutureScheduledLocalMessages(this.db, sessionId, now) } - countFutureScheduledBySessionIds(sessionIds: string[], now: number = Date.now()): Map { - return countFutureScheduledBySessionIds(this.db, sessionIds, now) - } - - minFutureScheduledAtBySessionIds(sessionIds: string[], now: number = Date.now()): Map { - return minFutureScheduledAtBySessionIds(this.db, sessionIds, now) + getFutureScheduledStatsBySessionIds(sessionIds: string[], now: number = Date.now()): Map { + return getFutureScheduledStatsBySessionIds(this.db, sessionIds, now) } // ponytail: scans through leading bookkeeping; index content if that prefix becomes costly. diff --git a/hub/src/store/messages.test.ts b/hub/src/store/messages.test.ts index 45065bb1..720f8645 100644 --- a/hub/src/store/messages.test.ts +++ b/hub/src/store/messages.test.ts @@ -1,7 +1,9 @@ -import { describe, expect, it } from 'bun:test' +import { describe, expect, it, spyOn } from 'bun:test' +import { Database } from 'bun:sqlite' import { AGENT_MESSAGE_PAYLOAD_TYPE } from '@hapi/protocol' import { getReasoningStreamId } from '@hapi/protocol/messages' import { Store } from './index' +import { getFutureScheduledStatsBySessionIds } from './messages' function makeStore(): Store { return new Store(':memory:') @@ -440,7 +442,7 @@ describe('countFutureScheduledLocalMessages', () => { expect(store.messages.countFutureScheduledLocalMessages(session.id, now)).toBe(1) }) - it('batch query returns counts keyed by session id', () => { + it('batch query returns counts and earliest timestamps keyed by session id', () => { const store = makeStore() const sessionA = makeSession(store, 'sched-batch-a') const sessionB = makeSession(store, 'sched-batch-b') @@ -464,13 +466,60 @@ describe('countFutureScheduledLocalMessages', () => { 'b-1' ) - const counts = store.messages.countFutureScheduledBySessionIds([sessionA.id, sessionB.id], now) - expect(counts.get(sessionA.id)).toBe(2) - expect(counts.get(sessionB.id)).toBeUndefined() + const stats = store.messages.getFutureScheduledStatsBySessionIds([sessionA.id, sessionB.id], now) + expect(stats.get(sessionA.id)).toEqual({ count: 2, nextScheduledAt: now + 60_000 }) + expect(stats.get(sessionB.id)).toBeUndefined() + expect(store.messages.getFutureScheduledStatsBySessionIds([], now).size).toBe(0) + }) - const nextAt = store.messages.minFutureScheduledAtBySessionIds([sessionA.id, sessionB.id], now) - expect(nextAt.get(sessionA.id)).toBe(now + 60_000) - expect(nextAt.get(sessionB.id)).toBeUndefined() + it('excludes invoked, mature, immediate, nonqueued, and other-session messages from batch statistics', () => { + const store = makeStore() + const session = makeSession(store, 'sched-filtered') + const otherSession = makeSession(store, 'sched-outside') + const now = Date.now() + + store.messages.addMessage(session.id, { text: 'later' }, 'later', now + 120_000) + store.messages.addMessage(session.id, { text: 'earliest' }, 'earliest', now + 60_000) + store.messages.addMessage(session.id, { text: 'immediate' }, 'immediate') + store.messages.addMessage(session.id, { text: 'mature' }, 'mature', now - 1) + store.messages.addMessage(session.id, { text: 'due now' }, 'due-now', now) + store.messages.addMessage(session.id, { text: 'invoked' }, 'invoked', now + 1) + store.messages.markMessagesInvoked(session.id, ['invoked'], now) + store.messages.addMessage(session.id, { text: 'dispatching' }, 'dispatching', now + 2) + store.messages.setMessagesDeliveryState(session.id, ['dispatching'], 'dispatching') + store.messages.addMessage(session.id, { text: 'indeterminate' }, 'indeterminate', now + 3) + store.messages.markMessagesIndeterminate(session.id, ['indeterminate']) + store.messages.addMessage(otherSession.id, { text: 'outside' }, 'outside', now + 4) + + expect(store.messages.getFutureScheduledStatsBySessionIds([session.id], now)).toEqual( + new Map([[session.id, { count: 2, nextScheduledAt: now + 60_000 }]]) + ) + }) + + it('uses the pending scheduled index instead of scanning historical local messages', () => { + const db = new Database(':memory:') + db.exec(` + CREATE TABLE messages ( + session_id TEXT, local_id TEXT, invoked_at INTEGER, + scheduled_at INTEGER, delivery_state TEXT + ); + CREATE INDEX idx_messages_local_id ON messages(session_id, local_id) + WHERE local_id IS NOT NULL; + CREATE INDEX idx_messages_scheduled_pending ON messages(scheduled_at) + WHERE scheduled_at IS NOT NULL AND invoked_at IS NULL; + `) + const prepare = spyOn(db, 'prepare') + try { + expect(getFutureScheduledStatsBySessionIds(db, ['session-1', 'session-2'], 1000).size).toBe(0) + expect(prepare).toHaveBeenCalledTimes(1) + const sql = prepare.mock.calls[0]![0] + const plan = db.query(`EXPLAIN QUERY PLAN ${sql}`).all('session-1', 'session-2', 1000) as { detail: string }[] + expect(plan.some((row) => row.detail.includes('idx_messages_scheduled_pending (scheduled_at>?)'))).toBe(true) + expect(plan.some((row) => row.detail.includes('idx_messages_local_id'))).toBe(false) + } finally { + prepare.mockRestore() + db.close() + } }) }) diff --git a/hub/src/store/messages.ts b/hub/src/store/messages.ts index eb8ae62b..b2f9954d 100644 --- a/hub/src/store/messages.ts +++ b/hub/src/store/messages.ts @@ -6,6 +6,8 @@ import { z } from 'zod' import { getLiveReasoningStreamId } from '@hapi/protocol/messages' import type { StoredMessage } from './types' +import { recordMessageUsage } from './usage' +import { bumpHistoryEpoch } from './historySyncStore' import { decodeMessageContent, encodeMessageContent, truncateOversizedMessageContent } from './contentCodec' type DbMessageRow = { @@ -75,10 +77,12 @@ export function addImportedMessage( local_id: localId, invoked_at: stampedAt }) - if (previousHead && stampedAt < previousHead.at) bumpMessageEpoch(db, sessionId) + if (previousHead && stampedAt < previousHead.at) bumpMessageEpoch(db, sessionId, false) const row = db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as DbMessageRow | undefined if (!row) throw new Error('Failed to create imported message') - return { message: toStoredMessage(row), inserted: true } + const message = toStoredMessage(row) + recordMessageUsage(db, message) + return { message, inserted: true } })() } @@ -111,8 +115,8 @@ export function addMessage( ): StoredMessage { const now = Date.now() // Client-provided origin timestamp (e.g. a Claude transcript entry's own - // `timestamp`), falling back to server-receive time when absent. Only - // agent-message callers (sessionHandlers' `on('message')`) pass this today. + // `timestamp`), falling back to server-receive time when absent. Native + // history callers also pass this for messages carrying stable localIds. const stampedAt = Number.isFinite(createdAt) ? Math.min(createdAt!, now) : now @@ -171,10 +175,12 @@ export function addMessage( }) const positionAt = invokedAt ?? stampedAt - if (previousHead && positionAt < previousHead.at) bumpMessageEpoch(db, sessionId) + if (previousHead && positionAt < previousHead.at) bumpMessageEpoch(db, sessionId, false) const row = db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as DbMessageRow | undefined if (!row) throw new Error('Failed to create message') - return toStoredMessage(row) + const message = toStoredMessage(row) + recordMessageUsage(db, message) + return message })() } @@ -248,7 +254,7 @@ export function copyMessageToSession( // Copies preserve the source display timestamp, so a new high-seq row can // still land behind a Web client's cached composite tail cursor. Mark the // target history as structurally changed so incremental readers reset. - bumpMessageEpoch(db, sessionId) + bumpMessageEpoch(db, sessionId, false) return toStoredMessage(row) } @@ -303,7 +309,7 @@ export function copyMessagesToSession( nextSeq += 1 } - bumpMessageEpoch(db, sessionId) + bumpMessageEpoch(db, sessionId, false) return messages.length })() } @@ -531,12 +537,13 @@ export function getMessageEpoch(db: Database, sessionId: string): number { return row?.epoch ?? 0 } -export function bumpMessageEpoch(db: Database, sessionId: string): number { +export function bumpMessageEpoch(db: Database, sessionId: string, invalidateHistory = true): number { db.prepare(` INSERT INTO message_epochs (session_id, epoch) VALUES (?, 1) ON CONFLICT(session_id) DO UPDATE SET epoch = epoch + 1 `).run(sessionId) + if (invalidateHistory) bumpHistoryEpoch(db, sessionId) return getMessageEpoch(db, sessionId) } @@ -665,51 +672,29 @@ export function countFutureScheduledLocalMessages( return row?.count ?? 0 } -/** Batch variant for GET /sessions — one query for all session IDs in a namespace. */ -export function countFutureScheduledBySessionIds( - db: Database, - sessionIds: string[], - now: number -): Map { - const counts = new Map() - if (sessionIds.length === 0) { - return counts - } - - const placeholders = sessionIds.map(() => '?').join(',') - const rows = db.prepare(` - SELECT session_id, COUNT(*) AS count - FROM messages - WHERE session_id IN (${placeholders}) - AND invoked_at IS NULL - AND local_id IS NOT NULL - AND scheduled_at IS NOT NULL - AND scheduled_at > ? - AND delivery_state = 'queued' - GROUP BY session_id - `).all(...sessionIds, now) as { session_id: string; count: number }[] - - for (const row of rows) { - counts.set(row.session_id, row.count) - } - return counts +export type FutureScheduledMessageStats = { + count: number + nextScheduledAt: number } -/** Earliest future scheduled_at per session (session-list clock tooltip). */ -export function minFutureScheduledAtBySessionIds( +/** Session-list statistics, computed together from pending scheduled messages. */ +export function getFutureScheduledStatsBySessionIds( db: Database, sessionIds: string[], now: number -): Map { - const nextAt = new Map() +): Map { + const stats = new Map() if (sessionIds.length === 0) { - return nextAt + return stats } const placeholders = sessionIds.map(() => '?').join(',') + // Without this index, SQLite can choose idx_messages_local_id and walk all + // historical local messages for every session, even when none are pending. + // This partial index exists on fresh databases and since the V9 migration. const rows = db.prepare(` - SELECT session_id, MIN(scheduled_at) AS next_at - FROM messages + SELECT session_id, COUNT(*) AS count, MIN(scheduled_at) AS next_at + FROM messages INDEXED BY idx_messages_scheduled_pending WHERE session_id IN (${placeholders}) AND invoked_at IS NULL AND local_id IS NOT NULL @@ -717,12 +702,12 @@ export function minFutureScheduledAtBySessionIds( AND scheduled_at > ? AND delivery_state = 'queued' GROUP BY session_id - `).all(...sessionIds, now) as { session_id: string; next_at: number }[] + `).all(...sessionIds, now) as { session_id: string; count: number; next_at: number }[] for (const row of rows) { - nextAt.set(row.session_id, row.next_at) + stats.set(row.session_id, { count: row.count, nextScheduledAt: row.next_at }) } - return nextAt + return stats } export function getMaxSeq(db: Database, sessionId: string): number { diff --git a/hub/src/store/sessionStore.ts b/hub/src/store/sessionStore.ts index 0efd403e..adb9f4a0 100644 --- a/hub/src/store/sessionStore.ts +++ b/hub/src/store/sessionStore.ts @@ -1,6 +1,8 @@ import type { Database } from 'bun:sqlite' import type { StoredSession, VersionedUpdateResult } from './types' +import { collectUsageEvents } from './usageStore' +import { rememberSessionUsageSources } from './usageSources' import { deleteSession, getOrCreateSession, @@ -41,7 +43,11 @@ export class SessionStore { modelReasoningEffort?: string, requestedId?: string ): StoredSession { - return getOrCreateSession(this.db, tag, metadata, agentState, namespace, model, effort, modelReasoningEffort, requestedId) + return this.db.transaction(() => { + const session = getOrCreateSession(this.db, tag, metadata, agentState, namespace, model, effort, modelReasoningEffort, requestedId) + rememberSessionUsageSources(this.db, session) + return session + })() } updateSessionMetadata( @@ -51,7 +57,14 @@ export class SessionStore { namespace: string, options?: { touchUpdatedAt?: boolean } ): VersionedUpdateResult { - return updateSessionMetadata(this.db, id, metadata, expectedVersion, namespace, options) + return this.db.transaction(() => { + const result = updateSessionMetadata(this.db, id, metadata, expectedVersion, namespace, options) + if (result.result === 'success') { + const session = getSessionByNamespace(this.db, id, namespace) + if (session) rememberSessionUsageSources(this.db, session) + } + return result + })() } updateSessionAgentState( @@ -133,6 +146,13 @@ export class SessionStore { } deleteSession(id: string, namespace: string): boolean { - return deleteSession(this.db, id, namespace) + return this.db.transaction(() => { + const session = getSessionByNamespace(this.db, id, namespace) + if (!session) return false + rememberSessionUsageSources(this.db, session) + // Capture unindexed legacy history before cascading message deletion. + collectUsageEvents(this.db, [session]) + return deleteSession(this.db, id, namespace) + })() } } diff --git a/hub/src/store/usage.ts b/hub/src/store/usage.ts index b3a1ddb2..d56f3aa5 100644 --- a/hub/src/store/usage.ts +++ b/hub/src/store/usage.ts @@ -1,4 +1,7 @@ import type { Database } from 'bun:sqlite' +import type { StoredMessage } from './types' +import { getSession } from './sessions' +import { parseUsageEvent, sessionModel } from './usageParser' export type UsageEventKind = 'delta' | 'cumulative' @@ -82,6 +85,109 @@ function toUsageEvent(row: UsageEventRow): UsageEvent { } } +export function recordUsageEvents(db: Database, namespace: string, events: UsageEvent[]): void { + if (events.length === 0) return + const statement = db.prepare(` + INSERT INTO usage_events ( + namespace, + session_id, + source_key, + source_seq, + created_at, + agent, + model, + kind, + input_tokens, + output_tokens, + cache_read_tokens, + cache_creation_tokens, + last_input_tokens, + last_output_tokens, + last_cache_read_tokens, + last_cache_creation_tokens + ) VALUES ( + @namespace, + @session_id, + @source_key, + @source_seq, + @created_at, + @agent, + @model, + @kind, + @input_tokens, + @output_tokens, + @cache_read_tokens, + @cache_creation_tokens, + @last_input_tokens, + @last_output_tokens, + @last_cache_read_tokens, + @last_cache_creation_tokens + ) + ON CONFLICT(session_id, source_key) + DO UPDATE SET + namespace = excluded.namespace, + source_seq = excluded.source_seq, + created_at = excluded.created_at, + agent = excluded.agent, + model = excluded.model, + kind = excluded.kind, + input_tokens = MAX(usage_events.input_tokens, excluded.input_tokens), + output_tokens = MAX(usage_events.output_tokens, excluded.output_tokens), + cache_read_tokens = MAX(usage_events.cache_read_tokens, excluded.cache_read_tokens), + cache_creation_tokens = MAX(usage_events.cache_creation_tokens, excluded.cache_creation_tokens), + last_input_tokens = excluded.last_input_tokens, + last_output_tokens = excluded.last_output_tokens, + last_cache_read_tokens = excluded.last_cache_read_tokens, + last_cache_creation_tokens = excluded.last_cache_creation_tokens + WHERE usage_events.kind = 'delta' + `) + const updateCumulativeModel = db.prepare(` + UPDATE usage_events + SET model = ? + WHERE session_id = ? + AND source_key = ? + AND kind = 'cumulative' + `) + + for (const event of events) { + statement.run({ + namespace, + session_id: event.sessionId, + source_key: event.sourceKey, + source_seq: event.sourceSeq, + created_at: event.createdAt, + agent: event.agent, + model: event.model, + kind: event.kind, + input_tokens: event.inputTokens, + output_tokens: event.outputTokens, + cache_read_tokens: event.cacheReadTokens, + cache_creation_tokens: event.cacheCreationTokens, + last_input_tokens: event.lastInputTokens, + last_output_tokens: event.lastOutputTokens, + last_cache_read_tokens: event.lastCacheReadTokens, + last_cache_creation_tokens: event.lastCacheCreationTokens + }) + if (event.kind === 'cumulative' && event.model !== null) { + updateCumulativeModel.run(event.model, event.sessionId, event.sourceKey) + } + } +} + +/** Called inside the message insertion transaction; scan cursors stay unchanged for legacy backfill. */ +export function recordMessageUsage(db: Database, message: StoredMessage): void { + const content = message.content + if (!content || typeof content !== 'object' || !('role' in content) || content.role !== 'agent') return + const session = getSession(db, message.sessionId) + if (!session) return + const event = parseUsageEvent(session, message) + if (!event) return + const previous = db.prepare('SELECT model FROM usage_events WHERE session_id = ? AND source_key = ?') + .get(event.sessionId, event.sourceKey) as { model: string | null } | undefined + event.model ??= previous?.model ?? sessionModel(session) + recordUsageEvents(db, session.namespace, [event]) +} + export function recordUsageScan( db: Database, sessionId: string, @@ -89,100 +195,20 @@ export function recordUsageScan( messageEpoch: number, lastSeq: number, events: UsageEvent[], - replaceEvents: boolean + _replaceEvents: boolean ): void { + // Bound standalone backfill writes so live hub acknowledgements can + // acquire SQLite's writer lock. A surrounding delete/merge transaction + // still makes its entire operation atomic. + const batchSize = 256 + for (let offset = 0; offset + batchSize < events.length; offset += batchSize) { + db.transaction(() => recordUsageEvents(db, namespace, events.slice(offset, offset + batchSize)))() + } + const finalOffset = events.length === 0 ? 0 : Math.floor((events.length - 1) / batchSize) * batchSize db.transaction(() => { - if (replaceEvents) { - db.prepare('DELETE FROM usage_events WHERE session_id = ?').run(sessionId) - } - - if (events.length > 0) { - const statement = db.prepare(` - INSERT INTO usage_events ( - namespace, - session_id, - source_key, - source_seq, - created_at, - agent, - model, - kind, - input_tokens, - output_tokens, - cache_read_tokens, - cache_creation_tokens, - last_input_tokens, - last_output_tokens, - last_cache_read_tokens, - last_cache_creation_tokens - ) VALUES ( - @namespace, - @session_id, - @source_key, - @source_seq, - @created_at, - @agent, - @model, - @kind, - @input_tokens, - @output_tokens, - @cache_read_tokens, - @cache_creation_tokens, - @last_input_tokens, - @last_output_tokens, - @last_cache_read_tokens, - @last_cache_creation_tokens - ) - ON CONFLICT(session_id, source_key) - DO UPDATE SET - namespace = excluded.namespace, - source_seq = excluded.source_seq, - created_at = excluded.created_at, - agent = excluded.agent, - model = excluded.model, - kind = excluded.kind, - input_tokens = excluded.input_tokens, - output_tokens = excluded.output_tokens, - cache_read_tokens = excluded.cache_read_tokens, - cache_creation_tokens = excluded.cache_creation_tokens, - last_input_tokens = excluded.last_input_tokens, - last_output_tokens = excluded.last_output_tokens, - last_cache_read_tokens = excluded.last_cache_read_tokens, - last_cache_creation_tokens = excluded.last_cache_creation_tokens - WHERE usage_events.kind = 'delta' - `) - const updateCumulativeModel = db.prepare(` - UPDATE usage_events - SET model = ? - WHERE session_id = ? - AND source_key = ? - AND kind = 'cumulative' - `) - - for (const event of events) { - statement.run({ - namespace, - session_id: event.sessionId, - source_key: event.sourceKey, - source_seq: event.sourceSeq, - created_at: event.createdAt, - agent: event.agent, - model: event.model, - kind: event.kind, - input_tokens: event.inputTokens, - output_tokens: event.outputTokens, - cache_read_tokens: event.cacheReadTokens, - cache_creation_tokens: event.cacheCreationTokens, - last_input_tokens: event.lastInputTokens, - last_output_tokens: event.lastOutputTokens, - last_cache_read_tokens: event.lastCacheReadTokens, - last_cache_creation_tokens: event.lastCacheCreationTokens - }) - if (event.kind === 'cumulative' && event.model !== null) { - updateCumulativeModel.run(event.model, event.sessionId, event.sourceKey) - } - } - } + // An epoch change invalidates the scan cursor, not actual consumption. + // Rewinds and transcript replacements cannot refund completed requests. + recordUsageEvents(db, namespace, events.slice(finalOffset)) db.prepare(` INSERT INTO usage_scan_state (namespace, session_id, message_epoch, last_seq) @@ -298,8 +324,8 @@ export function transferUsageSession(db: Database, fromSessionId: string, toSess last_cache_creation_tokens ) SELECT - ?, COALESCE((SELECT namespace FROM sessions WHERE id = ?), usage_events.namespace), + ?, source_key, source_seq, created_at, @@ -318,41 +344,17 @@ export function transferUsageSession(db: Database, fromSessionId: string, toSess WHERE session_id = ? `).run(toSessionId, toSessionId, fromSessionId) db.prepare('DELETE FROM usage_events WHERE session_id = ?').run(fromSessionId) - db.prepare('DELETE FROM usage_scan_state WHERE session_id IN (?, ?)').run(fromSessionId, toSessionId) - db.prepare(` - INSERT OR REPLACE INTO usage_reconciliation ( - namespace, - session_id, - day, - model, - agent, - input_tokens, - output_tokens, - cache_read_tokens, - cache_creation_tokens, - requests, - updated_at - ) - SELECT - ?, - ?, - day, - model, - agent, - input_tokens, - output_tokens, - cache_read_tokens, - cache_creation_tokens, - requests, - updated_at - FROM usage_reconciliation - WHERE session_id = ? - `).run( - (db.prepare('SELECT namespace FROM sessions WHERE id = ?').get(toSessionId) as { namespace: string } | undefined)?.namespace ?? 'default', - toSessionId, - fromSessionId - ) - db.prepare('DELETE FROM usage_reconciliation WHERE session_id = ?').run(fromSessionId) + // Preserve the source cursor: copied transcripts can remain there + // until deletion, and must not re-create the transferred ledger. + db.prepare('DELETE FROM usage_scan_state WHERE session_id = ?').run(toSessionId) + db.prepare(`INSERT OR IGNORE INTO usage_session_sources + (namespace, machine_id, agent, native_session_id, session_id) + SELECT COALESCE((SELECT namespace FROM sessions WHERE id = ?), namespace), + machine_id, agent, native_session_id, ? FROM usage_session_sources WHERE session_id = ?`) + .run(toSessionId, toSessionId, fromSessionId) + // Snapshots are already durable and source-specific. Moving them can + // overwrite a different native source's totals on the same day/model. + // Source aliases above make them cover the merged live ledger. })() } @@ -365,7 +367,13 @@ export function replaceReconciledUsage( updatedAt: number ): void { db.transaction(() => { - db.prepare('DELETE FROM usage_reconciliation WHERE session_id = ?').run(sessionId) + const agents = [...new Set(rows.map((row) => row.agent))] + for (const agent of agents) { + db.prepare('DELETE FROM usage_reconciliation WHERE session_id = ? AND agent = ?').run(sessionId, agent) + } + if (rows.length === 0) { + db.prepare('DELETE FROM usage_reconciliation WHERE session_id = ?').run(sessionId) + } if (rows.length === 0) return const statement = db.prepare(` INSERT INTO usage_reconciliation ( diff --git a/hub/src/store/usageParser.ts b/hub/src/store/usageParser.ts new file mode 100644 index 00000000..5a4bfff8 --- /dev/null +++ b/hub/src/store/usageParser.ts @@ -0,0 +1,227 @@ +import type { StoredMessage, StoredSession } from './types' +import type { UsageEvent } from './usage' + +type RecordValue = Record + +function asRecord(value: unknown): RecordValue | null { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? value as RecordValue + : null +} + +function asCount(value: unknown): number | null { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 + ? Math.floor(value) + : null +} + +function firstCount(record: RecordValue, ...keys: string[]): number { + for (const key of keys) { + const value = asCount(record[key]) + if (value !== null) return value + } + return 0 +} + +function normalizeInputTokens( + data: RecordValue, + inputTokens: number, + cacheReadTokens: number, + cacheCreationTokens: number, + legacySemantics: 'includes-cache' | 'excludes-cache' +): number { + // v1 generic usage messages make their input contract self-describing. + // Unknown/missing metadata intentionally falls back to the historical + // provider shape so already persisted transcripts remain readable. + const declaredSemantics = data.usageSchema === 'hapi.usage.v1' + && (data.inputTokenSemantics === 'includes-cache' || data.inputTokenSemantics === 'excludes-cache') + ? data.inputTokenSemantics + : null + const semantics = declaredSemantics ?? legacySemantics + return semantics === 'excludes-cache' + ? inputTokens + cacheReadTokens + cacheCreationTokens + : inputTokens +} + +function sessionAgent(session: Pick): string { + const metadata = asRecord(session.metadata) + const flavor = metadata?.flavor + return typeof flavor === 'string' && flavor.trim() ? flavor.trim() : 'unknown' +} + +export function sessionModel(session: StoredSession): string | null { + return typeof session.model === 'string' && session.model.trim() ? session.model.trim() : null +} + +export function parseUsageEvent( + session: Pick, + message: Pick +): UsageEvent | null { + const envelope = asRecord(message.content) + if (envelope?.role !== 'agent') return null + + const payload = asRecord(envelope.content) + if (!payload) return null + let data = asRecord(payload.data) + if (!data) return null + if (data.type === 'agent-run-trace' && sessionAgent(session) === 'codex') { + const trace = asRecord(data.message) + if (!trace) return null + const scope = asRecord(data.scope) + data = { + ...trace, + thread_id: trace.thread_id ?? scope?.threadId, + scope_role: 'child', + ...(data.hapiUsageScope ? { hapiUsageScope: data.hapiUsageScope } : {}) + } + } + + // Claude stream-json/SDK messages. A stream emits several updates for one + // assistant message, so the provider's message id is the stable upsert key. + if (payload.type === 'output' && data.type === 'assistant') { + const assistant = asRecord(data.message) + const usage = asRecord(assistant?.usage) + if (!usage) return null + const inputTokens = firstCount(usage, 'input_tokens', 'inputTokens') + const outputTokens = firstCount(usage, 'output_tokens', 'outputTokens') + const cacheReadTokens = firstCount(usage, 'cache_read_input_tokens', 'cacheReadTokens', 'cachedInputTokens') + const cacheCreationTokens = firstCount(usage, 'cache_creation_input_tokens', 'cacheCreationTokens', 'cacheWriteInputTokens') + if (inputTokens + outputTokens + cacheReadTokens + cacheCreationTokens <= 0) return null + const providerId = typeof assistant?.id === 'string' ? assistant.id : message.id + const model = typeof assistant?.model === 'string' && assistant.model.trim() + ? assistant.model.trim() + : null + return { + sessionId: session.id, + sourceKey: `claude|${providerId}`, + sourceSeq: message.seq, + createdAt: message.createdAt, + agent: 'claude', + model, + kind: 'delta', + inputTokens: normalizeInputTokens(data, inputTokens, cacheReadTokens, cacheCreationTokens, 'excludes-cache'), + outputTokens, + cacheReadTokens, + cacheCreationTokens, + lastInputTokens: null, + lastOutputTokens: null, + lastCacheReadTokens: null, + lastCacheCreationTokens: null + } + } + + // Codex forwards cumulative thread totals plus the most recent request. + // ACP-compatible backends wrap per-request usage in `total`, so only Codex + // should be diffed as a cumulative stream. + if (data.type === 'token_count' || data.type === 'usage') { + if (data.hapiUsageScope === 'imported-history') return null + const info = asRecord(data.info) ?? data + const agent = sessionAgent(session) + const explicitThreadId = typeof data.threadId === 'string' + ? data.threadId + : typeof data.thread_id === 'string' + ? data.thread_id + : null + const metadata = asRecord(session.metadata) + const hasImportedCodexHistory = typeof metadata?.codexSourceSessionId === 'string' + || metadata?.lifecycleState === 'imported' + if (agent === 'codex' && explicitThreadId === null && hasImportedCodexHistory) { + return null + } + const cumulativeTotal = agent === 'codex' + ? asRecord(info.total) + ?? asRecord(info.total_token_usage) + ?? asRecord(info.totalTokenUsage) + : null + const last = asRecord(info.last) + ?? asRecord(info.last_token_usage) + ?? asRecord(info.lastTokenUsage) + ?? (data.type === 'usage' ? info : null) + const total = cumulativeTotal ?? (agent === 'codex' ? last : asRecord(info.total) ?? info) + if (!total) return null + const rawInputTokens = firstCount(total, 'inputTokens', 'input_tokens') + const outputTokens = firstCount(total, 'outputTokens', 'output_tokens') + const cacheReadTokens = firstCount(total, 'cachedInputTokens', 'cached_input_tokens', 'cacheReadTokens', 'cache_read_input_tokens') + const cacheCreationTokens = firstCount(total, 'cacheWriteInputTokens', 'cache_write_input_tokens', 'cacheCreationTokens', 'cache_creation_input_tokens') + if (rawInputTokens + outputTokens + cacheReadTokens + cacheCreationTokens <= 0) return null + const threadId = explicitThreadId ?? session.id + const scope = typeof data.scopeRole === 'string' + ? data.scopeRole + : typeof data.scope_role === 'string' + ? data.scope_role + : 'parent' + const isCumulative = cumulativeTotal !== null + const turnId = typeof data.turnId === 'string' + ? data.turnId + : typeof data.turn_id === 'string' + ? data.turn_id + : '' + const model = typeof data.model === 'string' && data.model.trim() + ? data.model.trim() + : null + // Codex/Kimi provider formats have always reported inclusive input. + // Imported Pi usage is known-inclusive, and for generic ACP an own + // `model` property is the only strong provenance for the unmarked + // inclusive wire introduced with the usage dashboard. Older ambiguous + // payloads are conservatively treated as cache-exclusive. + const legacyInputSemantics = agent === 'codex' + || agent === 'kimi' + || (agent === 'pi' && message.localId?.startsWith('pi:')) + || Object.prototype.hasOwnProperty.call(data, 'model') + ? 'includes-cache' + : 'excludes-cache' + const inputTokens = normalizeInputTokens( + data, + rawInputTokens, + cacheReadTokens, + cacheCreationTokens, + legacyInputSemantics + ) + const lastOutputTokens = last ? firstCount(last, 'outputTokens', 'output_tokens') : null + const lastCacheReadTokens = last + ? firstCount(last, 'cachedInputTokens', 'cached_input_tokens', 'cacheReadTokens', 'cache_read_input_tokens') + : null + const lastCacheCreationTokens = last + ? firstCount(last, 'cacheWriteInputTokens', 'cache_write_input_tokens', 'cacheCreationTokens', 'cache_creation_input_tokens') + : null + const lastInputTokens = last + ? normalizeInputTokens( + data, + firstCount(last, 'inputTokens', 'input_tokens'), + lastCacheReadTokens ?? 0, + lastCacheCreationTokens ?? 0, + legacyInputSemantics + ) + : null + return { + sessionId: session.id, + sourceKey: isCumulative + ? [ + 'cumulative', + threadId, + scope, + turnId, + inputTokens, + outputTokens, + cacheReadTokens, + cacheCreationTokens + ].join('|') + : `delta|${message.id}`, + sourceSeq: message.seq, + createdAt: message.createdAt, + agent, + model, + kind: isCumulative ? 'cumulative' : 'delta', + inputTokens, + outputTokens, + cacheReadTokens, + cacheCreationTokens, + lastInputTokens, + lastOutputTokens, + lastCacheReadTokens, + lastCacheCreationTokens + } + } + + return null +} diff --git a/hub/src/store/usagePersistence.test.ts b/hub/src/store/usagePersistence.test.ts new file mode 100644 index 00000000..e03b68c8 --- /dev/null +++ b/hub/src/store/usagePersistence.test.ts @@ -0,0 +1,200 @@ +import { describe, expect, it } from 'bun:test' +import type { Database } from 'bun:sqlite' +import { Store } from './index' +import { getUsageSummary } from '../sync/usageService' + +function createSession(store: Store, namespace: string = 'default') { + return store.sessions.getOrCreateSession( + 'usage-persistence', + { path: '/tmp', host: 'test', flavor: 'claude' }, + null, + namespace, + 'test-model' + ) +} + +function usageMessage(id: string, inputTokens: number, outputTokens: number) { + return { + role: 'agent', + content: { + type: 'output', + data: { + type: 'assistant', + message: { id, usage: { input_tokens: inputTokens, output_tokens: outputTokens } } + } + } + } +} + +function addUnindexedHistory(store: Store, sessionId: string): void { + // Represents history persisted by a hub without realtime usage indexing. + store.messages.copyMessageToSession(sessionId, { + content: usageMessage('historical-request', 20, 4), + createdAt: Date.now(), + localId: null, + invokedAt: Date.now(), + scheduledAt: null + }) +} + +describe('durable usage persistence', () => { + it.each(['live', 'imported'] as const)('records %s usage before any dashboard request and keeps it after deletion', (source) => { + const store = new Store(':memory:') + try { + const session = createSession(store) + const content = usageMessage('request-1', 10, 2) + if (source === 'live') { + store.messages.addMessage(session.id, content) + } else { + store.messages.addImportedMessage(session.id, content, 'imported-1', Date.now()) + store.messages.addImportedMessage(session.id, content, 'imported-1', Date.now()) + } + + expect(store.usage.getEvents([session.id])).toEqual([ + expect.objectContaining({ inputTokens: 10, outputTokens: 2, model: 'test-model' }) + ]) + expect(store.sessions.deleteSession(session.id, 'default')).toBe(true) + expect(getUsageSummary(store, 'default', 'all').totals).toMatchObject({ + totalTokens: 12, requests: 1, sessions: 1 + }) + } finally { + store.close() + } + }) + + it('retains usage generated after the last dashboard request', () => { + const store = new Store(':memory:') + try { + const session = createSession(store) + store.messages.addMessage(session.id, usageMessage('request-1', 10, 2)) + expect(getUsageSummary(store, 'default', 'all').totals.totalTokens).toBe(12) + + store.messages.addMessage(session.id, usageMessage('request-2', 20, 4)) + // Streaming updates share the provider message id and count once. + store.messages.addMessage(session.id, usageMessage('request-2', 30, 6)) + expect(store.sessions.deleteSession(session.id, 'default')).toBe(true) + for (let read = 0; read < 2; read += 1) { + expect(getUsageSummary(store, 'default', 'all').totals).toMatchObject({ + totalTokens: 48, requests: 2, sessions: 1 + }) + } + } finally { + store.close() + } + }) + + it.each(['before-first-read', 'after-previous-read'] as const)('backfills unindexed history before deletion: %s', (scenario) => { + const store = new Store(':memory:') + try { + const session = createSession(store) + const previousRead = scenario === 'after-previous-read' + if (previousRead) { + store.messages.addMessage(session.id, usageMessage('request-1', 10, 2)) + expect(getUsageSummary(store, 'default', 'all').totals.totalTokens).toBe(12) + } + addUnindexedHistory(store, session.id) + + expect(store.sessions.deleteSession(session.id, 'default')).toBe(true) + expect(getUsageSummary(store, 'default', 'all').totals).toMatchObject({ + totalTokens: previousRead ? 36 : 24, + requests: previousRead ? 2 : 1 + }) + } finally { + store.close() + } + }) + + it('does not delete or index another namespace\'s session', () => { + const store = new Store(':memory:') + try { + const session = createSession(store, 'private') + addUnindexedHistory(store, session.id) + expect(store.sessions.deleteSession(session.id, 'default')).toBe(false) + expect(store.sessions.getSession(session.id)).not.toBeNull() + expect(store.usage.getEventsByNamespace('private')).toEqual([]) + + expect(store.sessions.deleteSession(session.id, 'private')).toBe(true) + expect(getUsageSummary(store, 'default', 'all').totals.totalTokens).toBe(0) + expect(getUsageSummary(store, 'private', 'all').totals.totalTokens).toBe(24) + } finally { + store.close() + } + }) + + it('indexes Codex cumulative snapshots once and preserves models before any dashboard request', () => { + const store = new Store(':memory:') + try { + const session = store.sessions.getOrCreateSession( + 'codex-usage-persistence', + { path: '/tmp', host: 'test', flavor: 'codex' }, + null, + 'default', + 'model-one' + ) + const addSnapshot = (turnId: string, inputTokens: number, outputTokens: number) => { + store.messages.addMessage(session.id, { + role: 'agent', + content: { + type: 'codex', + data: { + type: 'token_count', + thread_id: 'thread-1', + turn_id: turnId, + info: { + total_token_usage: { input_tokens: inputTokens, output_tokens: outputTokens }, + last_token_usage: { input_tokens: 100, output_tokens: 10 } + } + } + } + }) + } + addSnapshot('turn-1', 1_000, 100) + store.sessions.setSessionModel(session.id, 'model-two', 'default') + addSnapshot('turn-1', 1_000, 100) + addSnapshot('turn-2', 1_140, 115) + expect(store.usage.getEvents([session.id]).map((event) => event.model)).toEqual(['model-one', 'model-two']) + + expect(store.sessions.deleteSession(session.id, 'default')).toBe(true) + const summary = getUsageSummary(store, 'default', 'all') + expect(summary.totals).toMatchObject({ totalTokens: 265, requests: 2 }) + expect(summary.byModel).toEqual([ + expect.objectContaining({ key: 'model-two', totalTokens: 155 }), + expect.objectContaining({ key: 'model-one', totalTokens: 110 }) + ]) + } finally { + store.close() + } + }) + + it('rolls back the message when usage cannot be persisted', () => { + const store = new Store(':memory:') + try { + const session = createSession(store) + const db = (store as unknown as { db: Database }).db + db.exec("CREATE TRIGGER reject_usage BEFORE INSERT ON usage_events BEGIN SELECT RAISE(ABORT, 'usage rejected'); END") + + expect(() => store.messages.addMessage(session.id, usageMessage('request-1', 10, 2))).toThrow('usage rejected') + expect(store.messages.getAllMessages(session.id)).toEqual([]) + expect(store.usage.getEventsByNamespace('default')).toEqual([]) + } finally { + store.close() + } + }) + + it('rolls back the backfill when session deletion fails', () => { + const store = new Store(':memory:') + try { + const session = createSession(store) + addUnindexedHistory(store, session.id) + const db = (store as unknown as { db: Database }).db + db.exec("CREATE TRIGGER reject_delete BEFORE DELETE ON sessions BEGIN SELECT RAISE(ABORT, 'delete rejected'); END") + + expect(() => store.sessions.deleteSession(session.id, 'default')).toThrow('delete rejected') + expect(store.messages.getAllMessages(session.id)).toHaveLength(1) + expect(store.usage.getEventsByNamespace('default')).toEqual([]) + expect(store.usage.getScanStates([session.id]).size).toBe(0) + } finally { + store.close() + } + }) +}) diff --git a/hub/src/store/usageSources.ts b/hub/src/store/usageSources.ts new file mode 100644 index 00000000..56cf65a4 --- /dev/null +++ b/hub/src/store/usageSources.ts @@ -0,0 +1,53 @@ +import type { Database } from 'bun:sqlite' +import { createHash } from 'node:crypto' +import type { StoredSession } from './types' + +export type UsageSource = { + namespace: string + machineId: string + agent: string + nativeSessionId: string + sessionId: string +} + +export function usageSourceLedgerId(source: Omit): string { + return `usage-source:${createHash('sha256') + .update(JSON.stringify([source.namespace, source.machineId, source.agent, source.nativeSessionId])) + .digest('hex')}` +} + +export function rememberUsageSource(db: Database, source: UsageSource): void { + db.prepare(`INSERT OR IGNORE INTO usage_session_sources + (namespace, machine_id, agent, native_session_id, session_id) VALUES (?, ?, ?, ?, ?)`) + .run(source.namespace, source.machineId, source.agent, source.nativeSessionId, source.sessionId) + if (source.machineId) { + // Replace an incomplete binding once the owning machine is known. + db.prepare(`DELETE FROM usage_session_sources WHERE namespace = ? AND machine_id = '' + AND agent = ? AND native_session_id = ? AND session_id = ?`) + .run(source.namespace, source.agent, source.nativeSessionId, source.sessionId) + } +} + +export function rememberSessionUsageSources(db: Database, session: StoredSession): void { + const metadata = session.metadata + if (!metadata || typeof metadata !== 'object' || Array.isArray(metadata)) return + const data = metadata as Record + const machineId = typeof data.machineId === 'string' ? data.machineId : session.machineId ?? '' + for (const [agent, key] of [['opencode', 'opencodeSessionId'], ['codex', 'codexSessionId'], ['claude', 'claudeSessionId'], ['dsh', 'dshSessionId']] as const) { + const nativeSessionId = data[key] + if (typeof nativeSessionId !== 'string' || !nativeSessionId) continue + rememberUsageSource(db, { namespace: session.namespace, machineId, agent, nativeSessionId, sessionId: session.id }) + } +} + +export function getUsageSources(db: Database, namespace?: string, machineId?: string): UsageSource[] { + const clauses: string[] = [] + const params: string[] = [] + if (namespace !== undefined) { clauses.push('namespace = ?'); params.push(namespace) } + if (machineId !== undefined) { clauses.push('machine_id = ?'); params.push(machineId) } + return db.prepare(`SELECT namespace, machine_id AS machineId, agent, + native_session_id AS nativeSessionId, session_id AS sessionId + FROM usage_session_sources ${clauses.length ? `WHERE ${clauses.join(' AND ')}` : ''} + ORDER BY namespace, machine_id, agent, native_session_id, session_id`) + .all(...params) as UsageSource[] +} diff --git a/hub/src/store/usageStore.ts b/hub/src/store/usageStore.ts index 6af9aa61..537f80b4 100644 --- a/hub/src/store/usageStore.ts +++ b/hub/src/store/usageStore.ts @@ -1,4 +1,9 @@ import type { Database } from 'bun:sqlite' +import type { StoredSession } from './types' +import { getMessageEpoch, getMessagesAfterSeq } from './messages' +import { getSession } from './sessions' +import { parseUsageEvent, sessionModel } from './usageParser' +import { getUsageSources, rememberUsageSource, rememberSessionUsageSources, usageSourceLedgerId, type UsageSource } from './usageSources' import { getReconciledUsageByNamespace, @@ -6,6 +11,7 @@ import { getUsageEventsByNamespace, getUsageScanStates, recordUsageScan, + recordUsageEvents, replaceReconciledUsage, transferUsageSession, type ReconciledUsageRow, @@ -13,9 +19,131 @@ import { type UsageScanState } from './usage' +export function collectUsageEvents(db: Database, sessions: StoredSession[]): void { + const scanStates = getUsageScanStates(db, sessions.map((session) => session.id)) + for (const session of sessions) { + rememberSessionUsageSources(db, session) + const messageEpoch = getMessageEpoch(db, session.id) + const scanState = scanStates.get(session.id) + const replaceEvents = !scanState || scanState.messageEpoch !== messageEpoch + const afterSeq = replaceEvents ? 0 : scanState.lastSeq + const messages = getMessagesAfterSeq(db, session.id, afterSeq) + const events = new Map() + let indexedModels: Map | null = null + const getIndexedModel = (sourceKey: string): string | null => { + if (indexedModels === null) { + indexedModels = new Map( + getUsageEvents(db, [session.id]) + .filter((event): event is UsageEvent & { model: string } => event.model !== null) + .map((event) => [event.sourceKey, event.model]) + ) + } + return indexedModels.get(sourceKey) ?? null + } + const fallbackModel = sessionModel(session) + for (const message of messages) { + const event = parseUsageEvent(session, message) + if (!event) continue + const existingEvent = events.get(event.sourceKey) + const explicitModel = event.model + event.model = explicitModel + ?? existingEvent?.model + ?? getIndexedModel(event.sourceKey) + ?? fallbackModel + if (event.kind === 'delta' || !existingEvent) { + events.set(event.sourceKey, event) + } else if (explicitModel !== null) { + // A replay may add model metadata missing from the original snapshot. + existingEvent.model = explicitModel + } + } + const lastSeq = messages.at(-1)?.seq ?? afterSeq + if (messages.length > 0 || replaceEvents) { + recordUsageScan( + db, + session.id, + session.namespace, + messageEpoch, + lastSeq, + Array.from(events.values()), + replaceEvents + ) + } + } +} + export class UsageStore { constructor(private readonly db: Database) {} + collectSessions(sessions: StoredSession[]): void { + collectUsageEvents(this.db, sessions) + } + + rememberSource(source: UsageSource): void { + rememberUsageSource(this.db, source) + } + + rememberSessionSources(session: StoredSession): void { + rememberSessionUsageSources(this.db, session) + } + + getSources(namespace?: string, machineId?: string): UsageSource[] { + return getUsageSources(this.db, namespace, machineId) + } + + /** Repair records written with swapped namespace/session columns by the old merge path. */ + repairTransferredEvents(namespace: string): number { + return this.db.transaction(() => { + // Some deleted merge targets have no retained metadata. A native + // recovery can already have restored their exact provider keys; + // remove only those verified duplicate misplaced rows. + const duplicateRepair = this.db.prepare(`DELETE FROM usage_events AS broken + WHERE broken.session_id = ? AND length(broken.namespace) = 36 + AND substr(broken.namespace, 9, 1) = '-' + AND EXISTS (SELECT 1 FROM usage_events AS recovered + WHERE recovered.namespace = ? AND recovered.agent = broken.agent + AND recovered.source_key = broken.source_key)`) + .run(namespace, namespace).changes + const misplaced = this.db.prepare(`SELECT DISTINCT namespace FROM usage_events + WHERE session_id = ? AND namespace IN ( + SELECT id FROM sessions WHERE namespace = ? + UNION SELECT session_id FROM usage_session_sources WHERE namespace = ?)`) + .all(namespace, namespace, namespace) as Array<{ namespace: string }> + let repaired = duplicateRepair + for (const row of misplaced) { + const events = getUsageEventsByNamespace(this.db, row.namespace) + .filter((event) => event.sessionId === namespace) + .map((event) => ({ ...event, sessionId: row.namespace })) + recordUsageEvents(this.db, namespace, events) + this.db.prepare('DELETE FROM usage_events WHERE namespace = ? AND session_id = ?') + .run(row.namespace, namespace) + repaired += events.length + } + return repaired + })() + } + + reconcileSource(source: UsageSource, rows: Omit[], updatedAt: number): void { + if (rows.length === 0) return + this.db.transaction(() => { + rememberUsageSource(this.db, source) + const aliases = getUsageSources(this.db, source.namespace, source.machineId) + .filter((entry) => entry.agent === source.agent && entry.nativeSessionId === source.nativeSessionId) + const ledgerId = usageSourceLedgerId(source) + const ids = [ledgerId, ...aliases.map((alias) => alias.sessionId)] + const latest = this.db.prepare(`SELECT MAX(updated_at) AS updatedAt FROM usage_reconciliation + WHERE namespace = ? AND agent = ? AND session_id IN (${ids.map(() => '?').join(',')})`) + .get(source.namespace, source.agent, ...ids) as { updatedAt: number | null } + if (latest.updatedAt !== null && latest.updatedAt > updatedAt) return + for (const alias of aliases) { + this.db.prepare('DELETE FROM usage_reconciliation WHERE namespace = ? AND session_id = ? AND agent = ?') + .run(source.namespace, alias.sessionId, source.agent) + } + replaceReconciledUsage(this.db, source.namespace, ledgerId, + rows.map((row) => ({ ...row, agent: source.agent, sessionId: ledgerId })), updatedAt) + })() + } + recordScan( sessionId: string, namespace: string, @@ -40,7 +168,21 @@ export class UsageStore { } transferSession(fromSessionId: string, toSessionId: string): void { - transferUsageSession(this.db, fromSessionId, toSessionId) + this.db.transaction(() => { + const source = getSession(this.db, fromSessionId) + if (source) collectUsageEvents(this.db, [source]) + const legacy = this.db.prepare('SELECT DISTINCT namespace, agent FROM usage_reconciliation WHERE session_id = ?') + .all(fromSessionId) as Array<{ namespace: string; agent: string }> + for (const row of legacy) { + const hasSource = getUsageSources(this.db, row.namespace) + .some((entry) => entry.sessionId === fromSessionId && entry.agent === row.agent) + if (!hasSource) { + rememberUsageSource(this.db, { ...row, machineId: '', + nativeSessionId: `usage-ledger:${fromSessionId}`, sessionId: fromSessionId }) + } + } + transferUsageSession(this.db, fromSessionId, toSessionId) + })() } replaceReconciled(sessionId: string, namespace: string, rows: ReconciledUsageRow[], updatedAt: number): void { diff --git a/hub/src/sync/rpcGateway.test.ts b/hub/src/sync/rpcGateway.test.ts index 4f72d599..de3f2eae 100644 --- a/hub/src/sync/rpcGateway.test.ts +++ b/hub/src/sync/rpcGateway.test.ts @@ -51,6 +51,19 @@ function createGateway() { } describe('RpcGateway RPC timeouts', () => { + it('waits beyond the runner startup deadline for cold Codex resumes', async () => { + const { gateway, timeouts, calls } = createGateway() + + await gateway.spawnSession('machine-1', '/workspace', 'codex', undefined, undefined, + undefined, undefined, undefined, 'native-thread') + await gateway.spawnSession('machine-1', '/workspace', 'codex') + await gateway.spawnSession('machine-1', '/workspace', 'claude', undefined, undefined, + undefined, undefined, undefined, 'native-session') + + expect(timeouts).toEqual([130_000, 30_000, 30_000]) + expect(JSON.parse(calls[0].params).resumeSessionId).toBe('native-thread') + }) + it('uses the default RPC timeout for regular machine RPCs', async () => { const { gateway, timeouts } = createGateway() diff --git a/hub/src/sync/rpcGateway.ts b/hub/src/sync/rpcGateway.ts index 0fc0bbdb..828f70e2 100644 --- a/hub/src/sync/rpcGateway.ts +++ b/hub/src/sync/rpcGateway.ts @@ -1,6 +1,8 @@ import type { AgentFlavor, CodexCollaborationMode, CopilotAgentMode, PermissionMode } from '@hapi/protocol/types' import { RPC_METHODS } from '@hapi/protocol/rpcMethods' +import { sessionStartupTimeoutMs } from '@hapi/protocol' import { + CodexGoalResponseSchema, ArchiveCodexSessionRpcResponseSchema, AgentAvailabilityResponseSchema, CursorChatStoreStatusSchema, @@ -10,6 +12,8 @@ import { ListPiSessionsRpcResponseSchema } from '@hapi/protocol/apiTypes' import type { + CodexGoalRequest, + CodexGoalResponse, AgyModelsResponse, AgentAvailabilityResponse, CodexModelSummary, @@ -242,7 +246,9 @@ export class RpcGateway { startingMode, forkSession: forkSession === true, ...(team ? { team } : {}) - } + }, + // Allow the runner's startup deadline to settle before the RPC expires. + Math.max(DEFAULT_RPC_TIMEOUT_MS, sessionStartupTimeoutMs(agent, resumeSessionId) + 10_000) ) if (result && typeof result === 'object') { const obj = result as Record @@ -566,6 +572,10 @@ export class RpcGateway { return await this.sessionRpc(sessionId, RPC_METHODS.ImplementCodexPlan, { planId }, 60_000) as ImplementCodexPlanResult } + async codexGoal(sessionId: string, request: CodexGoalRequest): Promise { + return CodexGoalResponseSchema.parse(await this.sessionRpc(sessionId, RPC_METHODS.CodexGoal, request)) + } + async rewindConversation( sessionId: string, params: { messageLocalId: string } diff --git a/hub/src/sync/syncEngine.ts b/hub/src/sync/syncEngine.ts index ce91eae1..8d50a8e6 100644 --- a/hub/src/sync/syncEngine.ts +++ b/hub/src/sync/syncEngine.ts @@ -13,13 +13,14 @@ import { isMachineCapabilitySkewed, runnerSupportsAgentTeam, } from '@hapi/protocol/runnerCapabilities' -import type { CursorChatStoreStatus, CursorMigrateOutcome, CursorMigrateToAcpRequest, MessageDeliveryMode, MessagesResponse, QueuedStateResponse, RewindConversationErrorCode, SlashCommandsResponse, ImplementCodexPlanResult } from '@hapi/protocol/apiTypes' +import type { CodexGoalRequest, CodexGoalResponse, CursorChatStoreStatus, CursorMigrateOutcome, CursorMigrateToAcpRequest, MessageDeliveryMode, MessagesResponse, QueuedStateResponse, RewindConversationErrorCode, SlashCommandsResponse, ImplementCodexPlanResult } from '@hapi/protocol/apiTypes' import type { SteerQueuedMessageResponse } from '@hapi/protocol/schemas' import type { AgentFlavor, CodexCollaborationMode, CopilotAgentMode, DecryptedMessage, PermissionMode, Session, SyncEvent } from '@hapi/protocol/types' import { hasConversationMessageContent, unwrapRoleWrappedRecordEnvelope } from '@hapi/protocol/messages' import type { Server } from 'socket.io' import { randomUUID } from 'node:crypto' import type { Store, CancelQueuedMessageResult } from '../store' +import type { FutureScheduledMessageStats } from '../store/messages' import type { HapiSessionExportResult } from '@hapi/protocol/sessionExport' import type { RpcRegistry } from '../socket/rpcRegistry' import { clearAgentTerminalBuffer } from '../socket/agentTerminalBuffer' @@ -359,12 +360,8 @@ export class SyncEngine { this.sessionCache.setSessionPinMode(sessionId, mode) } - getFutureScheduledMessageCounts(sessionIds: string[], now: number = Date.now()): Map { - return this.store.messages.countFutureScheduledBySessionIds(sessionIds, now) - } - - getNextScheduledAtBySessionIds(sessionIds: string[], now: number = Date.now()): Map { - return this.store.messages.minFutureScheduledAtBySessionIds(sessionIds, now) + getFutureScheduledMessageStats(sessionIds: string[], now: number = Date.now()): Map { + return this.store.messages.getFutureScheduledStatsBySessionIds(sessionIds, now) } getSession(sessionId: string): Session | undefined { @@ -1993,6 +1990,15 @@ export class SyncEngine { return await this.rpcGateway.implementCodexPlan(access.sessionId, planId) } + async codexGoal(sessionId: string, namespace: string, request: CodexGoalRequest): Promise { + const access = this.sessionCache.resolveSessionAccess(sessionId, namespace) + if (!access.ok || !access.session.active || access.session.metadata?.flavor !== 'codex' + || !access.session.metadata.capabilities?.codexGoal) { + throw new Error('Goal controls require an active, updated Codex session') + } + return await this.rpcGateway.codexGoal(access.sessionId, request) + } + async switchSession(sessionId: string, to: 'remote' | 'local'): Promise { if (this.getSession(sessionId)?.metadata?.capabilities?.concurrentClients) throw new Error('control_mode_not_applicable') if (this.historyActionsInFlight.has(sessionId)) { diff --git a/hub/src/sync/usageReconciliation.ts b/hub/src/sync/usageReconciliation.ts index c6986ae3..88fd25a9 100644 --- a/hub/src/sync/usageReconciliation.ts +++ b/hub/src/sync/usageReconciliation.ts @@ -2,6 +2,7 @@ import { existsSync } from 'node:fs' import { readOpencodeUsageForSessions, resolveOpencodeDbPath } from '@hapi/protocol/opencodeUsage' import type { Store } from '../store' import type { ReconciledUsageRow } from '../store/usage' +import type { UsageSource } from '../store/usageSources' /** * Offline usage reconciliation for OpenCode sessions. @@ -22,7 +23,7 @@ import type { ReconciledUsageRow } from '../store/usage' * and cli/src/runner/opencodeUsageScanner.ts). */ -const RECONCILE_INTERVAL_MS = 30 * 60 * 1000 +const RECONCILE_INTERVAL_MS = 5 * 60 * 1000 const INITIAL_DELAY_MS = 15_000 export type UsageReconciliationResult = { @@ -53,19 +54,26 @@ export async function reconcileOpencodeUsage( return { dbPath: dbPath ?? null, sessions: 0, rows: 0, messages: 0 } } - const targets: Array<{ sessionId: string; namespace: string; opencodeSessionId: string }> = [] + const targets = new Map() for (const session of store.sessions.getSessions()) { + store.usage.rememberSessionSources(session) const opencodeSessionId = asString(asRecord(session.metadata)?.opencodeSessionId) if (!opencodeSessionId) continue - targets.push({ sessionId: session.id, namespace: session.namespace, opencodeSessionId }) + const source = store.usage.getSources(session.namespace).find((entry) => entry.agent === 'opencode' && entry.sessionId === session.id && entry.nativeSessionId === opencodeSessionId) + if (source) targets.set(`${source.namespace}|${source.machineId}|${source.nativeSessionId}`, source) + } + for (const source of store.usage.getSources()) { + if (source.agent !== 'opencode') continue + const key = `${source.namespace}|${source.machineId}|${source.nativeSessionId}` + if (!targets.has(key)) targets.set(key, source) } - const usageBySession = await readOpencodeUsageForSessions(dbPath, targets.map((target) => target.opencodeSessionId)) + const usageBySession = await readOpencodeUsageForSessions(dbPath, [...targets.values()].map((target) => target.nativeSessionId)) let sessionsTouched = 0 let rowsTotal = 0 let messagesTotal = 0 - for (const target of targets) { - const usage = usageBySession.get(target.opencodeSessionId) ?? { rows: [], messages: 0 } + for (const target of targets.values()) { + const usage = usageBySession.get(target.nativeSessionId) ?? { rows: [], messages: 0 } // No local messages means this session's OpenCode store lives on // another machine (its own process reports the rows) — leave the // existing snapshot alone instead of wiping it with an empty one. @@ -75,7 +83,7 @@ export async function reconcileOpencodeUsage( agent: 'opencode', ...row })) - store.usage.replaceReconciled(target.sessionId, target.namespace, rows, now) + store.usage.reconcileSource(target, rows, now) sessionsTouched += 1 rowsTotal += rows.length messagesTotal += usage.messages diff --git a/hub/src/sync/usageRecovery.test.ts b/hub/src/sync/usageRecovery.test.ts new file mode 100644 index 00000000..1973c45a --- /dev/null +++ b/hub/src/sync/usageRecovery.test.ts @@ -0,0 +1,179 @@ +import { describe, expect, it } from 'bun:test' +import type { Database } from 'bun:sqlite' +import { Store } from '../store' +import type { UsageEvent } from '../store/usage' +import { mergeRecoveredUsage } from './usageRecovery' +import { getUsageSummary } from './usageService' + +function event(key: string, sessionId = 'deleted', extra: Partial = {}): UsageEvent { + return { sessionId, sourceKey: key, sourceSeq: 1, createdAt: Date.now() - 1000, + agent: 'dsh', model: null, kind: 'delta', inputTokens: 100, outputTokens: 10, + cacheReadTokens: 80, cacheCreationTokens: 0, lastInputTokens: null, lastOutputTokens: null, + lastCacheReadTokens: null, lastCacheCreationTokens: null, ...extra } +} + +describe('usage recovery', () => { + it('can resume an interrupted backfill without advancing its cursor or duplicating usage', () => { + const store = new Store(':memory:') + const db = (store as unknown as { db: Database }).db + const events = Array.from({ length: 300 }, (_, index) => event(`delta|${index}`, 'deleted', { sourceSeq: index + 1 })) + db.exec(`CREATE TRIGGER interrupt_backfill BEFORE INSERT ON usage_events + WHEN NEW.source_key = 'delta|257' BEGIN SELECT RAISE(ABORT, 'interrupted'); END`) + expect(() => store.usage.recordScan('deleted', 'default', 0, 300, events, false)).toThrow('interrupted') + expect(store.usage.getScanStates(['deleted']).size).toBe(0) + expect(store.usage.getEvents(['deleted']).length).toBeGreaterThan(0) + db.exec('DROP TRIGGER interrupt_backfill') + store.usage.recordScan('deleted', 'default', 0, 300, events, false) + expect(store.usage.getScanStates(['deleted']).get('deleted')?.lastSeq).toBe(300) + expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(300) + store.close() + }) + + it('matches identical requests one-to-one and remains idempotent', () => { + const store = new Store(':memory:') + store.usage.recordScan('deleted', 'default', 0, 0, [event('delta|a')], false) + const recovered = [event('native|a'), event('native|b')] + expect(mergeRecoveredUsage(store, 'default', recovered)).toEqual({ inserted: 1, matched: 1, enriched: 0 }) + expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(2) + expect(mergeRecoveredUsage(store, 'default', recovered).inserted).toBe(0) + expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(2) + store.close() + }) + + it('indexes wrapped Codex child usage and ignores replay under a different scope label', () => { + const store = new Store(':memory:') + const session = store.sessions.getOrCreateSession('child-parent', { flavor: 'codex' }, null, 'default') + const data = { type: 'token_count', thread_id: 'child-thread', info: { + total_token_usage: { input_tokens: 100, output_tokens: 10, cached_input_tokens: 80 }, + last_token_usage: { input_tokens: 100, output_tokens: 10, cached_input_tokens: 80 } + }, model: 'child-model' } + store.messages.addMessage(session.id, { role: 'agent', content: { type: 'codex', data: { + type: 'agent-run-trace', scope: { role: 'child', threadId: 'child-thread' }, message: data + } } }) + store.messages.addMessage(session.id, { role: 'agent', content: { type: 'codex', data } }) + store.messages.addMessage(session.id, { role: 'agent', content: { type: 'codex', data: { + type: 'agent-run-trace', hapiUsageScope: 'imported-history', message: { ...data, thread_id: 'pre-hapi-child' } + } } }) + expect(store.usage.getEvents([session.id])).toHaveLength(2) + store.sessions.deleteSession(session.id, 'default') + const summary = getUsageSummary(store, 'default', 'all') + expect(summary.totals).toMatchObject({ inputTokens: 100, outputTokens: 10, requests: 1 }) + expect(summary.byModel[0]?.key).toBe('child-model') + store.close() + }) + + it('enriches a partial Claude update across session aliases without adding a request', () => { + const store = new Store(':memory:') + store.usage.recordScan('deleted', 'default', 0, 0, [event('claude|msg', 'deleted', { agent: 'claude', outputTokens: 2 })], false) + expect(mergeRecoveredUsage(store, 'default', [event('claude|msg', 'alias', { agent: 'claude', model: 'claude-test' })])) + .toEqual({ inserted: 0, matched: 1, enriched: 1 }) + expect(getUsageSummary(store, 'default', 'all').totals).toMatchObject({ requests: 1, outputTokens: 10 }) + store.close() + }) + + it('replaces an estimated model only when the recovered provider record identifies its actual model', () => { + const store = new Store(':memory:') + store.usage.recordScan('deleted', 'default', 0, 0, [event('delta|a', 'deleted', { model: 'estimated' })], false) + const recovered = [event('native|a', 'deleted', { model: 'actual' })] + expect(mergeRecoveredUsage(store, 'default', recovered).enriched).toBe(0) + expect(mergeRecoveredUsage(store, 'default', recovered, new Set(['dsh|native|a'])).enriched).toBe(1) + const summary = getUsageSummary(store, 'default', 'all') + expect(summary.totals.requests).toBe(1) + expect(summary.byModel[0]?.key).toBe('actual') + store.close() + }) + + it('matches Codex snapshots despite turn-id wire differences and preserves resets', () => { + const store = new Store(':memory:') + const first = event('cumulative|thread|parent||100|10|80|0', 'deleted', { + agent: 'codex', kind: 'cumulative', lastInputTokens: 100, lastOutputTokens: 10, + lastCacheReadTokens: 80, lastCacheCreationTokens: 0 + }) + store.usage.recordScan('deleted', 'default', 0, 0, [first], false) + const recovered = [{ ...first, sourceKey: 'cumulative|thread|parent|turn-1|100|10|80|0' }, + { ...first, sourceKey: 'cumulative|thread|parent|turn-2|50|5|40|0', sourceSeq: 2, + createdAt: first.createdAt + 100, inputTokens: 50, outputTokens: 5, cacheReadTokens: 40, + lastInputTokens: 50, lastOutputTokens: 5, lastCacheReadTokens: 40 }] + expect(mergeRecoveredUsage(store, 'default', recovered).inserted).toBe(1) + expect(getUsageSummary(store, 'default', 'all').totals).toMatchObject({ requests: 2, inputTokens: 150, outputTokens: 15 }) + expect(mergeRecoveredUsage(store, 'default', recovered).inserted).toBe(0) + store.close() + }) + + it('keeps native snapshots through deletion and merging multiple native sources', () => { + const store = new Store(':memory:') + const a = store.sessions.getOrCreateSession('a', { machineId: 'm', flavor: 'opencode', opencodeSessionId: 'native-a' }, null, 'default') + const b = store.sessions.getOrCreateSession('b', { machineId: 'm', flavor: 'opencode', opencodeSessionId: 'native-b' }, null, 'default') + const rows = [{ day: '2026-09-17', agent: 'opencode', model: 'test', inputTokens: 100, + outputTokens: 10, cacheReadTokens: 80, cacheCreationTokens: 0, requests: 1 }] + for (const source of store.usage.getSources()) store.usage.reconcileSource(source, rows, Date.now()) + store.usage.recordScan(a.id, 'default', 0, 0, [event('delta|a', a.id, { agent: 'opencode' })], false) + store.usage.transferSession(a.id, b.id) + store.sessions.deleteSession(a.id, 'default') + store.sessions.deleteSession(b.id, 'default') + expect(store.usage.getSources().filter((source) => source.sessionId === b.id)).toHaveLength(2) + expect(store.usage.getEventsByNamespace('default')).toHaveLength(1) + expect(store.usage.getEventsByNamespace('default')[0]?.sessionId).toBe(b.id) + expect(getUsageSummary(store, 'default', 'all').totals).toMatchObject({ requests: 2, inputTokens: 200, sessions: 1 }) + for (const source of store.usage.getSources()) store.usage.reconcileSource(source, rows, Date.now()) + expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(2) + store.close() + }) + + it('does not recreate transferred usage when a copied source is deleted', () => { + const store = new Store(':memory:') + const a = store.sessions.getOrCreateSession('source', { flavor: 'claude' }, null, 'default') + const b = store.sessions.getOrCreateSession('target', { flavor: 'claude' }, null, 'default') + const content = { role: 'agent', content: { type: 'output', data: { + type: 'assistant', message: { id: 'provider-id', usage: { input_tokens: 100, output_tokens: 10 } } + } } } + for (const session of [a, b]) store.messages.copyMessageToSession(session.id, { + content, createdAt: Date.now() - 100, localId: null, invokedAt: null, scheduledAt: null + }) + store.usage.transferSession(a.id, b.id) + store.sessions.deleteSession(a.id, 'default') + expect(getUsageSummary(store, 'default', 'all').totals).toMatchObject({ inputTokens: 100, requests: 1 }) + expect(store.usage.getEvents([a.id])).toHaveLength(0) + store.close() + }) + + it('ignores a snapshot collected before an already applied report', () => { + const store = new Store(':memory:') + const source = { namespace: 'default', machineId: 'm', agent: 'opencode', nativeSessionId: 'n', sessionId: 'deleted' } + const row = { day: '2026-09-17', agent: 'opencode', model: 'test', inputTokens: 100, + outputTokens: 10, cacheReadTokens: 80, cacheCreationTokens: 0, requests: 1 } + store.usage.reconcileSource(source, [{ ...row, requests: 2 }], 200) + store.usage.reconcileSource(source, [row], 100) + expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(2) + store.close() + }) + + it('retains both legacy snapshots when their day/model keys collide during merge', () => { + const store = new Store(':memory:') + const a = store.sessions.getOrCreateSession('legacy-a', { flavor: 'opencode' }, null, 'default') + const b = store.sessions.getOrCreateSession('legacy-b', { flavor: 'opencode' }, null, 'default') + const row = { day: '2026-09-17', agent: 'opencode', model: 'test', inputTokens: 100, + outputTokens: 10, cacheReadTokens: 80, cacheCreationTokens: 0, requests: 1 } + for (const session of [a, b]) { + store.usage.replaceReconciled(session.id, 'default', [{ ...row, sessionId: session.id }], Date.now()) + store.usage.recordScan(session.id, 'default', 0, 0, [event(`delta|${session.id}`, session.id, { agent: 'opencode' })], false) + } + store.usage.transferSession(a.id, b.id) + store.sessions.deleteSession(a.id, 'default') + expect(getUsageSummary(store, 'default', 'all').totals).toMatchObject({ inputTokens: 200, requests: 2 }) + store.close() + }) + + it('counts a native snapshot once when an older hub recreates its legacy alias', () => { + const store = new Store(':memory:') + const source = { namespace: 'default', machineId: 'm', agent: 'opencode', nativeSessionId: 'n', sessionId: 'old-session' } + const row = { day: '2026-09-17', agent: 'opencode', model: 'test', inputTokens: 100, + outputTokens: 10, cacheReadTokens: 80, cacheCreationTokens: 0, requests: 1 } + store.usage.reconcileSource(source, [row], 100) + store.usage.replaceReconciled(source.sessionId, 'default', [{ ...row, sessionId: source.sessionId }], 200) + expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(1) + store.usage.reconcileSource(source, [row], 150) + expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(1) + store.close() + }) +}) diff --git a/hub/src/sync/usageRecovery.ts b/hub/src/sync/usageRecovery.ts new file mode 100644 index 00000000..ec251d78 --- /dev/null +++ b/hub/src/sync/usageRecovery.ts @@ -0,0 +1,77 @@ +import type { Store } from '../store' +import type { UsageEvent } from '../store/usage' + +export type UsageRecoveryResult = { inserted: number; matched: number; enriched: number } + +function counters(event: UsageEvent): string { + return [event.agent, event.inputTokens, event.outputTokens, event.cacheReadTokens, event.cacheCreationTokens].join('|') +} + +function cumulativeFingerprint(event: UsageEvent): string { + return `${event.sourceKey.split('|')[1]}|${counters(event)}` +} + +/** Merge native/backup accounting without adding another copy of recorded requests. */ +export function mergeRecoveredUsage( + store: Store, + namespace: string, + recovered: UsageEvent[], + authoritativeModels: ReadonlySet = new Set() +): UsageRecoveryResult { + const existing = store.usage.getEventsByNamespace(namespace) + const stableKeys = new Map(existing.map((event) => [`${event.agent}|${event.sourceKey}`, event])) + const cumulative = new Map() + const deltas = new Map() + for (const event of existing) { + const index = event.kind === 'cumulative' ? cumulative : deltas + const key = event.kind === 'cumulative' ? cumulativeFingerprint(event) : `${event.sessionId}|${counters(event)}` + const values = index.get(key) ?? [] + values.push(event) + index.set(key, values) + } + const consumed = new Set() + const writes = new Map() + const result: UsageRecoveryResult = { inserted: 0, matched: 0, enriched: 0 } + for (const event of recovered) { + if (event.inputTokens + event.outputTokens <= 0) continue + const identity = `${event.agent}|${event.sourceKey}` + const exact = stableKeys.get(identity) + const candidates = event.kind === 'cumulative' + ? cumulative.get(cumulativeFingerprint(event)) + : event.agent === 'claude' ? undefined : deltas.get(`${event.sessionId}|${counters(event)}`) + const match = exact ?? candidates?.find((candidate) => !consumed.has(`${candidate.sessionId}|${candidate.sourceKey}`)) + if (match) { + consumed.add(`${match.sessionId}|${match.sourceKey}`) + result.matched += 1 + const moreCompleteDelta = event.kind === 'delta' && exact && ( + event.inputTokens > match.inputTokens || event.outputTokens > match.outputTokens + || event.cacheReadTokens > match.cacheReadTokens || event.cacheCreationTokens > match.cacheCreationTokens + ) + if (moreCompleteDelta || ((!match.model || match.model === 'unknown') && event.model) + || (authoritativeModels.has(identity) && event.model !== null && event.model !== match.model)) { + const values = writes.get(match.sessionId) ?? [] + values.push({ + ...match, + model: event.model ?? match.model, + inputTokens: Math.max(match.inputTokens, event.inputTokens), + outputTokens: Math.max(match.outputTokens, event.outputTokens), + cacheReadTokens: Math.max(match.cacheReadTokens, event.cacheReadTokens), + cacheCreationTokens: Math.max(match.cacheCreationTokens, event.cacheCreationTokens) + }) + writes.set(match.sessionId, values) + result.enriched += 1 + } + continue + } + const values = writes.get(event.sessionId) ?? [] + values.push(event) + writes.set(event.sessionId, values) + stableKeys.set(identity, event) + result.inserted += 1 + } + for (const [sessionId, events] of writes) { + const state = store.usage.getScanStates([sessionId]).get(sessionId) + store.usage.recordScan(sessionId, namespace, state?.messageEpoch ?? 0, state?.lastSeq ?? 0, events, false) + } + return result +} diff --git a/hub/src/sync/usageService.test.ts b/hub/src/sync/usageService.test.ts index 501954b7..39614017 100644 --- a/hub/src/sync/usageService.test.ts +++ b/hub/src/sync/usageService.test.ts @@ -615,14 +615,8 @@ describe('usage service', () => { ) store.messages.addMessage(session.id, { role: 'user', content: { type: 'text', text: 'hello' } }) - const afterSeqs: number[] = [] - const getMessagesAfterSeq = store.messages.getMessagesAfterSeq.bind(store.messages) - store.messages.getMessagesAfterSeq = (sessionId, afterSeq) => { - afterSeqs.push(afterSeq) - return getMessagesAfterSeq(sessionId, afterSeq) - } - expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(0) + expect(store.usage.getScanStates([session.id]).get(session.id)).toEqual({ messageEpoch: 0, lastSeq: 1 }) addAgentMessage(store, session.id, { type: 'output', data: { @@ -634,14 +628,15 @@ describe('usage service', () => { } }) expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(1) + expect(store.usage.getScanStates([session.id]).get(session.id)).toEqual({ messageEpoch: 0, lastSeq: 2 }) - store.messages.bumpMessageEpoch(session.id) + const messageEpoch = store.messages.bumpMessageEpoch(session.id) expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(1) - expect(afterSeqs).toEqual([0, 1, 0]) + expect(store.usage.getScanStates([session.id]).get(session.id)).toEqual({ messageEpoch, lastSeq: 2 }) store.close() }) - it('removes usage from transcript history discarded by a rewind', () => { + it('retains consumed usage when transcript history is discarded by a rewind', () => { const store = new Store(':memory:') const session = store.sessions.getOrCreateSession( 'rewound-usage-test', @@ -672,8 +667,8 @@ describe('usage service', () => { const result = store.messages.truncateMessagesFromLocalId(session.id, 'rewind-point') expect(result.deleted).toBe(2) - expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(1) - expect(store.usage.getEvents([session.id])).toHaveLength(1) + expect(getUsageSummary(store, 'default', 'all').totals.requests).toBe(2) + expect(store.usage.getEvents([session.id])).toHaveLength(2) store.close() }) diff --git a/hub/src/sync/usageService.ts b/hub/src/sync/usageService.ts index 32ae8b65..0b77bd63 100644 --- a/hub/src/sync/usageService.ts +++ b/hub/src/sync/usageService.ts @@ -1,269 +1,7 @@ import type { UsageSummaryBucket, UsageSummaryResponse } from '@hapi/protocol/apiTypes' -import type { StoredMessage, StoredSession } from '../store' import type { UsageEvent } from '../store/usage' import type { Store } from '../store' - -type RecordValue = Record - -function asRecord(value: unknown): RecordValue | null { - return value !== null && typeof value === 'object' && !Array.isArray(value) - ? value as RecordValue - : null -} - -function asCount(value: unknown): number | null { - return typeof value === 'number' && Number.isFinite(value) && value >= 0 - ? Math.floor(value) - : null -} - -function firstCount(record: RecordValue, ...keys: string[]): number { - for (const key of keys) { - const value = asCount(record[key]) - if (value !== null) return value - } - return 0 -} - -function normalizeInputTokens( - data: RecordValue, - inputTokens: number, - cacheReadTokens: number, - cacheCreationTokens: number, - legacySemantics: 'includes-cache' | 'excludes-cache' -): number { - // v1 generic usage messages make their input contract self-describing. - // Unknown/missing metadata intentionally falls back to the historical - // provider shape so already persisted transcripts remain readable. - const declaredSemantics = data.usageSchema === 'hapi.usage.v1' - && (data.inputTokenSemantics === 'includes-cache' || data.inputTokenSemantics === 'excludes-cache') - ? data.inputTokenSemantics - : null - const semantics = declaredSemantics ?? legacySemantics - return semantics === 'excludes-cache' - ? inputTokens + cacheReadTokens + cacheCreationTokens - : inputTokens -} - -function sessionAgent(session: StoredSession): string { - const metadata = asRecord(session.metadata) - const flavor = metadata?.flavor - return typeof flavor === 'string' && flavor.trim() ? flavor.trim() : 'unknown' -} - -function sessionModel(session: StoredSession): string | null { - return typeof session.model === 'string' && session.model.trim() ? session.model.trim() : null -} - -function parseUsageEvent(session: StoredSession, message: StoredMessage): UsageEvent | null { - const envelope = asRecord(message.content) - if (envelope?.role !== 'agent') return null - - const payload = asRecord(envelope.content) - if (!payload) return null - const data = asRecord(payload.data) - if (!data) return null - - // Claude stream-json/SDK messages. A stream emits several updates for one - // assistant message, so the provider's message id is the stable upsert key. - if (payload.type === 'output' && data.type === 'assistant') { - const assistant = asRecord(data.message) - const usage = asRecord(assistant?.usage) - if (!usage) return null - const inputTokens = firstCount(usage, 'input_tokens', 'inputTokens') - const outputTokens = firstCount(usage, 'output_tokens', 'outputTokens') - const cacheReadTokens = firstCount(usage, 'cache_read_input_tokens', 'cacheReadTokens', 'cachedInputTokens') - const cacheCreationTokens = firstCount(usage, 'cache_creation_input_tokens', 'cacheCreationTokens', 'cacheWriteInputTokens') - if (inputTokens + outputTokens + cacheReadTokens + cacheCreationTokens <= 0) return null - const providerId = typeof assistant?.id === 'string' ? assistant.id : message.id - const model = typeof assistant?.model === 'string' && assistant.model.trim() - ? assistant.model.trim() - : null - return { - sessionId: session.id, - sourceKey: `claude|${providerId}`, - sourceSeq: message.seq, - createdAt: message.createdAt, - agent: 'claude', - model, - kind: 'delta', - inputTokens: normalizeInputTokens(data, inputTokens, cacheReadTokens, cacheCreationTokens, 'excludes-cache'), - outputTokens, - cacheReadTokens, - cacheCreationTokens, - lastInputTokens: null, - lastOutputTokens: null, - lastCacheReadTokens: null, - lastCacheCreationTokens: null - } - } - - // Codex forwards cumulative thread totals plus the most recent request. - // ACP-compatible backends wrap per-request usage in `total`, so only Codex - // should be diffed as a cumulative stream. - if (data.type === 'token_count' || data.type === 'usage') { - if (data.hapiUsageScope === 'imported-history') return null - const info = asRecord(data.info) ?? data - const agent = sessionAgent(session) - const explicitThreadId = typeof data.threadId === 'string' - ? data.threadId - : typeof data.thread_id === 'string' - ? data.thread_id - : null - const metadata = asRecord(session.metadata) - const hasImportedCodexHistory = typeof metadata?.codexSourceSessionId === 'string' - || metadata?.lifecycleState === 'imported' - if (agent === 'codex' && explicitThreadId === null && hasImportedCodexHistory) { - return null - } - const cumulativeTotal = agent === 'codex' - ? asRecord(info.total) - ?? asRecord(info.total_token_usage) - ?? asRecord(info.totalTokenUsage) - : null - const last = asRecord(info.last) - ?? asRecord(info.last_token_usage) - ?? asRecord(info.lastTokenUsage) - ?? (data.type === 'usage' ? info : null) - const total = cumulativeTotal ?? (agent === 'codex' ? last : asRecord(info.total) ?? info) - if (!total) return null - const rawInputTokens = firstCount(total, 'inputTokens', 'input_tokens') - const outputTokens = firstCount(total, 'outputTokens', 'output_tokens') - const cacheReadTokens = firstCount(total, 'cachedInputTokens', 'cached_input_tokens', 'cacheReadTokens', 'cache_read_input_tokens') - const cacheCreationTokens = firstCount(total, 'cacheWriteInputTokens', 'cache_write_input_tokens', 'cacheCreationTokens', 'cache_creation_input_tokens') - if (rawInputTokens + outputTokens + cacheReadTokens + cacheCreationTokens <= 0) return null - const threadId = explicitThreadId ?? session.id - const scope = typeof data.scopeRole === 'string' - ? data.scopeRole - : typeof data.scope_role === 'string' - ? data.scope_role - : 'parent' - const isCumulative = cumulativeTotal !== null - const turnId = typeof data.turnId === 'string' - ? data.turnId - : typeof data.turn_id === 'string' - ? data.turn_id - : '' - const model = typeof data.model === 'string' && data.model.trim() - ? data.model.trim() - : null - // Codex/Kimi provider formats have always reported inclusive input. - // Imported Pi usage is known-inclusive, and for generic ACP an own - // `model` property is the only strong provenance for the unmarked - // inclusive wire introduced with the usage dashboard. Older ambiguous - // payloads are conservatively treated as cache-exclusive. - const legacyInputSemantics = agent === 'codex' - || agent === 'kimi' - || (agent === 'pi' && message.localId?.startsWith('pi:')) - || Object.prototype.hasOwnProperty.call(data, 'model') - ? 'includes-cache' - : 'excludes-cache' - const inputTokens = normalizeInputTokens( - data, - rawInputTokens, - cacheReadTokens, - cacheCreationTokens, - legacyInputSemantics - ) - const lastOutputTokens = last ? firstCount(last, 'outputTokens', 'output_tokens') : null - const lastCacheReadTokens = last - ? firstCount(last, 'cachedInputTokens', 'cached_input_tokens', 'cacheReadTokens', 'cache_read_input_tokens') - : null - const lastCacheCreationTokens = last - ? firstCount(last, 'cacheWriteInputTokens', 'cache_write_input_tokens', 'cacheCreationTokens', 'cache_creation_input_tokens') - : null - const lastInputTokens = last - ? normalizeInputTokens( - data, - firstCount(last, 'inputTokens', 'input_tokens'), - lastCacheReadTokens ?? 0, - lastCacheCreationTokens ?? 0, - legacyInputSemantics - ) - : null - return { - sessionId: session.id, - sourceKey: isCumulative - ? [ - 'cumulative', - threadId, - scope, - turnId, - inputTokens, - outputTokens, - cacheReadTokens, - cacheCreationTokens - ].join('|') - : `delta|${message.id}`, - sourceSeq: message.seq, - createdAt: message.createdAt, - agent, - model, - kind: isCumulative ? 'cumulative' : 'delta', - inputTokens, - outputTokens, - cacheReadTokens, - cacheCreationTokens, - lastInputTokens, - lastOutputTokens, - lastCacheReadTokens, - lastCacheCreationTokens - } - } - - return null -} - -function collectUsageEvents(store: Store, sessions: StoredSession[]): void { - const scanStates = store.usage.getScanStates(sessions.map((session) => session.id)) - for (const session of sessions) { - const messageEpoch = store.messages.getMessageEpoch(session.id) - const scanState = scanStates.get(session.id) - const replaceEvents = !scanState || scanState.messageEpoch !== messageEpoch - const afterSeq = replaceEvents ? 0 : scanState.lastSeq - const messages = store.messages.getMessagesAfterSeq(session.id, afterSeq) - const events = new Map() - let indexedModels: Map | null = null - const getIndexedModel = (sourceKey: string): string | null => { - if (indexedModels === null) { - indexedModels = new Map( - store.usage.getEvents([session.id]) - .filter((event): event is UsageEvent & { model: string } => event.model !== null) - .map((event) => [event.sourceKey, event.model]) - ) - } - return indexedModels.get(sourceKey) ?? null - } - const fallbackModel = sessionModel(session) - for (const message of messages) { - const event = parseUsageEvent(session, message) - if (!event) continue - const existingEvent = events.get(event.sourceKey) - const explicitModel = event.model - event.model = explicitModel - ?? existingEvent?.model - ?? getIndexedModel(event.sourceKey) - ?? fallbackModel - if (event.kind === 'delta' || !existingEvent) { - events.set(event.sourceKey, event) - } else if (explicitModel !== null) { - // A replay may add model metadata missing from the original snapshot. - existingEvent.model = explicitModel - } - } - const lastSeq = messages.at(-1)?.seq ?? afterSeq - if (messages.length > 0 || replaceEvents) { - store.usage.recordScan( - session.id, - session.namespace, - messageEpoch, - lastSeq, - Array.from(events.values()), - replaceEvents - ) - } - } -} +import { usageSourceLedgerId } from '../store/usageSources' type Totals = Omit @@ -344,10 +82,9 @@ export function getUsageSummary( timeZone: string = 'UTC' ): UsageSummaryResponse { const sessions = store.sessions.getSessionsByNamespace(namespace) - // This is intentionally lazy. Existing HAPI databases have no usage table; - // the first dashboard request backfills history, while later requests only - // update the idempotent event rows. - collectUsageEvents(store, sessions) + // New messages are indexed on insertion. Backfill history from older hubs + // and refresh usage after structural transcript changes. + store.usage.collectSessions(sessions) const now = Date.now() const days = range === '30d' ? 30 : range === 'all' ? null : 7 @@ -359,8 +96,41 @@ export function getUsageSummary( // same session: the live pipeline under-counts turns that report usage // only for their final step, and mixing both sources would double count // the steps it did see. See usageReconciliation.ts. - const reconciledRows = store.usage.getReconciledByNamespace(namespace) - const reconciledSessionIds = new Set(reconciledRows.map((row) => row.sessionId)) + const allReconciledRows = store.usage.getReconciledByNamespace(namespace) + const snapshotIds = new Set(allReconciledRows.map((row) => `${row.sessionId}|${row.agent}`)) + const sources = store.usage.getSources(namespace) + // Native ledger ids and old/resumed HAPI ids are aliases of a billed + // conversation. Count that conversation once across merges and clears. + const sessionAliases = new Map() + const canonicalSession = (id: string): string => { + const parent = sessionAliases.get(id) + if (!parent || parent === id) return id + const root = canonicalSession(parent) + sessionAliases.set(id, root) + return root + } + for (const source of sources) { + const native = canonicalSession(usageSourceLedgerId(source)) + const session = canonicalSession(source.sessionId) + if (native !== session) sessionAliases.set(native, session) + } + const supersededLegacySnapshots = new Set(sources + .filter((source) => snapshotIds.has(`${usageSourceLedgerId(source)}|${source.agent}`)) + .map((source) => `${source.sessionId}|${source.agent}`)) + // A previous hub can leave a newer session-keyed report beside a restored + // native snapshot. They describe the same requests, so count the native + // source once while the next reconciliation refreshes its full totals. + const reconciledRows = allReconciledRows.filter((row) => !supersededLegacySnapshots.has(`${row.sessionId}|${row.agent}`)) + const reconciledSessions = new Set(reconciledRows.map((row) => `${row.sessionId}|${row.agent}`)) + const reconciledSources = new Set(sources + .filter((source) => reconciledSessions.has(`${source.sessionId}|${source.agent}`) + || reconciledSessions.has(`${usageSourceLedgerId(source)}|${source.agent}`)) + .map((source) => `${source.machineId}|${source.agent}|${source.nativeSessionId}`)) + for (const source of sources) { + if (reconciledSources.has(`${source.machineId}|${source.agent}|${source.nativeSessionId}`)) { + reconciledSessions.add(`${source.sessionId}|${source.agent}`) + } + } const isInRange = (event: UsageEvent) => (from === null || event.createdAt >= from) && event.createdAt <= now const totals = emptyTotals() @@ -382,7 +152,10 @@ export function getUsageSummary( const sourceParts = event.sourceKey.split('|') // Provider thread ids are only unique within a HAPI session. Keep // deleted-session history from altering a newer session's delta. - const streamKey = `${event.sessionId}|${sourceParts.slice(0, 3).join('|')}` + // Parent/child labels describe presentation, not distinct billing + // streams. Native thread identity keeps replayed traces from + // starting another cumulative baseline under a different label. + const streamKey = `${event.sessionId}|${sourceParts.slice(0, 2).join('|')}` const previous = cumulativePrevious.get(streamKey) ?? null const current: UsageSnapshot = [ event.inputTokens, @@ -429,7 +202,7 @@ export function getUsageSummary( // Skip only the live OpenCode rows of a reconciled session: the // snapshot replaces them. Other agents' events on the same session // (e.g. after a flavor switch) still count. - if (event.agent === 'opencode' && reconciledSessionIds.has(event.sessionId)) continue + if (reconciledSessions.has(`${event.sessionId}|${event.agent}`)) continue // Cache reads and writes partition processed input. Preserve the // request and its primary token counts when a provider emits an // impossible partition, but conservatively decline to credit either @@ -450,7 +223,7 @@ export function getUsageSummary( const modelTotals = byModel.get(modelKey) ?? emptyTotals() addTotals(modelTotals, inputTokens, outputTokens, cacheReadTokens, cacheCreationTokens) byModel.set(modelKey, modelTotals) - sessionsWithUsage.add(event.sessionId) + sessionsWithUsage.add(canonicalSession(event.sessionId)) } // Reconciliation rows are absolute per-day totals, so range filtering @@ -472,7 +245,7 @@ export function getUsageSummary( const modelTotals = byModel.get(row.model) ?? emptyTotals() addTotals(modelTotals, row.inputTokens, row.outputTokens, row.cacheReadTokens, row.cacheCreationTokens, row.requests) byModel.set(row.model, modelTotals) - sessionsWithUsage.add(row.sessionId) + sessionsWithUsage.add(canonicalSession(row.sessionId)) } const sortBuckets = (values: Map): UsageSummaryBucket[] => Array.from(values.entries()) diff --git a/hub/src/web/routes/sessionShares.test.ts b/hub/src/web/routes/sessionShares.test.ts index e180ff91..5a47503c 100644 --- a/hub/src/web/routes/sessionShares.test.ts +++ b/hub/src/web/routes/sessionShares.test.ts @@ -70,8 +70,7 @@ describe("session share routes", () => { const engine = { resolveSessionAccess: () => ({ ok: true, sessionId: session.id, session }), getSessionsByNamespace: () => [session], - getFutureScheduledMessageCounts: () => new Map(), - getNextScheduledAtBySessionIds: () => new Map(), + getFutureScheduledMessageStats: () => new Map(), } as unknown as SyncEngine; const app = createApp(store, engine); const owner = await ownerToken(); @@ -122,8 +121,7 @@ describe("session share routes", () => { }, getSessionsByNamespace: (namespace: string) => [session, otherSession].filter((item) => item.namespace === namespace), - getFutureScheduledMessageCounts: () => new Map(), - getNextScheduledAtBySessionIds: () => new Map(), + getFutureScheduledMessageStats: () => new Map(), } as unknown as SyncEngine; const app = createApp(store, engine); const owner = await ownerToken(); @@ -271,8 +269,7 @@ describe("session share routes", () => { session, }), getSessionsByNamespace: () => [session], - getFutureScheduledMessageCounts: () => new Map(), - getNextScheduledAtBySessionIds: () => new Map(), + getFutureScheduledMessageStats: () => new Map(), } as unknown as SyncEngine; const app = createApp(store, engine); const owner = await ownerToken(); diff --git a/hub/src/web/routes/sessions.test.ts b/hub/src/web/routes/sessions.test.ts index bc1019ad..91aefae2 100644 --- a/hub/src/web/routes/sessions.test.ts +++ b/hub/src/web/routes/sessions.test.ts @@ -69,6 +69,8 @@ function createApp(session: Session, opts?: { forkConversation?: SyncEngine['forkConversation'] clearConversation?: SyncEngine['clearConversation'] implementCodexPlan?: SyncEngine['implementCodexPlan'] + codexGoal?: SyncEngine['codexGoal'] + guestSessionId?: string rewindConversation?: SyncEngine['rewindConversation'] suggestSessionTitle?: SyncEngine['suggestSessionTitle'] updateSessionSummary?: SyncEngine['updateSessionSummary'] @@ -172,6 +174,7 @@ function createApp(session: Session, opts?: { forkConversation: opts?.forkConversation ?? (async () => ({ type: 'success', sessionId: 'child-1' })), clearConversation: opts?.clearConversation, implementCodexPlan: opts?.implementCodexPlan, + codexGoal: opts?.codexGoal, rewindConversation: opts?.rewindConversation ?? (async () => ({ type: 'success' })), suggestSessionTitle: opts?.suggestSessionTitle ?? (async () => 'Generated title'), updateSessionSummary: opts?.updateSessionSummary ?? (async () => {}), @@ -181,6 +184,10 @@ function createApp(session: Session, opts?: { const app = new Hono() app.use('*', async (c, next) => { c.set('namespace', 'default') + if (opts?.guestSessionId) { + c.set('role', 'session-guest') + c.set('sessionId', opts.guestSessionId) + } await next() }) app.route('/api', createSessionsRoutes(() => engine as SyncEngine)) @@ -189,6 +196,50 @@ function createApp(session: Session, opts?: { } describe('sessions routes', () => { + it('routes Goal reads and changes to the owning session', async () => { + const calls: unknown[][] = [] + const session = createSession({ metadata: { path: '/tmp', host: 'test', flavor: 'codex', capabilities: { codexGoal: true } } }) + const { app } = createApp(session, { codexGoal: async (...args) => { calls.push(args); return { goal: null } } }) + expect((await app.request('/api/sessions/session-1/codex/goal')).status).toBe(200) + for (const action of [{ action: 'update', objective: 'Fix it', tokenBudget: 5000 }, { action: 'pause' }, { action: 'resume' }, { action: 'clear' }]) { + const response = await app.request('/api/sessions/session-1/codex/goal', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(action) + }) + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ goal: null }) + expect(calls.at(-1)).toEqual(['session-1', 'default', action]) + } + expect(calls[0]).toEqual(['session-1', 'default', { action: 'get' }]) + }) + + it('rejects unavailable, inactive and guest Goal controls before native RPC', async () => { + for (const [overrides, guestSessionId, status] of [ + [{ active: false }, undefined, 409], + [{}, undefined, 409], + [{ metadata: { path: '/tmp', host: 'test', flavor: 'claude', capabilities: { codexGoal: true } } }, undefined, 409], + [{ metadata: { path: '/tmp', host: 'test', flavor: 'codex', capabilities: { codexGoal: true } } }, 'session-1', 403] + ] as const) { + let called = false + const { app } = createApp(createSession(overrides), { guestSessionId, codexGoal: async () => { called = true; return { goal: null } } }) + const response = await app.request('/api/sessions/session-1/codex/goal', { method: 'POST', body: JSON.stringify({ action: 'clear' }) }) + expect(response.status).toBe(status) + expect(called).toBe(false) + } + }) + + it('validates Goal requests and surfaces native errors instead of claiming success', async () => { + const session = createSession({ metadata: { path: '/tmp', host: 'test', flavor: 'codex', capabilities: { codexGoal: true } } }) + let called = false + const { app } = createApp(session, { codexGoal: async () => { called = true; throw new Error('Goal update failed') } }) + for (const body of [null, { action: 'archive' }, { action: 'update', objective: '' }, { action: 'update', objective: 'Fix', tokenBudget: 1.5 }, { action: 'pause', threadId: 'other' }]) { + expect((await app.request('/api/sessions/session-1/codex/goal', { method: 'POST', body: JSON.stringify(body) })).status).toBe(400) + } + expect(called).toBe(false) + const response = await app.request('/api/sessions/session-1/codex/goal', { method: 'POST', body: JSON.stringify({ action: 'pause' }) }) + expect(response.status).toBe(502) + expect(await response.json()).toEqual({ error: 'Goal update failed' }) + }) + it('serves chat attachments with an immutable ETag and answers 304 revalidation', async () => { const session = createSession() const { app } = createApp(session, { @@ -1708,11 +1759,10 @@ describe('sessions routes', () => { const scheduledIds: string[][] = [] const engine = { getSessionsByNamespace: () => sessions, - getFutureScheduledMessageCounts: (ids: string[]) => { + getFutureScheduledMessageStats: (ids: string[]) => { scheduledIds.push(ids) - return new Map(ids.map((id) => [id, 0])) + return new Map([[ids[0]!, { count: 2, nextScheduledAt: 1000 }]]) }, - getNextScheduledAtBySessionIds: (_ids: string[]) => new Map(), resolveSessionAccess: () => ({ ok: false, reason: 'not-found' as const }) } as unknown as Partial @@ -1728,6 +1778,9 @@ describe('sessions routes', () => { const limitedBody = await limited.json() as { sessions: Array<{ id: string }> } expect(limitedBody.sessions.map((s) => s.id)).toEqual(['newer-active', 'older-active']) expect(scheduledIds.at(-1)).toEqual(['newer-active', 'older-active']) + expect(scheduledIds).toHaveLength(1) + expect(limitedBody.sessions[0]).toMatchObject({ futureScheduledMessageCount: 2, nextScheduledAt: 1000 }) + expect(limitedBody.sessions[1]).toMatchObject({ futureScheduledMessageCount: 0, nextScheduledAt: null }) const unlimited = await app.request('/api/sessions') expect(unlimited.status).toBe(200) @@ -1743,8 +1796,7 @@ describe('sessions routes', () => { ] const engine = { getSessionsByNamespace: () => sessions, - getFutureScheduledMessageCounts: (ids: string[]) => new Map(ids.map((id) => [id, 0])), - getNextScheduledAtBySessionIds: (_ids: string[]) => new Map(), + getFutureScheduledMessageStats: () => new Map(), resolveSessionAccess: () => ({ ok: false, reason: 'not-found' as const }) } as unknown as Partial diff --git a/hub/src/web/routes/sessions.ts b/hub/src/web/routes/sessions.ts index 42203a1f..3dd4cae0 100644 --- a/hub/src/web/routes/sessions.ts +++ b/hub/src/web/routes/sessions.ts @@ -1,4 +1,5 @@ import { + CodexGoalRequestSchema, CursorMigrateToAcpRequestSchema, DeleteUploadRequestSchema, ForkConversationRequestSchema, @@ -119,14 +120,14 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho if (limit !== null) { sessionRecords = sessionRecords.slice(0, limit) } - const scheduledCounts = engine.getFutureScheduledMessageCounts(sessionRecords.map((session) => session.id)) - const nextScheduledAt = engine.getNextScheduledAtBySessionIds(sessionRecords.map((session) => session.id)) + const scheduledStats = engine.getFutureScheduledMessageStats(sessionRecords.map((session) => session.id)) const sessions = sessionRecords.map((session) => { const summary = toSessionSummary(session) + const scheduled = scheduledStats.get(session.id) return { ...summary, - futureScheduledMessageCount: scheduledCounts.get(session.id) ?? 0, - nextScheduledAt: nextScheduledAt.get(session.id) ?? null + futureScheduledMessageCount: scheduled?.count ?? 0, + nextScheduledAt: scheduled?.nextScheduledAt ?? null } }) @@ -596,6 +597,25 @@ export function createSessionsRoutes(getSyncEngine: () => SyncEngine | null): Ho return c.json(outcome, status) }) + app.on(['GET', 'POST'], '/sessions/:id/codex/goal', async (c) => { + const engine = requireSyncEngine(c, getSyncEngine) + if (engine instanceof Response) return engine + const access = requireSessionFromParam(c, engine, { requireActive: true }) + if (access instanceof Response) return access + if (c.get('role') === 'session-guest') return c.json({ error: 'Goal controls require session owner access' }, 403) + if (access.session.metadata?.flavor !== 'codex' || !access.session.metadata.capabilities?.codexGoal) { + return c.json({ error: 'Goal controls require an updated Codex session', code: 'goal_unavailable' }, 409) + } + const parsed = CodexGoalRequestSchema.safeParse(c.req.method === 'GET' + ? { action: 'get' } : await c.req.json().catch(() => null)) + if (!parsed.success) return c.json({ error: 'Invalid Goal action' }, 400) + try { + return c.json(await engine.codexGoal(access.sessionId, c.get('namespace'), parsed.data)) + } catch (error) { + return c.json({ error: error instanceof Error ? error.message : 'Goal action failed' }, 502) + } + }) + app.post('/sessions/:id/codex/plan/implement', async (c) => { const engine = requireSyncEngine(c, getSyncEngine) if (engine instanceof Response) return engine diff --git a/scripts/deploy-local.sh b/scripts/deploy-local.sh index 94251070..6693ea14 100755 --- a/scripts/deploy-local.sh +++ b/scripts/deploy-local.sh @@ -26,6 +26,8 @@ if [ ! -x "$build" ]; then exit 1 fi +bash scripts/verify-deploy-build.sh "$build" + mkdir -p "$bin_dir" "$backup_dir" bash scripts/sign-build.sh "$build" @@ -82,6 +84,9 @@ echo "health ok" # Refresh the runner so its machine RPCs/capabilities match the new binary. # Compiled binaries never self-update: the heartbeat mtime check compares the # runner's own resolved exec path, which is fixed for the life of the process. +# The runner also inherits this shell's environment. Keep the full login PATH; +# scope temporary transfer-tool PATH overrides to the remote deploy command. +# A healthy hub does not prove the runner can resolve installed agent binaries. runner_state="$HOME/.hapi/runner.state.json" runner_pid=$(sed -n 's/.*"pid": *\([0-9][0-9]*\).*/\1/p' "$runner_state" 2>/dev/null | head -n 1 || true) if [ -n "${runner_pid:-}" ] && kill -0 "$runner_pid" 2>/dev/null; then diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 3f193dc2..b41033a9 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -32,6 +32,8 @@ if [ ! -x "$build" ]; then exit 1 fi +bash scripts/verify-deploy-build.sh "$build" + # Sign locally with the pinned identity so the remote keeps a stable signer. bash scripts/sign-build.sh "$build" @@ -48,7 +50,16 @@ echo "release: $stamp${tag:+-$tag}" # Upload next to the fixed path, then install by move (fresh inode + mtime). ssh "${ssh_opts[@]}" "$target" "mkdir -p '$remote_home/.hapi/bin/backups'" -scp -q -C "${ssh_opts[@]}" "$build" "$target:$remote_home/.hapi/bin/.hapi.incoming" +if command -v rsync >/dev/null 2>&1 && ssh "${ssh_opts[@]}" "$target" 'command -v rsync >/dev/null 2>&1'; then + # Seed the staging file from the installed binary so rsync can reuse its + # unchanged blocks. Only staging is modified; installation stays atomic. + ssh "${ssh_opts[@]}" "$target" "if [ ! -f '$remote_home/.hapi/bin/.hapi.incoming' ] && [ -f '$remote_home/.hapi/bin/hapi' ]; then cp -p '$remote_home/.hapi/bin/hapi' '$remote_home/.hapi/bin/.hapi.incoming'; fi" + rsync -az --no-whole-file --inplace --stats \ + -e 'ssh -o BatchMode=yes -o ConnectTimeout=10' \ + "$build" "$target:$remote_home/.hapi/bin/.hapi.incoming" +else + scp -q -C "${ssh_opts[@]}" "$build" "$target:$remote_home/.hapi/bin/.hapi.incoming" +fi ssh "${ssh_opts[@]}" "$target" "STAMP='$stamp' TAG='$tag' LABEL='$label' bash -s" <<'REMOTE' set -euo pipefail diff --git a/scripts/verify-deploy-build.sh b/scripts/verify-deploy-build.sh new file mode 100644 index 00000000..0b57c8d6 --- /dev/null +++ b/scripts/verify-deploy-build.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Reject release artifacts that omit locally required fixes before installing. +# An isolated checkout can build successfully while excluding uncommitted work. +set -euo pipefail + +build=${1:?usage: verify-deploy-build.sh } +if [ ! -f "$build" ]; then + echo "error: release artifact missing: $build" >&2 + exit 1 +fi + +# These wire/error/SQL literals survive the all-in-one bundle's minification. +# Check the artifact, not its version: local builds can share a version number. +for marker in \ + 'codex-history-sync' \ + 'codex_history_epochs' \ + 'History batch was not confirmed' \ + 'Transcript delivery unconfirmed' \ + 'usage_session_sources'; do + if ! LC_ALL=C grep -aqF -- "$marker" "$build"; then + echo "error: release artifact omits required feature: $marker" >&2 + echo "error: rebuild from the complete deployment workspace, including untracked source files" >&2 + exit 1 + fi +done + +echo "release features ok: durable Codex history, confirmed delivery, persistent usage" diff --git a/shared/src/apiTypes.ts b/shared/src/apiTypes.ts index 3cacf00c..135e0ce7 100644 --- a/shared/src/apiTypes.ts +++ b/shared/src/apiTypes.ts @@ -6,7 +6,8 @@ import { DecryptedMessageSchema, MachineSchema, PermissionModeSchema, - SessionSchema + SessionSchema, + ThreadGoalSchema } from './schemas' import { AgentFlavorSchema } from './modes' import type { @@ -808,6 +809,22 @@ export const ImplementCodexPlanRequestSchema = z.object({ export type ImplementCodexPlanRequest = z.infer +export const CodexGoalRequestSchema = z.discriminatedUnion('action', [ + z.object({ action: z.literal('get') }).strict(), + z.object({ + action: z.literal('update'), + objective: z.string().trim().min(1).refine(value => Array.from(value).length <= 4000, 'Goal must be at most 4000 characters'), + tokenBudget: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).nullable().optional() + }).strict(), + z.object({ action: z.literal('pause') }).strict(), + z.object({ action: z.literal('resume') }).strict(), + z.object({ action: z.literal('clear') }).strict() +]) + +export type CodexGoalRequest = z.infer +export const CodexGoalResponseSchema = z.object({ goal: ThreadGoalSchema.nullable() }) +export type CodexGoalResponse = z.infer + export type ImplementCodexPlanResult = { ok: true } | { ok: false code: 'stale_plan' | 'unavailable' | 'indeterminate' | 'failed' diff --git a/shared/src/historySync.ts b/shared/src/historySync.ts new file mode 100644 index 00000000..3bb7d557 --- /dev/null +++ b/shared/src/historySync.ts @@ -0,0 +1,30 @@ +import { z } from 'zod' + +/** Native positions, not Hub arrival sequences. The page containing the + * anchor turn is deliberately revisited so an active turn can grow. */ +export const CodexHistoryCheckpointSchema = z.object({ + version: z.literal(1), + pageCursor: z.string().nullable(), + turnId: z.string().nullable(), + itemId: z.string().nullable(), + turnComplete: z.boolean() +}) +export type CodexHistoryCheckpoint = z.infer +export const CodexHistoryStateSchema = z.object({ + epoch: z.number().int().nonnegative(), + revision: z.number().int().nonnegative(), + checkpoint: CodexHistoryCheckpointSchema.nullable() +}) +export type CodexHistoryState = z.infer +export const CodexHistorySyncRequestSchema = z.object({ + sid: z.string().min(1), + threadId: z.string().min(1), + commit: z.object({ + epoch: z.number().int().nonnegative(), + revision: z.number().int().nonnegative(), + checkpoint: CodexHistoryCheckpointSchema.nullable(), + localIds: z.array(z.string().min(1)).max(1000) + }).optional() +}) +export type CodexHistorySyncRequest = z.infer +export type CodexHistorySyncAck = { ok: true; state: CodexHistoryState } | { ok: false } diff --git a/shared/src/index.ts b/shared/src/index.ts index 4fa14330..ca950456 100644 --- a/shared/src/index.ts +++ b/shared/src/index.ts @@ -14,6 +14,8 @@ export * from './rpcMethods' export * from './runnerCapabilities' export * from './search' export * from './socket' +export * from './historySync' +export * from './messageTruncation' export * from './sessionSummary' export * from './sessionCitation' export * from './sessionExport' diff --git a/shared/src/messageTruncation.ts b/shared/src/messageTruncation.ts new file mode 100644 index 00000000..a9b711d2 --- /dev/null +++ b/shared/src/messageTruncation.ts @@ -0,0 +1,36 @@ +/** Display-only agent payloads. Preserve user inputs verbatim. This is shared + * by the CLI and Hub so truncation happens before network transmission too. */ +export const TRUNCATE_STRING_LIMIT = 64 * 1024 +const TRUNCATE_HEAD = 48 * 1024 +const TRUNCATE_TAIL = 12 * 1024 + +function truncateDeep(value: unknown): unknown { + if (typeof value === 'string') { + if (value.length <= TRUNCATE_STRING_LIMIT) return value + const removed = value.length - TRUNCATE_HEAD - TRUNCATE_TAIL + return `${value.slice(0, TRUNCATE_HEAD)}\n…[hapi: truncated ${removed} chars]…\n${value.slice(value.length - TRUNCATE_TAIL)}` + } + if (Array.isArray(value)) { + let copy: unknown[] | null = null + for (let i = 0; i < value.length; i++) { + const item = truncateDeep(value[i]) + if (item !== value[i]) { copy ??= value.slice(); copy[i] = item } + } + return copy ?? value + } + if (value !== null && typeof value === 'object') { + const original = value as Record + let copy: Record | null = null + for (const key of Object.keys(original)) { + const item = truncateDeep(original[key]) + if (item !== original[key]) { copy ??= { ...original }; copy[key] = item } + } + return copy ?? value + } + return value +} + +export function truncateOversizedMessageContent(content: unknown): unknown { + if (content === null || typeof content !== 'object' || (content as Record).role !== 'agent') return content + return truncateDeep(content) +} diff --git a/shared/src/resume.test.ts b/shared/src/resume.test.ts index 009fc46e..2795c5ea 100644 --- a/shared/src/resume.test.ts +++ b/shared/src/resume.test.ts @@ -1,9 +1,16 @@ import { describe, expect, it } from 'bun:test' -import { LocalResumeTargetSchema, ResumableSessionSchema } from './resume' +import { LocalResumeTargetSchema, ResumableSessionSchema, sessionStartupTimeoutMs } from './resume' import { SyncEventSchema } from './schemas' import { SessionEndReasonSchema } from './socket' describe('resume schemas', () => { + it('allows a longer startup only for cold Codex resumes', () => { + expect(sessionStartupTimeoutMs('codex', 'native-thread')).toBe(120_000) + expect(sessionStartupTimeoutMs('codex')).toBe(15_000) + expect(sessionStartupTimeoutMs('claude', 'native-session')).toBe(15_000) + expect(sessionStartupTimeoutMs()).toBe(15_000) + }) + it('accepts a local resume target', () => { const parsed = LocalResumeTargetSchema.safeParse({ sessionId: 'hapi-session-1', diff --git a/shared/src/resume.ts b/shared/src/resume.ts index 493ce56f..6e0ea8b9 100644 --- a/shared/src/resume.ts +++ b/shared/src/resume.ts @@ -1,6 +1,11 @@ import { z } from 'zod' import { CodexCollaborationModeSchema, CopilotAgentModeSchema, PermissionModeSchema } from './schemas' -import { AgentFlavorSchema } from './modes' +import { AgentFlavorSchema, type AgentFlavor } from './modes' + +/** Cold Codex resumes can spend substantially longer loading native thread state. */ +export function sessionStartupTimeoutMs(flavor?: AgentFlavor, resumeSessionId?: string): number { + return flavor === 'codex' && resumeSessionId ? 120_000 : 15_000 +} export const LocalResumeTargetSchema = z.object({ sessionId: z.string().min(1), diff --git a/shared/src/rpcMethods.ts b/shared/src/rpcMethods.ts index 20ef241d..3fca1264 100644 --- a/shared/src/rpcMethods.ts +++ b/shared/src/rpcMethods.ts @@ -55,6 +55,7 @@ export const RPC_METHODS = { RewindConversation: 'rewind-conversation', ClearConversation: 'clear-conversation', ImplementCodexPlan: 'implement-codex-plan', + CodexGoal: 'codex-goal', } as const export const RPC_TARGET_MISSING_ERROR_CODE = 'rpc_target_missing' as const diff --git a/shared/src/schemas.ts b/shared/src/schemas.ts index 5bf8721f..92a9ac6b 100644 --- a/shared/src/schemas.ts +++ b/shared/src/schemas.ts @@ -36,6 +36,7 @@ export type OpencodeClearOperation = z.infer void) => void // createdAt: optional client-provided timestamp (epoch ms) for the entry's - // own origin time (e.g. a Claude transcript entry's timestamp). Currently - // only honored by the hub for agent messages (no localId) — see - // messages.ts addMessage. Absent -> hub falls back to Date.now(). - message: (data: { sid: string; message: unknown; localId?: string; createdAt?: number }) => void + // own origin time (e.g. a Claude transcript or Codex history item). + // Absent -> hub falls back to Date.now(). + message: (data: { sid: string; message: unknown; localId?: string; createdAt?: number }, ack?: (response: { ok: boolean }) => void) => void 'session-alive': (data: { sid: string time: number diff --git a/web/README.md b/web/README.md index 4b65b18c..a13ad89e 100644 --- a/web/README.md +++ b/web/README.md @@ -5,6 +5,7 @@ React Mini App / PWA for monitoring and controlling hapi sessions. ## What it does - Session list with status, pending approvals, todos, and summaries. +- Native Codex Goal controls in the session status panel: edit objective/token budget, pause, resume, and delete. Available to owners of active sessions advertising `capabilities.codexGoal`; native Goal notifications refresh the displayed state. - Chat view with streaming updates and message sending. - Permission approval and denial workflows. - Codex `request_user_input` choices honor `isOther`: **None of the above** diff --git a/web/src/api/client.ts b/web/src/api/client.ts index dce3830f..7500b803 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -45,6 +45,8 @@ import type { } from '@/types/api' import type { TeamDetail, TeamMessage, TeamSummary, TeamTask, TeamWithMembers } from '@/types/team' import type { + CodexGoalRequest, + CodexGoalResponse, AgentAvailabilityResponse, AgyModelsResponse, CodexModelsResponse, @@ -906,6 +908,16 @@ export class ApiClient { }) } + async getCodexGoal(sessionId: string): Promise { + return await this.request(`/api/sessions/${encodeURIComponent(sessionId)}/codex/goal`) + } + + async manageCodexGoal(sessionId: string, action: CodexGoalRequest): Promise { + return await this.request(`/api/sessions/${encodeURIComponent(sessionId)}/codex/goal`, { + method: 'POST', body: JSON.stringify(action) + }) + } + async retryCodexTurn(sessionId: string): Promise<{ retried: boolean; error?: string; sessionId?: string }> { return await this.request<{ retried: boolean; error?: string; sessionId?: string }>( `/api/sessions/${encodeURIComponent(sessionId)}/retry-codex-turn`, diff --git a/web/src/components/CodexGoalControls.test.tsx b/web/src/components/CodexGoalControls.test.tsx new file mode 100644 index 00000000..bd3682b2 --- /dev/null +++ b/web/src/components/CodexGoalControls.test.tsx @@ -0,0 +1,77 @@ +import { fireEvent, render, screen, waitFor, within } from '@testing-library/react' +import { describe, expect, it, vi } from 'vitest' +import type { ThreadGoal } from '@/types/api' +import { I18nProvider } from '@/lib/i18n-context' +import { CodexGoalControls } from './CodexGoalControls' + +const goal: ThreadGoal = { threadId: 'root', objective: 'Ship the fix', status: 'paused', tokenBudget: 10000, + tokensUsed: 350, timeUsedSeconds: 12, createdAt: 1, updatedAt: 2 } + +function show(onAction: Parameters[0]['onAction'], status: ThreadGoal['status'] = 'paused') { + return render() +} + +describe('Codex Goal controls', () => { + it('edits the current objective and budget without sending a resume action', async () => { + const action = vi.fn(async () => {}) + show(action) + fireEvent.click(screen.getByRole('button', { name: 'Edit Goal' })) + const dialog = screen.getByRole('dialog') + expect(within(dialog).getByLabelText('Objective')).toHaveValue(goal.objective) + expect(within(dialog).getByLabelText('Token budget (optional)')).toHaveValue(10000) + fireEvent.change(within(dialog).getByLabelText('Objective'), { target: { value: 'Ship with tests' } }) + fireEvent.change(within(dialog).getByLabelText('Token budget (optional)'), { target: { value: '' } }) + fireEvent.click(within(dialog).getByRole('button', { name: 'Save' })) + await waitFor(() => expect(action).toHaveBeenCalledWith({ action: 'update', objective: 'Ship with tests', tokenBudget: null })) + await waitFor(() => expect(screen.queryByRole('dialog')).not.toBeInTheDocument()) + }) + + it('keeps edits on native failure and validates the objective', async () => { + const action = vi.fn(async () => { throw new Error('Codex disconnected') }) + show(action) + fireEvent.click(screen.getByRole('button', { name: 'Edit Goal' })) + const objective = screen.getByLabelText('Objective') + fireEvent.change(objective, { target: { value: 'Keep this draft' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save' })) + await waitFor(() => expect(screen.getByRole('alert')).toHaveTextContent('Codex disconnected')) + expect(objective).toHaveValue('Keep this draft') + action.mockClear() + fireEvent.change(objective, { target: { value: ' ' } }) + fireEvent.click(screen.getByRole('button', { name: 'Save' })) + expect(screen.getByRole('alert')).toHaveTextContent('1–4000') + expect(action).not.toHaveBeenCalled() + }) + + it('pauses an active Goal and disables other operations until confirmed', async () => { + let finish!: () => void + const action = vi.fn(() => new Promise(resolve => { finish = resolve })) + show(action, 'active') + fireEvent.click(screen.getByRole('button', { name: 'Pause' })) + expect(action).toHaveBeenCalledWith({ action: 'pause' }) + expect(screen.getByRole('button', { name: 'Edit Goal' })).toBeDisabled() + expect(screen.getByRole('button', { name: 'Delete Goal' })).toBeDisabled() + finish() + await waitFor(() => expect(screen.getByRole('button', { name: 'Edit Goal' })).toBeEnabled()) + }) + + it('resumes a paused Goal', async () => { + const action = vi.fn(async () => {}) + show(action) + fireEvent.click(screen.getByRole('button', { name: 'Resume' })) + await waitFor(() => expect(action).toHaveBeenCalledWith({ action: 'resume' })) + }) + + it('deletes only after confirmation and keeps failed deletion retryable', async () => { + const action = vi.fn(async () => { throw new Error('Goal clear failed') }) + show(action) + fireEvent.click(screen.getByRole('button', { name: 'Delete Goal' })) + expect(action).not.toHaveBeenCalled() + fireEvent.click(within(screen.getByRole('dialog')).getByRole('button', { name: 'Cancel' })) + expect(action).not.toHaveBeenCalled() + fireEvent.click(screen.getByRole('button', { name: 'Delete Goal' })) + fireEvent.click(within(screen.getByRole('dialog')).getByRole('button', { name: 'Delete Goal' })) + await waitFor(() => expect(screen.getByText('Goal clear failed')).toBeInTheDocument()) + expect(action).toHaveBeenCalledWith({ action: 'clear' }) + expect(within(screen.getByRole('dialog')).getByRole('button', { name: 'Delete Goal' })).toBeEnabled() + }) +}) diff --git a/web/src/components/CodexGoalControls.tsx b/web/src/components/CodexGoalControls.tsx new file mode 100644 index 00000000..33b41073 --- /dev/null +++ b/web/src/components/CodexGoalControls.tsx @@ -0,0 +1,88 @@ +import { useId, useState } from 'react' +import { CodexGoalRequestSchema, type CodexGoalRequest } from '@hapi/protocol/apiTypes' +import type { ThreadGoal } from '@/types/api' +import { Button } from '@/components/ui/button' +import { ConfirmDialog } from '@/components/ui/ConfirmDialog' +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { useTranslation } from '@/lib/use-translation' + +export function CodexGoalControls({ goal, onAction }: { + goal: ThreadGoal + onAction: (action: CodexGoalRequest) => Promise +}) { + const { t } = useTranslation() + const id = useId() + const [editing, setEditing] = useState(false) + const [deleting, setDeleting] = useState(false) + const [objective, setObjective] = useState('') + const [budget, setBudget] = useState('') + const [pending, setPending] = useState(false) + const [error, setError] = useState(null) + + const run = async (action: CodexGoalRequest) => { + setError(null) + setPending(true) + try { await onAction(action) } + finally { setPending(false) } + } + const report = (reason: unknown) => setError(reason instanceof Error ? reason.message : t('dialog.error.default')) + const openEditor = () => { + setObjective(goal.objective) + setBudget(goal.tokenBudget == null ? '' : String(goal.tokenBudget)) + setError(null) + setEditing(true) + } + const save = async () => { + const action = CodexGoalRequestSchema.safeParse({ + action: 'update', objective, + tokenBudget: budget.trim() === '' ? null : Number(budget) + }) + if (!action.success) { setError(t('session.goal.invalid')); return } + try { await run(action.data); setEditing(false) } + catch (reason) { report(reason) } + } + + return ( +
+
+ + + +
+ {error && !editing ?
{error}
: null} + { if (!pending) setEditing(open) }}> + + + {t('session.goal.edit')} + {t('session.goal.editDescription')} + +
{ event.preventDefault(); void save() }}> + +