mirror of
https://github.com/wu736139669/hapi.git
synced 2026-11-18 03:27:08 +00:00
fix: preserve deployed session fixes
Persist paged Codex history checkpoints and bounded, acknowledged delivery. Keep usage sources and accounting across deleted or merged sessions. Include native Goal controls, faster peer/session queries, and release guards. Track the complete deployed source so isolated builds cannot omit local fixes. Validation: all typechecks pass. The full suite reports 7,997 passing tests and 46 existing failures, each reproduced against the prior commit.
This commit is contained in:
@@ -6,6 +6,7 @@ const socketHarness = vi.hoisted(() => ({
|
||||
connected: boolean
|
||||
connectCalls: number
|
||||
connectImmediately: boolean
|
||||
ack: (event: string, ...args: unknown[]) => Promise<unknown>
|
||||
emitted: Array<{ event: string; args: unknown[] }>
|
||||
listeners: Map<string, Array<(...args: any[]) => void>>
|
||||
trigger: (event: string, ...args: any[]) => void
|
||||
@@ -24,6 +25,7 @@ vi.mock('socket.io-client', () => ({
|
||||
connected: false,
|
||||
connectCalls: 0,
|
||||
connectImmediately: true,
|
||||
ack: async () => ({}),
|
||||
emitted: [] as Array<{ event: string; args: unknown[] }>,
|
||||
listeners: new Map<string, Array<(...args: any[]) => void>>(),
|
||||
trigger: () => {},
|
||||
@@ -62,8 +64,8 @@ vi.mock('socket.io-client', () => ({
|
||||
state.emitted.push({ event, args })
|
||||
return socket
|
||||
},
|
||||
emitWithAck: async () => ({}),
|
||||
timeout: () => ({ emitWithAck: async () => ({}) }),
|
||||
emitWithAck: (event: string, ...args: unknown[]) => state.ack(event, ...args),
|
||||
timeout: () => ({ emitWithAck: (event: string, ...args: unknown[]) => state.ack(event, ...args) }),
|
||||
connect: () => {
|
||||
state.connectCalls += 1
|
||||
if (state.connectImmediately) {
|
||||
@@ -96,6 +98,63 @@ vi.mock('axios', () => ({
|
||||
|
||||
import { ApiSessionClient, isExternalUserMessage, IncomingMessageFilter } from './apiSession'
|
||||
|
||||
describe('ApiSessionClient confirmed Codex transcripts', () => {
|
||||
it('preserves native history times, validates durable progress, and trims agent output before upload', async () => {
|
||||
const client = new ApiSessionClient('token', createSession({
|
||||
metadata: { path: '/repo', host: 'test', capabilities: { concurrentClients: true } }
|
||||
}))
|
||||
const socket = socketHarness.sockets.at(-1)!
|
||||
const payloads: Array<{ message: { content: { data?: { message: string }; text?: string } }; createdAt?: number }> = []
|
||||
socket.ack = async (event, data) => {
|
||||
if (event === 'codex-history-sync') return { ok: true, state: { epoch: 0, revision: 1, checkpoint: null } }
|
||||
if (event === 'message') payloads.push(data as typeof payloads[number])
|
||||
return { ok: true }
|
||||
}
|
||||
client.sendAgentMessage({ type: 'message', message: 'x'.repeat(1024 * 1024) }, 'large', { replay: true, createdAt: 1000 })
|
||||
client.sendUserMessage('u'.repeat(100_000), undefined, 'user', { replay: true, createdAt: 2000 })
|
||||
expect(await client.waitForMessages(['large', 'user'])).toBe(true)
|
||||
expect(payloads.map(payload => payload.createdAt)).toEqual([1000, 2000])
|
||||
expect(payloads[0].message.content.data?.message.length).toBeLessThan(64 * 1024)
|
||||
expect(payloads[1].message.content.text?.length).toBe(100_000)
|
||||
expect(await client.syncCodexHistory('thread')).toEqual({ epoch: 0, revision: 1, checkpoint: null })
|
||||
socket.ack = async () => ({ ok: false })
|
||||
await expect(client.syncCodexHistory('thread')).rejects.toThrow('did not confirm')
|
||||
client.close()
|
||||
})
|
||||
it('waits for persistence before consumption and skips confirmed history on reconnect', async () => {
|
||||
const client = new ApiSessionClient('token', createSession({
|
||||
metadata: { path: '/repo', host: 'test', capabilities: { concurrentClients: true } }
|
||||
}))
|
||||
const socket = socketHarness.sockets.at(-1)!
|
||||
const persisted = deferred<unknown>()
|
||||
const ack = vi.fn(() => persisted.promise)
|
||||
socket.ack = ack
|
||||
client.sendAgentMessage({ type: 'message', message: 'completed' }, 'stable', { replay: true })
|
||||
expect(ack).toHaveBeenCalledTimes(1)
|
||||
expect(socket.emitted.filter(e => e.event === 'messages-consumed')).toHaveLength(0)
|
||||
persisted.resolve({ ok: true })
|
||||
await vi.waitFor(() => expect(socket.emitted.filter(e => e.event === 'messages-consumed')).toHaveLength(1))
|
||||
socket.connected = false
|
||||
socket.trigger('disconnect', 'transport close')
|
||||
socket.triggerConnect()
|
||||
client.sendAgentMessage({ type: 'message', message: 'completed' }, 'stable', { replay: true })
|
||||
expect(ack).toHaveBeenCalledTimes(1)
|
||||
client.close()
|
||||
})
|
||||
|
||||
it('keeps flush unconfirmed when the hub rejects transcript persistence', async () => {
|
||||
const client = new ApiSessionClient('token', createSession({
|
||||
metadata: { path: '/repo', host: 'test', capabilities: { concurrentClients: true } }
|
||||
}))
|
||||
const socket = socketHarness.sockets.at(-1)!
|
||||
socket.ack = async () => ({ ok: false })
|
||||
client.sendUserMessage('native history', undefined, 'user-id', { replay: true })
|
||||
expect(await client.flush({ timeoutMs: 20 })).toBe(false)
|
||||
expect(socket.emitted.filter(e => e.event === 'messages-consumed')).toHaveLength(0)
|
||||
client.close()
|
||||
})
|
||||
})
|
||||
|
||||
function createSession(overrides: Partial<Session> = {}): Session {
|
||||
return {
|
||||
id: '11111111-1111-4111-8111-111111111111',
|
||||
|
||||
@@ -41,6 +41,8 @@ import { cleanupUploadDir, preserveUploadDirOnExit } from '../modules/common/han
|
||||
import { TerminalManager } from '@/terminal/TerminalManager'
|
||||
import { applyVersionedAck } from './versionedUpdate'
|
||||
import { buildHubRequestHeaders, buildSocketIoExtraHeaderOptions } from './hubExtraHeaders'
|
||||
import { ConfirmedMessageQueue } from './confirmedMessageQueue'
|
||||
import { CodexHistoryStateSchema, truncateOversizedMessageContent, type CodexHistoryState, type CodexHistorySyncRequest } from '@hapi/protocol'
|
||||
|
||||
/**
|
||||
* XML tags that Claude Code injects as `type:'user'` messages.
|
||||
@@ -234,6 +236,17 @@ function hasSameJsonValue(left: unknown, right: unknown): boolean {
|
||||
export class ApiSessionClient extends EventEmitter {
|
||||
private reconnectHandler: (() => void) | null = null
|
||||
onReconnect(handler: (() => void) | null): void { this.reconnectHandler = handler }
|
||||
async syncCodexHistory(threadId: string, commit?: CodexHistorySyncRequest['commit']): Promise<CodexHistoryState> {
|
||||
if (!this.socket.connected) throw new Error('Hub disconnected during history synchronization')
|
||||
const response: unknown = await this.socket.timeout(15_000).emitWithAck('codex-history-sync', { sid: this.sessionId, threadId, ...(commit ? { commit } : {}) })
|
||||
if (typeof response !== 'object' || response === null || !('ok' in response) || response.ok !== true || !('state' in response)) {
|
||||
throw new Error('Hub did not confirm history synchronization progress')
|
||||
}
|
||||
return CodexHistoryStateSchema.parse(response.state)
|
||||
}
|
||||
waitForMessages(localIds: string[], signal?: AbortSignal): Promise<boolean> {
|
||||
return this.confirmedMessages.waitFor(localIds, 30_000, signal)
|
||||
}
|
||||
private readonly token: string
|
||||
readonly sessionId: string
|
||||
private metadata: Metadata | null
|
||||
@@ -281,6 +294,7 @@ export class ApiSessionClient extends EventEmitter {
|
||||
private agentStateChangedDuringAttempt = false
|
||||
private readonly pendingOutboundEvents: PendingOutboundEvent[] = []
|
||||
private didWarnPendingQueueFull = false
|
||||
private readonly confirmedMessages: ConfirmedMessageQueue
|
||||
|
||||
constructor(token: string, session: Session, options: ApiSessionClientOptions = {}) {
|
||||
super()
|
||||
@@ -319,6 +333,17 @@ export class ApiSessionClient extends EventEmitter {
|
||||
...buildSocketIoExtraHeaderOptions()
|
||||
})
|
||||
|
||||
this.confirmedMessages = new ConfirmedMessageQueue(
|
||||
async message => {
|
||||
const response: unknown = await this.socket.emitWithAck('message', message)
|
||||
if (typeof response !== 'object' || response === null || !('ok' in response) || response.ok !== true) {
|
||||
throw new Error('Hub did not confirm transcript persistence')
|
||||
}
|
||||
},
|
||||
localId => this.socket.emit('messages-consumed', { sid: this.sessionId, localIds: [localId] }),
|
||||
error => logger.debug('[API] Transcript delivery unconfirmed; retaining for reconnect', error)
|
||||
)
|
||||
|
||||
this.terminalManager = new TerminalManager({
|
||||
sessionId: this.sessionId,
|
||||
getSessionPath: () => this.metadata?.path ?? null,
|
||||
@@ -332,6 +357,7 @@ export class ApiSessionClient extends EventEmitter {
|
||||
logger.debug('Socket connected successfully')
|
||||
this.awaitingMaterializedConnection = false
|
||||
this.rpcHandlerManager.onSocketConnect(this.socket)
|
||||
this.confirmedMessages.setConnected(true)
|
||||
if (this.hasConnectedOnce) {
|
||||
this.needsBackfill = true
|
||||
}
|
||||
@@ -351,6 +377,7 @@ export class ApiSessionClient extends EventEmitter {
|
||||
|
||||
this.socket.on('disconnect', (reason) => {
|
||||
logger.debug('[API] Socket disconnected:', reason)
|
||||
this.confirmedMessages.setConnected(false)
|
||||
this.rpcHandlerManager.onSocketDisconnect()
|
||||
this.terminalManager.closeAll()
|
||||
if (this.hasConnectedOnce) {
|
||||
@@ -1027,7 +1054,7 @@ export class ApiSessionClient extends EventEmitter {
|
||||
})
|
||||
}
|
||||
|
||||
sendUserMessage(text: string, meta?: MessageMeta, localId?: string): void {
|
||||
sendUserMessage(text: string, meta?: MessageMeta, localId?: string, options?: { replay?: boolean; verifyPersistence?: boolean; createdAt?: number }): void {
|
||||
if (!text) {
|
||||
return
|
||||
}
|
||||
@@ -1045,6 +1072,10 @@ export class ApiSessionClient extends EventEmitter {
|
||||
}
|
||||
|
||||
this.emitOrQueue(() => {
|
||||
if (localId && this.metadata?.capabilities?.concurrentClients) {
|
||||
this.confirmedMessages.enqueue({ sid: this.sessionId, message: content, localId, ...(options?.createdAt !== undefined ? { createdAt: options.createdAt } : {}) }, options?.replay, options?.verifyPersistence)
|
||||
return
|
||||
}
|
||||
this.socket.emit('message', {
|
||||
sid: this.sessionId,
|
||||
message: content,
|
||||
@@ -1063,8 +1094,8 @@ export class ApiSessionClient extends EventEmitter {
|
||||
void this.materialize()
|
||||
}
|
||||
|
||||
sendAgentMessage(body: unknown, localId?: string): void {
|
||||
const content = {
|
||||
sendAgentMessage(body: unknown, localId?: string, options?: { replay?: boolean; verifyPersistence?: boolean; createdAt?: number }): void {
|
||||
const content = truncateOversizedMessageContent({
|
||||
role: 'agent',
|
||||
content: {
|
||||
type: AGENT_MESSAGE_PAYLOAD_TYPE,
|
||||
@@ -1073,8 +1104,12 @@ export class ApiSessionClient extends EventEmitter {
|
||||
meta: {
|
||||
sentFrom: 'cli'
|
||||
}
|
||||
}
|
||||
})
|
||||
this.emitOrQueue(() => {
|
||||
if (localId && this.metadata?.capabilities?.concurrentClients) {
|
||||
this.confirmedMessages.enqueue({ sid: this.sessionId, message: content, localId, ...(options?.createdAt !== undefined ? { createdAt: options.createdAt } : {}) }, options?.replay, options?.verifyPersistence)
|
||||
return
|
||||
}
|
||||
this.socket.emit('message', {
|
||||
sid: this.sessionId,
|
||||
message: content,
|
||||
@@ -1520,6 +1555,10 @@ export class ApiSessionClient extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
if (!await this.confirmedMessages.drain(remainingMs())) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (!await this.drainLock(this.metadataLock, remainingMs())) {
|
||||
return false
|
||||
}
|
||||
@@ -1556,6 +1595,7 @@ export class ApiSessionClient extends EventEmitter {
|
||||
this.materializationRetryAbortController = null
|
||||
this.awaitingMaterializedConnection = false
|
||||
this.pendingOutboundEvents.length = 0
|
||||
this.confirmedMessages.close()
|
||||
this.rpcHandlerManager.onSocketDisconnect()
|
||||
this.terminalManager.closeAll()
|
||||
this.socket.disconnect()
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import { createServer } from 'node:http'
|
||||
import { Server, type Socket as ServerSocket } from 'socket.io'
|
||||
import { io, type Socket } from 'socket.io-client'
|
||||
import WebSocket, { WebSocketServer } from 'ws'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { ConfirmedMessageQueue } from './confirmedMessageQueue'
|
||||
|
||||
async function slowConnection() {
|
||||
const http = createServer()
|
||||
const hub = new Server(http, { path: '/socket.io/', ...{ pingInterval: 1000, pingTimeout: 3000 } })
|
||||
await new Promise<void>(resolve => http.listen(0, '127.0.0.1', resolve))
|
||||
const address = http.address()
|
||||
if (!address || typeof address === 'string') throw new Error('No hub port')
|
||||
const proxy = new WebSocketServer({ port: 0, host: '127.0.0.1' })
|
||||
await new Promise<void>(resolve => proxy.once('listening', resolve))
|
||||
const proxyAddress = proxy.address()
|
||||
if (!proxyAddress || typeof proxyAddress === 'string') throw new Error('No proxy port')
|
||||
const cleanups: Array<() => void> = []
|
||||
let backlogBytes = 0
|
||||
let peakBacklogBytes = 0
|
||||
proxy.on('connection', (downstream, request) => {
|
||||
const upstream = new WebSocket(`ws://127.0.0.1:${address.port}${request.url}`)
|
||||
const packets: Array<{ data: Buffer; binary: boolean }> = []
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
let stopped = false
|
||||
const pump = () => {
|
||||
if (stopped || timer || upstream.readyState !== WebSocket.OPEN) return
|
||||
const packet = packets.shift()
|
||||
if (!packet) return
|
||||
// Throttle the entire uplink, including pongs and RPC replies.
|
||||
timer = setTimeout(() => {
|
||||
timer = undefined
|
||||
backlogBytes -= packet.data.length
|
||||
if (upstream.readyState === WebSocket.OPEN) upstream.send(packet.data, { binary: packet.binary })
|
||||
pump()
|
||||
}, Math.max(1, packet.data.length / (128 * 1024) * 1000))
|
||||
}
|
||||
upstream.on('open', pump)
|
||||
upstream.on('message', (data, binary) => {
|
||||
if (downstream.readyState === WebSocket.OPEN) downstream.send(data, { binary })
|
||||
})
|
||||
downstream.on('message', (raw, binary) => {
|
||||
const data = Buffer.isBuffer(raw) ? raw : Array.isArray(raw) ? Buffer.concat(raw) : Buffer.from(raw)
|
||||
packets.push({ data, binary })
|
||||
backlogBytes += data.length
|
||||
peakBacklogBytes = Math.max(peakBacklogBytes, backlogBytes)
|
||||
pump()
|
||||
})
|
||||
const stop = () => {
|
||||
if (stopped) return
|
||||
stopped = true
|
||||
clearTimeout(timer)
|
||||
downstream.terminate()
|
||||
upstream.terminate()
|
||||
}
|
||||
upstream.on('error', stop)
|
||||
downstream.on('error', stop)
|
||||
upstream.on('close', stop)
|
||||
downstream.on('close', stop)
|
||||
cleanups.push(stop)
|
||||
})
|
||||
const connection = new Promise<ServerSocket>(resolve => hub.once('connection', resolve))
|
||||
const socket: Socket = io(`http://127.0.0.1:${proxyAddress.port}`, { transports: ['websocket'], reconnection: false })
|
||||
await new Promise<void>(resolve => socket.once('connect', resolve))
|
||||
const serverSocket = await connection
|
||||
serverSocket.on('message', (_payload: unknown, ack?: (response: { ok: boolean }) => void) => ack?.({ ok: true }))
|
||||
socket.on('rpc-request', (ack: (value: string) => void) => ack('ok'))
|
||||
return {
|
||||
socket, serverSocket, peakBacklog: () => peakBacklogBytes,
|
||||
async close() {
|
||||
socket.disconnect()
|
||||
for (const cleanup of cleanups) cleanup()
|
||||
await new Promise<void>(resolve => proxy.close(() => resolve()))
|
||||
await new Promise<void>(resolve => hub.close(() => resolve()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe('transcript delivery over a slow uplink', () => {
|
||||
it('reproduces heartbeat failure when a long history floods the socket', async () => {
|
||||
const f = await slowConnection()
|
||||
try {
|
||||
const disconnected = new Promise<void>(resolve => f.socket.once('disconnect', () => resolve()))
|
||||
for (let i = 0; i < 1000; i++) f.socket.emit('message', { localId: `${i}`, text: 'x'.repeat(32_000) })
|
||||
await disconnected
|
||||
expect(f.socket.connected).toBe(false)
|
||||
expect(f.peakBacklog()).toBeGreaterThan(1024 * 1024)
|
||||
} finally { await f.close() }
|
||||
}, 10_000)
|
||||
|
||||
it('keeps heartbeats, RPCs and live content moving during the same history replay', async () => {
|
||||
const f = await slowConnection()
|
||||
const confirmed: string[] = []
|
||||
const queue = new ConfirmedMessageQueue(async payload => {
|
||||
const ack: unknown = await f.socket.emitWithAck('message', payload)
|
||||
if ((ack as { ok?: boolean })?.ok !== true) throw new Error('Not confirmed')
|
||||
}, id => confirmed.push(id), () => {})
|
||||
f.socket.on('disconnect', () => queue.setConnected(false))
|
||||
queue.setConnected(true)
|
||||
try {
|
||||
for (let i = 0; i < 1000; i++) queue.enqueue({ sid: 'sid', localId: `history-${i}`, message: 'x'.repeat(32_000) }, true)
|
||||
queue.enqueue({ sid: 'sid', localId: 'live', message: 'new response' })
|
||||
expect(await f.serverSocket.timeout(2800).emitWithAck('rpc-request')).toBe('ok')
|
||||
await new Promise(resolve => setTimeout(resolve, 5000))
|
||||
expect(f.socket.connected).toBe(true)
|
||||
expect(confirmed).toContain('live')
|
||||
expect(confirmed.length).toBeLessThan(1000)
|
||||
expect(f.peakBacklog()).toBeLessThan(270 * 1024)
|
||||
} finally { queue.close(); await f.close() }
|
||||
}, 12_000)
|
||||
})
|
||||
@@ -0,0 +1,96 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { ConfirmedMessageQueue } from './confirmedMessageQueue'
|
||||
|
||||
function fixture() {
|
||||
const attempts: Array<{ id: string; resolve(): void; reject(error: Error): void }> = []
|
||||
const confirmed = vi.fn()
|
||||
const error = vi.fn()
|
||||
const queue = new ConfirmedMessageQueue(message => new Promise<void>((resolve, reject) => {
|
||||
attempts.push({ id: message.localId, resolve, reject })
|
||||
}), confirmed, error)
|
||||
const enqueue = (id: string, bytes = 80_000, replay = true) => queue.enqueue({
|
||||
sid: 'sid', localId: id, message: { text: 'x'.repeat(bytes) }
|
||||
}, replay)
|
||||
return { queue, attempts, confirmed, error, enqueue }
|
||||
}
|
||||
|
||||
describe('confirmed transcript delivery', () => {
|
||||
it('confirms a history batch independently of continuous live traffic and allows restored-Hub verification', async () => {
|
||||
const f = fixture(); f.queue.setConnected(true)
|
||||
f.enqueue('history', 100); f.enqueue('live', 100, false)
|
||||
const done = f.queue.waitFor(['history'], 1000)
|
||||
f.attempts[0].resolve()
|
||||
expect(await done).toBe(true)
|
||||
expect(await f.queue.drain(0)).toBe(false)
|
||||
f.queue.enqueue({ sid: 'sid', localId: 'history', message: {} }, true, true)
|
||||
expect(f.attempts.map(attempt => attempt.id)).toEqual(['history', 'live', 'history'])
|
||||
const abort = new AbortController()
|
||||
const waiting = f.queue.waitFor(['history'], 1000, abort.signal)
|
||||
abort.abort(); expect(await waiting).toBe(false)
|
||||
f.queue.close()
|
||||
})
|
||||
it('bounds a long history and puts live messages ahead of unsent history', async () => {
|
||||
const f = fixture()
|
||||
f.queue.setConnected(true)
|
||||
for (let i = 0; i < 1000; i++) f.enqueue(`history-${i}`)
|
||||
expect(f.attempts.map(a => a.id)).toEqual(['history-0', 'history-1', 'history-2'])
|
||||
f.enqueue('live', 100, false)
|
||||
const drained = f.queue.drain(1000)
|
||||
f.attempts[0].resolve()
|
||||
await vi.waitFor(() => expect(f.attempts[3]?.id).toBe('live'))
|
||||
expect(f.confirmed).toHaveBeenCalledWith('history-0')
|
||||
f.queue.close()
|
||||
expect(await drained).toBe(false)
|
||||
})
|
||||
|
||||
it('retains only unconfirmed messages across reconnect and ignores stale ACKs', async () => {
|
||||
const f = fixture()
|
||||
f.queue.setConnected(true)
|
||||
f.enqueue('confirmed', 100)
|
||||
f.enqueue('uncertain', 100)
|
||||
f.attempts[0].resolve()
|
||||
await vi.waitFor(() => expect(f.confirmed).toHaveBeenCalledWith('confirmed'))
|
||||
f.queue.setConnected(false)
|
||||
f.attempts[1].resolve()
|
||||
await Promise.resolve()
|
||||
f.enqueue('confirmed', 100)
|
||||
f.enqueue('uncertain', 100)
|
||||
f.queue.setConnected(true)
|
||||
expect(f.attempts.map(a => a.id)).toEqual(['confirmed', 'uncertain', 'uncertain'])
|
||||
expect(f.confirmed).not.toHaveBeenCalledWith('uncertain')
|
||||
f.attempts[2].resolve()
|
||||
expect(await f.queue.drain(1000)).toBe(true)
|
||||
expect(f.confirmed).toHaveBeenCalledWith('uncertain')
|
||||
f.queue.close()
|
||||
})
|
||||
|
||||
it('does not turn a rejected persistence ACK into confirmation or a retry storm', async () => {
|
||||
const f = fixture()
|
||||
f.queue.setConnected(true)
|
||||
f.enqueue('failed', 100)
|
||||
f.attempts[0].reject(new Error('not persisted'))
|
||||
await vi.waitFor(() => expect(f.error).toHaveBeenCalledTimes(1))
|
||||
expect(f.confirmed).not.toHaveBeenCalled()
|
||||
expect(f.attempts).toHaveLength(1)
|
||||
expect(await f.queue.drain(0)).toBe(false)
|
||||
f.queue.setConnected(false)
|
||||
f.queue.setConnected(true)
|
||||
expect(f.attempts).toHaveLength(2)
|
||||
f.attempts[1].resolve()
|
||||
expect(await f.queue.drain(1000)).toBe(true)
|
||||
f.queue.close()
|
||||
})
|
||||
|
||||
it('sends one oversized result alone and counts UTF-8 bytes rather than characters', async () => {
|
||||
const f = fixture()
|
||||
f.queue.setConnected(true)
|
||||
f.queue.enqueue({ sid: 'sid', localId: 'large', message: '中'.repeat(100_000) }, true)
|
||||
f.enqueue('next', 100)
|
||||
expect(f.attempts).toHaveLength(1)
|
||||
f.attempts[0].resolve()
|
||||
await vi.waitFor(() => expect(f.attempts[1]?.id).toBe('next'))
|
||||
f.attempts[1].resolve()
|
||||
expect(await f.queue.drain(1000)).toBe(true)
|
||||
f.queue.close()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,154 @@
|
||||
type Message = {
|
||||
sid: string
|
||||
message: unknown
|
||||
localId: string
|
||||
createdAt?: number
|
||||
}
|
||||
|
||||
type Entry = {
|
||||
message: Message
|
||||
bytes: number
|
||||
replay: boolean
|
||||
inFlight: boolean
|
||||
failedGeneration?: number
|
||||
}
|
||||
|
||||
const MAX_IN_FLIGHT_BYTES = 256 * 1024
|
||||
const MAX_IN_FLIGHT_MESSAGES = 16
|
||||
|
||||
/** Transcript delivery only; native user submissions remain in the native queue.
|
||||
* Bound the socket backlog so pongs, keep-alives and RPC acks can pass between
|
||||
* history batches. Only a hub persistence ACK permits forgetting a message. */
|
||||
export class ConfirmedMessageQueue {
|
||||
private readonly pending = new Map<string, Entry>()
|
||||
private readonly confirmed = new Set<string>()
|
||||
private readonly drainListeners = new Set<(drained: boolean) => void>()
|
||||
private readonly progressListeners = new Set<() => void>()
|
||||
private connected = false
|
||||
private closed = false
|
||||
private generation = 0
|
||||
private inFlightBytes = 0
|
||||
private inFlightMessages = 0
|
||||
|
||||
constructor(
|
||||
private readonly send: (message: Message) => Promise<void>,
|
||||
private readonly onConfirmed: (localId: string) => void,
|
||||
private readonly onError: (error: unknown) => void
|
||||
) {}
|
||||
|
||||
enqueue(message: Message, replay = false, verifyPersistence = false): void {
|
||||
// The Hub checkpoint is authoritative after restore/rollback. A
|
||||
// process-local confirmation cannot prove that Hub still has a row.
|
||||
if (verifyPersistence) this.confirmed.delete(message.localId)
|
||||
if (this.closed || this.confirmed.has(message.localId)) return
|
||||
const existing = this.pending.get(message.localId)
|
||||
if (existing) {
|
||||
if (!replay) existing.replay = false
|
||||
} else {
|
||||
this.pending.set(message.localId, {
|
||||
message, replay, inFlight: false,
|
||||
bytes: Buffer.byteLength(JSON.stringify(message))
|
||||
})
|
||||
}
|
||||
this.pump()
|
||||
}
|
||||
|
||||
setConnected(connected: boolean): void {
|
||||
if (this.closed || this.connected === connected) return
|
||||
this.connected = connected
|
||||
this.generation++
|
||||
this.inFlightBytes = 0
|
||||
this.inFlightMessages = 0
|
||||
for (const entry of this.pending.values()) entry.inFlight = false
|
||||
if (connected) this.pump()
|
||||
}
|
||||
|
||||
private next(): Entry | undefined {
|
||||
let history: Entry | undefined
|
||||
for (const entry of this.pending.values()) {
|
||||
if (entry.inFlight || entry.failedGeneration === this.generation) continue
|
||||
if (!entry.replay) return entry
|
||||
history ??= entry
|
||||
}
|
||||
return history
|
||||
}
|
||||
|
||||
private pump(): void {
|
||||
while (this.connected && !this.closed && this.inFlightMessages < MAX_IN_FLIGHT_MESSAGES) {
|
||||
const entry = this.next()
|
||||
if (!entry) return
|
||||
// An individual large result may exceed the window; send it alone.
|
||||
if (this.inFlightMessages > 0 && this.inFlightBytes + entry.bytes > MAX_IN_FLIGHT_BYTES) return
|
||||
const generation = this.generation
|
||||
entry.inFlight = true
|
||||
this.inFlightBytes += entry.bytes
|
||||
this.inFlightMessages++
|
||||
void this.send(entry.message).then(() => {
|
||||
if (this.closed || generation !== this.generation) return
|
||||
this.pending.delete(entry.message.localId)
|
||||
this.confirmed.add(entry.message.localId)
|
||||
this.onConfirmed(entry.message.localId)
|
||||
for (const listener of this.progressListeners) listener()
|
||||
}).catch(error => {
|
||||
if (this.closed || generation !== this.generation) return
|
||||
// A failed ACK is uncertain. Retain it for the next connection,
|
||||
// rather than adding retries to an already congested socket.
|
||||
entry.failedGeneration = generation
|
||||
this.onError(error)
|
||||
}).finally(() => {
|
||||
if (this.closed || generation !== this.generation) return
|
||||
entry.inFlight = false
|
||||
this.inFlightBytes -= entry.bytes
|
||||
this.inFlightMessages--
|
||||
if (this.pending.size === 0) this.notifyDrained(true)
|
||||
this.pump()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
drain(timeoutMs: number): Promise<boolean> {
|
||||
if (this.closed) return Promise.resolve(false)
|
||||
if (this.pending.size === 0) return Promise.resolve(true)
|
||||
if (timeoutMs <= 0) return Promise.resolve(false)
|
||||
return new Promise(resolve => {
|
||||
const finish = (drained: boolean) => {
|
||||
clearTimeout(timer)
|
||||
this.drainListeners.delete(finish)
|
||||
resolve(drained)
|
||||
}
|
||||
const timer = setTimeout(() => finish(false), timeoutMs)
|
||||
this.drainListeners.add(finish)
|
||||
})
|
||||
}
|
||||
|
||||
/** Wait only for this history batch. Continuous live traffic must not
|
||||
* prevent a completed batch from advancing its durable checkpoint. */
|
||||
waitFor(localIds: string[], timeoutMs: number, signal?: AbortSignal): Promise<boolean> {
|
||||
const confirmed = () => localIds.every(id => this.confirmed.has(id))
|
||||
if (this.closed || signal?.aborted) return Promise.resolve(false)
|
||||
if (confirmed()) return Promise.resolve(true)
|
||||
return new Promise(resolve => {
|
||||
const finish = (result: boolean) => {
|
||||
clearTimeout(timer); this.progressListeners.delete(check)
|
||||
signal?.removeEventListener('abort', aborted); resolve(result)
|
||||
}
|
||||
const check = () => { if (this.closed) finish(false); else if (confirmed()) finish(true) }
|
||||
const aborted = () => finish(false)
|
||||
const timer = setTimeout(() => finish(false), timeoutMs)
|
||||
this.progressListeners.add(check)
|
||||
signal?.addEventListener('abort', aborted, { once: true })
|
||||
})
|
||||
}
|
||||
|
||||
private notifyDrained(drained: boolean): void {
|
||||
for (const listener of this.drainListeners) listener(drained)
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.closed = true
|
||||
this.pending.clear()
|
||||
this.confirmed.clear()
|
||||
for (const listener of this.progressListeners) listener()
|
||||
this.notifyDrained(false)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user