fix(pi): resume archived sessions safely (#1308)

* fix(pi): resume archived sessions safely

* fix(pi): harden native resume startup

* fix(pi): harden resume termination evidence

* fix(runner): persist resume process evidence

* fix(runner): track resume process generations

* fix(runner): verify full session tree shutdown

* fix(pi): block pre-mapping resume dedup
This commit is contained in:
KorenKrita
2026-08-02 21:15:52 +08:00
committed by GitHub
parent fb6f697555
commit abf9cb02a5
26 changed files with 1546 additions and 137 deletions
+7
View File
@@ -140,6 +140,13 @@ export class RpcGateway {
await this.sessionRpc(sessionId, RPC_METHODS.KillSession, {})
}
async stopRunnerSession(machineId: string, sessionId: string): Promise<'stopped' | 'already_gone' | 'still_alive'> {
const result = await this.machineRpc(machineId, RPC_METHODS.StopSession, { sessionId })
const status = result && typeof result === 'object' ? (result as { status?: unknown }).status : undefined
if (status === 'stopped' || status === 'already_gone' || status === 'still_alive') return status
throw new Error('Unexpected stop-session response')
}
async handoffSessionToLocal(sessionId: string): Promise<void> {
await this.sessionRpc(sessionId, RPC_METHODS.HandoffLocal, {})
}
+374
View File
@@ -16,6 +16,11 @@ function createPublisher(events: SyncEvent[]): EventPublisher {
} as unknown as EventPublisher
}
async function flushAsyncWork(): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, 0))
await new Promise((resolve) => setTimeout(resolve, 0))
}
function productionCodexMessage(event: Record<string, unknown>): Record<string, unknown> {
return {
role: 'agent',
@@ -2003,6 +2008,375 @@ describe('session model', () => {
}
})
it('reopens Pi in place only after native-ready, preserving its id and history', async () => {
const store = new Store(':memory:')
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
const session = engine.getOrCreateSession('pi-in-place', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-1',
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
}, null, 'default')
store.messages.addMessage(session.id, { role: 'user', content: { type: 'text', text: 'keep history' } })
engine.getOrCreateMachine('machine-1', { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, { status: 'running', capabilities: { piExistingSessionResume: true } }, 'default')
engine.handleMachineAlive({ machineId: 'machine-1', time: Date.now() })
engine.handleSessionEnd({ sid: session.id, time: Date.now() })
let existing: string | undefined
let merges = 0
;(engine as any).sessionCache.mergeSessions = async () => { merges += 1 }
;(engine as any).rpcGateway.spawnSession = async (...args: Parameters<SyncEngine['spawnSession']>) => {
existing = args[12]
engine.handleSessionAlive({ sid: session.id, time: Date.now() })
engine.handleSessionReady({ sid: session.id, time: Date.now() })
return { type: 'success', sessionId: session.id }
}
const result = await engine.reopenSession(session.id, 'default')
expect(result).toEqual({ type: 'success', sessionId: session.id, resumed: true })
expect(existing).toBe(session.id)
expect(merges).toBe(0)
expect(store.messages.getFirstMessages(session.id, 10)).toHaveLength(1)
} finally { engine.stop() }
})
it('kills and deletes an unexpected legacy Pi temp without touching the original row', async () => {
const store = new Store(':memory:')
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
const original = engine.getOrCreateSession('pi-original', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-old',
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
}, null, 'default')
const unexpected = engine.getOrCreateSession('pi-unexpected', { path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi' }, null, 'default')
engine.getOrCreateMachine('machine-1', { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, { status: 'running', capabilities: { piExistingSessionResume: true } }, 'default')
engine.handleMachineAlive({ machineId: 'machine-1', time: Date.now() })
engine.handleSessionEnd({ sid: original.id, time: Date.now() })
;(engine as any).rpcGateway.spawnSession = async () => ({ type: 'success', sessionId: unexpected.id })
;(engine as any).rpcGateway.stopRunnerSession = async (_machineId: string, sid: string) => {
engine.handleSessionEnd({ sid, time: Date.now(), reason: 'error' })
return 'stopped'
}
const result = await engine.reopenSession(original.id, 'default')
expect(result).toMatchObject({ type: 'error', code: 'resume_failed', message: expect.stringContaining('upgrade') })
expect(store.sessions.getSession(unexpected.id)).toBeNull()
expect(store.sessions.getSession(original.id)).not.toBeNull()
expect(engine.getSessionByNamespace(original.id, 'default')?.metadata?.lifecycleState).toBe('archived')
} finally { engine.stop() }
})
it('does not restore archive metadata over a live Pi child after kill failure', async () => {
const store = new Store(':memory:')
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
const session = engine.getOrCreateSession('pi-live-failure', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-live',
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
}, null, 'default')
engine.getOrCreateMachine('machine-1', { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, { status: 'running', capabilities: { piExistingSessionResume: true } }, 'default')
engine.handleMachineAlive({ machineId: 'machine-1', time: Date.now() })
engine.handleSessionEnd({ sid: session.id, time: Date.now() })
;(engine as any).rpcGateway.spawnSession = async () => {
engine.handleSessionAlive({ sid: session.id, time: Date.now() })
return { type: 'success', sessionId: session.id }
}
;(engine as any).waitForSessionReady = async () => 'timeout'
;(engine as any).rpcGateway.stopRunnerSession = async () => 'still_alive'
const result = await engine.reopenSession(session.id, 'default')
expect(result).toMatchObject({ type: 'error', message: expect.stringContaining('still active') })
expect(engine.getSessionByNamespace(session.id, 'default')?.active).toBe(true)
// Pi keeps the persisted archive snapshot until bootstrap succeeds,
// so a failed stop never needs to reconstruct it from memory.
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.lifecycleState).toBe('archived')
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt?.state).toBe('quarantined')
expect(await engine.reopenSession(session.id, 'default')).toMatchObject({ type: 'error', message: 'Pi resume is already in progress' })
engine.handleSessionEnd({ sid: session.id, time: Date.now(), reason: 'error' })
await flushAsyncWork()
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
} finally { engine.stop() }
})
it('rejects Pi resume before spawn when the runner lacks in-place capability', async () => {
const store = new Store(':memory:')
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
const session = engine.getOrCreateSession('pi-old-runner', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-old-runner',
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
}, null, 'default')
engine.getOrCreateMachine('machine-1', { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, { status: 'running' }, 'default')
engine.handleMachineAlive({ machineId: 'machine-1', time: Date.now() })
engine.handleSessionEnd({ sid: session.id, time: Date.now() })
let spawnCalls = 0
;(engine as any).rpcGateway.spawnSession = async () => { spawnCalls += 1; return { type: 'error', message: 'unexpected' } }
const result = await engine.reopenSession(session.id, 'default')
expect(result).toMatchObject({ type: 'error', message: 'Pi resume requires an upgraded runner' })
expect(spawnCalls).toBe(0)
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.lifecycleState).toBe('archived')
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
} finally { engine.stop() }
})
it('quarantines a Pi attempt when runner spawn fails before process termination is confirmed', async () => {
const store = new Store(':memory:')
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
const session = engine.getOrCreateSession('pi-spawn-error-live-child', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-spawn-error',
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
}, null, 'default')
engine.getOrCreateMachine('machine-1', { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, {
status: 'running', capabilities: { piExistingSessionResume: true }
}, 'default')
engine.handleMachineAlive({ machineId: 'machine-1', time: Date.now() })
engine.handleSessionEnd({ sid: session.id, time: Date.now() })
;(engine as any).rpcGateway.spawnSession = async () => ({ type: 'error', message: 'webhook timeout' })
;(engine as any).rpcGateway.stopRunnerSession = async () => 'still_alive'
expect(await engine.reopenSession(session.id, 'default')).toMatchObject({
type: 'error', message: 'webhook timeout'
})
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt?.state).toBe('quarantined')
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.lifecycleState).toBe('archived')
expect(await engine.reopenSession(session.id, 'default')).toMatchObject({
type: 'error', message: 'Pi resume is already in progress'
})
} finally { engine.stop() }
})
it('keeps persisted Pi quarantine across SyncEngine restart and clears it on end', async () => {
const store = new Store(':memory:')
const first = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
const persisted = first.getOrCreateSession('pi-persisted-attempt', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-persisted',
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
piResumeAttempt: { state: 'quarantined', machineId: 'machine-1', startedAt: 1 },
}, null, 'default')
first.stop()
const restarted = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
expect(await restarted.reopenSession(persisted.id, 'default')).toMatchObject({
type: 'error', message: 'Pi resume is already in progress'
})
restarted.handleSessionEnd({ sid: persisted.id, time: Date.now(), reason: 'error' })
await flushAsyncWork()
expect(restarted.getSessionByNamespace(persisted.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
} finally { restarted.stop() }
})
it('clears a persisted Pi attempt when native-ready arrives after Hub restart', async () => {
const store = new Store(':memory:')
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
const session = engine.getOrCreateSession('pi-ready-after-restart', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-ready',
piResumeAttempt: { state: 'resuming', machineId: 'machine-1', startedAt: 1 },
}, null, 'default')
engine.handleSessionReady({ sid: session.id, time: Date.now() })
await flushAsyncWork()
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
} finally { engine.stop() }
})
it('clears same-process Pi quarantine when a late validated ready arrives', async () => {
const store = new Store(':memory:')
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
const session = engine.getOrCreateSession('pi-late-ready', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-late-ready',
piResumeAttempt: { state: 'quarantined', machineId: 'machine-1', startedAt: 1 },
}, null, 'default')
;(engine as any).piResumeQuarantinedIds.add(session.id)
engine.handleSessionReady({ sid: session.id, time: Date.now() })
await flushAsyncWork()
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
expect((engine as any).piResumeQuarantinedIds.has(session.id)).toBe(false)
} finally { engine.stop() }
})
it('does not report active Pi attempts as reopened before validated ready', async () => {
for (const state of ['resuming', 'terminating'] as const) {
const store = new Store(':memory:')
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
const session = engine.getOrCreateSession(`pi-active-${state}`, {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: `pi-native-${state}`,
piResumeAttempt: { state, machineId: 'machine-1', startedAt: 1 },
}, null, 'default')
engine.handleSessionAlive({ sid: session.id, time: Date.now() })
expect(await engine.reopenSession(session.id, 'default')).toMatchObject({
type: 'error', message: 'Pi resume is already in progress'
})
} finally { engine.stop() }
}
})
it('reconciles persisted quarantined Pi state when the runner reports the child already gone', async () => {
const store = new Store(':memory:')
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
const session = engine.getOrCreateSession('pi-stale-quarantine', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-stale-quarantine',
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
piResumeAttempt: { state: 'quarantined', machineId: 'machine-1', startedAt: 1 },
}, null, 'default')
;(engine as any).rpcGateway.stopRunnerSession = async () => 'already_gone'
expect(await engine.reopenSession(session.id, 'default')).toMatchObject({
type: 'error', message: 'Previous Pi resume attempt was cleaned up; retry'
})
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
} finally { engine.stop() }
})
it('reconciles a persisted Pi attempt with an already-gone runner child without clearing archive state', async () => {
const store = new Store(':memory:')
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
const session = engine.getOrCreateSession('pi-stale-resume-attempt', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-stale',
// Mirrors bootstrapExistingSession before native get_state: the
// live metadata says running, while the attempt carries the
// exact archived snapshot needed if the child is already gone.
lifecycleState: 'running', lifecycleStateSince: 200,
piResumeAttempt: {
state: 'resuming', machineId: 'machine-1', startedAt: 1,
archiveSnapshot: {
lifecycleState: 'archived', lifecycleStateSince: 100,
archivedBy: 'cli', archiveReason: 'Pi exited',
},
},
}, null, 'default')
;(engine as any).rpcGateway.stopRunnerSession = async () => 'already_gone'
expect(await engine.reopenSession(session.id, 'default')).toMatchObject({
type: 'error', message: 'Previous Pi resume attempt was cleaned up; retry'
})
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.lifecycleState).toBe('archived')
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.lifecycleStateSince).toBe(100)
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.archivedBy).toBe('cli')
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.archiveReason).toBe('Pi exited')
} finally { engine.stop() }
})
it('does not quarantine an in-place Pi row when session-end wins the stop response race', async () => {
const store = new Store(':memory:')
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
const session = engine.getOrCreateSession('pi-stop-end-race', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-race',
piResumeAttempt: { state: 'resuming', machineId: 'machine-1', startedAt: 1 },
}, null, 'default')
engine.handleSessionAlive({ sid: session.id, time: Date.now() })
;(engine as any).rpcGateway.stopRunnerSession = async () => {
engine.handleSessionEnd({ sid: session.id, time: Date.now(), reason: 'error' })
return 'still_alive'
}
expect(await (engine as any).terminateInPlacePiResume('machine-1', session.id, 'default', true)).toBe(true)
await flushAsyncWork()
expect(engine.getSessionByNamespace(session.id, 'default')?.active).toBe(false)
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
} finally { engine.stop() }
})
it('does not persist an unexpected-child quarantine when child end wins the stop response race', async () => {
const store = new Store(':memory:')
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
const original = engine.getOrCreateSession('pi-original-stop-race', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-race',
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
piResumeAttempt: { state: 'resuming', machineId: 'machine-1', startedAt: 1 },
}, null, 'default')
const temp = engine.getOrCreateSession('pi-temp-stop-race', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi',
}, null, 'default')
engine.handleSessionAlive({ sid: temp.id, time: Date.now() })
;(engine as any).rpcGateway.stopRunnerSession = async () => {
engine.handleSessionEnd({ sid: temp.id, time: Date.now(), reason: 'error' })
return 'still_alive'
}
expect(await (engine as any).terminateUnexpectedPiTemp('machine-1', temp.id, original.id, 'default')).toBe(true)
await flushAsyncWork()
expect(store.sessions.getSession(temp.id)).toBeNull()
expect(engine.getSessionByNamespace(original.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
} finally { engine.stop() }
})
it('keeps still-alive Pi children quarantined even when Hub cache is inactive', async () => {
const store = new Store(':memory:')
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
const original = engine.getOrCreateSession('pi-still-alive-inactive', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-still-alive',
piResumeAttempt: { state: 'resuming', machineId: 'machine-1', startedAt: 1 },
}, null, 'default')
;(engine as any).rpcGateway.stopRunnerSession = async () => 'still_alive'
expect(await (engine as any).terminateInPlacePiResume('machine-1', original.id, 'default')).toBe(false)
const temp = engine.getOrCreateSession('pi-temp-still-alive-inactive', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi',
}, null, 'default')
expect(await (engine as any).terminateUnexpectedPiTemp('machine-1', temp.id, original.id, 'default')).toBe(false)
expect(store.sessions.getSession(temp.id)).not.toBeNull()
expect(engine.getSessionByNamespace(original.id, 'default')?.metadata?.piResumeAttempt).toMatchObject({
state: 'quarantined', childSessionId: temp.id
})
} finally { engine.stop() }
})
it('blocks dedup for a persisted unexpected Pi child and clears the original attempt on child end', async () => {
const store = new Store(':memory:')
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
const temp = engine.getOrCreateSession('pi-temp-mapped', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-mapped',
}, null, 'default')
const original = engine.getOrCreateSession('pi-original-mapped', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-mapped',
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
piResumeAttempt: { state: 'quarantined', machineId: 'machine-1', startedAt: 1, childSessionId: temp.id },
}, null, 'default')
let dedupCalls = 0
;(engine as any).sessionCache.deduplicateByAgentSessionId = async () => { dedupCalls += 1 }
;(engine as any).triggerDedupIfNeeded(temp.id)
await flushAsyncWork()
expect(dedupCalls).toBe(0)
expect(store.sessions.getSession(original.id)).not.toBeNull()
engine.handleSessionEnd({ sid: temp.id, time: Date.now(), reason: 'error' })
await flushAsyncWork()
expect(engine.getSessionByNamespace(original.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
expect(store.sessions.getSession(original.id)).not.toBeNull()
expect(dedupCalls).toBe(0)
} finally { engine.stop() }
})
it('blocks Pi dedup before an unexpected child ID is attached to the persisted attempt', async () => {
const store = new Store(':memory:')
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
try {
const original = engine.getOrCreateSession('pi-original-pre-mapping', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-pre-mapping',
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
piResumeAttempt: { state: 'resuming', machineId: 'machine-1', startedAt: 1 },
}, null, 'default')
const temp = engine.getOrCreateSession('pi-temp-pre-mapping', {
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-pre-mapping',
}, null, 'default')
engine.handleSessionAlive({ sid: temp.id, time: Date.now() })
let dedupCalls = 0
;(engine as any).sessionCache.deduplicateByAgentSessionId = async () => { dedupCalls += 1 }
;(engine as any).triggerDedupIfNeeded(temp.id)
await flushAsyncWork()
expect(dedupCalls).toBe(0)
expect(store.sessions.getSession(original.id)).not.toBeNull()
} finally { engine.stop() }
})
it('defers mergeSessions for cursor reopen until session-ready (load failure leaves old row)', async () => {
const store = new Store(':memory:')
const engine = new SyncEngine(
+395 -69
View File
@@ -46,6 +46,8 @@ import {
} from './rpcGateway'
import { SessionCache } from './sessionCache'
type PiResumeAttempt = NonNullable<NonNullable<Session['metadata']>['piResumeAttempt']>
export type { Session, SyncEvent } from '@hapi/protocol/types'
export type { Machine } from './machineCache'
export type { SyncEventListener } from './eventPublisher'
@@ -72,7 +74,7 @@ export type {
export type ResumeSessionResult =
| { type: 'success'; sessionId: string }
| { type: 'error'; message: string; code: 'session_not_found' | 'access_denied' | 'no_machine_online' | 'resume_unavailable' | 'resume_failed' }
| { type: 'error'; message: string; code: 'session_not_found' | 'access_denied' | 'no_machine_online' | 'resume_unavailable' | 'resume_failed'; rollbackSafe?: boolean }
export type ReopenSessionResult =
| { type: 'success'; sessionId: string; resumed: boolean; cursorSessionProtocol?: 'acp' | 'stream-json' }
@@ -147,8 +149,14 @@ export class SyncEngine {
private readonly messageService: MessageService
private readonly rpcGateway: RpcGateway
private inactivityTimer: NodeJS.Timeout | null = null
/** Sessions that emitted `session-ready` (Cursor ACP load/newSession complete). */
/** Sessions that emitted `session-ready` (Cursor ACP or validated Pi get_state). */
private readonly sessionReadyIds = new Set<string>()
/** Original Pi rows with a native resume currently in flight. */
private readonly piResumeInFlightIds = new Set<string>()
/** Pi rows whose runner child could not be confirmed terminated. */
private readonly piResumeQuarantinedIds = new Set<string>()
/** Unexpected version-skew temp child -> original row whose retry is blocked until child ends. */
private readonly piUnexpectedTempOriginalIds = new Map<string, string>()
/** Serialize scratchlist uploads per session so disk-byte caps cannot race. */
private readonly scratchlistUploadTails = new Map<string, Promise<unknown>>()
@@ -402,6 +410,15 @@ export class SyncEngine {
handleSessionReady(payload: { sid: string; time: number }): void {
this.sessionReadyIds.add(payload.sid)
const session = this.sessionCache.getSession(payload.sid)
if (session?.metadata?.piResumeAttempt) {
void this.writePiResumeAttempt(payload.sid, session.namespace, null)
.then(() => {
this.piResumeQuarantinedIds.delete(payload.sid)
this.triggerDedupIfNeeded(payload.sid)
})
.catch(() => {})
}
this.triggerDedupIfNeeded(payload.sid)
}
@@ -411,9 +428,14 @@ export class SyncEngine {
handleSessionEnd(payload: { sid: string; time: number; reason?: 'completed' | 'terminated' | 'error' }): void {
const before = this.sessionCache.getSession(payload.sid)
const ownsPiAttempt = before?.metadata?.piResumeAttempt !== undefined
const isPiAttemptChild = this.sessionCache.getSessions().some(
(session) => session.metadata?.piResumeAttempt?.childSessionId === payload.sid
)
const restorePiArchive = ownsPiAttempt && !this.sessionReadyIds.has(payload.sid)
const isCursorAcp = before?.metadata?.flavor === 'cursor'
&& before.metadata.cursorSessionProtocol === 'acp'
const shouldRetryDedup = !isCursorAcp || this.sessionReadyIds.has(payload.sid)
const shouldRetryDedup = !ownsPiAttempt && !isPiAttemptChild && (!isCursorAcp || this.sessionReadyIds.has(payload.sid))
this.sessionCache.handleSessionEnd(payload)
this.eventPublisher.emit({
@@ -428,6 +450,11 @@ export class SyncEngine {
this.triggerDedupIfNeeded(payload.sid)
}
this.sessionReadyIds.delete(payload.sid)
this.piResumeQuarantinedIds.delete(payload.sid)
this.piUnexpectedTempOriginalIds.delete(payload.sid)
if (ownsPiAttempt || isPiAttemptChild) {
void this.clearPiAttemptForEndedSession(payload.sid, restorePiArchive)
}
}
handleBackgroundTaskDelta(sessionId: string, delta: { started: number; completed: number }): void {
@@ -1539,6 +1566,35 @@ async uploadScratchlistAttachment(
return { type: 'error', message: 'No machine online', code: 'no_machine_online' }
}
if (flavor === 'pi' && resumeToken && targetMachine.runnerState?.capabilities?.piExistingSessionResume !== true) {
return { type: 'error', message: 'Pi resume requires an upgraded runner', code: 'resume_failed' }
}
const requiresPiNativeReady = flavor === 'pi' && resumeToken !== undefined
if (requiresPiNativeReady) {
if (this.isPiResumeBlocked(access.sessionId)) {
return { type: 'error', message: 'Pi resume is already in progress', code: 'resume_failed' }
}
this.piResumeInFlightIds.add(access.sessionId)
this.sessionReadyIds.delete(access.sessionId)
try {
await this.writePiResumeAttempt(access.sessionId, namespace, {
state: 'resuming',
machineId: targetMachine.id,
startedAt: Date.now(),
archiveSnapshot: {
lifecycleState: metadata.lifecycleState,
lifecycleStateSince: metadata.lifecycleStateSince,
archivedBy: metadata.archivedBy,
archiveReason: metadata.archiveReason,
},
})
} catch {
this.piResumeInFlightIds.delete(access.sessionId)
return { type: 'error', message: 'Failed to record Pi resume attempt', code: 'resume_failed' }
}
}
if (flavor === 'cursor' && resumeToken) {
try {
const chatStatus = await this.rpcGateway.getCursorChatStoreStatus(
@@ -1569,65 +1625,135 @@ async uploadScratchlistAttachment(
: opts?.permissionMode
?? session.permissionMode
?? metadataPermissionMode
const spawnResult = await this.rpcGateway.spawnSession(
targetMachine.id,
directory,
flavor,
session.model ?? undefined,
session.modelReasoningEffort ?? undefined,
undefined,
undefined,
undefined,
resumeToken,
session.effort ?? undefined,
preferredPermissionMode,
session.serviceTier ?? undefined,
access.sessionId,
session.collaborationMode ?? undefined
)
let piResumeSucceeded = false
try {
const spawnResult = await this.rpcGateway.spawnSession(
targetMachine.id,
directory,
flavor,
session.model ?? undefined,
session.modelReasoningEffort ?? undefined,
undefined,
undefined,
undefined,
resumeToken,
session.effort ?? undefined,
preferredPermissionMode,
session.serviceTier ?? undefined,
access.sessionId,
session.collaborationMode ?? undefined
)
if (spawnResult.type !== 'success') {
return { type: 'error', message: spawnResult.message, code: 'resume_failed' }
}
const becameActive = await this.waitForSessionActive(spawnResult.sessionId)
if (!becameActive) {
return { type: 'error', message: 'Session failed to become active', code: 'resume_failed' }
}
// permissionMode is passed to spawnSession above; do not call set-session-config here.
// session-alive can arrive before the CLI registers that RPC handler, which caused resume_failed.
const needsReadyBeforeMerge = spawnResult.sessionId !== access.sessionId
&& flavor === 'cursor'
&& metadata.cursorSessionProtocol === 'acp'
if (needsReadyBeforeMerge) {
const readyResult = await this.waitForSessionReady(spawnResult.sessionId)
if (readyResult !== 'ready') {
const message = readyResult === 'ended'
? 'Session ended before Cursor ACP load completed'
: 'Session failed to become ready'
return { type: 'error', message, code: 'resume_failed' }
if (spawnResult.type !== 'success') {
if (requiresPiNativeReady) {
const stopped = await this.terminateInPlacePiResume(
targetMachine.id,
access.sessionId,
namespace
)
if (!stopped) {
await this.quarantinePiResume(access.sessionId, namespace, targetMachine.id)
return {
type: 'error',
message: spawnResult.message,
code: 'resume_failed',
rollbackSafe: false,
}
}
}
return { type: 'error', message: spawnResult.message, code: 'resume_failed' }
}
}
if (spawnResult.sessionId !== access.sessionId) {
// The old session may have already been merged by the automatic dedup path
// (triggered when the spawned CLI sets its agent session ID in metadata).
// Only attempt the explicit merge if the old session still exists.
const oldSession = this.sessionCache.getSessionByNamespace(access.sessionId, namespace)
if (oldSession) {
try {
await this.sessionCache.mergeSessions(access.sessionId, spawnResult.sessionId, namespace)
} catch (error) {
const message = error instanceof Error ? error.message : 'Failed to merge resumed session'
if (requiresPiNativeReady && spawnResult.sessionId !== access.sessionId) {
const removed = await this.terminateUnexpectedPiTemp(
targetMachine.id,
spawnResult.sessionId,
access.sessionId,
namespace
)
return {
type: 'error',
message: removed
? 'Pi runner created an unexpected session; upgrade the runner and retry'
: 'Pi runner created an unexpected live session; upgrade the runner and retry',
code: 'resume_failed'
}
}
const becameActive = await this.waitForSessionActive(spawnResult.sessionId)
if (!becameActive) {
if (requiresPiNativeReady) {
const inactive = await this.terminateInPlacePiResume(
targetMachine.id,
access.sessionId,
namespace
)
if (!inactive) {
await this.quarantinePiResume(access.sessionId, namespace, targetMachine.id)
return { type: 'error', message: 'Pi resume failed and the child is still active', code: 'resume_failed', rollbackSafe: false }
}
}
return { type: 'error', message: 'Session failed to become active', code: 'resume_failed' }
}
const needsReadyBeforeSuccess = requiresPiNativeReady
|| (
spawnResult.sessionId !== access.sessionId
&& flavor === 'cursor'
&& metadata.cursorSessionProtocol === 'acp'
)
if (needsReadyBeforeSuccess) {
const readyResult = await this.waitForSessionReady(spawnResult.sessionId)
if (readyResult !== 'ready') {
if (requiresPiNativeReady && readyResult !== 'ended') {
const inactive = await this.terminateInPlacePiResume(
targetMachine.id,
access.sessionId,
namespace
)
if (!inactive) {
await this.quarantinePiResume(access.sessionId, namespace, targetMachine.id)
return { type: 'error', message: 'Pi native resume timed out and the child is still active', code: 'resume_failed', rollbackSafe: false }
}
}
const message = flavor === 'pi'
? readyResult === 'ended'
? 'Pi session ended before native resume completed'
: 'Pi session failed to become native-ready'
: readyResult === 'ended'
? 'Session ended before Cursor ACP load completed'
: 'Session failed to become ready'
return { type: 'error', message, code: 'resume_failed' }
}
}
}
this.sessionCache.markSessionActive(spawnResult.sessionId)
return { type: 'success', sessionId: spawnResult.sessionId }
if (spawnResult.sessionId !== access.sessionId) {
const oldSession = this.sessionCache.getSessionByNamespace(access.sessionId, namespace)
if (oldSession) {
try {
await this.sessionCache.mergeSessions(access.sessionId, spawnResult.sessionId, namespace)
} catch (error) {
const message = error instanceof Error ? error.message : 'Failed to merge resumed session'
return { type: 'error', message, code: 'resume_failed' }
}
}
}
this.sessionCache.markSessionActive(spawnResult.sessionId)
piResumeSucceeded = true
if (requiresPiNativeReady) await this.writePiResumeAttempt(access.sessionId, namespace, null)
return { type: 'success', sessionId: spawnResult.sessionId }
} finally {
if (requiresPiNativeReady) {
this.piResumeInFlightIds.delete(access.sessionId)
if (!piResumeSucceeded && this.sessionCache.getSession(access.sessionId)?.metadata?.piResumeAttempt?.state === 'resuming') {
await this.writePiResumeAttempt(access.sessionId, namespace, null, true).catch(() => {})
}
if (piResumeSucceeded) {
this.triggerDedupIfNeeded(access.sessionId)
}
}
}
}
/**
@@ -1664,6 +1790,23 @@ async uploadScratchlistAttachment(
const session = access.session
const metadata = session.metadata
if (metadata?.flavor === 'pi' && this.isPiResumeBlocked(access.sessionId)) {
if (session.active) {
return { type: 'error', message: 'Pi resume is already in progress', code: 'resume_failed' }
}
if (metadata.piResumeAttempt && !this.piResumeInFlightIds.has(access.sessionId)) {
const reconciled = await this.reconcilePersistedPiResumeAttempt(session)
return {
type: 'error',
message: reconciled
? 'Previous Pi resume attempt was cleaned up; retry'
: 'Pi resume is already in progress',
code: 'resume_failed'
}
}
return { type: 'error', message: 'Pi resume is already in progress', code: 'resume_failed' }
}
if (session.active) {
return { type: 'success', sessionId: access.sessionId, resumed: false }
}
@@ -1689,24 +1832,30 @@ async uploadScratchlistAttachment(
lifecycleStateSince: metadata.lifecycleStateSince
}
let applied: { cursorSessionProtocol?: 'acp' | 'stream-json' }
try {
applied = await this.sessionCache.clearSessionArchiveMetadata(access.sessionId)
} catch (error) {
const message = error instanceof Error ? error.message : 'Failed to clear archive metadata'
return { type: 'error', message, code: 'metadata_conflict' }
let applied: { cursorSessionProtocol?: 'acp' | 'stream-json' } = {}
// Pi reuses the original HAPI row. Keep its archive snapshot persisted
// until the CLI successfully bootstraps that row as running; this avoids
// an inactive, non-archived gap if the Hub restarts before spawn.
if (metadata.flavor !== 'pi') {
try {
applied = await this.sessionCache.clearSessionArchiveMetadata(access.sessionId)
} catch (error) {
const message = error instanceof Error ? error.message : 'Failed to clear archive metadata'
return { type: 'error', message, code: 'metadata_conflict' }
}
}
const resumeResult = await this.resumeSession(access.sessionId, namespace)
if (resumeResult.type === 'error') {
// Resume failed - put the archive flags back so the row stays archived in the UI
// and the operator can retry. Best-effort: a concurrent metadata write that
// succeeded between clear and restore (e.g. an unrelated rename) wins, in
// which case we surface the original resume error rather than masking it.
try {
await this.sessionCache.restoreSessionArchiveMetadata(access.sessionId, archiveSnapshot)
} catch {
// Swallow restore failures - the resume error is the more important signal.
// Never restore archived metadata over a live Pi child. A live
// row blocks retry by itself and must remain visible as active.
const current = this.sessionCache.getSessionByNamespace(access.sessionId, namespace)
if (resumeResult.rollbackSafe !== false && !current?.active) {
try {
await this.sessionCache.restoreSessionArchiveMetadata(access.sessionId, archiveSnapshot)
} catch {
// Swallow restore failures - the resume error is the more important signal.
}
}
return resumeResult
}
@@ -1928,6 +2077,16 @@ async uploadScratchlistAttachment(
}
private canRunCursorDedup(session: Session): boolean {
if (this.piResumeInFlightIds.has(session.id) || this.piResumeQuarantinedIds.has(session.id)) return false
if (session.metadata?.piResumeAttempt) return false
if (this.sessionCache.getSessions().some((candidate) => candidate.metadata?.piResumeAttempt?.childSessionId === session.id)) return false
const piSessionId = session.metadata?.piSessionId
if (piSessionId && this.sessionCache.getSessions().some((candidate) =>
candidate.id !== session.id
&& candidate.namespace === session.namespace
&& candidate.metadata?.piResumeAttempt !== undefined
&& candidate.metadata.piSessionId === piSessionId
)) return false
if (session.metadata?.flavor !== 'cursor') {
return true
}
@@ -1937,6 +2096,173 @@ async uploadScratchlistAttachment(
return this.sessionReadyIds.has(session.id)
}
private async terminateInPlacePiResume(
machineId: string,
sessionId: string,
namespace: string
): Promise<boolean> {
const existingAttempt = this.sessionCache.getSession(sessionId)?.metadata?.piResumeAttempt
await this.writePiResumeAttempt(sessionId, namespace, {
...existingAttempt,
state: 'terminating',
machineId,
startedAt: Date.now(),
})
let status: 'stopped' | 'already_gone' | 'still_alive'
try {
status = await this.rpcGateway.stopRunnerSession(machineId, sessionId)
} catch {
status = 'still_alive'
}
await new Promise((resolve) => setTimeout(resolve, 0))
const session = this.sessionCache.refreshSession(sessionId) ?? this.sessionCache.getSession(sessionId)
const attemptClearedByEnd = session?.metadata?.piResumeAttempt === undefined
if (status === 'still_alive') {
if (attemptClearedByEnd) return true
return false
}
if (session?.active) this.handleSessionEnd({ sid: sessionId, time: Date.now(), reason: 'error' })
await this.writePiResumeAttempt(sessionId, namespace, null, true).catch(() => {})
return true
}
private async terminateUnexpectedPiTemp(
machineId: string,
sessionId: string,
originalSessionId: string,
namespace: string
): Promise<boolean> {
this.piUnexpectedTempOriginalIds.set(sessionId, originalSessionId)
const existingAttempt = this.sessionCache.getSession(originalSessionId)?.metadata?.piResumeAttempt
await this.writePiResumeAttempt(originalSessionId, namespace, {
...existingAttempt,
state: 'terminating',
machineId,
startedAt: Date.now(),
childSessionId: sessionId,
})
let status: 'stopped' | 'already_gone' | 'still_alive'
try {
status = await this.rpcGateway.stopRunnerSession(machineId, sessionId)
} catch {
status = 'still_alive'
}
await new Promise((resolve) => setTimeout(resolve, 0))
const session = this.sessionCache.refreshSession(sessionId) ?? this.sessionCache.getSession(sessionId)
const original = this.sessionCache.refreshSession(originalSessionId) ?? this.sessionCache.getSession(originalSessionId)
const attemptClearedByEnd = original?.metadata?.piResumeAttempt === undefined
if (status === 'still_alive' && !attemptClearedByEnd) {
await this.writePiResumeAttempt(originalSessionId, namespace, {
...existingAttempt,
state: 'quarantined',
machineId,
startedAt: Date.now(),
childSessionId: sessionId,
})
return false
}
if (session?.active) this.handleSessionEnd({ sid: sessionId, time: Date.now(), reason: 'error' })
const remaining = this.sessionCache.getSession(sessionId)
if (remaining && !remaining.active) await this.sessionCache.deleteSession(sessionId)
this.piUnexpectedTempOriginalIds.delete(sessionId)
await this.writePiResumeAttempt(originalSessionId, namespace, null, true).catch(() => {})
return true
}
private async quarantinePiResume(sessionId: string, namespace: string, machineId: string): Promise<void> {
this.piResumeQuarantinedIds.add(sessionId)
const existingAttempt = this.sessionCache.getSession(sessionId)?.metadata?.piResumeAttempt
await this.writePiResumeAttempt(sessionId, namespace, {
...existingAttempt,
state: 'quarantined',
machineId,
startedAt: Date.now(),
})
}
private isPiResumeBlocked(sessionId: string): boolean {
const metadataAttempt = this.sessionCache.getSession(sessionId)?.metadata?.piResumeAttempt
return this.piResumeInFlightIds.has(sessionId)
|| this.piResumeQuarantinedIds.has(sessionId)
|| metadataAttempt !== undefined
|| [...this.piUnexpectedTempOriginalIds.values()].includes(sessionId)
}
private async writePiResumeAttempt(
sessionId: string,
namespace: string,
attempt: PiResumeAttempt | null,
restoreArchive = false
): Promise<void> {
for (let i = 0; i < 5; i += 1) {
const current = this.sessionCache.getSessionByNamespace(sessionId, namespace) ?? this.sessionCache.refreshSession(sessionId)
if (!current?.metadata) return
const next = { ...current.metadata }
if (attempt) next.piResumeAttempt = attempt
else {
const snapshot = current.metadata.piResumeAttempt?.archiveSnapshot
delete next.piResumeAttempt
if (restoreArchive && snapshot) {
if (snapshot.lifecycleState === undefined) delete next.lifecycleState
else next.lifecycleState = snapshot.lifecycleState
if (snapshot.lifecycleStateSince === undefined) delete next.lifecycleStateSince
else next.lifecycleStateSince = snapshot.lifecycleStateSince
if (snapshot.archivedBy === undefined) delete next.archivedBy
else next.archivedBy = snapshot.archivedBy
if (snapshot.archiveReason === undefined) delete next.archiveReason
else next.archiveReason = snapshot.archiveReason
}
}
const result = this.store.sessions.updateSessionMetadata(sessionId, next, current.metadataVersion, namespace, { touchUpdatedAt: false })
if (result.result === 'success') {
this.sessionCache.refreshSession(sessionId)
return
}
if (result.result !== 'version-mismatch') throw new Error('Failed to update Pi resume attempt')
this.sessionCache.refreshSession(sessionId)
}
throw new Error('Pi resume attempt metadata was modified concurrently')
}
private async clearPiAttemptForEndedSession(endedSessionId: string, restoreArchive: boolean): Promise<void> {
const ended = this.sessionCache.getSession(endedSessionId)
if (ended?.metadata?.piResumeAttempt) {
await this.writePiResumeAttempt(endedSessionId, ended.namespace, null, restoreArchive).catch(() => {})
return
}
for (const session of this.sessionCache.getSessions()) {
if (session.metadata?.piResumeAttempt?.childSessionId === endedSessionId) {
await this.writePiResumeAttempt(session.id, session.namespace, null, true).catch(() => {})
}
}
}
private async reconcilePersistedPiResumeAttempt(session: Session): Promise<boolean> {
const attempt = session.metadata?.piResumeAttempt
if (!attempt) return true
const childSessionId = attempt.childSessionId ?? session.id
let status: 'stopped' | 'already_gone' | 'still_alive'
try {
status = await this.rpcGateway.stopRunnerSession(attempt.machineId, childSessionId)
} catch {
return false
}
if (status === 'still_alive') return false
const child = this.sessionCache.getSession(childSessionId)
if (child?.active) this.handleSessionEnd({ sid: childSessionId, time: Date.now(), reason: 'error' })
if (childSessionId !== session.id) {
const remaining = this.sessionCache.getSession(childSessionId)
if (remaining && !remaining.active) await this.sessionCache.deleteSession(childSessionId)
}
await this.writePiResumeAttempt(session.id, session.namespace, null, true)
this.piResumeQuarantinedIds.delete(session.id)
this.piUnexpectedTempOriginalIds.delete(childSessionId)
return true
}
private triggerDedupIfNeeded(sessionId: string): void {
const session = this.sessionCache.getSession(sessionId)
if (session?.metadata) {