mirror of
https://github.com/wu736139669/hapi.git
synced 2026-08-05 06:24:37 +00:00
fix(pi): resume archived sessions safely (#1308)
* fix(pi): resume archived sessions safely * fix(pi): harden native resume startup * fix(pi): harden resume termination evidence * fix(runner): persist resume process evidence * fix(runner): track resume process generations * fix(runner): verify full session tree shutdown * fix(pi): block pre-mapping resume dedup
This commit is contained in:
@@ -140,6 +140,13 @@ export class RpcGateway {
|
||||
await this.sessionRpc(sessionId, RPC_METHODS.KillSession, {})
|
||||
}
|
||||
|
||||
async stopRunnerSession(machineId: string, sessionId: string): Promise<'stopped' | 'already_gone' | 'still_alive'> {
|
||||
const result = await this.machineRpc(machineId, RPC_METHODS.StopSession, { sessionId })
|
||||
const status = result && typeof result === 'object' ? (result as { status?: unknown }).status : undefined
|
||||
if (status === 'stopped' || status === 'already_gone' || status === 'still_alive') return status
|
||||
throw new Error('Unexpected stop-session response')
|
||||
}
|
||||
|
||||
async handoffSessionToLocal(sessionId: string): Promise<void> {
|
||||
await this.sessionRpc(sessionId, RPC_METHODS.HandoffLocal, {})
|
||||
}
|
||||
|
||||
@@ -16,6 +16,11 @@ function createPublisher(events: SyncEvent[]): EventPublisher {
|
||||
} as unknown as EventPublisher
|
||||
}
|
||||
|
||||
async function flushAsyncWork(): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, 0))
|
||||
await new Promise((resolve) => setTimeout(resolve, 0))
|
||||
}
|
||||
|
||||
function productionCodexMessage(event: Record<string, unknown>): Record<string, unknown> {
|
||||
return {
|
||||
role: 'agent',
|
||||
@@ -2003,6 +2008,375 @@ describe('session model', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('reopens Pi in place only after native-ready, preserving its id and history', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
const session = engine.getOrCreateSession('pi-in-place', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-1',
|
||||
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
|
||||
}, null, 'default')
|
||||
store.messages.addMessage(session.id, { role: 'user', content: { type: 'text', text: 'keep history' } })
|
||||
engine.getOrCreateMachine('machine-1', { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, { status: 'running', capabilities: { piExistingSessionResume: true } }, 'default')
|
||||
engine.handleMachineAlive({ machineId: 'machine-1', time: Date.now() })
|
||||
engine.handleSessionEnd({ sid: session.id, time: Date.now() })
|
||||
let existing: string | undefined
|
||||
let merges = 0
|
||||
;(engine as any).sessionCache.mergeSessions = async () => { merges += 1 }
|
||||
;(engine as any).rpcGateway.spawnSession = async (...args: Parameters<SyncEngine['spawnSession']>) => {
|
||||
existing = args[12]
|
||||
engine.handleSessionAlive({ sid: session.id, time: Date.now() })
|
||||
engine.handleSessionReady({ sid: session.id, time: Date.now() })
|
||||
return { type: 'success', sessionId: session.id }
|
||||
}
|
||||
const result = await engine.reopenSession(session.id, 'default')
|
||||
expect(result).toEqual({ type: 'success', sessionId: session.id, resumed: true })
|
||||
expect(existing).toBe(session.id)
|
||||
expect(merges).toBe(0)
|
||||
expect(store.messages.getFirstMessages(session.id, 10)).toHaveLength(1)
|
||||
} finally { engine.stop() }
|
||||
})
|
||||
|
||||
it('kills and deletes an unexpected legacy Pi temp without touching the original row', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
const original = engine.getOrCreateSession('pi-original', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-old',
|
||||
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
|
||||
}, null, 'default')
|
||||
const unexpected = engine.getOrCreateSession('pi-unexpected', { path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi' }, null, 'default')
|
||||
engine.getOrCreateMachine('machine-1', { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, { status: 'running', capabilities: { piExistingSessionResume: true } }, 'default')
|
||||
engine.handleMachineAlive({ machineId: 'machine-1', time: Date.now() })
|
||||
engine.handleSessionEnd({ sid: original.id, time: Date.now() })
|
||||
;(engine as any).rpcGateway.spawnSession = async () => ({ type: 'success', sessionId: unexpected.id })
|
||||
;(engine as any).rpcGateway.stopRunnerSession = async (_machineId: string, sid: string) => {
|
||||
engine.handleSessionEnd({ sid, time: Date.now(), reason: 'error' })
|
||||
return 'stopped'
|
||||
}
|
||||
const result = await engine.reopenSession(original.id, 'default')
|
||||
expect(result).toMatchObject({ type: 'error', code: 'resume_failed', message: expect.stringContaining('upgrade') })
|
||||
expect(store.sessions.getSession(unexpected.id)).toBeNull()
|
||||
expect(store.sessions.getSession(original.id)).not.toBeNull()
|
||||
expect(engine.getSessionByNamespace(original.id, 'default')?.metadata?.lifecycleState).toBe('archived')
|
||||
} finally { engine.stop() }
|
||||
})
|
||||
|
||||
it('does not restore archive metadata over a live Pi child after kill failure', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
const session = engine.getOrCreateSession('pi-live-failure', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-live',
|
||||
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
|
||||
}, null, 'default')
|
||||
engine.getOrCreateMachine('machine-1', { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, { status: 'running', capabilities: { piExistingSessionResume: true } }, 'default')
|
||||
engine.handleMachineAlive({ machineId: 'machine-1', time: Date.now() })
|
||||
engine.handleSessionEnd({ sid: session.id, time: Date.now() })
|
||||
;(engine as any).rpcGateway.spawnSession = async () => {
|
||||
engine.handleSessionAlive({ sid: session.id, time: Date.now() })
|
||||
return { type: 'success', sessionId: session.id }
|
||||
}
|
||||
;(engine as any).waitForSessionReady = async () => 'timeout'
|
||||
;(engine as any).rpcGateway.stopRunnerSession = async () => 'still_alive'
|
||||
const result = await engine.reopenSession(session.id, 'default')
|
||||
expect(result).toMatchObject({ type: 'error', message: expect.stringContaining('still active') })
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.active).toBe(true)
|
||||
// Pi keeps the persisted archive snapshot until bootstrap succeeds,
|
||||
// so a failed stop never needs to reconstruct it from memory.
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.lifecycleState).toBe('archived')
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt?.state).toBe('quarantined')
|
||||
expect(await engine.reopenSession(session.id, 'default')).toMatchObject({ type: 'error', message: 'Pi resume is already in progress' })
|
||||
|
||||
engine.handleSessionEnd({ sid: session.id, time: Date.now(), reason: 'error' })
|
||||
await flushAsyncWork()
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
|
||||
} finally { engine.stop() }
|
||||
})
|
||||
|
||||
it('rejects Pi resume before spawn when the runner lacks in-place capability', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
const session = engine.getOrCreateSession('pi-old-runner', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-old-runner',
|
||||
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
|
||||
}, null, 'default')
|
||||
engine.getOrCreateMachine('machine-1', { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, { status: 'running' }, 'default')
|
||||
engine.handleMachineAlive({ machineId: 'machine-1', time: Date.now() })
|
||||
engine.handleSessionEnd({ sid: session.id, time: Date.now() })
|
||||
let spawnCalls = 0
|
||||
;(engine as any).rpcGateway.spawnSession = async () => { spawnCalls += 1; return { type: 'error', message: 'unexpected' } }
|
||||
|
||||
const result = await engine.reopenSession(session.id, 'default')
|
||||
expect(result).toMatchObject({ type: 'error', message: 'Pi resume requires an upgraded runner' })
|
||||
expect(spawnCalls).toBe(0)
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.lifecycleState).toBe('archived')
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
|
||||
} finally { engine.stop() }
|
||||
})
|
||||
|
||||
it('quarantines a Pi attempt when runner spawn fails before process termination is confirmed', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
const session = engine.getOrCreateSession('pi-spawn-error-live-child', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-spawn-error',
|
||||
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
|
||||
}, null, 'default')
|
||||
engine.getOrCreateMachine('machine-1', { host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' }, {
|
||||
status: 'running', capabilities: { piExistingSessionResume: true }
|
||||
}, 'default')
|
||||
engine.handleMachineAlive({ machineId: 'machine-1', time: Date.now() })
|
||||
engine.handleSessionEnd({ sid: session.id, time: Date.now() })
|
||||
;(engine as any).rpcGateway.spawnSession = async () => ({ type: 'error', message: 'webhook timeout' })
|
||||
;(engine as any).rpcGateway.stopRunnerSession = async () => 'still_alive'
|
||||
|
||||
expect(await engine.reopenSession(session.id, 'default')).toMatchObject({
|
||||
type: 'error', message: 'webhook timeout'
|
||||
})
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt?.state).toBe('quarantined')
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.lifecycleState).toBe('archived')
|
||||
expect(await engine.reopenSession(session.id, 'default')).toMatchObject({
|
||||
type: 'error', message: 'Pi resume is already in progress'
|
||||
})
|
||||
} finally { engine.stop() }
|
||||
})
|
||||
|
||||
it('keeps persisted Pi quarantine across SyncEngine restart and clears it on end', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const first = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
const persisted = first.getOrCreateSession('pi-persisted-attempt', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-persisted',
|
||||
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
|
||||
piResumeAttempt: { state: 'quarantined', machineId: 'machine-1', startedAt: 1 },
|
||||
}, null, 'default')
|
||||
first.stop()
|
||||
|
||||
const restarted = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
expect(await restarted.reopenSession(persisted.id, 'default')).toMatchObject({
|
||||
type: 'error', message: 'Pi resume is already in progress'
|
||||
})
|
||||
restarted.handleSessionEnd({ sid: persisted.id, time: Date.now(), reason: 'error' })
|
||||
await flushAsyncWork()
|
||||
expect(restarted.getSessionByNamespace(persisted.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
|
||||
} finally { restarted.stop() }
|
||||
})
|
||||
|
||||
it('clears a persisted Pi attempt when native-ready arrives after Hub restart', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
const session = engine.getOrCreateSession('pi-ready-after-restart', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-ready',
|
||||
piResumeAttempt: { state: 'resuming', machineId: 'machine-1', startedAt: 1 },
|
||||
}, null, 'default')
|
||||
engine.handleSessionReady({ sid: session.id, time: Date.now() })
|
||||
await flushAsyncWork()
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
|
||||
} finally { engine.stop() }
|
||||
})
|
||||
|
||||
it('clears same-process Pi quarantine when a late validated ready arrives', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
const session = engine.getOrCreateSession('pi-late-ready', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-late-ready',
|
||||
piResumeAttempt: { state: 'quarantined', machineId: 'machine-1', startedAt: 1 },
|
||||
}, null, 'default')
|
||||
;(engine as any).piResumeQuarantinedIds.add(session.id)
|
||||
engine.handleSessionReady({ sid: session.id, time: Date.now() })
|
||||
await flushAsyncWork()
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
|
||||
expect((engine as any).piResumeQuarantinedIds.has(session.id)).toBe(false)
|
||||
} finally { engine.stop() }
|
||||
})
|
||||
|
||||
it('does not report active Pi attempts as reopened before validated ready', async () => {
|
||||
for (const state of ['resuming', 'terminating'] as const) {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
const session = engine.getOrCreateSession(`pi-active-${state}`, {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: `pi-native-${state}`,
|
||||
piResumeAttempt: { state, machineId: 'machine-1', startedAt: 1 },
|
||||
}, null, 'default')
|
||||
engine.handleSessionAlive({ sid: session.id, time: Date.now() })
|
||||
expect(await engine.reopenSession(session.id, 'default')).toMatchObject({
|
||||
type: 'error', message: 'Pi resume is already in progress'
|
||||
})
|
||||
} finally { engine.stop() }
|
||||
}
|
||||
})
|
||||
|
||||
it('reconciles persisted quarantined Pi state when the runner reports the child already gone', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
const session = engine.getOrCreateSession('pi-stale-quarantine', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-stale-quarantine',
|
||||
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
|
||||
piResumeAttempt: { state: 'quarantined', machineId: 'machine-1', startedAt: 1 },
|
||||
}, null, 'default')
|
||||
;(engine as any).rpcGateway.stopRunnerSession = async () => 'already_gone'
|
||||
expect(await engine.reopenSession(session.id, 'default')).toMatchObject({
|
||||
type: 'error', message: 'Previous Pi resume attempt was cleaned up; retry'
|
||||
})
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
|
||||
} finally { engine.stop() }
|
||||
})
|
||||
|
||||
it('reconciles a persisted Pi attempt with an already-gone runner child without clearing archive state', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
const session = engine.getOrCreateSession('pi-stale-resume-attempt', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-stale',
|
||||
// Mirrors bootstrapExistingSession before native get_state: the
|
||||
// live metadata says running, while the attempt carries the
|
||||
// exact archived snapshot needed if the child is already gone.
|
||||
lifecycleState: 'running', lifecycleStateSince: 200,
|
||||
piResumeAttempt: {
|
||||
state: 'resuming', machineId: 'machine-1', startedAt: 1,
|
||||
archiveSnapshot: {
|
||||
lifecycleState: 'archived', lifecycleStateSince: 100,
|
||||
archivedBy: 'cli', archiveReason: 'Pi exited',
|
||||
},
|
||||
},
|
||||
}, null, 'default')
|
||||
;(engine as any).rpcGateway.stopRunnerSession = async () => 'already_gone'
|
||||
|
||||
expect(await engine.reopenSession(session.id, 'default')).toMatchObject({
|
||||
type: 'error', message: 'Previous Pi resume attempt was cleaned up; retry'
|
||||
})
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.lifecycleState).toBe('archived')
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.lifecycleStateSince).toBe(100)
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.archivedBy).toBe('cli')
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.archiveReason).toBe('Pi exited')
|
||||
} finally { engine.stop() }
|
||||
})
|
||||
|
||||
it('does not quarantine an in-place Pi row when session-end wins the stop response race', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
const session = engine.getOrCreateSession('pi-stop-end-race', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-race',
|
||||
piResumeAttempt: { state: 'resuming', machineId: 'machine-1', startedAt: 1 },
|
||||
}, null, 'default')
|
||||
engine.handleSessionAlive({ sid: session.id, time: Date.now() })
|
||||
;(engine as any).rpcGateway.stopRunnerSession = async () => {
|
||||
engine.handleSessionEnd({ sid: session.id, time: Date.now(), reason: 'error' })
|
||||
return 'still_alive'
|
||||
}
|
||||
|
||||
expect(await (engine as any).terminateInPlacePiResume('machine-1', session.id, 'default', true)).toBe(true)
|
||||
await flushAsyncWork()
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.active).toBe(false)
|
||||
expect(engine.getSessionByNamespace(session.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
|
||||
} finally { engine.stop() }
|
||||
})
|
||||
|
||||
it('does not persist an unexpected-child quarantine when child end wins the stop response race', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
const original = engine.getOrCreateSession('pi-original-stop-race', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-race',
|
||||
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
|
||||
piResumeAttempt: { state: 'resuming', machineId: 'machine-1', startedAt: 1 },
|
||||
}, null, 'default')
|
||||
const temp = engine.getOrCreateSession('pi-temp-stop-race', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi',
|
||||
}, null, 'default')
|
||||
engine.handleSessionAlive({ sid: temp.id, time: Date.now() })
|
||||
;(engine as any).rpcGateway.stopRunnerSession = async () => {
|
||||
engine.handleSessionEnd({ sid: temp.id, time: Date.now(), reason: 'error' })
|
||||
return 'still_alive'
|
||||
}
|
||||
|
||||
expect(await (engine as any).terminateUnexpectedPiTemp('machine-1', temp.id, original.id, 'default')).toBe(true)
|
||||
await flushAsyncWork()
|
||||
expect(store.sessions.getSession(temp.id)).toBeNull()
|
||||
expect(engine.getSessionByNamespace(original.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
|
||||
} finally { engine.stop() }
|
||||
})
|
||||
|
||||
it('keeps still-alive Pi children quarantined even when Hub cache is inactive', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
const original = engine.getOrCreateSession('pi-still-alive-inactive', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-still-alive',
|
||||
piResumeAttempt: { state: 'resuming', machineId: 'machine-1', startedAt: 1 },
|
||||
}, null, 'default')
|
||||
;(engine as any).rpcGateway.stopRunnerSession = async () => 'still_alive'
|
||||
expect(await (engine as any).terminateInPlacePiResume('machine-1', original.id, 'default')).toBe(false)
|
||||
|
||||
const temp = engine.getOrCreateSession('pi-temp-still-alive-inactive', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi',
|
||||
}, null, 'default')
|
||||
expect(await (engine as any).terminateUnexpectedPiTemp('machine-1', temp.id, original.id, 'default')).toBe(false)
|
||||
expect(store.sessions.getSession(temp.id)).not.toBeNull()
|
||||
expect(engine.getSessionByNamespace(original.id, 'default')?.metadata?.piResumeAttempt).toMatchObject({
|
||||
state: 'quarantined', childSessionId: temp.id
|
||||
})
|
||||
} finally { engine.stop() }
|
||||
})
|
||||
|
||||
it('blocks dedup for a persisted unexpected Pi child and clears the original attempt on child end', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
const temp = engine.getOrCreateSession('pi-temp-mapped', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-mapped',
|
||||
}, null, 'default')
|
||||
const original = engine.getOrCreateSession('pi-original-mapped', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-mapped',
|
||||
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
|
||||
piResumeAttempt: { state: 'quarantined', machineId: 'machine-1', startedAt: 1, childSessionId: temp.id },
|
||||
}, null, 'default')
|
||||
let dedupCalls = 0
|
||||
;(engine as any).sessionCache.deduplicateByAgentSessionId = async () => { dedupCalls += 1 }
|
||||
;(engine as any).triggerDedupIfNeeded(temp.id)
|
||||
await flushAsyncWork()
|
||||
expect(dedupCalls).toBe(0)
|
||||
expect(store.sessions.getSession(original.id)).not.toBeNull()
|
||||
|
||||
engine.handleSessionEnd({ sid: temp.id, time: Date.now(), reason: 'error' })
|
||||
await flushAsyncWork()
|
||||
expect(engine.getSessionByNamespace(original.id, 'default')?.metadata?.piResumeAttempt).toBeUndefined()
|
||||
expect(store.sessions.getSession(original.id)).not.toBeNull()
|
||||
expect(dedupCalls).toBe(0)
|
||||
} finally { engine.stop() }
|
||||
})
|
||||
|
||||
it('blocks Pi dedup before an unexpected child ID is attached to the persisted attempt', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(store, {} as never, new RpcRegistry(), { broadcast() {} } as never)
|
||||
try {
|
||||
const original = engine.getOrCreateSession('pi-original-pre-mapping', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-pre-mapping',
|
||||
lifecycleState: 'archived', archivedBy: 'cli', archiveReason: 'Pi exited',
|
||||
piResumeAttempt: { state: 'resuming', machineId: 'machine-1', startedAt: 1 },
|
||||
}, null, 'default')
|
||||
const temp = engine.getOrCreateSession('pi-temp-pre-mapping', {
|
||||
path: '/tmp/project', host: 'localhost', machineId: 'machine-1', flavor: 'pi', piSessionId: 'pi-native-pre-mapping',
|
||||
}, null, 'default')
|
||||
engine.handleSessionAlive({ sid: temp.id, time: Date.now() })
|
||||
let dedupCalls = 0
|
||||
;(engine as any).sessionCache.deduplicateByAgentSessionId = async () => { dedupCalls += 1 }
|
||||
|
||||
;(engine as any).triggerDedupIfNeeded(temp.id)
|
||||
await flushAsyncWork()
|
||||
expect(dedupCalls).toBe(0)
|
||||
expect(store.sessions.getSession(original.id)).not.toBeNull()
|
||||
} finally { engine.stop() }
|
||||
})
|
||||
|
||||
it('defers mergeSessions for cursor reopen until session-ready (load failure leaves old row)', async () => {
|
||||
const store = new Store(':memory:')
|
||||
const engine = new SyncEngine(
|
||||
|
||||
+395
-69
@@ -46,6 +46,8 @@ import {
|
||||
} from './rpcGateway'
|
||||
import { SessionCache } from './sessionCache'
|
||||
|
||||
type PiResumeAttempt = NonNullable<NonNullable<Session['metadata']>['piResumeAttempt']>
|
||||
|
||||
export type { Session, SyncEvent } from '@hapi/protocol/types'
|
||||
export type { Machine } from './machineCache'
|
||||
export type { SyncEventListener } from './eventPublisher'
|
||||
@@ -72,7 +74,7 @@ export type {
|
||||
|
||||
export type ResumeSessionResult =
|
||||
| { type: 'success'; sessionId: string }
|
||||
| { type: 'error'; message: string; code: 'session_not_found' | 'access_denied' | 'no_machine_online' | 'resume_unavailable' | 'resume_failed' }
|
||||
| { type: 'error'; message: string; code: 'session_not_found' | 'access_denied' | 'no_machine_online' | 'resume_unavailable' | 'resume_failed'; rollbackSafe?: boolean }
|
||||
|
||||
export type ReopenSessionResult =
|
||||
| { type: 'success'; sessionId: string; resumed: boolean; cursorSessionProtocol?: 'acp' | 'stream-json' }
|
||||
@@ -147,8 +149,14 @@ export class SyncEngine {
|
||||
private readonly messageService: MessageService
|
||||
private readonly rpcGateway: RpcGateway
|
||||
private inactivityTimer: NodeJS.Timeout | null = null
|
||||
/** Sessions that emitted `session-ready` (Cursor ACP load/newSession complete). */
|
||||
/** Sessions that emitted `session-ready` (Cursor ACP or validated Pi get_state). */
|
||||
private readonly sessionReadyIds = new Set<string>()
|
||||
/** Original Pi rows with a native resume currently in flight. */
|
||||
private readonly piResumeInFlightIds = new Set<string>()
|
||||
/** Pi rows whose runner child could not be confirmed terminated. */
|
||||
private readonly piResumeQuarantinedIds = new Set<string>()
|
||||
/** Unexpected version-skew temp child -> original row whose retry is blocked until child ends. */
|
||||
private readonly piUnexpectedTempOriginalIds = new Map<string, string>()
|
||||
/** Serialize scratchlist uploads per session so disk-byte caps cannot race. */
|
||||
private readonly scratchlistUploadTails = new Map<string, Promise<unknown>>()
|
||||
|
||||
@@ -402,6 +410,15 @@ export class SyncEngine {
|
||||
|
||||
handleSessionReady(payload: { sid: string; time: number }): void {
|
||||
this.sessionReadyIds.add(payload.sid)
|
||||
const session = this.sessionCache.getSession(payload.sid)
|
||||
if (session?.metadata?.piResumeAttempt) {
|
||||
void this.writePiResumeAttempt(payload.sid, session.namespace, null)
|
||||
.then(() => {
|
||||
this.piResumeQuarantinedIds.delete(payload.sid)
|
||||
this.triggerDedupIfNeeded(payload.sid)
|
||||
})
|
||||
.catch(() => {})
|
||||
}
|
||||
this.triggerDedupIfNeeded(payload.sid)
|
||||
}
|
||||
|
||||
@@ -411,9 +428,14 @@ export class SyncEngine {
|
||||
|
||||
handleSessionEnd(payload: { sid: string; time: number; reason?: 'completed' | 'terminated' | 'error' }): void {
|
||||
const before = this.sessionCache.getSession(payload.sid)
|
||||
const ownsPiAttempt = before?.metadata?.piResumeAttempt !== undefined
|
||||
const isPiAttemptChild = this.sessionCache.getSessions().some(
|
||||
(session) => session.metadata?.piResumeAttempt?.childSessionId === payload.sid
|
||||
)
|
||||
const restorePiArchive = ownsPiAttempt && !this.sessionReadyIds.has(payload.sid)
|
||||
const isCursorAcp = before?.metadata?.flavor === 'cursor'
|
||||
&& before.metadata.cursorSessionProtocol === 'acp'
|
||||
const shouldRetryDedup = !isCursorAcp || this.sessionReadyIds.has(payload.sid)
|
||||
const shouldRetryDedup = !ownsPiAttempt && !isPiAttemptChild && (!isCursorAcp || this.sessionReadyIds.has(payload.sid))
|
||||
|
||||
this.sessionCache.handleSessionEnd(payload)
|
||||
this.eventPublisher.emit({
|
||||
@@ -428,6 +450,11 @@ export class SyncEngine {
|
||||
this.triggerDedupIfNeeded(payload.sid)
|
||||
}
|
||||
this.sessionReadyIds.delete(payload.sid)
|
||||
this.piResumeQuarantinedIds.delete(payload.sid)
|
||||
this.piUnexpectedTempOriginalIds.delete(payload.sid)
|
||||
if (ownsPiAttempt || isPiAttemptChild) {
|
||||
void this.clearPiAttemptForEndedSession(payload.sid, restorePiArchive)
|
||||
}
|
||||
}
|
||||
|
||||
handleBackgroundTaskDelta(sessionId: string, delta: { started: number; completed: number }): void {
|
||||
@@ -1539,6 +1566,35 @@ async uploadScratchlistAttachment(
|
||||
return { type: 'error', message: 'No machine online', code: 'no_machine_online' }
|
||||
}
|
||||
|
||||
if (flavor === 'pi' && resumeToken && targetMachine.runnerState?.capabilities?.piExistingSessionResume !== true) {
|
||||
return { type: 'error', message: 'Pi resume requires an upgraded runner', code: 'resume_failed' }
|
||||
}
|
||||
|
||||
const requiresPiNativeReady = flavor === 'pi' && resumeToken !== undefined
|
||||
if (requiresPiNativeReady) {
|
||||
if (this.isPiResumeBlocked(access.sessionId)) {
|
||||
return { type: 'error', message: 'Pi resume is already in progress', code: 'resume_failed' }
|
||||
}
|
||||
this.piResumeInFlightIds.add(access.sessionId)
|
||||
this.sessionReadyIds.delete(access.sessionId)
|
||||
try {
|
||||
await this.writePiResumeAttempt(access.sessionId, namespace, {
|
||||
state: 'resuming',
|
||||
machineId: targetMachine.id,
|
||||
startedAt: Date.now(),
|
||||
archiveSnapshot: {
|
||||
lifecycleState: metadata.lifecycleState,
|
||||
lifecycleStateSince: metadata.lifecycleStateSince,
|
||||
archivedBy: metadata.archivedBy,
|
||||
archiveReason: metadata.archiveReason,
|
||||
},
|
||||
})
|
||||
} catch {
|
||||
this.piResumeInFlightIds.delete(access.sessionId)
|
||||
return { type: 'error', message: 'Failed to record Pi resume attempt', code: 'resume_failed' }
|
||||
}
|
||||
}
|
||||
|
||||
if (flavor === 'cursor' && resumeToken) {
|
||||
try {
|
||||
const chatStatus = await this.rpcGateway.getCursorChatStoreStatus(
|
||||
@@ -1569,65 +1625,135 @@ async uploadScratchlistAttachment(
|
||||
: opts?.permissionMode
|
||||
?? session.permissionMode
|
||||
?? metadataPermissionMode
|
||||
const spawnResult = await this.rpcGateway.spawnSession(
|
||||
targetMachine.id,
|
||||
directory,
|
||||
flavor,
|
||||
session.model ?? undefined,
|
||||
session.modelReasoningEffort ?? undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
resumeToken,
|
||||
session.effort ?? undefined,
|
||||
preferredPermissionMode,
|
||||
session.serviceTier ?? undefined,
|
||||
access.sessionId,
|
||||
session.collaborationMode ?? undefined
|
||||
)
|
||||
let piResumeSucceeded = false
|
||||
try {
|
||||
const spawnResult = await this.rpcGateway.spawnSession(
|
||||
targetMachine.id,
|
||||
directory,
|
||||
flavor,
|
||||
session.model ?? undefined,
|
||||
session.modelReasoningEffort ?? undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
resumeToken,
|
||||
session.effort ?? undefined,
|
||||
preferredPermissionMode,
|
||||
session.serviceTier ?? undefined,
|
||||
access.sessionId,
|
||||
session.collaborationMode ?? undefined
|
||||
)
|
||||
|
||||
if (spawnResult.type !== 'success') {
|
||||
return { type: 'error', message: spawnResult.message, code: 'resume_failed' }
|
||||
}
|
||||
|
||||
const becameActive = await this.waitForSessionActive(spawnResult.sessionId)
|
||||
if (!becameActive) {
|
||||
return { type: 'error', message: 'Session failed to become active', code: 'resume_failed' }
|
||||
}
|
||||
|
||||
// permissionMode is passed to spawnSession above; do not call set-session-config here.
|
||||
// session-alive can arrive before the CLI registers that RPC handler, which caused resume_failed.
|
||||
|
||||
const needsReadyBeforeMerge = spawnResult.sessionId !== access.sessionId
|
||||
&& flavor === 'cursor'
|
||||
&& metadata.cursorSessionProtocol === 'acp'
|
||||
if (needsReadyBeforeMerge) {
|
||||
const readyResult = await this.waitForSessionReady(spawnResult.sessionId)
|
||||
if (readyResult !== 'ready') {
|
||||
const message = readyResult === 'ended'
|
||||
? 'Session ended before Cursor ACP load completed'
|
||||
: 'Session failed to become ready'
|
||||
return { type: 'error', message, code: 'resume_failed' }
|
||||
if (spawnResult.type !== 'success') {
|
||||
if (requiresPiNativeReady) {
|
||||
const stopped = await this.terminateInPlacePiResume(
|
||||
targetMachine.id,
|
||||
access.sessionId,
|
||||
namespace
|
||||
)
|
||||
if (!stopped) {
|
||||
await this.quarantinePiResume(access.sessionId, namespace, targetMachine.id)
|
||||
return {
|
||||
type: 'error',
|
||||
message: spawnResult.message,
|
||||
code: 'resume_failed',
|
||||
rollbackSafe: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
return { type: 'error', message: spawnResult.message, code: 'resume_failed' }
|
||||
}
|
||||
}
|
||||
|
||||
if (spawnResult.sessionId !== access.sessionId) {
|
||||
// The old session may have already been merged by the automatic dedup path
|
||||
// (triggered when the spawned CLI sets its agent session ID in metadata).
|
||||
// Only attempt the explicit merge if the old session still exists.
|
||||
const oldSession = this.sessionCache.getSessionByNamespace(access.sessionId, namespace)
|
||||
if (oldSession) {
|
||||
try {
|
||||
await this.sessionCache.mergeSessions(access.sessionId, spawnResult.sessionId, namespace)
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Failed to merge resumed session'
|
||||
if (requiresPiNativeReady && spawnResult.sessionId !== access.sessionId) {
|
||||
const removed = await this.terminateUnexpectedPiTemp(
|
||||
targetMachine.id,
|
||||
spawnResult.sessionId,
|
||||
access.sessionId,
|
||||
namespace
|
||||
)
|
||||
return {
|
||||
type: 'error',
|
||||
message: removed
|
||||
? 'Pi runner created an unexpected session; upgrade the runner and retry'
|
||||
: 'Pi runner created an unexpected live session; upgrade the runner and retry',
|
||||
code: 'resume_failed'
|
||||
}
|
||||
}
|
||||
|
||||
const becameActive = await this.waitForSessionActive(spawnResult.sessionId)
|
||||
if (!becameActive) {
|
||||
if (requiresPiNativeReady) {
|
||||
const inactive = await this.terminateInPlacePiResume(
|
||||
targetMachine.id,
|
||||
access.sessionId,
|
||||
namespace
|
||||
)
|
||||
if (!inactive) {
|
||||
await this.quarantinePiResume(access.sessionId, namespace, targetMachine.id)
|
||||
return { type: 'error', message: 'Pi resume failed and the child is still active', code: 'resume_failed', rollbackSafe: false }
|
||||
}
|
||||
}
|
||||
return { type: 'error', message: 'Session failed to become active', code: 'resume_failed' }
|
||||
}
|
||||
|
||||
const needsReadyBeforeSuccess = requiresPiNativeReady
|
||||
|| (
|
||||
spawnResult.sessionId !== access.sessionId
|
||||
&& flavor === 'cursor'
|
||||
&& metadata.cursorSessionProtocol === 'acp'
|
||||
)
|
||||
if (needsReadyBeforeSuccess) {
|
||||
const readyResult = await this.waitForSessionReady(spawnResult.sessionId)
|
||||
if (readyResult !== 'ready') {
|
||||
if (requiresPiNativeReady && readyResult !== 'ended') {
|
||||
const inactive = await this.terminateInPlacePiResume(
|
||||
targetMachine.id,
|
||||
access.sessionId,
|
||||
namespace
|
||||
)
|
||||
if (!inactive) {
|
||||
await this.quarantinePiResume(access.sessionId, namespace, targetMachine.id)
|
||||
return { type: 'error', message: 'Pi native resume timed out and the child is still active', code: 'resume_failed', rollbackSafe: false }
|
||||
}
|
||||
}
|
||||
const message = flavor === 'pi'
|
||||
? readyResult === 'ended'
|
||||
? 'Pi session ended before native resume completed'
|
||||
: 'Pi session failed to become native-ready'
|
||||
: readyResult === 'ended'
|
||||
? 'Session ended before Cursor ACP load completed'
|
||||
: 'Session failed to become ready'
|
||||
return { type: 'error', message, code: 'resume_failed' }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.sessionCache.markSessionActive(spawnResult.sessionId)
|
||||
return { type: 'success', sessionId: spawnResult.sessionId }
|
||||
if (spawnResult.sessionId !== access.sessionId) {
|
||||
const oldSession = this.sessionCache.getSessionByNamespace(access.sessionId, namespace)
|
||||
if (oldSession) {
|
||||
try {
|
||||
await this.sessionCache.mergeSessions(access.sessionId, spawnResult.sessionId, namespace)
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Failed to merge resumed session'
|
||||
return { type: 'error', message, code: 'resume_failed' }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.sessionCache.markSessionActive(spawnResult.sessionId)
|
||||
piResumeSucceeded = true
|
||||
if (requiresPiNativeReady) await this.writePiResumeAttempt(access.sessionId, namespace, null)
|
||||
return { type: 'success', sessionId: spawnResult.sessionId }
|
||||
} finally {
|
||||
if (requiresPiNativeReady) {
|
||||
this.piResumeInFlightIds.delete(access.sessionId)
|
||||
if (!piResumeSucceeded && this.sessionCache.getSession(access.sessionId)?.metadata?.piResumeAttempt?.state === 'resuming') {
|
||||
await this.writePiResumeAttempt(access.sessionId, namespace, null, true).catch(() => {})
|
||||
}
|
||||
if (piResumeSucceeded) {
|
||||
this.triggerDedupIfNeeded(access.sessionId)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1664,6 +1790,23 @@ async uploadScratchlistAttachment(
|
||||
const session = access.session
|
||||
const metadata = session.metadata
|
||||
|
||||
if (metadata?.flavor === 'pi' && this.isPiResumeBlocked(access.sessionId)) {
|
||||
if (session.active) {
|
||||
return { type: 'error', message: 'Pi resume is already in progress', code: 'resume_failed' }
|
||||
}
|
||||
if (metadata.piResumeAttempt && !this.piResumeInFlightIds.has(access.sessionId)) {
|
||||
const reconciled = await this.reconcilePersistedPiResumeAttempt(session)
|
||||
return {
|
||||
type: 'error',
|
||||
message: reconciled
|
||||
? 'Previous Pi resume attempt was cleaned up; retry'
|
||||
: 'Pi resume is already in progress',
|
||||
code: 'resume_failed'
|
||||
}
|
||||
}
|
||||
return { type: 'error', message: 'Pi resume is already in progress', code: 'resume_failed' }
|
||||
}
|
||||
|
||||
if (session.active) {
|
||||
return { type: 'success', sessionId: access.sessionId, resumed: false }
|
||||
}
|
||||
@@ -1689,24 +1832,30 @@ async uploadScratchlistAttachment(
|
||||
lifecycleStateSince: metadata.lifecycleStateSince
|
||||
}
|
||||
|
||||
let applied: { cursorSessionProtocol?: 'acp' | 'stream-json' }
|
||||
try {
|
||||
applied = await this.sessionCache.clearSessionArchiveMetadata(access.sessionId)
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Failed to clear archive metadata'
|
||||
return { type: 'error', message, code: 'metadata_conflict' }
|
||||
let applied: { cursorSessionProtocol?: 'acp' | 'stream-json' } = {}
|
||||
// Pi reuses the original HAPI row. Keep its archive snapshot persisted
|
||||
// until the CLI successfully bootstraps that row as running; this avoids
|
||||
// an inactive, non-archived gap if the Hub restarts before spawn.
|
||||
if (metadata.flavor !== 'pi') {
|
||||
try {
|
||||
applied = await this.sessionCache.clearSessionArchiveMetadata(access.sessionId)
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Failed to clear archive metadata'
|
||||
return { type: 'error', message, code: 'metadata_conflict' }
|
||||
}
|
||||
}
|
||||
|
||||
const resumeResult = await this.resumeSession(access.sessionId, namespace)
|
||||
if (resumeResult.type === 'error') {
|
||||
// Resume failed - put the archive flags back so the row stays archived in the UI
|
||||
// and the operator can retry. Best-effort: a concurrent metadata write that
|
||||
// succeeded between clear and restore (e.g. an unrelated rename) wins, in
|
||||
// which case we surface the original resume error rather than masking it.
|
||||
try {
|
||||
await this.sessionCache.restoreSessionArchiveMetadata(access.sessionId, archiveSnapshot)
|
||||
} catch {
|
||||
// Swallow restore failures - the resume error is the more important signal.
|
||||
// Never restore archived metadata over a live Pi child. A live
|
||||
// row blocks retry by itself and must remain visible as active.
|
||||
const current = this.sessionCache.getSessionByNamespace(access.sessionId, namespace)
|
||||
if (resumeResult.rollbackSafe !== false && !current?.active) {
|
||||
try {
|
||||
await this.sessionCache.restoreSessionArchiveMetadata(access.sessionId, archiveSnapshot)
|
||||
} catch {
|
||||
// Swallow restore failures - the resume error is the more important signal.
|
||||
}
|
||||
}
|
||||
return resumeResult
|
||||
}
|
||||
@@ -1928,6 +2077,16 @@ async uploadScratchlistAttachment(
|
||||
}
|
||||
|
||||
private canRunCursorDedup(session: Session): boolean {
|
||||
if (this.piResumeInFlightIds.has(session.id) || this.piResumeQuarantinedIds.has(session.id)) return false
|
||||
if (session.metadata?.piResumeAttempt) return false
|
||||
if (this.sessionCache.getSessions().some((candidate) => candidate.metadata?.piResumeAttempt?.childSessionId === session.id)) return false
|
||||
const piSessionId = session.metadata?.piSessionId
|
||||
if (piSessionId && this.sessionCache.getSessions().some((candidate) =>
|
||||
candidate.id !== session.id
|
||||
&& candidate.namespace === session.namespace
|
||||
&& candidate.metadata?.piResumeAttempt !== undefined
|
||||
&& candidate.metadata.piSessionId === piSessionId
|
||||
)) return false
|
||||
if (session.metadata?.flavor !== 'cursor') {
|
||||
return true
|
||||
}
|
||||
@@ -1937,6 +2096,173 @@ async uploadScratchlistAttachment(
|
||||
return this.sessionReadyIds.has(session.id)
|
||||
}
|
||||
|
||||
private async terminateInPlacePiResume(
|
||||
machineId: string,
|
||||
sessionId: string,
|
||||
namespace: string
|
||||
): Promise<boolean> {
|
||||
const existingAttempt = this.sessionCache.getSession(sessionId)?.metadata?.piResumeAttempt
|
||||
await this.writePiResumeAttempt(sessionId, namespace, {
|
||||
...existingAttempt,
|
||||
state: 'terminating',
|
||||
machineId,
|
||||
startedAt: Date.now(),
|
||||
})
|
||||
let status: 'stopped' | 'already_gone' | 'still_alive'
|
||||
try {
|
||||
status = await this.rpcGateway.stopRunnerSession(machineId, sessionId)
|
||||
} catch {
|
||||
status = 'still_alive'
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 0))
|
||||
const session = this.sessionCache.refreshSession(sessionId) ?? this.sessionCache.getSession(sessionId)
|
||||
const attemptClearedByEnd = session?.metadata?.piResumeAttempt === undefined
|
||||
if (status === 'still_alive') {
|
||||
if (attemptClearedByEnd) return true
|
||||
return false
|
||||
}
|
||||
if (session?.active) this.handleSessionEnd({ sid: sessionId, time: Date.now(), reason: 'error' })
|
||||
await this.writePiResumeAttempt(sessionId, namespace, null, true).catch(() => {})
|
||||
return true
|
||||
}
|
||||
|
||||
private async terminateUnexpectedPiTemp(
|
||||
machineId: string,
|
||||
sessionId: string,
|
||||
originalSessionId: string,
|
||||
namespace: string
|
||||
): Promise<boolean> {
|
||||
this.piUnexpectedTempOriginalIds.set(sessionId, originalSessionId)
|
||||
const existingAttempt = this.sessionCache.getSession(originalSessionId)?.metadata?.piResumeAttempt
|
||||
await this.writePiResumeAttempt(originalSessionId, namespace, {
|
||||
...existingAttempt,
|
||||
state: 'terminating',
|
||||
machineId,
|
||||
startedAt: Date.now(),
|
||||
childSessionId: sessionId,
|
||||
})
|
||||
let status: 'stopped' | 'already_gone' | 'still_alive'
|
||||
try {
|
||||
status = await this.rpcGateway.stopRunnerSession(machineId, sessionId)
|
||||
} catch {
|
||||
status = 'still_alive'
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 0))
|
||||
const session = this.sessionCache.refreshSession(sessionId) ?? this.sessionCache.getSession(sessionId)
|
||||
const original = this.sessionCache.refreshSession(originalSessionId) ?? this.sessionCache.getSession(originalSessionId)
|
||||
const attemptClearedByEnd = original?.metadata?.piResumeAttempt === undefined
|
||||
if (status === 'still_alive' && !attemptClearedByEnd) {
|
||||
await this.writePiResumeAttempt(originalSessionId, namespace, {
|
||||
...existingAttempt,
|
||||
state: 'quarantined',
|
||||
machineId,
|
||||
startedAt: Date.now(),
|
||||
childSessionId: sessionId,
|
||||
})
|
||||
return false
|
||||
}
|
||||
if (session?.active) this.handleSessionEnd({ sid: sessionId, time: Date.now(), reason: 'error' })
|
||||
const remaining = this.sessionCache.getSession(sessionId)
|
||||
if (remaining && !remaining.active) await this.sessionCache.deleteSession(sessionId)
|
||||
this.piUnexpectedTempOriginalIds.delete(sessionId)
|
||||
await this.writePiResumeAttempt(originalSessionId, namespace, null, true).catch(() => {})
|
||||
return true
|
||||
}
|
||||
|
||||
private async quarantinePiResume(sessionId: string, namespace: string, machineId: string): Promise<void> {
|
||||
this.piResumeQuarantinedIds.add(sessionId)
|
||||
const existingAttempt = this.sessionCache.getSession(sessionId)?.metadata?.piResumeAttempt
|
||||
await this.writePiResumeAttempt(sessionId, namespace, {
|
||||
...existingAttempt,
|
||||
state: 'quarantined',
|
||||
machineId,
|
||||
startedAt: Date.now(),
|
||||
})
|
||||
}
|
||||
|
||||
private isPiResumeBlocked(sessionId: string): boolean {
|
||||
const metadataAttempt = this.sessionCache.getSession(sessionId)?.metadata?.piResumeAttempt
|
||||
return this.piResumeInFlightIds.has(sessionId)
|
||||
|| this.piResumeQuarantinedIds.has(sessionId)
|
||||
|| metadataAttempt !== undefined
|
||||
|| [...this.piUnexpectedTempOriginalIds.values()].includes(sessionId)
|
||||
}
|
||||
|
||||
private async writePiResumeAttempt(
|
||||
sessionId: string,
|
||||
namespace: string,
|
||||
attempt: PiResumeAttempt | null,
|
||||
restoreArchive = false
|
||||
): Promise<void> {
|
||||
for (let i = 0; i < 5; i += 1) {
|
||||
const current = this.sessionCache.getSessionByNamespace(sessionId, namespace) ?? this.sessionCache.refreshSession(sessionId)
|
||||
if (!current?.metadata) return
|
||||
const next = { ...current.metadata }
|
||||
if (attempt) next.piResumeAttempt = attempt
|
||||
else {
|
||||
const snapshot = current.metadata.piResumeAttempt?.archiveSnapshot
|
||||
delete next.piResumeAttempt
|
||||
if (restoreArchive && snapshot) {
|
||||
if (snapshot.lifecycleState === undefined) delete next.lifecycleState
|
||||
else next.lifecycleState = snapshot.lifecycleState
|
||||
if (snapshot.lifecycleStateSince === undefined) delete next.lifecycleStateSince
|
||||
else next.lifecycleStateSince = snapshot.lifecycleStateSince
|
||||
if (snapshot.archivedBy === undefined) delete next.archivedBy
|
||||
else next.archivedBy = snapshot.archivedBy
|
||||
if (snapshot.archiveReason === undefined) delete next.archiveReason
|
||||
else next.archiveReason = snapshot.archiveReason
|
||||
}
|
||||
}
|
||||
const result = this.store.sessions.updateSessionMetadata(sessionId, next, current.metadataVersion, namespace, { touchUpdatedAt: false })
|
||||
if (result.result === 'success') {
|
||||
this.sessionCache.refreshSession(sessionId)
|
||||
return
|
||||
}
|
||||
if (result.result !== 'version-mismatch') throw new Error('Failed to update Pi resume attempt')
|
||||
this.sessionCache.refreshSession(sessionId)
|
||||
}
|
||||
throw new Error('Pi resume attempt metadata was modified concurrently')
|
||||
}
|
||||
|
||||
private async clearPiAttemptForEndedSession(endedSessionId: string, restoreArchive: boolean): Promise<void> {
|
||||
const ended = this.sessionCache.getSession(endedSessionId)
|
||||
if (ended?.metadata?.piResumeAttempt) {
|
||||
await this.writePiResumeAttempt(endedSessionId, ended.namespace, null, restoreArchive).catch(() => {})
|
||||
return
|
||||
}
|
||||
for (const session of this.sessionCache.getSessions()) {
|
||||
if (session.metadata?.piResumeAttempt?.childSessionId === endedSessionId) {
|
||||
await this.writePiResumeAttempt(session.id, session.namespace, null, true).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async reconcilePersistedPiResumeAttempt(session: Session): Promise<boolean> {
|
||||
const attempt = session.metadata?.piResumeAttempt
|
||||
if (!attempt) return true
|
||||
const childSessionId = attempt.childSessionId ?? session.id
|
||||
let status: 'stopped' | 'already_gone' | 'still_alive'
|
||||
try {
|
||||
status = await this.rpcGateway.stopRunnerSession(attempt.machineId, childSessionId)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
if (status === 'still_alive') return false
|
||||
|
||||
const child = this.sessionCache.getSession(childSessionId)
|
||||
if (child?.active) this.handleSessionEnd({ sid: childSessionId, time: Date.now(), reason: 'error' })
|
||||
if (childSessionId !== session.id) {
|
||||
const remaining = this.sessionCache.getSession(childSessionId)
|
||||
if (remaining && !remaining.active) await this.sessionCache.deleteSession(childSessionId)
|
||||
}
|
||||
await this.writePiResumeAttempt(session.id, session.namespace, null, true)
|
||||
this.piResumeQuarantinedIds.delete(session.id)
|
||||
this.piUnexpectedTempOriginalIds.delete(childSessionId)
|
||||
return true
|
||||
}
|
||||
|
||||
private triggerDedupIfNeeded(sessionId: string): void {
|
||||
const session = this.sessionCache.getSession(sessionId)
|
||||
if (session?.metadata) {
|
||||
|
||||
Reference in New Issue
Block a user