fix(sync): add SSE heartbeat and alive status (#223)

This commit is contained in:
Lihengwannafly
2026-02-28 01:46:21 +08:00
committed by GitHub
parent 4ace791aa4
commit 1942f088ff
11 changed files with 547 additions and 17 deletions
+1
View File
@@ -40,3 +40,4 @@ execplan/
# Generated npm bundle output (local)
cli/npm/main/
.ace-tool/
+64
View File
@@ -0,0 +1,64 @@
import { describe, expect, it } from 'bun:test'
import type { SyncEvent } from '@hapi/protocol/types'
import { Store } from '../store'
import type { EventPublisher } from './eventPublisher'
import { MachineCache } from './machineCache'
import { SessionCache } from './sessionCache'
function createPublisher(events: SyncEvent[]): EventPublisher {
return {
emit: (event: SyncEvent) => {
events.push(event)
}
} as unknown as EventPublisher
}
describe('alive incremental events', () => {
it('includes active=true in session alive updates', () => {
const store = new Store(':memory:')
const events: SyncEvent[] = []
const cache = new SessionCache(store, createPublisher(events))
const session = cache.getOrCreateSession(
'session-alive-test',
{ path: '/tmp/project', host: 'localhost' },
{ requests: {}, completedRequests: {} },
'default'
)
events.length = 0
cache.handleSessionAlive({ sid: session.id, time: Date.now(), thinking: false })
const update = events.find((event) => event.type === 'session-updated')
expect(update).toBeDefined()
if (!update || update.type !== 'session-updated') {
return
}
expect(update.data).toEqual(expect.objectContaining({ active: true }))
})
it('emits full active machine object on machine alive', () => {
const store = new Store(':memory:')
const events: SyncEvent[] = []
const cache = new MachineCache(store, createPublisher(events))
const machine = cache.getOrCreateMachine(
'machine-alive-test',
{ host: 'localhost', platform: 'linux', happyCliVersion: '0.1.0' },
null,
'default'
)
events.length = 0
cache.handleMachineAlive({ machineId: machine.id, time: Date.now() })
const update = events.find((event) => event.type === 'machine-updated')
expect(update).toBeDefined()
if (!update || update.type !== 'machine-updated') {
return
}
expect(update.data).toEqual(expect.objectContaining({ id: machine.id, active: true }))
})
})
+1 -1
View File
@@ -150,7 +150,7 @@ export class MachineCache {
const shouldBroadcast = (!wasActive && machine.active) || (now - lastBroadcastAt > 10_000)
if (shouldBroadcast) {
this.lastBroadcastAtByMachineId.set(machine.id, now)
this.publisher.emit({ type: 'machine-updated', machineId: machine.id, data: { activeAt: machine.activeAt } })
this.publisher.emit({ type: 'machine-updated', machineId: machine.id, data: machine })
}
}
+1
View File
@@ -179,6 +179,7 @@ export class SessionCache {
type: 'session-updated',
sessionId: session.id,
data: {
active: true,
activeAt: session.activeAt,
thinking: session.thinking,
permissionMode: session.permissionMode,
+9 -1
View File
@@ -87,7 +87,15 @@ export function createEventsRoutes(
visibility,
send: (event) => stream.writeSSE({ data: JSON.stringify(event) }),
sendHeartbeat: async () => {
await stream.write(': heartbeat\n\n')
await stream.writeSSE({
data: JSON.stringify({
type: 'heartbeat',
namespace,
data: {
timestamp: Date.now()
}
})
})
}
})
+6
View File
@@ -181,6 +181,12 @@ export const SyncEventSchema = z.discriminatedUnion('type', [
url: z.string()
})
}),
SessionEventBaseSchema.extend({
type: z.literal('heartbeat'),
data: z.object({
timestamp: z.number()
}).optional()
}),
SessionEventBaseSchema.extend({
type: z.literal('connection-changed'),
data: z.object({
+8 -2
View File
@@ -117,6 +117,7 @@ function AppInner() {
const selectedSessionId = sessionMatch && sessionMatch.sessionId !== 'new' ? sessionMatch.sessionId : null
const { isSyncing, startSync, endSync } = useSyncingState()
const [sseDisconnected, setSseDisconnected] = useState(false)
const [sseDisconnectReason, setSseDisconnectReason] = useState<string | null>(null)
const syncTokenRef = useRef(0)
const isFirstConnectRef = useRef(true)
const baseUrlRef = useRef(baseUrl)
@@ -181,6 +182,7 @@ function AppInner() {
const handleSseConnect = useCallback(() => {
// Clear disconnected state on successful connection
setSseDisconnected(false)
setSseDisconnectReason(null)
// Increment token to track this specific connection
const token = ++syncTokenRef.current
@@ -215,10 +217,11 @@ function AppInner() {
})
}, [api, queryClient, selectedSessionId, startSync, endSync])
const handleSseDisconnect = useCallback(() => {
const handleSseDisconnect = useCallback((reason: string) => {
// Only show reconnecting banner if we've already connected once
if (!isFirstConnectRef.current) {
setSseDisconnected(true)
setSseDisconnectReason(reason)
}
}, [])
@@ -338,7 +341,10 @@ function AppInner() {
<AppContextProvider value={{ api, token, baseUrl }}>
<VoiceProvider>
<SyncingBanner isSyncing={isSyncing} />
<ReconnectingBanner isReconnecting={sseDisconnected && !isSyncing} />
<ReconnectingBanner
isReconnecting={sseDisconnected && !isSyncing}
reason={sseDisconnectReason}
/>
<VoiceErrorBanner />
<OfflineBanner />
<div className="h-full flex flex-col">
+22 -1
View File
@@ -1,9 +1,29 @@
import { useOnlineStatus } from '@/hooks/useOnlineStatus'
import { useTranslation } from '@/lib/use-translation'
export function ReconnectingBanner({ isReconnecting }: { isReconnecting: boolean }) {
function getReasonLabel(reason: string, t: (key: string) => string): string {
if (reason === 'heartbeat-timeout') {
return t('reconnecting.reason.heartbeatTimeout')
}
if (reason === 'closed') {
return t('reconnecting.reason.closed')
}
if (reason === 'error') {
return t('reconnecting.reason.error')
}
return reason
}
export function ReconnectingBanner({
isReconnecting,
reason
}: {
isReconnecting: boolean
reason?: string | null
}) {
const { t } = useTranslation()
const isOnline = useOnlineStatus()
const reasonLabel = reason ? getReasonLabel(reason, t) : null
// Don't show if offline (OfflineBanner takes precedence) or if not reconnecting
if (!isReconnecting || !isOnline) {
@@ -14,6 +34,7 @@ export function ReconnectingBanner({ isReconnecting }: { isReconnecting: boolean
<div className="fixed top-0 left-0 right-0 bg-amber-500 text-white text-center py-2 text-sm font-medium z-50 flex items-center justify-center gap-2">
<span className="animate-spin h-4 w-4 border-2 border-white border-t-transparent rounded-full" />
{t('reconnecting.message')}
{reasonLabel ? <span className="opacity-90">({reasonLabel})</span> : null}
</div>
)
}
+429 -12
View File
@@ -1,7 +1,15 @@
import { useEffect, useMemo, useRef, useState } from 'react'
import { useQueryClient } from '@tanstack/react-query'
import { isObject } from '@hapi/protocol'
import type { SyncEvent } from '@/types/api'
import { isObject, toSessionSummary } from '@hapi/protocol'
import type {
Machine,
MachinesResponse,
Session,
SessionResponse,
SessionsResponse,
SessionSummary,
SyncEvent
} from '@/types/api'
import { queryKeys } from '@/lib/query-keys'
import { clearMessageWindow, ingestIncomingMessages } from '@/lib/message-window-store'
@@ -15,6 +23,109 @@ type VisibilityState = 'visible' | 'hidden'
type ToastEvent = Extract<SyncEvent, { type: 'toast' }>
const HEARTBEAT_STALE_MS = 90_000
const HEARTBEAT_WATCHDOG_INTERVAL_MS = 10_000
const RECONNECT_BASE_DELAY_MS = 1_000
const RECONNECT_MAX_DELAY_MS = 30_000
const RECONNECT_JITTER_MS = 500
const INVALIDATION_BATCH_MS = 16
type SessionPatch = Partial<Pick<Session, 'active' | 'thinking' | 'activeAt' | 'updatedAt' | 'permissionMode' | 'modelMode'>>
function sortSessionSummaries(left: SessionSummary, right: SessionSummary): number {
if (left.active !== right.active) {
return left.active ? -1 : 1
}
if (left.active && left.pendingRequestsCount !== right.pendingRequestsCount) {
return right.pendingRequestsCount - left.pendingRequestsCount
}
return right.updatedAt - left.updatedAt
}
function hasRecordShape(value: unknown): value is Record<string, unknown> {
return isObject(value)
}
function isSessionRecord(value: unknown): value is Session {
if (!hasRecordShape(value)) {
return false
}
return typeof value.id === 'string'
&& typeof value.active === 'boolean'
&& typeof value.activeAt === 'number'
&& typeof value.updatedAt === 'number'
&& typeof value.thinking === 'boolean'
}
function getSessionPatch(value: unknown): SessionPatch | null {
if (!hasRecordShape(value)) {
return null
}
const patch: SessionPatch = {}
let hasKnownPatch = false
if (typeof value.active === 'boolean') {
patch.active = value.active
hasKnownPatch = true
}
if (typeof value.thinking === 'boolean') {
patch.thinking = value.thinking
hasKnownPatch = true
}
if (typeof value.activeAt === 'number') {
patch.activeAt = value.activeAt
hasKnownPatch = true
}
if (typeof value.updatedAt === 'number') {
patch.updatedAt = value.updatedAt
hasKnownPatch = true
}
if (typeof value.permissionMode === 'string') {
patch.permissionMode = value.permissionMode as Session['permissionMode']
hasKnownPatch = true
}
if (typeof value.modelMode === 'string') {
patch.modelMode = value.modelMode as Session['modelMode']
hasKnownPatch = true
}
return hasKnownPatch ? patch : null
}
function hasUnknownSessionPatchKeys(value: unknown): boolean {
if (!hasRecordShape(value)) {
return false
}
const knownKeys = new Set(['active', 'thinking', 'activeAt', 'updatedAt', 'permissionMode', 'modelMode'])
return Object.keys(value).some((key) => !knownKeys.has(key))
}
function isMachineMetadata(value: unknown): value is Machine['metadata'] {
if (value === null) {
return true
}
if (!hasRecordShape(value)) {
return false
}
return typeof value.host === 'string'
&& typeof value.platform === 'string'
&& typeof value.happyCliVersion === 'string'
}
function isMachineRecord(value: unknown): value is Machine {
if (!hasRecordShape(value)) {
return false
}
return typeof value.id === 'string'
&& typeof value.active === 'boolean'
&& isMachineMetadata(value.metadata)
}
function isInactiveMachinePatch(value: unknown): boolean {
return hasRecordShape(value) && value.active === false
}
function getVisibilityState(): VisibilityState {
if (typeof document === 'undefined') {
return 'hidden'
@@ -67,6 +178,16 @@ export function useSSE(options: {
const onErrorRef = useRef(options.onError)
const onToastRef = useRef(options.onToast)
const eventSourceRef = useRef<EventSource | null>(null)
const invalidationTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
const pendingInvalidationsRef = useRef<{
sessions: boolean
machines: boolean
sessionIds: Set<string>
}>({ sessions: false, machines: false, sessionIds: new Set() })
const reconnectTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
const reconnectAttemptRef = useRef(0)
const lastActivityAtRef = useRef(0)
const [reconnectNonce, setReconnectNonce] = useState(0)
const [subscriptionId, setSubscriptionId] = useState<string | null>(null)
useEffect(() => {
@@ -99,6 +220,18 @@ export function useSSE(options: {
if (!options.enabled) {
eventSourceRef.current?.close()
eventSourceRef.current = null
if (invalidationTimerRef.current) {
clearTimeout(invalidationTimerRef.current)
invalidationTimerRef.current = null
}
pendingInvalidationsRef.current.sessions = false
pendingInvalidationsRef.current.machines = false
pendingInvalidationsRef.current.sessionIds.clear()
if (reconnectTimerRef.current) {
clearTimeout(reconnectTimerRef.current)
reconnectTimerRef.current = null
}
reconnectAttemptRef.current = 0
setSubscriptionId(null)
return
}
@@ -109,9 +242,233 @@ export function useSSE(options: {
sessionId: subscription.sessionId ?? undefined
}, getVisibilityState())
const eventSource = new EventSource(url)
let disconnectNotified = false
let reconnectRequested = false
eventSourceRef.current = eventSource
lastActivityAtRef.current = Date.now()
const scheduleReconnect = () => {
const attempt = reconnectAttemptRef.current
const exponentialDelay = Math.min(RECONNECT_MAX_DELAY_MS, RECONNECT_BASE_DELAY_MS * (2 ** attempt))
const jitter = Math.floor(Math.random() * (RECONNECT_JITTER_MS + 1))
reconnectAttemptRef.current = attempt + 1
if (reconnectTimerRef.current) {
clearTimeout(reconnectTimerRef.current)
}
reconnectTimerRef.current = setTimeout(() => {
reconnectTimerRef.current = null
setReconnectNonce((value) => value + 1)
}, exponentialDelay + jitter)
}
const notifyDisconnect = (reason: string) => {
if (disconnectNotified) {
return
}
disconnectNotified = true
onDisconnectRef.current?.(reason)
}
const requestReconnect = (reason: string) => {
if (reconnectRequested) {
return
}
reconnectRequested = true
notifyDisconnect(reason)
eventSource.close()
if (eventSourceRef.current === eventSource) {
eventSourceRef.current = null
}
setSubscriptionId(null)
scheduleReconnect()
}
const flushInvalidations = () => {
const pending = pendingInvalidationsRef.current
if (!pending.sessions && !pending.machines && pending.sessionIds.size === 0) {
return
}
const shouldInvalidateSessions = pending.sessions
const shouldInvalidateMachines = pending.machines
const sessionIds = Array.from(pending.sessionIds)
pending.sessions = false
pending.machines = false
pending.sessionIds.clear()
const tasks: Array<Promise<unknown>> = []
if (shouldInvalidateSessions) {
tasks.push(queryClient.invalidateQueries({ queryKey: queryKeys.sessions }))
}
for (const sessionId of sessionIds) {
tasks.push(queryClient.invalidateQueries({ queryKey: queryKeys.session(sessionId) }))
}
if (shouldInvalidateMachines) {
tasks.push(queryClient.invalidateQueries({ queryKey: queryKeys.machines }))
}
if (tasks.length === 0) {
return
}
void Promise.all(tasks).catch(() => {})
}
const scheduleInvalidationFlush = () => {
if (invalidationTimerRef.current) {
return
}
invalidationTimerRef.current = setTimeout(() => {
invalidationTimerRef.current = null
flushInvalidations()
}, INVALIDATION_BATCH_MS)
}
const queueSessionListInvalidation = () => {
pendingInvalidationsRef.current.sessions = true
scheduleInvalidationFlush()
}
const queueSessionDetailInvalidation = (sessionId: string) => {
pendingInvalidationsRef.current.sessionIds.add(sessionId)
scheduleInvalidationFlush()
}
const queueMachinesInvalidation = () => {
pendingInvalidationsRef.current.machines = true
scheduleInvalidationFlush()
}
const upsertSessionSummary = (session: Session) => {
queryClient.setQueryData<SessionsResponse | undefined>(queryKeys.sessions, (previous) => {
if (!previous) {
return previous
}
const summary = toSessionSummary(session)
const nextSessions = previous.sessions.slice()
const existingIndex = nextSessions.findIndex((item) => item.id === session.id)
if (existingIndex >= 0) {
nextSessions[existingIndex] = summary
} else {
nextSessions.push(summary)
}
nextSessions.sort(sortSessionSummaries)
return { ...previous, sessions: nextSessions }
})
}
const patchSessionSummary = (sessionId: string, patch: SessionPatch): boolean => {
let patched = false
queryClient.setQueryData<SessionsResponse | undefined>(queryKeys.sessions, (previous) => {
if (!previous) {
return previous
}
const nextSessions = previous.sessions.slice()
const index = nextSessions.findIndex((item) => item.id === sessionId)
if (index < 0) {
return previous
}
const current = nextSessions[index]
if (!current) {
return previous
}
const nextSummary: SessionSummary = {
...current,
active: patch.active ?? current.active,
thinking: patch.thinking ?? current.thinking,
activeAt: patch.activeAt ?? current.activeAt,
updatedAt: patch.updatedAt ?? current.updatedAt,
modelMode: patch.modelMode ?? current.modelMode
}
patched = true
nextSessions[index] = nextSummary
nextSessions.sort(sortSessionSummaries)
return { ...previous, sessions: nextSessions }
})
return patched
}
const patchSessionDetail = (sessionId: string, patch: SessionPatch): boolean => {
let patched = false
queryClient.setQueryData<SessionResponse | undefined>(queryKeys.session(sessionId), (previous) => {
if (!previous?.session) {
return previous
}
patched = true
return {
...previous,
session: {
...previous.session,
...patch
}
}
})
return patched
}
const removeSessionSummary = (sessionId: string) => {
queryClient.setQueryData<SessionsResponse | undefined>(queryKeys.sessions, (previous) => {
if (!previous) {
return previous
}
const nextSessions = previous.sessions.filter((item) => item.id !== sessionId)
if (nextSessions.length === previous.sessions.length) {
return previous
}
return { ...previous, sessions: nextSessions }
})
}
const upsertMachine = (machine: Machine) => {
queryClient.setQueryData<MachinesResponse | undefined>(queryKeys.machines, (previous) => {
if (!previous) {
return previous
}
const nextMachines = previous.machines.slice()
const index = nextMachines.findIndex((item) => item.id === machine.id)
if (!machine.active) {
if (index >= 0) {
nextMachines.splice(index, 1)
return { ...previous, machines: nextMachines }
}
return previous
}
if (index >= 0) {
nextMachines[index] = machine
} else {
nextMachines.push(machine)
}
return { ...previous, machines: nextMachines }
})
}
const removeMachine = (machineId: string) => {
queryClient.setQueryData<MachinesResponse | undefined>(queryKeys.machines, (previous) => {
if (!previous) {
return previous
}
const nextMachines = previous.machines.filter((item) => item.id !== machineId)
if (nextMachines.length === previous.machines.length) {
return previous
}
return { ...previous, machines: nextMachines }
})
}
const handleSyncEvent = (event: SyncEvent) => {
lastActivityAtRef.current = Date.now()
if (event.type === 'heartbeat') {
return
}
if (event.type === 'connection-changed') {
const data = event.data
if (data && typeof data === 'object' && 'subscriptionId' in data) {
@@ -132,19 +489,44 @@ export function useSSE(options: {
}
if (event.type === 'session-added' || event.type === 'session-updated' || event.type === 'session-removed') {
void queryClient.invalidateQueries({ queryKey: queryKeys.sessions })
if ('sessionId' in event) {
if (event.type === 'session-removed') {
void queryClient.removeQueries({ queryKey: queryKeys.session(event.sessionId) })
clearMessageWindow(event.sessionId)
if (event.type === 'session-removed') {
removeSessionSummary(event.sessionId)
void queryClient.removeQueries({ queryKey: queryKeys.session(event.sessionId) })
clearMessageWindow(event.sessionId)
} else if (isSessionRecord(event.data) && event.data.id === event.sessionId) {
queryClient.setQueryData<SessionResponse>(queryKeys.session(event.sessionId), { session: event.data })
upsertSessionSummary(event.data)
} else {
const patch = getSessionPatch(event.data)
if (patch) {
const detailPatched = patchSessionDetail(event.sessionId, patch)
const summaryPatched = patchSessionSummary(event.sessionId, patch)
if (!detailPatched) {
queueSessionDetailInvalidation(event.sessionId)
}
if (!summaryPatched) {
queueSessionListInvalidation()
}
if (hasUnknownSessionPatchKeys(event.data)) {
queueSessionDetailInvalidation(event.sessionId)
queueSessionListInvalidation()
}
} else {
void queryClient.invalidateQueries({ queryKey: queryKeys.session(event.sessionId) })
queueSessionDetailInvalidation(event.sessionId)
queueSessionListInvalidation()
}
}
}
if (event.type === 'machine-updated') {
void queryClient.invalidateQueries({ queryKey: queryKeys.machines })
if (isMachineRecord(event.data)) {
upsertMachine(event.data)
} else if (event.data === null || isInactiveMachinePatch(event.data)) {
removeMachine(event.machineId)
} else if (!hasRecordShape(event.data) || typeof event.data.activeAt !== 'number') {
queueMachinesInvalidation()
}
}
onEventRef.current(event)
@@ -174,22 +556,57 @@ export function useSSE(options: {
eventSource.onmessage = handleMessage
eventSource.onopen = () => {
if (reconnectTimerRef.current) {
clearTimeout(reconnectTimerRef.current)
reconnectTimerRef.current = null
}
reconnectAttemptRef.current = 0
disconnectNotified = false
lastActivityAtRef.current = Date.now()
onConnectRef.current?.()
}
eventSource.onerror = (error) => {
onErrorRef.current?.(error)
const reason = eventSource.readyState === EventSource.CLOSED ? 'closed' : 'error'
onDisconnectRef.current?.(reason)
if (eventSource.readyState === EventSource.CLOSED) {
requestReconnect('closed')
return
}
notifyDisconnect('error')
}
const watchdogTimer = setInterval(() => {
if (eventSourceRef.current !== eventSource) {
return
}
if (getVisibilityState() === 'hidden') {
return
}
if (Date.now() - lastActivityAtRef.current < HEARTBEAT_STALE_MS) {
return
}
requestReconnect('heartbeat-timeout')
}, HEARTBEAT_WATCHDOG_INTERVAL_MS)
return () => {
clearInterval(watchdogTimer)
if (invalidationTimerRef.current) {
clearTimeout(invalidationTimerRef.current)
invalidationTimerRef.current = null
}
pendingInvalidationsRef.current.sessions = false
pendingInvalidationsRef.current.machines = false
pendingInvalidationsRef.current.sessionIds.clear()
if (reconnectTimerRef.current) {
clearTimeout(reconnectTimerRef.current)
reconnectTimerRef.current = null
}
eventSource.close()
if (eventSourceRef.current === eventSource) {
eventSourceRef.current = null
}
setSubscriptionId(null)
}
}, [options.baseUrl, options.enabled, options.token, subscriptionKey, queryClient])
}, [options.baseUrl, options.enabled, options.token, subscriptionKey, queryClient, reconnectNonce])
return { subscriptionId }
}
+3
View File
@@ -225,6 +225,9 @@ export default {
'syncing.title': 'Syncing…',
'syncing.message': 'Your data is being synchronized.',
'reconnecting.message': 'Reconnecting...',
'reconnecting.reason.error': 'stream error',
'reconnecting.reason.closed': 'stream closed',
'reconnecting.reason.heartbeatTimeout': 'heartbeat timeout',
// Send blocked
'send.blocked.title': 'Cannot send message',
+3
View File
@@ -227,6 +227,9 @@ export default {
'syncing.title': '同步中…',
'syncing.message': '正在同步您的数据。',
'reconnecting.message': '正在重新连接...',
'reconnecting.reason.error': '流连接错误',
'reconnecting.reason.closed': '流连接已关闭',
'reconnecting.reason.heartbeatTimeout': '心跳超时',
// Send blocked
'send.blocked.title': '无法发送消息',