feat(web,hub,cli): show machine health in session sidebar (#962)

* feat(web,hub,cli): show machine load in session sidebar Runners attach OS health snapshots to machine-alive heartbeats; the hub caches them and the web session list renders load or CPU between the machine label and session count. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(web,cli): show CPU and RAM pressure in machine health badge Sidebar label now combines CPU and RAM percentages for overload signaling; load stays in the tooltip on Unix. Prime CPU sampling so the first heartbeat includes usage, not just memory. Co-authored-by: Cursor <cursoragent@cursor.com> * feat(web): visual machine health meters with tooltip Replace bare CPU/RAM text with labeled mini bar gauges, chip border tint by severity, and a HoverTooltip explaining capacity and overload guidance. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(web): widen machine health tooltip with horizontal layout Allow a generous popover width and lay CPU/RAM/load out side by side so the capacity tooltip reads wider and less tall than the chip. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(web): anchor machine health tooltip to row left edge Wide tooltip was align=end on the chip, so it grew left off-screen. Use row-span positioning on the machine tile button instead. Co-authored-by: Cursor <cursoragent@cursor.com> * feat(web): machine host card with OS label and inline health Turn the session sidebar machine row into a bordered host panel with OS metadata and side-by-side CPU/RAM meters embedded in the tile instead of a flat label line matching project rows. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(web): keep machine host tile single-row height Collapse the machine header back to one py-1.5 row with OS and compact inline health beside the name, and restore the original project indent without the extra nested rail or second header line. Co-authored-by: Cursor <cursoragent@cursor.com> * feat(web): show CPU core count in machine health tooltip When the runner reports cpuCount, the tooltip reads "CPU across all 6 cores" instead of the generic all-cores label. Co-authored-by: Cursor <cursoragent@cursor.com> * docs: add machine health sidebar screenshots Dogfood captures for the session sidebar machine tile and capacity tooltip, for upstream PR review. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): clear machine-alive priming timeout on disconnect Track the 50ms CPU priming setTimeout and clear it in stopKeepAlive so disconnect/shutdown during the delay cannot leave a stray interval alive. Co-authored-by: Cursor <cursoragent@cursor.com> * chore: drop dogfood screenshots from upstream PR diff Review evidence lives in the PR discussion only; no need to ship PNGs in the repo long-term. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(web): truncate long machine OS/host metadata in sidebar row Bound the metadata span so a long hostname cannot push the health chip or session count off-screen in narrow sidebars. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(web): reveal machine health tooltip on keyboard row focus Wire MACHINE_ROW_TOOLTIP_FOCUS_CLASS and aria-describedby on the machine header button so keyboard users can read the health tooltip like session rows. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): use MemAvailable for Linux RAM pressure on Bun Bun's os.freemem() reflects MemFree (~1% on cache-heavy hosts), which made sidebar RAM read ~99% while btop showed ~40% used. Parse /proc/meminfo MemAvailable instead so used percent matches operator tools. Co-authored-by: Cursor <cursoragent@cursor.com> * feat(web,cli): show machine uptime in sidebar tiles and tooltip Collect os.uptime() as uptimeSeconds on keepalive and render compact up 1h 54m in the machine meta row plus an Uptime line in the health tooltip. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(web): anchor machine health tooltip to chip not row align=row positioned the tooltip below the full machine header button, so the collapsible project panel painted over it on hover. Use align=end with a min-width panel so mouse and keyboard tooltips stay visible. Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-05 06:24:37 +00:00 · 2026-06-29 11:41:53 +08:00
parent 5ade952218
commit 26a24bb6ce
24 changed files with 1194 additions and 50 deletions
@@ -144,3 +144,62 @@ describe('ApiMachineClient listOpencodeModelsForCwd handler', () => {
        }
    })
 })
+
+describe('ApiMachineClient keepAlive lifecycle', () => {
+    beforeEach(() => {
+        vi.useFakeTimers()
+    })
+
+    afterEach(() => {
+        vi.useRealTimers()
+    })
+
+    it('clears priming timeout on shutdown before first machine-alive emit', () => {
+        const machine = makeMachine('machine-keepalive')
+        const client = new ApiMachineClient('cli-token', machine)
+        const emit = vi.fn()
+        ;(client as unknown as { socket: { emit: typeof emit; close: () => void } }).socket = {
+            emit,
+            close: vi.fn(),
+        } as never
+
+        const priv = client as unknown as {
+            startKeepAlive: () => void
+            keepAliveInterval: NodeJS.Timeout | null
+            keepAliveStartTimeout: ReturnType<typeof setTimeout> | null
+        }
+
+        priv.startKeepAlive()
+        client.shutdown()
+        vi.advanceTimersByTime(100)
+
+        expect(emit).not.toHaveBeenCalled()
+        expect(priv.keepAliveInterval).toBeNull()
+        expect(priv.keepAliveStartTimeout).toBeNull()
+    })
+
+    it('clears running keepAlive interval on shutdown', () => {
+        const machine = makeMachine('machine-keepalive-2')
+        const client = new ApiMachineClient('cli-token', machine)
+        const emit = vi.fn()
+        ;(client as unknown as { socket: { emit: typeof emit; close: () => void } }).socket = {
+            emit,
+            close: vi.fn(),
+        } as never
+
+        const priv = client as unknown as {
+            startKeepAlive: () => void
+            keepAliveInterval: NodeJS.Timeout | null
+        }
+
+        priv.startKeepAlive()
+        vi.advanceTimersByTime(50)
+        expect(emit).toHaveBeenCalledTimes(1)
+
+        client.shutdown()
+        vi.advanceTimersByTime(20_000)
+
+        expect(emit).toHaveBeenCalledTimes(1)
+        expect(priv.keepAliveInterval).toBeNull()
+    })
+})
@@ -25,6 +25,7 @@ import {
 import type { SpawnSessionOptions, SpawnSessionResult } from '../modules/common/rpcTypes'
 import { applyVersionedAck } from './versionedUpdate'
 import { buildSocketIoExtraHeaderOptions } from './hubExtraHeaders'
+import { collectMachineHealth } from '@/utils/machineHealth'

 type MachineRpcHandlers = {
    spawnSession: (options: SpawnSessionOptions) => Promise<SpawnSessionResult>
@@ -73,6 +74,7 @@ function formatWorkspaceRoots(paths?: string[]): string {
 export class ApiMachineClient {
    private socket!: Socket<ServerToClientEvents, ClientToServerEvents>
    private keepAliveInterval: NodeJS.Timeout | null = null
+    private keepAliveStartTimeout: ReturnType<typeof setTimeout> | null = null
    private rpcHandlerManager: RpcHandlerManager

    private readonly normalizedWorkspaceRoots: string[] | undefined
@@ -489,15 +491,27 @@ export class ApiMachineClient {

    private startKeepAlive(): void {
        this.stopKeepAlive()
-        this.keepAliveInterval = setInterval(() => {
+        const emitAlive = () => {
            this.socket.emit('machine-alive', {
                machineId: this.machine.id,
-                time: Date.now()
+                time: Date.now(),
+                health: collectMachineHealth()
            })
-        }, 20_000)
+        }
+        // Prime CPU sampling so the first heartbeat already includes CPU %.
+        collectMachineHealth()
+        this.keepAliveStartTimeout = setTimeout(() => {
+            this.keepAliveStartTimeout = null
+            emitAlive()
+            this.keepAliveInterval = setInterval(emitAlive, 20_000)
+        }, 50)
    }

    private stopKeepAlive(): void {
+        if (this.keepAliveStartTimeout) {
+            clearTimeout(this.keepAliveStartTimeout)
+            this.keepAliveStartTimeout = null
+        }
        if (this.keepAliveInterval) {
            clearInterval(this.keepAliveInterval)
            this.keepAliveInterval = null
@@ -0,0 +1,39 @@
+import { describe, expect, it } from 'vitest'
+import { collectMachineHealth, readLinuxMemoryUsedPercent, resetMachineHealthSamplerForTests } from './machineHealth'
+
+describe('readLinuxMemoryUsedPercent', () => {
+    it('uses MemAvailable, not MemFree, so page cache does not read as pressure', () => {
+        const meminfo = `
+MemTotal:       32793696 kB
+MemFree:          578248 kB
+MemAvailable:   18312444 kB
+Buffers:         1196580 kB
+Cached:          9758076 kB
+`.trim()
+
+        expect(readLinuxMemoryUsedPercent(meminfo)).toBe(44)
+    })
+})
+
+describe('collectMachineHealth', () => {
+    it('returns schema-valid health with memory, uptime, and cpu count', () => {
+        resetMachineHealthSamplerForTests()
+        const health = collectMachineHealth(1_700_000_000_000)
+        expect(health.collectedAt).toBe(1_700_000_000_000)
+        expect(health.cpuCount).toBeGreaterThan(0)
+        expect(health.memoryPercent).toBeGreaterThanOrEqual(0)
+        expect(health.memoryPercent).toBeLessThanOrEqual(100)
+        expect(health.uptimeSeconds).toBeGreaterThan(0)
+    })
+
+    it('computes cpu percent after a second sample', async () => {
+        resetMachineHealthSamplerForTests()
+        collectMachineHealth()
+        await new Promise((resolve) => setTimeout(resolve, 50))
+        const second = collectMachineHealth()
+        if (second.cpuPercent !== undefined) {
+            expect(second.cpuPercent).toBeGreaterThanOrEqual(0)
+            expect(second.cpuPercent).toBeLessThanOrEqual(100)
+        }
+    })
+})
@@ -0,0 +1,142 @@
+import { readFileSync } from 'node:fs'
+import { availableParallelism, cpus, freemem, loadavg, platform, totalmem, uptime } from 'node:os'
+import type { MachineHealth } from '@hapi/protocol/types'
+import { MachineHealthSchema } from '@hapi/protocol/schemas'
+
+type CpuTimesSnapshot = {
+    idle: number
+    total: number
+}
+
+let previousCpuSnapshot: CpuTimesSnapshot | null = null
+
+function sumCpuTimes(): CpuTimesSnapshot | null {
+    const cores = cpus()
+    if (cores.length === 0) {
+        return null
+    }
+
+    let idle = 0
+    let total = 0
+    for (const core of cores) {
+        const times = core.times
+        idle += times.idle
+        total += times.user + times.nice + times.sys + times.idle + times.irq
+    }
+
+    return { idle, total }
+}
+
+function computeCpuPercent(current: CpuTimesSnapshot, previous: CpuTimesSnapshot): number | undefined {
+    const idleDelta = current.idle - previous.idle
+    const totalDelta = current.total - previous.total
+    if (totalDelta <= 0) {
+        return undefined
+    }
+
+    const usage = 1 - idleDelta / totalDelta
+    return Math.max(0, Math.min(100, Math.round(usage * 100)))
+}
+
+function parseMeminfoKbValue(meminfo: string, key: string): number | undefined {
+    for (const line of meminfo.split('\n')) {
+        if (!line.startsWith(`${key}:`)) {
+            continue
+        }
+        const kb = Number(line.split(/\s+/)[1])
+        return Number.isFinite(kb) ? kb * 1024 : undefined
+    }
+    return undefined
+}
+
+/** Linux pressure percent: (MemTotal - MemAvailable) / MemTotal. Testable without /proc. */
+export function readLinuxMemoryUsedPercent(meminfo: string): number | undefined {
+    const total = parseMeminfoKbValue(meminfo, 'MemTotal')
+    if (!total || total <= 0) {
+        return undefined
+    }
+
+    const available = parseMeminfoKbValue(meminfo, 'MemAvailable')
+    if (available !== undefined) {
+        return Math.max(0, Math.min(100, Math.round(((total - available) / total) * 100)))
+    }
+
+    // Pre-3.14 kernels: approximate available as free + reclaimable cache.
+    const free = parseMeminfoKbValue(meminfo, 'MemFree')
+    if (free === undefined) {
+        return undefined
+    }
+    const buffers = parseMeminfoKbValue(meminfo, 'Buffers') ?? 0
+    const cached = parseMeminfoKbValue(meminfo, 'Cached') ?? 0
+    const approxAvailable = free + buffers + cached
+    return Math.max(0, Math.min(100, Math.round(((total - approxAvailable) / total) * 100)))
+}
+
+function computeMemoryPercent(): number | undefined {
+    if (platform() === 'linux') {
+        try {
+            const fromProc = readLinuxMemoryUsedPercent(readFileSync('/proc/meminfo', 'utf8'))
+            if (fromProc !== undefined) {
+                return fromProc
+            }
+        } catch {
+            // fall through to os.freemem()
+        }
+    }
+
+    const total = totalmem()
+    if (total <= 0) {
+        return undefined
+    }
+
+    const used = total - freemem()
+    return Math.max(0, Math.min(100, Math.round((used / total) * 100)))
+}
+
+function isUnixLikeLoadPlatform(): boolean {
+    return platform() !== 'win32'
+}
+
+function computeUptimeSeconds(): number | undefined {
+    const seconds = uptime()
+    if (!Number.isFinite(seconds) || seconds < 0) {
+        return undefined
+    }
+    return Math.floor(seconds)
+}
+
+export function collectMachineHealth(now: number = Date.now()): MachineHealth {
+    const cpuCount = availableParallelism()
+    const memoryPercent = computeMemoryPercent()
+    const uptimeSeconds = computeUptimeSeconds()
+    const load1m = isUnixLikeLoadPlatform() ? loadavg()[0] : undefined
+
+    const cpuSnapshot = sumCpuTimes()
+    let cpuPercent: number | undefined
+    if (cpuSnapshot && previousCpuSnapshot) {
+        cpuPercent = computeCpuPercent(cpuSnapshot, previousCpuSnapshot)
+    }
+    if (cpuSnapshot) {
+        previousCpuSnapshot = cpuSnapshot
+    }
+
+    const health = {
+        collectedAt: now,
+        cpuCount,
+        ...(load1m !== undefined ? { load1m } : {}),
+        ...(cpuPercent !== undefined ? { cpuPercent } : {}),
+        ...(memoryPercent !== undefined ? { memoryPercent } : {}),
+        ...(uptimeSeconds !== undefined ? { uptimeSeconds } : {})
+    }
+
+    const parsed = MachineHealthSchema.safeParse(health)
+    if (!parsed.success) {
+        return { collectedAt: now }
+    }
+    return parsed.data
+}
+
+/** Test helper */
+export function resetMachineHealthSamplerForTests(): void {
+    previousCpuSnapshot = null
+}