fix(hub,cli): run OpenCode usage scans off the process event loop

The 5-minute reconciliation read the multi-GB OpenCode store on the hub's
event loop with bun:sqlite; a cold scan froze every request for ~88s and
stalled each machine's runner the same way.

- scan in a spawned `hapi usage-scan` child process (stdin/stdout JSON)
- refresh on demand from the usage page (debounced) + a daily fallback
- a runner whose hub is localhost skips the store the hub already owns
- machine scans run every 30min and read the message table sequentially
  in one pass instead of per-session indexed lookups
This commit is contained in:
2026-10-16 19:38:07 +08:00
parent a759a62b77
commit 2d57763177
10 changed files with 474 additions and 50 deletions
+5 -1
View File
@@ -21,6 +21,7 @@ import { notifyCommand } from './notify'
import { hubCommand } from './hub'
import { pingPeerCommand } from './pingPeer'
import { inspectPeerCommand } from './inspectPeer'
import { usageScanCommand } from './usageScan'
import type { CommandContext, CommandDefinition } from './types'
// Gemini CLI was sunset (Google stopped serving the consumer Gemini CLI on
@@ -63,7 +64,10 @@ const COMMANDS: CommandDefinition[] = [
runnerCommand,
notifyCommand,
pingPeerCommand,
inspectPeerCommand
inspectPeerCommand,
// Internal: spawned by the hub to scan the local OpenCode store without
// blocking its event loop (see hub/src/sync/usageReconciliation.ts).
usageScanCommand
]
const commandMap = new Map<string, CommandDefinition>()
+86
View File
@@ -0,0 +1,86 @@
import { Database } from 'bun:sqlite'
import { describe, expect, it } from 'bun:test'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { parseUsageScanRequest, runUsageScan } from './usageScan'
function createOpencodeStore(): string {
const dir = mkdtempSync(join(tmpdir(), 'hapi-usage-scan-'))
const dbPath = join(dir, 'opencode.db')
const db = new Database(dbPath, { create: true })
db.exec(`
CREATE TABLE message (
id TEXT PRIMARY KEY,
session_id TEXT NOT NULL,
time_created INTEGER NOT NULL,
time_updated INTEGER NOT NULL,
data TEXT NOT NULL
)
`)
db.prepare('INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (?, ?, ?, ?, ?)').run(
'msg-1',
'ses_scan',
Date.parse('2026-09-17T10:00:00'),
Date.parse('2026-09-17T10:00:00'),
JSON.stringify({
role: 'assistant',
providerID: 'opencode-go',
modelID: 'deepseek-v4.1-flash',
time: { created: Date.parse('2026-09-17T10:00:00') },
tokens: { input: 7, output: 3, reasoning: 1, cache: { read: 5, write: 2 } }
})
)
db.close()
return dbPath
}
describe('usage-scan command', () => {
it('returns the hub-facing payload for the requested sessions', async () => {
const dbPath = createOpencodeStore()
try {
const payload = await runUsageScan({ dbPath, sessionIds: ['ses_scan', 'ses_missing'] })
expect(payload.results.ses_scan.messages).toBe(1)
expect(payload.results.ses_scan.rows).toEqual([
expect.objectContaining({
day: '2026-09-17',
model: 'opencode-go/deepseek-v4.1-flash',
inputTokens: 14,
outputTokens: 4,
requests: 1
})
])
expect(payload.results.ses_missing).toEqual({ rows: [], messages: 0 })
} finally {
rmSync(join(dbPath, '..'), { recursive: true, force: true })
}
})
it('listAll scans every session present in the store', async () => {
const dbPath = createOpencodeStore()
try {
const payload = await runUsageScan({ dbPath, listAll: true })
expect(Object.keys(payload.results)).toEqual(['ses_scan'])
expect(payload.results.ses_scan.messages).toBe(1)
// The sequential all-sessions pass must match the indexed
// per-session reads (this path feeds the runner's reports).
const perSession = await runUsageScan({ dbPath, sessionIds: ['ses_scan'] })
expect(payload.results.ses_scan).toEqual(perSession.results.ses_scan)
} finally {
rmSync(join(dbPath, '..'), { recursive: true, force: true })
}
})
it('rejects malformed requests', () => {
expect(() => parseUsageScanRequest('not json')).toThrow()
expect(() => parseUsageScanRequest('{"sessionIds":[]}')).toThrow()
expect(parseUsageScanRequest('{"dbPath":"/x","sessionIds":["ses_a",3,""]}')).toEqual({
dbPath: '/x',
sessionIds: ['ses_a']
})
expect(parseUsageScanRequest('{"dbPath":"/x","listAll":true}')).toEqual({
dbPath: '/x',
listAll: true
})
})
})
+62
View File
@@ -0,0 +1,62 @@
import { readFileSync } from 'node:fs'
import type { CommandDefinition } from './types'
/**
* Internal command: aggregate OpenCode token usage.
*
* The hub and the runner spawn `hapi usage-scan` as a child process so the
* synchronous bun:sqlite reads over the multi-GB OpenCode store never block
* their event loops (a cold scan used to freeze the hub for a minute or
* more). The request arrives as JSON on stdin, the result leaves as JSON on
* stdout: `{ dbPath, sessionIds }` for known sessions, or
* `{ dbPath, listAll: true }` to scan every session found in the store.
*/
export type UsageScanRequest = {
dbPath: string
sessionIds?: string[]
listAll?: boolean
}
function readStdin(): string {
// fd 0 works for both pipe and file-backed stdin; Bun's async iteration
// over a file-backed stdin yields nothing.
return readFileSync(0, 'utf8')
}
export function parseUsageScanRequest(raw: string): UsageScanRequest {
const parsed = JSON.parse(raw) as { dbPath?: unknown; sessionIds?: unknown; listAll?: unknown }
if (typeof parsed.dbPath !== 'string') {
throw new Error('usage-scan expects { dbPath: string, sessionIds?: string[], listAll?: boolean } on stdin')
}
if (parsed.listAll === true) {
return { dbPath: parsed.dbPath, listAll: true }
}
if (!Array.isArray(parsed.sessionIds)) {
throw new Error('usage-scan expects sessionIds or listAll: true on stdin')
}
const sessionIds = parsed.sessionIds.filter((id): id is string => typeof id === 'string' && id.length > 0)
return { dbPath: parsed.dbPath, sessionIds }
}
/** Payload shape the hub/runner parse (see hub/src/sync/usageReconciliation.ts). */
export async function runUsageScan(request: UsageScanRequest): Promise<{ results: Record<string, { rows: unknown[]; messages: number }> }> {
const { readOpencodeUsageForAllSessions, readOpencodeUsageForSessions } = await import('@hapi/protocol/opencodeUsage')
const usage = request.listAll === true
? await readOpencodeUsageForAllSessions(request.dbPath)
: await readOpencodeUsageForSessions(request.dbPath, request.sessionIds ?? [])
const results: Record<string, { rows: unknown[]; messages: number }> = {}
for (const [sessionId, entry] of usage) {
results[sessionId] = { rows: entry.rows, messages: entry.messages }
}
return { results }
}
export const usageScanCommand: CommandDefinition = {
name: 'usage-scan',
requiresRuntimeAssets: false,
run: async () => {
const request = parseUsageScanRequest(readStdin())
process.stdout.write(JSON.stringify(await runUsageScan(request)))
}
}
+81 -8
View File
@@ -1,5 +1,5 @@
import { listOpencodeSessionIds, readOpencodeUsageForSessions, resolveOpencodeDbPath } from '@hapi/protocol/opencodeUsage'
import type { OpencodeUsageSessionReport } from '@hapi/protocol/usage'
import type { OpencodeUsageReportRow, OpencodeUsageSessionReport } from '@hapi/protocol/usage'
import { logger } from '@/ui/logger'
/**
@@ -12,12 +12,18 @@ import { logger } from '@/ui/logger'
* their reconciliation rows, which makes repeated or concurrent reports
* idempotent (see hub/src/sync/usageReconciliation.ts).
*
* The scan runs in a spawned `hapi usage-scan` child process: the OpenCode
* store is multi-GB in practice and `bun:sqlite` is synchronous, so an
* in-process scan would stall this runner's event loop — machine RPCs
* (spawn, model listing) queue behind it and can time out.
*
* The live chat pipeline is not involved at all — a report never becomes a
* timeline message.
*/
const SCAN_INTERVAL_MS = 5 * 60 * 1000
const SCAN_INTERVAL_MS = 30 * 60 * 1000
const INITIAL_DELAY_MS = 20_000
const SCAN_TIMEOUT_MS = 2 * 60 * 1000
export type OpencodeUsageScannerOptions = {
/** Best-effort delivery; the next scan re-sends on failure. */
@@ -35,15 +41,82 @@ export type OpencodeUsageScanner = {
runNow: () => void
}
type ScannedUsage = Map<string, { rows: OpencodeUsageReportRow[]; messages: number }>
async function spawnUsageScan(request: { dbPath: string; listAll: true }): Promise<ScannedUsage | null> {
if (typeof Bun === 'undefined') return null
const executable = process.env.HAPI_CLI_EXECUTABLE?.trim() || process.execPath
if (!executable) return null
try {
const proc = Bun.spawn({
cmd: [executable, 'usage-scan'],
stdin: 'pipe',
stdout: 'pipe',
stderr: 'ignore'
})
const timer = setTimeout(() => {
try {
proc.kill()
} catch {
}
}, SCAN_TIMEOUT_MS)
try {
proc.stdin.write(JSON.stringify(request))
proc.stdin.end()
const [exitCode, stdout] = await Promise.all([proc.exited, new Response(proc.stdout).text()])
if (exitCode !== 0 || stdout.trim().length === 0) return null
const parsed = JSON.parse(stdout) as { results?: unknown }
const results = parsed.results
if (results === null || typeof results !== 'object' || Array.isArray(results)) return null
const usage: ScannedUsage = new Map()
for (const [sessionId, value] of Object.entries(results as Record<string, unknown>)) {
const entry = value as { rows?: unknown; messages?: unknown } | null
if (!entry || typeof entry !== 'object' || !Array.isArray(entry.rows)) return null
usage.set(sessionId, {
rows: entry.rows as OpencodeUsageReportRow[],
messages: typeof entry.messages === 'number' ? entry.messages : entry.rows.length
})
}
return usage
} finally {
clearTimeout(timer)
}
} catch {
return null
}
}
/** Probe once whether the current executable can actually run `usage-scan`. */
let cliScanAvailablePromise: Promise<boolean> | null = null
function cliScanAvailable(): Promise<boolean> {
cliScanAvailablePromise ??= (async () => {
const probe = await spawnUsageScan({ dbPath: '', listAll: true })
return probe !== null
})()
return cliScanAvailablePromise
}
async function scanOpencodeStore(dbPath: string): Promise<ScannedUsage> {
if (await cliScanAvailable()) {
const scanned = await spawnUsageScan({ dbPath, listAll: true })
if (scanned) return scanned
throw new Error('opencode usage scan failed')
}
// Dev/test fallback (no hapi binary to spawn): read in-process. This is
// deliberately not used in production — the multi-GB scan must never run
// on this process's event loop.
const opencodeSessionIds = await listOpencodeSessionIds(dbPath)
if (opencodeSessionIds.length === 0) return new Map()
return await readOpencodeUsageForSessions(dbPath, opencodeSessionIds)
}
/** Collect snapshot rows for every session present in the OpenCode store. */
export async function collectOpencodeUsageReports(dbPath: string): Promise<OpencodeUsageSessionReport[]> {
const opencodeSessionIds = await listOpencodeSessionIds(dbPath)
if (opencodeSessionIds.length === 0) return []
const usageBySession = await readOpencodeUsageForSessions(dbPath, opencodeSessionIds)
const usageBySession = await scanOpencodeStore(dbPath)
const reports: OpencodeUsageSessionReport[] = []
for (const opencodeSessionId of opencodeSessionIds) {
const usage = usageBySession.get(opencodeSessionId)
if (!usage || usage.rows.length === 0) continue
for (const [opencodeSessionId, usage] of usageBySession) {
if (usage.rows.length === 0) continue
reports.push({ opencodeSessionId, rows: usage.rows })
}
return reports
+6 -3
View File
@@ -1166,8 +1166,11 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}):
apiMachine.connect();
// Report this machine's OpenCode usage snapshots so the hub's token
// dashboard does not under-count tool-loop turns on non-hub machines.
// The scan reads the local OpenCode store; best-effort, never blocks.
const opencodeUsageScanner = startOpencodeUsageScanner({
// A runner whose hub is localhost would only rescan the store the hub
// already reconciles itself (see hub/src/sync/usageReconciliation.ts),
// so skip it there. The scan itself runs in a child process.
const hubIsLocal = /^https?:\/\/(localhost|127\.0\.0\.1|\[::1\])(?::|\/|$)/i.test(configuration.apiUrl);
const opencodeUsageScanner = hubIsLocal ? null : startOpencodeUsageScanner({
report: (sessions) => apiMachine.reportOpencodeUsage(sessions)
});
scheduleCursorModelsPrewarm();
@@ -1435,7 +1438,7 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}):
clearInterval(restartOnStaleVersionAndHeartbeat);
logger.debug('[RUNNER RUN] Health check interval cleared');
}
opencodeUsageScanner.stop();
opencodeUsageScanner?.stop();
// Update runner state before shutting down
await apiMachine.updateRunnerState((state: RunnerState | null) => ({