mirror of
https://github.com/wu736139669/hapi.git
synced 2026-11-09 01:51:39 +00:00
fix(hub,cli): run OpenCode usage scans off the process event loop
The 5-minute reconciliation read the multi-GB OpenCode store on the hub's event loop with bun:sqlite; a cold scan froze every request for ~88s and stalled each machine's runner the same way. - scan in a spawned `hapi usage-scan` child process (stdin/stdout JSON) - refresh on demand from the usage page (debounced) + a daily fallback - a runner whose hub is localhost skips the store the hub already owns - machine scans run every 30min and read the message table sequentially in one pass instead of per-session indexed lookups
This commit is contained in:
@@ -21,6 +21,7 @@ import { notifyCommand } from './notify'
|
||||
import { hubCommand } from './hub'
|
||||
import { pingPeerCommand } from './pingPeer'
|
||||
import { inspectPeerCommand } from './inspectPeer'
|
||||
import { usageScanCommand } from './usageScan'
|
||||
import type { CommandContext, CommandDefinition } from './types'
|
||||
|
||||
// Gemini CLI was sunset (Google stopped serving the consumer Gemini CLI on
|
||||
@@ -63,7 +64,10 @@ const COMMANDS: CommandDefinition[] = [
|
||||
runnerCommand,
|
||||
notifyCommand,
|
||||
pingPeerCommand,
|
||||
inspectPeerCommand
|
||||
inspectPeerCommand,
|
||||
// Internal: spawned by the hub to scan the local OpenCode store without
|
||||
// blocking its event loop (see hub/src/sync/usageReconciliation.ts).
|
||||
usageScanCommand
|
||||
]
|
||||
|
||||
const commandMap = new Map<string, CommandDefinition>()
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import { Database } from 'bun:sqlite'
|
||||
import { describe, expect, it } from 'bun:test'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { parseUsageScanRequest, runUsageScan } from './usageScan'
|
||||
|
||||
function createOpencodeStore(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'hapi-usage-scan-'))
|
||||
const dbPath = join(dir, 'opencode.db')
|
||||
const db = new Database(dbPath, { create: true })
|
||||
db.exec(`
|
||||
CREATE TABLE message (
|
||||
id TEXT PRIMARY KEY,
|
||||
session_id TEXT NOT NULL,
|
||||
time_created INTEGER NOT NULL,
|
||||
time_updated INTEGER NOT NULL,
|
||||
data TEXT NOT NULL
|
||||
)
|
||||
`)
|
||||
db.prepare('INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (?, ?, ?, ?, ?)').run(
|
||||
'msg-1',
|
||||
'ses_scan',
|
||||
Date.parse('2026-09-17T10:00:00'),
|
||||
Date.parse('2026-09-17T10:00:00'),
|
||||
JSON.stringify({
|
||||
role: 'assistant',
|
||||
providerID: 'opencode-go',
|
||||
modelID: 'deepseek-v4.1-flash',
|
||||
time: { created: Date.parse('2026-09-17T10:00:00') },
|
||||
tokens: { input: 7, output: 3, reasoning: 1, cache: { read: 5, write: 2 } }
|
||||
})
|
||||
)
|
||||
db.close()
|
||||
return dbPath
|
||||
}
|
||||
|
||||
describe('usage-scan command', () => {
|
||||
it('returns the hub-facing payload for the requested sessions', async () => {
|
||||
const dbPath = createOpencodeStore()
|
||||
try {
|
||||
const payload = await runUsageScan({ dbPath, sessionIds: ['ses_scan', 'ses_missing'] })
|
||||
expect(payload.results.ses_scan.messages).toBe(1)
|
||||
expect(payload.results.ses_scan.rows).toEqual([
|
||||
expect.objectContaining({
|
||||
day: '2026-09-17',
|
||||
model: 'opencode-go/deepseek-v4.1-flash',
|
||||
inputTokens: 14,
|
||||
outputTokens: 4,
|
||||
requests: 1
|
||||
})
|
||||
])
|
||||
expect(payload.results.ses_missing).toEqual({ rows: [], messages: 0 })
|
||||
} finally {
|
||||
rmSync(join(dbPath, '..'), { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('listAll scans every session present in the store', async () => {
|
||||
const dbPath = createOpencodeStore()
|
||||
try {
|
||||
const payload = await runUsageScan({ dbPath, listAll: true })
|
||||
expect(Object.keys(payload.results)).toEqual(['ses_scan'])
|
||||
expect(payload.results.ses_scan.messages).toBe(1)
|
||||
// The sequential all-sessions pass must match the indexed
|
||||
// per-session reads (this path feeds the runner's reports).
|
||||
const perSession = await runUsageScan({ dbPath, sessionIds: ['ses_scan'] })
|
||||
expect(payload.results.ses_scan).toEqual(perSession.results.ses_scan)
|
||||
} finally {
|
||||
rmSync(join(dbPath, '..'), { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects malformed requests', () => {
|
||||
expect(() => parseUsageScanRequest('not json')).toThrow()
|
||||
expect(() => parseUsageScanRequest('{"sessionIds":[]}')).toThrow()
|
||||
expect(parseUsageScanRequest('{"dbPath":"/x","sessionIds":["ses_a",3,""]}')).toEqual({
|
||||
dbPath: '/x',
|
||||
sessionIds: ['ses_a']
|
||||
})
|
||||
expect(parseUsageScanRequest('{"dbPath":"/x","listAll":true}')).toEqual({
|
||||
dbPath: '/x',
|
||||
listAll: true
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,62 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import type { CommandDefinition } from './types'
|
||||
|
||||
/**
|
||||
* Internal command: aggregate OpenCode token usage.
|
||||
*
|
||||
* The hub and the runner spawn `hapi usage-scan` as a child process so the
|
||||
* synchronous bun:sqlite reads over the multi-GB OpenCode store never block
|
||||
* their event loops (a cold scan used to freeze the hub for a minute or
|
||||
* more). The request arrives as JSON on stdin, the result leaves as JSON on
|
||||
* stdout: `{ dbPath, sessionIds }` for known sessions, or
|
||||
* `{ dbPath, listAll: true }` to scan every session found in the store.
|
||||
*/
|
||||
|
||||
export type UsageScanRequest = {
|
||||
dbPath: string
|
||||
sessionIds?: string[]
|
||||
listAll?: boolean
|
||||
}
|
||||
|
||||
function readStdin(): string {
|
||||
// fd 0 works for both pipe and file-backed stdin; Bun's async iteration
|
||||
// over a file-backed stdin yields nothing.
|
||||
return readFileSync(0, 'utf8')
|
||||
}
|
||||
|
||||
export function parseUsageScanRequest(raw: string): UsageScanRequest {
|
||||
const parsed = JSON.parse(raw) as { dbPath?: unknown; sessionIds?: unknown; listAll?: unknown }
|
||||
if (typeof parsed.dbPath !== 'string') {
|
||||
throw new Error('usage-scan expects { dbPath: string, sessionIds?: string[], listAll?: boolean } on stdin')
|
||||
}
|
||||
if (parsed.listAll === true) {
|
||||
return { dbPath: parsed.dbPath, listAll: true }
|
||||
}
|
||||
if (!Array.isArray(parsed.sessionIds)) {
|
||||
throw new Error('usage-scan expects sessionIds or listAll: true on stdin')
|
||||
}
|
||||
const sessionIds = parsed.sessionIds.filter((id): id is string => typeof id === 'string' && id.length > 0)
|
||||
return { dbPath: parsed.dbPath, sessionIds }
|
||||
}
|
||||
|
||||
/** Payload shape the hub/runner parse (see hub/src/sync/usageReconciliation.ts). */
|
||||
export async function runUsageScan(request: UsageScanRequest): Promise<{ results: Record<string, { rows: unknown[]; messages: number }> }> {
|
||||
const { readOpencodeUsageForAllSessions, readOpencodeUsageForSessions } = await import('@hapi/protocol/opencodeUsage')
|
||||
const usage = request.listAll === true
|
||||
? await readOpencodeUsageForAllSessions(request.dbPath)
|
||||
: await readOpencodeUsageForSessions(request.dbPath, request.sessionIds ?? [])
|
||||
const results: Record<string, { rows: unknown[]; messages: number }> = {}
|
||||
for (const [sessionId, entry] of usage) {
|
||||
results[sessionId] = { rows: entry.rows, messages: entry.messages }
|
||||
}
|
||||
return { results }
|
||||
}
|
||||
|
||||
export const usageScanCommand: CommandDefinition = {
|
||||
name: 'usage-scan',
|
||||
requiresRuntimeAssets: false,
|
||||
run: async () => {
|
||||
const request = parseUsageScanRequest(readStdin())
|
||||
process.stdout.write(JSON.stringify(await runUsageScan(request)))
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { listOpencodeSessionIds, readOpencodeUsageForSessions, resolveOpencodeDbPath } from '@hapi/protocol/opencodeUsage'
|
||||
import type { OpencodeUsageSessionReport } from '@hapi/protocol/usage'
|
||||
import type { OpencodeUsageReportRow, OpencodeUsageSessionReport } from '@hapi/protocol/usage'
|
||||
import { logger } from '@/ui/logger'
|
||||
|
||||
/**
|
||||
@@ -12,12 +12,18 @@ import { logger } from '@/ui/logger'
|
||||
* their reconciliation rows, which makes repeated or concurrent reports
|
||||
* idempotent (see hub/src/sync/usageReconciliation.ts).
|
||||
*
|
||||
* The scan runs in a spawned `hapi usage-scan` child process: the OpenCode
|
||||
* store is multi-GB in practice and `bun:sqlite` is synchronous, so an
|
||||
* in-process scan would stall this runner's event loop — machine RPCs
|
||||
* (spawn, model listing) queue behind it and can time out.
|
||||
*
|
||||
* The live chat pipeline is not involved at all — a report never becomes a
|
||||
* timeline message.
|
||||
*/
|
||||
|
||||
const SCAN_INTERVAL_MS = 5 * 60 * 1000
|
||||
const SCAN_INTERVAL_MS = 30 * 60 * 1000
|
||||
const INITIAL_DELAY_MS = 20_000
|
||||
const SCAN_TIMEOUT_MS = 2 * 60 * 1000
|
||||
|
||||
export type OpencodeUsageScannerOptions = {
|
||||
/** Best-effort delivery; the next scan re-sends on failure. */
|
||||
@@ -35,15 +41,82 @@ export type OpencodeUsageScanner = {
|
||||
runNow: () => void
|
||||
}
|
||||
|
||||
type ScannedUsage = Map<string, { rows: OpencodeUsageReportRow[]; messages: number }>
|
||||
|
||||
async function spawnUsageScan(request: { dbPath: string; listAll: true }): Promise<ScannedUsage | null> {
|
||||
if (typeof Bun === 'undefined') return null
|
||||
const executable = process.env.HAPI_CLI_EXECUTABLE?.trim() || process.execPath
|
||||
if (!executable) return null
|
||||
try {
|
||||
const proc = Bun.spawn({
|
||||
cmd: [executable, 'usage-scan'],
|
||||
stdin: 'pipe',
|
||||
stdout: 'pipe',
|
||||
stderr: 'ignore'
|
||||
})
|
||||
const timer = setTimeout(() => {
|
||||
try {
|
||||
proc.kill()
|
||||
} catch {
|
||||
}
|
||||
}, SCAN_TIMEOUT_MS)
|
||||
try {
|
||||
proc.stdin.write(JSON.stringify(request))
|
||||
proc.stdin.end()
|
||||
const [exitCode, stdout] = await Promise.all([proc.exited, new Response(proc.stdout).text()])
|
||||
if (exitCode !== 0 || stdout.trim().length === 0) return null
|
||||
const parsed = JSON.parse(stdout) as { results?: unknown }
|
||||
const results = parsed.results
|
||||
if (results === null || typeof results !== 'object' || Array.isArray(results)) return null
|
||||
const usage: ScannedUsage = new Map()
|
||||
for (const [sessionId, value] of Object.entries(results as Record<string, unknown>)) {
|
||||
const entry = value as { rows?: unknown; messages?: unknown } | null
|
||||
if (!entry || typeof entry !== 'object' || !Array.isArray(entry.rows)) return null
|
||||
usage.set(sessionId, {
|
||||
rows: entry.rows as OpencodeUsageReportRow[],
|
||||
messages: typeof entry.messages === 'number' ? entry.messages : entry.rows.length
|
||||
})
|
||||
}
|
||||
return usage
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/** Probe once whether the current executable can actually run `usage-scan`. */
|
||||
let cliScanAvailablePromise: Promise<boolean> | null = null
|
||||
|
||||
function cliScanAvailable(): Promise<boolean> {
|
||||
cliScanAvailablePromise ??= (async () => {
|
||||
const probe = await spawnUsageScan({ dbPath: '', listAll: true })
|
||||
return probe !== null
|
||||
})()
|
||||
return cliScanAvailablePromise
|
||||
}
|
||||
|
||||
async function scanOpencodeStore(dbPath: string): Promise<ScannedUsage> {
|
||||
if (await cliScanAvailable()) {
|
||||
const scanned = await spawnUsageScan({ dbPath, listAll: true })
|
||||
if (scanned) return scanned
|
||||
throw new Error('opencode usage scan failed')
|
||||
}
|
||||
// Dev/test fallback (no hapi binary to spawn): read in-process. This is
|
||||
// deliberately not used in production — the multi-GB scan must never run
|
||||
// on this process's event loop.
|
||||
const opencodeSessionIds = await listOpencodeSessionIds(dbPath)
|
||||
if (opencodeSessionIds.length === 0) return new Map()
|
||||
return await readOpencodeUsageForSessions(dbPath, opencodeSessionIds)
|
||||
}
|
||||
|
||||
/** Collect snapshot rows for every session present in the OpenCode store. */
|
||||
export async function collectOpencodeUsageReports(dbPath: string): Promise<OpencodeUsageSessionReport[]> {
|
||||
const opencodeSessionIds = await listOpencodeSessionIds(dbPath)
|
||||
if (opencodeSessionIds.length === 0) return []
|
||||
const usageBySession = await readOpencodeUsageForSessions(dbPath, opencodeSessionIds)
|
||||
const usageBySession = await scanOpencodeStore(dbPath)
|
||||
const reports: OpencodeUsageSessionReport[] = []
|
||||
for (const opencodeSessionId of opencodeSessionIds) {
|
||||
const usage = usageBySession.get(opencodeSessionId)
|
||||
if (!usage || usage.rows.length === 0) continue
|
||||
for (const [opencodeSessionId, usage] of usageBySession) {
|
||||
if (usage.rows.length === 0) continue
|
||||
reports.push({ opencodeSessionId, rows: usage.rows })
|
||||
}
|
||||
return reports
|
||||
|
||||
@@ -1166,8 +1166,11 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}):
|
||||
apiMachine.connect();
|
||||
// Report this machine's OpenCode usage snapshots so the hub's token
|
||||
// dashboard does not under-count tool-loop turns on non-hub machines.
|
||||
// The scan reads the local OpenCode store; best-effort, never blocks.
|
||||
const opencodeUsageScanner = startOpencodeUsageScanner({
|
||||
// A runner whose hub is localhost would only rescan the store the hub
|
||||
// already reconciles itself (see hub/src/sync/usageReconciliation.ts),
|
||||
// so skip it there. The scan itself runs in a child process.
|
||||
const hubIsLocal = /^https?:\/\/(localhost|127\.0\.0\.1|\[::1\])(?::|\/|$)/i.test(configuration.apiUrl);
|
||||
const opencodeUsageScanner = hubIsLocal ? null : startOpencodeUsageScanner({
|
||||
report: (sessions) => apiMachine.reportOpencodeUsage(sessions)
|
||||
});
|
||||
scheduleCursorModelsPrewarm();
|
||||
@@ -1435,7 +1438,7 @@ export async function startRunner(options: { workspaceRoots?: string[] } = {}):
|
||||
clearInterval(restartOnStaleVersionAndHeartbeat);
|
||||
logger.debug('[RUNNER RUN] Health check interval cleared');
|
||||
}
|
||||
opencodeUsageScanner.stop();
|
||||
opencodeUsageScanner?.stop();
|
||||
|
||||
// Update runner state before shutting down
|
||||
await apiMachine.updateRunnerState((state: RunnerState | null) => ({
|
||||
|
||||
Reference in New Issue
Block a user