fix: add retry logic to daemon machine registration for transient connection errors

Add exponential backoff retry mechanism to handle ECONNREFUSED errors when the
server isn't ready yet during daemon startup. This includes:

- New errorUtils module with error classification helpers
- withRetry function in time.ts supporting configurable exponential backoff
- Machine registration retry with sensible defaults (60 attempts, 1-30s delays)
- Error refactoring to consolidate extractErrorInfo utility

close #35
This commit is contained in:
weishu
2026-01-08 11:37:29 +08:00
parent c0446766a5
commit 1665c63d95
4 changed files with 170 additions and 42 deletions
+1 -35
View File
@@ -10,43 +10,9 @@ import { initializeToken } from '@/ui/tokenInit'
import { spawnHappyCLI } from '@/utils/spawnHappyCLI'
import { maybeAutoStartServer } from '@/utils/autoStartServer'
import { withBunRuntimeEnv } from '@/utils/bunRuntime'
import { extractErrorInfo } from '@/utils/errorUtils'
import type { CommandDefinition } from './types'
function extractErrorInfo(error: unknown): {
message: string
messageLower: string
axiosCode?: string
httpStatus?: number
responseErrorText: string
} {
const message = error instanceof Error ? error.message : 'Unknown error'
const messageLower = message.toLowerCase()
if (typeof error !== 'object' || error === null) {
return { message, messageLower, responseErrorText: '' }
}
const record = error as Record<string, unknown>
const axiosCode = typeof record.code === 'string' ? record.code : undefined
const response = typeof record.response === 'object' && record.response !== null
? (record.response as Record<string, unknown>)
: undefined
const httpStatus = typeof response?.status === 'number' ? response.status : undefined
const responseData = response?.data
const responseError = typeof responseData === 'object' && responseData !== null
? (responseData as Record<string, unknown>).error
: undefined
const responseErrorText = typeof responseError === 'string' ? responseError : ''
return {
message,
messageLower,
axiosCode,
httpStatus,
responseErrorText
}
}
export const claudeCommand: CommandDefinition = {
name: 'default',
requiresRuntimeAssets: true,
+20 -6
View File
@@ -12,6 +12,8 @@ import { getEnvironmentInfo } from '@/ui/doctor';
import { spawnHappyCLI } from '@/utils/spawnHappyCLI';
import { writeDaemonState, DaemonLocallyPersistedState, readDaemonState, acquireDaemonLock, releaseDaemonLock } from '@/persistence';
import { isProcessAlive, isWindows, killProcess, killProcessByChildProcess } from '@/utils/process';
import { withRetry } from '@/utils/time';
import { isRetryableConnectionError } from '@/utils/errorUtils';
import { cleanupDaemonState, getInstalledCliMtimeMs, isDaemonRunningCurrentlyInstalledHappyVersion, stopDaemon } from './controlClient';
import { startDaemonControlServer } from './controlServer';
@@ -522,12 +524,24 @@ export async function startDaemon(): Promise<void> {
// Create API client
const api = await ApiClient.create();
// Get or create machine
const machine = await api.getOrCreateMachine({
machineId,
metadata: buildMachineMetadata(),
daemonState: initialDaemonState
});
// Get or create machine (with retry for transient connection errors)
const machine = await withRetry(
() => api.getOrCreateMachine({
machineId,
metadata: buildMachineMetadata(),
daemonState: initialDaemonState
}),
{
maxAttempts: 60,
minDelay: 1000,
maxDelay: 30000,
shouldRetry: isRetryableConnectionError,
onRetry: (error, attempt, nextDelayMs) => {
const errorMsg = error instanceof Error ? error.message : String(error)
logger.debug(`[DAEMON RUN] Failed to register machine (attempt ${attempt}), retrying in ${nextDelayMs}ms: ${errorMsg}`)
}
}
);
logger.debug(`[DAEMON RUN] Machine registered: ${machine.id}`);
// Create realtime machine session
+81
View File
@@ -0,0 +1,81 @@
/**
* Error handling utilities for API requests
*/
export type ErrorInfo = {
message: string
messageLower: string
axiosCode?: string
httpStatus?: number
responseErrorText: string
}
/**
* Extract structured error information from an unknown error
*/
export function extractErrorInfo(error: unknown): ErrorInfo {
const message = error instanceof Error ? error.message : 'Unknown error'
const messageLower = message.toLowerCase()
if (typeof error !== 'object' || error === null) {
return { message, messageLower, responseErrorText: '' }
}
const record = error as Record<string, unknown>
const axiosCode = typeof record.code === 'string' ? record.code : undefined
const response = typeof record.response === 'object' && record.response !== null
? (record.response as Record<string, unknown>)
: undefined
const httpStatus = typeof response?.status === 'number' ? response.status : undefined
const responseData = response?.data
const responseError = typeof responseData === 'object' && responseData !== null
? (responseData as Record<string, unknown>).error
: undefined
const responseErrorText = typeof responseError === 'string' ? responseError : ''
return {
message,
messageLower,
axiosCode,
httpStatus,
responseErrorText
}
}
/**
* Check if an error is a retryable connection error
*
* Retryable errors:
* - ECONNREFUSED - server not started
* - ETIMEDOUT - connection timeout
* - ENOTFOUND - DNS resolution failed
* - ENETUNREACH - network unreachable
* - ECONNRESET - connection reset
* - 5xx - server errors
*
* Non-retryable errors:
* - 401 - authentication failed
* - 403 - permission denied
* - 404 - endpoint not found
* - other 4xx errors
*/
export function isRetryableConnectionError(error: unknown): boolean {
const { axiosCode, httpStatus } = extractErrorInfo(error)
// Retryable network errors
if (axiosCode === 'ECONNREFUSED' ||
axiosCode === 'ETIMEDOUT' ||
axiosCode === 'ENOTFOUND' ||
axiosCode === 'ENETUNREACH' ||
axiosCode === 'ECONNRESET') {
return true
}
// 5xx server errors are retryable
if (httpStatus && httpStatus >= 500) {
return true
}
// Other errors (401, 403, 404, etc.) are not retryable
return false
}
+68 -1
View File
@@ -38,4 +38,71 @@ export function createBackoff(
};
}
export let backoff = createBackoff();
export let backoff = createBackoff();
/**
* Options for withRetry function
*/
export type RetryOptions = {
/** Maximum number of retry attempts. Default: unlimited */
maxAttempts?: number
/** Minimum delay between retries in ms. Default: 1000 */
minDelay?: number
/** Maximum delay between retries in ms. Default: 30000 */
maxDelay?: number
/** Function to determine if error is retryable. Default: retry all errors */
shouldRetry?: (error: unknown) => boolean
/** Callback when a retry is about to happen */
onRetry?: (error: unknown, attempt: number, nextDelayMs: number) => void
}
/**
* Execute a function with retry logic and exponential backoff
*
* Unlike createBackoff, this function:
* - Supports a shouldRetry predicate to skip non-retryable errors
* - Has sensible defaults for daemon-style long-running processes
* - Uses clearer exponential backoff (2^n with jitter)
*/
export async function withRetry<T>(
fn: () => Promise<T>,
options?: RetryOptions
): Promise<T> {
const maxAttempts = options?.maxAttempts ?? Infinity
const minDelay = options?.minDelay ?? 1000
const maxDelay = options?.maxDelay ?? 30000
const shouldRetry = options?.shouldRetry ?? (() => true)
const onRetry = options?.onRetry
let attempt = 0
while (true) {
try {
return await fn()
} catch (error) {
attempt++
// Check if we should retry this error
if (!shouldRetry(error)) {
throw error
}
// Check if we've exceeded max attempts
if (attempt >= maxAttempts) {
throw error
}
// Calculate delay with exponential backoff and jitter
const exponentialDelay = minDelay * Math.pow(2, attempt - 1)
const cappedDelay = Math.min(exponentialDelay, maxDelay)
const jitter = Math.random() * 0.3 * cappedDelay // 0-30% jitter
const nextDelayMs = Math.round(cappedDelay + jitter)
if (onRetry) {
onRetry(error, attempt, nextDelayMs)
}
await delay(nextDelayMs)
}
}
}