mirror of
https://github.com/wu736139669/hapi.git
synced 2026-08-05 06:24:37 +00:00
fix: add retry logic to daemon machine registration for transient connection errors
Add exponential backoff retry mechanism to handle ECONNREFUSED errors when the server isn't ready yet during daemon startup. This includes: - New errorUtils module with error classification helpers - withRetry function in time.ts supporting configurable exponential backoff - Machine registration retry with sensible defaults (60 attempts, 1-30s delays) - Error refactoring to consolidate extractErrorInfo utility close #35
This commit is contained in:
@@ -10,43 +10,9 @@ import { initializeToken } from '@/ui/tokenInit'
|
||||
import { spawnHappyCLI } from '@/utils/spawnHappyCLI'
|
||||
import { maybeAutoStartServer } from '@/utils/autoStartServer'
|
||||
import { withBunRuntimeEnv } from '@/utils/bunRuntime'
|
||||
import { extractErrorInfo } from '@/utils/errorUtils'
|
||||
import type { CommandDefinition } from './types'
|
||||
|
||||
function extractErrorInfo(error: unknown): {
|
||||
message: string
|
||||
messageLower: string
|
||||
axiosCode?: string
|
||||
httpStatus?: number
|
||||
responseErrorText: string
|
||||
} {
|
||||
const message = error instanceof Error ? error.message : 'Unknown error'
|
||||
const messageLower = message.toLowerCase()
|
||||
|
||||
if (typeof error !== 'object' || error === null) {
|
||||
return { message, messageLower, responseErrorText: '' }
|
||||
}
|
||||
|
||||
const record = error as Record<string, unknown>
|
||||
const axiosCode = typeof record.code === 'string' ? record.code : undefined
|
||||
const response = typeof record.response === 'object' && record.response !== null
|
||||
? (record.response as Record<string, unknown>)
|
||||
: undefined
|
||||
const httpStatus = typeof response?.status === 'number' ? response.status : undefined
|
||||
const responseData = response?.data
|
||||
const responseError = typeof responseData === 'object' && responseData !== null
|
||||
? (responseData as Record<string, unknown>).error
|
||||
: undefined
|
||||
const responseErrorText = typeof responseError === 'string' ? responseError : ''
|
||||
|
||||
return {
|
||||
message,
|
||||
messageLower,
|
||||
axiosCode,
|
||||
httpStatus,
|
||||
responseErrorText
|
||||
}
|
||||
}
|
||||
|
||||
export const claudeCommand: CommandDefinition = {
|
||||
name: 'default',
|
||||
requiresRuntimeAssets: true,
|
||||
|
||||
+20
-6
@@ -12,6 +12,8 @@ import { getEnvironmentInfo } from '@/ui/doctor';
|
||||
import { spawnHappyCLI } from '@/utils/spawnHappyCLI';
|
||||
import { writeDaemonState, DaemonLocallyPersistedState, readDaemonState, acquireDaemonLock, releaseDaemonLock } from '@/persistence';
|
||||
import { isProcessAlive, isWindows, killProcess, killProcessByChildProcess } from '@/utils/process';
|
||||
import { withRetry } from '@/utils/time';
|
||||
import { isRetryableConnectionError } from '@/utils/errorUtils';
|
||||
|
||||
import { cleanupDaemonState, getInstalledCliMtimeMs, isDaemonRunningCurrentlyInstalledHappyVersion, stopDaemon } from './controlClient';
|
||||
import { startDaemonControlServer } from './controlServer';
|
||||
@@ -522,12 +524,24 @@ export async function startDaemon(): Promise<void> {
|
||||
// Create API client
|
||||
const api = await ApiClient.create();
|
||||
|
||||
// Get or create machine
|
||||
const machine = await api.getOrCreateMachine({
|
||||
machineId,
|
||||
metadata: buildMachineMetadata(),
|
||||
daemonState: initialDaemonState
|
||||
});
|
||||
// Get or create machine (with retry for transient connection errors)
|
||||
const machine = await withRetry(
|
||||
() => api.getOrCreateMachine({
|
||||
machineId,
|
||||
metadata: buildMachineMetadata(),
|
||||
daemonState: initialDaemonState
|
||||
}),
|
||||
{
|
||||
maxAttempts: 60,
|
||||
minDelay: 1000,
|
||||
maxDelay: 30000,
|
||||
shouldRetry: isRetryableConnectionError,
|
||||
onRetry: (error, attempt, nextDelayMs) => {
|
||||
const errorMsg = error instanceof Error ? error.message : String(error)
|
||||
logger.debug(`[DAEMON RUN] Failed to register machine (attempt ${attempt}), retrying in ${nextDelayMs}ms: ${errorMsg}`)
|
||||
}
|
||||
}
|
||||
);
|
||||
logger.debug(`[DAEMON RUN] Machine registered: ${machine.id}`);
|
||||
|
||||
// Create realtime machine session
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
/**
|
||||
* Error handling utilities for API requests
|
||||
*/
|
||||
|
||||
export type ErrorInfo = {
|
||||
message: string
|
||||
messageLower: string
|
||||
axiosCode?: string
|
||||
httpStatus?: number
|
||||
responseErrorText: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract structured error information from an unknown error
|
||||
*/
|
||||
export function extractErrorInfo(error: unknown): ErrorInfo {
|
||||
const message = error instanceof Error ? error.message : 'Unknown error'
|
||||
const messageLower = message.toLowerCase()
|
||||
|
||||
if (typeof error !== 'object' || error === null) {
|
||||
return { message, messageLower, responseErrorText: '' }
|
||||
}
|
||||
|
||||
const record = error as Record<string, unknown>
|
||||
const axiosCode = typeof record.code === 'string' ? record.code : undefined
|
||||
const response = typeof record.response === 'object' && record.response !== null
|
||||
? (record.response as Record<string, unknown>)
|
||||
: undefined
|
||||
const httpStatus = typeof response?.status === 'number' ? response.status : undefined
|
||||
const responseData = response?.data
|
||||
const responseError = typeof responseData === 'object' && responseData !== null
|
||||
? (responseData as Record<string, unknown>).error
|
||||
: undefined
|
||||
const responseErrorText = typeof responseError === 'string' ? responseError : ''
|
||||
|
||||
return {
|
||||
message,
|
||||
messageLower,
|
||||
axiosCode,
|
||||
httpStatus,
|
||||
responseErrorText
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an error is a retryable connection error
|
||||
*
|
||||
* Retryable errors:
|
||||
* - ECONNREFUSED - server not started
|
||||
* - ETIMEDOUT - connection timeout
|
||||
* - ENOTFOUND - DNS resolution failed
|
||||
* - ENETUNREACH - network unreachable
|
||||
* - ECONNRESET - connection reset
|
||||
* - 5xx - server errors
|
||||
*
|
||||
* Non-retryable errors:
|
||||
* - 401 - authentication failed
|
||||
* - 403 - permission denied
|
||||
* - 404 - endpoint not found
|
||||
* - other 4xx errors
|
||||
*/
|
||||
export function isRetryableConnectionError(error: unknown): boolean {
|
||||
const { axiosCode, httpStatus } = extractErrorInfo(error)
|
||||
|
||||
// Retryable network errors
|
||||
if (axiosCode === 'ECONNREFUSED' ||
|
||||
axiosCode === 'ETIMEDOUT' ||
|
||||
axiosCode === 'ENOTFOUND' ||
|
||||
axiosCode === 'ENETUNREACH' ||
|
||||
axiosCode === 'ECONNRESET') {
|
||||
return true
|
||||
}
|
||||
|
||||
// 5xx server errors are retryable
|
||||
if (httpStatus && httpStatus >= 500) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Other errors (401, 403, 404, etc.) are not retryable
|
||||
return false
|
||||
}
|
||||
+68
-1
@@ -38,4 +38,71 @@ export function createBackoff(
|
||||
};
|
||||
}
|
||||
|
||||
export let backoff = createBackoff();
|
||||
export let backoff = createBackoff();
|
||||
|
||||
/**
|
||||
* Options for withRetry function
|
||||
*/
|
||||
export type RetryOptions = {
|
||||
/** Maximum number of retry attempts. Default: unlimited */
|
||||
maxAttempts?: number
|
||||
/** Minimum delay between retries in ms. Default: 1000 */
|
||||
minDelay?: number
|
||||
/** Maximum delay between retries in ms. Default: 30000 */
|
||||
maxDelay?: number
|
||||
/** Function to determine if error is retryable. Default: retry all errors */
|
||||
shouldRetry?: (error: unknown) => boolean
|
||||
/** Callback when a retry is about to happen */
|
||||
onRetry?: (error: unknown, attempt: number, nextDelayMs: number) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a function with retry logic and exponential backoff
|
||||
*
|
||||
* Unlike createBackoff, this function:
|
||||
* - Supports a shouldRetry predicate to skip non-retryable errors
|
||||
* - Has sensible defaults for daemon-style long-running processes
|
||||
* - Uses clearer exponential backoff (2^n with jitter)
|
||||
*/
|
||||
export async function withRetry<T>(
|
||||
fn: () => Promise<T>,
|
||||
options?: RetryOptions
|
||||
): Promise<T> {
|
||||
const maxAttempts = options?.maxAttempts ?? Infinity
|
||||
const minDelay = options?.minDelay ?? 1000
|
||||
const maxDelay = options?.maxDelay ?? 30000
|
||||
const shouldRetry = options?.shouldRetry ?? (() => true)
|
||||
const onRetry = options?.onRetry
|
||||
|
||||
let attempt = 0
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
return await fn()
|
||||
} catch (error) {
|
||||
attempt++
|
||||
|
||||
// Check if we should retry this error
|
||||
if (!shouldRetry(error)) {
|
||||
throw error
|
||||
}
|
||||
|
||||
// Check if we've exceeded max attempts
|
||||
if (attempt >= maxAttempts) {
|
||||
throw error
|
||||
}
|
||||
|
||||
// Calculate delay with exponential backoff and jitter
|
||||
const exponentialDelay = minDelay * Math.pow(2, attempt - 1)
|
||||
const cappedDelay = Math.min(exponentialDelay, maxDelay)
|
||||
const jitter = Math.random() * 0.3 * cappedDelay // 0-30% jitter
|
||||
const nextDelayMs = Math.round(cappedDelay + jitter)
|
||||
|
||||
if (onRetry) {
|
||||
onRetry(error, attempt, nextDelayMs)
|
||||
}
|
||||
|
||||
await delay(nextDelayMs)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user