feat: improve spawn error handling and reporting across full stack (#249)

* feat: improve spawn error handling and reporting across full stack - Return error result instead of throwing in apiMachine spawn handler - Add lastSpawnError field to RunnerState for persistent error tracking - Add error awaiter system for early process exit/error detection before webhook - Build detailed webhook failure messages with exit code, signal, and stderr tail - Report spawn outcomes to hub via runner state updates - Handle more spawn result types in rpcGateway with better error messages - Display runner last spawn error in web UI (NewSession & SpawnSession) - Extract shared formatRunnerSpawnError utility to avoid duplication Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix(cli): narrow spawnResult type check to fix TS2339 error Use `type === 'error'` instead of `type !== 'success'` to properly narrow the discriminated union, allowing TypeScript to infer errorMessage. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-08-05 06:24:37 +00:00 · 2026-03-09 20:55:50 +08:00
parent b4b73d4405
commit 4716d315b7
8 changed files with 234 additions and 7 deletions
@@ -127,7 +127,7 @@ export class ApiMachineClient {
                case 'requestToApproveDirectoryCreation':
                    return { type: 'requestToApproveDirectoryCreation', directory: result.directory }
                case 'error':
-                    throw new Error(result.errorMessage)
+                    return { type: 'error', errorMessage: result.errorMessage }
            }
        })

@@ -43,7 +43,14 @@ export const RunnerStateSchema = z.object({
    httpPort: z.number().optional(),
    startedAt: z.number().optional(),
    shutdownRequestedAt: z.number().optional(),
-    shutdownSource: z.union([z.enum(['mobile-app', 'cli', 'os-signal', 'unknown']), z.string()]).optional()
+    shutdownSource: z.union([z.enum(['mobile-app', 'cli', 'os-signal', 'unknown']), z.string()]).optional(),
+    lastSpawnError: z.object({
+        message: z.string(),
+        pid: z.number().optional(),
+        exitCode: z.number().nullable().optional(),
+        signal: z.string().nullable().optional(),
+        at: z.number()
+    }).nullable().optional()
 })

 export type RunnerState = z.infer<typeof RunnerStateSchema>
@@ -127,6 +127,20 @@ export async function startRunner(): Promise<void> {

    // Session spawning awaiter system
    const pidToAwaiter = new Map<number, (session: TrackedSession) => void>();
+    const pidToErrorAwaiter = new Map<number, (errorMessage: string) => void>();
+    type SpawnFailureDetails = {
+      message: string
+      pid?: number
+      exitCode?: number | null
+      signal?: NodeJS.Signals | null
+    };
+    let reportSpawnOutcomeToHub: ((outcome: { type: 'success' } | { type: 'error'; details: SpawnFailureDetails }) => void) | null = null;
+    const formatSpawnError = (error: unknown): string => {
+      if (error instanceof Error) {
+        return error.message;
+      }
+      return String(error);
+    };

    // Helper functions
    const getCurrentChildren = () => Array.from(pidToTrackedSession.values());
@@ -157,6 +171,7 @@ export async function startRunner(): Promise<void> {
        const awaiter = pidToAwaiter.get(pid);
        if (awaiter) {
          pidToAwaiter.delete(pid);
+          pidToErrorAwaiter.delete(pid);
          awaiter(existingSession);
          logger.debug(`[RUNNER RUN] Resolved session awaiter for PID ${pid}`);
        }
@@ -383,17 +398,66 @@ export async function startRunner(): Promise<void> {
          stderrTail = appendTail(stderrTail, data);
        });

+        let spawnErrorBeforePidCheck: Error | null = null;
+        const captureSpawnErrorBeforePidCheck = (error: Error) => {
+          spawnErrorBeforePidCheck = error;
+        };
+        happyProcess.once('error', captureSpawnErrorBeforePidCheck);
+
        if (!happyProcess.pid) {
-          logger.debug('[RUNNER RUN] Failed to spawn process - no PID returned');
+          // Allow the async 'error' event to fire before we read it
+          await new Promise((resolve) => setImmediate(resolve));
+          const details = [`cwd=${spawnDirectory}`];
+          if (spawnErrorBeforePidCheck) {
+            details.push(formatSpawnError(spawnErrorBeforePidCheck));
+          }
+          const errorMessage = `Failed to spawn HAPI process - no PID returned (${details.join('; ')})`;
+          logger.debug('[RUNNER RUN] Failed to spawn process - no PID returned', spawnErrorBeforePidCheck ?? null);
+          reportSpawnOutcomeToHub?.({
+            type: 'error',
+            details: {
+              message: errorMessage
+            }
+          });
          await maybeCleanupWorktree('no-pid');
          return {
            type: 'error',
-            errorMessage: 'Failed to spawn HAPI process - no PID returned'
+            errorMessage
          };
        }
+        happyProcess.removeListener('error', captureSpawnErrorBeforePidCheck);

        const pid = happyProcess.pid;
        logger.debug(`[RUNNER RUN] Spawned process with PID ${pid}`);
+        let observedExitCode: number | null = null;
+        let observedExitSignal: NodeJS.Signals | null = null;
+        const buildWebhookFailureMessage = (reason: 'timeout' | 'exit-before-webhook' | 'process-error-before-webhook'): string => {
+          let message = '';
+          if (reason === 'exit-before-webhook') {
+            message = `Session process exited before webhook for PID ${pid}`;
+          } else if (reason === 'process-error-before-webhook') {
+            message = `Session process error before webhook for PID ${pid}`;
+          } else {
+            message = `Session webhook timeout for PID ${pid}`;
+          }
+
+          if (observedExitCode !== null || observedExitSignal) {
+            if (observedExitCode !== null) {
+              message += ` (exit code ${observedExitCode})`;
+            } else {
+              message += ` (signal ${observedExitSignal})`;
+            }
+          }
+
+          const trimmedTail = stderrTail.trim();
+          if (trimmedTail) {
+            const compactTail = trimmedTail.replace(/\s+/g, ' ');
+            const tailForMessage = compactTail.length > 800 ? compactTail.slice(-800) : compactTail;
+            message += `. stderr: ${tailForMessage}`;
+          }
+
+          return message;
+        };

        const trackedSession: TrackedSession = {
          startedBy: 'runner',
@@ -406,15 +470,29 @@ export async function startRunner(): Promise<void> {
        pidToTrackedSession.set(pid, trackedSession);

        happyProcess.on('exit', (code, signal) => {
+          observedExitCode = typeof code === 'number' ? code : null;
+          observedExitSignal = signal ?? null;
          logger.debug(`[RUNNER RUN] Child PID ${pid} exited with code ${code}, signal ${signal}`);
          if (code !== 0 || signal) {
            logStderrTail();
          }
+          const errorAwaiter = pidToErrorAwaiter.get(pid);
+          if (errorAwaiter) {
+            pidToErrorAwaiter.delete(pid);
+            pidToAwaiter.delete(pid);
+            errorAwaiter(buildWebhookFailureMessage('exit-before-webhook'));
+          }
          onChildExited(pid);
        });

        happyProcess.on('error', (error) => {
          logger.debug(`[RUNNER RUN] Child process error:`, error);
+          const errorAwaiter = pidToErrorAwaiter.get(pid);
+          if (errorAwaiter) {
+            pidToErrorAwaiter.delete(pid);
+            pidToAwaiter.delete(pid);
+            errorAwaiter(buildWebhookFailureMessage('process-error-before-webhook'));
+          }
          onChildExited(pid);
        });

@@ -425,11 +503,12 @@ export async function startRunner(): Promise<void> {
          // Set timeout for webhook
          const timeout = setTimeout(() => {
            pidToAwaiter.delete(pid);
+            pidToErrorAwaiter.delete(pid);
            logger.debug(`[RUNNER RUN] Session webhook timeout for PID ${pid}`);
            logStderrTail();
            resolve({
              type: 'error',
-              errorMessage: `Session webhook timeout for PID ${pid}`
+              errorMessage: buildWebhookFailureMessage('timeout')
            });
            // 15 second timeout - I have seen timeouts on 10 seconds
            // even though session was still created successfully in ~2 more seconds
@@ -438,21 +517,46 @@ export async function startRunner(): Promise<void> {
          // Register awaiter
          pidToAwaiter.set(pid, (completedSession) => {
            clearTimeout(timeout);
+            pidToErrorAwaiter.delete(pid);
            logger.debug(`[RUNNER RUN] Session ${completedSession.happySessionId} fully spawned with webhook`);
            resolve({
              type: 'success',
              sessionId: completedSession.happySessionId!
            });
          });
+          pidToErrorAwaiter.set(pid, (errorMessage) => {
+            clearTimeout(timeout);
+            resolve({
+              type: 'error',
+              errorMessage
+            });
+          });
        });
-        if (spawnResult.type !== 'success') {
+        if (spawnResult.type === 'error') {
+          reportSpawnOutcomeToHub?.({
+            type: 'error',
+            details: {
+              message: spawnResult.errorMessage,
+              pid,
+              exitCode: observedExitCode,
+              signal: observedExitSignal
+            }
+          });
          await maybeCleanupWorktree('spawn-error');
+        } else {
+          reportSpawnOutcomeToHub?.({ type: 'success' });
        }
        return spawnResult;
      } catch (error) {
        const errorMessage = error instanceof Error ? error.message : String(error);
        logger.debug('[RUNNER RUN] Failed to spawn session:', error);
        await maybeCleanupWorktree('exception');
+        reportSpawnOutcomeToHub?.({
+          type: 'error',
+          details: {
+            message: `Failed to spawn session: ${errorMessage}`
+          }
+        });
        return {
          type: 'error',
          errorMessage: `Failed to spawn session: ${errorMessage}`
@@ -500,6 +604,8 @@ export async function startRunner(): Promise<void> {
    const onChildExited = (pid: number) => {
      logger.debug(`[RUNNER RUN] Removing exited process PID ${pid} from tracking`);
      pidToTrackedSession.delete(pid);
+      pidToAwaiter.delete(pid);
+      pidToErrorAwaiter.delete(pid);
    };

    // Start control server
@@ -569,6 +675,44 @@ export async function startRunner(): Promise<void> {
    // Connect to server
    apiMachine.connect();

+    reportSpawnOutcomeToHub = (outcome) => {
+      void apiMachine.updateRunnerState((state: RunnerState | null) => {
+        const baseState: RunnerState = state
+          ? { ...state }
+          : { status: 'running' };
+
+        if (typeof baseState.pid !== 'number') {
+          baseState.pid = process.pid;
+        }
+        if (typeof baseState.httpPort !== 'number') {
+          baseState.httpPort = controlPort;
+        }
+        if (typeof baseState.startedAt !== 'number') {
+          baseState.startedAt = Date.now();
+        }
+
+        if (outcome.type === 'success') {
+          return {
+            ...baseState,
+            lastSpawnError: null
+          };
+        }
+
+        return {
+          ...baseState,
+          lastSpawnError: {
+            message: outcome.details.message,
+            pid: outcome.details.pid,
+            exitCode: outcome.details.exitCode ?? null,
+            signal: outcome.details.signal ?? null,
+            at: Date.now()
+          }
+        };
+      }).catch((error) => {
+        logger.debug('[RUNNER RUN] Failed to update runner state with spawn outcome', error);
+      });
+    };
+
    // Every 60 seconds:
    // 1. Prune stale sessions
    // 2. Check if runner needs update