fix: Implement production-ready shutdown timeout system

GhostTypes · GhostTypes · commit 195e85a8a33f · 2026-01-28T15:14:19.000-05:00
Fixes indefinite hangs during graceful shutdown (Ctrl+C) by implementing a
three-tier timeout strategy that ensures the application always exits within
10 seconds, even when printers are unresponsive or HTTP connections are stuck.

**Changes:**

- **New timeout utility** (src/utils/ShutdownTimeout.ts):
  - TimeoutError class for timeout failures
  - withTimeout() wrapper for promises with timeout enforcement
  - createHardDeadline() for absolute maximum shutdown time

- **ConnectionFlowManager.disconnectContext()**:
  - Added 5s timeout wrapper (configurable)
  - On timeout: forces cleanup (removes context, marks disconnected)
  - Backward compatible with existing callers

- **WebUIManager.stop()**:
  - Added 3s timeout wrapper (configurable)
  - On timeout: calls closeAllConnections() to force-close stuck HTTP connections
  - Added isStopping guard flag to prevent concurrent stop calls
  - Added timing information logging

- **Main shutdown logic** (src/index.ts):
  - Replaced sequential disconnect loop with parallel Promise.allSettled()
  - All printers disconnect concurrently instead of one-by-one
  - Added 10s hard deadline that calls process.exit(1) if exceeded
  - Added step-by-step logging with clear progress indicators
  - Preserved second Ctrl+C force-exit as safety net

**Benefits:**
- Single printer: &lt; 5s shutdown
- Multiple printers: &lt; 10s (parallelized)
- Hard deadline: Always exits within 10s maximum
- Second Ctrl+C: Immediate force exit
- Clear logging: Step-by-step progress visible

**Timeout hierarchy:**
```
HARD DEADLINE (10s) ─────────────────────────────────┐
  ├── Stop polling (immediate)                        │
  ├── Parallel disconnects (5s each, concurrent)    ─┤
  └── WebUI server stop (3s, force close if timeout)─┘
```

All changes are backward compatible - optional timeout parameters have
sensible defaults and existing callers work without modification.
diff --git a/src/index.ts b/src/index.ts
@@ -28,6 +28,7 @@ import { getRtspStreamService } from './services/RtspStreamService';
 import { initializeSpoolmanIntegrationService } from './services/SpoolmanIntegrationService';
 import { getSavedPrinterService } from './services/SavedPrinterService';
 import { parseHeadlessArguments, validateHeadlessConfig } from './utils/HeadlessArguments';
+import { withTimeout, createHardDeadline } from './utils/ShutdownTimeout';
 import type { HeadlessConfig, PrinterSpec } from './utils/HeadlessArguments';
 import type { PrinterDetails, PrinterClientType } from './types/printer';
 import { initializeDataDirectory } from './utils/setup';
@@ -46,6 +47,25 @@ const _cameraProxyService = getCameraProxyService();
 
 let connectedContexts: string[] = [];
 let isInitialized = false;
+let isShuttingDown = false;
+
+/**
+ * Shutdown timeout configuration
+ *
+ * Layered timeout strategy:
+ * 1. Per-operation timeouts (disconnect: 5s, webui: 3s)
+ * 2. Hard deadline (10s absolute maximum)
+ *
+ * This prevents hangs from unresponsive printers or stuck HTTP connections
+ */
+const SHUTDOWN_CONFIG = {
+  /** Hard deadline - forces process.exit(1) if exceeded */
+  HARD_DEADLINE_MS: 10000,
+  /** Per-printer disconnect timeout (parallelized, so 3 printers = ~5s total) */
+  DISCONNECT_TIMEOUT_MS: 5000,
+  /** WebUI server graceful close timeout */
+  WEBUI_STOP_TIMEOUT_MS: 3000
+} as const;
 
 /**
  * Apply configuration overrides from CLI arguments
@@ -247,7 +267,12 @@ async function initializeCameraProxies(): Promise<void> {
 function setupSignalHandlers(): void {
   // Handle Ctrl+C (works on all platforms including Windows)
   process.on('SIGINT', () => {
+    if (isShuttingDown) {
+      console.log('\n[Shutdown] Force exit (second Ctrl+C)');
+      process.exit(1);
+    }
     console.log('\n[Shutdown] Received SIGINT signal (Ctrl+C)');
+    isShuttingDown = true;
     void shutdown().then(() => {
       process.exit(0);
     }).catch((error) => {
@@ -283,36 +308,75 @@ function setupSignalHandlers(): void {
 
 /**
  * Gracefully shutdown the application
+ *
+ * Implements a three-tier timeout strategy:
+ * 1. Hard deadline (10s) - ultimate fallback with process.exit(1)
+ * 2. Parallel disconnects (5s each, concurrent) - one hung printer doesn't block others
+ * 3. WebUI stop (3s) - force-close connections if timeout
+ *
+ * This ensures the application always exits within 10 seconds, even if
+ * printers are unresponsive or HTTP connections are stuck.
  */
 async function shutdown(): Promise<void> {
   if (!isInitialized) {
     return;
   }
 
-  console.log('[Shutdown] Stopping services...');
+  const startTime = Date.now();
+  console.log('[Shutdown] Starting graceful shutdown...');
+
+  // Set hard deadline - ultimate fallback to prevent indefinite hangs
+  const hardDeadline = createHardDeadline(SHUTDOWN_CONFIG.HARD_DEADLINE_MS);
 
   try {
-    // Stop all polling
+    // Step 1: Stop polling (immediate)
+    console.log('[Shutdown] Step 1/4: Stopping polling...');
     pollingCoordinator.stopAllPolling();
-    console.log('[Shutdown] Polling stopped');
+    console.log('[Shutdown] ✓ Polling stopped');
 
-    // Disconnect all printers
-    for (const contextId of connectedContexts) {
-      try {
-        await connectionManager.disconnectContext(contextId);
-        console.log(`[Shutdown] Disconnected context: ${contextId}`);
-      } catch (error) {
-        console.error(`[Shutdown] Error disconnecting context ${contextId}:`, error);
-      }
+    // Step 2: Parallel disconnects (all printers disconnect concurrently)
+    console.log(`[Shutdown] Step 2/4: Disconnecting ${connectedContexts.length} context(s)...`);
+    if (connectedContexts.length > 0) {
+      const results = await Promise.allSettled(
+        connectedContexts.map(contextId =>
+          withTimeout(
+            connectionManager.disconnectContext(contextId),
+            {
+              timeoutMs: SHUTDOWN_CONFIG.DISCONNECT_TIMEOUT_MS,
+              operation: `disconnectContext(${contextId})`
+            }
+          )
+        )
+      );
+
+      const succeeded = results.filter(r => r.status === 'fulfilled').length;
+      const failed = results.filter(r => r.status === 'rejected').length;
+
+      console.log(`[Shutdown] ✓ Disconnect: ${succeeded} succeeded, ${failed} failed`);
+
+      // Log individual failures for debugging
+      results.forEach((result, index) => {
+        if (result.status === 'rejected') {
+          console.warn(`[Shutdown] Context ${connectedContexts[index]} failed:`, result.reason);
+        }
+      });
+    } else {
+      console.log('[Shutdown] ✓ No contexts to disconnect');
     }
 
-    // Stop WebUI
-    await webUIManager.stop();
-    console.log('[Shutdown] WebUI server stopped');
+    // Step 3: Stop WebUI (with timeout and force-close fallback)
+    console.log('[Shutdown] Step 3/4: Stopping WebUI...');
+    await webUIManager.stop(SHUTDOWN_CONFIG.WEBUI_STOP_TIMEOUT_MS);
+    console.log('[Shutdown] ✓ WebUI stopped');
+
+    // Step 4: Complete (cancel hard deadline)
+    clearTimeout(hardDeadline);
+    const duration = Date.now() - startTime;
+    console.log(`[Shutdown] ✓ Complete (${duration}ms)`);
 
-    console.log('[Shutdown] Graceful shutdown complete');
   } catch (error) {
-    console.error('[Shutdown] Error during shutdown:', error);
+    console.error('[Shutdown] Error:', error);
+    // Hard deadline will still fire if we exceed max time
   }
 }
 
diff --git a/src/managers/ConnectionFlowManager.ts b/src/managers/ConnectionFlowManager.ts
@@ -33,6 +33,7 @@ import { getLoadingManager } from './LoadingManager';
 import { getPrinterBackendManager } from './PrinterBackendManager';
 import { getPrinterContextManager } from './PrinterContextManager';
 import { getPrinterDiscoveryService } from '../services/PrinterDiscoveryService';
+import { withTimeout, TimeoutError } from '../utils/ShutdownTimeout';
 import { getThumbnailRequestQueue } from '../services/ThumbnailRequestQueue';
 import { getSavedPrinterService } from '../services/SavedPrinterService';
 import { getAutoConnectService } from '../services/AutoConnectService';
@@ -451,7 +452,7 @@ export class ConnectionFlowManager extends EventEmitter {
   }
 
   /** Disconnect a specific printer context with proper cleanup */
-  public async disconnectContext(contextId: string): Promise<void> {
+  public async disconnectContext(contextId: string, timeoutMs = 5000): Promise<void> {
     const context = this.contextManager.getContext(contextId);
     if (!context) {
       console.warn(`Cannot disconnect - context ${contextId} not found`);
@@ -461,37 +462,48 @@ export class ConnectionFlowManager extends EventEmitter {
     const currentDetails = context.printerDetails;
 
     try {
-      console.log(`Starting disconnect sequence for context ${contextId}...`);
-
-      // Stop polling first
-      this.emit('pre-disconnect', contextId);
-      await new Promise(resolve => setTimeout(resolve, 100));
+      await withTimeout(
+        (async () => {
+          console.log(`Starting disconnect sequence for context ${contextId}...`);
+
+          // Stop polling first
+          this.emit('pre-disconnect', contextId);
+          await new Promise(resolve => setTimeout(resolve, 100));
+
+          // Get clients for disposal from connection state
+          const primaryClient = this.connectionStateManager.getPrimaryClient(contextId);
+          const secondaryClient = this.connectionStateManager.getSecondaryClient(contextId);
+
+          // Dispose backend for this context
+          await this.backendManager.disposeContext(contextId);
+
+          // Dispose clients through connection service (handles logout)
+          await this.connectionService.disposeClients(
+            primaryClient,
+            secondaryClient,
+            currentDetails?.ClientType
+          );
 
-      // Get clients for disposal from connection state
-      const primaryClient = this.connectionStateManager.getPrimaryClient(contextId);
-      const secondaryClient = this.connectionStateManager.getSecondaryClient(contextId);
+          // Update connection state
+          this.connectionStateManager.setDisconnected(contextId);
 
-      // Dispose backend for this context
-      await this.backendManager.disposeContext(contextId);
+          // Remove context from manager
+          this.contextManager.removeContext(contextId);
 
-      // Dispose clients through connection service (handles logout)
-      await this.connectionService.disposeClients(
-        primaryClient,
-        secondaryClient,
-        currentDetails?.ClientType
+          // Emit disconnected event
+          this.emit('disconnected', currentDetails?.Name);
+        })(),
+        { timeoutMs, operation: `disconnectContext(${contextId})` }
       );
-
-      // Update connection state
-      this.connectionStateManager.setDisconnected(contextId);
-
-      // Remove context from manager
-      this.contextManager.removeContext(contextId);
-
-      // Emit disconnected event
-      this.emit('disconnected', currentDetails?.Name);
-
     } catch (error) {
-      console.error(`Error during disconnect for context ${contextId}:`, error);
+      if (error instanceof TimeoutError) {
+        console.error(`[Shutdown] Context ${contextId} timed out, forcing cleanup`);
+        // Force cleanup on timeout
+        this.contextManager.removeContext(contextId);
+        this.connectionStateManager.setDisconnected(contextId);
+      } else {
+        console.error(`Error during disconnect for context ${contextId}:`, error);
+      }
     }
   }
 
diff --git a/src/utils/ShutdownTimeout.ts b/src/utils/ShutdownTimeout.ts
@@ -0,0 +1,106 @@
+/**
+ * @fileoverview Timeout utilities for graceful shutdown operations.
+ *
+ * Provides timeout wrappers and deadline enforcement for async operations
+ * during application shutdown. Prevents indefinite hangs from unresponsive
+ * printers or stuck HTTP connections.
+ *
+ * Key exports:
+ * - TimeoutError: Custom error class for timeout failures
+ * - withTimeout(): Promise wrapper with timeout enforcement
+ * - createHardDeadline(): Sets absolute maximum shutdown time with process.exit(1)
+ */
+
+/**
+ * Custom error thrown when an operation exceeds its timeout
+ */
+export class TimeoutError extends Error {
+  constructor(operation: string, timeoutMs: number) {
+    super(`Timeout: ${operation} exceeded ${timeoutMs}ms`);
+    this.name = 'TimeoutError';
+  }
+}
+
+/**
+ * Wrap a promise with timeout enforcement
+ *
+ * Races the provided promise against a timeout. If the timeout fires first,
+ * the promise is rejected with TimeoutError. Properly cleans up timeout
+ * handle to prevent memory leaks.
+ *
+ * @template T - Promise result type
+ * @param promise - The promise to wrap with timeout
+ * @param options - Timeout configuration
+ * @returns Promise that rejects on timeout
+ *
+ * @example
+ * ```typescript
+ * await withTimeout(
+ *   disconnectContext(contextId),
+ *   { timeoutMs: 5000, operation: 'disconnectContext' }
+ * );
+ * ```
+ */
+export async function withTimeout<T>(
+  promise: Promise<T>,
+  options: { timeoutMs: number; operation: string; silent?: boolean }
+): Promise<T> {
+  const { timeoutMs, operation, silent = false } = options;
+
+  let timeoutHandle: NodeJS.Timeout | undefined;
+
+  const timeoutPromise = new Promise<T>((_, reject) => {
+    timeoutHandle = setTimeout(() => {
+      if (!silent) {
+        console.warn(`[Shutdown] Timeout: ${operation} (${timeoutMs}ms)`);
+      }
+      reject(new TimeoutError(operation, timeoutMs));
+    }, timeoutMs);
+  });
+
+  try {
+    const result = await Promise.race([promise, timeoutPromise]);
+
+    // Clear timeout if promise won the race
+    if (timeoutHandle) {
+      clearTimeout(timeoutHandle);
+    }
+
+    return result;
+  } catch (error) {
+    // Ensure timeout is cleared even on error
+    if (timeoutHandle) {
+      clearTimeout(timeoutHandle);
+    }
+    throw error;
+  }
+}
+
+/**
+ * Create a hard deadline that forces process termination
+ *
+ * Sets a timeout that calls process.exit(1) when elapsed. This is the
+ * ultimate fallback to prevent the application from hanging indefinitely
+ * during shutdown. Returns the timeout handle so the deadline can be
+ * cleared if shutdown completes successfully.
+ *
+ * @param timeoutMs - Deadline duration in milliseconds
+ * @returns NodeJS.Timeout handle for deadline cancellation
+ *
+ * @example
+ * ```typescript
+ * const deadline = createHardDeadline(10000);
+ * try {
+ *   await shutdown();
+ *   clearTimeout(deadline); // Shutdown succeeded, cancel deadline
+ * } catch (error) {
+ *   // Error logged, deadline will fire if exceeded
+ * }
+ * ```
+ */
+export function createHardDeadline(timeoutMs: number): NodeJS.Timeout {
+  return setTimeout(() => {
+    console.error(`[Shutdown] HARD DEADLINE (${timeoutMs}ms) exceeded - forcing exit`);
+    process.exit(1);
+  }, timeoutMs);
+}
diff --git a/src/webui/server/WebUIManager.ts b/src/webui/server/WebUIManager.ts
