Improve TaskRunProcess health detection so we don't try and reuse an unhealthy process

This was happening after the process was killed internally, like by an OOM error

Author: ericallam

packages/cli-v3/src/dev/taskRunProcessPool.ts

@@ -208,10 +208,18 @@ export class TaskRunProcessPool {
 
   private isProcessHealthy(process: TaskRunProcess): boolean {
     // Basic health checks - we can expand this later
-    return !process.isBeingKilled && process.pid !== undefined;
+    return process.isHealthy;
   }
 
   private async killProcess(process: TaskRunProcess): Promise<void> {
+    if (!process.isHealthy) {
+      logger.debug("[TaskRunProcessPool] Process is not healthy, skipping cleanup", {
+        processId: process.pid,
+      });
+
+      return;
+    }
+
     try {
       await process.cleanup(true);
     } catch (error) {

packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts

@@ -135,7 +135,7 @@ export class TaskRunProcessProvider {
       this.sendDebugLog("Not keeping TaskRunProcess alive, cleaning up", {
         executionCount: this.executionCount,
         maxExecutionCount: this.processKeepAliveMaxExecutionCount,
-        isHealthy: this.isProcessHealthy(process),
+        isHealthy: process.isHealthy,
       });
 
       // Cleanup the process completely
@@ -284,19 +284,12 @@ export class TaskRunProcessProvider {
     return (
       !!this.persistentProcess &&
       this.executionCount < this.processKeepAliveMaxExecutionCount &&
-      this.isProcessHealthy(this.persistentProcess)
+      this.persistentProcess.isHealthy
     );
   }
 
   private shouldKeepProcessAlive(process: TaskRunProcess): boolean {
-    return (
-      this.executionCount < this.processKeepAliveMaxExecutionCount && this.isProcessHealthy(process)
-    );
-  }
-
-  private isProcessHealthy(process: TaskRunProcess): boolean {
-    // Basic health check - TaskRunProcess will handle more detailed internal health checks
-    return !process.isBeingKilled && process.pid !== undefined;
+    return this.executionCount < this.processKeepAliveMaxExecutionCount && process.isHealthy;
   }
 
   private async cleanupProcess(taskRunProcess: TaskRunProcess): Promise<void> {

packages/cli-v3/src/executions/taskRunProcess.ts

@@ -442,10 +442,26 @@ export class TaskRunProcess {
     return this._isBeingKilled || this._child?.killed;
   }
 
+  get isBeingSuspended() {
+    return this._isBeingSuspended;
+  }
+
   get pid() {
     return this._childPid;
   }
 
+  get isHealthy() {
+    if (!this._child) {
+      return false;
+    }
+
+    if (this.isBeingKilled || this.isBeingSuspended) {
+      return false;
+    }
+
+    return this._child.connected;
+  }
+
   static parseExecuteError(error: unknown, dockerMode = true): TaskRunInternalError {
     if (error instanceof CancelledProcessError) {
       return {

references/hello-world/src/trigger/oom.ts

@@ -55,3 +55,33 @@ export const oomTask = task({
     }
   },
 });
+
+export const oomTask2 = task({
+  id: "oom-task-2",
+  machine: "micro",
+  run: async (payload: any, { ctx }) => {
+    await runMemoryLeakScenario();
+  },
+});
+
+async function runMemoryLeakScenario() {
+  console.log("🧠 Starting memory leak simulation");
+  const memoryHogs = [];
+  let iteration = 0;
+
+  while (iteration < 1000) {
+    // Allocate large chunks of memory
+    const bigArray = new Array(10000000).fill(`memory-leak-data-${iteration}`);
+    memoryHogs.push(bigArray);
+
+    await setTimeout(200);
+    iteration++;
+
+    const memUsage = process.memoryUsage();
+    console.log(
+      `🧠 Memory leak iteration ${iteration}, RSS: ${Math.round(memUsage.rss / 1024 / 1024)} MB`
+    );
+  }
+
+  console.log("🧠 Memory leak scenario completed");
+}