fix(@probitas/runner): abort all scenarios when maxFailures is reached

Previously, when maxFailures was reached, the Runner would throw an error,
leaving remaining scenarios unrecorded. This caused incomplete test results
and prevented proper cleanup.

Now, the Runner:
- Calls controller.abort() with Skip reason when maxFailures is reached
- Aborts in-progress scenarios via signal propagation (status: "skipped")
- Creates skip results for unexecuted scenarios (status: "skipped")
- Preserves signal.reason from external aborts (e.g., TimeoutError)

This ensures all scenarios are properly recorded and reporter events are
emitted consistently, providing complete test results even when stopping
early due to maxFailures.

Author: lambdalisue

packages/probitas-runner/runner.ts

@@ -11,6 +11,7 @@ import { toScenarioMetadata } from "./metadata.ts";
 import { timeit } from "./utils/timeit.ts";
 import { mergeSignals } from "./utils/signal.ts";
 import { ScenarioTimeoutError } from "./errors.ts";
+import { Skip } from "./skip.ts";
 
 /**
  * Top-level test runner that orchestrates execution of multiple scenarios.
@@ -69,7 +70,6 @@ export class Runner {
 
     // Create abort controller for outer context
     const controller = new AbortController();
-    const { signal } = controller;
     options?.signal?.addEventListener("abort", () => {
       // Pass the reason from external signal to internal controller
       controller.abort(options.signal?.reason);
@@ -88,7 +88,7 @@ export class Runner {
         maxConcurrency,
         maxFailures,
         timeout,
-        signal,
+        controller,
         options?.stepOptions,
       )
     );
@@ -145,20 +145,43 @@ export class Runner {
     maxConcurrency: number,
     maxFailures: number,
     timeout: number,
-    signal?: AbortSignal,
+    controller: AbortController,
     stepOptions?: StepOptions,
   ): Promise<void> {
     // Parallel execution with concurrency control
     // maxConcurrency=1 means sequential execution
     const concurrency = maxConcurrency || scenarios.length;
     const scenarioRunner = new ScenarioRunner(this.reporter, stepOptions);
+    const { signal } = controller;
 
     let failureCount = 0;
+
     for (const batch of chunk(scenarios, concurrency)) {
-      signal?.throwIfAborted();
+      // Don't throw - just skip remaining batches if aborted
+      if (signal.aborted) {
+        break;
+      }
+
       await Promise.all(
         batch.map(async (scenario: ScenarioDefinition) => {
-          signal?.throwIfAborted();
+          // Check if already aborted (by maxFailures or external signal)
+          if (signal.aborted) {
+            const skipResult: ScenarioResult = {
+              status: "skipped",
+              metadata: toScenarioMetadata(scenario),
+              duration: 0,
+              steps: [],
+              error: signal.reason ??
+                new Skip("Skipped due to previous failures"),
+            };
+            scenarioResults.push(skipResult);
+            await this.reporter.onScenarioStart?.(skipResult.metadata);
+            await this.reporter.onScenarioEnd?.(
+              skipResult.metadata,
+              skipResult,
+            );
+            return;
+          }
 
           // Execute scenario with optional timeout
           // Priority: scenario timeout > RunOptions timeout
@@ -173,14 +196,35 @@ export class Runner {
             : await scenarioRunner.run(scenario, { signal });
 
           scenarioResults.push(scenarioResult);
+
+          // Check if we've reached maxFailures - abort all remaining scenarios
           if (scenarioResult.status === "failed") {
             failureCount++;
             if (maxFailures !== 0 && failureCount >= maxFailures) {
-              throw scenarioResult.error;
+              controller.abort(new Skip("Skipped due to previous failures"));
             }
           }
         }),
       );
     }
+
+    // Add skip results for any remaining scenarios that weren't executed
+    // This handles the case where we broke out of the batch loop early
+    const executedCount = scenarioResults.length;
+    if (executedCount < scenarios.length) {
+      for (let i = executedCount; i < scenarios.length; i++) {
+        const scenario = scenarios[i];
+        const skipResult: ScenarioResult = {
+          status: "skipped",
+          metadata: toScenarioMetadata(scenario),
+          duration: 0,
+          steps: [],
+          error: signal.reason ?? new Skip("Skipped due to previous failures"),
+        };
+        scenarioResults.push(skipResult);
+        await this.reporter.onScenarioStart?.(skipResult.metadata);
+        await this.reporter.onScenarioEnd?.(skipResult.metadata, skipResult);
+      }
+    }
   }
 }

packages/probitas-runner/runner_test.ts

@@ -141,7 +141,7 @@ Deno.test("Runner run runs multiple scenarios (skip)", async () => {
   });
 });
 
-Deno.test("Runner run runs multiple scenarios (failed)", async () => {
+Deno.test("Runner run runs multiple scenarios (failed) - aborts on maxFailures", async () => {
   using signal = createScopedSignal();
   const runner = new Runner(reporter);
   const scenarios = [
@@ -150,8 +150,8 @@ Deno.test("Runner run runs multiple scenarios (failed)", async () => {
       steps: [
         createTestStep({
           name: "Step 1",
-          fn: async () => {
-            await delay(0, { signal });
+          fn: async (ctx) => {
+            await delay(0, { signal: ctx.signal });
             return "1";
           },
         }),
@@ -162,8 +162,8 @@ Deno.test("Runner run runs multiple scenarios (failed)", async () => {
       steps: [
         createTestStep({
           name: "Step 1",
-          fn: async () => {
-            await delay(50, { signal });
+          fn: async (ctx) => {
+            await delay(50, { signal: ctx.signal });
             throw "2";
           },
         }),
@@ -174,8 +174,8 @@ Deno.test("Runner run runs multiple scenarios (failed)", async () => {
       steps: [
         createTestStep({
           name: "Step 1",
-          fn: async () => {
-            await delay(100, { signal });
+          fn: async (ctx) => {
+            await delay(100, { signal: ctx.signal });
             return "3";
           },
         }),
@@ -187,12 +187,219 @@ Deno.test("Runner run runs multiple scenarios (failed)", async () => {
     signal,
     maxFailures: 1,
   });
+
+  // When maxFailures is reached, all scenarios (running or pending) should be aborted
+  // Scenario 1 passes (fastest, 0ms delay)
+  // Scenario 2 fails (50ms delay, triggers maxFailures)
+  // Scenario 3 should be aborted/skipped (100ms delay, still running when maxFailures reached)
   expect(summary).toMatchObject({
     total: 3,
     passed: 1,
-    skipped: 1,
+    skipped: 1, // Scenario 3 aborted
+    failed: 1, // Scenario 2
+  });
+});
+
+Deno.test("Runner run with maxFailures skips remaining scenarios (sequential)", async () => {
+  const runner = new Runner(reporter);
+  const executionOrder: string[] = [];
+  const scenarios = [
+    createTestScenario({
+      name: "Scenario 1",
+      steps: [
+        createTestStep({
+          name: "Step 1",
+          fn: () => {
+            executionOrder.push("Scenario 1");
+            return "1";
+          },
+        }),
+      ],
+    }),
+    createTestScenario({
+      name: "Scenario 2",
+      steps: [
+        createTestStep({
+          name: "Step 1",
+          fn: () => {
+            executionOrder.push("Scenario 2");
+            throw new Error("Scenario 2 failed");
+          },
+        }),
+      ],
+    }),
+    createTestScenario({
+      name: "Scenario 3",
+      steps: [
+        createTestStep({
+          name: "Step 1",
+          fn: () => {
+            executionOrder.push("Scenario 3");
+            return "3";
+          },
+        }),
+      ],
+    }),
+    createTestScenario({
+      name: "Scenario 4",
+      steps: [
+        createTestStep({
+          name: "Step 1",
+          fn: () => {
+            executionOrder.push("Scenario 4");
+            return "4";
+          },
+        }),
+      ],
+    }),
+  ];
+
+  const summary = await runner.run(scenarios, {
+    maxConcurrency: 1, // Sequential execution
+    maxFailures: 1,
+  });
+
+  // Scenario 1 should pass, Scenario 2 should fail, Scenario 3 and 4 should be skipped
+  expect(summary).toMatchObject({
+    total: 4,
+    passed: 1,
     failed: 1,
+    skipped: 2,
+  });
+
+  // Only Scenario 1 and 2 should have been executed
+  expect(executionOrder).toEqual(["Scenario 1", "Scenario 2"]);
+
+  // Check scenario results
+  const results = summary.scenarios;
+  expect(results.length).toBe(4);
+  expect(results[0].status).toBe("passed");
+  expect(results[0].metadata.name).toBe("Scenario 1");
+  expect(results[1].status).toBe("failed");
+  expect(results[1].metadata.name).toBe("Scenario 2");
+  expect(results[2].status).toBe("skipped");
+  expect(results[2].metadata.name).toBe("Scenario 3");
+  expect(results[3].status).toBe("skipped");
+  expect(results[3].metadata.name).toBe("Scenario 4");
+});
+
+Deno.test("Runner run with maxFailures aborts in-progress scenarios (parallel)", async () => {
+  using signal = createScopedSignal();
+  const runner = new Runner(reporter);
+  const executionOrder: string[] = [];
+  const scenarios = [
+    createTestScenario({
+      name: "Scenario 1",
+      steps: [
+        createTestStep({
+          name: "Step 1",
+          fn: async (ctx) => {
+            executionOrder.push("Scenario 1 start");
+            try {
+              await delay(100, { signal: ctx.signal }); // Slow - will be aborted
+              executionOrder.push("Scenario 1 end");
+              return "1";
+            } catch (err) {
+              executionOrder.push("Scenario 1 aborted");
+              throw err;
+            }
+          },
+        }),
+      ],
+    }),
+    createTestScenario({
+      name: "Scenario 2",
+      steps: [
+        createTestStep({
+          name: "Step 1",
+          fn: async (ctx) => {
+            executionOrder.push("Scenario 2 start");
+            await delay(10, { signal: ctx.signal }); // Fast
+            executionOrder.push("Scenario 2 end");
+            throw new Error("Scenario 2 failed");
+          },
+        }),
+      ],
+    }),
+    createTestScenario({
+      name: "Scenario 3",
+      steps: [
+        createTestStep({
+          name: "Step 1",
+          fn: async (ctx) => {
+            executionOrder.push("Scenario 3 start");
+            try {
+              await delay(100, { signal: ctx.signal }); // Slow - will be aborted
+              executionOrder.push("Scenario 3 end");
+              return "3";
+            } catch (err) {
+              executionOrder.push("Scenario 3 aborted");
+              throw err;
+            }
+          },
+        }),
+      ],
+    }),
+    createTestScenario({
+      name: "Scenario 4",
+      steps: [
+        createTestStep({
+          name: "Step 1",
+          fn: () => {
+            executionOrder.push("Scenario 4 start");
+            return "4";
+          },
+        }),
+      ],
+    }),
+  ];
+
+  const summary = await runner.run(scenarios, {
+    signal,
+    maxConcurrency: 3, // Scenarios 1-3 run in parallel, 4 is in next batch
+    maxFailures: 1,
   });
+
+  // Scenario 1, 2, 3 all start in parallel
+  // Scenario 2 finishes first and fails, triggers maxFailures
+  // Scenarios 1, 3 are aborted immediately (signal.abort called)
+  // Scenario 4 is in the next batch, never starts
+  expect(summary).toMatchObject({
+    total: 4,
+    passed: 0,
+    failed: 1, // Scenario 2
+    skipped: 3, // Scenarios 1, 3, 4 all skipped
+  });
+
+  // Verify execution order
+  expect(executionOrder).toContain("Scenario 1 start");
+  expect(executionOrder).toContain("Scenario 1 aborted"); // Aborted, not completed
+  expect(executionOrder).not.toContain("Scenario 1 end");
+
+  expect(executionOrder).toContain("Scenario 2 start");
+  expect(executionOrder).toContain("Scenario 2 end");
+
+  expect(executionOrder).toContain("Scenario 3 start");
+  expect(executionOrder).toContain("Scenario 3 aborted"); // Aborted, not completed
+  expect(executionOrder).not.toContain("Scenario 3 end");
+
+  // Scenario 4 should never start
+  expect(executionOrder).not.toContain("Scenario 4 start");
+
+  // Check scenario results
+  const results = summary.scenarios;
+  expect(results.length).toBe(4);
+
+  // Find scenarios by name since order may vary in parallel execution
+  const scenario1 = results.find((r) => r.metadata.name === "Scenario 1")!;
+  const scenario2 = results.find((r) => r.metadata.name === "Scenario 2")!;
+  const scenario3 = results.find((r) => r.metadata.name === "Scenario 3")!;
+  const scenario4 = results.find((r) => r.metadata.name === "Scenario 4")!;
+
+  expect(scenario1.status).toBe("skipped");
+  expect(scenario2.status).toBe("failed");
+  expect(scenario3.status).toBe("skipped");
+  expect(scenario4.status).toBe("skipped");
 });
 
 Deno.test({