feat(cli): categorize execution results with resultType for better failure tracking

packages/cli/src/batch-runner.ts

@@ -355,7 +355,28 @@ class BatchRunner {
 
     for (const context of executedContexts) {
       const { file, player, duration } = context;
-      const success = player.status !== 'error';
+      // Determine result type based on player and task statuses
+      const hasFailedTasks =
+        player.taskStatusList?.some((task) => task.status === 'error') ?? false;
+      const hasPlayerError = player.status === 'error';
+
+      let success: boolean;
+      let resultType: 'success' | 'failed' | 'partialFailed';
+
+      if (hasPlayerError) {
+        // Complete failure - player itself failed
+        success = false;
+        resultType = 'failed';
+      } else if (hasFailedTasks) {
+        // Partial failure - some tasks failed but execution continued (continueOnError)
+        success = false;
+        resultType = 'partialFailed';
+      } else {
+        // Success - all tasks completed successfully
+        success = true;
+        resultType = 'success';
+      }
+
       let reportFile: string | undefined;
 
       if (player.reportFile) {
@@ -375,9 +396,11 @@ class BatchRunner {
         output: outputPath,
         report: reportFile,
         duration,
+        resultType,
         error:
           player.errorInSetup?.message ||
-          (player.status === 'error' ? 'Execution failed' : undefined),
+          (hasPlayerError ? 'Execution failed' : undefined) ||
+          (hasFailedTasks ? 'Some tasks failed' : undefined),
       });
     }
 
@@ -389,6 +412,7 @@ class BatchRunner {
         output: undefined,
         report: undefined,
         duration: 0,
+        resultType: 'notExecuted',
         error: 'Not executed (previous task failed)',
       });
     }
@@ -435,8 +459,15 @@ class BatchRunner {
       const indexData = {
         summary: {
           total: this.results.length,
-          successful: this.results.filter((r) => r.success).length,
-          failed: this.results.filter((r) => !r.success).length,
+          successful: this.results.filter((r) => r.resultType === 'success')
+            .length,
+          failed: this.results.filter((r) => r.resultType === 'failed').length,
+          partialFailed: this.results.filter(
+            (r) => r.resultType === 'partialFailed',
+          ).length,
+          notExecuted: this.results.filter(
+            (r) => r.resultType === 'notExecuted',
+          ).length,
           totalDuration: this.results.reduce(
             (sum, r) => sum + (r.duration || 0),
             0,
@@ -446,6 +477,7 @@ class BatchRunner {
         results: this.results.map((result) => ({
           script: relative(outputDir, result.file),
           success: result.success,
+          resultType: result.resultType,
           output: result.output
             ? (() => {
                 const relativePath = relative(outputDir, result.output);
@@ -473,17 +505,26 @@ class BatchRunner {
     total: number;
     successful: number;
     failed: number;
+    partialFailed: number;
     notExecuted: number;
     totalDuration: number;
   } {
-    const successful = this.results.filter((r) => r.success).length;
-    const notExecuted = this.results.filter((r) => !r.executed).length;
-    const failed = this.results.filter((r) => r.executed && !r.success).length;
+    const successful = this.results.filter(
+      (r) => r.resultType === 'success',
+    ).length;
+    const failed = this.results.filter((r) => r.resultType === 'failed').length;
+    const partialFailed = this.results.filter(
+      (r) => r.resultType === 'partialFailed',
+    ).length;
+    const notExecuted = this.results.filter(
+      (r) => r.resultType === 'notExecuted',
+    ).length;
 
     return {
       total: this.results.length,
       successful,
       failed,
+      partialFailed,
       notExecuted,
       totalDuration: this.results.reduce(
         (sum, r) => sum + (r.duration || 0),
@@ -494,16 +535,26 @@ class BatchRunner {
 
   getFailedFiles(): string[] {
     return this.results
-      .filter((r) => r.executed && !r.success)
+      .filter((r) => r.resultType === 'failed')
+      .map((r) => r.file);
+  }
+
+  getPartialFailedFiles(): string[] {
+    return this.results
+      .filter((r) => r.resultType === 'partialFailed')
       .map((r) => r.file);
   }
 
   getNotExecutedFiles(): string[] {
-    return this.results.filter((r) => !r.executed).map((r) => r.file);
+    return this.results
+      .filter((r) => r.resultType === 'notExecuted')
+      .map((r) => r.file);
   }
 
   getSuccessfulFiles(): string[] {
-    return this.results.filter((r) => r.success).map((r) => r.file);
+    return this.results
+      .filter((r) => r.resultType === 'success')
+      .map((r) => r.file);
   }
 
   getResults(): MidsceneYamlConfigResult[] {
@@ -512,12 +563,16 @@ class BatchRunner {
 
   printExecutionSummary(): boolean {
     const summary = this.getExecutionSummary();
-    const success = summary.failed === 0 && summary.notExecuted === 0;
+    const success =
+      summary.failed === 0 &&
+      summary.partialFailed === 0 &&
+      summary.notExecuted === 0;
 
     console.log('\n📊 Execution Summary:');
     console.log(`   Total files: ${summary.total}`);
     console.log(`   Successful: ${summary.successful}`);
     console.log(`   Failed: ${summary.failed}`);
+    console.log(`   Partial failed: ${summary.partialFailed}`);
     console.log(`   Not executed: ${summary.notExecuted}`);
     console.log(`   Duration: ${(summary.totalDuration / 1000).toFixed(2)}s`);
     console.log(`   Summary: ${this.getSummaryAbsolutePath()}`);
@@ -536,6 +591,15 @@ class BatchRunner {
       });
     }
 
+    if (summary.partialFailed > 0) {
+      console.log(
+        '\n⚠️  Partial failed files (some tasks failed with continueOnError)',
+      );
+      this.getPartialFailedFiles().forEach((file) => {
+        console.log(`   ${file}`);
+      });
+    }
+
     if (summary.notExecuted > 0) {
       console.log('\n⏸️ Not executed files');
       this.getNotExecutedFiles().forEach((file) => {

packages/cli/tests/unit-test/batch-runner.test.ts

@@ -369,6 +369,69 @@ describe('BatchRunner', () => {
       );
       consoleSpy.mockRestore();
     });
+
+    test('continueOnError: failed tasks should be counted as failed files', async () => {
+      const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+
+      // Create a mock player that simulates continueOnError behavior:
+      // - player.status = 'done' (execution completed)
+      // - but taskStatusList contains failed tasks
+      const createMockPlayerWithFailedTasks = (
+        fileName: string,
+      ): ScriptPlayer<MidsceneYamlScriptEnv> => {
+        const isFile1 = fileName === 'file1.yml';
+        const mockPlayer = {
+          status: 'done' as ScriptPlayerStatusValue, // Always 'done' with continueOnError
+          output: '/test/output/file.json',
+          reportFile: '/test/report.html',
+          result: { test: 'data' },
+          errorInSetup: null,
+          taskStatusList: isFile1
+            ? [
+                {
+                  status: 'error',
+                  error: new Error(
+                    'Assertion failed: this is not a search engine',
+                  ),
+                },
+                { status: 'done' },
+              ]
+            : [{ status: 'done' }],
+          run: vi.fn().mockImplementation(async () => {
+            return undefined;
+          }),
+          script: mockYamlScript,
+          setupAgent: vi.fn(),
+          unnamedResultIndex: 0,
+          pageAgent: null,
+          currentTaskIndex: undefined,
+          agentStatusTip: '',
+        };
+        return mockPlayer as unknown as ScriptPlayer<MidsceneYamlScriptEnv>;
+      };
+
+      vi.mocked(createYamlPlayer).mockImplementation(async (file) =>
+        createMockPlayerWithFailedTasks(file),
+      );
+
+      const config = { ...mockBatchConfig, continueOnError: true };
+      const executor = new BatchRunner(config);
+      await executor.run();
+
+      const summary = executor.getExecutionSummary();
+      const success = executor.printExecutionSummary();
+
+      // Files with failed tasks and continueOnError should be counted as partialFailed
+      expect(summary.partialFailed).toBe(1);
+      expect(summary.failed).toBe(0); // No complete failures
+      expect(summary.successful).toBe(2); // The other two files succeeded
+      expect(success).toBe(false); // Overall should still be false due to partial failure
+      expect(consoleSpy).toHaveBeenCalledWith(
+        expect.stringContaining('⚠️  Partial failed files'),
+      );
+
+      consoleSpy.mockRestore();
+    });
   });
 
   describe('BatchRunner output file existence check', () => {

packages/core/src/yaml.ts

@@ -235,4 +235,12 @@ export interface MidsceneYamlConfigResult {
   report?: string | null;
   error?: string;
   duration?: number;
+  /**
+   * Type of result:
+   * - 'success': All tasks completed successfully
+   * - 'failed': Execution failed (player error)
+   * - 'partialFailed': Some tasks failed but execution continued (continueOnError)
+   * - 'notExecuted': Not executed due to previous failures
+   */
+  resultType?: 'success' | 'failed' | 'partialFailed' | 'notExecuted';
 }