feat(cli): add resultType to categorize execution results

- Added 'resultType' field to MidsceneYamlConfigResult to distinguish between:
  - 'success': All tasks completed successfully
  - 'failed': Complete execution failure (player error)
  - 'partialFailed': Some tasks failed with continueOnError=true
  - 'notExecuted': Not executed due to previous failures

- Updated BatchRunner to track and display partial failures separately
- Added 'Partial failed' count to execution summary
- Updated tests to verify correct categorization

This provides better visibility into what actually happened during execution,
distinguishing between complete failures and partial failures where tasks
failed but execution continued due to continueOnError setting.

Co-authored-by: quanru <[email protected]>

Author: Copilot

packages/cli/src/batch-runner.ts

@@ -355,12 +355,28 @@ class BatchRunner {
 
     for (const context of executedContexts) {
       const { file, player, duration } = context;
-      // A file is successful only if:
-      // 1. Player status is not 'error', AND
-      // 2. No individual tasks have failed status
+      // Determine result type based on player and task statuses
       const hasFailedTasks =
         player.taskStatusList?.some((task) => task.status === 'error') ?? false;
-      const success = player.status !== 'error' && !hasFailedTasks;
+      const hasPlayerError = player.status === 'error';
+
+      let success: boolean;
+      let resultType: 'success' | 'failed' | 'partialFailed';
+
+      if (hasPlayerError) {
+        // Complete failure - player itself failed
+        success = false;
+        resultType = 'failed';
+      } else if (hasFailedTasks) {
+        // Partial failure - some tasks failed but execution continued (continueOnError)
+        success = false;
+        resultType = 'partialFailed';
+      } else {
+        // Success - all tasks completed successfully
+        success = true;
+        resultType = 'success';
+      }
+
       let reportFile: string | undefined;
 
       if (player.reportFile) {
@@ -380,9 +396,10 @@ class BatchRunner {
         output: outputPath,
         report: reportFile,
         duration,
+        resultType,
         error:
           player.errorInSetup?.message ||
-          (player.status === 'error' ? 'Execution failed' : undefined) ||
+          (hasPlayerError ? 'Execution failed' : undefined) ||
           (hasFailedTasks ? 'Some tasks failed' : undefined),
       });
     }
@@ -395,6 +412,7 @@ class BatchRunner {
         output: undefined,
         report: undefined,
         duration: 0,
+        resultType: 'notExecuted',
         error: 'Not executed (previous task failed)',
       });
     }
@@ -441,8 +459,15 @@ class BatchRunner {
       const indexData = {
         summary: {
           total: this.results.length,
-          successful: this.results.filter((r) => r.success).length,
-          failed: this.results.filter((r) => !r.success).length,
+          successful: this.results.filter((r) => r.resultType === 'success')
+            .length,
+          failed: this.results.filter((r) => r.resultType === 'failed').length,
+          partialFailed: this.results.filter(
+            (r) => r.resultType === 'partialFailed',
+          ).length,
+          notExecuted: this.results.filter(
+            (r) => r.resultType === 'notExecuted',
+          ).length,
           totalDuration: this.results.reduce(
             (sum, r) => sum + (r.duration || 0),
             0,
@@ -452,6 +477,7 @@ class BatchRunner {
         results: this.results.map((result) => ({
           script: relative(outputDir, result.file),
           success: result.success,
+          resultType: result.resultType,
           output: result.output
             ? (() => {
                 const relativePath = relative(outputDir, result.output);
@@ -479,17 +505,26 @@ class BatchRunner {
     total: number;
     successful: number;
     failed: number;
+    partialFailed: number;
     notExecuted: number;
     totalDuration: number;
   } {
-    const successful = this.results.filter((r) => r.success).length;
-    const notExecuted = this.results.filter((r) => !r.executed).length;
-    const failed = this.results.filter((r) => r.executed && !r.success).length;
+    const successful = this.results.filter(
+      (r) => r.resultType === 'success',
+    ).length;
+    const failed = this.results.filter((r) => r.resultType === 'failed').length;
+    const partialFailed = this.results.filter(
+      (r) => r.resultType === 'partialFailed',
+    ).length;
+    const notExecuted = this.results.filter(
+      (r) => r.resultType === 'notExecuted',
+    ).length;
 
     return {
       total: this.results.length,
       successful,
       failed,
+      partialFailed,
       notExecuted,
       totalDuration: this.results.reduce(
         (sum, r) => sum + (r.duration || 0),
@@ -500,16 +535,26 @@ class BatchRunner {
 
   getFailedFiles(): string[] {
     return this.results
-      .filter((r) => r.executed && !r.success)
+      .filter((r) => r.resultType === 'failed')
+      .map((r) => r.file);
+  }
+
+  getPartialFailedFiles(): string[] {
+    return this.results
+      .filter((r) => r.resultType === 'partialFailed')
       .map((r) => r.file);
   }
 
   getNotExecutedFiles(): string[] {
-    return this.results.filter((r) => !r.executed).map((r) => r.file);
+    return this.results
+      .filter((r) => r.resultType === 'notExecuted')
+      .map((r) => r.file);
   }
 
   getSuccessfulFiles(): string[] {
-    return this.results.filter((r) => r.success).map((r) => r.file);
+    return this.results
+      .filter((r) => r.resultType === 'success')
+      .map((r) => r.file);
   }
 
   getResults(): MidsceneYamlConfigResult[] {
@@ -518,12 +563,16 @@ class BatchRunner {
 
   printExecutionSummary(): boolean {
     const summary = this.getExecutionSummary();
-    const success = summary.failed === 0 && summary.notExecuted === 0;
+    const success =
+      summary.failed === 0 &&
+      summary.partialFailed === 0 &&
+      summary.notExecuted === 0;
 
     console.log('\n📊 Execution Summary:');
     console.log(`   Total files: ${summary.total}`);
     console.log(`   Successful: ${summary.successful}`);
     console.log(`   Failed: ${summary.failed}`);
+    console.log(`   Partial failed: ${summary.partialFailed}`);
     console.log(`   Not executed: ${summary.notExecuted}`);
     console.log(`   Duration: ${(summary.totalDuration / 1000).toFixed(2)}s`);
     console.log(`   Summary: ${this.getSummaryAbsolutePath()}`);
@@ -542,6 +591,15 @@ class BatchRunner {
       });
     }
 
+    if (summary.partialFailed > 0) {
+      console.log(
+        '\n⚠️  Partial failed files (some tasks failed with continueOnError)',
+      );
+      this.getPartialFailedFiles().forEach((file) => {
+        console.log(`   ${file}`);
+      });
+    }
+
     if (summary.notExecuted > 0) {
       console.log('\n⏸️ Not executed files');
       this.getNotExecutedFiles().forEach((file) => {

packages/cli/tests/unit-test/batch-runner.test.ts

@@ -372,7 +372,7 @@ describe('BatchRunner', () => {
 
     test('continueOnError: failed tasks should be counted as failed files', async () => {
       const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
-      
+
       // Create a mock player that simulates continueOnError behavior:
       // - player.status = 'done' (execution completed)
       // - but taskStatusList contains failed tasks
@@ -386,10 +386,15 @@ describe('BatchRunner', () => {
           reportFile: '/test/report.html',
           result: { test: 'data' },
           errorInSetup: null,
-          taskStatusList: isFile1 
+          taskStatusList: isFile1
             ? [
-                { status: 'error', error: new Error('Assertion failed: this is not a search engine') },
-                { status: 'done' }
+                {
+                  status: 'error',
+                  error: new Error(
+                    'Assertion failed: this is not a search engine',
+                  ),
+                },
+                { status: 'done' },
               ]
             : [{ status: 'done' }],
           run: vi.fn().mockImplementation(async () => {
@@ -408,22 +413,23 @@ describe('BatchRunner', () => {
       vi.mocked(createYamlPlayer).mockImplementation(async (file) =>
         createMockPlayerWithFailedTasks(file),
       );
-      
+
       const config = { ...mockBatchConfig, continueOnError: true };
       const executor = new BatchRunner(config);
       await executor.run();
-      
+
       const summary = executor.getExecutionSummary();
       const success = executor.printExecutionSummary();
-      
-      // This currently fails - demonstrates the bug
-      // Should show 1 failed file, but currently shows 0
-      expect(summary.failed).toBe(1); // This test will fail, showing the bug
-      expect(success).toBe(false);
+
+      // Files with failed tasks and continueOnError should be counted as partialFailed
+      expect(summary.partialFailed).toBe(1);
+      expect(summary.failed).toBe(0); // No complete failures
+      expect(summary.successful).toBe(2); // The other two files succeeded
+      expect(success).toBe(false); // Overall should still be false due to partial failure
       expect(consoleSpy).toHaveBeenCalledWith(
-        expect.stringContaining('❌ Failed files'),
+        expect.stringContaining('⚠️  Partial failed files'),
       );
-      
+
       consoleSpy.mockRestore();
     });
   });

packages/core/src/yaml.ts

@@ -235,4 +235,12 @@ export interface MidsceneYamlConfigResult {
   report?: string | null;
   error?: string;
   duration?: number;
+  /**
+   * Type of result:
+   * - 'success': All tasks completed successfully
+   * - 'failed': Execution failed (player error)
+   * - 'partialFailed': Some tasks failed but execution continued (continueOnError)
+   * - 'notExecuted': Not executed due to previous failures
+   */
+  resultType?: 'success' | 'failed' | 'partialFailed' | 'notExecuted';
 }