diff --git a/packages/cli/src/batch-runner.ts b/packages/cli/src/batch-runner.ts index ceccb27dc..bd46f205a 100644 --- a/packages/cli/src/batch-runner.ts +++ b/packages/cli/src/batch-runner.ts @@ -355,7 +355,28 @@ class BatchRunner { for (const context of executedContexts) { const { file, player, duration } = context; - const success = player.status !== 'error'; + // Determine result type based on player and task statuses + const hasFailedTasks = + player.taskStatusList?.some((task) => task.status === 'error') ?? false; + const hasPlayerError = player.status === 'error'; + + let success: boolean; + let resultType: 'success' | 'failed' | 'partialFailed'; + + if (hasPlayerError) { + // Complete failure - player itself failed + success = false; + resultType = 'failed'; + } else if (hasFailedTasks) { + // Partial failure - some tasks failed but execution continued (continueOnError) + success = false; + resultType = 'partialFailed'; + } else { + // Success - all tasks completed successfully + success = true; + resultType = 'success'; + } + let reportFile: string | undefined; if (player.reportFile) { @@ -375,9 +396,11 @@ class BatchRunner { output: outputPath, report: reportFile, duration, + resultType, error: player.errorInSetup?.message || - (player.status === 'error' ? 'Execution failed' : undefined), + (hasPlayerError ? 'Execution failed' : undefined) || + (hasFailedTasks ? 'Some tasks failed' : undefined), }); } @@ -389,6 +412,7 @@ class BatchRunner { output: undefined, report: undefined, duration: 0, + resultType: 'notExecuted', error: 'Not executed (previous task failed)', }); } @@ -435,8 +459,15 @@ class BatchRunner { const indexData = { summary: { total: this.results.length, - successful: this.results.filter((r) => r.success).length, - failed: this.results.filter((r) => !r.success).length, + successful: this.results.filter((r) => r.resultType === 'success') + .length, + failed: this.results.filter((r) => r.resultType === 'failed').length, + partialFailed: this.results.filter( + (r) => r.resultType === 'partialFailed', + ).length, + notExecuted: this.results.filter( + (r) => r.resultType === 'notExecuted', + ).length, totalDuration: this.results.reduce( (sum, r) => sum + (r.duration || 0), 0, @@ -446,6 +477,7 @@ class BatchRunner { results: this.results.map((result) => ({ script: relative(outputDir, result.file), success: result.success, + resultType: result.resultType, output: result.output ? (() => { const relativePath = relative(outputDir, result.output); @@ -473,17 +505,26 @@ class BatchRunner { total: number; successful: number; failed: number; + partialFailed: number; notExecuted: number; totalDuration: number; } { - const successful = this.results.filter((r) => r.success).length; - const notExecuted = this.results.filter((r) => !r.executed).length; - const failed = this.results.filter((r) => r.executed && !r.success).length; + const successful = this.results.filter( + (r) => r.resultType === 'success', + ).length; + const failed = this.results.filter((r) => r.resultType === 'failed').length; + const partialFailed = this.results.filter( + (r) => r.resultType === 'partialFailed', + ).length; + const notExecuted = this.results.filter( + (r) => r.resultType === 'notExecuted', + ).length; return { total: this.results.length, successful, failed, + partialFailed, notExecuted, totalDuration: this.results.reduce( (sum, r) => sum + (r.duration || 0), @@ -494,16 +535,26 @@ class BatchRunner { getFailedFiles(): string[] { return this.results - .filter((r) => r.executed && !r.success) + .filter((r) => r.resultType === 'failed') + .map((r) => r.file); + } + + getPartialFailedFiles(): string[] { + return this.results + .filter((r) => r.resultType === 'partialFailed') .map((r) => r.file); } getNotExecutedFiles(): string[] { - return this.results.filter((r) => !r.executed).map((r) => r.file); + return this.results + .filter((r) => r.resultType === 'notExecuted') + .map((r) => r.file); } getSuccessfulFiles(): string[] { - return this.results.filter((r) => r.success).map((r) => r.file); + return this.results + .filter((r) => r.resultType === 'success') + .map((r) => r.file); } getResults(): MidsceneYamlConfigResult[] { @@ -512,12 +563,16 @@ class BatchRunner { printExecutionSummary(): boolean { const summary = this.getExecutionSummary(); - const success = summary.failed === 0 && summary.notExecuted === 0; + const success = + summary.failed === 0 && + summary.partialFailed === 0 && + summary.notExecuted === 0; console.log('\n📊 Execution Summary:'); console.log(` Total files: ${summary.total}`); console.log(` Successful: ${summary.successful}`); console.log(` Failed: ${summary.failed}`); + console.log(` Partial failed: ${summary.partialFailed}`); console.log(` Not executed: ${summary.notExecuted}`); console.log(` Duration: ${(summary.totalDuration / 1000).toFixed(2)}s`); console.log(` Summary: ${this.getSummaryAbsolutePath()}`); @@ -536,6 +591,15 @@ class BatchRunner { }); } + if (summary.partialFailed > 0) { + console.log( + '\n⚠️ Partial failed files (some tasks failed with continueOnError)', + ); + this.getPartialFailedFiles().forEach((file) => { + console.log(` ${file}`); + }); + } + if (summary.notExecuted > 0) { console.log('\n⏸️ Not executed files'); this.getNotExecutedFiles().forEach((file) => { diff --git a/packages/cli/tests/unit-test/batch-runner.test.ts b/packages/cli/tests/unit-test/batch-runner.test.ts index 1819f071e..10dc98d64 100644 --- a/packages/cli/tests/unit-test/batch-runner.test.ts +++ b/packages/cli/tests/unit-test/batch-runner.test.ts @@ -369,6 +369,69 @@ describe('BatchRunner', () => { ); consoleSpy.mockRestore(); }); + + test('continueOnError: failed tasks should be counted as failed files', async () => { + const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + + // Create a mock player that simulates continueOnError behavior: + // - player.status = 'done' (execution completed) + // - but taskStatusList contains failed tasks + const createMockPlayerWithFailedTasks = ( + fileName: string, + ): ScriptPlayer => { + const isFile1 = fileName === 'file1.yml'; + const mockPlayer = { + status: 'done' as ScriptPlayerStatusValue, // Always 'done' with continueOnError + output: '/test/output/file.json', + reportFile: '/test/report.html', + result: { test: 'data' }, + errorInSetup: null, + taskStatusList: isFile1 + ? [ + { + status: 'error', + error: new Error( + 'Assertion failed: this is not a search engine', + ), + }, + { status: 'done' }, + ] + : [{ status: 'done' }], + run: vi.fn().mockImplementation(async () => { + return undefined; + }), + script: mockYamlScript, + setupAgent: vi.fn(), + unnamedResultIndex: 0, + pageAgent: null, + currentTaskIndex: undefined, + agentStatusTip: '', + }; + return mockPlayer as unknown as ScriptPlayer; + }; + + vi.mocked(createYamlPlayer).mockImplementation(async (file) => + createMockPlayerWithFailedTasks(file), + ); + + const config = { ...mockBatchConfig, continueOnError: true }; + const executor = new BatchRunner(config); + await executor.run(); + + const summary = executor.getExecutionSummary(); + const success = executor.printExecutionSummary(); + + // Files with failed tasks and continueOnError should be counted as partialFailed + expect(summary.partialFailed).toBe(1); + expect(summary.failed).toBe(0); // No complete failures + expect(summary.successful).toBe(2); // The other two files succeeded + expect(success).toBe(false); // Overall should still be false due to partial failure + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining('⚠️ Partial failed files'), + ); + + consoleSpy.mockRestore(); + }); }); describe('BatchRunner output file existence check', () => { diff --git a/packages/core/src/yaml.ts b/packages/core/src/yaml.ts index 7fb43c3ed..2f9ff5376 100644 --- a/packages/core/src/yaml.ts +++ b/packages/core/src/yaml.ts @@ -235,4 +235,12 @@ export interface MidsceneYamlConfigResult { report?: string | null; error?: string; duration?: number; + /** + * Type of result: + * - 'success': All tasks completed successfully + * - 'failed': Execution failed (player error) + * - 'partialFailed': Some tasks failed but execution continued (continueOnError) + * - 'notExecuted': Not executed due to previous failures + */ + resultType?: 'success' | 'failed' | 'partialFailed' | 'notExecuted'; }