diff --git a/scripts/cicd/extract-playwright-counts.ts b/scripts/cicd/extract-playwright-counts.ts index ff6f44db30..7b1c759275 100755 --- a/scripts/cicd/extract-playwright-counts.ts +++ b/scripts/cicd/extract-playwright-counts.ts @@ -10,8 +10,56 @@ interface TestStats { finished?: number } +interface TestLocation { + file: string + line: number + column: number +} + +interface TestAttachment { + name: string + path?: string + contentType: string +} + +interface TestResult { + status: string + duration: number + errors?: Array<{ message?: string; stack?: string }> + attachments?: TestAttachment[] +} + +interface Test { + title: string + location?: TestLocation + results?: TestResult[] +} + +interface Suite { + title: string + suites?: Suite[] + tests?: Test[] +} + interface ReportData { stats?: TestStats + suites?: Suite[] +} + +interface FailingTest { + name: string + filePath: string + line: number + error: string + tracePath?: string + failureType?: 'screenshot' | 'expectation' | 'timeout' | 'other' +} + +interface FailureTypeCounts { + screenshot: number + expectation: number + timeout: number + other: number } interface TestCounts { @@ -20,12 +68,106 @@ interface TestCounts { flaky: number skipped: number total: number + failingTests?: FailingTest[] + failureTypes?: FailureTypeCounts +} + +/** + * Categorize the failure type based on error message + */ +function categorizeFailureType( + error: string, + status: string +): 'screenshot' | 'expectation' | 'timeout' | 'other' { + if (status === 'timedOut') { + return 'timeout' + } + + const errorLower = error.toLowerCase() + + // Screenshot-related errors + if ( + errorLower.includes('screenshot') || + errorLower.includes('snapshot') || + errorLower.includes('toHaveScreenshot') || + errorLower.includes('image comparison') || + errorLower.includes('pixel') || + errorLower.includes('visual') + ) { + return 'screenshot' + } + + // Expectation errors + if ( + errorLower.includes('expect') || + errorLower.includes('assertion') || + errorLower.includes('toEqual') || + errorLower.includes('toBe') || + errorLower.includes('toContain') || + errorLower.includes('toHave') || + errorLower.includes('toMatch') + ) { + return 'expectation' + } + + return 'other' +} + +/** + * Recursively extract failing tests from suite structure + */ +function extractFailingTests(suite: Suite, failingTests: FailingTest[]): void { + // Process tests in this suite + if (suite.tests) { + for (const test of suite.tests) { + if (!test.results) continue + + for (const result of test.results) { + if (result.status === 'failed' || result.status === 'timedOut') { + const error = + result.errors?.[0]?.message || + result.errors?.[0]?.stack || + 'Test failed' + + // Find trace attachment + let tracePath: string | undefined + if (result.attachments) { + const traceAttachment = result.attachments.find( + (att) => + att.name === 'trace' || att.contentType === 'application/zip' + ) + if (traceAttachment?.path) { + tracePath = traceAttachment.path + } + } + + const failureType = categorizeFailureType(error, result.status) + + failingTests.push({ + name: test.title, + filePath: test.location?.file || 'unknown', + line: test.location?.line || 0, + error: error.split('\n')[0], // First line of error + tracePath, + failureType + }) + } + } + } + } + + // Recursively process nested suites + if (suite.suites) { + for (const nestedSuite of suite.suites) { + extractFailingTests(nestedSuite, failingTests) + } + } } /** * Extract test counts from Playwright HTML report * @param reportDir - Path to the playwright-report directory - * @returns Test counts { passed, failed, flaky, skipped, total } + * @returns Test counts { passed, failed, flaky, skipped, total, failingTests } */ function extractTestCounts(reportDir: string): TestCounts { const counts: TestCounts = { @@ -33,7 +175,14 @@ function extractTestCounts(reportDir: string): TestCounts { failed: 0, flaky: 0, skipped: 0, - total: 0 + total: 0, + failingTests: [], + failureTypes: { + screenshot: 0, + expectation: 0, + timeout: 0, + other: 0 + } } try { @@ -54,6 +203,22 @@ function extractTestCounts(reportDir: string): TestCounts { counts.failed = stats.unexpected || 0 counts.flaky = stats.flaky || 0 counts.skipped = stats.skipped || 0 + + // Extract failing test details + if (reportJson.suites) { + for (const suite of reportJson.suites) { + extractFailingTests(suite, counts.failingTests) + } + } + + // Count failure types + if (counts.failingTests) { + for (const test of counts.failingTests) { + const type = test.failureType || 'other' + counts.failureTypes![type]++ + } + } + return counts } } @@ -86,6 +251,22 @@ function extractTestCounts(reportDir: string): TestCounts { counts.failed = stats.unexpected || 0 counts.flaky = stats.flaky || 0 counts.skipped = stats.skipped || 0 + + // Extract failing test details + if (reportData.suites) { + for (const suite of reportData.suites) { + extractFailingTests(suite, counts.failingTests!) + } + } + + // Count failure types + if (counts.failingTests) { + for (const test of counts.failingTests) { + const type = test.failureType || 'other' + counts.failureTypes![type]++ + } + } + return counts } } catch (e) { @@ -113,6 +294,22 @@ function extractTestCounts(reportDir: string): TestCounts { counts.failed = stats.unexpected || 0 counts.flaky = stats.flaky || 0 counts.skipped = stats.skipped || 0 + + // Extract failing test details + if (reportData.suites) { + for (const suite of reportData.suites) { + extractFailingTests(suite, counts.failingTests!) + } + } + + // Count failure types + if (counts.failingTests) { + for (const test of counts.failingTests) { + const type = test.failureType || 'other' + counts.failureTypes![type]++ + } + } + return counts } } catch (e) { @@ -161,7 +358,7 @@ function extractTestCounts(reportDir: string): TestCounts { } } } catch (error) { - console.error(`Error reading report from ${reportDir}:`, error) + process.stderr.write(`Error reading report from ${reportDir}: ${error}\n`) } return counts @@ -171,13 +368,15 @@ function extractTestCounts(reportDir: string): TestCounts { const reportDir = process.argv[2] if (!reportDir) { - console.error('Usage: extract-playwright-counts.ts ') + process.stderr.write( + 'Usage: extract-playwright-counts.ts \n' + ) process.exit(1) } const counts = extractTestCounts(reportDir) // Output as JSON for easy parsing in shell script -console.log(JSON.stringify(counts)) +process.stdout.write(JSON.stringify(counts) + '\n') export { extractTestCounts } diff --git a/scripts/cicd/pr-playwright-deploy-and-comment.sh b/scripts/cicd/pr-playwright-deploy-and-comment.sh index 840203f44a..ad7c1eede2 100755 --- a/scripts/cicd/pr-playwright-deploy-and-comment.sh +++ b/scripts/cicd/pr-playwright-deploy-and-comment.sh @@ -252,6 +252,10 @@ else total_flaky=0 total_skipped=0 total_tests=0 + total_screenshot_failures=0 + total_expectation_failures=0 + total_timeout_failures=0 + total_other_failures=0 # Parse counts and calculate totals IFS='|' read -r -a counts_array <<< "$all_counts" @@ -265,6 +269,10 @@ else flaky=$(echo "$counts_json" | jq -r '.flaky // 0') skipped=$(echo "$counts_json" | jq -r '.skipped // 0') total=$(echo "$counts_json" | jq -r '.total // 0') + screenshot=$(echo "$counts_json" | jq -r '.failureTypes.screenshot // 0') + expectation=$(echo "$counts_json" | jq -r '.failureTypes.expectation // 0') + timeout=$(echo "$counts_json" | jq -r '.failureTypes.timeout // 0') + other=$(echo "$counts_json" | jq -r '.failureTypes.other // 0') else # Fallback parsing without jq passed=$(echo "$counts_json" | sed -n 's/.*"passed":\([0-9]*\).*/\1/p') @@ -272,13 +280,21 @@ else flaky=$(echo "$counts_json" | sed -n 's/.*"flaky":\([0-9]*\).*/\1/p') skipped=$(echo "$counts_json" | sed -n 's/.*"skipped":\([0-9]*\).*/\1/p') total=$(echo "$counts_json" | sed -n 's/.*"total":\([0-9]*\).*/\1/p') + screenshot=0 + expectation=0 + timeout=0 + other=0 fi - + total_passed=$((total_passed + ${passed:-0})) total_failed=$((total_failed + ${failed:-0})) total_flaky=$((total_flaky + ${flaky:-0})) total_skipped=$((total_skipped + ${skipped:-0})) total_tests=$((total_tests + ${total:-0})) + total_screenshot_failures=$((total_screenshot_failures + ${screenshot:-0})) + total_expectation_failures=$((total_expectation_failures + ${expectation:-0})) + total_timeout_failures=$((total_timeout_failures + ${timeout:-0})) + total_other_failures=$((total_other_failures + ${other:-0})) fi done unset IFS @@ -302,35 +318,98 @@ else comment="$COMMENT_MARKER ## 🎭 Playwright Test Results -$status_icon **$status_text** - -⏰ Completed at: $(date -u '+%m/%d/%Y, %I:%M:%S %p') UTC" +$status_icon **$status_text** • ⏰ $(date -u '+%m/%d/%Y, %I:%M:%S %p') UTC" # Add summary counts if we have test data if [ $total_tests -gt 0 ]; then comment="$comment -### 📈 Summary -- **Total Tests:** $total_tests -- **Passed:** $total_passed ✅ -- **Failed:** $total_failed $([ $total_failed -gt 0 ] && echo '❌' || echo '') -- **Flaky:** $total_flaky $([ $total_flaky -gt 0 ] && echo '⚠️' || echo '') -- **Skipped:** $total_skipped $([ $total_skipped -gt 0 ] && echo '⏭️' || echo '')" +**$total_passed** ✅ • **$total_failed** $([ $total_failed -gt 0 ] && echo '❌' || echo '✅') • **$total_flaky** $([ $total_flaky -gt 0 ] && echo '⚠️' || echo '✅') • **$total_skipped** ⏭️ • **$total_tests** total" + + # Add failure breakdown if there are failures + if [ $total_failed -gt 0 ]; then + comment="$comment + +**Failure Breakdown:** 📸 $total_screenshot_failures screenshot • ✓ $total_expectation_failures expectation • ⏱️ $total_timeout_failures timeout • ❓ $total_other_failures other" + fi fi + # Collect all failing tests across browsers + all_failing_tests="" + i=0 + IFS=' ' read -r -a browser_array <<< "$BROWSERS" + for counts_json in "${counts_array[@]}"; do + [ -z "$counts_json" ] && { i=$((i + 1)); continue; } + browser="${browser_array[$i]:-}" + + if [ "$counts_json" != "{}" ] && [ -n "$counts_json" ]; then + if command -v jq > /dev/null 2>&1; then + failing_tests=$(echo "$counts_json" | jq -r '.failingTests // [] | .[]' 2>/dev/null || echo "") + if [ -n "$failing_tests" ]; then + # Process each failing test + while IFS= read -r test_json; do + [ -z "$test_json" ] && continue + test_name=$(echo "$test_json" | jq -r '.name // "Unknown test"') + test_file=$(echo "$test_json" | jq -r '.filePath // "unknown"') + test_line=$(echo "$test_json" | jq -r '.line // 0') + trace_path=$(echo "$test_json" | jq -r '.tracePath // ""') + + # Build GitHub source link (assumes ComfyUI_frontend repo) + source_link="https://github.com/$GITHUB_REPOSITORY/blob/$BRANCH_NAME/$test_file#L$test_line" + + # Build trace viewer link if trace exists + if [ -n "$trace_path" ] && [ "$trace_path" != "null" ]; then + # Extract trace filename from path + trace_file=$(basename "$trace_path") + url="${url_array[$i]:-}" + if [ "$url" != "failed" ] && [ -n "$url" ]; then + base_url="${url%/index.html}" + trace_viewer_link="${base_url}/trace/?trace=${base_url}/data/${trace_file}" + fi + fi + + # Format failing test entry + if [ -n "$all_failing_tests" ]; then + all_failing_tests="$all_failing_tests +" + fi + + if [ -n "$trace_viewer_link" ]; then + all_failing_tests="${all_failing_tests}- **[$test_name]($source_link)** \`$browser\` • [View trace]($trace_viewer_link)" + else + all_failing_tests="${all_failing_tests}- **[$test_name]($source_link)** \`$browser\`" + fi + done < <(echo "$counts_json" | jq -c '.failingTests[]?' 2>/dev/null || echo "") + fi + fi + fi + i=$((i + 1)) + done + unset IFS + + # Add failing tests section if there are failures + if [ $total_failed -gt 0 ] && [ -n "$all_failing_tests" ]; then + comment="$comment + +### ❌ Failed Tests +$all_failing_tests" + fi + comment="$comment -### 📊 Test Reports by Browser" - +
+📊 Test Reports by Browser + +" + # Add browser results with individual counts i=0 - IFS=' ' read -r -a browser_array <<< "$BROWSERS" IFS=' ' read -r -a url_array <<< "$urls" for counts_json in "${counts_array[@]}"; do [ -z "$counts_json" ] && { i=$((i + 1)); continue; } browser="${browser_array[$i]:-}" url="${url_array[$i]:-}" - + if [ "$url" != "failed" ] && [ -n "$url" ]; then # Parse individual browser counts if [ "$counts_json" != "{}" ] && [ -n "$counts_json" ]; then @@ -347,7 +426,7 @@ $status_icon **$status_text** b_skipped=$(echo "$counts_json" | sed -n 's/.*"skipped":\([0-9]*\).*/\1/p') b_total=$(echo "$counts_json" | sed -n 's/.*"total":\([0-9]*\).*/\1/p') fi - + if [ -n "$b_total" ] && [ "$b_total" != "0" ]; then counts_str=" • ✅ $b_passed / ❌ $b_failed / ⚠️ $b_flaky / ⏭️ $b_skipped" else @@ -356,21 +435,20 @@ $status_icon **$status_text** else counts_str="" fi - + comment="$comment -- ✅ **${browser}**: [View Report](${url})${counts_str}" +- **${browser}**: [View Report](${url})${counts_str}" else comment="$comment -- ❌ **${browser}**: Deployment failed" +- **${browser}**: Deployment failed" fi i=$((i + 1)) done unset IFS - + comment="$comment ---- -🎉 Click on the links above to view detailed test results for each browser configuration." +
" post_comment "$comment" fi