diff --git a/.github/workflows/tests-ci.yaml b/.github/workflows/tests-ci.yaml index 1e069ea11f..ac49e551e2 100644 --- a/.github/workflows/tests-ci.yaml +++ b/.github/workflows/tests-ci.yaml @@ -212,7 +212,7 @@ jobs: merge-reports: needs: [playwright-tests-chromium-sharded] runs-on: ubuntu-latest - if: ${{ !cancelled() }} + if: ${{ always() && !cancelled() }} steps: - name: Checkout ComfyUI_frontend uses: actions/checkout@v5 @@ -252,6 +252,22 @@ jobs: pnpm exec playwright merge-reports --reporter=json ./all-blob-reports working-directory: ComfyUI_frontend + - name: Build failed screenshot manifest + if: ${{ needs.playwright-tests-chromium-sharded.result == 'failure' }} + run: | + set -euo pipefail + pnpm tsx scripts/cicd/build-failed-screenshot-manifest.ts + working-directory: ComfyUI_frontend + + - name: Upload failed screenshot manifest + if: ${{ needs.playwright-tests-chromium-sharded.result == 'failure' }} + uses: actions/upload-artifact@v4 + with: + name: failed-screenshot-tests + path: ComfyUI_frontend/ci-rerun/*.txt + retention-days: 7 + if-no-files-found: ignore + - name: Upload HTML report uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/update-playwright-expectations.yaml b/.github/workflows/update-playwright-expectations.yaml index 82b99baa87..5d08a3be0a 100644 --- a/.github/workflows/update-playwright-expectations.yaml +++ b/.github/workflows/update-playwright-expectations.yaml @@ -23,38 +23,145 @@ jobs: steps: - name: Initial Checkout uses: actions/checkout@v5 + - name: Pull Request Checkout - run: gh pr checkout ${{ github.event.issue.number }} if: github.event.issue.pull_request && github.event_name == 'issue_comment' + run: gh pr checkout ${{ github.event.issue.number }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Setup Frontend uses: ./.github/actions/setup-frontend + - name: Setup Playwright uses: ./.github/actions/setup-playwright - - name: Run Playwright tests and update snapshots + + - name: Locate failed screenshot manifest artifact + id: locate-manifest + uses: actions/github-script@v8 + with: + script: | + const { owner, repo } = context.repo + let headSha = '' + if (context.eventName === 'pull_request') { + headSha = context.payload.pull_request.head.sha + } else if (context.eventName === 'issue_comment') { + const prNumber = context.payload.issue.number + const pr = await github.rest.pulls.get({ owner, repo, pull_number: prNumber }) + headSha = pr.data.head.sha + } + + if (!headSha) { + core.setOutput('run_id', '') + core.setOutput('has_manifest', 'false') + return + } + + const { data } = await github.rest.actions.listWorkflowRuns({ + owner, + repo, + workflow_id: 'tests-ci.yaml', + head_sha: headSha, + event: 'pull_request', + per_page: 1, + }) + const run = data.workflow_runs?.[0] + + let has = 'false' + let runId = '' + if (run) { + runId = String(run.id) + const { data: { artifacts = [] } } = await github.rest.actions.listWorkflowRunArtifacts({ + owner, + repo, + run_id: run.id, + per_page: 100, + }) + if (artifacts.some(a => a.name === 'failed-screenshot-tests' && !a.expired)) has = 'true' + } + core.setOutput('run_id', runId) + core.setOutput('has_manifest', has) + + - name: Download failed screenshot manifest + if: steps.locate-manifest.outputs.has_manifest == 'true' + uses: actions/download-artifact@v4 + with: + run-id: ${{ steps.locate-manifest.outputs.run_id }} + name: failed-screenshot-tests + path: ComfyUI_frontend/ci-rerun + + - name: Re-run failed screenshot tests and update snapshots id: playwright-tests - run: pnpm exec playwright test --update-snapshots - continue-on-error: true + shell: bash working-directory: ComfyUI_frontend + continue-on-error: true + run: | + set -euo pipefail + if [ ! -d ci-rerun ]; then + echo "No manifest found; running full suite as fallback" + PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \ + pnpm exec playwright test --update-snapshots \ + --reporter=line --reporter=html + exit 0 + fi + shopt -s nullglob + files=(ci-rerun/*.txt) + if [ ${#files[@]} -eq 0 ]; then + echo "Manifest is empty; running full suite as fallback" + PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \ + pnpm exec playwright test --update-snapshots \ + --reporter=line --reporter=html + exit 0 + fi + for f in "${files[@]}"; do + project="$(basename "$f" .txt)" + mapfile -t lines < "$f" + filtered=( ) + for l in "${lines[@]}"; do + [ -n "$l" ] && filtered+=("$l") + done + if [ ${#filtered[@]} -eq 0 ]; then + continue + fi + echo "Re-running ${#filtered[@]} tests for project $project" + PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \ + pnpm exec playwright test --project="$project" --update-snapshots \ + --reporter=line --reporter=html \ + "${filtered[@]}" + done + - uses: actions/upload-artifact@v4 if: always() with: name: playwright-report path: ComfyUI_frontend/playwright-report/ retention-days: 30 + - name: Debugging info + working-directory: ComfyUI_frontend run: | - echo "PR: ${{ github.event.issue.number }}" + echo "Branch: ${{ github.head_ref }}" git status - working-directory: ComfyUI_frontend + - name: Commit updated expectations + working-directory: ComfyUI_frontend run: | git config --global user.name 'github-actions' git config --global user.email 'github-actions@github.com' + if [ "${{ github.event_name }}" = "issue_comment" ]; then + true + else + git fetch origin ${{ github.head_ref }} + git checkout -B ${{ github.head_ref }} origin/${{ github.head_ref }} + fi git add browser_tests - git diff --cached --quiet || git commit -m "[automated] Update test expectations" - git push - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - working-directory: ComfyUI_frontend + if git diff --cached --quiet; then + echo "No expectation updates detected; skipping commit." + else + git commit -m "[automated] Update test expectations" + if [ "${{ github.event_name }}" = "issue_comment" ]; then + git push + else + git push origin HEAD:${{ github.head_ref }} + fi + fi diff --git a/scripts/cicd/build-failed-screenshot-manifest.ts b/scripts/cicd/build-failed-screenshot-manifest.ts new file mode 100644 index 0000000000..9ad7c9f83c --- /dev/null +++ b/scripts/cicd/build-failed-screenshot-manifest.ts @@ -0,0 +1,74 @@ +import type { + JSONReport, + JSONReportSpec, + JSONReportSuite, + JSONReportTestResult +} from '@playwright/test/reporter' +import fs from 'node:fs' +import fsp from 'node:fs/promises' +import path from 'node:path' + +const argv = process.argv.slice(2) +const getArg = (flag: string, fallback: string) => { + const i = argv.indexOf(flag) + if (i >= 0 && i + 1 < argv.length) return argv[i + 1] + return fallback +} + +async function main() { + // Defaults mirror the workflow layout + const reportPath = getArg( + '--report', + path.join('playwright-report', 'report.json') + ) + const outDir = getArg('--out', path.join('ci-rerun')) + + if (!fs.existsSync(reportPath)) { + throw Error(`Report not found at ${reportPath}`) + } + + const raw = await fsp.readFile(reportPath, 'utf8') + const data = JSON.parse(raw) + + const hasScreenshotSignal = (r: JSONReportTestResult) => { + return r.attachments.some((att) => att?.contentType?.startsWith('image/')) + } + + const out = new Map>() + + const collectFailedScreenshots = (suite?: JSONReportSuite) => { + if (!suite) return + const childSuites = suite.suites ?? [] + for (const childSuite of childSuites) collectFailedScreenshots(childSuite) + const specs: JSONReportSpec[] = suite.specs ?? [] + for (const spec of specs) { + const file = spec.file + const line = spec.line + const loc = `${file}:${line}` + for (const test of spec.tests) { + const project = test.projectId + const last = test.results[test.results.length - 1] + const failedScreenshot = + last && last.status === 'failed' && hasScreenshotSignal(last) + if (!failedScreenshot) continue + if (!out.has(project)) out.set(project, new Set()) + out.get(project)!.add(loc) + } + } + } + + const report: JSONReport = data + const rootSuites = report.suites ?? [] + for (const suite of rootSuites) collectFailedScreenshots(suite) + + await fsp.mkdir(outDir, { recursive: true }) + for (const [project, set] of out.entries()) { + const f = path.join(outDir, `${project}.txt`) + await fsp.writeFile(f, Array.from(set).join('\n') + '\n', 'utf8') + } +} + +main().catch((err) => { + console.error('Manifest generation failed:', err) + process.exit(1) +})