Only run failed tests for snapshot update

.github/workflows/tests-ci.yaml

@@ -212,7 +212,7 @@ jobs:
   merge-reports:
     needs: [playwright-tests-chromium-sharded]
     runs-on: ubuntu-latest
-    if: ${{ !cancelled() }}
+    if: ${{ always() && !cancelled() }}
     steps:
       - name: Checkout ComfyUI_frontend
         uses: actions/checkout@v5
@@ -252,6 +252,22 @@ jobs:
           pnpm exec playwright merge-reports --reporter=json ./all-blob-reports
         working-directory: ComfyUI_frontend
 
+      - name: Build failed screenshot manifest
+        if: ${{ needs.playwright-tests-chromium-sharded.result == 'failure' }}
+        run: |
+          set -euo pipefail
+          pnpm tsx scripts/cicd/build-failed-screenshot-manifest.ts
+        working-directory: ComfyUI_frontend
+
+      - name: Upload failed screenshot manifest
+        if: ${{ needs.playwright-tests-chromium-sharded.result == 'failure' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: failed-screenshot-tests
+          path: ComfyUI_frontend/ci-rerun/*.txt
+          retention-days: 7
+          if-no-files-found: ignore
+
       - name: Upload HTML report
         uses: actions/upload-artifact@v4
         with:

.github/workflows/update-playwright-expectations.yaml

@@ -23,38 +23,145 @@ jobs:
     steps:
       - name: Initial Checkout
         uses: actions/checkout@v5
+
       - name: Pull Request Checkout
-        run: gh pr checkout ${{ github.event.issue.number }}
         if: github.event.issue.pull_request && github.event_name == 'issue_comment'
+        run: gh pr checkout ${{ github.event.issue.number }}
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Setup Frontend
         uses: ./.github/actions/setup-frontend
+
       - name: Setup Playwright
         uses: ./.github/actions/setup-playwright
-      - name: Run Playwright tests and update snapshots
+
+      - name: Locate failed screenshot manifest artifact
+        id: locate-manifest
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const { owner, repo } = context.repo
+            let headSha = ''
+            if (context.eventName === 'pull_request') {
+              headSha = context.payload.pull_request.head.sha
+            } else if (context.eventName === 'issue_comment') {
+              const prNumber = context.payload.issue.number
+              const pr = await github.rest.pulls.get({ owner, repo, pull_number: prNumber })
+              headSha = pr.data.head.sha
+            }
+
+            if (!headSha) {
+              core.setOutput('run_id', '')
+              core.setOutput('has_manifest', 'false')
+              return
+            }
+
+            const { data } = await github.rest.actions.listWorkflowRuns({
+              owner,
+              repo,
+              workflow_id: 'tests-ci.yaml',
+              head_sha: headSha,
+              event: 'pull_request',
+              per_page: 1,
+            })
+            const run = data.workflow_runs?.[0]
+
+            let has = 'false'
+            let runId = ''
+            if (run) {
+              runId = String(run.id)
+              const { data: { artifacts = [] } } = await github.rest.actions.listWorkflowRunArtifacts({
+                owner,
+                repo,
+                run_id: run.id,
+                per_page: 100,
+              })
+              if (artifacts.some(a => a.name === 'failed-screenshot-tests' && !a.expired)) has = 'true'
+            }
+            core.setOutput('run_id', runId)
+            core.setOutput('has_manifest', has)
+
+      - name: Download failed screenshot manifest
+        if: steps.locate-manifest.outputs.has_manifest == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          run-id: ${{ steps.locate-manifest.outputs.run_id }}
+          name: failed-screenshot-tests
+          path: ComfyUI_frontend/ci-rerun
+
+      - name: Re-run failed screenshot tests and update snapshots
         id: playwright-tests
-        run: pnpm exec playwright test --update-snapshots
-        continue-on-error: true
+        shell: bash
         working-directory: ComfyUI_frontend
+        continue-on-error: true
+        run: |
+          set -euo pipefail
+          if [ ! -d ci-rerun ]; then
+            echo "No manifest found; running full suite as fallback"
+            PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \
+            pnpm exec playwright test --update-snapshots \
+              --reporter=line --reporter=html
+            exit 0
+          fi
+          shopt -s nullglob
+          files=(ci-rerun/*.txt)
+          if [ ${#files[@]} -eq 0 ]; then
+            echo "Manifest is empty; running full suite as fallback"
+            PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \
+            pnpm exec playwright test --update-snapshots \
+              --reporter=line --reporter=html
+            exit 0
+          fi
+          for f in "${files[@]}"; do
+            project="$(basename "$f" .txt)"
+            mapfile -t lines < "$f"
+            filtered=( )
+            for l in "${lines[@]}"; do
+              [ -n "$l" ] && filtered+=("$l")
+            done
+            if [ ${#filtered[@]} -eq 0 ]; then
+              continue
+            fi
+            echo "Re-running ${#filtered[@]} tests for project $project"
+            PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-report/report.json \
+            pnpm exec playwright test --project="$project" --update-snapshots \
+              --reporter=line --reporter=html \
+              "${filtered[@]}"
+          done
+
       - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: playwright-report
           path: ComfyUI_frontend/playwright-report/
           retention-days: 30
+
       - name: Debugging info
+        working-directory: ComfyUI_frontend
         run: |
-          echo "PR: ${{ github.event.issue.number }}"
+          echo "Branch: ${{ github.head_ref }}"
           git status
-        working-directory: ComfyUI_frontend
+
       - name: Commit updated expectations
+        working-directory: ComfyUI_frontend
         run: |
           git config --global user.name 'github-actions'
           git config --global user.email '[email protected]'
+          if [ "${{ github.event_name }}" = "issue_comment" ]; then
+            true
+          else
+            git fetch origin ${{ github.head_ref }}
+            git checkout -B ${{ github.head_ref }} origin/${{ github.head_ref }}
+          fi
           git add browser_tests
-          git diff --cached --quiet || git commit -m "[automated] Update test expectations"
-          git push
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        working-directory: ComfyUI_frontend
+          if git diff --cached --quiet; then
+            echo "No expectation updates detected; skipping commit."
+          else
+            git commit -m "[automated] Update test expectations"
+            if [ "${{ github.event_name }}" = "issue_comment" ]; then
+              git push
+            else
+              git push origin HEAD:${{ github.head_ref }}
+            fi
+          fi

scripts/cicd/build-failed-screenshot-manifest.ts

@@ -0,0 +1,74 @@
+import type {
+  JSONReport,
+  JSONReportSpec,
+  JSONReportSuite,
+  JSONReportTestResult
+} from '@playwright/test/reporter'
+import fs from 'node:fs'
+import fsp from 'node:fs/promises'
+import path from 'node:path'
+
+const argv = process.argv.slice(2)
+const getArg = (flag: string, fallback: string) => {
+  const i = argv.indexOf(flag)
+  if (i >= 0 && i + 1 < argv.length) return argv[i + 1]
+  return fallback
+}
+
+async function main() {
+  // Defaults mirror the workflow layout
+  const reportPath = getArg(
+    '--report',
+    path.join('playwright-report', 'report.json')
+  )
+  const outDir = getArg('--out', path.join('ci-rerun'))
+
+  if (!fs.existsSync(reportPath)) {
+    throw Error(`Report not found at ${reportPath}`)
+  }
+
+  const raw = await fsp.readFile(reportPath, 'utf8')
+  const data = JSON.parse(raw)
+
+  const hasScreenshotSignal = (r: JSONReportTestResult) => {
+    return r.attachments.some((att) => att?.contentType?.startsWith('image/'))
+  }
+
+  const out = new Map<string, Set<string>>()
+
+  const collectFailedScreenshots = (suite?: JSONReportSuite) => {
+    if (!suite) return
+    const childSuites = suite.suites ?? []
+    for (const childSuite of childSuites) collectFailedScreenshots(childSuite)
+    const specs: JSONReportSpec[] = suite.specs ?? []
+    for (const spec of specs) {
+      const file = spec.file
+      const line = spec.line
+      const loc = `${file}:${line}`
+      for (const test of spec.tests) {
+        const project = test.projectId
+        const last = test.results[test.results.length - 1]
+        const failedScreenshot =
+          last && last.status === 'failed' && hasScreenshotSignal(last)
+        if (!failedScreenshot) continue
+        if (!out.has(project)) out.set(project, new Set())
+        out.get(project)!.add(loc)
+      }
+    }
+  }
+
+  const report: JSONReport = data
+  const rootSuites = report.suites ?? []
+  for (const suite of rootSuites) collectFailedScreenshots(suite)
+
+  await fsp.mkdir(outDir, { recursive: true })
+  for (const [project, set] of out.entries()) {
+    const f = path.join(outDir, `${project}.txt`)
+    await fsp.writeFile(f, Array.from(set).join('\n') + '\n', 'utf8')
+  }
+}
+
+main().catch((err) => {
+  console.error('Manifest generation failed:', err)
+  process.exit(1)
+})