add snapshot checking in CI (#1587)

* add snapshot checking in CI

* linting

* tests

Author: shakyShane

.github/workflows/snapshots-update.yml

@@ -0,0 +1,84 @@
+name: Update Snapshots on a PR
+description: |
+    Runs previously failed snapshot tests, and commits the changes.
+
+    Your PR will receive a commit with the changes so you can manually verify before merging.
+
+on:
+    workflow_dispatch:
+        inputs:
+            pr_number:
+                description: 'Pull Request Number (Warning: This action will push a commit to the referenced PR)'
+                required: true
+                type: number
+
+permissions:
+    pull-requests: write
+    contents: write
+
+jobs:
+    update-pr-with-snapshots:
+        name: Update PR With Snapshots
+        runs-on: macos-14
+        steps:
+            - uses: actions/checkout@v4
+            - name: Checkout PR ${{ github.event.inputs.pr_number }}
+              if: github.event_name == 'workflow_dispatch'
+              run: gh pr checkout ${{ github.event.inputs.pr_number }}
+              env:
+                  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Use Node.js
+              uses: actions/setup-node@v4
+              with:
+                  node-version-file: '.nvmrc'
+            - uses: actions/cache@v4
+              with:
+                  path: |
+                      ~/.npm
+                      ~/.cache/ms-playwright
+                  key: ${{ runner.os }}-node-playwright-${{ hashFiles('**/package-lock.json') }}
+                  restore-keys: |
+                      ${{ runner.os }}-node-playwright-
+                      ${{ runner.os }}-node-
+
+            - name: Install dependencies
+              run: npm ci
+
+            - name: Build all
+              run: npm run build
+
+            - name: Install Playwright Browsers
+              run: npx playwright install --with-deps
+
+            - name: Run Screenshot tests
+              id: screenshot_tests
+              run: npm run test-int-snapshots
+
+            - if: ${{ steps.screenshot_tests.conclusion == 'success' }}
+              run: |
+                  echo "nothing to update - tests all passed"
+
+            - name: Re-Running Playwright to update snapshots
+              id: screenshot_tests_update
+              if: ${{ failure() && steps.screenshot_tests.conclusion == 'failure' }}
+              run: npm run test-int-snapshots-update
+
+            - name: Commit the updated files to the PR branch
+              if: ${{ failure() && steps.screenshot_tests_update.conclusion == 'success' }}
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              run: |
+                  # Configure Git with a bot's username and email for committing changes
+                  # This makes it easy to identify in the PR
+                  git config user.name "github-actions[bot]"
+                  git config user.email "github-actions[bot]@users.noreply.github.com"
+
+                  # Stage all updated PNG files for commit
+                  git add "*.png"
+
+                  # Commit the changes with a descriptive message
+                  git commit -m "Updated snapshots via workflow"
+
+                  # Push the changes to the current branch in the PR
+                  git push origin HEAD

.github/workflows/snapshots.yml

@@ -0,0 +1,66 @@
+name: Test Snapshots
+description: |
+    Runs snapshot tests and uploads test reports as artifacts
+
+    If this workflow fails, you can trigger `update-snapshots.yml` from the
+    GitHub UI.
+
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+
+permissions:
+    contents: read
+
+jobs:
+    snapshots:
+        timeout-minutes: 5
+        runs-on: macos-14
+        steps:
+            - uses: actions/checkout@v4
+            - name: Use Node.js
+              uses: actions/setup-node@v4
+              with:
+                  node-version-file: '.nvmrc'
+            - uses: actions/cache@v4
+              with:
+                  path: |
+                      ~/.npm
+                      ~/.cache/ms-playwright
+                  key: ${{ runner.os }}-node-playwright-${{ hashFiles('**/package-lock.json') }}
+                  restore-keys: |
+                      ${{ runner.os }}-node-playwright-
+                      ${{ runner.os }}-node-
+
+            - run: npm ci
+            - run: npm run build
+
+            - run: npm run lint
+              continue-on-error: true
+
+            - run: npm run stylelint
+              continue-on-error: true
+
+            - run: npm run test-unit
+              continue-on-error: true
+
+            - name: 'Clean tree'
+              run: 'npm run test-clean-tree'
+
+            - name: Install Playwright Browsers
+              run: npx playwright install --with-deps
+
+            - run: npm run test-int-snapshots
+
+            - uses: actions/upload-artifact@v4
+              if: always()
+              with:
+                  name: playwright-report-pages
+                  path: |
+                      special-pages/playwright-report
+                      special-pages/test-results
+                      injected/playwright-report
+                      injected/test-results
+                  retention-days: 5

injected/integration-test/duckplayer-mobile.spec.js-snapshots/overlay-android-darwin.png

@@ -11,8 +11,10 @@
     "build-types": "node scripts/types.mjs",
     "bundle-trackers": "node scripts/bundleTrackers.mjs --output ../build/tracker-lookup.json",
     "test-unit": "jasmine --config=unit-test/config.json",
-    "test-int": "npm run playwright",
+    "test-int": "playwright test --grep-invert '@screenshots'",
     "test-int-x": "xvfb-run --server-args='-screen 0 1024x768x24' npm run test-int",
+    "test-int-snapshots": "playwright test --grep '@screenshots'",
+    "test-int-snapshots-update": "playwright test --grep '@screenshots' --update-snapshots --last-failed",
     "test": "npm run lint && npm run test-unit && npm run test-int && npm run playwright",
     "serve": "http-server -c-1 --port 3220 integration-test/test-pages",
     "playwright": "playwright test --grep-invert '@screenshots'",

injected/package.json

@@ -14,6 +14,8 @@
     "test-unit": "npm run test-unit --workspaces --if-present",
     "test-int": "npm run test-int --workspaces --if-present",
     "test-int-x": "npm run test-int-x --workspaces --if-present",
+    "test-int-snapshots": "npm run test-int-snapshots --workspaces --if-present",
+    "test-int-snapshots-update": "npm run test-int-snapshots-update --workspaces --if-present",
     "test-clean-tree": "npm run build && sh scripts/check-for-changes.sh",
     "docs": "typedoc",
     "docs-watch": "typedoc --watch",

package.json

@@ -11,9 +11,11 @@
     "build.dev": "npm run build -- --env development",
     "lint-fix": "cd ../ && npm run lint-fix",
     "test-unit": "node --test \"unit-test/*\" \"pages/history/unit-tests/*\" \"pages/duckplayer/unit-tests/*\" \"pages/new-tab/app/freemium-pir-banner/unit-tests/*\"",
-    "test-int": "npm run test-unit && npm run build.dev && playwright test --grep-invert '@screenshots'",
+    "test-int": "playwright test --grep-invert '@screenshots'",
     "test-int-x": "npm run test-int",
-    "test.screenshots": "npm run test-unit && npm run build.dev && playwright test --grep '@screenshots'",
+    "test-int-snapshots": "playwright test --grep '@screenshots'",
+    "test-int-snapshots-update": "playwright test --grep '@screenshots' --update-snapshots --last-failed",
+    "test.screenshots": "playwright test --grep '@screenshots'",
     "test.windows": "npm run test-int -- --project windows",
     "test.macos": "npm run test-int -- --project macos",
     "test.ios": "npm run test-int -- --project ios",