enable snapshot tests in CI (#1588)

Author: shakyShane

.github/workflows/tests.yml

@@ -64,14 +64,12 @@ jobs:
                   key: docs-output-${{ github.run_id }}
             - name: Install Playwright Browsers
               run: npx playwright install --with-deps
-            - name: Install dependencies for CI integration tests
-              run: sudo apt-get install xvfb
             - run: npm run test-int-x
             - uses: actions/upload-artifact@v4
               if: always()
               with:
                   name: playwright-report-pages
-                  path: special-pages/test-results
+                  path: special-pages/playwright-report
                   retention-days: 5
             - name: Build docs
               run: npm run docs

special-pages/package.json

@@ -14,7 +14,7 @@
     "test-int": "playwright test --grep-invert '@screenshots'",
     "test-int-x": "npm run test-int",
     "test-int-snapshots": "playwright test --grep '@screenshots'",
-    "test-int-snapshots-update": "playwright test --grep '@screenshots' --update-snapshots --last-failed",
+    "test-int-snapshots-update": "playwright test --grep '@screenshots' --update-snapshots --last-failed --pass-with-no-tests",
     "test.screenshots": "playwright test --grep '@screenshots'",
     "test.windows": "npm run test-int -- --project windows",
     "test.macos": "npm run test-int -- --project macos",

special-pages/pages/duckplayer/integration-tests/duckplayer-screenshots.spec.js

@@ -1,9 +1,7 @@
-/* global process */
 import { expect, test } from '@playwright/test';
 import { DuckPlayerPage } from './duck-player.js';
 
 test.describe('screenshots @screenshots', () => {
-    test.skip(process.env.CI === 'true');
     test('regular layout', async ({ page }, workerInfo) => {
         const duckplayer = DuckPlayerPage.create(page, workerInfo);
         // load as normal

special-pages/pages/new-tab/app/favorites/integration-tests/favorites.page.js

@@ -201,8 +201,7 @@ export class FavoritesPage {
         await body.press('Tab');
         await body.press('Tab');
         const statsToggle = page.getByLabel('Hide recent activity');
-        const isActive = await statsToggle.evaluate((handle) => handle === document.activeElement);
-        expect(isActive).toBe(true);
+        await expect(statsToggle).toBeFocused();
     }
 
     /**

special-pages/pages/new-tab/app/privacy-stats/integration-tests/privacy-stats.page.js

@@ -41,7 +41,7 @@ export class PrivacyStatsPage {
      */
     async hasRows(count) {
         const rows = this.rows();
-        expect(await rows.count()).toBe(count);
+        await expect(rows).toHaveCount(count);
     }
 
     /**
@@ -76,7 +76,7 @@ export class PrivacyStatsPage {
         await this.ntp.openPage({ additional: { stats: 'manyOnlyTop' } });
 
         // control, ensure we have 5 rows
-        expect(await rows.count()).toBe(5);
+        await expect(rows).toHaveCount(5);
 
         // check the initial values
         expect(await rows.nth(0).textContent()).toBe('Facebook310');
@@ -103,8 +103,8 @@ export class PrivacyStatsPage {
         // open the NTP with the correct param
         await this.ntp.openPage({ additional: { stats: 'fewOnlyTop' } });
 
-        // control, ensure we have 5 rows
-        expect(await rows.count()).toBe(3);
+        // control, ensure we have 3 rows
+        await expect(rows).toHaveCount(3);
 
         // check the initial values
         expect(await rows.nth(0).textContent()).toBe('Facebook310');
@@ -126,7 +126,7 @@ export class PrivacyStatsPage {
         await this.ntp.openPage({ additional: { stats: 'topAndOneOther' } });
 
         // should show all 5 items
-        expect(await rows.count()).toBe(5);
+        await expect(rows).toHaveCount(5);
 
         // And should also show the text, but no expander
         // ensure the toggle is absent
@@ -145,7 +145,7 @@ export class PrivacyStatsPage {
         await this.ntp.openPage({ additional: { stats: 'manyTopAndOther' } });
 
         // control, ensure we have 5 rows
-        expect(await rows.count()).toBe(5);
+        await expect(rows).toHaveCount(5);
 
         // control: ensure 'other' text is absent here
         await expect(this.page.getByTestId('ListFooter')).not.toContainText('2 attempts from other networks');

special-pages/pages/new-tab/app/privacy-stats/integration-tests/privacy-stats.spec.js

@@ -95,6 +95,7 @@ test.describe('newtab privacy stats', () => {
             const psp = new PrivacyStatsPage(page, ntp);
             await ntp.reducedMotion();
             await ntp.openPage({ additional: { stats: 'none' } });
+            await page.getByText('Tracking protections active').waitFor();
 
             await psp.receiveData({
                 totalCount: 2,

special-pages/pages/special-error/integration-tests/special-error-screenshots.spec.js

@@ -1,12 +1,9 @@
-/* global process */
 import { expect, test } from '@playwright/test';
 import { SpecialErrorPage } from './special-error';
 
 const maxDiffPixels = 20;
 
 test.describe('screenshots @screenshots', () => {
-    test.skip(process.env.CI === 'true');
-
     test('SSL expired cert error', async ({ page }, workerInfo) => {
         const special = SpecialErrorPage.create(page, workerInfo);
         await special.openPage({ errorId: 'ssl.expired' });

special-pages/playwright.config.js

@@ -96,7 +96,8 @@ export default defineConfig({
     retries: process.env.CI ? 2 : 0,
     /* Opt out of parallel tests on CI. */
     workers: process.env.CI ? 1 : undefined,
-    reporter: process.env.CI ? 'github' : [['html', { open: 'never' }]],
+    /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+    reporter: 'html',
     // @ts-expect-error - Type 'undefined' is not assignable to type 'string'. process.env
     webServer: {
         command: process.env.PAGE ? `npm run watch -- --page ${process.env.PAGE}` : 'npm run serve',
@@ -107,5 +108,6 @@ export default defineConfig({
     use: {
         actionTimeout: 5000,
         trace: 'on-first-retry',
+        video: { mode: 'on-first-retry' },
     },
 });