feat: Added playwright to example site + gh action for CI

Author: lamATnginx

.github/workflows/playwright.yml

@@ -0,0 +1,75 @@
+name: Testing for visual regression on old theme
+
+# Run the workflow when code is pushed or when a pull request is created
+on:
+    push:
+      branches:
+        - '**'
+    pull_request:
+      branches:
+        - main
+
+jobs:
+    playwright:
+        name: Run Playwright
+        runs-on: ubuntu-latest
+        steps: 
+          # Checkout the repository so the workflow has access to the code
+          - name: Checkout code
+            uses: actions/checkout@v3
+          - name: Install dependencies
+            run: cd tests && npm ci
+          - name: Setup Hugo
+            uses: peaceiris/actions-hugo@75d2e84710de30f6ff7268e08f310b60ef14033f # v3.0.0
+            with:
+                hugo-version: "0.134.2"
+                extended: true
+          - name: Install Playwright browsers
+            run: npx playwright install --with-deps
+          - name: Run Playwright tests
+            id: test-visual
+            run: |
+              make tests | tee output.log
+              if grep -q -e "Error: A snapshot doesn't exist at" -e "Screenshot comparison failed" output.log; then
+                echo "Playwright tests failed due to a snapshot issue."
+                echo "SNAPSHOT_DIFFERENCES=true" >> $GITHUB_ENV
+                exit 1
+              elif grep -q "failed" output.log; then
+                echo "Playwright tests failed due to a non-snapshot issue."
+                exit 1
+              fi
+          - uses: actions/upload-artifact@v4
+            id: artifact-upload
+            if: ${{ !cancelled() && failure() && steps.test-visual.conclusion == 'failure' }}
+            with:
+              name: playwright-report
+              path: tests/playwright-report/
+              retention-days: 3
+          - name: Comment on PR with Playwright report
+            uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+            if: ${{ failure() && env.SNAPSHOT_DIFFERENCES == 'true' }}
+            with:
+              script: |
+                const body = `### <span aria-hidden="true">❌</span> Playwright visual snapshot differences were detected.
+                
+                View the [Playwright report](${{ steps.artifact-upload.outputs.artifact-url }}) 
+                **To approve the snapshot changes and update the snapshots, please comment:** /approve-snapshots`;
+
+                await github.rest.issues.createComment({
+                  issue_number: context.issue.number,
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  body: body,
+                });
+
+          # - name: Update the screenshots based on Ubuntu env on GH
+          #   id: test-visual-update
+          #   if: ${{ failure() && steps.test-visual.conclusion == 'failure' }}
+          #   run: make tests-update-screenshots
+          # - name: Upload updated Playwright screenshots
+          #   uses: actions/upload-artifact@v4
+          #   if: ${{ failure() && steps.test-visual-update.conclusion == 'success' }}
+          #   with:
+          #       name: playwright-screenshots-updated
+          #       path: tests/src/__screenshots__/**/*.png
+          #       retention-days: 3

.gitignore

@@ -15,7 +15,7 @@ Thumbs.db
 ########
 *.log
 
-node_modules/*
+*/node_modules
 .markdownlint.json
 resources/*
 
@@ -27,4 +27,9 @@ public/*
 exampleSite/public
 
 # Python
-.venv
+.venv
+
+# Playwright
+/coverage
+*/test-results
+*/playwright-report

Makefile

@@ -6,15 +6,19 @@ WRITE_FLAG := --write
 list help::
 	$(info Available Make targets:)
 	@echo "<COMMON>"
-	@echo "		list | help:  		Print these available make targets"
+	@echo "				 list | help:  		Print these available make targets"
 	@echo "<LINTING AND FORMATTING>"
-	@echo "		biome-format: 		Runs the biome formatter."
-	@echo "		  biome-lint: 		Runs the biome linter."
-	@echo "		   biome-all:		Runs both the lint and formatting commands."
+	@echo "				biome-format: 		Runs the biome formatter."
+	@echo "				  biome-lint: 		Runs the biome linter."
+	@echo "		   		   biome-all:		Runs both the lint and formatting commands."
 	@echo "  (Set BIOME_ARGS to add additional arguments to biome command (ex: make biome-all BIOME_ARGS=write))"
+	@echo "<PRE-COMMIT>"
+	@echo "			    setup-pre-commit:		Sets up pre-commit (assuming it is installed)"
+	@echo "<PLAYWRIGHT TESTS>"
+	@echo "		     			test: 		Runs playwright against the old theme."
+	@echo "		    tests-update-screenshots:		Runs playwright against the old theme."
 
-.PHONY: biome-format biome-lint biome-all setup-pre-commit
-BIOME_ARGS ?= 
+.PHONY: biome-format biome-lint biome-all setup-pre-commit tests
 FLAG :=
 ifeq ($(BIOME_ARGS), write)
 	FLAG := $(WRITE_FLAG)
@@ -36,3 +40,8 @@ setup-pre-commit:
 		pre-commit install --hook-type commit-msg; \
 		echo "pre-commit hooks have been successfully installed."; \
 	fi
+
+tests:
+	cd tests && npx playwright test
+tests-update-screenshots:
+	cd tests && npx playwright test --update-snapshots

biome.json

@@ -22,7 +22,9 @@
       "**/coveo.css",
       "**/f5-hugo.css",
       "**/highlight.css",
-      "**/*-overrides.css"
+      "**/*-overrides.css",
+      "public/*",
+      "test-results/*"
     ]
   },
   "formatter": {

tests/package-lock.json

@@ -0,0 +1,8 @@
+{
+  "name": "nginx-docs-theme-test",
+  "version": "1.0.0",
+  "private": "true",
+  "devDependencies": {
+    "@playwright/test": "^1.48.0"
+  }
+}

tests/package.json

@@ -0,0 +1,45 @@
+import { defineConfig, devices } from '@playwright/test';
+
+const BASE_URL = 'http://127.0.0.1';
+const PORT = 1313;
+export default defineConfig({
+  testDir: './src',
+  fullyParallel: true,
+  workers: 1,
+  outputDir: './test-results',
+  snapshotPathTemplate: '{testDir}/__screenshots__/{testFilePath}/{arg}{ext}',
+  reporter: [['html', { outputFolder: './playwright-report' }]],
+  use: {
+    baseURL: `${BASE_URL}:${PORT}`,
+    screenshots: 'only-on-failure',
+    video: 'retain-on-failure',
+    trace: 'on-first-retry',
+    timezoneId: 'America/Los_Angeles',
+  },
+  projects: [
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] },
+    },
+    {
+      name: 'webkit',
+      use: { ...devices['Desktop Safari'] },
+    },
+    {
+      name: 'Mobile Chrome',
+      use: { ...devices['Pixel 5'] },
+    },
+  ],
+  webServer: {
+    command: `cd ../exampleSite && hugo mod get && hugo --gc -e production && hugo serve --port ${PORT}`,
+    url: `${BASE_URL}:${PORT}`,
+    stdout: 'ignore',
+  },
+  expect: {
+    toHaveScreenshot: {
+      pathTemplate:
+        '{testDir}/__screenshots__{/projectName}/{testFilePath}/{arg}{ext}',
+      maxDiffPixels: 10,
+    },
+  },
+});

tests/playwright.config.js

@@ -0,0 +1,17 @@
+import { expect, test } from '@playwright/test';
+
+test.describe('Testing old theme', () => {
+  test('should ensure the old theme does not visually regress', async ({
+    page,
+  }) => {
+    const currentPlus = 'nginx/installing-nginx-open-source/';
+    await page.goto(`/${currentPlus}`, { waitUntil: 'networkidle' });
+    await page.evaluate(() => {
+      // switch to old theme
+      useNewTheme(false);
+      window.scrollTo(0, 0);
+    });
+    await page.waitForFunction(() => window.scrollY === 0);
+    await expect(page).toHaveScreenshot('example-site-screenshot.png');
+  });
+});