diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 9481f49d..664025ce 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -9,18 +9,19 @@ on: jobs: test: runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write + contents: write # Still need this for Prettier patch, but not for docs/pr push anymore + pull-requests: write # Still need this for the sticky PR comment steps: #--------------------------------------------------- - # 0 – Checkout + # 0 – Checkout first #--------------------------------------------------- - name: Checkout code uses: actions/checkout@v3 - with: { fetch-depth: 0 } + with: + fetch-depth: 0 # Needed to fetch origin/main later + ref: ${{ github.event.pull_request.head.ref }} # Checkout PR branch first #--------------------------------------------------- # 1 – reviewdog CLI @@ -59,27 +60,119 @@ jobs: - name: Install dependencies run: npm install - #--------------------------------------------------- - # 4 – Install Playwright browsers - #--------------------------------------------------- + # --- IMPORTANT: Install Playwright browsers early --- - name: Install Playwright and browsers run: npx playwright install --with-deps + ## CORE LOGIC FOR BEFORE/AFTER COMPARISON ## + + # Step 1: Save current PR's test files (before checking out main's version) + - name: Save PR test files + run: | + mkdir -p tests_pr_backup/ + cp -r tests/ tests_pr_backup/ # Copy the 'tests' directory and its contents + echo "Contents of tests_pr_backup/:" + ls -la tests_pr_backup/ # Debugging: verify backup content + + # Step 2: Checkout base branch version of test files (from friend's suggestion) + - name: Checkout base version of test files + run: | + git fetch origin main # Ensure origin/main is up-to-date + # This will put 'tests/' from origin/main into your working directory, overwriting current 'tests/' + git checkout --force origin/main -- tests/ + echo "Contents of tests/ after checking out main's version:" + ls -la tests/ # Debugging + + # Step 3: Run Playwright on base version (for "before" view) + - name: Run Playwright "before" tests on Main's version + env: + # This ENV variable is picked up by playwright.config.js for the 'output' path + PLAYWRIGHT_SCREENSHOT_DIR: playwright-artifacts-before + run: | + mkdir -p $PLAYWRIGHT_SCREENSHOT_DIR # Ensure the directory exists + echo "Running Playwright 'before' tests, outputting to: $PLAYWRIGHT_SCREENSHOT_DIR" + # Using --reporter=json to ensure results.json is generated here for potential future use + # Using --debug for more verbose output to diagnose why tests might not be running + npx playwright test --output=$PLAYWRIGHT_SCREENSHOT_DIR --reporter=json --debug || true + # The '|| true' allows the workflow to continue even if Playwright fails, so we can inspect artifacts + + # Debugging: List contents after run + echo "Contents of $PLAYWRIGHT_SCREENSHOT_DIR after 'before' tests:" + ls -laR $PLAYWRIGHT_SCREENSHOT_DIR || true # List even if directory is empty or missing + + # Step 4: Restore PR version of tests + - name: Restore PR test files + run: | + rm -rf tests/ # Remove tests/ from main's version + mv tests_pr_backup/ tests/ # Move the backup back to become the PR's tests/ + echo "Contents of tests/ after restoring PR's version:" + ls -la tests/ # Debugging + #--------------------------------------------------- - # 5 – Run Playwright tests + # 5 – Run Playwright tests (This now runs on the PR branch's tests) #--------------------------------------------------- - - name: Run Playwright tests - run: npx playwright test + - name: Run Playwright tests (PR version) + env: + # This ENV variable is picked up by playwright.config.js for the 'output' path + PLAYWRIGHT_SCREENSHOT_DIR: playwright-artifacts-pr + run: | + mkdir -p $PLAYWRIGHT_SCREENSHOT_DIR # Ensure the directory exists + echo "Running Playwright 'PR' tests, outputting to: $PLAYWRIGHT_SCREENSHOT_DIR" + npx playwright test --output=$PLAYWRIGHT_SCREENSHOT_DIR --reporter=html,json --debug || true + # The '|| true' allows the workflow to continue even if Playwright fails + + # Debugging: List contents after run + echo "Contents of $PLAYWRIGHT_SCREENSHOT_DIR after 'PR' tests:" + ls -laR $PLAYWRIGHT_SCREENSHOT_DIR || true + + # --- Debugging: Verify Playwright Report Existence AFTER tests --- + - name: Verify Playwright Report Existence (PR version) + run: | + # Check in the explicitly set output directory + REPORT_FILE="playwright-artifacts-pr/results.json" + echo "Checking for report file at: $REPORT_FILE" + if [ -f "$REPORT_FILE" ]; then + echo "$REPORT_FILE exists." + else + echo "ERROR: $REPORT_FILE DOES NOT EXIST. Listing parent directory contents:" + ls -la playwright-artifacts-pr/ || true # List even if directory is empty or missing + exit 1 + fi + # --- End Debugging --- + + # --- START OF ARTIFACT UPLOADS --- + + - name: Upload 'Before' Playwright Artifacts + uses: actions/upload-artifact@v4 + with: + name: playwright-artifacts-before + path: playwright-artifacts-before/ + retention-days: 7 # Optional: How long to keep the artifact + + - name: Upload 'After' Playwright Artifacts + uses: actions/upload-artifact@v4 + with: + name: playwright-artifacts-pr + path: playwright-artifacts-pr/ + retention-days: 7 # Optional: How long to keep the artifact + + # --- END OF ARTIFACT UPLOADS --- + + # Removed: Prepare PR Screenshots for GitHub Pages (After) + # Removed: Prepare PR Screenshots for GitHub Pages (Before) + # Removed: Commit and Push PR Screenshots to GitHub Pages + # Removed: Generate Screenshot Markdown for PR Comment + #--------------------------------------------------- - # 6 – Upload HTML report + # 6 – Upload HTML report (Still useful, points to the PR run's html folder) #--------------------------------------------------- - name: Upload HTML report if: always() uses: actions/upload-artifact@v4 with: - name: playwright-report - path: playwright-report/ + name: playwright-html-report + path: playwright-artifacts-pr/html/ # Playwright's HTML report is usually in a subfolder like 'html/' within the output directory #--------------------------------------------------- # 7 – Extract test summary @@ -87,16 +180,34 @@ jobs: - name: Extract test summary id: summary run: | - REPORT=metrics.json - TOTAL=$(jq '[..|objects|select(has("status"))] | length' "$REPORT") - PASSED=$(jq '[..|objects|select(.status?=="expected")] | length' "$REPORT") - FAILED=$(jq '[..|objects|select(.status?=="failed")] | length' "$REPORT") - SKIPPED=$(jq '[..|objects|select(.status?=="skipped")] | length' "$REPORT") - DURATION=$(jq '.stats.duration*1000|floor' "$REPORT") + # Playwright's JSON report contains the summary statistics. + # It's now in the 'playwright-artifacts-pr/' directory. + REPORT_FILE="playwright-artifacts-pr/results.json" + if [ ! -f "$REPORT_FILE" ]; then + echo "Error: Playwright report file not found at $REPORT_FILE" + # Exit 0 here so the sticky comment can still be posted with an error message + echo "total=N/A" >> $GITHUB_OUTPUT + echo "passed=N/A" >> $GITHUB_OUTPUT + echo "failed=N/A" >> $GITHUB_OUTPUT + echo "skipped=N/A" >> $GITHUB_OUTPUT + echo "duration=N/A" >> $GITHUB_OUTPUT + echo "passrate=N/A" >> $GITHUB_OUTPUT + exit 0 # Allow workflow to continue for debugging info in PR comment + fi + + TOTAL=$(jq '.stats.total' "$REPORT_FILE") + PASSED=$(jq '.stats.expected' "$REPORT_FILE") + FAILED=$(jq '.stats.failures' "$REPORT_FILE") + SKIPPED=$(jq '.stats.skipped' "$REPORT_FILE") + DURATION=$(jq '.stats.duration' "$REPORT_FILE") # Playwright duration is already in ms PASS_RATE=$(awk "BEGIN{printf \"%.2f\", ($TOTAL==0)?0:($PASSED/$TOTAL)*100}") - for k in total passed failed skipped duration passrate; do - eval "echo \"$k=\${${k^^}}\" >> \$GITHUB_OUTPUT" - done + + echo "total=$TOTAL" >> $GITHUB_OUTPUT + echo "passed=$PASSED" >> $GITHUB_OUTPUT + echo "failed=$FAILED" >> $GITHUB_OUTPUT + echo "skipped=$SKIPPED" >> $GITHUB_OUTPUT + echo "duration=$DURATION" >> $GITHUB_OUTPUT + echo "passrate=$PASS_RATE" >> $GITHUB_OUTPUT #--------------------------------------------------- # 8 – ESLint (tests only) @@ -104,6 +215,7 @@ jobs: - name: Run ESLint on GUI tests shell: bash run: | + # Run ESLint and redirect output to a file. The `|| true` prevents the step from failing if lint issues are found. npx eslint "tests/**/*.{js,ts,tsx}" -f stylish > eslint-tests.txt || true - name: Upload ESLint report @@ -116,11 +228,16 @@ jobs: - name: Read ESLint report (preview) id: lint_summary run: | - echo 'summary<> $GITHUB_OUTPUT - head -n 20 eslint-tests.txt >> $GITHUB_OUTPUT - echo 'EOF' >> $GITHUB_OUTPUT - - + # Only read if the file exists, to prevent errors + if [ -f "eslint-tests.txt" ]; then + echo 'summary<> $GITHUB_OUTPUT + head -n 20 eslint-tests.txt >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT + else + echo 'summary<> $GITHUB_OUTPUT + echo 'ESLint report not generated or found.' >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT + fi #--------------------------------------------------- # 9 – Generate Suite→Spec Mermaid chart (flowchart.png) @@ -129,17 +246,24 @@ jobs: shell: bash run: | set -e + # The report file for the flowchart should be from the PR's run + REPORT_FILE="playwright-artifacts-pr/results.json" + if [ ! -f "$REPORT_FILE" ]; then + echo "Error: Playwright report file not found at $REPORT_FILE for flowchart generation. Skipping chart." + exit 0 # Exit successfully if report not found, but log message + fi + echo "graph TD" > flowchart.mmd jq -r ' .suites[] as $file | - ($file.title // "NO_FILE_TITLE") as $fileTitle | - $file.suites[]? as $suite | - ($suite.title // "NO_SUITE_TITLE") as $suiteTitle | - $suite.specs[]? as $spec | - ($spec.title // "NO_SPEC_TITLE") as $specTitle | - [$fileTitle, $suiteTitle, $specTitle] | @tsv - ' metrics.json | + ($file.title // "NO_FILE_TITLE") as $fileTitle | + $file.suites[]? as $suite | + ($suite.title // "NO_SUITE_TITLE") as $suiteTitle | + $suite.specs[]? as $spec | + ($spec.title // "NO_SPEC_TITLE") as $specTitle | + [$fileTitle, $suiteTitle, $specTitle] | @tsv + ' "$REPORT_FILE" | while IFS=$'\t' read -r fileTitle suiteTitle specTitle; do # Build unique, safe IDs by combining parent and child fileId=$(echo "$fileTitle" | tr -c 'A-Za-z0-9' '_' | sed 's/^_*\|_*$//g') @@ -168,7 +292,7 @@ jobs: ls -lh flowchart.png - - name: Show flowchart.mmd for debugging + - name: Show flowchart.mmd for debugging run: cat flowchart.mmd - name: Upload test-flow chart @@ -179,19 +303,19 @@ jobs: path: flowchart.png #--------------------------------------------------- - # 10 – Sticky PR comment + # 10 – Sticky PR comment (Simplified) #--------------------------------------------------- - name: Comment on PR with results uses: marocchino/sticky-pull-request-comment@v2 with: message: | ## Playwright Test Metrics - *Total:* **${{ steps.summary.outputs.total }}** - *Passed:* **${{ steps.summary.outputs.passed }}** - *Failed:* **${{ steps.summary.outputs.failed }}** + *Total:* **${{ steps.summary.outputs.total }}** + *Passed:* **${{ steps.summary.outputs.passed }}** + *Failed:* **${{ steps.summary.outputs.failed }}** *Skipped:* **${{ steps.summary.outputs.skipped }}** - Duration: **${{ steps.summary.outputs.duration }} ms** + Duration: **${{ steps.summary.outputs.duration }} ms** Pass Rate: **${{ steps.summary.outputs.passrate }} %** ## ESLint (GUI tests) @@ -199,7 +323,16 @@ jobs: ${{ steps.lint_summary.outputs.summary }} ``` - ## Test-Flow Chart + ## Test-Flow Chart Artifact: **test-flow-chart → flowchart.png** + --- + + **View Before/After Screenshots and Full Reports:** + You can download the following artifacts from this workflow run's summary page: + * **`playwright-artifacts-before`**: Contains screenshots and data from tests run against the `main` branch. + * **`playwright-artifacts-pr`**: Contains screenshots and data from tests run against this PR's branch. + * **`playwright-html-report`**: The full interactive Playwright HTML report for this PR's branch. + * **`eslint-test-report`**: Detailed ESLint results. + _Full run details:_ [link](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..ec60f667 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +# Ignore generated screenshots +published-screenshots/ + +# You might also want to ignore Node.js dependencies +node_modules/ + +# And Playwright test results/reports +playwright-report/ diff --git a/docs/pr/82/screenshots/example-domain.png b/docs/pr/82/screenshots/example-domain.png new file mode 100644 index 00000000..cb216b97 Binary files /dev/null and b/docs/pr/82/screenshots/example-domain.png differ diff --git a/docs/pr/82/screenshots/navigation.png b/docs/pr/82/screenshots/navigation.png new file mode 100644 index 00000000..2611ab42 Binary files /dev/null and b/docs/pr/82/screenshots/navigation.png differ diff --git a/playwright.config.js b/playwright.config.js index 3cb86394..9391f137 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -1,16 +1,86 @@ -const { defineConfig } = require('@playwright/test'); +// @ts-check +const { defineConfig, devices } = require('@playwright/test'); +/** + * Read environment variables from .env file. + * https://github.com/motdotla/dotenv + */ +// require('dotenv').config(); + +/** + * @see https://playwright.dev/docs/test-configuration + */ module.exports = defineConfig({ testDir: './tests', + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', // Default reporter, can be overridden by CLI + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { - headless: true, - screenshot: 'on', - trace: 'on', - video: 'off', - ignoreHTTPSErrors: true, + /* Base URL to use in actions like `await page.goto('/')`. */ + // baseURL: 'http://127.0.0.1:3000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + + // --- IMPORTANT: Dynamic screenshot output directory --- + // If PLAYWRIGHT_SCREENSHOT_DIR is set (by our workflow), use it as the base output directory. + // Otherwise, default to 'test-results' (Playwright's default for general artifacts). + // Playwright will create subdirectories within this 'output' path (e.g., 'output/test-name-platform/'). + screenshot: 'only-on-failure', // or 'on' if you want screenshots for all steps + video: 'on-first-retry', + output: process.env.PLAYWRIGHT_SCREENSHOT_DIR || 'test-results', // This is the key change! }, - reporter: [ - ['json', { outputFile: 'metrics.json' }], - ['html', { outputFile: 'report.html', open: 'never' }], + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + + { + name: 'firefox', + use: { ...devices['Desktop Firefox'] }, + }, + + { + name: 'webkit', + use: { ...devices['Desktop Safari'] }, + }, + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, ], -}); \ No newline at end of file + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // url: 'http://127.0.0.1:3000', + // reuseExistingServer: !process.env.CI, + // }, +}); diff --git a/published-screenshots/.gitkeep b/published-screenshots/.gitkeep new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/published-screenshots/.gitkeep @@ -0,0 +1 @@ + diff --git a/tests/example.spec.js b/tests/example.spec.js index c5ace5e9..56ae6593 100644 --- a/tests/example.spec.js +++ b/tests/example.spec.js @@ -1,16 +1,19 @@ +// tests/example.spec.js const { test, expect } = require('@playwright/test'); test.describe('Playwright Basic Workflow Tests', () => { test('should load example.com and verify title', async ({ page }) => { await page.goto('https://example.com'); await expect(page).toHaveTitle(/Example Domain/); - await page.screenshot({ path: 'playwright-report/example-domain.png' }); + // THIS LINE IS CRUCIAL FOR PBI 2.2 + await page.screenshot({ path: `${process.env.PLAYWRIGHT_SCREENSHOT_DIR || 'published-screenshots'}/example-domain.png` }); }); - + test('should find and click the More information link', async ({ page }) => { await page.goto('https://example.com'); await page.click('text=More information'); await expect(page).toHaveURL(/iana.org/); - await page.screenshot({ path: 'playwright-report/navigation.png' }); + // THIS LINE IS CRUCIAL FOR PBI 2.2 + await page.screenshot({ path: `${process.env.PLAYWRIGHT_SCREENSHOT_DIR || 'published-screenshots'}/navigation.png` }); }); -}); \ No newline at end of file +});