Feature/pr screenshots

.github/workflows/playwright.yml

@@ -9,18 +9,19 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
-
     permissions:
-      contents: read
-      pull-requests: write
+      contents: write # Still need this for Prettier patch, but not for docs/pr push anymore
+      pull-requests: write # Still need this for the sticky PR comment
 
     steps:
     #---------------------------------------------------
-    # 0 – Checkout
+    # 0 – Checkout first
     #---------------------------------------------------
     - name: Checkout code
       uses: actions/checkout@v3
-      with: { fetch-depth: 0 }
+      with:
+        fetch-depth: 0 # Needed to fetch origin/main later
+        ref: ${{ github.event.pull_request.head.ref }} # Checkout PR branch first
 
     #---------------------------------------------------
     # 1 – reviewdog CLI
@@ -59,51 +60,162 @@ jobs:
     - name: Install dependencies
       run: npm install
 
-    #---------------------------------------------------
-    # 4 – Install Playwright browsers
-    #---------------------------------------------------
+    # --- IMPORTANT: Install Playwright browsers early ---
     - name: Install Playwright and browsers
       run: npx playwright install --with-deps
 
+    ## CORE LOGIC FOR BEFORE/AFTER COMPARISON ##
+
+    # Step 1: Save current PR's test files (before checking out main's version)
+    - name: Save PR test files
+      run: |
+        mkdir -p tests_pr_backup/
+        cp -r tests/ tests_pr_backup/ # Copy the 'tests' directory and its contents
+        echo "Contents of tests_pr_backup/:"
+        ls -la tests_pr_backup/ # Debugging: verify backup content
+
+    # Step 2: Checkout base branch version of test files (from friend's suggestion)
+    - name: Checkout base version of test files
+      run: |
+        git fetch origin main # Ensure origin/main is up-to-date
+        # This will put 'tests/' from origin/main into your working directory, overwriting current 'tests/'
+        git checkout --force origin/main -- tests/
+        echo "Contents of tests/ after checking out main's version:"
+        ls -la tests/ # Debugging
+
+    # Step 3: Run Playwright on base version (for "before" view)
+    - name: Run Playwright "before" tests on Main's version
+      env:
+        # This ENV variable is picked up by playwright.config.js for the 'output' path
+        PLAYWRIGHT_SCREENSHOT_DIR: playwright-artifacts-before
+      run: |
+        mkdir -p $PLAYWRIGHT_SCREENSHOT_DIR # Ensure the directory exists
+        echo "Running Playwright 'before' tests, outputting to: $PLAYWRIGHT_SCREENSHOT_DIR"
+        # Using --reporter=json to ensure results.json is generated here for potential future use
+        # Using --debug for more verbose output to diagnose why tests might not be running
+        npx playwright test --output=$PLAYWRIGHT_SCREENSHOT_DIR --reporter=json --debug || true
+        # The '|| true' allows the workflow to continue even if Playwright fails, so we can inspect artifacts
+
+        # Debugging: List contents after run
+        echo "Contents of $PLAYWRIGHT_SCREENSHOT_DIR after 'before' tests:"
+        ls -laR $PLAYWRIGHT_SCREENSHOT_DIR || true # List even if directory is empty or missing
+
+    # Step 4: Restore PR version of tests
+    - name: Restore PR test files
+      run: |
+        rm -rf tests/ # Remove tests/ from main's version
+        mv tests_pr_backup/ tests/ # Move the backup back to become the PR's tests/
+        echo "Contents of tests/ after restoring PR's version:"
+        ls -la tests/ # Debugging
+
     #---------------------------------------------------
-    # 5 – Run Playwright tests
+    # 5 – Run Playwright tests (This now runs on the PR branch's tests)
     #---------------------------------------------------
-    - name: Run Playwright tests
-      run: npx playwright test
+    - name: Run Playwright tests (PR version)
+      env:
+        # This ENV variable is picked up by playwright.config.js for the 'output' path
+        PLAYWRIGHT_SCREENSHOT_DIR: playwright-artifacts-pr
+      run: |
+        mkdir -p $PLAYWRIGHT_SCREENSHOT_DIR # Ensure the directory exists
+        echo "Running Playwright 'PR' tests, outputting to: $PLAYWRIGHT_SCREENSHOT_DIR"
+        npx playwright test --output=$PLAYWRIGHT_SCREENSHOT_DIR --reporter=html,json --debug || true
+        # The '|| true' allows the workflow to continue even if Playwright fails
+
+        # Debugging: List contents after run
+        echo "Contents of $PLAYWRIGHT_SCREENSHOT_DIR after 'PR' tests:"
+        ls -laR $PLAYWRIGHT_SCREENSHOT_DIR || true
+
+    # --- Debugging: Verify Playwright Report Existence AFTER tests ---
+    - name: Verify Playwright Report Existence (PR version)
+      run: |
+        # Check in the explicitly set output directory
+        REPORT_FILE="playwright-artifacts-pr/results.json"
+        echo "Checking for report file at: $REPORT_FILE"
+        if [ -f "$REPORT_FILE" ]; then
+          echo "$REPORT_FILE exists."
+        else
+          echo "ERROR: $REPORT_FILE DOES NOT EXIST. Listing parent directory contents:"
+          ls -la playwright-artifacts-pr/ || true # List even if directory is empty or missing
+          exit 1
+        fi
+    # --- End Debugging ---
+
+    # --- START OF ARTIFACT UPLOADS ---
+
+    - name: Upload 'Before' Playwright Artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: playwright-artifacts-before
+        path: playwright-artifacts-before/
+        retention-days: 7 # Optional: How long to keep the artifact
+
+    - name: Upload 'After' Playwright Artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: playwright-artifacts-pr
+        path: playwright-artifacts-pr/
+        retention-days: 7 # Optional: How long to keep the artifact
+
+    # --- END OF ARTIFACT UPLOADS ---
+
+    # Removed: Prepare PR Screenshots for GitHub Pages (After)
+    # Removed: Prepare PR Screenshots for GitHub Pages (Before)
+    # Removed: Commit and Push PR Screenshots to GitHub Pages
+    # Removed: Generate Screenshot Markdown for PR Comment
+
 
     #---------------------------------------------------
-    # 6 – Upload HTML report
+    # 6 – Upload HTML report (Still useful, points to the PR run's html folder)
     #---------------------------------------------------
     - name: Upload HTML report
       if: always()
       uses: actions/upload-artifact@v4
       with:
-        name: playwright-report
-        path: playwright-report/
+        name: playwright-html-report
+        path: playwright-artifacts-pr/html/ # Playwright's HTML report is usually in a subfolder like 'html/' within the output directory
 
     #---------------------------------------------------
     # 7 – Extract test summary
     #---------------------------------------------------
     - name: Extract test summary
       id: summary
       run: |
-        REPORT=metrics.json
-        TOTAL=$(jq '[..|objects|select(has("status"))] | length' "$REPORT")
-        PASSED=$(jq '[..|objects|select(.status?=="expected")] | length' "$REPORT")
-        FAILED=$(jq '[..|objects|select(.status?=="failed")]   | length' "$REPORT")
-        SKIPPED=$(jq '[..|objects|select(.status?=="skipped")] | length' "$REPORT")
-        DURATION=$(jq '.stats.duration*1000|floor' "$REPORT")
+        # Playwright's JSON report contains the summary statistics.
+        # It's now in the 'playwright-artifacts-pr/' directory.
+        REPORT_FILE="playwright-artifacts-pr/results.json"
+        if [ ! -f "$REPORT_FILE" ]; then
+          echo "Error: Playwright report file not found at $REPORT_FILE"
+          # Exit 0 here so the sticky comment can still be posted with an error message
+          echo "total=N/A" >> $GITHUB_OUTPUT
+          echo "passed=N/A" >> $GITHUB_OUTPUT
+          echo "failed=N/A" >> $GITHUB_OUTPUT
+          echo "skipped=N/A" >> $GITHUB_OUTPUT
+          echo "duration=N/A" >> $GITHUB_OUTPUT
+          echo "passrate=N/A" >> $GITHUB_OUTPUT
+          exit 0 # Allow workflow to continue for debugging info in PR comment
+        fi
+
+        TOTAL=$(jq '.stats.total' "$REPORT_FILE")
+        PASSED=$(jq '.stats.expected' "$REPORT_FILE")
+        FAILED=$(jq '.stats.failures' "$REPORT_FILE")
+        SKIPPED=$(jq '.stats.skipped' "$REPORT_FILE")
+        DURATION=$(jq '.stats.duration' "$REPORT_FILE") # Playwright duration is already in ms
         PASS_RATE=$(awk "BEGIN{printf \"%.2f\", ($TOTAL==0)?0:($PASSED/$TOTAL)*100}")
-        for k in total passed failed skipped duration passrate; do
-          eval "echo \"$k=\${${k^^}}\" >> \$GITHUB_OUTPUT"
-        done
+
+        echo "total=$TOTAL" >> $GITHUB_OUTPUT
+        echo "passed=$PASSED" >> $GITHUB_OUTPUT
+        echo "failed=$FAILED" >> $GITHUB_OUTPUT
+        echo "skipped=$SKIPPED" >> $GITHUB_OUTPUT
+        echo "duration=$DURATION" >> $GITHUB_OUTPUT
+        echo "passrate=$PASS_RATE" >> $GITHUB_OUTPUT
 
     #---------------------------------------------------
     # 8 – ESLint (tests only)
     #---------------------------------------------------
     - name: Run ESLint on GUI tests
       shell: bash
       run: |
+        # Run ESLint and redirect output to a file. The `|| true` prevents the step from failing if lint issues are found.
         npx eslint "tests/**/*.{js,ts,tsx}" -f stylish > eslint-tests.txt || true
 
     - name: Upload ESLint report
@@ -116,11 +228,16 @@ jobs:
     - name: Read ESLint report (preview)
       id: lint_summary
       run: |
-        echo 'summary<<EOF' >> $GITHUB_OUTPUT
-        head -n 20 eslint-tests.txt >> $GITHUB_OUTPUT
-        echo 'EOF' >> $GITHUB_OUTPUT
-
-
+        # Only read if the file exists, to prevent errors
+        if [ -f "eslint-tests.txt" ]; then
+          echo 'summary<<EOF' >> $GITHUB_OUTPUT
+          head -n 20 eslint-tests.txt >> $GITHUB_OUTPUT
+          echo 'EOF' >> $GITHUB_OUTPUT
+        else
+          echo 'summary<<EOF' >> $GITHUB_OUTPUT
+          echo 'ESLint report not generated or found.' >> $GITHUB_OUTPUT
+          echo 'EOF' >> $GITHUB_OUTPUT
+        fi
 
     #---------------------------------------------------
     # 9 – Generate Suite→Spec Mermaid chart (flowchart.png)
@@ -129,17 +246,24 @@ jobs:
       shell: bash
       run: |
         set -e
+        # The report file for the flowchart should be from the PR's run
+        REPORT_FILE="playwright-artifacts-pr/results.json"
+        if [ ! -f "$REPORT_FILE" ]; then
+          echo "Error: Playwright report file not found at $REPORT_FILE for flowchart generation. Skipping chart."
+          exit 0 # Exit successfully if report not found, but log message
+        fi
+
         echo "graph TD" > flowchart.mmd
 
         jq -r '
           .suites[] as $file |
-            ($file.title // "NO_FILE_TITLE") as $fileTitle |
-            $file.suites[]? as $suite |
-              ($suite.title // "NO_SUITE_TITLE") as $suiteTitle |
-              $suite.specs[]? as $spec |
-                ($spec.title // "NO_SPEC_TITLE") as $specTitle |
-                [$fileTitle, $suiteTitle, $specTitle] | @tsv
-        ' metrics.json |
+          ($file.title // "NO_FILE_TITLE") as $fileTitle |
+          $file.suites[]? as $suite |
+            ($suite.title // "NO_SUITE_TITLE") as $suiteTitle |
+            $suite.specs[]? as $spec |
+              ($spec.title // "NO_SPEC_TITLE") as $specTitle |
+              [$fileTitle, $suiteTitle, $specTitle] | @tsv
+        ' "$REPORT_FILE" |
         while IFS=$'\t' read -r fileTitle suiteTitle specTitle; do
           # Build unique, safe IDs by combining parent and child
           fileId=$(echo "$fileTitle" | tr -c 'A-Za-z0-9' '_' | sed 's/^_*\|_*$//g')
@@ -168,7 +292,7 @@ jobs:
 
         ls -lh flowchart.png
 
-    - name: Show flowchart.mmd for debugging  
+    - name: Show flowchart.mmd for debugging
       run: cat flowchart.mmd
 
     - name: Upload test-flow chart
@@ -179,27 +303,36 @@ jobs:
         path: flowchart.png
 
     #---------------------------------------------------
-    # 10 – Sticky PR comment
+    # 10 – Sticky PR comment (Simplified)
     #---------------------------------------------------
     - name: Comment on PR with results
       uses: marocchino/sticky-pull-request-comment@v2
       with:
         message: |
           ## Playwright Test Metrics
-          *Total:* **${{ steps.summary.outputs.total }}**  
-          *Passed:* **${{ steps.summary.outputs.passed }}**  
-          *Failed:* **${{ steps.summary.outputs.failed }}**  
+          *Total:* **${{ steps.summary.outputs.total }}**
+          *Passed:* **${{ steps.summary.outputs.passed }}**
+          *Failed:* **${{ steps.summary.outputs.failed }}**
           *Skipped:* **${{ steps.summary.outputs.skipped }}**
 
-          Duration: **${{ steps.summary.outputs.duration }} ms**  
+          Duration: **${{ steps.summary.outputs.duration }} ms**
           Pass Rate: **${{ steps.summary.outputs.passrate }} %**
 
           ## ESLint (GUI tests)
           ```
           ${{ steps.lint_summary.outputs.summary }}
           ```
 
-          ## Test-Flow Chart  
+          ## Test-Flow Chart
           Artifact: **test-flow-chart → flowchart.png**
 
+          ---
+
+          **View Before/After Screenshots and Full Reports:**
+          You can download the following artifacts from this workflow run's summary page:
+          * **`playwright-artifacts-before`**: Contains screenshots and data from tests run against the `main` branch.
+          * **`playwright-artifacts-pr`**: Contains screenshots and data from tests run against this PR's branch.
+          * **`playwright-html-report`**: The full interactive Playwright HTML report for this PR's branch.
+          * **`eslint-test-report`**: Detailed ESLint results.
+
           _Full run details:_ [link](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})

.gitignore

@@ -0,0 +1,8 @@
+# Ignore generated screenshots
+published-screenshots/
+
+# You might also want to ignore Node.js dependencies
+node_modules/
+
+# And Playwright test results/reports
+playwright-report/