[E2E] Ensure version coverage for WooCommerce and PHP

.github/actions/e2e/run-log-tests/action.yml

@@ -1,4 +1,4 @@
-name: 'Run Tests'
+name: 'Run E2E Tests with Retry & Upload Logs'
 description: 'Runs E2E tests with retry & upload logs and screenshots'
 
 runs:
@@ -10,12 +10,17 @@ runs:
       shell: /bin/bash +e {0}
       run: |
         npm run test:e2e-ci
-
-        if [[ -f "$E2E_RESULT_FILEPATH" ]]; then
-          E2E_NUM_FAILED_TEST_SUITES=$(cat "$E2E_RESULT_FILEPATH" | jq '.stats["unexpected"]')
-          echo "FIRST_RUN_FAILED_TEST_SUITES=$(echo $E2E_NUM_FAILED_TEST_SUITES)" >> $GITHUB_OUTPUT
-          if [[ ${E2E_NUM_FAILED_TEST_SUITES} -gt 0 ]]; then
-            echo "::notice::${E2E_NUM_FAILED_TEST_SUITES} test suite(s) failed in the first run but we will try (it) them again in the second run."
+        RESULTS_JSON="$E2E_RESULT_FILEPATH"
+        if [[ -f "$RESULTS_JSON" ]]; then
+          # Build unique list of spec files with any failed/unexpected/timedOut/interrupted result and count them.
+          FAILED_SPECS_COUNT=$( jq -r '[ .. | objects | select(has("specs")) | .specs[]
+              | select( ( any(.tests[]?; (.status=="unexpected") or (.status=="failed")) )
+                        or ( any(.tests[]?.results[]?; (.status=="failed") or (.status=="timedOut") or (.status=="interrupted")) ) )
+              | .file ] | unique | length' "$RESULTS_JSON" )
+          echo "FIRST_RUN_FAILED_TEST_SUITES=$FAILED_SPECS_COUNT" >> $GITHUB_OUTPUT
+          echo "RESULTS_JSON=$RESULTS_JSON" >> $GITHUB_OUTPUT
+          if [[ ${FAILED_SPECS_COUNT} -gt 0 ]]; then
+            echo "::notice::${FAILED_SPECS_COUNT} spec file(s) failed in the first run. We will re-run only the failed specs."
             exit 0
           fi
         else
@@ -27,9 +32,32 @@ runs:
     - name: Re-try Failed E2E Files
       if: ${{ steps.first_run_e2e_tests.outputs.FIRST_RUN_FAILED_TEST_SUITES > 0  }}
       shell: bash
-      # Filter failed E2E files from the result JSON file, and re-run them.
       run: |
-        npm run test:e2e-ci $(cat $E2E_RESULT_FILEPATH | jq -r '[.suites[] | (if has("suites") then .suites[] | .specs[] else .specs[] end) | select(.tests[].status == "unexpected") | .file] | unique | .[]')
+        set -e
+        RESULTS_JSON="${{ steps.first_run_e2e_tests.outputs.RESULTS_JSON }}"
+        echo "Using results file: $RESULTS_JSON"
+
+        # Build a unique list of spec files that had unexpected/failed outcomes.
+        # This uses a recursive jq search to find any node with a 'specs' array, then selects specs
+        # where any test result is marked unexpected/failed.
+        mapfile -t FAILED_SPECS < <( jq -r '[ .. | objects | select(has("specs")) | .specs[]
+          | select( ( any(.tests[]?; (.status=="unexpected") or (.status=="failed")) )
+            or ( any(.tests[]?.results[]?; (.status=="failed") or (.status=="timedOut") or (.status=="interrupted")) ) )
+          | .file ] | unique | .[]' "$RESULTS_JSON" )
+
+        if [[ ${#FAILED_SPECS[@]} -eq 0 ]]; then
+          echo "::notice::No failed specs found in results file. Re-running full suite instead."
+          npm run test:e2e-ci
+          exit 0
+        fi
+
+        echo "Retrying failed specs (${#FAILED_SPECS[@]}):"
+        for f in "${FAILED_SPECS[@]}"; do
+          echo " - $f"
+        done
+
+        # Re-run only the failed spec files using Playwright directly (npm script does not accept args)
+        npx playwright test --config=tests/e2e/playwright.config.ts --grep-invert @todo "${FAILED_SPECS[@]}"
 
     # Archive screenshots if any
     - name: Archive e2e test screenshots & logs
@@ -41,5 +69,6 @@ runs:
             playwright-report/
             tests/e2e/test-results
             ${{ env.E2E_RESULT_FILEPATH }}
+            ${{ steps.first_run_e2e_tests.outputs.RESULTS_JSON }}
           if-no-files-found: ignore
           retention-days: 14

.github/actions/setup-php/action.yml

@@ -1,5 +1,5 @@
 name: "Set up PHP"
-description: "Extracts the required PHP version from plugin file and uses it to build PHP."
+description: "Extracts the required PHP version from plugin file and uses it to build PHP. Can be overridden with E2E_PHP_VERSION environment variable."
 
 runs:
   using: composite
@@ -14,6 +14,6 @@ runs:
     - name: "Setup PHP"
       uses: shivammathur/setup-php@v2
       with:
-        php-version: ${{ steps.get_min_php_version.outputs.MIN_PHP_VERSION }}
+        php-version: ${{ env.E2E_PHP_VERSION || steps.get_min_php_version.outputs.MIN_PHP_VERSION }}
         tools: composer
         coverage: none

.github/scripts/README.md

@@ -0,0 +1,111 @@
+# GitHub Actions Scripts
+
+This directory contains scripts used by GitHub Actions workflows for dynamic version management and matrix generation.
+
+## Scripts
+
+### `generate-wc-matrix.sh`
+
+Generates the WooCommerce version matrix for E2E tests with dynamic version resolution and optimized PHP version strategy.
+
+**Usage:**
+
+```bash
+.github/scripts/generate-wc-matrix.sh
+```
+
+**Output:**
+Single JSON object containing versions array and metadata:
+
+```json
+{
+  "versions": [
+    "7.7.0",
+    "9.9.5",
+    "latest",
+    "10.1.0-rc.2"
+  ],
+  "metadata": {
+    "l1_version": "9.9.5",
+    "rc_version": "10.1.0-rc.2",
+    "beta_version": null
+  }
+}
+```
+
+**Features:**
+
+- Fetches latest WC version from WordPress.org API
+- Dynamically calculates L-1 version (latest stable in previous major branch)
+- Includes only L-1 and current major versions (skipping intermediate versions)
+- Dynamically resolves beta and RC versions from current major branch
+- Outputs structured JSON for easy parsing
+- Skips beta versions when not available
+- Provides debug output to stderr for troubleshooting
+
+## Matrix Generation Strategy
+
+### PHP Version Strategy
+
+The workflow uses an optimized PHP version strategy to reduce job count while maintaining comprehensive coverage:
+
+- **WC 7.7.0**: PHP 7.3 (legacy support - minimum required version)
+- **WC L-1**: PHP 8.3 (stable)
+- **WC latest**: PHP 8.3 (stable)
+- **WC beta**: PHP 8.3 (stable) - only when available
+- **WC rc**: PHP 8.4 (latest)
+
+### Version Resolution
+
+- **L-1 Version**: Extracted from JSON metadata
+- **Beta Version**: Extracted from JSON metadata, only included when available
+- **RC Version**: Always included - extracted from JSON metadata or falls back to string "rc"
+
+## How It Works
+
+### Script Execution
+
+1. Fetches the latest WooCommerce version from `https://api.wordpress.org/plugins/info/1.0/woocommerce.json`
+2. Dynamically calculates the L-1 version by finding the latest stable version in the previous major branch
+3. Fetches beta and RC versions from the current major branch only
+4. Outputs JSON object to stdout for matrix generation
+
+### Workflow Integration
+
+1. Script runs and outputs structured JSON with versions and metadata
+2. Workflow extracts specific versions using standard JSON parsing
+3. Workflow builds optimized matrix with selective PHP version testing
+4. Matrix includes only necessary combinations to reduce job count
+
+### Version Extraction
+
+```bash
+# Get script result
+SCRIPT_RESULT=$( .github/scripts/generate-wc-matrix.sh )
+
+# Extract versions and metadata using jq
+WC_VERSIONS=$(echo "$SCRIPT_RESULT" | jq -r '.versions')
+L1_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.l1_version')
+RC_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.rc_version')
+BETA_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.beta_version')
+```
+
+## Dependencies
+
+- `curl`: For API requests
+- `jq`: For JSON parsing and array generation
+- `bash`: For script execution
+
+## Error Handling
+
+- Scripts use `set -e` to exit on any error
+- Version extraction includes validation checks
+- Graceful handling of missing beta versions
+- If the API is unavailable or returns unexpected data, the workflow will fail gracefully
+
+## Future Considerations
+
+- Automatically adapts to new WooCommerce releases
+- Will include beta versions when they become available
+- Supports L-2 policy implementation if needed
+- Maintains business continuity with WC 7.7.0 support

.github/scripts/generate-wc-matrix.sh

@@ -0,0 +1,157 @@
+#!/bin/bash
+
+# Script to dynamically generate WooCommerce version matrix for L-1 policy
+# This script fetches the latest WC version and calculates the L-1 version
+
+set -e
+
+# Function to get the latest WooCommerce version from WordPress.org API
+get_latest_wc_version() {
+    curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | jq -r '.version'
+}
+
+# Function to get the latest stable version for a specific major version
+get_latest_stable_for_major() {
+    local major_version=$1
+    curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | \
+    jq -r --arg major "$major_version" '.versions | with_entries(select(.key | startswith($major + ".") and (contains("-") | not))) | keys | sort_by( . | split(".") | map(tonumber) ) | last'
+}
+
+# Function to get the L-1 version (previous major version's latest stable)
+get_l1_version() {
+    local latest_version=$1
+    local major_version=$(echo "$latest_version" | cut -d. -f1)
+    local l1_major=$((major_version - 1))
+    get_latest_stable_for_major "$l1_major"
+}
+
+# Function to get specific major versions' latest stable
+get_major_versions_latest() {
+    local latest_version=$1
+    local major_version=$(echo "$latest_version" | cut -d. -f1)
+    local versions=()
+
+    # Dynamically calculate L-1 major version
+    local l1_major=$((major_version - 1))
+
+    # Only get L-1 version (previous major) and current major
+    # Skip intermediate major versions as they don't align with L-1 policy
+    for ((i=l1_major; i<=major_version; i++)); do
+        latest_stable=$(get_latest_stable_for_major "$i")
+        if [[ -n "$latest_stable" && "$latest_stable" != "null" ]]; then
+            versions+=("$latest_stable")
+        fi
+    done
+
+    echo "${versions[@]}"
+}
+
+# Function to get the latest RC version from WordPress.org API
+get_latest_rc_version() {
+    curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | \
+    jq -r '.versions | with_entries(select(.key|match("rc";"i"))) | keys | sort_by( . | split("-")[0] | split(".") | map(tonumber) ) | last'
+}
+
+# Function to get the latest beta version from WordPress.org API
+get_latest_beta_version() {
+    local latest_version=$1
+    local major_version=$(echo "$latest_version" | cut -d. -f1)
+    curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | \
+    jq -r --arg major "$major_version" '.versions | with_entries(select(.key | startswith($major + ".") and contains("beta"))) | keys | sort_by( . | split("-")[0] | split(".") | map(tonumber) ) | last'
+}
+
+# Get the latest WooCommerce version
+echo "Fetching latest WooCommerce version..." >&2
+LATEST_WC_VERSION=$(get_latest_wc_version)
+echo "Latest WC version: $LATEST_WC_VERSION" >&2
+
+# Get the L-1 version
+L1_VERSION=$(get_l1_version "$LATEST_WC_VERSION")
+echo "L-1 version: $L1_VERSION" >&2
+
+# Get major versions latest stable
+MAJOR_VERSIONS=($(get_major_versions_latest "$LATEST_WC_VERSION"))
+echo "Major versions latest stable: ${MAJOR_VERSIONS[*]}" >&2
+
+# Get latest RC and beta versions
+echo "Fetching latest RC and beta versions..." >&2
+LATEST_RC_VERSION=$(get_latest_rc_version)
+LATEST_BETA_VERSION=$(get_latest_beta_version "$LATEST_WC_VERSION")
+echo "Latest RC version: $LATEST_RC_VERSION" >&2
+echo "Latest beta version: $LATEST_BETA_VERSION" >&2
+
+# Build the version array
+VERSIONS=("7.7.0")  # Keep for business reasons (significant TPV)
+
+# Add major versions latest stable (excluding current major since we'll use 'latest')
+for version in "${MAJOR_VERSIONS[@]}"; do
+    # Skip the current major version since we'll use 'latest' instead
+    if [[ "$version" != "$LATEST_WC_VERSION" ]]; then
+        VERSIONS+=("$version")
+    fi
+done
+
+# Add latest, beta, rc (with actual versions)
+VERSIONS+=("latest")
+if [[ -n "$LATEST_BETA_VERSION" && "$LATEST_BETA_VERSION" != "null" ]]; then
+    VERSIONS+=("$LATEST_BETA_VERSION")
+    echo "Including beta version: $LATEST_BETA_VERSION" >&2
+else
+    echo "No beta version available, skipping beta tests" >&2
+fi
+
+# Decide whether to include RC: only include if RC base version (without suffix) is strictly greater than the latest stable.
+INCLUDED_RC_VERSION=""
+if [[ -n "$LATEST_RC_VERSION" && "$LATEST_RC_VERSION" != "null" ]]; then
+    RC_BASE="${LATEST_RC_VERSION%%-*}"
+    # Compare RC_BASE vs LATEST_WC_VERSION using sort -V
+    HIGHEST=$(printf '%s\n%s\n' "$RC_BASE" "$LATEST_WC_VERSION" | sort -V | tail -n1)
+    if [[ "$HIGHEST" == "$RC_BASE" && "$RC_BASE" != "$LATEST_WC_VERSION" ]]; then
+        INCLUDED_RC_VERSION="$LATEST_RC_VERSION"
+        VERSIONS+=("$LATEST_RC_VERSION")
+        echo "Including RC version: $LATEST_RC_VERSION (base $RC_BASE > latest $LATEST_WC_VERSION)" >&2
+    else
+        echo "Skipping RC version $LATEST_RC_VERSION because stable $LATEST_WC_VERSION is already released for this line." >&2
+    fi
+else
+    echo "No RC version available, skipping rc tests" >&2
+fi
+
+# Validate versions before output
+if [[ -z "$L1_VERSION" || "$L1_VERSION" == "null" ]]; then
+    echo "Error: Could not extract L-1 version" >&2
+    exit 1
+fi
+
+if [[ ! "$L1_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    echo "Error: Invalid L-1 version: $L1_VERSION" >&2
+    exit 1
+fi
+
+# RC is optional; do not fail if not present or skipped
+
+# Only validate beta if it's available
+if [[ -n "$LATEST_BETA_VERSION" && "$LATEST_BETA_VERSION" != "null" ]]; then
+    if [[ ! "$LATEST_BETA_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+        echo "Error: Invalid beta version: $LATEST_BETA_VERSION" >&2
+        exit 1
+    fi
+fi
+
+# Convert to JSON array and output only the JSON (no extra whitespace or newlines)
+# Output a single JSON object with both versions and metadata
+RESULT=$(jq -n \
+    --argjson versions "$(printf '%s\n' "${VERSIONS[@]}" | jq -R . | jq -s .)" \
+    --arg l1_version "$L1_VERSION" \
+    --arg rc_version "${INCLUDED_RC_VERSION}" \
+    --arg beta_version "${LATEST_BETA_VERSION}" \
+    '{
+        versions: $versions,
+        metadata: {
+            l1_version: $l1_version,
+            rc_version: (if ($rc_version // "") == "" or ($rc_version == "null") then null else $rc_version end),
+            beta_version: (if ($beta_version // "") == "" or ($beta_version == "null") then null else $beta_version end)
+        }
+    }')
+
+echo "$RESULT"