Merge branch 'dev/woopmnt-5249-e2e-ensure-version-coverage-for-woocommerce-and-php' into dev/woopmnt-5251-e2e-investigate-and-re-enable-woocommerce-blocks-e2e-tests

Author: mgascam

.github/actions/e2e/run-log-tests/action.yml

@@ -4,32 +4,14 @@ description: 'Runs E2E tests with retry & upload logs and screenshots'
 runs:
   using: "composite"
   steps:
-    - name: First Run E2E Tests
-      id: first_run_e2e_tests
-      # Use +e to trap errors when running E2E tests.
-      shell: /bin/bash +e {0}
-      run: |
-        npm run test:e2e-ci
-        
-        if [[ -f "$E2E_RESULT_FILEPATH" ]]; then
-          E2E_NUM_FAILED_TEST_SUITES=$(cat "$E2E_RESULT_FILEPATH" | jq '.stats["unexpected"]')
-          echo "FIRST_RUN_FAILED_TEST_SUITES=$(echo $E2E_NUM_FAILED_TEST_SUITES)" >> $GITHUB_OUTPUT
-          if [[ ${E2E_NUM_FAILED_TEST_SUITES} -gt 0 ]]; then
-            echo "::notice::${E2E_NUM_FAILED_TEST_SUITES} test suite(s) failed in the first run but we will try (it) them again in the second run."
-            exit 0
-          fi
-        else
-          echo "FIRST_RUN_FAILED_TEST_SUITES=0" >> $GITHUB_OUTPUT
-          exit 0
-        fi
-
-    # Retry failed E2E tests
-    - name: Re-try Failed E2E Files
-      if: ${{ steps.first_run_e2e_tests.outputs.FIRST_RUN_FAILED_TEST_SUITES > 0  }}
+    - name: Run E2E Tests
       shell: bash
-      # Filter failed E2E files from the result JSON file, and re-run them.
       run: |
-        npm run test:e2e-ci $(cat $E2E_RESULT_FILEPATH | jq -r '[.suites[] | (if has("suites") then .suites[] | .specs[] else .specs[] end) | select(.tests[].status == "unexpected") | .file] | unique | .[]')
+        set -e
+        npm run test:e2e-ci || {
+          echo "::notice::Some tests failed, retrying only failed tests with --last-failed flag"
+          npx playwright test --config=tests/e2e/playwright.config.ts --grep-invert @todo --last-failed
+        }
 
     # Archive screenshots if any
     - name: Archive e2e test screenshots & logs

.github/scripts/generate-wc-matrix.sh

@@ -105,6 +105,30 @@ else
     VERSIONS+=("rc")  # Fallback to string if no RC found
 fi
 
+# Validate versions before output
+if [[ -z "$L1_VERSION" || "$L1_VERSION" == "null" ]]; then
+    echo "Error: Could not extract L-1 version" >&2
+    exit 1
+fi
+
+if [[ ! "$L1_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    echo "Error: Invalid L-1 version: $L1_VERSION" >&2
+    exit 1
+fi
+
+if [[ -z "$LATEST_RC_VERSION" || "$LATEST_RC_VERSION" == "null" ]]; then
+    echo "Error: Could not extract RC version" >&2
+    exit 1
+fi
+
+# Only validate beta if it's available
+if [[ -n "$LATEST_BETA_VERSION" && "$LATEST_BETA_VERSION" != "null" ]]; then
+    if [[ ! "$LATEST_BETA_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+        echo "Error: Invalid beta version: $LATEST_BETA_VERSION" >&2
+        exit 1
+    fi
+fi
+
 # Convert to JSON array and output only the JSON (no extra whitespace or newlines)
 # Output a single JSON object with both versions and metadata
 RESULT=$(jq -n \

.github/workflows/e2e-pull-request.yml

@@ -58,22 +58,8 @@ jobs:
           SCRIPT_RESULT=$( .github/scripts/generate-wc-matrix.sh )
 
           # Extract versions and metadata from JSON
-          WC_VERSIONS=$(echo "$SCRIPT_RESULT" | jq -r '.versions')
           L1_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.l1_version')
 
-          # Validate that we got proper versions
-          if [[ -z "$L1_VERSION" || "$L1_VERSION" == "null" ]]; then
-            echo "Error: Could not extract L-1 version from script output" >&2
-            echo "Script result: $SCRIPT_RESULT" >&2
-            exit 1
-          fi
-
-          if [[ ! "$L1_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-            echo "Error: Invalid L-1 version extracted: $L1_VERSION" >&2
-            echo "WC_VERSIONS: $WC_VERSIONS" >&2
-            exit 1
-          fi
-
           echo "Using L-1 version: $L1_VERSION" >&2
 
           # Create PR matrix with L-1 and latest versions

.github/workflows/e2e-test.yml

@@ -44,49 +44,17 @@ jobs:
           SCRIPT_RESULT=$( .github/scripts/generate-wc-matrix.sh )
 
           # Extract versions and metadata from JSON
-          WC_VERSIONS=$(echo "$SCRIPT_RESULT" | jq -r '.versions')
           L1_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.l1_version')
           RC_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.rc_version')
           BETA_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.beta_version')
 
-          # Check if beta version is actually a version (not null or empty)
-          if [[ "$BETA_VERSION" == "null" || -z "$BETA_VERSION" ]]; then
+          if [[ "$BETA_VERSION" == "null" ]]; then
             BETA_VERSION=""
-            echo "No beta version available" >&2
-          fi
-
-          # Validate that we got proper versions
-          if [[ -z "$L1_VERSION" || "$L1_VERSION" == "null" ]]; then
-            echo "Error: Could not extract L-1 version from script output" >&2
-            echo "Script result: $SCRIPT_RESULT" >&2
-            exit 1
-          fi
-
-          if [[ ! "$L1_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-            echo "Error: Invalid L-1 version extracted: $L1_VERSION" >&2
-            echo "WC_VERSIONS: $WC_VERSIONS" >&2
-            exit 1
-          fi
-
-          # Validate RC version
-          if [[ -z "$RC_VERSION" || "$RC_VERSION" == "null" ]]; then
-            echo "Error: Could not extract RC version from script output" >&2
-            echo "WC_VERSIONS: $WC_VERSIONS" >&2
-            exit 1
-          fi
-
-          # Only validate beta if it's available
-          if [[ -n "$BETA_VERSION" && "$BETA_VERSION" != "null" ]]; then
-            if [[ ! "$BETA_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+ ]]; then
-              echo "Error: Invalid beta version extracted: $BETA_VERSION" >&2
-              echo "WC_VERSIONS: $WC_VERSIONS" >&2
-              exit 1
-            fi
           fi
 
           echo "Using L-1 version: $L1_VERSION" >&2
           echo "Using RC version: $RC_VERSION" >&2
-          if [[ -n "$BETA_VERSION" && "$BETA_VERSION" != "null" ]]; then
+          if [[ -n "$BETA_VERSION" ]]; then
             echo "Using beta version: $BETA_VERSION" >&2
           else
             echo "No beta version available" >&2

.github/workflows/release-code-freeze.yml

@@ -86,6 +86,7 @@ jobs:
       RELEASE_VERSION: ${{ needs.check-code-freeze.outputs.nextReleaseVersion }}
       RELEASE_DATE: ${{ needs.check-code-freeze.outputs.nextReleaseDate }}
       RELEASE_PR_ID: ${{ needs.create-release-pr.outputs.release-pr-id }}
+      GITHUB_RUN_ID: ${{ github.run_id }}
     steps:
       - name: "Slack ping"
         uses: slackapi/[email protected]
@@ -116,7 +117,7 @@ jobs:
                   "type": "section",
                   "text": {
                     "type": "mrkdwn",
-                    "text": "• Created a release branch `release/${{ env.RELEASE_VERSION }}` \n • Raised a <https://github.com/Automattic/woocommerce-payments/pull/${{ env.RELEASE_PR_ID }}|Pull Request> to `trunk`\n • Built a <https://github.com/Automattic/woocommerce-payments/actions/runs/$GITHUB_RUN_ID|zip file and ran smoke tests> against it"
+                    "text": "• Created a release branch `release/${{ env.RELEASE_VERSION }}` \n • Raised a <https://github.com/Automattic/woocommerce-payments/pull/${{ env.RELEASE_PR_ID }}|Pull Request> to `trunk`\n • Built a <https://github.com/Automattic/woocommerce-payments/actions/runs/${{ env.GITHUB_RUN_ID }}|zip file and ran smoke tests> against it"
                   }
                 },
                 {

changelog/fix-run-id-for-github-action

@@ -0,0 +1,5 @@
+Significance: patch
+Type: dev
+Comment: Fix GITHUB_RUN_ID env variable
+
+

tests/e2e/README.md

@@ -4,6 +4,24 @@ WooPayments e2e tests can be found in the `./tests/e2e/specs` directory. These t
 
 E2E tests can be run locally or in GitHub Actions. Github Actions are already configured and don't require any changes to run the tests.
 
+## Recent Improvements
+
+### Retry Mechanism
+The E2E tests now include an intelligent retry mechanism that:
+- **First run**: Executes all tests normally
+- **Automatic retry**: If any tests fail, only the failed tests are retried using Playwright's `--last-failed` flag
+
+### Improved Timeout Handling
+- **Increased timeouts**: UI interaction timeouts increased from 100ms to 10 seconds for better reliability
+- **Better error handling**: More robust page loading and element waiting strategies
+- **DevTools reliability**: Improved devtools page navigation and interaction
+
+### Dynamic Matrix Generation
+- **L-1 Policy**: Tests automatically run against the latest WooCommerce version and the L-1 (previous major) version
+- **Dynamic version resolution**: Automatically fetches latest WC, RC, and beta versions from WordPress.org API
+- **Optimized PHP strategy**: Reduces job count while maintaining comprehensive coverage
+- **Business continuity**: Maintains support for WC 7.7.0 for significant TPV reasons
+
 ## Setting up & running E2E tests
 
 For running E2E tests locally, create a new file named `local.env` under `tests/e2e/config` folder with the following env variables (replace values as required).
@@ -207,6 +225,19 @@ Currently, the best way to debug tests is to use the Playwright UI mode. This mo
 You can use the locator functionality to help correctly determine the locator syntax to correctly target the HTML element you need. Lastly, you can also use
 `console.log()` to assist with debugging tests in UI mode. To run tests in UI mode, use the `npm run test:e2e-ui path/to/test.spec` command.
 
+### Understanding Test Failures and Retries
+
+When tests fail in CI, the retry mechanism automatically kicks in:
+
+1. **First run**: All tests execute normally
+2. **If failures occur**: The system automatically retries only the failed tests
+3. **Retry logs**: Look for the message "Some tests failed, retrying only failed tests with --last-failed flag" in the logs
+4. **Final results**: The test run will show both the initial results and retry results
+
+This approach helps distinguish between:
+- **Flaky tests**: Tests that fail occasionally but pass on retry
+- **Consistent failures**: Tests that fail both initially and on retry (indicating real issues)
+
 ## Slack integration
 
 The Slack reporter is a custom reporter that sends e2e test failures to a public Slack channel (search Slack channel ID `CQ0Q6N62D`). The reporter is configured to only send the first failure of a test to Slack. If the retry also fails it will not be sent to prevent spamming the channel.
@@ -254,6 +285,16 @@ await page.getByRole( 'button', { name: /submit/i } ).click();
 
 In some cases, you may need to wait for the page to reach a certain load state before interacting with it. You can use `await page.waitForLoadState( 'domcontentloaded' );` to wait for the page to finish loading.
 
+**What timeout values are used for UI interactions?**
+
+The E2E tests use optimized timeout values for better reliability:
+- **Global expect timeout**: 20 seconds (configured in `playwright.config.ts`)
+- **UI interaction timeouts**: 10 seconds for critical UI elements (buttons, forms, etc.)
+- **Page load timeouts**: 120 seconds for test execution
+- **Network idle waits**: Used for dynamic content loading
+
+These timeouts have been increased from the previous 100ms values to provide better stability, especially for slower environments or complex UI interactions.
+
 **What is the best way to target elements in the page?**
 
 Prefer the use of [user-facing attribute or test-id locators](https://playwright.dev/docs/locators#locating-elements) to target elements in the page. This will make the tests more resilient to changes to implementation details, such as class names.
@@ -314,15 +355,29 @@ test.describe( 'Sign in as customer', () => {
 } );
 ```
 
+**How does the dynamic matrix generation work?**
+
+The E2E test matrix is dynamically generated using the `.github/scripts/generate-wc-matrix.sh` script:
+
+- **L-1 Policy**: Automatically tests against the latest WooCommerce version and the L-1 (previous major) version
+- **Version Resolution**: Fetches latest WC, RC, and beta versions from WordPress.org API
+- **PHP Strategy**:
+  - WC 7.7.0: PHP 7.3 (legacy support)
+  - WC L-1 & Latest: PHP 8.3 (stable)
+  - WC RC: PHP 8.4 (latest)
+- **Business Continuity**: Maintains WC 7.7.0 support for significant TPV reasons
+
+This ensures comprehensive test coverage while optimizing CI execution time and resource usage.
+
 **How can I investigate and interact with a test failures?**
 
 - **Github Action test runs**
   - View GitHub checks in the "Checks" tab of a PR
-  - There are currently four E2E test workflows:
-    - E2E Tests - Pull Request / WC - latest | wcpay - merchant (pull_request)
-    - E2E Tests - Pull Request / WC - latest | wcpay - shopper (pull_request)
-    - E2E Tests - Pull Request / WC - latest | subscriptions - merchant (pull_request)
-    - E2E Tests - Pull Request / WC - latest | subscriptions - shopper (pull_request)
+  - The E2E test matrix now dynamically generates test combinations based on:
+    - **WooCommerce versions**: Latest, L-1 (previous major), RC, and beta versions
+    - **PHP versions**: 7.3 (legacy), 8.3 (stable), 8.4 (latest)
+    - **Test groups**: wcpay, subscriptions
+    - **Test branches**: merchant, shopper
   - Click on the details link to the right of the failed job to see the summary
   - In the job summary, click on the "Run tests, upload screenshots & logs" section.
   - Click on the artifact download link at the end of the section, then extract and copy the `playwright-report` directory to the root of the WooPayments repository

tests/e2e/playwright.config.ts

@@ -51,8 +51,6 @@ export default defineConfig( {
 	fullyParallel: false,
 	/* Fail the build on CI if you accidentally left test.only in the source code. */
 	forbidOnly: !! process.env.CI,
-	/* Retry on CI only */
-	retries: process.env.CI ? 2 : 0,
 	/* Opt out of parallel tests. */
 	workers: 1,
 	/* Reporters to use. See https://playwright.dev/docs/test-reporters */

tests/e2e/specs/wcpay/shopper/shopper-myaccount-saved-cards.spec.ts

@@ -235,11 +235,6 @@ test.describe( 'Shopper can save and delete cards', () => {
 						// Take note of the time when we added this card
 						cardTimingHelper.markCardAdded();
 
-						await expect(
-							shopperPage.getByText(
-								`${ card2.expires.month }/${ card2.expires.year }`
-							)
-						).toBeVisible();
 						await setDefaultPaymentMethod( shopperPage, card2 );
 						// Verify that the card was set as default
 						await expect(
@@ -255,6 +250,15 @@ test.describe( 'Shopper can save and delete cards', () => {
 					{ tag: '@critical' },
 					async () => {
 						await goToMyAccount( shopperPage, 'payment-methods' );
+
+						// Verify both cards are visible before trying to delete them
+						await expect(
+							shopperPage.getByText( card.label )
+						).toBeVisible( { timeout: 10000 } );
+						await expect(
+							shopperPage.getByText( card2.label )
+						).toBeVisible( { timeout: 10000 } );
+
 						await deleteSavedCard( shopperPage, card );
 						await expect(
 							shopperPage.getByText( 'Payment method deleted.' )

tests/e2e/utils/devtools.ts

@@ -3,15 +3,29 @@
  */
 import { Page, expect } from '@playwright/test';
 
-const goToDevToolsSettings = ( page: Page ) =>
-	page.goto( '/wp-admin/admin.php?page=wcpaydev', {
-		waitUntil: 'load',
+const goToDevToolsSettings = async ( page: Page ) => {
+	await page.goto( '/wp-admin/admin.php?page=wcpaydev', {
+		waitUntil: 'domcontentloaded',
 	} );
 
+	// Wait for the page to be fully loaded
+	await page.waitForLoadState( 'networkidle' );
+};
+
 const saveDevToolsSettings = async ( page: Page ) => {
-	await page.getByRole( 'button', { name: 'Save Changes' } ).click();
+	// Wait for the page to be fully loaded before trying to interact
+	await page.waitForLoadState( 'domcontentloaded' );
+
+	// Wait for the Save Changes button to be available
+	const saveButton = page.getByRole( 'button', { name: 'Save Changes' } );
+	await expect( saveButton ).toBeVisible( { timeout: 10000 } );
+	await expect( saveButton ).toBeEnabled( { timeout: 10000 } );
+
+	await saveButton.click();
 	await page.waitForLoadState( 'networkidle' );
-	await expect( page.getByText( /Settings saved/ ) ).toBeVisible();
+	await expect( page.getByText( /Settings saved/ ) ).toBeVisible( {
+		timeout: 10000,
+	} );
 };
 
 const getIsCardTestingProtectionEnabled = ( page: Page ) =>