Merge pull request #6160 from mozilla/refactor-e2e-relay-only-MPP-4466

fix #MPP-4466 - refactor(e2e): replace health check tags with relay-only test suite

Author: groovecoder

.github/workflows/playwright.yml

@@ -8,10 +8,11 @@ on:
         required: true
         default: stage
         type: string
-      health_check:
+      suite:
+        description: Test suite to run
         required: false
-        default: false
-        type: boolean
+        default: relay-only
+        type: string
     secrets:
       E2E_TEST_ACCOUNT_FREE: 
         required: true
@@ -32,14 +33,17 @@ on:
           - stage
           - prod
           - dev
-      health_check:
-        description: Enable health check
+      suite:
+        description: Test suite to run
         required: false
-        default: false
-        type: boolean
+        default: full
+        type: choice
+        options:
+          - relay-only
+          - full
 jobs:
   relaye2e:
-    name: ${{ inputs.health_check && 'Relay e2e health check' || 'Relay e2e all tests' }} 
+    name: ${{ inputs.suite == 'relay-only' && 'Relay e2e relay-only suite' || 'Relay e2e full suite' }}
     timeout-minutes: 60
     runs-on: ubuntu-latest
     steps:
@@ -55,15 +59,9 @@ jobs:
           npm install -D @playwright/test --with-deps
           npx playwright install
       - name: Run Playwright tests
-        run: >
-          if $HEALTH_CHECK; then
-            E2E_TEST_ENV=$E2E_TEST_ENV npx playwright test --grep "@health_check"
-          else
-            npm run test:$E2E_TEST_ENV
-          fi
+        run: |
+          npm run test:${{ inputs.suite }}:${{ inputs.environment }}
         env:
-          HEALTH_CHECK: ${{ contains(inputs.health_check, 'true') }}
-          E2E_TEST_ENV: ${{ inputs.environment != null && inputs.environment || 'stage' }}
           E2E_TEST_ACCOUNT_FREE: ${{ secrets.E2E_TEST_ACCOUNT_FREE }}
           E2E_TEST_ACCOUNT_PASSWORD: ${{ secrets.E2E_TEST_ACCOUNT_PASSWORD }}
           E2E_TEST_ACCOUNT_PREMIUM: ${{ secrets.E2E_TEST_ACCOUNT_PREMIUM }}
@@ -101,4 +99,3 @@ jobs:
                     text: "*Ref:*\n ${{ github.ref }}"
                   - type: mrkdwn
                     text: "*Triggered by:*\n ${{ github.triggering_actor }}"
-

.github/workflows/relay_e2e_health.yml

@@ -64,14 +64,14 @@ another more appropriate party.
 2. SRE processes ticket to release the tag to production
 3. Update [Github Release][github-releases] to current release
 4. Monitor [Sentry Releases][sentry-releases] for new production issues
-5. [Run e2e tests][run-e2e-tests] against the prod environment via GitHub Actions.
+5. [Run the relay-only e2e test suite][run-e2e-tests] against the prod environment via GitHub Actions.
 6. (On your 3rd Tuesday) Hand-off base load duties to next engineer in rotation
 
 ## Wednesdays
 
 1. Daily routine
-2. Before releasing to Stage, [run e2e tests][run-e2e-tests] against the dev environment via GitHub Actions.
-   - Ensure that the e2e tests are passing.
+2. Before releasing to Stage, [run the relay-only e2e test suite][run-e2e-tests] against the dev environment via GitHub Actions.
+   - Ensure that the relay-only e2e tests are passing.
    - If e2e Playwright tests are flaky--fails to pass for reasons outside of legitimate Relay bug--consider making the tests more reliable by using the [locators][playwright-locators], [auto-retrying assertions][playwright-auto-retrying-assertions], or [fixtures][playwright-fixtures]. For more suggestions on making Playwright tests more reliable or efficient, see [documentation on FxA test improvements][fxa-test-improvements].
 3. [Release to stage][Release-to-stage] (tag, Github release notes)
    - Ping all the engineers who have changes in the release to:

docs/base-load-engineer-playbook.md

@@ -77,11 +77,11 @@ Check out the update branch and run:
 ```
 npm install
 npx playwright install
-npx playwright test
+npm run test:relay-only:stage
 ```
 
 This will confirm that `dotenv` picked up the settings from `.env`, and that
-the new playwright runs against stage.
+the new playwright runs the relay-only test suite against stage.
 
 Once the PR is merged, you can wait for the daily test run, or manually run the
 test by going to the [Relay e2e Tests][] GitHub Action page, select "Run

docs/dependency-updates.md

@@ -185,7 +185,7 @@ On Tuesday, after the Release Readiness review with QA:
    - Spot-check the site for basic functionality
    - Check [sentry prod project](https://mozilla.sentry.io/releases/?environment=prod) for a spike in any new issues
    - Check [grafana dashboard](https://yardstick.mozilla.org/) for any unexpected spike in ops
-   - (optional) [Run end-to-end tests](https://github.com/mozilla/fx-private-relay/actions/workflows/playwright.yml) on prod
+   - (optional) [Run the relay-only e2e test suite](https://github.com/mozilla/fx-private-relay/actions/workflows/playwright.yml) on prod
 4. Update the GitHub release:
    - Update the summary:
 

docs/release_process.md

@@ -1,4 +1,4 @@
-# Relay e2e Test Suite
+# Relay e2e Test Suites
 
 ---
 
@@ -53,13 +53,37 @@ The premium account needs to have a chosen subdomain for the premium tests to pa
 
 ### 6. Run Tests
 
+#### Running the full suite
+
+```
+npm run test:e2e           # Runs on stage (default)
+npm run test:full:stage    # Explicit full suite on stage
+npm run test:full:prod     # Full suite on production
+```
+
+By default, `npm run test:e2e` will run all tests on https://relay.allizom.org/.
+
+#### Running the relay-only suite
+
+The relay-only suite excludes tests that depend on Mozilla Monitor, SubPlat payment flows, or third-party sites like developmentthatpays.com.
+
 ```
-npm run test:e2e
+npm run test:relay-only         # Runs on stage (default)
+npm run test:relay-only:stage   # Relay-only on stage
+npm run test:relay-only:prod    # Relay-only on production
 ```
 
-By default, `npm run test:e2e` will run the tests on https://relay.allizom.org/.
+**Test suites:**
 
-You can also run tests locally, on our dev server (https://relay-dev.allizom.org/), and in production (https://relay.firefox.com/). You can find the commands [here](https://github.com/mozilla/fx-private-relay/blob/main/package.json#L26-L31), or you can run `E2E_TEST_ENV=<env (prod, dev, stage)> npx playwright test`.
+- **Relay-only**: Landing page, free user functionality, premium functionality, upgrade flow
+- **Full**: All relay-only tests plus Monitor integration, subscription flows, tracker tests
+
+**External dependencies:**
+
+- Relay-only requires: Relay deployment, FXA, Restmail.net
+- Full suite additionally requires: Mozilla Monitor, SubPlat, developmentthatpays.com
+
+You can also run tests locally or on our dev server. See all commands [here](https://github.com/mozilla/fx-private-relay/blob/main/package.json), or use `E2E_TEST_ENV=<env (prod, dev, stage, local)> npx playwright test`.
 
 To view the tests live in the browser, you can add `--headed` to the end of the command:
 
@@ -87,13 +111,22 @@ Error: A snapshot doesn't exist at example.spec.ts-snapshots/example-test-1-chro
 
 This is because playwright needs to create an image initially. On the following runs, it will compare that a screenshot of the respective element matches the one added initially. Do not push your local images into the repo, the only ones that are needed for CI end in `linux`.
 
-### 8. Health check
+### 8. Relay-Only Suite
 
-Our ![health check](https://github.com/mozilla/fx-private-relay/actions/workflows/relay_e2e_health.yml) runs a subset of the entire e2e test suite everyday. This subset of tests focuses on critical tests for free and premium users for the overall health of the relay application. To add a test into the healthcheck CI, add `@health_check` into the title of your test or test group. See the following as an example,
+The [Relay e2e tests workflow](https://github.com/mozilla/fx-private-relay/actions/workflows/playwright.yml) runs the relay-only test suite daily at 8 AM UTC. This suite focuses on core Relay functionality without dependencies on external services like Monitor or payment processors.
 
-`test.describe("Subscription flows @health_check", ...)`
+The relay-only suite runs automatically on a schedule, but you can also trigger it manually:
 
-To run the health check manually, go to ![Relay e2e tests](https://github.com/mozilla/fx-private-relay/actions/workflows/playwright.yml), click run workflow, and check off "enable health check" before clicking "run workflow".
+1. Go to [Relay e2e tests](https://github.com/mozilla/fx-private-relay/actions/workflows/playwright.yml)
+2. Click "Run workflow"
+3. Select "relay-only" from the suite dropdown
+4. Click "Run workflow"
+
+To run the relay-only suite locally:
+
+```bash
+npm run test:relay-only:stage
+```
 
 ### 9. Diagnosing Test Failures
 

e2e-tests/README.md

@@ -2,7 +2,7 @@ import test, { expect } from "../fixtures/basePages";
 import { checkAuthState } from "../e2eTestUtils/helpers";
 
 test.describe.configure({ mode: "parallel" });
-test.describe("FxA auth, random mask generation, and email forwarding @health_check", () => {
+test.describe("FxA auth, random mask generation, and email forwarding", () => {
   test.skip(
     process.env.E2E_TEST_ENV === "prod",
     "This test only works on stage/dev environments because you cannot use masks to sign up for monitor, see 'Email forwarding and trackers removal' for a similar test",
@@ -67,7 +67,7 @@ test.skip("Email forwarding and trackers removal", () => {
   });
 });
 
-test.describe("Subscription flows with PlanGrid @health_check", () => {
+test.describe("Subscription flows with PlanGrid", () => {
   /**
    * Verifies that all plans correctly redirect to their corresponding subscriptions page.
    */

e2e-tests/specs/relay-e2e.spec.ts

@@ -11,7 +11,7 @@ test.describe("Free - General Functionalities, Desktop", () => {
     await dashboardPage.maybeDeleteMasks();
   });
 
-  test("Check the free user can only create 5 masks, C1553067 @health_check", async ({
+  test("Check the free user can only create 5 masks, C1553067", async ({
     dashboardPage,
     page,
   }) => {
@@ -84,7 +84,7 @@ test.describe("Free - General Functionalities, Desktop - Visual Regression", ()
     );
   });
 
-  test("Check that the user can delete an mask, and is prompted to confirm before they delete, C1553071 @health_check", async ({
+  test("Check that the user can delete an mask, and is prompted to confirm before they delete, C1553071", async ({
     dashboardPage,
     page,
   }) => {

e2e-tests/specs/relay-general-functionality.spec.ts

@@ -1,7 +1,7 @@
 import test, { expect } from "../fixtures/basePages";
 import { defaultScreenshotOpts } from "../e2eTestUtils/helpers";
 
-test.describe("Firefox Relay - Landing Page - Visual Regression @health_check", () => {
+test.describe("Firefox Relay - Landing Page - Visual Regression", () => {
   test.skip(
     ({ browserName }) => browserName !== "firefox",
     "firefox only image comparisons!",
@@ -28,7 +28,7 @@ test.describe("Firefox Relay - Landing Page - Visual Regression @health_check",
   });
 });
 
-test.describe("Check header buttons and their redirects, C1812638 @health_check", () => {
+test.describe("Check header buttons and their redirects, C1812638", () => {
   test.beforeEach(async ({ landingPage }) => {
     await landingPage.open();
   });

e2e-tests/specs/relay-home-page.spec.ts

@@ -9,7 +9,7 @@ test.describe("Premium - General Functionalities, Desktop", () => {
     await dashboardPage.maybeDeleteMasks(true, parseInt(totalMasks as string));
   });
 
-  test("Verify that a premium user can make more than 5 masks @health_check", async ({
+  test("Verify that a premium user can make more than 5 masks", async ({
     dashboardPage,
   }) => {
     expect(await dashboardPage.emailMasksUsedAmount.textContent()).toBe("0");
@@ -41,7 +41,7 @@ test.describe("Premium - General Functionalities, Desktop", () => {
     );
   });
 
-  test("Verify that a premium user can generate a custom mask @health_check", async ({
+  test("Verify that a premium user can generate a custom mask", async ({
     dashboardPage,
   }) => {
     // When there are zero masks, a random mask must be generated first