More E2E test improvements

Author: rwood-moz

.github/workflows/deploy-production.yml

@@ -155,8 +155,8 @@ jobs:
           project-name: 'Thunderbird Appointment'
           build-name: 'Production Deployment Tests: BUILD_INFO'
 
-      - name: Run Playwright Tests on Browserstack
+      - name: Production Sanity E2E Tests on Desktop Firefox
         run: |
           cd ./test/e2e
           cp .env.prod.example .env
-          npm run prod-sanity-test-browserstack-gha
+          npm run prod-sanity-test-browserstack-firefox

.github/workflows/deploy-staging.yml

@@ -262,9 +262,8 @@ jobs:
           project-name: 'Thunderbird Appointment'
           build-name: 'E2E Tests: BUILD_INFO'
 
-      - name: Run E2E Tests on stage via Browserstack
+      - name: Stage E2E Tests on Desktop Firefox
         run: |
           cd ./test/e2e
           cp .env.stage.example .env
-          npm run e2e-test-browserstack-gha
-
+          npm run e2e-test-browserstack-firefox

.github/workflows/nightly-tests-desktop.yml

@@ -15,8 +15,8 @@ permissions:
   contents: read    # This is required for actions/checkout
 
 jobs:
-  prod-sanity-browserstack:
-    name: prod-sanity-browserstack
+  nightly-prod-desktop-e2e:
+    name: nightly-prod-desktop-e2e
     runs-on: ubuntu-latest
     environment: production
     env:
@@ -47,10 +47,26 @@ jobs:
           project-name: 'Thunderbird Appointment'
           build-name: 'TB Appointment Nightly Tests (Desktop): BUILD_INFO'
 
-      - name: Run E2E Tests on Production on Browserstack (Desktop)
+      - name: Prod E2E Tests on Desktop Firefox
         # don't send GHA failure email if any of the E2E tests fail, can be annoying (I check the jobs each day in BrowserStack)
         continue-on-error: true
         run: |
           cd ./test/e2e
           cp .env.prod.example .env
-          npm run prod-nightly-tests-browserstack-gha
+          npm run prod-nightly-tests-browserstack-firefox
+
+      - name: Prod E2E Tests on Desktop Safari Webkit
+        # don't send GHA failure email if any of the E2E tests fail, can be annoying (I check the jobs each day in BrowserStack)
+        continue-on-error: true
+        run: |
+          cd ./test/e2e
+          cp .env.prod.example .env
+          npm run prod-nightly-tests-browserstack-safari
+
+      - name: Prod E2E Tests on Desktop Chromium
+        # don't send GHA failure email if any of the E2E tests fail, can be annoying (I check the jobs each day in BrowserStack)
+        continue-on-error: true
+        run: |
+          cd ./test/e2e
+          cp .env.prod.example .env
+          npm run prod-nightly-tests-browserstack-chromium

.github/workflows/nightly-tests-mobile.yml

@@ -15,8 +15,8 @@ permissions:
   contents: read    # This is required for actions/checkout
 
 jobs:
-  prod-nightly-mobile-browserstack:
-    name: prod-nighlty-mobile-browserstack
+  nightly-prod-mobile-e2e:
+    name: nightly-prod-mobile-e2e
     runs-on: ubuntu-latest
     environment: production
     env:
@@ -53,4 +53,4 @@ jobs:
         run: |
           cd ./test/e2e
           cp .env.prod.example .env
-          npm run prod-nightly-tests-mobile-browserstack-gha
+          npm run prod-nightly-tests-mobile-browserstack-android-chrome

test/e2e/browserstack-desktop-nightly.yml

@@ -44,7 +44,7 @@ platforms:
     playwrightConfigOptions:
       name: Safari-OSX
       setup: 
-      - name: 'setup_safari_osx'
+      - name: 'setup_webkit'
         testMatch: 'auth.desktop.setup.ts'
       use:
         storageState: 'test-results/.auth/user.json'
@@ -56,19 +56,7 @@ platforms:
     playwrightConfigOptions:
       name: Chromium-Win11
       setup: 
-      - name: 'setup_chromium_win'
-        testMatch: 'auth.desktop.setup.ts'
-      use:
-        storageState: 'test-results/.auth/user.json'
-
-  - os: Windows
-    osVersion: 11
-    browserName: edge
-    browserVersion: latest
-    playwrightConfigOptions:
-      name: Edge-Win11
-      setup: 
-      - name: 'setup_edge_win'
+      - name: 'setup_chromium'
         testMatch: 'auth.desktop.setup.ts'
       use:
         storageState: 'test-results/.auth/user.json'

test/e2e/package.json

@@ -5,15 +5,14 @@
   "scripts": {
     "e2e-test": "npx playwright test --grep e2e-suite --project=firefox",
     "e2e-test-headed": "npx playwright test --grep e2e-suite --project=firefox --headed",
-    "e2e-test-browserstack": "npx browserstack-node-sdk playwright test --grep e2e-suite --browserstack.buildName 'Appointment E2E Tests (Desktop)' --browserstack.config 'browserstack-desktop.yml'",
-    "e2e-test-browserstack-gha": "npx browserstack-node-sdk playwright test --grep e2e-suite --project=Firefox-OSX --browserstack.config 'browserstack-desktop.yml'",
+    "e2e-test-browserstack-firefox": "npx browserstack-node-sdk playwright test --grep e2e-suite --project=Firefox-OSX --browserstack.config 'browserstack-desktop.yml'",
     "prod-sanity-test": "npx playwright test --grep prod-sanity --project=firefox",
     "prod-sanity-test-headed": "npx playwright test --grep prod-sanity --project=firefox --headed",
-    "prod-sanity-test-browserstack": "npx browserstack-node-sdk playwright test --grep prod-sanity --project=Firefox-OSX --browserstack.buildName 'Production Sanity Test (Desktop)' --browserstack.config 'browserstack-desktop.yml'",
-    "prod-sanity-test-browserstack-gha": "npx browserstack-node-sdk playwright test --grep prod-sanity --project=Firefox-OSX --browserstack.config 'browserstack-desktop.yml'",
-    "prod-nightly-tests-browserstack-gha": "npx browserstack-node-sdk playwright test --grep prod-nightly --browserstack.config 'browserstack-desktop-nightly.yml'",
-    "e2e-test-mobile-browserstack": "npx browserstack-node-sdk playwright test --grep e2e-mobile-suite --browserstack.buildName 'Appointment E2E Tests (Mobile)' --browserstack.config 'browserstack-mobile.yml'",
-    "prod-nightly-tests-mobile-browserstack-gha": "npx browserstack-node-sdk playwright test --grep prod-mobile-nightly --browserstack.config 'browserstack-mobile-nightly.yml'",
+    "prod-sanity-test-browserstack-firefox": "npx browserstack-node-sdk playwright test --grep prod-sanity --project=Firefox-OSX --browserstack.buildName 'Appointment Production Sanity Test (Desktop)' --browserstack.config 'browserstack-desktop.yml'",
+    "prod-nightly-tests-browserstack-firefox": "npx browserstack-node-sdk playwright test --grep prod-nightly --project=Firefox-OSX --browserstack.buildName 'Appointment Nightly Tests Firefox Desktop' --browserstack.config 'browserstack-desktop-nightly.yml'",
+    "prod-nightly-tests-browserstack-safari": "npx browserstack-node-sdk playwright test --grep prod-nightly --project=Safari-OSX --browserstack.buildName 'Appointment Nightly Tests Safari Desktop' --browserstack.config 'browserstack-desktop-nightly.yml'",
+    "prod-nightly-tests-browserstack-chromium": "npx browserstack-node-sdk playwright test --grep prod-nightly --project=Chromium-Win11 --browserstack.buildName 'Appointment Nightly Tests Chromium Desktop' --browserstack.config 'browserstack-desktop-nightly.yml'",
+    "prod-nightly-tests-mobile-browserstack-android-chrome": "npx browserstack-node-sdk playwright test --grep prod-mobile-nightly --project=Chrome-GooglePixel10 --browserstack.buildName 'Appointment Nightly Tests Android Chrome' --browserstack.config 'browserstack-mobile-nightly.yml'",
     "postinstall": "npm update browserstack-node-sdk"
   },
   "keywords": [],

test/e2e/pages/availability-page.ts

@@ -2,9 +2,10 @@ import { expect, type Page, type Locator } from '@playwright/test';
 
 import {
   APPT_AVAILABILITY_PAGE,
+  APPT_TIMEZONE_SETTING_PRIMARY,
   TIMEOUT_1_SECOND,
   TIMEOUT_3_SECONDS,
-  APPT_TIMEZONE_SETTING_PRIMARY,
+  TIMEOUT_10_SECONDS,
   TIMEOUT_30_SECONDS,
 } from '../const/constants';
 
@@ -93,7 +94,7 @@ export class AvailabilityPage {
   async gotoAvailabilityPage() {
     // go to availability page, sometimes takes a bit to load all element values!
     await this.page.goto(APPT_AVAILABILITY_PAGE, { timeout: TIMEOUT_30_SECONDS });
-    await this.page.waitForTimeout(TIMEOUT_3_SECONDS);
+    await this.page.waitForTimeout(TIMEOUT_10_SECONDS);
     await this.bookableToggleContainer.waitFor({ timeout: TIMEOUT_30_SECONDS });
     await this.allStartTimeInput.waitFor({ timeout: TIMEOUT_30_SECONDS });
     await this.bookingPageMtgDur15MinBtn.waitFor({ timeout: TIMEOUT_30_SECONDS });

test/e2e/pages/settings-page.ts

@@ -6,7 +6,7 @@ import {
   TIMEOUT_1_SECOND,
   TIMEOUT_2_SECONDS,
   TIMEOUT_3_SECONDS,
-  TIMEOUT_10_SECONDS,
+  TIMEOUT_5_SECONDS,
   TIMEOUT_30_SECONDS,
   APPT_LANGUAGE_SETTING_EN,
   } from '../const/constants';
@@ -86,9 +86,9 @@ export class SettingsPage {
     this.connectedAppsBtn = this.page.getByTestId('settings-connectedApplications-settings-btn');
     this.connectedAppsHdr = this.page.getByRole('heading', { name: 'Connected Applications' });
     this.addCaldavBtn = this.page.getByRole('button', { name: 'Add CalDAV' });
-    this.addCaldavUsernameInput = this.page.getByRole('textbox', { name: 'user' });
-    this.addCaldavLocationInput = this.page.getByRole('textbox', { name: 'Location The URL or hostname' });
-    this.addCaldavPasswordInput = this.page.getByRole('textbox', { name: 'password' });
+    this.addCaldavUsernameInput = this.page.getByLabel('Username');
+    this.addCaldavLocationInput = this.page.getByLabel('Location');
+    this.addCaldavPasswordInput = this.page.getByLabel('Password');
     this.addCaldavCloseModalBtn = this.page.getByRole('img', { name: 'Close' });
     this.addGoogleBtn = this.page.getByRole('button', { name: 'Add Google Calendar' });
     this.defaultCalendarConnectedCbox = this.page.locator('div').filter({ hasText: /^Default*/ }).getByTestId('checkbox-input');
@@ -99,7 +99,7 @@ export class SettingsPage {
    */
   async gotoAccountSettings() {
     await this.page.goto(APPT_SETTINGS_PAGE);
-    await this.page.waitForTimeout(TIMEOUT_3_SECONDS);
+    await this.page.waitForTimeout(TIMEOUT_5_SECONDS);
     await this.accountSettingsBtn.scrollIntoViewIfNeeded();
     await this.accountSettingsBtn.click();
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
@@ -110,7 +110,7 @@ export class SettingsPage {
    */
   async gotoPreferencesSettings() {
     await this.page.goto(APPT_SETTINGS_PAGE);
-    await this.page.waitForTimeout(TIMEOUT_3_SECONDS);
+    await this.page.waitForTimeout(TIMEOUT_5_SECONDS);
     await this.preferencesBtn.scrollIntoViewIfNeeded({ timeout: TIMEOUT_30_SECONDS });
     await this.preferencesBtn.click();
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
@@ -121,7 +121,7 @@ export class SettingsPage {
    */
   async gotoConnectedAppSettings() {
     await this.page.goto(APPT_SETTINGS_PAGE);
-    await this.page.waitForTimeout(TIMEOUT_3_SECONDS);
+    await this.page.waitForTimeout(TIMEOUT_5_SECONDS);
     await this.connectedAppsBtn.scrollIntoViewIfNeeded();
     await this.connectedAppsBtn.click();
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
@@ -174,7 +174,7 @@ export class SettingsPage {
     await this.saveBtnEN.click();
     await expect(this.savedSuccessfullyTextEN).toBeVisible({ timeout: TIMEOUT_30_SECONDS });
     // wait for theme to take affect, can take time especially on browserstack
-    await this.page.waitForTimeout(TIMEOUT_10_SECONDS);
+    await this.page.waitForTimeout(TIMEOUT_5_SECONDS);
   }
 
   /**

test/e2e/pages/tb-accts-page.ts

@@ -14,7 +14,7 @@ export class TBAcctsPage {
 
   constructor(page: Page) {
     this.page = page;
-    this.signInHeaderText = this.page.getByText('Enter your password');
+    this.signInHeaderText = this.page.getByText('Sign in to your account');
     this.userAvatar = this.page.getByTestId('avatar-default');
     this.emailInput = this.page.getByTestId('username-input');
     this.passwordInput = this.page.getByTestId('password-input');

test/e2e/playwright.config.ts

@@ -48,8 +48,16 @@ export default defineConfig({
 
   /* Configure projects for major browsers */
   projects: [
-    // Setup project - signs into Appointment once for all the tests
-    { name: 'desktop-setup', testMatch: /.*\.desktop.setup\.ts/ },
+    // Setup browsers - signs into Appointment once for all the tests and saves auth
+    { name: 'desktop-setup',
+      testMatch: /.*\.desktop.setup\.ts/,
+      use: {
+        ...devices['Desktop Chrome'],
+        screenshot: 'only-on-failure',
+       },
+    },
+
+    // Main tests; each browser runs the setup above and saves auth state which is loaded for each test in the suite
     {
       name: 'chromium',
       use: {
@@ -91,6 +99,7 @@ export default defineConfig({
         screenshot: 'only-on-failure',
        },
     },
+    // no dependency as mobile browsers don't support loading auth state so must sign-in for each test
   ],
 
   /* Run your local dev server before starting the tests */