Improve nightly E2E test suite authentication issues in BrowserStack

.github/workflows/deploy-production.yml

@@ -155,8 +155,8 @@ jobs:
           project-name: 'Thunderbird Appointment'
           build-name: 'Production Deployment Tests: BUILD_INFO'
 
-      - name: Run Playwright Tests on Browserstack
+      - name: Production Sanity E2E Tests on Desktop Firefox
         run: |
           cd ./test/e2e
           cp .env.prod.example .env
-          npm run prod-sanity-test-browserstack-gha
+          npm run prod-sanity-test-browserstack-firefox

.github/workflows/deploy-staging.yml

@@ -262,9 +262,8 @@ jobs:
           project-name: 'Thunderbird Appointment'
           build-name: 'E2E Tests: BUILD_INFO'
 
-      - name: Run E2E Tests on stage via Browserstack
+      - name: Stage E2E Tests on Desktop Firefox
         run: |
           cd ./test/e2e
           cp .env.stage.example .env
-          npm run e2e-test-browserstack-gha
-
+          npm run e2e-test-browserstack-firefox

.github/workflows/nightly-tests-desktop.yml

@@ -15,8 +15,8 @@ permissions:
   contents: read    # This is required for actions/checkout
 
 jobs:
-  prod-sanity-browserstack:
-    name: prod-sanity-browserstack
+  nightly-prod-desktop-e2e:
+    name: nightly-prod-desktop-e2e
     runs-on: ubuntu-latest
     environment: production
     env:
@@ -47,10 +47,26 @@ jobs:
           project-name: 'Thunderbird Appointment'
           build-name: 'TB Appointment Nightly Tests (Desktop): BUILD_INFO'
 
-      - name: Run E2E Tests on Production on Browserstack (Desktop)
+      - name: Prod E2E Tests on Desktop Firefox
         # don't send GHA failure email if any of the E2E tests fail, can be annoying (I check the jobs each day in BrowserStack)
         continue-on-error: true
         run: |
           cd ./test/e2e
           cp .env.prod.example .env
-          npm run prod-nightly-tests-browserstack-gha
+          npm run prod-nightly-tests-browserstack-firefox
+
+      - name: Prod E2E Tests on Desktop Safari Webkit
+        # don't send GHA failure email if any of the E2E tests fail, can be annoying (I check the jobs each day in BrowserStack)
+        continue-on-error: true
+        run: |
+          cd ./test/e2e
+          cp .env.prod.example .env
+          npm run prod-nightly-tests-browserstack-safari
+
+      - name: Prod E2E Tests on Desktop Chromium
+        # don't send GHA failure email if any of the E2E tests fail, can be annoying (I check the jobs each day in BrowserStack)
+        continue-on-error: true
+        run: |
+          cd ./test/e2e
+          cp .env.prod.example .env
+          npm run prod-nightly-tests-browserstack-chromium

.github/workflows/nightly-tests-mobile.yml

@@ -15,8 +15,8 @@ permissions:
   contents: read    # This is required for actions/checkout
 
 jobs:
-  prod-nightly-mobile-browserstack:
-    name: prod-nighlty-mobile-browserstack
+  nightly-prod-mobile-e2e:
+    name: nightly-prod-mobile-e2e
     runs-on: ubuntu-latest
     environment: production
     env:
@@ -53,4 +53,4 @@ jobs:
         run: |
           cd ./test/e2e
           cp .env.prod.example .env
-          npm run prod-nightly-tests-mobile-browserstack-gha
+          npm run prod-nightly-tests-mobile-browserstack-android-chrome

test/e2e/browserstack-desktop-nightly.yml

@@ -44,7 +44,7 @@ platforms:
     playwrightConfigOptions:
       name: Safari-OSX
       setup: 
-      - name: 'setup_safari_osx'
+      - name: 'setup_webkit'
         testMatch: 'auth.desktop.setup.ts'
       use:
         storageState: 'test-results/.auth/user.json'
@@ -56,19 +56,7 @@ platforms:
     playwrightConfigOptions:
       name: Chromium-Win11
       setup: 
-      - name: 'setup_chromium_win'
-        testMatch: 'auth.desktop.setup.ts'
-      use:
-        storageState: 'test-results/.auth/user.json'
-
-  - os: Windows
-    osVersion: 11
-    browserName: edge
-    browserVersion: latest
-    playwrightConfigOptions:
-      name: Edge-Win11
-      setup: 
-      - name: 'setup_edge_win'
+      - name: 'setup_chromium'
         testMatch: 'auth.desktop.setup.ts'
       use:
         storageState: 'test-results/.auth/user.json'

test/e2e/package.json

@@ -5,15 +5,14 @@
   "scripts": {
     "e2e-test": "npx playwright test --grep e2e-suite --project=firefox",
     "e2e-test-headed": "npx playwright test --grep e2e-suite --project=firefox --headed",
-    "e2e-test-browserstack": "npx browserstack-node-sdk playwright test --grep e2e-suite --browserstack.buildName 'Appointment E2E Tests (Desktop)' --browserstack.config 'browserstack-desktop.yml'",
-    "e2e-test-browserstack-gha": "npx browserstack-node-sdk playwright test --grep e2e-suite --project=Firefox-OSX --browserstack.config 'browserstack-desktop.yml'",
+    "e2e-test-browserstack-firefox": "npx browserstack-node-sdk playwright test --grep e2e-suite --project=Firefox-OSX --browserstack.buildName 'Appointment E2E Tests Firefox Desktop' --browserstack.config 'browserstack-desktop.yml'",
     "prod-sanity-test": "npx playwright test --grep prod-sanity --project=firefox",
     "prod-sanity-test-headed": "npx playwright test --grep prod-sanity --project=firefox --headed",
-    "prod-sanity-test-browserstack": "npx browserstack-node-sdk playwright test --grep prod-sanity --project=Firefox-OSX --browserstack.buildName 'Production Sanity Test (Desktop)' --browserstack.config 'browserstack-desktop.yml'",
-    "prod-sanity-test-browserstack-gha": "npx browserstack-node-sdk playwright test --grep prod-sanity --project=Firefox-OSX --browserstack.config 'browserstack-desktop.yml'",
-    "prod-nightly-tests-browserstack-gha": "npx browserstack-node-sdk playwright test --grep prod-nightly --browserstack.config 'browserstack-desktop-nightly.yml'",
-    "e2e-test-mobile-browserstack": "npx browserstack-node-sdk playwright test --grep e2e-mobile-suite --browserstack.buildName 'Appointment E2E Tests (Mobile)' --browserstack.config 'browserstack-mobile.yml'",
-    "prod-nightly-tests-mobile-browserstack-gha": "npx browserstack-node-sdk playwright test --grep prod-mobile-nightly --browserstack.config 'browserstack-mobile-nightly.yml'",
+    "prod-sanity-test-browserstack-firefox": "npx browserstack-node-sdk playwright test --grep prod-sanity --project=Firefox-OSX --browserstack.buildName 'Appointment Production Sanity Test Firefox Desktop' --browserstack.config 'browserstack-desktop.yml'",
+    "prod-nightly-tests-browserstack-firefox": "npx browserstack-node-sdk playwright test --grep prod-nightly --project=Firefox-OSX --browserstack.buildName 'Appointment Nightly Tests Firefox Desktop' --browserstack.config 'browserstack-desktop-nightly.yml'",
+    "prod-nightly-tests-browserstack-safari": "npx browserstack-node-sdk playwright test --grep prod-nightly --project=Safari-OSX --browserstack.buildName 'Appointment Nightly Tests Safari Desktop' --browserstack.config 'browserstack-desktop-nightly.yml'",
+    "prod-nightly-tests-browserstack-chromium": "npx browserstack-node-sdk playwright test --grep prod-nightly --project=Chromium-Win11 --browserstack.buildName 'Appointment Nightly Tests Chromium Desktop' --browserstack.config 'browserstack-desktop-nightly.yml'",
+    "prod-nightly-tests-mobile-browserstack-android-chrome": "npx browserstack-node-sdk playwright test --grep prod-mobile-nightly --project=Chrome-GooglePixel10 --browserstack.buildName 'Appointment Nightly Tests Android Chrome' --browserstack.config 'browserstack-mobile-nightly.yml'",
     "postinstall": "npm update browserstack-node-sdk"
   },
   "keywords": [],

test/e2e/pages/availability-page.ts

@@ -2,9 +2,10 @@ import { expect, type Page, type Locator } from '@playwright/test';
 
 import {
   APPT_AVAILABILITY_PAGE,
+  APPT_TIMEZONE_SETTING_PRIMARY,
   TIMEOUT_1_SECOND,
   TIMEOUT_3_SECONDS,
-  APPT_TIMEZONE_SETTING_PRIMARY,
+  TIMEOUT_10_SECONDS,
   TIMEOUT_30_SECONDS,
 } from '../const/constants';
 
@@ -93,7 +94,7 @@ export class AvailabilityPage {
   async gotoAvailabilityPage() {
     // go to availability page, sometimes takes a bit to load all element values!
     await this.page.goto(APPT_AVAILABILITY_PAGE, { timeout: TIMEOUT_30_SECONDS });
-    await this.page.waitForTimeout(TIMEOUT_3_SECONDS);
+    await this.page.waitForTimeout(TIMEOUT_10_SECONDS);
     await this.bookableToggleContainer.waitFor({ timeout: TIMEOUT_30_SECONDS });
     await this.allStartTimeInput.waitFor({ timeout: TIMEOUT_30_SECONDS });
     await this.bookingPageMtgDur15MinBtn.waitFor({ timeout: TIMEOUT_30_SECONDS });

test/e2e/pages/settings-page.ts

@@ -6,7 +6,7 @@ import {
   TIMEOUT_1_SECOND,
   TIMEOUT_2_SECONDS,
   TIMEOUT_3_SECONDS,
-  TIMEOUT_10_SECONDS,
+  TIMEOUT_5_SECONDS,
   TIMEOUT_30_SECONDS,
   APPT_LANGUAGE_SETTING_EN,
   } from '../const/constants';
@@ -86,9 +86,9 @@ export class SettingsPage {
     this.connectedAppsBtn = this.page.getByTestId('settings-connectedApplications-settings-btn');
     this.connectedAppsHdr = this.page.getByRole('heading', { name: 'Connected Applications' });
     this.addCaldavBtn = this.page.getByRole('button', { name: 'Add CalDAV' });
-    this.addCaldavUsernameInput = this.page.getByRole('textbox', { name: 'user' });
-    this.addCaldavLocationInput = this.page.getByRole('textbox', { name: 'Location The URL or hostname' });
-    this.addCaldavPasswordInput = this.page.getByRole('textbox', { name: 'password' });
+    this.addCaldavUsernameInput = this.page.getByLabel('Username');
+    this.addCaldavLocationInput = this.page.getByLabel('Location');
+    this.addCaldavPasswordInput = this.page.getByLabel('Password');
     this.addCaldavCloseModalBtn = this.page.getByRole('img', { name: 'Close' });
     this.addGoogleBtn = this.page.getByRole('button', { name: 'Add Google Calendar' });
     this.defaultCalendarConnectedCbox = this.page.locator('div').filter({ hasText: /^Default*/ }).getByTestId('checkbox-input');
@@ -99,7 +99,7 @@ export class SettingsPage {
    */
   async gotoAccountSettings() {
     await this.page.goto(APPT_SETTINGS_PAGE);
-    await this.page.waitForTimeout(TIMEOUT_3_SECONDS);
+    await this.page.waitForTimeout(TIMEOUT_5_SECONDS);
     await this.accountSettingsBtn.scrollIntoViewIfNeeded();
     await this.accountSettingsBtn.click();
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
@@ -110,7 +110,7 @@ export class SettingsPage {
    */
   async gotoPreferencesSettings() {
     await this.page.goto(APPT_SETTINGS_PAGE);
-    await this.page.waitForTimeout(TIMEOUT_3_SECONDS);
+    await this.page.waitForTimeout(TIMEOUT_5_SECONDS);
     await this.preferencesBtn.scrollIntoViewIfNeeded({ timeout: TIMEOUT_30_SECONDS });
     await this.preferencesBtn.click();
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
@@ -121,7 +121,7 @@ export class SettingsPage {
    */
   async gotoConnectedAppSettings() {
     await this.page.goto(APPT_SETTINGS_PAGE);
-    await this.page.waitForTimeout(TIMEOUT_3_SECONDS);
+    await this.page.waitForTimeout(TIMEOUT_5_SECONDS);
     await this.connectedAppsBtn.scrollIntoViewIfNeeded();
     await this.connectedAppsBtn.click();
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
@@ -161,6 +161,7 @@ export class SettingsPage {
       await this.page.waitForTimeout(TIMEOUT_1_SECOND);
       await expect(this.savedSuccessfullyTextEN).toBeVisible();
     }
+    await this.page.waitForTimeout(TIMEOUT_5_SECONDS);
   }
 
   /**
@@ -174,7 +175,7 @@ export class SettingsPage {
     await this.saveBtnEN.click();
     await expect(this.savedSuccessfullyTextEN).toBeVisible({ timeout: TIMEOUT_30_SECONDS });
     // wait for theme to take affect, can take time especially on browserstack
-    await this.page.waitForTimeout(TIMEOUT_10_SECONDS);
+    await this.page.waitForTimeout(TIMEOUT_5_SECONDS);
   }
 
   /**
@@ -202,18 +203,6 @@ export class SettingsPage {
     await this.saveBtnEN.click();
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
     await expect(this.savedSuccessfullyTextEN).toBeVisible();
-  }
-
-  /**
-   * Change the display name setting
-   */
-  async changeDisplaName(newName: string) {
-    await this.displayNameInput.scrollIntoViewIfNeeded();
-    await this.page.waitForTimeout(TIMEOUT_1_SECOND);
-    await this.displayNameInput.fill(newName);
-    await this.page.waitForTimeout(TIMEOUT_1_SECOND);
-    await this.saveBtnEN.scrollIntoViewIfNeeded();
-    await this.saveBtnEN.click();
-    await this.page.waitForTimeout(TIMEOUT_3_SECONDS);
+    await this.page.waitForTimeout(TIMEOUT_5_SECONDS);
   }
 }

test/e2e/pages/tb-accts-page.ts

@@ -14,7 +14,7 @@ export class TBAcctsPage {
 
   constructor(page: Page) {
     this.page = page;
-    this.signInHeaderText = this.page.getByText('Enter your password');
+    this.signInHeaderText = this.page.getByText('Sign in to your account');
     this.userAvatar = this.page.getByTestId('avatar-default');
     this.emailInput = this.page.getByTestId('username-input');
     this.passwordInput = this.page.getByTestId('password-input');

test/e2e/playwright.config.ts

@@ -48,8 +48,16 @@ export default defineConfig({
 
   /* Configure projects for major browsers */
   projects: [
-    // Setup project - signs into Appointment once for all the tests
-    { name: 'desktop-setup', testMatch: /.*\.desktop.setup\.ts/ },
+    // Setup browsers - signs into Appointment once for all the tests and saves auth
+    { name: 'desktop-setup',
+      testMatch: /.*\.desktop.setup\.ts/,
+      use: {
+        ...devices['Desktop Chrome'],
+        screenshot: 'only-on-failure',
+       },
+    },
+
+    // Main tests; each browser runs the setup above and saves auth state which is loaded for each test in the suite
     {
       name: 'chromium',
       use: {
@@ -91,6 +99,7 @@ export default defineConfig({
         screenshot: 'only-on-failure',
        },
     },
+    // no dependency as mobile browsers don't support loading auth state so must sign-in for each test
   ],
 
   /* Run your local dev server before starting the tests */

test/e2e/tests/desktop/auth.desktop.setup.ts

@@ -10,7 +10,7 @@ import {
     APPT_DISPLAY_NAME,
     TIMEOUT_1_SECOND,
     TIMEOUT_2_SECONDS,
-    TIMEOUT_10_SECONDS,
+    TIMEOUT_5_SECONDS,
 } from "../../const/constants";
 
 const fs = require('fs');
@@ -43,16 +43,16 @@ setup('desktop browser authenticate', async ({ page }) => {
   // Sometimes login flow sets cookies in the process of several redirects.
   // Wait for the final URL to ensure that the cookies are actually set.
   await page.waitForURL(APPT_DASHBOARD_HOME_PAGE);
-  await page.waitForTimeout(TIMEOUT_2_SECONDS);
+  await page.waitForTimeout(TIMEOUT_5_SECONDS);
 
   // ensure our settings are set to what the tests expect as default (in case a
   // previous test run failed and left the settings in an incorrect state)
   await page.goto(APPT_SETTINGS_PAGE);
-  await page.waitForTimeout(TIMEOUT_2_SECONDS);
+  await page.waitForTimeout(TIMEOUT_5_SECONDS);
   await setDefaultUserSettingsLocalStore(page);
   await page.waitForTimeout(TIMEOUT_2_SECONDS);
 
-  // End of authentication steps.
+  // End of authentication steps, save the auth
   await page.context().storageState({ path: authFile });
 
   // Now also ensure the test account is bookable (availability panel) before we start
@@ -61,6 +61,7 @@ setup('desktop browser authenticate', async ({ page }) => {
   await availabilityPage.bookableToggleContainer.scrollIntoViewIfNeeded();
   if (! await availabilityPage.bookableToggle.isChecked()) {
     await availabilityPage.bookableToggleContainer.click();
+    await page.waitForTimeout(TIMEOUT_5_SECONDS)
     console.log('turned booking availibility on');
   }
 
@@ -71,11 +72,13 @@ setup('desktop browser authenticate', async ({ page }) => {
 
   if (await availabilityPage.allStartTimeInput.inputValue() != '09:00') {
     await availabilityPage.allStartTimeInput.fill('09:00');
+    await page.waitForTimeout(TIMEOUT_1_SECOND);
     console.log('set availability start time to 09:00');
   }
 
   if (await availabilityPage.allEndTimeInput.inputValue() != '17:00') {
     await availabilityPage.allEndTimeInput.fill('17:00');
+    await page.waitForTimeout(TIMEOUT_1_SECOND);
     console.log('set availability end time to 17:00');
   }
 
@@ -88,13 +91,14 @@ setup('desktop browser authenticate', async ({ page }) => {
   await availabilityPage.bookingPageNameInput.fill(APPT_DISPLAY_NAME);
   await page.waitForTimeout(TIMEOUT_1_SECOND);
   await availabilityPage.bookingPageDescInput.clear();
+  await page.waitForTimeout(TIMEOUT_1_SECOND);
   console.log('set booking page name and description');
 
   // if availability changes were made, save them
-  const saveBtnVisible = await availabilityPage.saveChangesBtn.isVisible({ timeout: TIMEOUT_10_SECONDS });
+  const saveBtnVisible = await availabilityPage.saveChangesBtn.isVisible({ timeout: TIMEOUT_5_SECONDS });
   if (saveBtnVisible) {
     await availabilityPage.saveChangesBtn.click();
-    await page.waitForTimeout(TIMEOUT_1_SECOND);
+    await page.waitForTimeout(TIMEOUT_2_SECONDS);
     await expect(availabilityPage.savedSuccessfullyText).toBeVisible();
   }
 });

test/e2e/tests/desktop/availability-booking-details.spec.ts

@@ -1,6 +1,7 @@
 import { test, expect } from '@playwright/test';
 import { AvailabilityPage } from '../../pages/availability-page';
 import { BookingPage } from '../../pages/booking-page';
+import { ensureWeAreSignedIn } from '../../utils/utils';
 
 import {
   PLAYWRIGHT_TAG_E2E_SUITE,
@@ -22,7 +23,7 @@ test.describe('availability - booking page details on desktop browser', {
   tag: [PLAYWRIGHT_TAG_E2E_SUITE, PLAYWRIGHT_TAG_PROD_NIGHTLY],
 }, () => {
   test.beforeEach(async ({ page }) => {
-    // note: we are already signed into Appointment with our default settings (via our auth-setup)
+    await ensureWeAreSignedIn(page);
     // availability panel is displayed open as default
     bookApptPage = new BookingPage(page);
     availabilityPage = new AvailabilityPage(page);

test/e2e/tests/desktop/availability-booking-link.spec.ts

@@ -1,5 +1,6 @@
 import { test, expect } from '@playwright/test';
 import { AvailabilityPage } from '../../pages/availability-page';
+import { ensureWeAreSignedIn } from '../../utils/utils';
 
 import {
   PLAYWRIGHT_TAG_E2E_SUITE,
@@ -14,7 +15,7 @@ test.describe('availability - booking page link on desktop browser', {
   tag: [PLAYWRIGHT_TAG_E2E_SUITE, PLAYWRIGHT_TAG_PROD_NIGHTLY],
 }, () => {
   test.beforeEach(async ({ page }) => {
-    // note: we are already signed into Appointment with our default settings (via our auth-setup)
+    await ensureWeAreSignedIn(page);
     // availability panel is displayed open as default
     availabilityPage = new AvailabilityPage(page);
     await availabilityPage.gotoAvailabilityPage();

test/e2e/tests/desktop/availability-set.spec.ts

@@ -1,6 +1,7 @@
 import { test, expect } from '@playwright/test';
 import { AvailabilityPage } from '../../pages/availability-page';
 import { BookingPage } from '../../pages/booking-page';
+import { ensureWeAreSignedIn } from '../../utils/utils';
 
 import {
   PLAYWRIGHT_TAG_E2E_SUITE,
@@ -20,7 +21,7 @@ test.describe('set availability on desktop browser', {
   tag: [PLAYWRIGHT_TAG_E2E_SUITE, PLAYWRIGHT_TAG_PROD_NIGHTLY],
 }, () => {
   test.beforeEach(async ({ page }) => {
-    // note: we are already signed into Appointment with our default settings (via our auth-setup)
+    await ensureWeAreSignedIn(page);
     // availability panel is displayed open as default
     bookApptPage = new BookingPage(page);
     availabilityPage = new AvailabilityPage(page);