First set of E2E tests on iOS

Author: rwood-moz

.github/workflows/nightly-tests-mobile.yml

@@ -47,10 +47,18 @@ jobs:
           project-name: 'Thunderbird Appointment'
           build-name: 'TB Appointment Nightly Tests (Mobile): BUILD_INFO'
 
-      - name: Run E2E Tests on Production on Browserstack (Mobile)
+      - name: Prod E2E Tests on Android Chrome
         # don't send GHA failure email if any of the E2E tests fail, can be annoying (I check the jobs each day in BrowserStack)
         continue-on-error: true
         run: |
           cd ./test/e2e
           cp .env.prod.example .env
           npm run prod-nightly-tests-mobile-browserstack-android-chrome
+
+      - name: Prod E2E Tests on iOS Safari
+        # don't send GHA failure email if any of the E2E tests fail, can be annoying (I check the jobs each day in BrowserStack)
+        continue-on-error: true
+        run: |
+          cd ./test/e2e
+          cp .env.prod.example .env
+          npm run prod-nightly-tests-mobile-browserstack-ios-safari

backend/test/unit/test_schedule_availability.py

@@ -4,7 +4,6 @@
 
 
 class TestScheduleAvailability:
-
     def test_empty_availability_is_valid(self):
         # Test empty availability is valid
         schedule = schemas.ScheduleValidationIn(
@@ -17,7 +16,6 @@ def test_empty_availability_is_valid(self):
         )
         assert all_availability_is_valid(schedule)
 
-
     def test_all_availability_is_valid(self):
         # Test already sorted availabilities
         schedule = schemas.ScheduleValidationIn(
@@ -37,7 +35,7 @@ def test_all_availability_is_valid(self):
                 schemas.AvailabilityValidationIn(
                     schedule_id=1, day_of_week=2, start_time=time(9, 0), end_time=time(10, 0)
                 ),
-            ]
+            ],
         )
         assert all_availability_is_valid(schedule)
 
@@ -49,16 +47,13 @@ def test_all_availability_is_valid(self):
             schemas.AvailabilityValidationIn(
                 schedule_id=1, day_of_week=1, start_time=time(10, 0), end_time=time(11, 0)
             ),
-            schemas.AvailabilityValidationIn(
-                schedule_id=1, day_of_week=1, start_time=time(9, 0), end_time=time(10, 0)
-            ),
+            schemas.AvailabilityValidationIn(schedule_id=1, day_of_week=1, start_time=time(9, 0), end_time=time(10, 0)),
             schemas.AvailabilityValidationIn(
                 schedule_id=1, day_of_week=1, start_time=time(17, 0), end_time=time(18, 0)
             ),
         ]
         assert all_availability_is_valid(schedule)
 
-
     def test_all_availability_is_invalid(self):
         # Test overlapping end-start times
         schedule = schemas.ScheduleValidationIn(
@@ -75,15 +70,13 @@ def test_all_availability_is_invalid(self):
                 schemas.AvailabilityValidationIn(
                     schedule_id=1, day_of_week=1, start_time=time(10, 0), end_time=time(12, 0)
                 ),
-            ]
+            ],
         )
         assert not all_availability_is_valid(schedule)
 
         # Test completely overlapping slots
         schedule.availabilities = [
-            schemas.AvailabilityValidationIn(
-                schedule_id=1, day_of_week=1, start_time=time(9, 0), end_time=time(12, 0)
-            ),
+            schemas.AvailabilityValidationIn(schedule_id=1, day_of_week=1, start_time=time(9, 0), end_time=time(12, 0)),
             schemas.AvailabilityValidationIn(
                 schedule_id=1, day_of_week=1, start_time=time(10, 0), end_time=time(11, 0)
             ),
@@ -92,9 +85,7 @@ def test_all_availability_is_invalid(self):
 
         # Test slots with invalid start/end time
         schedule.availabilities = [
-            schemas.AvailabilityValidationIn(
-                schedule_id=1, day_of_week=1, start_time=time(9, 0), end_time=time(12, 0)
-            ),
+            schemas.AvailabilityValidationIn(schedule_id=1, day_of_week=1, start_time=time(9, 0), end_time=time(12, 0)),
             schemas.AvailabilityValidationIn(
                 schedule_id=1, day_of_week=1, start_time=time(14, 0), end_time=time(13, 0)
             ),
@@ -103,8 +94,6 @@ def test_all_availability_is_invalid(self):
 
         # Test slots that are too small for the defined duration
         schedule.availabilities = [
-            schemas.AvailabilityValidationIn(
-                schedule_id=1, day_of_week=1, start_time=time(9, 0), end_time=time(9, 15)
-            ),
+            schemas.AvailabilityValidationIn(schedule_id=1, day_of_week=1, start_time=time(9, 0), end_time=time(9, 15)),
         ]
         assert not all_availability_is_valid(schedule)

test/e2e/README.md

@@ -175,28 +175,28 @@ export BROWSERSTACK_USERNAME=<your-browserstack-user-name>
 export BROWSERSTACK_ACCESS_KEY=<your-browserstack-access-key>
 ```
 
-To run the E2E tests on BrowserStack (still in `test/e2e`):
+To run the E2E tests on Firefox Desktop on BrowserStack (still in `test/e2e`):
 
 ```bash
-npm run e2e-test-browserstack
+npm run e2e-test-browserstack-firefox
 ```
 
-To run the production sanity test on BrowserStack (still in `test/e2e`):
+To run the production sanity test on Firefox Desktop on BrowserStack (still in `test/e2e`):
 
 ```bash
-npm run prod-sanity-test-browserstack
+npm run prod-sanity-test-browserstack-firefox
 ```
 
-To run the E2E tests on mobile devices on BrowserStack (still in `test/e2e`):
+To run the E2E tests on Android Chrome on a real Google Pixel device on BrowserStack (still in `test/e2e`):
 
 ```bash
-npm run e2e-test-mobile-browserstack
+npm run e2e-tests-mobile-browserstack-android-chrome
 ```
 
-To run the nightly test suite on real mobile devices in BrowserStack (still in `test/e2e`):
+To run the E2E tests on iOS Safari on a real iPhone device on BrowserStack (still in `test/e2e`):
 
 ```bash
-npm run prod-nightly-tests-mobile-browserstack-gha
+npm run e2e-tests-mobile-browserstack-ios-safari
 ```
 
 After the tests finish in your local console you'll see a link to the BrowserStack test session; when signed into your BrowserStack account you'll be able to use that link to see the test session results including video playback.

test/e2e/browserstack-mobile-nightly.yml

@@ -29,14 +29,20 @@ platforms:
     osVersion: 16.0
     deviceName: Google Pixel 10
     playwrightConfigOptions:
-      name: Chrome-GooglePixel10
+      name: android-chrome
     chrome:
       chromeOptions:
         prefs:
           credentials_enable_service: false
         profile:
           password_manager_enabled: false
 
+  - browserName: safari
+    osVersion: 26
+    deviceName: iPhone 17
+    playwrightConfigOptions:
+      name: ios-safari
+
 # =======================
 # Parallels per Platform
 # =======================
@@ -68,6 +74,7 @@ debug: false # <boolean> # Set to true if you need screenshots for every seleniu
 networkLogs: false # <boolean> Set to true to enable HAR logs capturing; off as may contain sensitive info like login API requests
 consoleLogs: info # <string> Remote browser's console debug levels to be printed (`disable`, `errors`, `warnings`, `info`, or `verbose`)
 framework: playwright
+idleTimeout: 500 # seeing if this helps rid BROWSERSTACK_IDLE_TIMEOUTs on ios
 # the e2e tests use the same node.js project so the playwright version dependecy must work on both desktop
 # and mobile browsers; and therefore must be the same in all of our browserstack-*.yml config files
 # for playwright ver we need one compatible with our desktop and mobile browsers, see:

test/e2e/browserstack-mobile.yml

@@ -29,14 +29,20 @@ platforms:
     osVersion: 16.0
     deviceName: Google Pixel 10
     playwrightConfigOptions:
-      name: Chrome-GooglePixel10
+      name: android-chrome
     chrome:
       chromeOptions:
         prefs:
           credentials_enable_service: false
         profile:
           password_manager_enabled: false
 
+  - browserName: safari
+    osVersion: 26
+    deviceName: iPhone 17
+    playwrightConfigOptions:
+      name: ios-safari
+  
 # =======================
 # Parallels per Platform
 # =======================
@@ -68,6 +74,7 @@ debug: false # <boolean> # Set to true if you need screenshots for every seleniu
 networkLogs: false # <boolean> Set to true to enable HAR logs capturing; off as may contain sensitive info like login API requests
 consoleLogs: info # <string> Remote browser's console debug levels to be printed (`disable`, `errors`, `warnings`, `info`, or `verbose`)
 framework: playwright
+idleTimeout: 500 # seeing if this helps rid BROWSERSTACK_IDLE_TIMEOUTs on ios
 # the e2e tests use the same node.js project so the playwright version dependecy must work on both desktop
 # and mobile browsers; and therefore must be the same in all of our browserstack-*.yml config files
 # for playwright ver we need one compatible with our desktop and mobile browsers, see:

test/e2e/package.json

@@ -5,14 +5,17 @@
   "scripts": {
     "e2e-test": "npx playwright test --grep e2e-suite --project=firefox",
     "e2e-test-headed": "npx playwright test --grep e2e-suite --project=firefox --headed",
-    "e2e-test-browserstack-firefox": "npx browserstack-node-sdk playwright test --grep e2e-suite --project=Firefox-OSX --browserstack.buildName 'Appointment E2E Tests Firefox Desktop' --browserstack.config 'browserstack-desktop.yml'",
+    "e2e-test-browserstack-firefox": "npx browserstack-node-sdk playwright test --grep e2e-suite --project=Firefox-OSX --max-failures=2 --browserstack.buildName 'Appointment E2E Tests Firefox Desktop' --browserstack.config 'browserstack-desktop.yml'",
+    "e2e-tests-mobile-browserstack-android-chrome": "npx browserstack-node-sdk playwright test --grep e2e-mobile-suite --project=android-chrome --max-failures=2 --browserstack.buildName 'Appointment E2E Tests Android Chrome' --browserstack.config 'browserstack-mobile.yml'",
+    "e2e-tests-mobile-browserstack-ios-safari": "npx browserstack-node-sdk playwright test --grep e2e-mobile-suite --project=ios-safari --max-failures=2 --browserstack.buildName 'Appointment E2E Tests iOS Chrome' --browserstack.config 'browserstack-mobile.yml'",
     "prod-sanity-test": "npx playwright test --grep prod-sanity --project=firefox",
     "prod-sanity-test-headed": "npx playwright test --grep prod-sanity --project=firefox --headed",
-    "prod-sanity-test-browserstack-firefox": "npx browserstack-node-sdk playwright test --grep prod-sanity --project=Firefox-OSX --browserstack.buildName 'Appointment Production Sanity Test Firefox Desktop' --browserstack.config 'browserstack-desktop.yml'",
+    "prod-sanity-test-browserstack-firefox": "npx browserstack-node-sdk playwright test --grep prod-sanity --project=Firefox-OSX --max-failures=2 --browserstack.buildName 'Appointment Production Sanity Test Firefox Desktop' --browserstack.config 'browserstack-desktop.yml'",
     "prod-nightly-tests-browserstack-firefox": "npx browserstack-node-sdk playwright test --grep prod-nightly --project=Firefox-OSX --browserstack.buildName 'Appointment Nightly Tests Firefox Desktop' --browserstack.config 'browserstack-desktop-nightly.yml'",
     "prod-nightly-tests-browserstack-safari": "npx browserstack-node-sdk playwright test --grep prod-nightly --project=Safari-OSX --browserstack.buildName 'Appointment Nightly Tests Safari Desktop' --browserstack.config 'browserstack-desktop-nightly.yml'",
     "prod-nightly-tests-browserstack-chromium": "npx browserstack-node-sdk playwright test --grep prod-nightly --project=Chromium-Win11 --browserstack.buildName 'Appointment Nightly Tests Chromium Desktop' --browserstack.config 'browserstack-desktop-nightly.yml'",
-    "prod-nightly-tests-mobile-browserstack-android-chrome": "npx browserstack-node-sdk playwright test --grep prod-mobile-nightly --project=Chrome-GooglePixel10 --browserstack.buildName 'Appointment Nightly Tests Android Chrome' --browserstack.config 'browserstack-mobile-nightly.yml'",
+    "prod-nightly-tests-mobile-browserstack-android-chrome": "npx browserstack-node-sdk playwright test --grep prod-mobile-nightly --project=android-chrome --browserstack.buildName 'Appointment Nightly Tests Android Chrome' --browserstack.config 'browserstack-mobile-nightly.yml'",
+    "prod-nightly-tests-mobile-browserstack-ios-safari": "npx browserstack-node-sdk playwright test --grep prod-mobile-nightly --project=ios-safari --browserstack.buildName 'Appointment Nightly Tests iOS Safari' --browserstack.config 'browserstack-mobile-nightly.yml'",
     "postinstall": "npm update browserstack-node-sdk"
   },
   "keywords": [],

test/e2e/pages/settings-page.ts

@@ -5,15 +5,16 @@ import {
   APPT_HTML_DARK_MODE_CLASS,
   TIMEOUT_1_SECOND,
   TIMEOUT_2_SECONDS,
-  TIMEOUT_3_SECONDS,
   TIMEOUT_5_SECONDS,
   TIMEOUT_30_SECONDS,
   APPT_LANGUAGE_SETTING_EN,
   } from '../const/constants';
+import test from 'node:test';
 
 
 export class SettingsPage {
   readonly page: Page;
+  readonly testPlatform: string;
   readonly accountSettingsBtn: Locator;
   readonly connectedAppsBtn: Locator;
   readonly preferencesBtn: Locator;
@@ -47,10 +48,11 @@ export class SettingsPage {
   readonly savedSuccessfullyTextDE: Locator;
   readonly saveBtnDE: Locator;
   readonly revertBtn: Locator;
+  readonly googleSignInHdr: Locator;
 
-
-  constructor(page: Page) {
+  constructor(page: Page, testPlatform: string = 'desktop') {
     this.page = page;
+    this.testPlatform = testPlatform;
 
     // main settings view
     this.settingsHeaderEN = this.page.getByRole('main').getByText('Settings', { exact: true });
@@ -92,6 +94,16 @@ export class SettingsPage {
     this.addCaldavCloseModalBtn = this.page.getByRole('img', { name: 'Close' });
     this.addGoogleBtn = this.page.getByRole('button', { name: 'Add Google Calendar' });
     this.defaultCalendarConnectedCbox = this.page.locator('div').filter({ hasText: /^Default*/ }).getByTestId('checkbox-input');
+    this.googleSignInHdr = this.page.getByText('Sign in with Google');
+  }
+
+  /**
+   * Scroll the given element into view. The reason why we do this here is because playright doesn't yet supported this on ios.
+   */
+  async scrollIntoView(targetElement: Locator, timeout: number = 10000) {
+    if (!this.testPlatform.includes('ios')) {
+      await targetElement.scrollIntoViewIfNeeded({ timeout: timeout });
+    }
   }
 
   /**
@@ -100,7 +112,7 @@ export class SettingsPage {
   async gotoAccountSettings() {
     await this.page.goto(APPT_SETTINGS_PAGE);
     await this.page.waitForTimeout(TIMEOUT_5_SECONDS);
-    await this.accountSettingsBtn.scrollIntoViewIfNeeded();
+    await this.scrollIntoView(this.accountSettingsBtn);
     await this.accountSettingsBtn.click();
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
   }
@@ -111,7 +123,7 @@ export class SettingsPage {
   async gotoPreferencesSettings() {
     await this.page.goto(APPT_SETTINGS_PAGE);
     await this.page.waitForTimeout(TIMEOUT_5_SECONDS);
-    await this.preferencesBtn.scrollIntoViewIfNeeded({ timeout: TIMEOUT_30_SECONDS });
+    await this.scrollIntoView(this.preferencesBtn, TIMEOUT_30_SECONDS);
     await this.preferencesBtn.click();
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
   }
@@ -122,7 +134,7 @@ export class SettingsPage {
   async gotoConnectedAppSettings() {
     await this.page.goto(APPT_SETTINGS_PAGE);
     await this.page.waitForTimeout(TIMEOUT_5_SECONDS);
-    await this.connectedAppsBtn.scrollIntoViewIfNeeded();
+    await this.scrollIntoView(this.connectedAppsBtn);
     await this.connectedAppsBtn.click();
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
   }
@@ -132,31 +144,32 @@ export class SettingsPage {
    */
   async changeDefaultTimezoneSetting(timezone: string) {
     await this.defaultTimeZoneSelect.waitFor( { timeout: TIMEOUT_30_SECONDS });
-    await this.defaultTimeZoneSelect.scrollIntoViewIfNeeded();
+    await this.scrollIntoView(this.defaultTimeZoneSelect);
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
     await this.defaultTimeZoneSelect.selectOption(timezone, { timeout: TIMEOUT_30_SECONDS });
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
-    await this.saveBtnEN.scrollIntoViewIfNeeded();
+    await this.scrollIntoView(this.saveBtnEN);
     await this.saveBtnEN.click();
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
     await expect(this.savedSuccessfullyTextEN).toBeVisible();
+    await this.page.waitForTimeout(TIMEOUT_2_SECONDS);
   }
 
   /**
    * Change the language setting
    */
   async changeLanguageSetting(currentLanguage: string, newLanguage: string) {
-    await this.languageSelect.scrollIntoViewIfNeeded();
+    await this.scrollIntoView(this.languageSelect);
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
     await this.languageSelect.selectOption(newLanguage, { timeout: TIMEOUT_30_SECONDS });
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
     if (currentLanguage == APPT_LANGUAGE_SETTING_EN) {
-      await this.saveBtnEN.scrollIntoViewIfNeeded();
+      await this.scrollIntoView(this.saveBtnEN);
       await this.saveBtnEN.click();
       await this.page.waitForTimeout(TIMEOUT_1_SECOND);
       await expect(this.savedSuccessfullyTextDE).toBeVisible();
     } else {
-      await this.saveBtnDE.scrollIntoViewIfNeeded();
+      await this.scrollIntoView(this.saveBtnDE);
       await this.saveBtnDE.click();
       await this.page.waitForTimeout(TIMEOUT_1_SECOND);
       await expect(this.savedSuccessfullyTextEN).toBeVisible();
@@ -169,10 +182,11 @@ export class SettingsPage {
    */
   async changeThemeSetting(theme: string) {
     await this.themeSelect.waitFor({ timeout: TIMEOUT_30_SECONDS });
-    await this.themeSelect.scrollIntoViewIfNeeded();
+    await this.scrollIntoView(this.themeSelect);
     await this.themeSelect.selectOption(theme);
     await this.page.waitForTimeout(TIMEOUT_2_SECONDS);
     await this.saveBtnEN.click();
+    await this.page.waitForTimeout(TIMEOUT_2_SECONDS);
     await expect(this.savedSuccessfullyTextEN).toBeVisible({ timeout: TIMEOUT_30_SECONDS });
     // wait for theme to take affect, can take time especially on browserstack
     await this.page.waitForTimeout(TIMEOUT_5_SECONDS);
@@ -199,7 +213,7 @@ export class SettingsPage {
       await this.startOfWeekSundayBtn.click({ timeout: TIMEOUT_30_SECONDS });
     }
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
-    await this.saveBtnEN.scrollIntoViewIfNeeded();
+    await this.scrollIntoView(this.saveBtnEN);
     await this.saveBtnEN.click();
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
     await expect(this.savedSuccessfullyTextEN).toBeVisible();

test/e2e/pages/tb-accts-page.ts

@@ -18,16 +18,17 @@ export class TBAcctsPage {
     this.userAvatar = this.page.getByTestId('avatar-default');
     this.emailInput = this.page.getByTestId('username-input');
     this.passwordInput = this.page.getByTestId('password-input');
-    this.signInButton = this.page.getByRole('button', { name: 'Sign in' });
+    this.signInButton = this.page.getByTestId('submit-btn');
     this.loginEmailInput = this.page.getByLabel('Email');
     this.localDevpasswordInput = this.page.getByLabel('Password');
     this.loginDialogContinueBtn = this.page.getByTitle('Continue');
   }
 
   /**
-   * Sign in to TB Accounts using the provided email and password.
+   * Sign in to TB Accounts using the provided email and password. We provide the playwright
+   * test project name (i.e. 'android-chrome') as some actions differ on different mobile platforms.
    */
-  async signIn() {
+  async signIn(testProjectName: string = 'desktop') {
     console.log('signing in to TB Accounts');
     expect(TB_ACCTS_EMAIL, 'getting TB_ACCTS_EMAIL env var').toBeTruthy();
     expect(TB_ACCTS_PWORD, 'getting TB_ACCTS_PWORD env var').toBeTruthy();
@@ -37,7 +38,12 @@ export class TBAcctsPage {
     await this.passwordInput.fill(String(TB_ACCTS_PWORD));
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
     await this.page.waitForTimeout(TIMEOUT_1_SECOND);
-    await this.signInButton.click({ force: true });
+    // 'force' is needed for android but doesn't work on ios
+    if (testProjectName.includes('android')) {
+      await this.signInButton.click({ force: true });
+    } else {
+      await this.signInButton.click({ timeout: TIMEOUT_10_SECONDS });
+    }
     await this.page.waitForTimeout(TIMEOUT_10_SECONDS);
   }