Skip to content

Commit 85f95c9

Browse files
authored
Update the 'submit an issue' E2E test and have it run nightly in a new test job (#480)
1 parent 3459832 commit 85f95c9

18 files changed

+518
-226
lines changed

.github/workflows/merge.yml

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -366,16 +366,10 @@ jobs:
366366
runs-on: ubuntu-latest
367367
environment: staging
368368
env:
369-
ACCTS_FXA_EMAIL: ${{ secrets.E2E_ACCTS_FXA_EMAIL }}
370-
ACCTS_FXA_PWORD: ${{ secrets.E2E_ACCTS_FXA_PWORD }}
371-
FXA_CLIENT_ID: ${{ secrets.E2E_ACCTS_FXA_CLIENT_ID }}
372-
FXA_SECRET: ${{ secrets.E2E_ACCTS_FXA_SECRET }}
373-
SECRET_KEY: ${{ secrets.E2E_ACCTS_SECRET_KEY }}
374-
LOGIN_CODE_SECRET: ${{ secrets.E2E_ACCTS_LOGIN_CODE_SECRET }}
375-
FXA_ENCRYPT_SECRET: ${{ secrets.E2E_ACCTS_FXA_ENCRYPT_SECRET }}
376-
THUNDERMAIL_USERNAME: ${{ secrets.E2E_THUNDERMAIL_USERNAME }}
377-
THUNDERMAIL_EMAIL_ADDRESS: ${{ secrets.E2E_THUNDERMAIL_EMAIL_ADDRESS }}
378-
EMAIL_SIGN_UP_ADDRESS: ${{ secrets.E2E_ACCTS_PADDLE_EMAIL_SIGN_UP_ADDRESS }}
369+
ACCTS_OIDC_EMAIL: ${{ secrets.E2E_ACCTS_OIDC_EMAIL }}
370+
ACCTS_OIDC_PWORD: ${{ secrets.E2E_ACCTS_OIDC_PWORD }}
371+
ACCTS_RECOVERY_EMAIL: ${{ secrets.E2E_ACCTS_RECOVERY_EMAIL }}
372+
PRIMARY_THUNDERMAIL_EMAIL: ${{ secrets.E2E_PRIMARY_THUNDERMAIL_EMAIL }}
379373
steps:
380374
- uses: actions/checkout@v4
381375

@@ -402,4 +396,4 @@ jobs:
402396
run: |
403397
cd ./test/e2e
404398
cp .env.stage.example .env
405-
npm run e2e-test-browserstack-gha
399+
npm run e2e-test-browserstack-desktop-firefox
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: nightly-tests-desktop
2+
3+
concurrency:
4+
group: nightly-tests-desktop
5+
cancel-in-progress: true
6+
7+
on:
8+
schedule:
9+
# run every day at 4am UTC (11PM EST)
10+
- cron: '0 4 * * *'
11+
# Allows you to run this workflow manually from the Actions tab
12+
workflow_dispatch:
13+
14+
permissions:
15+
contents: read # This is required for actions/checkout
16+
17+
jobs:
18+
prod-sanity-browserstack:
19+
name: nightly-tests-desktop-browserstack
20+
runs-on: ubuntu-latest
21+
environment: production
22+
env:
23+
ACCTS_OIDC_EMAIL: ${{ secrets.E2E_ACCTS_OIDC_EMAIL }}
24+
ACCTS_OIDC_PWORD: ${{ secrets.E2E_ACCTS_OIDC_PWORD }}
25+
ACCTS_RECOVERY_EMAIL: ${{ secrets.E2E_ACCTS_RECOVERY_EMAIL }}
26+
PRIMARY_THUNDERMAIL_EMAIL: ${{ secrets.E2E_PRIMARY_THUNDERMAIL_EMAIL }}
27+
steps:
28+
- uses: actions/checkout@v4
29+
30+
- uses: actions/setup-node@v4
31+
with:
32+
node-version: 20
33+
cache: 'npm'
34+
cache-dependency-path: 'test/e2e/package-lock.json'
35+
36+
- name: Install dependencies
37+
run: |
38+
cd ./test/e2e
39+
npm install
40+
41+
- name: BrowserStack Env Setup
42+
uses: browserstack/github-actions/setup-env@master
43+
with:
44+
username: ${{ secrets.BROWSERSTACK_USERNAME }}
45+
access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
46+
project-name: 'Thunderbird Accounts'
47+
build-name: 'TB Accounts Nightly Tests (Desktop): BUILD_INFO'
48+
49+
- name: Prod E2E Tests on Desktop Firefox
50+
# don't send GHA failure email if any of the E2E tests fail, can be annoying (I check the jobs each day in BrowserStack)
51+
continue-on-error: true
52+
run: |
53+
cd ./test/e2e
54+
cp .env.prod.example .env
55+
npm run prod-nightly-tests-browserstack-desktop-firefox
56+
57+
- name: Prod E2E Tests on Desktop Chromium
58+
# don't send GHA failure email if any of the E2E tests fail, can be annoying (I check the jobs each day in BrowserStack)
59+
continue-on-error: true
60+
run: |
61+
cd ./test/e2e
62+
cp .env.prod.example .env
63+
npm run prod-nightly-tests-browserstack-desktop-chromium

test/e2e/.env.dev.example

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
ACCTS_TARGET_ENV=dev
33

44
# URLs
5-
ACCTS_CONTACT_URL=http://localhost:8087/contact
5+
ACCTS_HUB_URL=http://localhost:8087
66

77
# Host / ports / domains
88
ACCTS_HOST=localhost
@@ -16,17 +16,14 @@ SMTP_TLS="None"
1616
# Sign-in credentials
1717
ACCTS_OIDC_EMAIL=admin@example.org
1818
ACCTS_OIDC_PWORD=admin
19+
ACCTS_RECOVERY_EMAIL=admin@example.com
1920

20-
# A Thundermail email is required to be set up within the tb account
21-
THUNDERMAIL_USERNAME=admin@example.org
22-
THUNDERMAIL_EMAIL_ADDRESS=admin@example.org
23-
24-
# The customer email address to use in Paddle checkout form
25-
EMAIL_SIGN_UP_ADDRESS=melissa+test@thunderbird.net
21+
# Thundermail associated with the above account
22+
PRIMARY_THUNDERMAIL_EMAIL=
2623

2724
# Custom OIDC paths since this runs from outside the Docker context
2825
OIDC_URL_AUTH=http://localhost:8999/realms/tbpro/protocol/openid-connect/auth
2926
OIDC_URL_TOKEN=http://localhost:8999/realms/tbpro/protocol/openid-connect/token
3027
OIDC_URL_USER=http://localhost:8999/realms/tbpro/protocol/openid-connect/userinfo
3128
OIDC_URL_JWKS=http://localhost:8999/realms/tbpro/protocol/openid-connect/certs
32-
OIDC_URL_LOGOUT=http://localhost:8999/realms/tbpro/protocol/openid-connect/logout
29+
OIDC_URL_LOGOUT=http://localhost:8999/realms/tbpro/protocol/openid-connect/logout

test/e2e/.env.prod.example

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Accounts E2E Test Configuration
2+
ACCTS_TARGET_ENV=prod
3+
4+
# URLs
5+
ACCTS_HUB_URL=https://accounts.tb.pro
6+
7+
# Host / ports / domains
8+
ACCTS_HOST=mail.thundermail.com
9+
IMAP_PORT=993
10+
JMAP_PORT=443
11+
SMTP_PORT=465
12+
13+
# Sign-in credentials
14+
ACCTS_OIDC_EMAIL=
15+
ACCTS_OIDC_PWORD=
16+
ACCTS_RECOVERY_EMAIL=
17+
18+
# Thundermail associated with the above account
19+
PRIMARY_THUNDERMAIL_EMAIL=

test/e2e/.env.stage.example

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
ACCTS_TARGET_ENV=stage
33

44
# URLs
5-
ACCTS_CONTACT_URL=https://accounts-stage.tb.pro/contact
5+
ACCTS_HUB_URL=https://accounts-stage.tb.pro
66

77
# Host / ports / domains
88
ACCTS_HOST=mail.stage-thundermail.com
@@ -11,12 +11,9 @@ JMAP_PORT=443
1111
SMTP_PORT=465
1212

1313
# Sign-in credentials
14-
ACCTS_OIDC_EMAIL=admin@example.org
14+
ACCTS_OIDC_EMAIL=
1515
ACCTS_OIDC_PWORD=
16+
ACCTS_RECOVERY_EMAIL=
1617

17-
# A Thundermail email is required to be set up within the tb account
18-
THUNDERMAIL_USERNAME=
19-
THUNDERMAIL_EMAIL_ADDRESS=
20-
21-
# The customer email address to use in Paddle checkout form
22-
EMAIL_SIGN_UP_ADDRESS=
18+
# Thundermail associated with the above account
19+
PRIMARY_THUNDERMAIL_EMAIL=

test/e2e/README.md

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,7 @@ npx playwright install
1919

2020
## E2E Test Prerequisites
2121

22-
The E2E tests require an existing TB Pro (keycloak) account used to sign-in to TB Accounts, and reads this from your local .env file. This includes:
23-
24-
- Credentials for an existing TB Pro (keycloak) account
22+
The E2E tests require an existing TB Pro (keycloak) account used to sign-in to TB Accounts, and reads this from your local .env file.
2523

2624
In addition, you will need the env vars for the Paddle sandbox (found in the Services vault in 1Password)
2725

@@ -39,7 +37,7 @@ PADDLE_ENV=sandbox
3937

4038
### Add TB Pro Account Information
4139

42-
When running the TB Accounts local dev stack it uses the TB Pro stage env for sign-in, and then redirects back to your localhost afterwords. So you will use your TB Pro stage test account (keycloak) also when running the E2E tests against your local dev stack.
40+
When running the TB Accounts local dev stack setup your existing local OIDC/Keycloak credentials in your .env file as follows.
4341

4442
First copy over the provided `.env.dev.example` to a local `.env`:
4543

@@ -51,8 +49,11 @@ cp .env.dev.example .env
5149
Then edit your local `.env` file and provide the following values:
5250

5351
```dotenv
54-
ACCTS_OIDC_EMAIL=<existing-tbpro-stage-user-email>
55-
ACCTS_OIDC_PWORD=<exisiting-tbro-stage-password>
52+
ACCTS_HUB_URL=<URL-for-TB-Accts-Hub>
53+
ACCTS_OIDC_EMAIL=<existing-tbpro-user-email>
54+
ACCTS_OIDC_PWORD=<exisiting-tbro-password>
55+
ACCTS_RECOVERY_EMAIL=<tbpro-recovery-email>
56+
PRIMARY_THUNDERMAIL_EMAIL=<primary-thundermail-email-address>
5657
```
5758

5859
## Setup Paddle backend for subscription tests (optional)
@@ -110,10 +111,11 @@ cp .env.stage.example .env
110111
Then edit your local `.env` file and provide the following values:
111112

112113
```dotenv
113-
ACCTS_OIDC_EMAIL=<existing-tbpro-stage-user-email>
114-
ACCTS_OIDC_PWORD=<exisiting-tbro-stage-password>
115-
THUNDERMAIL_USERNAME=<thundermail username associated with the above acct>
116-
THUNDERMAIL_EMAIL_ADDRESS=<thundermail email address associated with the above acct>
114+
ACCTS_HUB_URL=<URL-for-TB-Accts-Hub>
115+
ACCTS_OIDC_EMAIL=<existing-tbpro-user-email>
116+
ACCTS_OIDC_PWORD=<exisiting-tbro-password>
117+
ACCTS_RECOVERY_EMAIL=<tbpro-recovery-email>
118+
PRIMARY_THUNDERMAIL_EMAIL=<primary-thundermail-email-address>
117119
```
118120

119121
To run the E2E tests headless (still in `test/e2e`):
@@ -143,10 +145,11 @@ You can run the E2E tests from your local machine pointed at the stage environme
143145
Once you have credentials for an existing TB Accounts test account, edit your local `.env` file (that you first copied from `.env.stage.example`) and add these details (more information found above):
144146

145147
```dotenv
146-
ACCTS_OIDC_EMAIL=<existing-tbpro-stage-user-email>
147-
ACCTS_OIDC_PWORD=<exisiting-tbro-stage-password>
148-
THUNDERMAIL_USERNAME=<thundermail username associated with the above acct>
149-
THUNDERMAIL_EMAIL_ADDRESS=<thundermail email address associated with the above acct>
148+
ACCTS_HUB_URL=<URL-for-TB-Accts-Hub>
149+
ACCTS_OIDC_EMAIL=<existing-tbpro-user-email>
150+
ACCTS_OIDC_PWORD=<exisiting-tbro-password>
151+
ACCTS_RECOVERY_EMAIL=<tbpro-recovery-email>
152+
PRIMARY_THUNDERMAIL_EMAIL=<primary-thundermail-email-address>
150153
```
151154

152155
Also in order to run on BrowserStack you need to provide your BrowserStack credentials. Sign into your BrowserStack account and navigate to your `User Profile` and find your auth username and access key. In your local terminal export the following env vars to set the BrowserStack credentials that the tests will use:
@@ -162,7 +165,7 @@ export BROWSERSTACK_ACCESS_KEY=<your-browserstack-access-key>
162165
To run the E2E tests on BrowserStack (still in `test/e2e`):
163166

164167
```bash
165-
npm run e2e-test-browserstack
168+
npm run e2e-test-browserstack-desktop-firefox
166169
```
167170

168171
After the tests finish in your local console you'll see a link to the BrowserStack test session; when signed into your BrowserStack account you'll be able to use that link to see the test session results including video playback.
Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ platforms:
3333
name: Firefox-OSX
3434
setup:
3535
- name: 'setup_firefox_auth'
36-
testMatch: 'auth.setup.ts'
36+
testMatch: 'auth.desktop.setup.ts'
3737
use:
3838
storageState: 'test-results/.auth/user.json'
3939

@@ -45,7 +45,7 @@ platforms:
4545
name: Chromium-OSX
4646
setup:
4747
- name: 'setup_chrome_auth'
48-
testMatch: 'auth.setup.ts'
48+
testMatch: 'auth.desktop.setup.ts'
4949
use:
5050
storageState: 'test-results/.auth/user.json'
5151

test/e2e/browserstack-desktop.yml

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# =============================
2+
# Set BrowserStack Credentials
3+
# =============================
4+
# In the terminal export BROWSERSTACK_USERNAME=<secret>
5+
# In the terminal export BROWSERSTACK_ACCESS_KEY=<secret>
6+
7+
# ======================
8+
# BrowserStack Reporting
9+
# ======================
10+
# The following capabilities are used to set up reporting on BrowserStack:
11+
# Set 'projectName' to the name of your project. Example, Marketing Website
12+
projectName: Thunderbird Accounts
13+
# Set `buildName` as the name of the job / testsuite being run
14+
buildName: TB Accounts E2E Tests
15+
# `buildIdentifier` is a unique id to differentiate every execution that gets appended to
16+
# buildName. Choose your buildIdentifier format from the available expressions:
17+
# ${BUILD_NUMBER} (Default): Generates an incremental counter with every execution
18+
# ${DATE_TIME}: Generates a Timestamp with every execution. Eg. 05-Nov-19:30
19+
# Read more about buildIdentifiers here -> https://www.browserstack.com/docs/automate/selenium/organize-tests
20+
buildIdentifier: '${DATE_TIME}'
21+
22+
# =======================================
23+
# Platforms (Browsers / Devices to test)
24+
# =======================================
25+
# Platforms object contains all the browser / device combinations you want to test on.
26+
# Entire list available here -> (https://www.browserstack.com/list-of-browsers-and-platforms/automate)
27+
platforms:
28+
- os: OS X
29+
osVersion: Sequoia
30+
browserName: playwright-firefox
31+
browserVersion: latest
32+
playwrightConfigOptions:
33+
name: Firefox-OSX
34+
setup:
35+
- name: 'setup_firefox_auth'
36+
testMatch: 'auth.desktop.setup.ts'
37+
use:
38+
storageState: 'test-results/.auth/user.json'
39+
40+
# =======================
41+
# Parallels per Platform
42+
# =======================
43+
# The number of parallel threads to be used for each platform set.
44+
# BrowserStack's SDK runner will select the best strategy based on the configured value
45+
#
46+
# Example 1 - If you have configured 3 platforms and set `parallelsPerPlatform` as 2, a total of 6 (2 * 3) parallel threads will be used on BrowserStack
47+
#
48+
# Example 2 - If you have configured 1 platform and set `parallelsPerPlatform` as 5, a total of 5 (1 * 5) parallel threads will be used on BrowserStack
49+
parallelsPerPlatform: 1
50+
51+
# ==========================================
52+
# BrowserStack Local
53+
# (For localhost, staging/private websites)
54+
# ==========================================
55+
# Set browserStackLocal to true if your website under test is not accessible publicly over the internet
56+
# Learn more about how BrowserStack Local works here -> https://www.browserstack.com/docs/automate/selenium/local-testing-introduction
57+
browserstackLocal: false # <boolean> (Default false)
58+
# browserStackLocalOptions:
59+
# Options to be passed to BrowserStack local in-case of advanced configurations
60+
# localIdentifier: # <string> (Default: null) Needed if you need to run multiple instances of local.
61+
# forceLocal: true # <boolean> (Default: false) Set to true if you need to resolve all your traffic via BrowserStack Local tunnel.
62+
# Entire list of arguments available here -> https://www.browserstack.com/docs/automate/selenium/manage-incoming-connections
63+
64+
# ===================
65+
# Debugging features
66+
# ===================
67+
debug: false # <boolean> # Set to true if you need screenshots for every selenium command ran
68+
networkLogs: false # <boolean> Set to true to enable HAR logs capturing; off as may contain sensitive info like login API requests
69+
consoleLogs: info # <string> Remote browser's console debug levels to be printed (`disable`, `errors`, `warnings`, `info`, or `verbose`)
70+
framework: playwright
71+
browserstack.playwrightVersion: 1.55.0 # must match our client playwright version in package.json; this is the latest supported in BrowserStack
72+
browserstack.playwrightLogs: false # disable playwright logs appearing on browserstack builds as may contain sensitive info like credentials
73+
# CUSTOM_TAG_<INT>: # <string> (Default: parent folder name of the test file) Custom tag for your test suite
74+
browserstack.maskBasicAuth: true # mask username and passwords from browserstack session logs
75+
browserstack.maskCommands: sendType, sendPress, setHTTPCredentials, setStorageState, setWebAuthnCredentials # prevent sensitive info appearing in browserstack session logs
76+
# Test Observability is an intelligent test reporting & debugging product. It collects data using the SDK. Read more about what data is collected at https://www.browserstack.com/docs/test-observability/references/terms-and-conditions
77+
# Visit observability.browserstack.com to see your test reports and insights. To disable test observability, specify `testObservability: false` in the key below.
78+
testObservability: false

test/e2e/const/constants.ts

Lines changed: 7 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,24 @@
22
export const ACCTS_TARGET_ENV = String(process.env.ACCTS_TARGET_ENV);
33

44
// tb accounts urls
5-
export const ACCTS_CONTACT_URL = String(process.env.ACCTS_CONTACT_URL);
5+
export const ACCTS_HUB_URL = String(process.env.ACCTS_HUB_URL);
6+
export const ACCTS_CONTACT_URL = String(`${ACCTS_HUB_URL}/contact`);
67

7-
// sign-in credentials and corresponding account display name
8+
// sign-in credentials and corresponding info
89
export const ACCTS_OIDC_EMAIL = String(process.env.ACCTS_OIDC_EMAIL);
910
export const ACCTS_OIDC_PWORD = String(process.env.ACCTS_OIDC_PWORD);
11+
export const ACCTS_RECOVERY_EMAIL = String(process.env.ACCTS_RECOVERY_EMAIL);
12+
export const PRIMARY_THUNDERMAIL_EMAIL = String(process.env.PRIMARY_THUNDERMAIL_EMAIL);
1013

1114
// playwright test tags
1215
export const PLAYWRIGHT_TAG_E2E_SUITE = '@e2e-suite';
16+
export const PLAYWRIGHT_TAG_E2E_PROD_DESKTOP_NIGHTLY = '@e2e-prod-desktop-nighlty';
1317

1418
// timeouts
1519
export const TIMEOUT_2_SECONDS = 2000;
1620
export const TIMEOUT_5_SECONDS = 5000;
21+
export const TIMEOUT_10_SECONDS = 5000;
1722
export const TIMEOUT_30_SECONDS = 30000;
18-
export const TIMEOUT_60_SECONDS = 60000;
1923

2024
// connection info
2125
export const ACCTS_HOST = String(process.env.ACCTS_HOST);
@@ -25,16 +29,3 @@ export const SMTP_PORT = Number(process.env.SMTP_PORT);
2529
export const IMAP_TLS = String(process.env.IMAP_TLS) ?? 'SSL/TLS';
2630
export const JMAP_TLS = String(process.env.JMAP_TLS) ?? 'SSL/TLS';
2731
export const SMTP_TLS = String(process.env.SMTP_TLS) ?? 'SSL/TLS';
28-
export const APP_PASSWORD = 'Your App Password';
29-
export const YOUR_EMAIL_LBL = 'Your email address:';
30-
export const THUNDERMAIL_USERNAME = String(process.env.THUNDERMAIL_USERNAME);
31-
export const THUNDERMAIL_EMAIL_ADDRESS = String(process.env.THUNDERMAIL_EMAIL_ADDRESS);
32-
33-
// mock responses
34-
export const MOCK_RESPONSE_OK = {
35-
status: 200,
36-
contentType: 'application/json',
37-
body: JSON.stringify({
38-
fake_data: 'fake response data',
39-
}),
40-
};

test/e2e/package.json

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@
66
"e2e-test": "npx playwright test --grep e2e-suite --project=firefox",
77
"e2e-test-headed": "npx playwright test --grep e2e-suite --project=firefox --headed",
88
"e2e-test-debug": "npx playwright test --grep e2e-suite --project=firefox --headed --ui",
9-
"e2e-test-browserstack": "npx browserstack-node-sdk playwright test --grep e2e-suite --browserstack.buildName 'TB Accounts E2E Tests'",
10-
"e2e-test-browserstack-gha": "npx browserstack-node-sdk playwright test --grep e2e-suite",
9+
"e2e-test-browserstack-desktop-firefox": "npx browserstack-node-sdk playwright test --grep e2e-suite --project=Firefox-OSX --browserstack.config 'browserstack-desktop.yml'",
10+
"prod-nightly-tests-browserstack-desktop-firefox": "npx browserstack-node-sdk playwright test --grep e2e-prod-desktop-nighlty --project=Firefox-OSX --browserstack.config 'browserstack-desktop-nightly.yml'",
11+
"prod-nightly-tests-browserstack-desktop-chromium": "npx browserstack-node-sdk playwright test --grep e2e-prod-desktop-nighlty --project=Chromium-OSX --browserstack.config 'browserstack-desktop-nightly.yml'",
1112
"postinstall": "npm update browserstack-node-sdk"
1213
},
1314
"keywords": [],

0 commit comments

Comments
 (0)