Skip to content

Commit 9527fdc

Browse files
jprusikjprusik
authored
[PM-19288] Utilize production feature flag configuration in testing (#319)
* create config fetching script * update actions workflows to use configuration matching * add flags.json artifact to test-all-custom-flags * remove references to notification "bar" * add known-failure cases * update workflows * add known-failure cases
1 parent 1a7e9d5 commit 9527fdc

File tree

8 files changed

+258
-46
lines changed

8 files changed

+258
-46
lines changed

‎.env.example

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ VAULT_HOST_PORT=8443
2323
VAULT_HOST_INSECURE_PORT=8080
2424
VAULT_HOST_URL="https://localhost"
2525

26+
# (Optional) Specify an an API endpoint returning a vault configuration.
27+
# Used to match a remote server configuration.
28+
REMOTE_VAULT_CONFIG_MATCH="https://localhost:8443/api/config"
29+
2630
# The extension that will have tests running against it
2731
EXTENSION_BUILD_PATH="clients/apps/browser/build"
2832

‎.github/workflows/test-all-custom-flags.yml

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
name: Test-all-custom-flags
2+
run-name: All tests with extension build ${{ inputs.CLIENTS_BRANCH }} by @${{ github.actor }}
3+
on:
4+
push:
5+
branches:
6+
- "main"
7+
pull_request:
8+
workflow_dispatch:
9+
inputs:
10+
CLIENTS_BRANCH:
11+
default: "main"
12+
description: "clients branch of browser build to use"
13+
required: true
14+
type: string
15+
FEATURE_FLAGS:
16+
default: "{}"
17+
description: 'JSON key-value pairs representing feature flag states. (e.g. {"autofill-v2": true, "autofill-overlay": false})'
18+
required: true
19+
type: string
20+
jobs:
21+
build-and-test:
22+
name: Build and test
23+
runs-on: ubuntu-24.04
24+
steps:
25+
- name: Checkout
26+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
27+
- name: Setup Node
28+
uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0
29+
with:
30+
cache: "npm"
31+
cache-dependency-path: "**/package-lock.json"
32+
node-version: "20"
33+
34+
- name: Create dotenv file
35+
run: |
36+
sudo setcap 'cap_net_bind_service=+ep' `which node`
37+
echo "${{ secrets.ENV_FILE }}" | base64 --decode > .env
38+
echo "BW_INSTALLATION_ID=${{ secrets.BW_INSTALLATION_ID }}" >> .env
39+
echo "BW_INSTALLATION_KEY=${{ secrets.BW_INSTALLATION_KEY }}" >> .env
40+
41+
- name: Create feature flags file
42+
run: echo "{\"flagValues\":${{ inputs.FEATURE_FLAGS || '{}' }}}" > flags.json
43+
44+
- name: Download extension artifact
45+
uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3.1.4
46+
with:
47+
github_token: ${{ secrets.GITHUB_TOKEN }}
48+
workflow: build-browser.yml
49+
workflow_conclusion: ""
50+
branch: ${{ inputs.CLIENTS_BRANCH || 'main' }}
51+
name: ^dist-chrome-MV3-\w{7}\.zip$
52+
name_is_regexp: true
53+
repo: bitwarden/clients
54+
if_no_artifact_found: fail
55+
skip_unpack: true
56+
57+
- name: Unzip extension artifact
58+
run: |
59+
unzip -o *dist-chrome-*.zip.zip
60+
unzip -o 'dist-chrome*.zip' -d build
61+
62+
- name: Generate and install certs
63+
run: |
64+
npm run setup:ssl
65+
sudo apt-get install libnss3-tools=2:3.98-1build1
66+
. .env
67+
mkdir -p $HOME/.pki/nssdb
68+
certutil -d $HOME/.pki/nssdb -N --empty-password
69+
certutil -d sql:$HOME/.pki/nssdb -A -t "CP,CP," -n TestAutomationSSL -i ./$BW_SSL_CERT
70+
71+
- name: Install Bitwarden CLI
72+
run: npm install -g @bitwarden/[email protected]
73+
74+
- name: Install project dependencies
75+
run: |
76+
npm ci
77+
npx playwright install --with-deps chromium
78+
79+
- name: Build and start the test vault
80+
run: docker compose up -d --build --remove-orphans --wait --wait-timeout 60
81+
82+
- name: Setup the vault test account
83+
run: npm run seed:vault:account
84+
85+
- name: Setup the vault test account ciphers
86+
run: |
87+
npm run start:cli
88+
npm run seed:vault:ciphers
89+
90+
- name: Download test site build
91+
uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3.1.4
92+
with:
93+
github_token: ${{ secrets.GITHUB_TOKEN }}
94+
workflow: build.yml
95+
workflow_conclusion: ""
96+
branch: main
97+
name: build-files
98+
path: test-site
99+
repo: bitwarden/test-the-web
100+
if_no_artifact_found: fail
101+
102+
- name: Copy over certs and install test site dependencies
103+
run: |
104+
cp ssl.crt test-site/api/
105+
cp ssl.key test-site/api/
106+
cd test-site
107+
npm ci
108+
109+
- name: Run all tests
110+
run: npm run test:static:ci
111+
112+
- name: Update job summary
113+
if: always()
114+
run: echo "$(<./test-summary/test-summary.md)" >> $GITHUB_STEP_SUMMARY
115+
116+
- name: Upload results as artifact
117+
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
118+
if: always()
119+
with:
120+
name: test-summary
121+
path: |
122+
./flags.json
123+
./test-summary
124+
./tests-out/videos
125+
./tests-out/screenshots

‎.github/workflows/test-all.yml

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,9 @@ on:
1212
description: "clients branch of browser build to use"
1313
required: true
1414
type: string
15-
FEATURE_FLAGS:
16-
default: "{}"
17-
description: 'JSON key-value pairs representing feature flag states. (e.g. {"autofill-v2": true, "autofill-overlay": false})'
18-
required: true
15+
REMOTE_VAULT_CONFIG_MATCH:
16+
description: "URL to an endpoint returning a vault configuration (will default to US prod configuration)"
17+
required: false
1918
type: string
2019
jobs:
2120
build-and-test:
@@ -37,9 +36,7 @@ jobs:
3736
echo "${{ secrets.ENV_FILE }}" | base64 --decode > .env
3837
echo "BW_INSTALLATION_ID=${{ secrets.BW_INSTALLATION_ID }}" >> .env
3938
echo "BW_INSTALLATION_KEY=${{ secrets.BW_INSTALLATION_KEY }}" >> .env
40-
41-
- name: Create feature flags file
42-
run: echo "{\"flagValues\":${{ inputs.FEATURE_FLAGS || '{}' }}}" > flags.json
39+
echo "REMOTE_VAULT_CONFIG_MATCH=${{ inputs.REMOTE_VAULT_CONFIG_MATCH || vars.BW_REMOTE_VAULT_CONFIG_MATCH }}" >> .env
4340
4441
- name: Download extension artifact
4542
uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3.1.4
@@ -48,7 +45,7 @@ jobs:
4845
workflow: build-browser.yml
4946
workflow_conclusion: ""
5047
branch: ${{ inputs.CLIENTS_BRANCH || 'main' }}
51-
name: dist-chrome-MV3-\w{7}\.zip
48+
name: ^dist-chrome-MV3-\w{7}\.zip$
5249
name_is_regexp: true
5350
repo: bitwarden/clients
5451
if_no_artifact_found: fail
@@ -76,6 +73,11 @@ jobs:
7673
npm ci
7774
npx playwright install --with-deps chromium
7875
76+
- name: Match feature flags file to remote config
77+
run: |
78+
touch flags.json
79+
npm run setup:flags
80+
7981
- name: Build and start the test vault
8082
run: docker compose up -d --build --remove-orphans --wait --wait-timeout 60
8183

@@ -119,6 +121,7 @@ jobs:
119121
with:
120122
name: test-summary
121123
path: |
124+
./flags.json
122125
./test-summary
123126
./tests-out/videos
124127
./tests-out/screenshots

‎constants/test-pages.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ export const testPages: PageTest[] = [
8181
skipTests: [
8282
TestNames.InlineMenuAutofill, // @TODO known failure - inline menu appears, but fails to autofill (PM-8693)
8383
TestNames.MessageAutofill, // @TODO known failure - fails to autofill (PM-8693)
84+
TestNames.NewCredentialsNotification, // @TODO known failure - regression (PM-19363)
85+
TestNames.PasswordUpdateNotification, // @TODO known failure - regression (PM-19363)
8486
],
8587
},
8688
{
@@ -123,6 +125,7 @@ export const testPages: PageTest[] = [
123125
},
124126
},
125127
skipTests: [
128+
TestNames.MessageAutofill, // @TODO known failure - notification appears inappropriately (PM-19376)
126129
TestNames.NewCredentialsNotification, // @TODO known failure - save prompt does not appear (PM-8697)
127130
TestNames.PasswordUpdateNotification, // @TODO known failure - update prompt does not appear (PM-8697)
128131
],
@@ -189,6 +192,7 @@ export const testPages: PageTest[] = [
189192
password: { selector: "#password", value: "fakeMultiStepPassword" },
190193
},
191194
skipTests: [
195+
TestNames.MessageAutofill, // @TODO known failure - notification appears inappropriately (PM-19376)
192196
TestNames.NewCredentialsNotification, // @TODO known failure - save prompt does not appear (PM-8697)
193197
TestNames.PasswordUpdateNotification, // @TODO known failure - update prompt does not appear (PM-8697)
194198
],

‎package.json

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
"setup:all": "./scripts/first-time-setup.sh",
4646
"setup:extension": "rimraf clients && git clone https://github.com/bitwarden/clients.git clients && cd clients && npm ci",
4747
"setup:install": "ts-node ./scripts/generate-installation.ts",
48+
"setup:flags": "ts-node ./scripts/match-server-config.ts",
4849
"setup:ssl": "./scripts/generate-certs.sh",
4950
"setup:test-site": "rimraf test-site && ./scripts/setup-test-site.sh",
5051
"setup:vault": "npm run seed:vault:account && npm run seed:vault:ciphers",
@@ -72,9 +73,9 @@
7273
"test:static:inline-menu": "npm run pretest && NODE_EXTRA_CA_CERTS=ssl.crt playwright test tests/static/inline-menu.spec.ts",
7374
"test:static:inline-menu:ci": "npm run pretest && NODE_EXTRA_CA_CERTS=ssl.crt xvfb-run playwright test tests/static/inline-menu.spec.ts",
7475
"test:static:inline-menu:headless": "npm run pretest && NODE_EXTRA_CA_CERTS=ssl.crt HEADLESS=true playwright test tests/static/inline-menu.spec.ts",
75-
"test:static:notification": "npm run pretest && NODE_EXTRA_CA_CERTS=ssl.crt playwright test tests/static/notification-bar.spec.ts",
76-
"test:static:notification:ci": "npm run pretest && NODE_EXTRA_CA_CERTS=ssl.crt xvfb-run playwright test tests/static/notification-bar.spec.ts",
77-
"test:static:notification:headless": "npm run pretest && NODE_EXTRA_CA_CERTS=ssl.crt HEADLESS=true playwright test tests/static/notification-bar.spec.ts",
76+
"test:static:notification": "npm run pretest && NODE_EXTRA_CA_CERTS=ssl.crt playwright test tests/static/notifications.spec.ts",
77+
"test:static:notification:ci": "npm run pretest && NODE_EXTRA_CA_CERTS=ssl.crt xvfb-run playwright test tests/static/notifications.spec.ts",
78+
"test:static:notification:headless": "npm run pretest && NODE_EXTRA_CA_CERTS=ssl.crt HEADLESS=true playwright test tests/static/notifications.spec.ts",
7879
"test:webserve": "cd clients/apps/web && npm run build:bit:watch",
7980
"test:webserve:bitwarden": "cd clients/apps/web && ENV=cloud npm run build:oss:watch"
8081
},

‎scripts/match-server-config.ts

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import fetch from "cross-fetch";
2+
import fs from "fs";
3+
import { configDotenv } from "dotenv";
4+
5+
configDotenv();
6+
7+
type VaultConfigurationResponseData = {
8+
version: string;
9+
gitHash: string;
10+
server: string | null;
11+
environment: {
12+
cloudRegion: string;
13+
vault: string;
14+
api: string;
15+
identity: string;
16+
notifications: string;
17+
sso: string;
18+
};
19+
featureStates: {
20+
[key: string]: boolean;
21+
};
22+
settings: {
23+
disableUserRegistration: false;
24+
};
25+
object: string;
26+
};
27+
28+
async function matchRemoteFeatureFlags() {
29+
const { REMOTE_VAULT_CONFIG_MATCH } = process.env;
30+
if (REMOTE_VAULT_CONFIG_MATCH) {
31+
try {
32+
const response = await fetch(REMOTE_VAULT_CONFIG_MATCH);
33+
34+
const { featureStates } =
35+
((await response.json()) as VaultConfigurationResponseData) || {};
36+
37+
await fs.readFile("flags.json", "utf8", async (error, fileContent) => {
38+
if (error) {
39+
throw error;
40+
}
41+
42+
let parsedFile = {};
43+
44+
if (fileContent) {
45+
parsedFile = JSON.parse(fileContent);
46+
}
47+
48+
const fileData = { ...parsedFile, flagValues: { ...featureStates } };
49+
50+
const newFileContent = JSON.stringify(fileData);
51+
52+
await fs.writeFile("flags.json", newFileContent, "utf8", () => {});
53+
54+
console.log(
55+
"\x1b[1m\x1b[32m%s\x1b[0m", // bold, light green foreground
56+
`Feature flag values from ${REMOTE_VAULT_CONFIG_MATCH} have been successfully written to 'flags.json'!\n`,
57+
);
58+
});
59+
} catch (error) {
60+
throw error;
61+
}
62+
} else {
63+
console.warn(
64+
"\x1b[1m\x1b[33m%s\x1b[0m", // bold, yellow foreground
65+
"No remote config URL was provided!\n",
66+
);
67+
}
68+
69+
return;
70+
}
71+
72+
matchRemoteFeatureFlags();

‎tests/static/autofill-forms.spec.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -149,13 +149,13 @@ test.describe("Extension autofills forms when triggered", () => {
149149
// Submit
150150
await testPage.keyboard.press("Enter");
151151

152-
// Target notification close button since it's present on all notification bar cases
153-
const notificationBarCloseButtonLocator = testPage
152+
// Target notification close button since it's present on all notification cases
153+
const notificationCloseButtonLocator = testPage
154154
.frameLocator("#bit-notification-bar-iframe")
155155
.getByRole("button", { name: "Close" })
156156
.first();
157157

158-
await expect(notificationBarCloseButtonLocator).not.toBeVisible();
158+
await expect(notificationCloseButtonLocator).not.toBeVisible();
159159
});
160160
}
161161

0 commit comments

Comments
 (0)