Introduce WebkitHeadless browser testing (#8491)

* Introduce WebkitHeadless browser testing

Replace Safari with WebkitHeadless in list of supported karma browsers

Install playwright browsers in CI

Set WEBKIT_HEADLESS_BIN

Update launcher version

Run firestore test changed on push

Fix

Don't use  in test-changed-firestore

Ignore failing tests

* Formatting

* Respond to review comments

Author: dlarocque

.github/workflows/test-changed-auth.yml

@@ -102,3 +102,28 @@ jobs:
         run: xvfb-run yarn test:changed auth
         env:
           BROWSERS: 'Firefox'
+
+  test-webkit:
+    name: Test Auth on Webkit if Changed
+    runs-on: macos-latest
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Node (20)
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+      - name: Test setup and yarn install
+        run: |
+          cp config/ci.config.json config/project.json
+          yarn
+          npx playwright install webkit
+      - name: build
+        run: yarn build:changed auth
+      - name: Run tests on changed packages
+        run: yarn test:changed auth
+        env:
+          BROWSERS: 'WebkitHeadless'

.github/workflows/test-changed-firestore.yml

@@ -231,6 +231,62 @@ jobs:
           BROWSERS: 'Firefox'
           EXPERIMENTAL_MODE: true
 
+  compat-test-webkit:
+    name: Test Firestore Compatible on Webkit
+    runs-on: macos-latest
+    needs: build
+    if: ${{ needs.build.outputs.changed == 'true'}}
+    steps:
+      - name: Set up Node (20)
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+      - name: Download build archive
+        uses: actions/download-artifact@v3
+        with:
+          name: build.tar.gz
+      - name: Unzip build artifact
+        run: tar xf build.tar.gz
+      - name: Test setup
+        run: |
+          cp config/ci.config.json config/project.json
+          npx playwright install webkit
+      - name: Run compat tests
+        run: cd packages/firestore-compat && yarn run test:ci
+        env:
+          BROWSERS: 'WebkitHeadless'
+
+  test-webkit:
+    name: Test Firestore on Webkit
+    strategy:
+      matrix:
+        # TODO (dlarocque): Add test:travis once the browser tests are isolated
+        # Exclude test:travis for now, since it includes node tests, which are failing for
+        # some reason.
+        test-name: ["test:browser", "test:lite:browser", "test:browser:prod:nameddb", "test:lite:browser:nameddb"]
+    runs-on: macos-latest
+    needs: build
+    if: ${{ needs.build.outputs.changed == 'true'}}
+    steps:
+      - name: Download build archive
+        uses: actions/download-artifact@v3
+        with:
+          name: build.tar.gz
+      - name: Unzip build artifact
+        run: tar xf build.tar.gz
+      - name: Set up Node (20)
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+      - name: Test setup
+        run: |
+          cp config/ci.config.json config/project.json
+          npx playwright install webkit
+      - name: Run tests
+        run: cd packages/firestore && yarn run ${{ matrix.test-name }}
+        env:
+          BROWSERS: 'WebkitHeadless'
+          EXPERIMENTAL_MODE: true
   # A job that fails if any required job in the test matrix fails,
   # to be used as a required check for merging.
   check-required-tests:
@@ -241,4 +297,4 @@ jobs:
     steps:
       - name: Check test matrix
         if: needs.build.result == 'failure' || needs.test-chrome.result == 'failure' || needs.compat-test-chrome.result == 'failure'
-        run: exit 1
+        run: exit 1

.github/workflows/test-changed.yml

@@ -78,3 +78,29 @@ jobs:
         run: xvfb-run yarn test:changed core
         env:
           BROWSERS: 'Firefox'
+          
+          
+  test-webkit:
+    name: Test Packages With Changed Files in Webkit
+    runs-on: macos-latest
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Node (20)
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+      - name: Test setup and yarn install
+        run: |
+          cp config/ci.config.json config/project.json
+          yarn
+          npx playwright install webkit
+      - name: build
+        run: yarn build:changed core
+      - name: Run tests on changed packages
+        run: yarn test:changed core
+        env:
+          BROWSERS: 'WebkitHeadless'

config/karma.base.js

@@ -20,6 +20,33 @@ const path = require('path');
 const webpackTestConfig = require('./webpack.test');
 const { argv } = require('yargs');
 
+function determineBrowsers() {
+  const supportedBrowsers = ['ChromeHeadless', 'WebkitHeadless', 'Firefox'];
+
+  if (process.env.BROWSERS) {
+    const browsers = process.env.BROWSERS.split(',');
+
+    const validBrowsers = browsers.filter(browser =>
+      supportedBrowsers.includes(browser)
+    );
+    if (validBrowsers.length === 0) {
+      console.error(
+        `The \'BROWSER\' environment variable was set, but no supported browsers were listed. The supported browsers are ${JSON.stringify(
+          supportedBrowsers
+        )}.`
+      );
+      return [];
+    } else {
+      return validBrowsers;
+    }
+  } else {
+    console.log(
+      "The 'BROWSER' environment variable is undefined. Defaulting to 'ChromeHeadless'."
+    );
+    return ['ChromeHeadless'];
+  }
+}
+
 const config = {
   // disable watcher
   autoWatch: false,
@@ -57,10 +84,11 @@ const config = {
   // changes
   autoWatch: false,
 
-  // start these browsers
-  // available browser launchers:
-  // https://npmjs.org/browse/keyword/karma-launcher
-  browsers: process.env?.BROWSERS?.split(',') ?? ['ChromeHeadless'],
+  // Browsers to launch for testing
+  // To use a custom set of browsers, define the BROWSERS environment variable as a comma-seperated list.
+  // Supported browsers are 'ChromeHeadless', 'WebkitHeadless', and 'Firefox'.
+  // See: https://karma-runner.github.io/6.4/config/browsers.html
+  browsers: determineBrowsers(),
 
   webpack: webpackTestConfig,
 

package.json

@@ -120,10 +120,10 @@
     "karma-firefox-launcher": "2.1.3",
     "karma-mocha": "2.0.1",
     "karma-mocha-reporter": "2.2.5",
-    "karma-safari-launcher": "1.0.0",
     "karma-sourcemap-loader": "0.4.0",
     "karma-spec-reporter": "0.0.36",
     "karma-summary-reporter": "3.1.1",
+    "karma-webkit-launcher": "2.6.0",
     "karma-webpack": "5.0.0",
     "lcov-result-merger": "3.3.0",
     "lerna": "4.0.0",
@@ -139,6 +139,7 @@
     "nyc": "15.1.0",
     "ora": "5.4.1",
     "patch-package": "7.0.2",
+    "playwright": "1.46.1",
     "postinstall-postinstall": "2.1.0",
     "prettier": "2.8.7",
     "protractor": "5.4.2",

scripts/ci-test/testConfig.ts

@@ -28,11 +28,25 @@ export interface TestConfig {
   alwaysIncludePackages?: string[];
 }
 
+// These tests are flaky on WebkitHeadless for some reason, so skip them.
+// TODO (dlarocque): Fix the flakes and remove this
+const ignoredWebkitCoreTests = process.env?.BROWSERS?.includes('WebkitHeadless')
+  ? [
+      '@firebase/app-check',
+      '@firebase/installations',
+      '@firebase/storage',
+      '@firebase/storage-compat',
+      '@firebase/database',
+      '@firebase/database-compat'
+    ]
+  : [];
+
 export const testConfig: {
   [key: string]: TestConfig | undefined;
 } = {
   'core': {
     'ignorePackages': [
+      ...ignoredWebkitCoreTests,
       '@firebase/firestore',
       '@firebase/firestore-compat',
       'firebase-firestore-integration-test',

yarn.lock

@@ -8653,6 +8653,11 @@ fs.realpath@^1.0.0:
   resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz"
   integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8=
 
+[email protected], fsevents@^2.3.2, fsevents@~2.3.2:
+  version "2.3.2"
+  resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz"
+  integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
+
 fsevents@^1.2.7:
   version "1.2.13"
   resolved "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz"
@@ -8661,11 +8666,6 @@ fsevents@^1.2.7:
     bindings "^1.5.0"
     nan "^2.12.1"
 
-fsevents@^2.3.2, fsevents@~2.3.2:
-  version "2.3.2"
-  resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz"
-  integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
-
 ftp@^0.3.10:
   version "0.3.10"
   resolved "https://registry.npmjs.org/ftp/-/ftp-0.3.10.tgz"
@@ -11339,11 +11339,6 @@ [email protected]:
   dependencies:
     minimist "^1.2.3"
 
-[email protected]:
-  version "1.0.0"
-  resolved "https://registry.npmjs.org/karma-safari-launcher/-/karma-safari-launcher-1.0.0.tgz"
-  integrity sha1-lpgqLMR9BmquccVTursoMZEVos4=
-
 [email protected]:
   version "0.4.0"
   resolved "https://registry.npmjs.org/karma-sourcemap-loader/-/karma-sourcemap-loader-0.4.0.tgz"
@@ -11414,6 +11409,14 @@ [email protected]:
     util "^0.12.1"
     vm-browserify "^1.1.2"
 
+[email protected]:
+  version "2.6.0"
+  resolved "https://registry.npmjs.org/karma-webkit-launcher/-/karma-webkit-launcher-2.6.0.tgz#2e3ba096b69139e608d9ce6e89816a17f358b700"
+  integrity sha512-IDURopxJ1SbuqnvPaE+lP2qiP2Ie7I+ojwJRBpr0tfGwObsaVdjMkUkmZ1BcXUtYRt5ogs9cyCH2Wb9sNv0BbQ==
+  dependencies:
+    is-ci "^3.0.1"
+    uuid "^10.0.0"
+
 [email protected]:
   version "5.0.0"
   resolved "https://registry.npmjs.org/karma-webpack/-/karma-webpack-5.0.0.tgz"
@@ -14119,6 +14122,20 @@ pkg-dir@^4.1.0, pkg-dir@^4.2.0:
   dependencies:
     find-up "^4.0.0"
 
+[email protected]:
+  version "1.46.1"
+  resolved "https://registry.npmjs.org/playwright-core/-/playwright-core-1.46.1.tgz#28f3ab35312135dda75b0c92a3e5c0e7edb9cc8b"
+  integrity sha512-h9LqIQaAv+CYvWzsZ+h3RsrqCStkBHlgo6/TJlFst3cOTlLghBQlJwPOZKQJTKNaD3QIB7aAVQ+gfWbN3NXB7A==
+
+[email protected]:
+  version "1.46.1"
+  resolved "https://registry.npmjs.org/playwright/-/playwright-1.46.1.tgz#ea562bc48373648e10420a10c16842f0b227c218"
+  integrity sha512-oPcr1yqoXLCkgKtD5eNUPLiN40rYEM39odNpIb6VE6S7/15gJmA1NzVv6zJYusV0e7tzvkU/utBFNa/Kpxmwng==
+  dependencies:
+    playwright-core "1.46.1"
+  optionalDependencies:
+    fsevents "2.3.2"
+
 plugin-error@^1.0.1:
   version "1.0.1"
   resolved "https://registry.npmjs.org/plugin-error/-/plugin-error-1.0.1.tgz"
@@ -17657,6 +17674,11 @@ [email protected]:
   resolved "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz"
   integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=
 
+uuid@^10.0.0:
+  version "10.0.0"
+  resolved "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz#5a95aa454e6e002725c79055fd42aaba30ca6294"
+  integrity sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==
+
 uuid@^3.3.2, uuid@^3.3.3:
   version "3.4.0"
   resolved "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz"