(#8569) - Switch browser tests to playwright

Co-authored-by: alxndrsn <alxndrsn>

Author: alxndrsn

.github/actions/install-deps/action.yml

@@ -25,3 +25,6 @@ runs:
     - uses: iamssen/couchdb-github-action@master
       with:
         couchdb-version: ${{ inputs.couchdb-version }}
+
+    - run: sudo npx playwright install-deps
+      shell: bash

.github/workflows/ci.yml

@@ -15,7 +15,6 @@ on:
 
 env:
   NODE_VERSION: 14
-  SELENIUM_HUB_HOST: hub
   TEST_HOST: localhost
 
 jobs:
@@ -74,17 +73,11 @@ jobs:
 
   couchdb-browser:
     needs: lint
-    services:
-      hub:
-        image: selenium/hub:3.141.59-gold
-      firefox:
-        image: selenium/node-firefox:3.141.59-gold
-        env: { HUB_HOST: hub, HUB_PORT: 4444 }
     strategy:
       fail-fast: false
       matrix:
         couchdb: ['2.3','3.1']
-        client: ['selenium:firefox', 'selenium:chrome']
+        client: ['firefox', 'chromium']
         cmd:
           - npm test
           - TYPE=find PLUGINS=pouchdb-find ADAPTERS=http npm test
@@ -156,16 +149,10 @@ jobs:
 
   browser-adapter:
     needs: lint
-    services:
-      hub:
-        image: selenium/hub:3.141.59-gold
-      firefox:
-        image: selenium/node-firefox:3.141.59-gold
-        env: { HUB_HOST: hub, HUB_PORT: 4444 }
     strategy:
       fail-fast: false
       matrix:
-        client: ['selenium:firefox', 'selenium:chrome']
+        client: ['firefox', 'chromium']
         adapter: ['idb', 'indexeddb', 'memory']
         cmd:
           - npm test
@@ -201,16 +188,10 @@ jobs:
 
   pouchdb-server:
     needs: lint
-    services:
-      hub:
-        image: selenium/hub:3.141.59-gold
-      firefox:
-        image: selenium/node-firefox:3.141.59-gold
-        env: { HUB_HOST: hub, HUB_PORT: 4444 }
     strategy:
       fail-fast: false
       matrix:
-        client: ['node', 'selenium:firefox', 'selenium:chrome']
+        client: ['node', 'firefox', 'chromium']
         cmd:
           - npm test
           - TYPE=find PLUGINS=pouchdb-find ADAPTERS=http npm test
@@ -246,7 +227,7 @@ jobs:
       matrix:
         node: [14, 16]
         cmd:
-          - CLIENT=selenium:firefox npm run test-webpack
+          - CLIENT=firefox npm run test-webpack
           - AUTO_COMPACTION=true npm test
           - PERF=1 npm test
           - npm run test-unit

TESTING.md

@@ -86,10 +86,7 @@ this to `0` to prevent this behaviour.
 #### `CLIENT` (default: `node`)
 
 Sets the target platform the tests will execute on. Set this to
-`selenium:firefox` or `selenium:chrome` to execute in the tests in the browser.
-
-You can also specify a custom Firefox binary to run using the `FIREFOX_BIN`
-variable.
+`firefox`, `chromium` or `webkit` to execute the tests in the browser.
 
 #### `COUCH_HOST`
 
@@ -180,7 +177,7 @@ for example:
     $ TYPE=find PLUGINS=pouchdb-find CLIENT=node ADAPTERS=memory npm test
 
     # run the "mapreduce" tests with indexeddb in firefox
-    $ TYPE=mapreduce CLIENT=selenium:firefox ADAPTERS=indexeddb npm test
+    $ TYPE=mapreduce CLIENT=firefox ADAPTERS=indexeddb npm test
 
 It's also important to check these tests against server-side adapters,
 specifically we need to ensure compatibility with CouchDB itself. We do this by
@@ -221,7 +218,7 @@ Sets the number of iterations each test uses by default.
 
 ### Running tests in the browser
 
-Normally we use `CLIENT=selenium:firefox` to run a set of tests in the browser
+Normally we use `CLIENT=firefox` to run a set of tests in the browser
 automatically. This opens a browser window, automatically runs the requested
 tests in it, and reports the results back to the shell.
 

bin/test-browser.js

@@ -1,34 +1,28 @@
 #!/usr/bin/env node
 'use strict';
 
-var wd = require('wd');
-wd.configureHttp({timeout: 180000}); // 3 minutes
+const playwright = require('playwright');
 
-var selenium = require('selenium-standalone');
 var querystring = require("querystring");
 
 var MochaSpecReporter = require('mocha').reporters.Spec;
 
 var devserver = require('./dev-server.js');
 
-var testTimeout = 30 * 60 * 1000;
-
-var SELENIUM_VERSION = process.env.SELENIUM_VERSION || '3.141.0';
-var CHROME_BIN = process.env.CHROME_BIN;
-var FIREFOX_BIN = process.env.FIREFOX_BIN;
-
 // BAIL=0 to disable bailing
 var bail = process.env.BAIL !== '0';
 
-// process.env.CLIENT is a colon separated list of
-// selenium:browserName:browserVerion:platform
-var tmp = (process.env.CLIENT || 'selenium:firefox').split(':');
-var client = {
-  runner: tmp[0] || 'selenium',
-  browser: tmp[1] || 'firefox',
-  version: tmp[2] || null, // Latest
-  platform: tmp[3] || null
-};
+// Playwright BrowserType whitelist.
+// See: https://playwright.dev/docs/api/class-playwright
+const SUPPORTED_BROWSERS = [ 'chromium', 'firefox', 'webkit' ];
+const browserName = process.env.CLIENT || 'firefox';
+if (!SUPPORTED_BROWSERS.includes(browserName)) {
+  console.log(`
+    !!! Requested browser not supported: '${browserName}'.
+    !!! Available browsers: ${SUPPORTED_BROWSERS.map(b => `'${b}'`).join(', ')}
+  `);
+  process.exit(1);
+}
 
 var testRoot = 'http://127.0.0.1:8000/tests/';
 var testUrl;
@@ -46,9 +40,6 @@ if (process.env.PERF) {
 
 var qs = { remote: 1 };
 
-var seleniumClient;
-var tunnelId = process.env.TRAVIS_JOB_NUMBER || 'tunnel-' + Date.now();
-
 if (process.env.INVERT) {
   qs.invert = process.env.INVERT;
 }
@@ -86,36 +77,6 @@ if (process.env.ITERATIONS) {
 testUrl += '?';
 testUrl += querystring.stringify(qs);
 
-function testError(e) {
-  console.error(e);
-  console.error('Doh, tests failed');
-
-  closeClient(function () {
-    process.exit(3);
-  });
-}
-
-function startSelenium(callback) {
-  // Start selenium
-  var opts = {version: SELENIUM_VERSION};
-  selenium.install(opts, function (err) {
-    if (err) {
-      console.error('Failed to install selenium');
-      process.exit(1);
-    }
-    selenium.start(opts, function () {
-      seleniumClient = wd.promiseChainRemote();
-      callback();
-    });
-  });
-}
-
-function closeClient(callback) {
-  seleniumClient.quit().then(function () {
-    callback();
-  });
-}
-
 class RemoteRunner {
   constructor() {
     this.handlers = {};
@@ -181,90 +142,60 @@ function BenchmarkReporter(runner) {
   });
 }
 
-function startTest() {
+async function startTest() {
 
-  console.log('Starting', client, 'on', testUrl);
+  console.log('Starting', browserName, 'on', testUrl);
 
-  var opts = {
-    browserName: client.browser,
-    version: client.version,
-    platform: client.platform,
-    tunnelTimeout: testTimeout,
-    name: client.browser + ' - ' + tunnelId,
-    'max-duration': 60 * 45,
-    'command-timeout': 599,
-    'idle-timeout': 599,
-    'tunnel-identifier': tunnelId
-  };
-  if (CHROME_BIN) {
-    opts.chromeOptions = {
-      binary: CHROME_BIN,
-      args: ['--headless', '--disable-gpu', '--no-sandbox', '--disable-setuid-sandbox']
-    };
-  }
-  if (FIREFOX_BIN) {
-    opts.firefox_binary = FIREFOX_BIN;
-  }
-
-  var runner = new RemoteRunner();
+  const runner = new RemoteRunner();
   new MochaSpecReporter(runner);
   new BenchmarkReporter(runner);
 
-  seleniumClient.init(opts, function (err) {
-    if (err) {
-      testError(err);
-      return;
-    }
-    console.log('Initialized');
+  const options = {
+    headless: true,
+  };
+  const browser = await playwright[browserName].launch(options);
+  const page = await browser.newPage();
+  if (process.env.BROWSER_CONSOLE) {
+    page.on('console', message => {
+      const { url, lineNumber } = message.location();
+      console.log('BROWSER', message.type().toUpperCase(), `${url}:${lineNumber}`, message.text());
+    });
+  }
+  await page.goto(testUrl);
+
+  const userAgent = await page.evaluate('navigator.userAgent');
+  console.log('Testing on:', userAgent);
+
+  const interval = setInterval(async () => {
+    try {
+      const events = await page.evaluate('window.testEvents()');
+      runner.handleEvents(events);
+
+      if (runner.completed || (runner.failed && bail)) {
+        if (!runner.completed && runner.failed) {
+          try {
+            runner.bail();
+          } catch (e) {
+            // Temporary debugging of bailing failure
+            console.log('An error occurred while bailing:');
+            console.log(e);
+          }
+        }
 
-    seleniumClient.get(testUrl, function (err) {
-      if (err) {
-        testError(err);
-        return;
+        clearInterval(interval);
+        await browser.close();
+        process.exit(!process.env.PERF && runner.failed ? 1 : 0);
       }
-      console.log('Successfully started');
-
-      seleniumClient.eval('navigator.userAgent', function (err, userAgent) {
-        if (err) {
-          testError(err);
-        } else {
-          console.log('Testing on:', userAgent);
-
-          var interval = setInterval(function () {
-            seleniumClient.eval('window.testEvents()', function (err, events) {
-                if (err) {
-                  clearInterval(interval);
-                  testError(err);
-                } else if (events) {
-                  runner.handleEvents(events);
-
-                  if (runner.completed || (runner.failed && bail)) {
-                    if (!runner.completed && runner.failed) {
-                      try {
-                        runner.bail();
-                      } catch (e) {
-                        // Temporary debugging of bailing failure
-                        console.log('An error occurred while bailing:');
-                        console.log(e);
-                      }
-                    }
-
-                    clearInterval(interval);
-
-                    closeClient(function () {
-                      process.exit(!process.env.PERF && runner.failed ? 1 : 0);
-                    });
-                  }
-                }
-            });
-          }, 10 * 1000);
+    } catch (e) {
+      console.error('Tests failed:', e);
 
-        }
-      });
-    });
-  });
+      clearInterval(interval);
+      await browser.close();
+      process.exit(3);
+    }
+  }, 1000);
 }
 
 devserver.start(function () {
-  startSelenium(startTest);
+  startTest();
 });

package.json

@@ -94,6 +94,7 @@
     "mocha": "3.5.0",
     "mockery": "2.1.0",
     "ncp": "2.0.0",
+    "playwright": "1.36.1",
     "pouchdb-express-router": "0.0.11",
     "query-string": "6.10.1",
     "replace": "1.2.1",
@@ -103,14 +104,12 @@
     "rollup-plugin-node-resolve": "4.2.4",
     "rollup-plugin-replace": "1.2.1",
     "seedrandom": "3.0.5",
-    "selenium-standalone": "6.16.0",
     "stream-to-promise": "1.1.1",
     "tape": "4.13.0",
     "terser": "4.8.0",
     "throw-max-listeners-error": "1.0.1",
     "ua-parser-js": "0.7.24",
-    "watch-glob": "0.1.3",
-    "wd": "1.11.4"
+    "watch-glob": "0.1.3"
   },
   "// greenkeeper": [
     "// chai-as-promised is pinned because of breaking changes in 6.0.0",

tests/integration/test.attachments.js

@@ -3609,11 +3609,7 @@ repl_adapters.forEach(function (adapters) {
       });
     });
 
-    // Currently this test is causing occasional CI selenium:firefox
-    // failures. Under advice of @daleharvey, we will skip this test
-    // to not block other development/tests and track this issue.
-    // See issue #6835 and #6831 for further info
-    it.skip('#3961 Many attachments on same doc', function () {
+    it('#3961 Many attachments on same doc', function () {
         var doc = {_id: 'foo', _attachments: {}};
 
         var db = new PouchDB(dbs.name);

tests/integration/webrunner.js

@@ -8,7 +8,7 @@
     window.removeEventListener("load", startTests);
 
     if (remote) {
-      // Capture logs for selenium output
+      // Capture logs for test runner output
       var logs = [];
 
       (function () {
@@ -41,7 +41,7 @@
 
       })();
 
-      // Capture test events for selenium output
+      // Capture test events for test runner output
       var testEventsBuffer = [];
 
       window.testEvents = function () {