test(wtr): run in multiple browsers and with API_VERSION=66

.github/workflows/web-test-runner.yml

@@ -60,6 +60,7 @@ jobs:
             - run: API_VERSION=60 yarn test
             - run: API_VERSION=61 yarn test
             - run: API_VERSION=62 yarn test
+            - run: API_VERSION=66 yarn test
             - run: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 yarn test || true
             - run: DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test
             - run: ENABLE_ARIA_REFLECTION_GLOBAL_POLYFILL=1 yarn test

packages/@lwc/engine-core/src/libs/signal-tracker/index.ts

@@ -44,6 +44,19 @@ export function unsubscribeFromSignals(target: object) {
 
 type CallbackFunction = () => void;
 
+/**
+ * A normalized string representation of an error, because browsers behave differently
+ */
+const errorWithStack = (err: unknown): string => {
+    if (typeof err !== 'object' || err === null) {
+        return String(err);
+    }
+    const stack = 'stack' in err ? String(err.stack) : '';
+    const message = 'message' in err ? String(err.message) : '';
+    const constructor = err.constructor.name;
+    return stack.includes(message) ? stack : `${constructor}: ${message}\n${stack}`;
+};
+
 /**
  * This class is used to keep track of the signals associated to a given object.
  * It is used to prevent the LWC engine from subscribing duplicate callbacks multiple times
@@ -67,9 +80,9 @@ class SignalTracker {
             }
         } catch (err: any) {
             logWarnOnce(
-                `Attempted to subscribe to an object that has the shape of a signal but received the following error: ${
-                    err?.stack ?? err
-                }`
+                `Attempted to subscribe to an object that has the shape of a signal but received the following error: ${errorWithStack(
+                    err
+                )}`
             );
         }
     }
@@ -79,9 +92,9 @@ class SignalTracker {
             this.signalToUnsubscribeMap.forEach((unsubscribe) => unsubscribe());
         } catch (err: any) {
             logWarnOnce(
-                `Attempted to call a signal's unsubscribe callback but received the following error: ${
-                    err?.stack ?? err
-                }`
+                `Attempted to call a signal's unsubscribe callback but received the following error: ${errorWithStack(
+                    err
+                )}`
             );
         }
     }

packages/@lwc/integration-not-karma/configs/hydration.js

@@ -1,5 +1,5 @@
 import * as options from '../helpers/options.js';
-import createConfig from './base.js';
+import createConfig from './shared/base-config.js';
 import hydrationTestPlugin from './plugins/serve-hydration.js';
 
 const SHADOW_MODE = options.SHADOW_MODE_OVERRIDE ?? 'native';

packages/@lwc/integration-not-karma/configs/integration.js

@@ -1,6 +1,6 @@
 import { importMapsPlugin } from '@web/dev-server-import-maps';
 import * as options from '../helpers/options.js';
-import createConfig from './base.js';
+import createConfig from './shared/base-config.js';
 import testPlugin from './plugins/serve-integration.js';
 
 const SHADOW_MODE = options.SHADOW_MODE_OVERRIDE ?? 'synthetic';

packages/@lwc/integration-not-karma/configs/base.js → packages/@lwc/integration-not-karma/configs/shared/base-config.js

@@ -1,6 +1,7 @@
 import { join } from 'node:path';
 import { LWC_VERSION } from '@lwc/shared';
-import { resolvePathOutsideRoot } from '../helpers/utils.js';
+import { resolvePathOutsideRoot } from '../../helpers/utils.js';
+import { getBrowsers } from './browsers.js';
 
 /**
  * We want to convert from parsed options (true/false) to a `process.env` with only strings.
@@ -37,13 +38,15 @@ export default (options) => {
     });
 
     return {
+        browsers: getBrowsers(options),
+        browserLogs: false,
         // FIXME: Parallelism breaks tests that rely on focus/requestAnimationFrame, because they often
         // time out before they receive focus. But it also makes the full suite take 3x longer to run...
         // Potential workaround: https://github.com/modernweb-dev/web/issues/2588
         concurrency: 1,
-        browserLogs: false,
+        concurrentBrowsers: 3,
         nodeResolve: true,
-        rootDir: join(import.meta.dirname, '..'),
+        rootDir: join(import.meta.dirname, '../..'),
         plugins: [
             {
                 name: 'lwc-base-plugin',
@@ -64,7 +67,8 @@ export default (options) => {
             },
         ],
         testRunnerHtml: (testFramework) =>
-            `<!DOCTYPE html>
+            `
+        <!DOCTYPE html>
         <html>
           <head>
             <script type="module">
@@ -82,6 +86,7 @@ export default (options) => {
             <script type="module" src="./helpers/setup.js"></script>
             <script type="module" src="${testFramework}"></script>
           </head>
-        </html>`,
+        </html>
+        `,
     };
 };

packages/@lwc/integration-not-karma/configs/shared/browsers.js

@@ -0,0 +1,56 @@
+import { playwrightLauncher } from '@web/test-runner-playwright';
+import { createSauceLabsLauncher } from '@web/test-runner-saucelabs';
+
+/** @type {(options: typeof import('../../helpers/options.js')) => import("@web/test-runner").BrowserLauncher[]} */
+export function getBrowsers(options) {
+    if (options.IS_CI) {
+        if (!options.SAUCE_USERNAME || !options.SAUCE_ACCESS_KEY || !options.SAUCE_TUNNEL_ID) {
+            throw new Error(
+                `SAUCE_USERNAME, SAUCE_ACCESS_KEY, and SAUCE_TUNNEL_ID must be configured in CI`
+            );
+        }
+        const sauceLabsLauncher = createSauceLabsLauncher(
+            {
+                user: options.SAUCE_USERNAME,
+                key: options.SAUCE_ACCESS_KEY,
+            },
+            {
+                tunnelName: options.SAUCE_TUNNEL_ID,
+            }
+        );
+        return [
+            sauceLabsLauncher({
+                browserName: 'chrome',
+                browserVersion: 'latest',
+            }),
+            sauceLabsLauncher({
+                browserName: 'firefox',
+                browserVersion: 'latest',
+            }),
+            sauceLabsLauncher({
+                browserName: 'safari',
+                browserVersion: 'latest',
+                platformName: 'macOS 15', // Update this with new Mac releases
+            }),
+            ...(options.LEGACY_BROWSERS
+                ? [
+                      sauceLabsLauncher({
+                          browserName: 'chrome',
+                          browserVersion: 'latest-2',
+                      }),
+                      sauceLabsLauncher({
+                          browserName: 'safari',
+                          browserVersion: 'latest-2',
+                          platformName: 'macOS 13', // Should be 2 behind latest
+                      }),
+                  ]
+                : []),
+        ];
+    } else {
+        return [
+            playwrightLauncher({ product: 'chromium' }),
+            playwrightLauncher({ product: 'firefox' }),
+            playwrightLauncher({ product: 'webkit' }),
+        ];
+    }
+}

packages/@lwc/integration-not-karma/package.json

@@ -4,6 +4,7 @@
     "version": "8.22.4",
     "type": "module",
     "scripts": {
+        "build": "playwright install",
         "start": "web-test-runner --manual",
         "test": "web-test-runner --config configs/integration.js",
         "test:hydration": "web-test-runner --config configs/hydration.js"
@@ -21,7 +22,10 @@
         "@web/dev-server-import-maps": "^0.2.1",
         "@web/dev-server-rollup": "^0.6.4",
         "@web/test-runner": "^0.20.2",
-        "chai": "^6.2.0"
+        "@web/test-runner-playwright": "^0.11.1",
+        "@web/test-runner-saucelabs": "^0.13.0",
+        "chai": "^6.2.0",
+        "playwright": "^1.56.0"
     },
     "volta": {
         "extends": "../../../package.json"

packages/@lwc/integration-not-karma/test/api/CustomElementConstructor-getter/index.spec.js

@@ -34,7 +34,9 @@ it('CustomElementConstructor cannot be `new`ed before being defined', () => {
         new UndefinedComponent.CustomElementConstructor();
     };
     expect(func).toThrowError(TypeError);
-    expect(func).toThrowError(/(Illegal constructor|does not define a custom element)/);
+    expect(func).toThrowError(
+        /(Illegal constructor|does not define a custom element|is not a valid custom element constructor)/
+    );
 });
 
 it('CustomElementConstructor can be `new`ed after being defined', () => {

packages/@lwc/integration-not-karma/test/signal/protocol/index.spec.js

@@ -11,7 +11,7 @@ import Throws from 'x/throws';
 // Note for testing purposes the signal implementation uses LWC module resolution to simplify things.
 // In production the signal will come from a 3rd party library.
 import { Signal } from 'x/signal';
-import { jasmine } from '../../../helpers/jasmine.js';
+import { fn as mockFn } from '@vitest/spy';
 import { resetDOM } from '../../../helpers/reset.js';
 
 describe('signal protocol', () => {
@@ -190,7 +190,7 @@ describe('signal protocol', () => {
 
     it('does not subscribe if the signal shape is incorrect', async () => {
         const elm = createElement('x-child', { is: Child });
-        const subscribe = jasmine.createSpy();
+        const subscribe = mockFn();
         // Note the signals property is value's' and not value
         const signal = { values: 'initial value', subscribe };
         elm.signal = signal;
@@ -202,7 +202,7 @@ describe('signal protocol', () => {
 
     it('does not subscribe if the signal is not added as trusted signal', async () => {
         const elm = createElement('x-child', { is: Child });
-        const subscribe = jasmine.createSpy();
+        const subscribe = mockFn();
         // Note this follows the shape of the signal implementation
         // but it's not added as a trusted signal (add using lwc.addTrustedSignal)
         const signal = {

yarn.lock

@@ -2091,9 +2091,11 @@
 
 "@lwc/eslint-plugin-lwc-internal@link:./scripts/eslint-plugin":
   version "0.0.0"
+  uid ""
 
 "@lwc/test-utils-lwc-internals@link:./scripts/test-utils":
   version "0.0.0"
+  uid ""
 
 "@napi-rs/[email protected]":
   version "0.2.4"
@@ -4026,6 +4028,35 @@
   dependencies:
     "@web/test-runner-core" "^0.13.0"
 
+"@web/test-runner-playwright@^0.11.1":
+  version "0.11.1"
+  resolved "https://registry.yarnpkg.com/@web/test-runner-playwright/-/test-runner-playwright-0.11.1.tgz#d993112ae2126eb74c1c5a171d6ea44c2dd24b4e"
+  integrity sha512-l9tmX0LtBqMaKAApS4WshpB87A/M8sOHZyfCobSGuYqnREgz5rqQpX314yx+4fwHXLLTa5N64mTrawsYkLjliw==
+  dependencies:
+    "@web/test-runner-core" "^0.13.0"
+    "@web/test-runner-coverage-v8" "^0.8.0"
+    playwright "^1.53.0"
+
+"@web/test-runner-saucelabs@^0.13.0":
+  version "0.13.0"
+  resolved "https://registry.yarnpkg.com/@web/test-runner-saucelabs/-/test-runner-saucelabs-0.13.0.tgz#66b50d4cf6ca29163e8c67e0d6a0b5863ac69172"
+  integrity sha512-RHVDIFVaiOoxydigx1S4wrA20wjjLF9IAxlT7HxYHxO1qx1UIuWLabd6jEGlxluC0xFhuBJLm+zFj8zMEXJNyg==
+  dependencies:
+    "@web/test-runner-webdriver" "^0.9.0"
+    internal-ip "^6.2.0"
+    nanoid "^3.1.25"
+    saucelabs "^9.0.0"
+    webdriver "^9.0.0"
+    webdriverio "^9.0.0"
+
+"@web/test-runner-webdriver@^0.9.0":
+  version "0.9.0"
+  resolved "https://registry.yarnpkg.com/@web/test-runner-webdriver/-/test-runner-webdriver-0.9.0.tgz#18fb3299ddc258a2782028de19f21ee4c27895c5"
+  integrity sha512-G2io6ph0v/sX0U/DEgh/EUHsLsq8/Gs/uUw8N7rcUaXSdIFCUbxlwzx/qZv2ZKY52q84oYUyeOhViNZ2OqYl6Q==
+  dependencies:
+    "@web/test-runner-core" "^0.13.0"
+    webdriverio "^9.0.0"
+
 "@web/test-runner@^0.20.2":
   version "0.20.2"
   resolved "https://registry.yarnpkg.com/@web/test-runner/-/test-runner-0.20.2.tgz#3045de8f14eb9f9b8aa51aa7fd9849dd95bb0a93"
@@ -7605,6 +7636,11 @@ fs.realpath@^1.0.0:
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
   integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==
 
+[email protected]:
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
+  integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
+
 fsevents@~2.3.2, fsevents@~2.3.3:
   version "2.3.3"
   resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6"
@@ -11279,6 +11315,20 @@ pkg-up@^4.0.0:
   dependencies:
     find-up "^6.2.0"
 
+[email protected]:
+  version "1.56.0"
+  resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.56.0.tgz#14b40ea436551b0bcefe19c5bfb8d1804c83739c"
+  integrity sha512-1SXl7pMfemAMSDn5rkPeZljxOCYAmQnYLBTExuh6E8USHXGSX3dx6lYZN/xPpTz1vimXmPA9CDnILvmJaB8aSQ==
+
+playwright@^1.53.0, playwright@^1.56.0:
+  version "1.56.0"
+  resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.56.0.tgz#71c533c61da33e95812f8c6fa53960e073548d9a"
+  integrity sha512-X5Q1b8lOdWIE4KAoHpW3SE8HvUB+ZZsUoN64ZhjnN8dOb1UpujxBtENGiZFE+9F/yhzJwYa+ca3u43FeLbboHA==
+  dependencies:
+    playwright-core "1.56.0"
+  optionalDependencies:
+    fsevents "2.3.2"
+
 portfinder@^1.0.32:
   version "1.0.37"
   resolved "https://registry.yarnpkg.com/portfinder/-/portfinder-1.0.37.tgz#92b754ef89a11801c8efe4b0e5cd845b0064c212"
@@ -12132,7 +12182,7 @@ [email protected]:
     tunnel "0.0.6"
     yargs "^17.0.1"
 
-saucelabs@^9.0.1:
+saucelabs@^9.0.0, saucelabs@^9.0.1:
   version "9.0.2"
   resolved "https://registry.yarnpkg.com/saucelabs/-/saucelabs-9.0.2.tgz#99f6170f3d789fcb0be2f270f7d37a9d7cdf5187"
   integrity sha512-37QGEOgp9BP1re6S06qpNcBZ0Hw+ZSkZkDepbXHT9VjYoRQwRzUoLtKqE4yyVeK7dzcQXQapmTGF1kp1jO2VDw==
@@ -13856,7 +13906,7 @@ [email protected]:
     ky "^0.30.0"
     lodash.merge "^4.6.1"
 
-[email protected]:
+[email protected], webdriver@^9.0.0:
   version "9.20.0"
   resolved "https://registry.yarnpkg.com/webdriver/-/webdriver-9.20.0.tgz#518fa61abd0b0435509548ef54fb0824f0fa04fd"
   integrity sha512-Kk+AGV1xWLNHVpzUynQJDULMzbcO3IjXo3s0BzfC30OpGxhpaNmoazMQodhtv0Lp242Mb1VYXD89dCb4oAHc4w==
@@ -13906,7 +13956,7 @@ [email protected]:
     serialize-error "^8.0.0"
     webdriver "7.19.5"
 
-[email protected], webdriverio@^9.19.2:
+[email protected], webdriverio@^9.0.0, webdriverio@^9.19.2:
   version "9.20.0"
   resolved "https://registry.yarnpkg.com/webdriverio/-/webdriverio-9.20.0.tgz#46f45268f2c4021a18045a7e7dfd738225514ece"
   integrity sha512-cqaXfahTzCFaQLlk++feZaze6tAsW8OSdaVRgmOGJRII1z2A4uh4YGHtusTpqOiZAST7OBPqycOwfh01G/Ktbg==