test(wtr): run in multiple browsers and with API_VERSION=66

.github/workflows/web-test-runner.yml

@@ -18,8 +18,10 @@ env:
     NODE_VERSION: '20.19.4'
 
 jobs:
+    # We run tests in parallel to speed up CI, with an arbitrary goal of ~20 minutes per job.
+    # Test grouping is also somewhat arbitrary.
     # TODO: upload result artifacts
-    # TODO: make it saucy 🥫
+
     integration-tests:
         name: Integration tests (${{ matrix.shadow_mode }} shadow)
         strategy:
@@ -47,23 +49,61 @@ jobs:
               run: yarn install --frozen-lockfile
               working-directory: ./
 
-            # - uses: saucelabs/[email protected]
-            #   with:
-            #       username: ${{ secrets.SAUCE_USERNAME }}
-            #       accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
-            #       tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
-            #       region: us
+            - uses: saucelabs/[email protected]
+              with:
+                  username: ${{ secrets.SAUCE_USERNAME }}
+                  accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
+                  tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
+                  region: us
 
+            - run: yarn exec -- playwright install chrome firefox webkit --with-deps
             - run: yarn test
+            - run: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 yarn test || true
+            - run: DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test
+            - run: ENABLE_ARIA_REFLECTION_GLOBAL_POLYFILL=1 yarn test
+            - run: NODE_ENV_FOR_TEST=production yarn test
+
+    integration-tests-api-versions:
+        name: Integration tests (${{ matrix.shadow_mode }} shadow) - API versions
+        strategy:
+            matrix:
+                shadow_mode: [native, synthetic]
+
+        runs-on: ubuntu-22.04
+        env:
+            SAUCE_TUNNEL_ID: github-action-tunnel-wtr-${{github.run_id}}-group-1
+            SHADOW_MODE_OVERRIDE: ${{ matrix.shadow_mode }}
+        defaults:
+            run:
+                working-directory: ./packages/@lwc/integration-not-karma
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+
+            - name: Setup Node
+              uses: actions/setup-node@v4
+              with:
+                  node-version: ${{ env.NODE_VERSION }}
+                  cache: 'yarn'
+
+            - name: Install dependencies
+              run: yarn install --frozen-lockfile
+              working-directory: ./
+
+            - uses: saucelabs/[email protected]
+              with:
+                  username: ${{ secrets.SAUCE_USERNAME }}
+                  accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
+                  tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
+                  region: us
+
+            - run: yarn exec -- playwright install chrome firefox webkit --with-deps
             - run: API_VERSION=58 yarn test
             - run: API_VERSION=59 yarn test
             - run: API_VERSION=60 yarn test
             - run: API_VERSION=61 yarn test
             - run: API_VERSION=62 yarn test
-            - run: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 yarn test || true
-            - run: DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test
-            - run: ENABLE_ARIA_REFLECTION_GLOBAL_POLYFILL=1 yarn test
-            - run: NODE_ENV_FOR_TEST=production yarn test
+            - run: API_VERSION=66 yarn test
 
     integration-tests-not-both-modes:
         # Tests that should run in only synthetic or native shadow, not both
@@ -89,13 +129,14 @@ jobs:
               run: yarn install --frozen-lockfile
               working-directory: ./
 
-            # - uses: saucelabs/[email protected]
-            #   with:
-            #       username: ${{ secrets.SAUCE_USERNAME }}
-            #       accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
-            #       tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
-            #       region: us
+            - uses: saucelabs/[email protected]
+              with:
+                  username: ${{ secrets.SAUCE_USERNAME }}
+                  accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
+                  tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
+                  region: us
 
+            - run: yarn exec -- playwright install chrome firefox webkit --with-deps
             # Synthetic shadow only
             - run: LEGACY_BROWSERS=1 yarn test || true
             - run: FORCE_NATIVE_SHADOW_MODE_FOR_TEST=1 yarn test
@@ -106,10 +147,48 @@ jobs:
             - run: SHADOW_MODE_OVERRIDE=native DISABLE_SYNTHETIC_SHADOW_SUPPORT_IN_COMPILER=1 yarn test
             - run: SHADOW_MODE_OVERRIDE=native DISABLE_SYNTHETIC_SHADOW_SUPPORT_IN_COMPILER=1 DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test
 
-    hydration-tests:
+    hydration-tests-ssr-v2:
+        name: Hydration tests (SSR v2)
+        runs-on: ubuntu-22.04
+        env:
+            SAUCE_TUNNEL_ID: github-action-tunnel-wtr-${{github.run_id}}-group-1
+        defaults:
+            run:
+                working-directory: ./packages/@lwc/integration-not-karma
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+
+            - name: Setup Node
+              uses: actions/setup-node@v4
+              with:
+                  node-version: ${{ env.NODE_VERSION }}
+                  cache: 'yarn'
+
+            - name: Install dependencies
+              run: yarn install --frozen-lockfile
+              working-directory: ./
+
+            - uses: saucelabs/[email protected]
+              with:
+                  username: ${{ secrets.SAUCE_USERNAME }}
+                  accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
+                  tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
+                  region: us
+
+            - run: yarn exec -- playwright install chrome firefox webkit --with-deps
+            - run: yarn test:hydration
+            - run: SHADOW_MODE_OVERRIDE=synthetic yarn test:hydration
+            - run: NODE_ENV_FOR_TEST=production yarn test:hydration
+            - run: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 yarn test:hydration
+            - run: DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test:hydration
+
+    hydration-tests-engine-server:
+        name: Hydration tests (engine-server)
         runs-on: ubuntu-22.04
         env:
             SAUCE_TUNNEL_ID: github-action-tunnel-wtr-${{github.run_id}}-group-1
+            ENGINE_SERVER: 1
         defaults:
             run:
                 working-directory: ./packages/@lwc/integration-not-karma
@@ -127,21 +206,19 @@ jobs:
               run: yarn install --frozen-lockfile
               working-directory: ./
 
-            # - uses: saucelabs/[email protected]
-            #   with:
-            #       username: ${{ secrets.SAUCE_USERNAME }}
-            #       accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
-            #       tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
-            #       region: us
-            - run: ENGINE_SERVER=1 yarn test:hydration
-            - run: ENGINE_SERVER=1 SHADOW_MODE_OVERRIDE=synthetic yarn test:hydration
-            - run: ENGINE_SERVER=1 NODE_ENV_FOR_TEST=production yarn test:hydration
-            - run: ENGINE_SERVER=1 DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 yarn test:hydration
-            - run: ENGINE_SERVER=1 DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test:hydration
-            - run: ENGINE_SERVER=1 DISABLE_DETACHED_REHYDRATION=1 yarn test:hydration
-            - run: ENGINE_SERVER=1 DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 DISABLE_DETACHED_REHYDRATION=1 yarn test:hydration
+            - uses: saucelabs/[email protected]
+              with:
+                  username: ${{ secrets.SAUCE_USERNAME }}
+                  accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
+                  tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
+                  region: us
+
+            - run: yarn exec -- playwright install chrome firefox webkit --with-deps
+            # NOTE: All tests have ENGINE_SERVER=1 already set
             - run: yarn test:hydration
             - run: SHADOW_MODE_OVERRIDE=synthetic yarn test:hydration
             - run: NODE_ENV_FOR_TEST=production yarn test:hydration
             - run: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 yarn test:hydration
             - run: DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test:hydration
+            - run: DISABLE_DETACHED_REHYDRATION=1 yarn test:hydration
+            - run: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 DISABLE_DETACHED_REHYDRATION=1 yarn test:hydration

packages/@lwc/engine-core/src/libs/signal-tracker/index.ts

@@ -44,6 +44,19 @@ export function unsubscribeFromSignals(target: object) {
 
 type CallbackFunction = () => void;
 
+/**
+ * A normalized string representation of an error, because browsers behave differently
+ */
+const errorWithStack = (err: unknown): string => {
+    if (typeof err !== 'object' || err === null) {
+        return String(err);
+    }
+    const stack = 'stack' in err ? String(err.stack) : '';
+    const message = 'message' in err ? String(err.message) : '';
+    const constructor = err.constructor.name;
+    return stack.includes(message) ? stack : `${constructor}: ${message}\n${stack}`;
+};
+
 /**
  * This class is used to keep track of the signals associated to a given object.
  * It is used to prevent the LWC engine from subscribing duplicate callbacks multiple times
@@ -67,9 +80,9 @@ class SignalTracker {
             }
         } catch (err: any) {
             logWarnOnce(
-                `Attempted to subscribe to an object that has the shape of a signal but received the following error: ${
-                    err?.stack ?? err
-                }`
+                `Attempted to subscribe to an object that has the shape of a signal but received the following error: ${errorWithStack(
+                    err
+                )}`
             );
         }
     }
@@ -79,9 +92,9 @@ class SignalTracker {
             this.signalToUnsubscribeMap.forEach((unsubscribe) => unsubscribe());
         } catch (err: any) {
             logWarnOnce(
-                `Attempted to call a signal's unsubscribe callback but received the following error: ${
-                    err?.stack ?? err
-                }`
+                `Attempted to call a signal's unsubscribe callback but received the following error: ${errorWithStack(
+                    err
+                )}`
             );
         }
     }

packages/@lwc/integration-not-karma/configs/hydration.js

@@ -1,5 +1,5 @@
 import * as options from '../helpers/options.js';
-import createConfig from './base.js';
+import createConfig from './shared/base-config.js';
 import hydrationTestPlugin from './plugins/serve-hydration.js';
 
 const SHADOW_MODE = options.SHADOW_MODE_OVERRIDE ?? 'native';

packages/@lwc/integration-not-karma/configs/integration.js

@@ -1,6 +1,6 @@
 import { importMapsPlugin } from '@web/dev-server-import-maps';
 import * as options from '../helpers/options.js';
-import createConfig from './base.js';
+import createConfig from './shared/base-config.js';
 import testPlugin from './plugins/serve-integration.js';
 
 const SHADOW_MODE = options.SHADOW_MODE_OVERRIDE ?? 'synthetic';
@@ -13,12 +13,7 @@ const baseConfig = createConfig({
 /** @type {import("@web/test-runner").TestRunnerConfig} */
 export default {
     ...baseConfig,
-    files: [
-        'test/**/*.spec.js',
-        // Make John fix this after his PR is merged
-        '!test/template-expressions/errors/index.spec.js',
-        '!test/template-expressions/smoke-test/index.spec.js',
-    ],
+    files: ['test/**/*.spec.js'],
     plugins: [
         ...baseConfig.plugins,
         importMapsPlugin({ inject: { importMap: { imports: { lwc: './mocks/lwc.js' } } } }),

packages/@lwc/integration-not-karma/configs/plugins/serve-hydration.js

@@ -1,20 +1,16 @@
 import path from 'node:path';
 import vm from 'node:vm';
 import { fileURLToPath } from 'node:url';
+import { readFileSync } from 'node:fs';
 import { rollup } from 'rollup';
 import lwcRollupPlugin from '@lwc/rollup-plugin';
 import { DISABLE_STATIC_CONTENT_OPTIMIZATION, ENGINE_SERVER } from '../../helpers/options.js';
-/** LWC SSR module to use when server-side rendering components. */
-const lwcSsr = await (ENGINE_SERVER
-    ? // Using import('literal') rather than import(variable) so static analysis tools work
-      import('@lwc/engine-server')
-    : import('@lwc/ssr-runtime'));
 
-lwcSsr.setHooks({
-    sanitizeHtmlContent(content) {
-        return content;
-    },
-});
+/** Code for the LWC SSR module. */
+const LWC_SSR = readFileSync(
+    new URL(import.meta.resolve(ENGINE_SERVER ? '@lwc/engine-server' : '@lwc/ssr-runtime')),
+    'utf8'
+);
 
 const ROOT_DIR = path.join(import.meta.dirname, '../..');
 const COMPONENT_NAME = 'x-main';
@@ -68,16 +64,27 @@ async function compileModule(input, targetSSR, format) {
  */
 async function getSsrMarkup(componentEntrypoint, configPath) {
     const componentIife = await compileModule(componentEntrypoint, !ENGINE_SERVER, 'iife');
-    // To minimize the amount of code in the generated script, ideally we'd do `import Component`
-    // and delegate the bundling to the loader. However, that's complicated to configure and using
-    // imports with vm.Script/vm.Module is still experimental, so we use an IIFE for simplicity.
-    // Additionally, we could import LWC, but the framework requires configuration before each test
-    // (setHooks/setFeatureFlagForTest), so instead we configure it once in the top-level context
-    // and inject it as a global variable.
+    // Ideally, we'd be able to do `import Component` and delegate bundling to the loader. We also
+    // need each import of LWC to be isolated, but by all server-side imports share a global state.
+    // We could solve this with the right `vm.Script`/`vm.Module` setup, but that's complicated and
+    // still experimental. Therefore, we just inline everything.
     const script = new vm.Script(
         `(async () => {
-            const {default: config} = await import('./${configPath}');
-            ${componentIife /* var Component = ... */}
+            // node.js / CommonJS setup
+            const process = { env: ${JSON.stringify(process.env)} };
+            const exports = Object.create(null);
+            const LWC = exports;
+
+            // LWC / test setup
+            ${LWC_SSR};
+            LWC.setHooks({ sanitizeHtmlContent: (v) => v });
+            const { default: config } = await import('./${configPath}');
+            config.requiredFeatureFlags?.forEach(ff => {
+                LWC.setFeatureFlagForTest(ff, true);
+            });
+
+            // Component code
+            ${componentIife};
             return LWC.renderComponent(
                 '${COMPONENT_NAME}',
                 Component,
@@ -87,44 +94,32 @@ async function getSsrMarkup(componentEntrypoint, configPath) {
             );
         })()`,
         {
-            filename: `[SSR] ${configPath}`,
+            filename: `(virtual SSR file for) ${configPath}`,
             importModuleDynamically: vm.constants.USE_MAIN_CONTEXT_DEFAULT_LOADER,
         }
     );
 
-    return await script.runInContext(vm.createContext({ LWC: lwcSsr }));
+    return await script.runInNewContext();
 }
 
 /**
  * Hydration test `index.spec.js` files are actually config files, not spec files.
  * This function wraps those configs in the test code to be executed.
  */
 async function wrapHydrationTest(configPath) {
-    const { default: config } = await import(path.join(ROOT_DIR, configPath));
+    const suiteDir = path.dirname(configPath);
+    const componentEntrypoint = path.join(suiteDir, COMPONENT_ENTRYPOINT);
+    const ssrOutput = await getSsrMarkup(componentEntrypoint, configPath);
 
-    try {
-        config.requiredFeatureFlags?.forEach((featureFlag) => {
-            lwcSsr.setFeatureFlagForTest(featureFlag, true);
-        });
-
-        const suiteDir = path.dirname(configPath);
-        const componentEntrypoint = path.join(suiteDir, COMPONENT_ENTRYPOINT);
-        const ssrOutput = await getSsrMarkup(componentEntrypoint, configPath);
-
-        return `
-        import * as LWC from 'lwc';
-        import { runTest } from '/configs/plugins/test-hydration.js';
-        runTest(
-            '/${configPath}?original=1',
-            '/${componentEntrypoint}',
-            ${JSON.stringify(ssrOutput) /* escape quotes */}
-        );
-        `;
-    } finally {
-        config.requiredFeatureFlags?.forEach((featureFlag) => {
-            lwcSsr.setFeatureFlagForTest(featureFlag, false);
-        });
-    }
+    return `
+    import * as LWC from 'lwc';
+    import { runTest } from '/configs/plugins/test-hydration.js';
+    runTest(
+        '/${configPath}?original=1',
+        '/${componentEntrypoint}',
+        ${JSON.stringify(ssrOutput) /* escape quotes */}
+    );
+    `;
 }
 
 /** @type {import('@web/dev-server-core').Plugin} */