chore(build): Move WPT support build-time to runtime config (#257)

Instead of requiring a CMake build config option to enable WPT support, with this change that's moved to the new `--wpt-mode` runtime flag.

Author: tschneidereit

.github/workflows/main.yml

@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        build: [release, weval]
+        build: [release, debug, weval]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -39,55 +39,28 @@ jobs:
       with:
         node-version: 'lts/*'
 
-    - name: Build StarlingMonkey
+    - name: Configure StarlingMonkey
       run: |
         cmake -S . -B cmake-build-${{ matrix.build }}\
           -DCMAKE_BUILD_TYPE=${{ (matrix.build == 'release' || matrix.build == 'weval') && 'Release' || 'Debug' }}\
           ${{matrix.build == 'weval' && '-DUSE_WASM_OPT=OFF -DWEVAL=ON' || ''}}
-        cmake --build cmake-build-${{ matrix.build }} --parallel 4 --target all integration-test-server
-
-    - name: StarlingMonkey E2E & Integration Tests
-      run: |
-        CTEST_OUTPUT_ON_FAILURE=1 ctest --test-dir cmake-build-${{ matrix.build }} -j4
-
-  wpt:
-    name: Web Platform Tests
-    strategy:
-      matrix:
-        build: [release, debug, weval]
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        submodules: recursive
 
-    - name: Install Rust 1.80.0
+    - name: Build StarlingMonkey
       run: |
-        rustup toolchain install 1.80.0
-        rustup target add wasm32-wasip1 --toolchain 1.80.0
+        cmake --build cmake-build-${{ matrix.build }} --parallel $(nproc) --target all
 
-    - uses: actions/setup-node@v2
-      with:
-        node-version: 'lts/*'
-
-    - name: Build StarlingMonkey WPT
-      if: matrix.build != 'weval'
+    - name: Build Integration Test Server
       run: |
-        cmake -S . -B cmake-build-${{ matrix.build }} -DENABLE_WPT:BOOL=ON -DCMAKE_BUILD_TYPE=${{ matrix.build == 'release' && 'Release' || 'Debug' }}
-        cmake --build cmake-build-${{ matrix.build }} --parallel 4 --target wpt-runtime
+        cmake --build cmake-build-${{ matrix.build }} --parallel $(nproc) --target integration-test-server
 
-    - name: Build StarlingMonkey WPT Weval
-      if: matrix.build == 'weval'
+    - name: Build WPT Runtime
       run: |
-        cmake -S . -B cmake-build-weval -DCMAKE_BUILD_TYPE=Release -DUSE_WASM_OPT=OFF -DWEVAL=ON -DENABLE_WPT:BOOL=ON
-        cmake --build cmake-build-weval --parallel 4 --target starling-ics.wevalcache
-        cmake --build cmake-build-weval --parallel 4 --target wpt-runtime
+        cmake --build cmake-build-${{ matrix.build }} --parallel $(nproc) --target wpt-runtime
 
     - name: Prepare WPT hosts
       run: |
         cat deps/wpt-hosts | sudo tee -a /etc/hosts
 
-    - name: StarlingMonkey WPT Test
-      env:
-        CTEST_OUTPUT_ON_FAILURE: 1
-      run: ctest -R wpt --test-dir cmake-build-${{ matrix.build }} --verbose --no-tests=error
+    - name: StarlingMonkey E2E, Integration, and WPT Tests
+      run: |
+        CTEST_OUTPUT_ON_FAILURE=1 ctest --test-dir cmake-build-${{ matrix.build }} -j$(nproc) --verbose

CMakeLists.txt

@@ -50,10 +50,7 @@ target_include_directories(extension_api INTERFACE include deps/include runtime)
 
 include("builtins")
 
-option(ENABLE_WPT "Enable WPT harness support" OFF)
-if (ENABLE_WPT)
-    include("tests/wpt-harness/wpt.cmake")
-endif()
+include("tests/wpt-harness/wpt.cmake")
 
 set(SOURCES
     runtime/js.cpp

componentize.sh.in

@@ -16,6 +16,7 @@ usage() {
   echo "       Specifying '-i' or '--initializer-script-path' allows specifying an initializer script"
   echo "       Specifying '--strip-path-prefix' will cause the provided prefix to be stripped from paths in stack traces and the debugger"
   echo "       Specifying '--legacy-script' causes evaluation as a legacy JS script instead of a module"
+  echo "       Specifying '--wpt-mode' enables WPT compatibility mode"
   exit 1
 }
 
@@ -50,6 +51,10 @@ do
             STARLING_ARGS="$STARLING_ARGS $1 $2"
             shift 2
             ;;
+        --wpt-mode)
+            STARLING_ARGS="$STARLING_ARGS $1 $2"
+            shift 2
+            ;;
         -v|--verbose)
             STARLING_ARGS="$1 $STARLING_ARGS"
             VERBOSE=1

docs/src/getting-started/testing.md

@@ -11,7 +11,7 @@ cmake --build cmake-build-debug --target integration-test-server
 Then tests can be run with `ctest` directly via:
 
 ```console
-ctest --test-dir cmake-build-debug -j8 --output-on-failure
+ctest --test-dir cmake-build-debug -j$(nproc) --output-on-failure
 ```
 
 Alternatively, the integration test server can be directly run with `wasmtime serve` via:
@@ -25,12 +25,11 @@ Then visit `http://0.0.0.0:8080/timers`, or any test name and filter of the form
 ## Web Platform Tests
 
 To run the [Web Platform Tests](https://web-platform-tests.org/) suite, the WPT runner requires
-`Node.js` above v18.0 to be installed, and during build configuration the option `ENABLE_WPT:BOOL=ON` 
-must be set.
+`Node.js` above v18.0 to be installed, and the list of hosts in `deps/wpt-hosts` needs to be added to `etc/hosts`.
 
 ```console
-cmake -S . -B cmake-build-debug -DENABLE_WPT:BOOL=ON -DCMAKE_BUILD_TYPE=Debug
-cmake --build cmake-build-debug --parallel 8 --target wpt-runtime
+cmake -S . -B cmake-build-debug -DCMAKE_BUILD_TYPE=Debug
+cmake --build cmake-build-debug --parallel $(nproc) --target wpt-runtime
 cat deps/wpt-hosts | sudo tee -a /etc/hosts # Required to resolve test server hostnames
 cd cmake-build-debug
 ctest -R wpt --verbose # Note: some of the tests run fairly slowly in debug builds, so be patient

include/config-parser.h

@@ -97,6 +97,8 @@ class ConfigParser {
           config_->content_script_path = mozilla::Some(args[i + 1]);
           i++;
         }
+      } else if (args[i] == "--wpt-mode") {
+        config_->wpt_mode = true;
       } else if (args[i].starts_with("--")) {
         std::cerr << "Unknown option: " << args[i] << std::endl;
         exit(1);

include/extension-api.h

@@ -65,6 +65,13 @@ struct EngineConfig {
    */
   bool debugging = false;
 
+  /**
+   * Whether to enable Web Platform Test mode. Specifically, this means installing a
+   * few global properties required to make WPT work, that wouldn't be made available
+   * to content.
+   */
+  bool wpt_mode = false;
+
   EngineConfig() = default;
 };
 
@@ -82,6 +89,7 @@ class Engine {
   HandleObject global();
   EngineState state();
   bool debugging_enabled();
+  bool wpt_mode();
 
   void finish_pre_initialization();
 

justfile

@@ -58,26 +58,22 @@ format *ARGS:
     {{ justdir }}/scripts/clang-format.sh {{ ARGS }}
 
 # Run integration test
-test regex="": (build "integration-test-server")
+test regex="": (build "integration-test-server") (build "wpt-runtime")
     ctest --test-dir {{ builddir }} -j {{ ncpus }} --output-on-failure {{ if regex == "" { regex } else { "-R " + regex } }}
 
-# Build web platform test suite
-[group('wpt')]
-wpt-build: (build "wpt-runtime" "-DENABLE_WPT:BOOL=ON")
-
 # Run web platform test suite
 [group('wpt')]
-wpt-test filter="": wpt-build
+wpt-test filter="": (build "wpt-runtime")
     WPT_FILTER={{ filter }} ctest --test-dir {{ builddir }} -R wpt --verbose
 
 # Update web platform test expectations
 [group('wpt')]
-wpt-update filter="": wpt-build
+wpt-update filter="": (build "wpt-runtime")
     WPT_FLAGS="--update-expectations" WPT_FILTER={{ filter }} ctest --test-dir {{ builddir }} -R wpt --verbose
 
 # Run wpt server
 [group('wpt')]
-wpt-server: wpt-build
+wpt-server: (build "wpt-runtime")
     #!/usr/bin/env bash
     set -euo pipefail
     cd {{ builddir }}

runtime/engine.cpp

@@ -518,6 +518,7 @@ EngineState Engine::state() { return state_; }
 bool Engine::debugging_enabled() {
   return config_->debugging;
 }
+bool Engine::wpt_mode() { return config_->wpt_mode; }
 
 void Engine::finish_pre_initialization() {
   MOZ_ASSERT(state_ == EngineState::ScriptPreInitializing);

tests/wpt-harness/build-wpt-runtime.sh

@@ -18,4 +18,4 @@ inputs=(
 )
 
 cat "${inputs[@]}" > wpt-test-runner.js
-./componentize.sh $componentize_flags --verbose --legacy-script wpt-test-runner.js wpt-runtime.wasm
+./componentize.sh $componentize_flags --verbose --wpt-mode --legacy-script wpt-test-runner.js wpt-runtime.wasm

tests/wpt-harness/run-wpt.mjs

@@ -312,8 +312,10 @@ async function timeout(millis, message) {
 
 async function wasmtimeReady(wasmtime, config) {
   // Wait until Wasmtime has fully initialized and extract host from output.
+  let output = "";
   for await(const [chunk] of on(wasmtime.stderr, "data")) {
-    let result = chunk.match(/Serving HTTP on (.+)/);
+    output += chunk;
+    let result = output.match(/Serving HTTP on (http:\/\/.+\/)/);
     if (result) {
       return { process: wasmtime, host: result[1], ...config };
     }
@@ -333,6 +335,9 @@ async function startWasmtime(runtime, addr, logLevel) {
   let backtrace = "";
   let backtrace_re = /Error \{\s+context: "error while executing at wasm backtrace:(.+?)",\s+source: (.+?)/s;
   wasmtime.stderr.on("data", data => {
+    if (logLevel >= LogLevel.VeryVerbose) {
+      console.log(`wasmtime stderr: ${stripTrailingNewline(data)}`);
+    }
     if (backtrace.length > 0 || data.includes("error while executing at wasm backtrace:")) {
       backtrace += data;
       let match = backtrace.match(backtrace_re);
@@ -353,11 +358,11 @@ async function startWasmtime(runtime, addr, logLevel) {
     });
   }
 
-  // give wasmtime 20 seconds to become available
-  const WASMTIME_READY_TIMEOUT = 20000;
+  // give wasmtime 60 seconds to become available
+  const WASMTIME_READY_TIMEOUT = 60000;
   return await Promise.race([
     wasmtimeReady(wasmtime, config),
-    timeout(WASMTIME_READY_TIMEOUT, "Wasmtime failed to start"),
+    timeout(WASMTIME_READY_TIMEOUT, `Wasmtime start timeout after ${WASMTIME_READY_TIMEOUT}ms`),
   ]);
 }