Configurable resource prefetching (#70)

This addresses issue #58 and is useful for local profiling.

If the prefetchResources is disabled, we now no longer preload them eagerly, but rather use the live URL and either script-inject it or use it directly with workers.
This means we can easily get the original source positions for profiling.

- `--no-prefetch` command ilne support
- `&prefetchResources=0` URLParam support
- Improve test coverage and add --no-prefetch tests
- Check that test shell output for failures to catch JSC errors (it does not report exit != 0 on rejected promises)

Drive-by-fix:
- Move FileLoader out of the IIFE
- Check that JetStreamDriver has at least one active benchmark
- Add assertions for valid scores
- Fix benchmarks selection with tags

Author: camillobruni

JetStreamDriver.js

@@ -40,17 +40,21 @@ globalThis.dumpJSONResults ??= false;
 globalThis.testList ??= undefined;
 globalThis.startDelay ??= undefined;
 globalThis.shouldReport ??= false;
+globalThis.prefetchResources ??= true;
 
 function getIntParam(urlParams, key) {
-    if (!urlParams.has(key))
-        return undefined
     const rawValue = urlParams.get(key);
     const value = parseInt(rawValue);
     if (value <= 0)
         throw new Error(`Expected positive value for ${key}, but got ${rawValue}`);
     return value;
 }
 
+function getBoolParam(urlParams, key) {
+    const rawValue = urlParams.get(key).toLowerCase()
+    return !(rawValue === "false" || rawValue === "0")
+ }
+
 function getTestListParam(urlParams, key) {
     if (globalThis.testList?.length)
         throw new Error(`Overriding previous testList=${globalThis.testList.join()} with ${key} url-parameter.`);
@@ -73,8 +77,13 @@ if (typeof(URLSearchParams) !== "undefined") {
         globalThis.testIterationCount = getIntParam(urlParameters, "iterationCount");
     if (urlParameters.has("worstCaseCount"))
         globalThis.testWorstCaseCount = getIntParam(urlParameters, "worstCaseCount");
+    if (urlParameters.has("prefetchResources"))
+        globalThis.prefetchResources = getBoolParam(urlParameters, "prefetchResources");
 }
 
+if (!globalThis.prefetchResources)
+    console.warn("Disabling resource prefetching!");
+
 // Used for the promise representing the current benchmark run.
 this.currentResolve = null;
 this.currentReject = null;
@@ -88,7 +97,7 @@ function displayCategoryScores() {
 
     let summaryElement = document.getElementById("result-summary");
     for (let [category, scores] of categoryScores)
-        summaryElement.innerHTML += `<p> ${category}: ${uiFriendlyScore(geomean(scores))}</p>`
+        summaryElement.innerHTML += `<p> ${category}: ${uiFriendlyScore(geomeanScore(scores))}</p>`
 
     categoryScores = null;
 }
@@ -138,12 +147,15 @@ function mean(values) {
     return sum / values.length;
 }
 
-function geomean(values) {
+function geomeanScore(values) {
     assert(values instanceof Array);
     let product = 1;
     for (let x of values)
         product *= x;
-    return product ** (1 / values.length);
+    const score = product ** (1 / values.length);
+    // Allow 0 for uninitialized subScores().
+    assert(score >= 0, `Got invalid score: ${score}`)
+    return score;
 }
 
 function toScore(timeValue) {
@@ -180,28 +192,29 @@ function uiFriendlyDuration(time) {
 // TODO: Cleanup / remove / merge. This is only used for caching loads in the
 // non-browser setting. In the browser we use exclusively `loadCache`, 
 // `loadBlob`, `doLoadBlob`, `prefetchResourcesForBrowser` etc., see below.
-const fileLoader = (function() {
-    class Loader {
-        constructor() {
-            this.requests = new Map;
-        }
-
-        // Cache / memoize previously read files, because some workloads
-        // share common code.
-        load(url) {
-            assert(!isInBrowser);
+class ShellFileLoader {
+    constructor() {
+        this.requests = new Map;
+    }
 
-            if (this.requests.has(url)) {
-                return this.requests.get(url);
-            }
+    // Cache / memoize previously read files, because some workloads
+    // share common code.
+    load(url) {
+        assert(!isInBrowser);
+        if (!globalThis.prefetchResources)
+            return `load("${url}");`
 
-            const contents = readFile(url);
-            this.requests.set(url, contents);
-            return contents;
+        if (this.requests.has(url)) {
+            return this.requests.get(url);
         }
+
+        const contents = readFile(url);
+        this.requests.set(url, contents);
+        return contents;
     }
-    return new Loader;
-})();
+};
+
+const shellFileLoader = new ShellFileLoader();
 
 class Driver {
     constructor(benchmarks) {
@@ -211,6 +224,7 @@ class Driver {
         // Make benchmark list unique and sort it.
         this.benchmarks = Array.from(new Set(benchmarks));
         this.benchmarks.sort((a, b) => a.plan.name.toLowerCase() < b.plan.name.toLowerCase() ? 1 : -1);
+        assert(this.benchmarks.length, "No benchmarks selected");
         // TODO: Cleanup / remove / merge `blobDataCache` and `loadCache` vs.
         // the global `fileLoader` cache.
         this.blobDataCache = { };
@@ -248,7 +262,7 @@ class Driver {
             performance.mark("update-ui");
             benchmark.updateUIAfterRun();
 
-            if (isInBrowser) {
+            if (isInBrowser && globalThis.prefetchResources) {
                 const cache = JetStream.blobDataCache;
                 for (const file of benchmark.plan.files) {
                     const blobData = cache[file];
@@ -270,8 +284,11 @@ class Driver {
         }
 
         const allScores = [];
-        for (const benchmark of this.benchmarks)
-            allScores.push(benchmark.score);
+        for (const benchmark of this.benchmarks) {
+            const score = benchmark.score;
+            assert(score > 0, `Invalid ${benchmark.name} score: ${score}`);
+            allScores.push(score);
+        }
 
         categoryScores = new Map;
         for (const benchmark of this.benchmarks) {
@@ -282,23 +299,27 @@ class Driver {
         for (const benchmark of this.benchmarks) {
             for (let [category, value] of Object.entries(benchmark.subScores())) {
                 const arr = categoryScores.get(category);
+                assert(value > 0, `Invalid ${benchmark.name} ${category} score: ${value}`);
                 arr.push(value);
             }
         }
 
+        const totalScore = geomeanScore(allScores);
+        assert(totalScore > 0, `Invalid total score: ${totalScore}`);
+
         if (isInBrowser) {
-            summaryElement.classList.add('done');
-            summaryElement.innerHTML = `<div class="score">${uiFriendlyScore(geomean(allScores))}</div><label>Score</label>`;
+            summaryElement.classList.add("done");
+            summaryElement.innerHTML = `<div class="score">${uiFriendlyScore(totalScore)}</div><label>Score</label>`;
             summaryElement.onclick = displayCategoryScores;
             if (showScoreDetails)
                 displayCategoryScores();
-            statusElement.innerHTML = '';
+            statusElement.innerHTML = "";
         } else if (!dumpJSONResults) {
             console.log("\n");
             for (let [category, scores] of categoryScores)
-                console.log(`${category}: ${uiFriendlyScore(geomean(scores))}`);
+                console.log(`${category}: ${uiFriendlyScore(geomeanScore(scores))}`);
 
-            console.log("\nTotal Score: ", uiFriendlyScore(geomean(allScores)), "\n");
+            console.log("\nTotal Score: ", uiFriendlyScore(totalScore), "\n");
         }
 
         this.reportScoreToRunBenchmarkRunner();
@@ -727,7 +748,7 @@ class Benchmark {
 
     get score() {
         const subScores = Object.values(this.subScores());
-        return geomean(subScores);
+        return geomeanScore(subScores);
     }
 
     subScores() {
@@ -788,8 +809,9 @@ class Benchmark {
                 scripts.add(text);
         } else {
             const cache = JetStream.blobDataCache;
-            for (const file of this.plan.files)
-                scripts.addWithURL(cache[file].blobURL);
+            for (const file of this.plan.files) {
+                scripts.addWithURL(globalThis.prefetchResources ? cache[file].blobURL : file);
+            }
         }
 
         const promise = new Promise((resolve, reject) => {
@@ -838,6 +860,11 @@ class Benchmark {
     }
 
     async doLoadBlob(resource) {
+        const blobData = JetStream.blobDataCache[resource];
+        if (!globalThis.prefetchResources) {
+            blobData.blobURL = resource;
+            return blobData;
+        }
         let response;
         let tries = 3;
         while (tries--) {
@@ -854,7 +881,6 @@ class Benchmark {
             throw new Error("Fetch failed");
         }
         const blob = await response.blob();
-        const blobData = JetStream.blobDataCache[resource];
         blobData.blob = blob;
         blobData.blobURL = URL.createObjectURL(blob);
         return blobData;
@@ -987,7 +1013,7 @@ class Benchmark {
         assert(!isInBrowser);
 
         assert(this.scripts === null, "This initialization should be called only once.");
-        this.scripts = this.plan.files.map(file => fileLoader.load(file));
+        this.scripts = this.plan.files.map(file => shellFileLoader.load(file));
 
         assert(this.preloads === null, "This initialization should be called only once.");
         this.preloads = Object.entries(this.plan.preload ?? {});
@@ -2417,8 +2443,7 @@ for (const benchmark of BENCHMARKS) {
 }
 
 
-function processTestList(testList)
-{
+function processTestList(testList) {
     let benchmarkNames = [];
     let benchmarks = [];
 
@@ -2430,7 +2455,7 @@ function processTestList(testList)
     for (let name of benchmarkNames) {
         name = name.toLowerCase();
         if (benchmarksByTag.has(name))
-            benchmarks.concat(findBenchmarksByTag(name));
+            benchmarks = benchmarks.concat(findBenchmarksByTag(name));
         else
             benchmarks.push(findBenchmarkByName(name));
     }

cli.js

@@ -56,6 +56,8 @@ if (typeof runMode !== "undefined" && runMode == "RAMification")
     globalThis.RAMification = true;
 if ("--ramification" in cliFlags)
     globalThis.RAMification = true;
+if ("--no-prefetch" in cliFlags)
+    globalThis.prefetchResources = false;
 if (cliArgs.length)
     globalThis.testList = cliArgs;
 
@@ -78,9 +80,11 @@ if ("--help" in cliFlags) {
     console.log("");
 
     console.log("Options:");
-    console.log("   --iteration-count:  Set the default iteration count.");
-    console.log("   --worst-case-count: Set the default worst-case count");
+    console.log("   --iteration-count:   Set the default iteration count.");
+    console.log("   --worst-case-count:  Set the default worst-case count");
     console.log("   --dump-json-results: Print summary json to the console.");
+    console.log("   --ramification:      Enable ramification support. See RAMification.py for more details.");
+    console.log("   --no-prefetch:       Do not prefetch resources. Will add network overhead to measurements!");
     console.log("");
 
     console.log("Available tags:");

package.json

@@ -23,6 +23,7 @@
         "test:firefox": "node tests/run.mjs --browser firefox",
         "test:safari": "node tests/run.mjs --browser safari",
         "test:edge": "node tests/run.mjs --browser edge",
+        "test:shell": "npm run test:v8 && npm run test:jsc && npm run test:spidermonkey",
         "test:v8": "node tests/run-shell.mjs --shell v8",
         "test:jsc": "node tests/run-shell.mjs --shell jsc",
         "test:spidermonkey": "node tests/run-shell.mjs --shell spidermonkey"

shell-config.js

@@ -27,6 +27,7 @@ const isInBrowser = false;
 console = {
     log: globalThis?.console?.log ?? print,
     error: globalThis?.console?.error ?? print,
+    warn: globalThis?.console?.warn ?? print,
 }
 
 const isD8 = typeof Realm !== "undefined";

tests/run-shell.mjs

@@ -1,7 +1,7 @@
 #! /usr/bin/env node
 
 import commandLineArgs from "command-line-args";
-import { spawnSync } from  "child_process";
+import { spawn } from  "child_process";
 import { fileURLToPath } from "url";
 import { styleText } from "node:util";
 import * as path from "path";
@@ -59,7 +59,7 @@ const SPAWN_OPTIONS =  {
   stdio: ["inherit", "inherit", "inherit"]
 };
 
-function sh(binary, ...args) {
+async function sh(binary, ...args) {
   const cmd = `${binary} ${args.join(" ")}`;
   if (GITHUB_ACTIONS_OUTPUT) {
     core.startGroup(binary);
@@ -68,22 +68,48 @@ function sh(binary, ...args) {
     console.log(styleText("blue", cmd));
   }
   try {
-    const result = spawnSync(binary, args, SPAWN_OPTIONS);
+    const result = await spawnCaptureStdout(binary, args, SPAWN_OPTIONS);
     if (result.status || result.error) {
       logError(result.error);
       throw new Error(`Shell CMD failed: ${binary} ${args.join(" ")}`);
     }
+    return result;
   } finally {
     if (GITHUB_ACTIONS_OUTPUT)
       core.endGroup();
   }
 }
 
+async function spawnCaptureStdout(binary, args) {
+  const childProcess = spawn(binary, args);
+  childProcess.stdout.pipe(process.stdout);
+  return new Promise((resolve, reject) => {
+    childProcess.stdoutString = "";
+    childProcess.stdio[1].on("data", (data) => {
+      childProcess.stdoutString += data.toString();
+    });
+    childProcess.on('close', (code) => {
+      if (code === 0) {
+        resolve(childProcess);
+      } else {
+        // Reject the Promise with an Error on failure
+        const error = new Error(`Command failed with exit code ${code}: ${binary} ${args.join(" ")}`);
+        error.process = childProcess;
+        error.stdout = childProcess.stdoutString;
+        error.exitCode = code;
+        reject(error);
+      }
+    });
+    childProcess.on('error', reject);
+  })
+}
+
 async function runTests() {
     const shellBinary = await logGroup(`Installing JavaScript Shell: ${SHELL_NAME}`, testSetup);
     let success = true;
     success &&= await runTest("Run UnitTests", () => sh(shellBinary, UNIT_TEST_PATH));
     success &&= await runCLITest("Run Single Suite", shellBinary, "proxy-mobx");
+    success &&= await runCLITest("Run Tag No Prefetch", shellBinary, "proxy", "--no-prefetch");
     success &&= await runCLITest("Run Disabled Suite", shellBinary, "disabled");
     success &&= await runCLITest("Run Default Suite",  shellBinary);
     if (!success)
@@ -111,8 +137,8 @@ function jsvuOSName() {
 
 const DEFAULT_JSC_LOCATION = "/System/Library/Frameworks/JavaScriptCore.framework/Versions/Current/Helpers/jsc"
 
-function testSetup() {
-    sh("jsvu", `--engines=${SHELL_NAME}`, `--os=${jsvuOSName()}`);
+async function testSetup() {
+    await sh("jsvu", `--engines=${SHELL_NAME}`, `--os=${jsvuOSName()}`);
     let shellBinary = path.join(os.homedir(), ".jsvu/bin", SHELL_NAME);
     if (!fs.existsSync(shellBinary) && SHELL_NAME == "javascriptcore")
       shellBinary = DEFAULT_JSC_LOCATION;
@@ -123,7 +149,13 @@ function testSetup() {
 }
 
 function runCLITest(name, shellBinary, ...args) {
-  return runTest(name, () => sh(shellBinary, ...convertCliArgs(CLI_PATH, ...args)));
+  return runTest(name, () => runShell(shellBinary, ...convertCliArgs(CLI_PATH, ...args)));
+}
+
+async function runShell(shellBinary, ...args) {
+  const result = await sh(shellBinary, ...args);
+  if (result.stdoutString.includes("JetStream3 failed"))
+    throw new Error("test failed")
 }
 
 setImmediate(runTests);