Merge pull request #41 from camillobruni/2025-01-28_runner_log

kmiller68 · web-flow · commit dd08c17e6ebd · 2025-04-07T10:53:13.000-07:00
Improve end2end browser tests
diff --git a/JetStream.css b/JetStream.css
@@ -312,6 +312,7 @@ a.button {
     background-image: none;
 }
 
+
 .benchmark-done h3, .benchmark-done h4, .benchmark-done .result, .benchmark-done label {
     background-color: transparent;
     background-image: none;
@@ -322,6 +323,16 @@ a.button {
     user-select: text;
 }
 
+.benchmark-error h4, .benchmark-error .result, .benchmark-error label {
+    color: #ff8686;
+    background-color: #ff8686;
+    background-image: none;
+}
+
+.benchmark-error h3 {
+    color: #ff8686;
+}
+
 .benchmark h3 {
     font-weight: 400;
     font-size: 1.6rem;
diff --git a/JetStreamDriver.js b/JetStreamDriver.js
@@ -239,6 +239,8 @@ const fileLoader = (function() {
 class Driver {
     constructor() {
         this.isReady = false;
+        this.isDone = false;
+        this.errors = [];
         this.benchmarks = [];
         this.blobDataCache = { };
         this.loadCache = { };
@@ -274,11 +276,12 @@ class Driver {
             try {
                 await benchmark.run();
             } catch(e) {
-                JetStream.reportError(benchmark);
+                this.reportError(benchmark, e);
                 throw e;
             }
 
             benchmark.updateUIAfterRun();
+            console.log(benchmark.name)
 
             if (isInBrowser) {
                 const cache = JetStream.blobDataCache;
@@ -334,6 +337,8 @@ class Driver {
 
         this.reportScoreToRunBenchmarkRunner();
         this.dumpJSONResultsIfNeeded();
+        this.isDone = true;
+
         if (isInBrowser) {
             globalThis.dispatchEvent(new CustomEvent("JetStreamDone", {
                 detail: this.resultsObject()
@@ -438,16 +443,32 @@ class Driver {
         });
     }
 
-    reportError(benchmark)
+    reportError(benchmark, error)
     {
+        this.pushError(benchmark.name, error);
+
         if (!isInBrowser)
             return;
 
-        for (const id of benchmark.scoreIdentifiers())
+        for (const id of benchmark.scoreIdentifiers()) {
             document.getElementById(id).innerHTML = "error";
+            const benchmarkResultsUI = document.getElementById(`benchmark-${benchmark.name}`);
+            benchmarkResultsUI.classList.remove("benchmark-running");
+            benchmarkResultsUI.classList.add("benchmark-error");
+        }
+    }
+
+    pushError(name, error) {
+        this.errors.push({
+            benchmark: name,
+            error: error.toString(),
+            stack: error.stack
+        });
     }
 
     async initialize() {
+        if (isInBrowser)
+            window.addEventListener("error", (e) => this.pushError("driver startup", e.error));
         await this.prefetchResourcesForBrowser();
         await this.fetchResources();
         this.prepareToRun();
@@ -508,7 +529,18 @@ class Driver {
         }
     }
 
-    resultsObject()
+    resultsObject(format = "run-benchmark") {
+        switch(format) {
+            case "run-benchmark":
+                return this.runBenchmarkResultsObject();
+            case "simple":
+                return this.simpleResultsObject();
+            default:
+                throw Error(`Unknown result format '${format}'`);
+        }
+    }
+
+    runBenchmarkResultsObject()
     {
         let results = {};
         for (const benchmark of this.benchmarks) {
@@ -528,12 +560,29 @@ class Driver {
 
         results = {"JetStream3.0": {"metrics" : {"Score" : ["Geometric"]}, "tests" : results}};
         return results;
+    }
 
+    simpleResultsObject() {
+        const results = {__proto__: null};
+        for (const benchmark of this.benchmarks) {
+            if (!benchmark.isDone)
+                continue;
+            if (!benchmark.isSuccess) {
+                results[benchmark.name] = "FAILED";
+            } else {
+                results[benchmark.name] = {
+                    Score: benchmark.score,
+                    ...benchmark.subScores(),
+
+                };
+            }
+        }
+        return results;
     }
 
-    resultsJSON()
+    resultsJSON(format = "run-benchmark")
     {
-        return JSON.stringify(this.resultsObject());
+        return JSON.stringify(this.resultsObject(format));
     }
 
     dumpJSONResultsIfNeeded()
@@ -566,6 +615,15 @@ class Driver {
     }
 };
 
+const BenchmarkState = Object.freeze({
+    READY: "READY",
+    SETUP: "SETUP",
+    RUNNING: "RUNNING",
+    FINALIZE: "FINALIZE",
+    ERROR: "ERROR",
+    DONE: "DONE"
+})
+
 class Benchmark {
     constructor(plan)
     {
@@ -575,10 +633,16 @@ class Benchmark {
         this.isAsync = !!plan.isAsync;
         this.scripts = null;
         this._resourcesPromise = null;
+        this._state = BenchmarkState.READY;
     }
 
     get name() { return this.plan.name; }
 
+    get isDone() {
+        return this._state == BenchmarkState.DONE || this._state == BenchmarkState.ERROR;
+    }
+    get isSuccess() { return this._state = BenchmarkState.DONE; }
+
     get runnerCode() {
         return `
             let __benchmark = new Benchmark(${this.iterations});
@@ -640,6 +704,9 @@ class Benchmark {
     }
 
     async run() {
+        if (this.isDone)
+            throw new Error(`Cannot run Benchmark ${this.name} twice`);
+        this._state = BenchmarkState.PREPARE;
         let code;
         if (isInBrowser)
             code = "";
@@ -748,11 +815,15 @@ class Benchmark {
 
         let magicFrame;
         try {
+            this._state = BenchmarkState.RUNNING;
             magicFrame = JetStream.runCode(code);
         } catch(e) {
+            this._state = BenchmarkState.ERROR;
             console.log("Error in runCode: ", e);
             console.log(e.stack)
             throw e;
+        } finally {
+            this._state = BenchmarkState.FINALIZE;
         }
         const results = await promise;
 
@@ -765,6 +836,8 @@ class Benchmark {
         }
 
         this.processResults(results);
+        this._state = BenchmarkState.DONE;
+
         if (isInBrowser)
             magicFrame.contentDocument.close();
         else if (isD8)
diff --git a/tests/run.mjs b/tests/run.mjs
@@ -4,7 +4,6 @@ import serve from "./server.mjs";
 import { Builder, Capabilities } from "selenium-webdriver";
 import commandLineArgs from "command-line-args";
 import commandLineUsage from "command-line-usage";
-import assert from "assert";
 
 const optionDefinitions = [
     { name: "browser", type: String, description: "Set the browser to test, choices are [safari, firefox, chrome]. By default the $BROWSER env variable is used." },
@@ -81,7 +80,9 @@ async function testEnd2End() {
     const driver = await new Builder().withCapabilities(capabilities).build();
     let results;
     try {
-        await driver.get(`http://localhost:${PORT}/index.html?worstCaseCount=2&iterationCount=3`);
+        const url = `http://localhost:${PORT}/index.html?worstCaseCount=2&iterationCount=3`;
+        console.log(`JetStream PREPARE ${url}`);
+        await driver.get(url);
         await driver.executeAsyncScript((callback) => {
             // callback() is explicitly called without the default event
             // as argument to avoid serialization issues with chromedriver.
@@ -91,14 +92,9 @@ async function testEnd2End() {
             if (globalThis?.JetStream?.isReady)
                 callback()
         });
-        await driver.manage().setTimeouts({ script: 3 * 60_000 });
-        results = await driver.executeAsyncScript((callback) => {
-            globalThis.addEventListener("JetStreamDone", event => callback(event.detail));
-            JetStream.start();
-        });
+        results = await benchmarkResults(driver);
+        // FIXME: validate results;
         console.log("\n✅ Tests completed!");
-        console.log("RESULTS:")
-        console.log(results)
     } catch(e) {
         console.error("\n❌ Tests failed!");
         console.error(e);
@@ -109,4 +105,59 @@ async function testEnd2End() {
     }
 }
 
+async function benchmarkResults(driver) {
+    console.log("JetStream START");
+    await driver.manage().setTimeouts({ script: 60_000 });
+    await driver.executeAsyncScript((callback) => {
+        globalThis.JetStream.start();
+        callback();
+    });
+
+    await new Promise((resolve, reject) => pollResultsUntilDone(driver, resolve, reject));
+    const resultsJSON = await driver.executeScript(() => {
+        return globalThis.JetStream.resultsJSON();
+    });
+    return JSON.parse(resultsJSON);
+}
+
+class JetStreamTestError extends Error {
+    constructor(errors) {
+        super(`Tests failed: ${errors.map(e => e.name).join(", ")}`);
+        this.errors = errors;
+    }
+
+}
+
+const UPDATE_INTERVAL = 250;
+async function pollResultsUntilDone(driver, resolve, reject) {
+    const previousResults = new Set();
+    const intervalId = setInterval(async function logResult()  {
+        const {done, errors, resultsJSON} = await driver.executeScript(() => {
+            return {
+                done: globalThis.JetStream.isDone,
+                errors: globalThis.JetStream.errors,
+                resultsJSON: JSON.stringify(globalThis.JetStream.resultsObject("simple")),
+            };
+        });
+        if (errors.length) {
+            clearInterval(intervalId);
+            reject(new JetStreamTestError(errors));
+        }
+        logIncrementalResult(previousResults, JSON.parse(resultsJSON));
+        if (done) {
+            clearInterval(intervalId);
+            resolve();
+        }
+    }, UPDATE_INTERVAL)
+}
+
+function logIncrementalResult(previousResults, benchmarkResults) {
+    for (const [testName, testResults] of Object.entries(benchmarkResults)) {
+        if (previousResults.has(testName))
+            continue;
+        console.log(testName, testResults);
+        previousResults.add(testName);
+    }
+}
+
 setImmediate(testEnd2End);
