cleanup

Author: camillobruni

JetStream.css

@@ -312,6 +312,7 @@ a.button {
     background-image: none;
 }
 
+
 .benchmark-done h3, .benchmark-done h4, .benchmark-done .result, .benchmark-done label {
     background-color: transparent;
     background-image: none;
@@ -322,6 +323,16 @@ a.button {
     user-select: text;
 }
 
+.benchmark-error h4, .benchmark-error .result, .benchmark-error label {
+    color: #ff8686;
+    background-color: #ff8686;
+    background-image: none;
+}
+
+.benchmark-error h3 {
+    color: #ff8686;
+}
+
 .benchmark h3 {
     font-weight: 400;
     font-size: 1.6rem;

JetStreamDriver.js

@@ -240,7 +240,7 @@ class Driver {
     constructor() {
         this.isReady = false;
         this.isDone = false;
-        this.incrementalResults = [];
+        this.errors = [];
         this.benchmarks = [];
         this.blobDataCache = { };
         this.loadCache = { };
@@ -276,21 +276,14 @@ class Driver {
             try {
                 await benchmark.run();
             } catch(e) {
-                JetStream.reportError(benchmark);
+                this.reportError(benchmark, e);
                 throw e;
             }
 
             benchmark.updateUIAfterRun();
             console.log(benchmark.name)
 
             if (isInBrowser) {
-                this.incrementalResults.push({
-                    name: benchmark.name,
-                    results: {
-                        Score: benchmark.score,
-                        ...benchmark.subScores(),
-                    }
-                });
                 const cache = JetStream.blobDataCache;
                 for (const file of benchmark.plan.files) {
                     const blobData = cache[file];
@@ -345,19 +338,14 @@ class Driver {
         this.reportScoreToRunBenchmarkRunner();
         this.dumpJSONResultsIfNeeded();
         this.isDone = true;
+
         if (isInBrowser) {
             globalThis.dispatchEvent(new CustomEvent("JetStreamDone", {
-                detail: this.resultsObject()
+                detail: this.resultsObjectNew()
             }));
         }
     }
 
-    drainIncrementalResults() {
-        const currentIncrementalResults = this.incrementalResults;
-        this.incrementalResults = [];
-        return currentIncrementalResults
-    }
-
     runCode(string)
     {
         if (!isInBrowser) {
@@ -455,16 +443,31 @@ class Driver {
         });
     }
 
-    reportError(benchmark)
+    reportError(benchmark, error)
     {
+        this.pushError(benchmark.name, error);
+
         if (!isInBrowser)
             return;
 
-        for (const id of benchmark.scoreIdentifiers())
+        for (const id of benchmark.scoreIdentifiers()) {
             document.getElementById(id).innerHTML = "error";
+            const benchmarkResultsUI = document.getElementById(`benchmark-${benchmark.name}`);
+            benchmarkResultsUI.classList.remove("benchmark-running");
+            benchmarkResultsUI.classList.add("benchmark-error");
+        }
+    }
+
+    pushError(name, error) {
+        this.errors.push({
+            benchmark: name,
+            error: error.toString(),
+            stack: error.stack
+        });
     }
 
     async initialize() {
+        window.addEventListener("error", (e) => this.pushError(benchmark, e.error));
         await this.prefetchResourcesForBrowser();
         await this.fetchResources();
         this.prepareToRun();
@@ -525,16 +528,43 @@ class Driver {
         }
     }
 
-    resultsObject()
+    resultsObject(newStyle = false) {
+        if (newStyle)
+            return this.resultsObjectNew();
+        return this.resultsObjectOld();
+    }
+
+    resultsObjectNew() {
+        const results = {__proto__: null};
+        for (const benchmark of this.benchmarks) {
+            if (!benchmark.isDone)
+                continue;
+            results[benchmark.name] = {
+                Score: benchmark.score,
+                ...benchmark.subScores(),
+
+            };
+        }
+        return results;
+    }
+
+
+    resultsObjectOld()
     {
-        let results = {};
+        let testResults = {};
         for (const benchmark of this.benchmarks) {
             const subResults = {}
             const subScores = benchmark.subScores();
             for (const name in subScores) {
-                subResults[name] = {"metrics": {"Time": {"current": [toTimeValue(subScores[name])]}}};
+                subResults[name] = {
+                    "metrics": {
+                        "Time": {
+                            "current": [toTimeValue(subScores[name])]
+                        }
+                    }
+                };
             }
-            results[benchmark.name] = {
+            testResults[benchmark.name] = {
                 "metrics" : {
                     "Score" : {"current" : [benchmark.score]},
                     "Time": ["Geometric"],
@@ -543,9 +573,15 @@ class Driver {
             };
         }
 
-        results = {"JetStream3.0": {"metrics" : {"Score" : ["Geometric"]}, "tests" : results}};
+        const results = {
+            "JetStream3.0": {
+                "metrics" : {
+                    "Score" : ["Geometric"]
+                },
+                "tests" : testResults
+            }
+        };
         return results;
-
     }
 
     resultsJSON()
@@ -595,10 +631,13 @@ class Benchmark {
         this.scripts = null;
 
         this._resourcesPromise = null;
+        this._isDone = false;
     }
 
     get name() { return this.plan.name; }
 
+    get isDone() { return this._isDone; }
+
     get runnerCode() {
         return `
             let __benchmark = new Benchmark(${this.iterations});
@@ -660,6 +699,8 @@ class Benchmark {
     }
 
     async run() {
+        if (this.isDone)
+            throw new Error(`Cannot run Benchmark ${this.name} twice`);
         let code;
         if (isInBrowser)
             code = "";
@@ -777,6 +818,7 @@ class Benchmark {
         const results = await promise;
 
         this.endTime = performance.now();
+        this._isDone = true;
 
         if (RAMification) {
             const memoryFootprint = MemoryFootprint();

tests/run.mjs

@@ -98,6 +98,7 @@ async function testEnd2End() {
         console.error(e);
         throw e;
     } finally {
+        driver.close();
         driver.quit();
         server.close();
     }
@@ -110,32 +111,53 @@ async function benchmarkResults(driver) {
         globalThis.JetStream.start();
         callback();
     });
-    await new Promise(resolve => pollIncrementalResults(driver, resolve));
-    const resultString = await driver.executeScript(() => {
-        return JSON.stringify(globalThis.JetStream.resultsObject());
+
+    await new Promise((resolve, reject) => pollResultsUntilDone(driver, resolve, reject));
+    const resultsJSON = await driver.executeScript(() => {
+        return globalThis.JetStream.resultsJSON();
     });
-    return  JSON.parse(resultString);
+    return JSON.parse(resultsJSON);
+}
+
+class JetStreamTestError extends Error {
+    constructor(errors) {
+        super(`Tests failed: ${errors.map(e => e.name).join(", ")}`)
+        this.errors = errors
+    }
+
 }
 
 const UPDATE_INTERVAL = 250;
-async function pollIncrementalResults(driver, resolve) {
+async function pollResultsUntilDone(driver, resolve, reject) {
+    const previousResults = new Set();
     const intervalId = setInterval(async function logResult()  {
-        const {done, results} = await driver.executeScript(() => {
+        const {done, errors, resultsJSON} = await driver.executeScript(() => {
             return {
                 done: globalThis.JetStream.isDone,
-                results: JSON.stringify(globalThis.JetStream.drainIncrementalResults()),
+                errors: globalThis.JetStream.errors,
+                resultsJSON: JSON.stringify(globalThis.JetStream.resultsObjectNew()),
             };
         });
-        JSON.parse(results).forEach(logIncrementalResult);
+        if (errors.length) {
+            clearInterval(intervalId);
+            reject(new JetStreamTestError(errors));
+        }
+        logIncrementalResult(previousResults, JSON.parse(resultsJSON))
         if (done) {
             clearInterval(intervalId);
             resolve()
         }
     }, UPDATE_INTERVAL)
 }
 
-function logIncrementalResult(benchmarkResult) {
-    console.log(benchmarkResult.name, benchmarkResult.results)
+function logIncrementalResult(previousResults, benchmarkResults) {
+    console.log(benchmarkResults)
+    for (const [testName, testResults] of Object.entries(benchmarkResults)) {
+        if (previousResults.has(testName))
+            continue;
+        console.log(testName, testResults);
+        previousResults.add(testName);
+    }
 }
 
 setImmediate(testEnd2End);