Improve Benchmark constructor and instance variables (#255)

- Use plan as options bag for constructor arguments
- Prefix "protected" instance variables with "_"
- Rename Benchmark.preload to Benchmark._preloadEntries
- Move iteration and worstCase count processing to the Benchmark class
- Add files and preloadEntries getters so we can override it in GroupedBenchmark
- Rename plan.arguments to plan.args due to "argument" being a reserved word
- Rename WasmLegacyBenchmark to AsyncWasmLegacyBenchmark

Author: camillobruni

JetStreamDriver.js

@@ -56,25 +56,6 @@ function displayCategoryScores() {
     document.body.classList.add("details");
 }
 
-function getIterationCount(plan) {
-    if (plan.name in JetStreamParams.testIterationCountMap)
-        return JetStreamParams.testIterationCountMap[plan.name];
-    if (JetStreamParams.testIterationCount)
-        return JetStreamParams.testIterationCount;
-    if (plan.iterations)
-        return plan.iterations;
-    return defaultIterationCount;
-}
-
-function getWorstCaseCount(plan) {
-    if (plan.name in JetStreamParams.testWorstCaseCountMap)
-        return JetStreamParams.testWorstCaseCountMap[plan.name];
-    if (JetStreamParams.testWorstCaseCount)
-        return JetStreamParams.testWorstCaseCount;
-    if (plan.worstCaseCount !== undefined)
-        return plan.worstCaseCount;
-    return defaultWorstCaseCount;
-}
 
 if (isInBrowser) {
     document.onkeydown = (keyboardEvent) => {
@@ -336,7 +317,7 @@ class Driver {
         this.errors = [];
         // Make benchmark list unique and sort it.
         this.benchmarks = Array.from(new Set(benchmarks));
-        this.benchmarks.sort((a, b) => a.plan.name.toLowerCase() < b.plan.name.toLowerCase() ? 1 : -1);
+        this.benchmarks.sort((a, b) => a.name.toLowerCase() < b.name.toLowerCase() ? 1 : -1);
         console.assert(this.benchmarks.length, "No benchmarks selected");
     }
 
@@ -504,7 +485,7 @@ class Driver {
         if (isInBrowser)
             window.addEventListener("error", (e) => this.pushError("driver startup", e.error));
         await this.prefetchResources();
-        this.benchmarks.sort((a, b) => a.plan.name.toLowerCase() < b.plan.name.toLowerCase() ? 1 : -1);
+        this.benchmarks.sort((a, b) => a.name.toLowerCase() < b.name.toLowerCase() ? 1 : -1);
         if (isInBrowser)
             this.prepareBrowserUI();
         this.isReady = true;
@@ -859,21 +840,45 @@ class BrowserScripts extends Scripts {
 
 
 class Benchmark {
-    constructor(plan)
-    {
-        this.plan = plan;
-        this.tags = this.processTags(plan.tags)
-        this.iterations = getIterationCount(plan);
-        this.isAsync = !!plan.isAsync;
-        this.allowUtf16 = !!plan.allowUtf16;
-        this.scripts = null;
-        this.preloads = [];
-        this.shellPrefetchedResources = null;
-        this.results = [];
+    constructor({
+            name, 
+            files,
+            preload = {},
+            tags, 
+            iterations,
+            deterministicRandom = false,
+            exposeBrowserTest = false,
+            allowUtf16 = false,
+            args = {} }) {
         this._state = BenchmarkState.READY;
+        this.results = [];
+
+        this.name = name
+        this.tags = this._processTags(tags)
+        this._arguments = args;
+        
+        this.iterations = this._processIterationCount(iterations);
+        this._deterministicRandom = deterministicRandom;
+        this._exposeBrowserTest = exposeBrowserTest;
+        this.allowUtf16 = !!allowUtf16;
+
+        // Resource handling:
+        this._scripts = null;
+        this._files = files;
+        this._preloadEntries = Object.entries(preload);
+        this._preloadBlobData = [];
+        this._shellPrefetchedResources = null;
     }
 
-    processTags(rawTags) {
+    // Use getter so it can be overridden in subclasses (GroupedBenchmark).
+    get files() {
+        return this._files;
+    }
+    get preloadEntries() { 
+        return this._preloadEntries;
+    }
+
+    _processTags(rawTags) {
         const tags = new Set(rawTags.map(each => each.toLowerCase()));
         if (tags.size != rawTags.length)
             throw new Error(`${this.name} got duplicate tags: ${rawTags.join()}`);
@@ -883,9 +888,25 @@ class Benchmark {
         return tags;
     }
 
-    get name() { return this.plan.name; }
-    get files() { return this.plan.files; }
-    get preloadFiles() { return Object.values(this.plan.preload ?? {}); }
+    _processIterationCount(iterations) {
+        if (this.name in JetStreamParams.testIterationCountMap)
+            return JetStreamParams.testIterationCountMap[this.name];
+        if (JetStreamParams.testIterationCount)
+            return JetStreamParams.testIterationCount;
+        if (iterations)
+            return iterations;
+        return defaultIterationCount;
+    }
+
+    _processWorstCaseCount(worstCaseCount) {
+        if (this.name in JetStreamParams.testWorstCaseCountMap)
+            return JetStreamParams.testWorstCaseCountMap[plan.name];
+        if (JetStreamParams.testWorstCaseCount)
+            return JetStreamParams.testWorstCaseCount;
+        if (worstCaseCount !== undefined)
+            return worstCaseCount;
+        return defaultWorstCaseCount;
+    }
 
     get isDone() {
         return this._state == BenchmarkState.DONE || this._state == BenchmarkState.ERROR;
@@ -898,7 +919,7 @@ class Benchmark {
 
     get benchmarkArguments() {
         return {
-            ...this.plan.arguments,
+            ...this._arguments,
             iterationCount: this.iterations,
         };
     }
@@ -975,7 +996,7 @@ class Benchmark {
 
     get preIterationCode() {
         let code = this.prepareForNextIterationCode ;
-        if (this.plan.deterministicRandom)
+        if (this._deterministicRandom)
             code += `Math.random.__resetSeed();`;
 
         if (JetStreamParams.customPreIterationCode)
@@ -1036,24 +1057,26 @@ class Benchmark {
             globalThis?.gc();
         }
 
-        const scripts = isInBrowser ? new BrowserScripts(this.preloads) : new ShellScripts(this.preloads);
+        const scripts = isInBrowser ? 
+                new BrowserScripts(this._preloadBlobData) :
+                new ShellScripts(this._preloadBlobData);
 
-        if (!!this.plan.deterministicRandom)
+        if (this._deterministicRandom)
             scripts.addDeterministicRandom()
-        if (!!this.plan.exposeBrowserTest)
+        if (this._exposeBrowserTest)
             scripts.addBrowserTest();
 
-        if (this.shellPrefetchedResources) {
-            scripts.addPrefetchedResources(this.shellPrefetchedResources);
+        if (this._shellPrefetchedResources) {
+            scripts.addPrefetchedResources(this._shellPrefetchedResources);
         }
 
         const prerunCode = this.prerunCode;
         if (prerunCode)
             scripts.add(prerunCode);
 
         if (!isInBrowser) {
-            console.assert(this.scripts && this.scripts.length === this.files.length);
-            for (const text of this.scripts)
+            console.assert(this._scripts && this._scripts.length === this.files.length);
+            for (const text of this._scripts)
                 scripts.add(text);
         } else {
             for (const file of this.files) {
@@ -1110,30 +1133,27 @@ class Benchmark {
     async prefetchResourcesForBrowser() {
         console.assert(isInBrowser);
         const promises = this.files.map((file) => browserFileLoader.prefetchResourceFile(file));
-        for (const [name, resource] of Object.entries(this.plan.preload ?? {})) {
+        for (const [name, resource] of this.preloadEntries) {
             promises.push(this.prefetchResourcePreload(name, resource));
         }
         await Promise.all(promises);
     }
 
     async prefetchResourcePreload(name, resource) {
         const preloadData = await browserFileLoader.prefetchResourcePreload(name, resource);
-        this.preloads.push(preloadData);
+        this._preloadBlobData.push(preloadData);
     }
 
     prefetchResourcesForShell() {
         // FIXME: move to ShellFileLoader.
         console.assert(!isInBrowser);
 
-        console.assert(this.scripts === null, "This initialization should be called only once.");
-        this.scripts = this.files.map(file => shellFileLoader.load(file));
+        console.assert(this._scripts === null, "This initialization should be called only once.");
+        this._scripts = this.files.map(file => shellFileLoader.load(file));
 
-        console.assert(this.preloads.length === 0, "This initialization should be called only once.");
-        this.shellPrefetchedResources = Object.create(null);
-        if (!this.plan.preload) {
-            return;
-        }
-        for (let [name, resource] of Object.entries(this.plan.preload)) {
+        console.assert(this._preloadBlobData.length === 0, "This initialization should be called only once.");
+        this._shellPrefetchedResources = Object.create(null);
+        for (let [name, resource] of this.preloadEntries) {
             const compressed = isCompressed(resource);
             if (compressed && !JetStreamParams.prefetchResources) {
                 resource = uncompressedName(resource);
@@ -1144,10 +1164,10 @@ class Benchmark {
                 if (compressed) {
                     bytes = zlib.decompress(bytes);
                 }
-                this.shellPrefetchedResources[resource] = bytes;
+                this._shellPrefetchedResources[resource] = bytes;
             }
 
-            this.preloads.push({ name, resource, blobURLOrPath: resource });
+            this._preloadBlobData.push({ name, resource, blobURLOrPath: resource });
         }
     }
 
@@ -1267,7 +1287,8 @@ class Benchmark {
     tearDown() {
         if (isInBrowser) {
             browserFileLoader.free(this.files);
-            browserFileLoader.free(this.preloadFiles);
+            const preloadFiles = this.preloadEntries.map(([_, file]) => file);
+            browserFileLoader.free(preloadFiles);
         }
     }
 };
@@ -1315,10 +1336,11 @@ class GroupedBenchmark extends Benchmark {
     }
 
     get files() {
-        let files = [];
-        for (const benchmark of this.benchmarks)
-            files = files.concat(benchmark.files);
-        return files;
+        return this.benchmarks.flatMap(benchmark => benchmark.files)
+    }
+
+    get preloadEntries() {
+        return this.benchmarks.flatMap(benchmark => benchmark.preloadEntries)
     }
 
     async run() {
@@ -1398,10 +1420,10 @@ class GroupedBenchmark extends Benchmark {
 };
 
 class DefaultBenchmark extends Benchmark {
-    constructor(...args) {
-        super(...args);
+    constructor({worstCaseCount, ...args}) {
+        super(args);
 
-        this.worstCaseCount = getWorstCaseCount(this.plan);
+        this.worstCaseCount = this._processWorstCaseCount(worstCaseCount);
         this.firstIterationTime = null;
         this.firstIterationScore = null;
         this.worstTime = null;
@@ -1582,8 +1604,8 @@ class WasmEMCCBenchmark extends AsyncBenchmark {
 };
 
 class WSLBenchmark extends Benchmark {
-    constructor(...args) {
-        super(...args);
+    constructor(plan) {
+        super(plan);
 
         this.stdlibTime = null;
         this.stdlibScore = null;
@@ -1645,10 +1667,9 @@ class WSLBenchmark extends Benchmark {
     }
 };
 
-class WasmLegacyBenchmark extends Benchmark {
-    constructor(...args) {
-        super(...args);
-
+class AsyncWasmLegacyBenchmark extends Benchmark {
+    constructor(plan) {
+        super(plan);
         this.startupTime = null;
         this.startupScore = null;
         this.runTime = null;
@@ -1751,19 +1772,15 @@ class WasmLegacyBenchmark extends Benchmark {
 
         str += "};\n";
         let preloadCount = 0;
-        for (const [name, resource] of Object.entries(this.plan.preload)) {
+        for (const [name, resource] of this.preloadEntries) {
             preloadCount++;
             str += `JetStream.loadBlob(${JSON.stringify(name)}, "${resource}", () => {\n`;
         }
-        if (this.plan.async) {
-            str += `doRun().catch((e) => {
-                console.log("error running wasm:", e);
-                console.log(e.stack)
-                throw e;
-            });`;
-        } else {
-            str += `doRun();`
-        }
+        str += `doRun().catch((e) => {
+            console.log("error running wasm:", e);
+            console.log(e.stack)
+            throw e;
+        });`;
         for (let i = 0; i < preloadCount; ++i) {
             str += `})`;
         }
@@ -2556,7 +2573,7 @@ let BENCHMARKS = [
         allowUtf16: true,
         tags: ["Wasm", "transformersjs"],
     }),
-    new WasmLegacyBenchmark({
+    new AsyncWasmLegacyBenchmark({
         name: "tfjs-wasm",
         files: [
             "./wasm/tfjs-model-helpers.js",
@@ -2572,13 +2589,12 @@ let BENCHMARKS = [
         preload: {
             tfjsBackendWasmBlob: "./wasm/tfjs-backend-wasm.wasm",
         },
-        async: true,
         deterministicRandom: true,
         exposeBrowserTest: true,
         allowUtf16: true,
         tags: ["Wasm"],
     }),
-    new WasmLegacyBenchmark({
+    new AsyncWasmLegacyBenchmark({
         name: "tfjs-wasm-simd",
         files: [
             "./wasm/tfjs-model-helpers.js",
@@ -2594,7 +2610,6 @@ let BENCHMARKS = [
         preload: {
             tfjsBackendWasmSimdBlob: "./wasm/tfjs-backend-wasm-simd.wasm",
         },
-        async: true,
         deterministicRandom: true,
         exposeBrowserTest: true,
         allowUtf16: true,
@@ -2624,7 +2639,7 @@ let BENCHMARKS = [
         preload: {
             BUNDLE: "./babylonjs/dist/bundle.es5.min.js",
         },
-        arguments: {
+        args: {
             expectedCacheCommentCount: 23988,
         },
         tags: ["startup",  "js", "class", "es5", "babylonjs"],
@@ -2639,7 +2654,7 @@ let BENCHMARKS = [
         preload: {
             BUNDLE: "./babylonjs/dist/bundle.es6.min.js",
         },
-        arguments: {
+        args: {
             expectedCacheCommentCount: 21222,
         },
         tags: ["Default",  "js", "startup", "class", "es6", "babylonjs"],

tests/unit-tests.js

@@ -186,10 +186,8 @@ function assertThrows(message, func) {
         checkFile(benchmark.name, file, "file");
     }
 
-    if (benchmark.plan.preload) {
-        for (const [name, file] of Object.entries(benchmark.plan.preload)) {
-            checkFile(benchmark.name, file, `preload.${name}`);
-        }
+    for (const [name, file] of benchmark.preloadEntries) {
+        checkFile(benchmark.name, file, `preload.${name}`);
     }
   }
 })();