WIP

Move existing benchmarks to WasmLegacyBenchamrk and add a new WasmBenchmark that supports
running iterations. It's currently an async benchmark subclass since the compile is async.

As a proof of concept this patch converts tsf-wasm to use the new scoring. As part of this
process tsf needed to be recompiled so I added a step to the build.sh script which wraps
emcc's output .js file with some glue code. This should make it easier to rebuild in the
future too since we're not directly editing the module loading file. I went with this
route rather than hooking into emcc's API since I wanted to use the js glue as close to
"out of the box" as possible.

Lastly, get emcc to ouput the .symbols object so we know which wasm function is which index.
We could emit a name section into he binary but my assumption is that most production code
does not ship with that so it could change the benchmark in an unrealistic way.

TODO:
* Profile perf differences
* Generalize the wrapping process
* Remove postIterationHook code for a different PR.

Author: kmiller68

JetStreamDriver.js

@@ -1,7 +1,7 @@
 "use strict";
 
 /*
- * Copyright (C) 2018-2022 Apple Inc. All rights reserved.
+ * Copyright (C) 2018-2024 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -330,7 +330,7 @@ class Driver {
             } else
                 globalObject = runString("");
 
-            globalObject.console = { log: globalObject.print, warn: (e) => { print("Warn: " + e); /*$vm.abort();*/ } }
+            globalObject.console = { log: globalObject.print, warn: (e) => { print("Warn: " + e); /*$vm.abort();*/ }, error: (e) => { print("Error: " + e); /*$vm.abort();*/ } }
             globalObject.self = globalObject;
             globalObject.top = {
                 currentResolve,
@@ -543,10 +543,11 @@ class Benchmark {
                 if (__benchmark.prepareForNextIteration)
                     __benchmark.prepareForNextIteration();
 
-                ${this.preiterationCode}
+                ${this.preIterationCode}
                 let start = performance.now();
                 __benchmark.runIteration();
                 let end = performance.now();
+                ${this.postIterationCode}
 
                 results.push(Math.max(1, end - start));
             }
@@ -565,11 +566,24 @@ class Benchmark {
 
     get prerunCode() { return null; }
 
-    get preiterationCode() {
+    get preIterationCode() {
+        let code = "";
         if (this.plan.deterministicRandom)
-            return `Math.random.__resetSeed();`;
+            code += `Math.random.__resetSeed();`;
 
-        return "";
+        if (globalThis.customPreIterationCode)
+            code += customPreIterationCode;
+
+        return code;
+    }
+
+    get postIterationCode() {
+        let code = "";
+
+        if (globalThis.customPostIterationCode)
+            code += customPostIterationCode;
+
+        return code;
     }
 
     async run() {
@@ -972,10 +986,11 @@ class AsyncBenchmark extends DefaultBenchmark {
             let __benchmark = new Benchmark();
             let results = [];
             for (let i = 0; i < ${this.iterations}; i++) {
-                ${this.preiterationCode}
+                ${this.preIterationCode}
                 let start = performance.now();
                 await __benchmark.runIteration();
                 let end = performance.now();
+                ${this.postIterationCode}
                 results.push(Math.max(1, end - start));
             }
             if (__benchmark.validate)
@@ -986,6 +1001,96 @@ class AsyncBenchmark extends DefaultBenchmark {
     }
 };
 
+class WasmBenchmark extends AsyncBenchmark {
+    get prerunCode() {
+        let str = `
+            let verbose = true;
+
+            let globalObject = this;
+
+            abort = quit = function() {
+                if (verbose)
+                    console.log('Intercepted quit/abort');
+            };
+
+            oldPrint = globalObject.print;
+            globalObject.print = globalObject.printErr = (...args) => {
+                if (verbose)
+                    console.log('Intercepted print: ', ...args);
+            };
+
+            let instanceReady;
+            let instancePromise = new Promise((resolve) => {
+                instanceReady = resolve;
+            });
+
+            let Module = {
+                preRun: [],
+                postRun: [],
+                print: function() { },
+                printErr: function() { },
+                setStatus: function(text) {
+                },
+                totalDependencies: 0,
+                monitorRunDependencies: function(left) {
+                    this.totalDependencies = Math.max(this.totalDependencies, left);
+                    Module.setStatus(left ? 'Preparing... (' + (this.totalDependencies-left) + '/' + this.totalDependencies + ')' : 'All downloads complete.');
+                },
+            };
+            globalObject.Module = Module;
+            `;
+        return str;
+    }
+
+    get runnerCode() {
+        let str = `function loadBlob(key, path, andThen) {`;
+
+        if (isInBrowser) {
+            str += `
+                var xhr = new XMLHttpRequest();
+                xhr.open('GET', path, true);
+                xhr.responseType = 'arraybuffer';
+                xhr.onload = function() {
+                    Module[key] = new Int8Array(xhr.response);
+                    andThen();
+                };
+                xhr.send(null);
+            `;
+        } else {
+            str += `
+            Module[key] = new Int8Array(read(path, "binary"));
+
+            Module.setStatus = null;
+            Module.monitorRunDependencies = null;
+
+            Promise.resolve(42).then(() => {
+                try {
+                    andThen();
+                } catch(e) {
+                    console.log("error running wasm:", e);
+                    console.log(e.stack);
+                    throw e;
+                }
+            })
+            `;
+        }
+
+        str += "}";
+
+        let keys = Object.keys(this.plan.preload);
+        for (let i = 0; i < keys.length; ++i) {
+            str += `loadBlob("${keys[i]}", "${this.plan.preload[keys[i]]}", () => {\n`;
+        }
+        str += super.runnerCode;
+        for (let i = 0; i < keys.length; ++i) {
+            str += `})`;
+        }
+        str += `;`;
+
+        return str;
+    }
+};
+
 class WSLBenchmark extends Benchmark {
     constructor(...args) {
         super(...args);
@@ -1062,7 +1167,7 @@ class WSLBenchmark extends Benchmark {
     }
 };
 
-class WasmBenchmark extends Benchmark {
+class WasmLegacyBenchmark extends Benchmark {
     constructor(...args) {
         super(...args);
 
@@ -1776,16 +1881,16 @@ const testPlans = [
         preload: {
             wasmBinary: "./wasm/HashSet.wasm"
         },
-        benchmarkClass: WasmBenchmark,
+        benchmarkClass: WasmLegacyBenchmark,
         testGroup: WasmGroup
     },
     {
         name: "tsf-wasm",
         files: [
-            "./wasm/tsf.js"
+            "./wasm/TSF/tsf.js"
         ],
         preload: {
-            wasmBinary: "./wasm/tsf.wasm"
+            wasmBinary: "./wasm/TSF/tsf.wasm"
         },
         benchmarkClass: WasmBenchmark,
         testGroup: WasmGroup
@@ -1798,7 +1903,7 @@ const testPlans = [
         preload: {
             wasmBinary: "./wasm/quicksort.wasm"
         },
-        benchmarkClass: WasmBenchmark,
+        benchmarkClass: WasmLegacyBenchmark,
         testGroup: WasmGroup
     },
     {
@@ -1809,7 +1914,7 @@ const testPlans = [
         preload: {
             wasmBinary: "./wasm/gcc-loops.wasm"
         },
-        benchmarkClass: WasmBenchmark,
+        benchmarkClass: WasmLegacyBenchmark,
         testGroup: WasmGroup
     },
     {
@@ -1820,7 +1925,7 @@ const testPlans = [
         preload: {
             wasmBinary: "./wasm/richards.wasm"
         },
-        benchmarkClass: WasmBenchmark,
+        benchmarkClass: WasmLegacyBenchmark,
         testGroup: WasmGroup
     },
     {
@@ -1852,7 +1957,7 @@ const testPlans = [
         preload: {
             tfjsBackendWasmBlob: "./wasm/tfjs-backend-wasm.wasm",
         },
-        benchmarkClass: WasmBenchmark,
+        benchmarkClass: WasmLegacyBenchmark,
         async: true,
         deterministicRandom: true,
         testGroup: WasmGroup
@@ -1873,7 +1978,7 @@ const testPlans = [
         preload: {
             tfjsBackendWasmSimdBlob: "./wasm/tfjs-backend-wasm-simd.wasm",
         },
-        benchmarkClass: WasmBenchmark,
+        benchmarkClass: WasmLegacyBenchmark,
         async: true,
         deterministicRandom: true,
         testGroup: WasmGroup
@@ -1888,7 +1993,7 @@ const testPlans = [
         preload: {
             argon2WasmBlob: "./wasm/argon2.wasm",
         },
-        benchmarkClass: WasmBenchmark,
+        benchmarkClass: WasmLegacyBenchmark,
         testGroup: WasmGroup
     },
     {
@@ -1901,7 +2006,7 @@ const testPlans = [
         preload: {
             argon2WasmSimdBlob: "./wasm/argon2-simd.wasm",
         },
-        benchmarkClass: WasmBenchmark,
+        benchmarkClass: WasmLegacyBenchmark,
         testGroup: WasmGroup
     },
     // WorkerTests
@@ -1975,7 +2080,7 @@ const testPlans = [
             romBinary: "./8bitbench/assets/program.bin"
         },
         async: true,
-        benchmarkClass: WasmBenchmark,
+        benchmarkClass: WasmLegacyBenchmark,
         testGroup: WasmGroup
     }
 ];

wasm/TSF/build.sh

@@ -1,6 +1,9 @@
 #!/bin/sh
+
+set -euo pipefail
+
 emcc \
-    -o tsf.html -o tsf.js -O2 -s WASM=1 -s TOTAL_MEMORY=52428800 -g1 \
+    -o tsf.js -O2 -s WASM=1 -s TOTAL_MEMORY=52428800 -g1 --emit-symbol-map -s EXPORTED_FUNCTIONS=_main,_runIteration \
     -I. -DTSF_BUILD_SYSTEM=1 \
     tsf_asprintf.c\
     tsf_buffer.c\
@@ -53,3 +56,23 @@ emcc \
     tsf_ir_different.c\
     tsf_ir_speed.c
 
+TEMPFILE=`mktemp /tmp/tsf.XXXXXX`
+
+(echo 'function setup(Module) {'; cat tsf.js; echo '}
+
+class Benchmark {
+    async runIteration() {
+        if (!Module["_main"]) {
+            let runtimeInitializedCallback;
+            let runtimeInitialized = new Promise((success) => runtimeInitializedCallback = success);
+            Module.onRuntimeInitialized = function() {
+                runtimeInitializedCallback();
+            }
+            setup(Module);
+            await runtimeInitialized;
+        }
+
+        Module["_runIteration"]();
+    }
+}') > $TEMPFILE
+mv $TEMPFILE tsf.js