Skip to content

Improve Benchmark constructor and instance variables #255

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
camillobruni merged 6 commits into from
Jan 14, 2026
Merged

Improve Benchmark constructor and instance variables #255

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ff8753e
fixes
camillobruni Jan 14, 2026
562464d
cleanups
camillobruni Jan 14, 2026
a783d24
more-cleanup
camillobruni Jan 14, 2026
467488d
fix-recursiooooooon
camillobruni Jan 14, 2026
7cc4a16
fix-test
camillobruni Jan 14, 2026
b7a471e
AsyncWasmLegacyBenchmark
camillobruni Jan 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
189 changes: 102 additions & 87 deletions JetStreamDriver.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,25 +56,6 @@ function displayCategoryScores() {
document.body.classList.add("details");
}

function getIterationCount(plan) {
if (plan.name in JetStreamParams.testIterationCountMap)
return JetStreamParams.testIterationCountMap[plan.name];
if (JetStreamParams.testIterationCount)
return JetStreamParams.testIterationCount;
if (plan.iterations)
return plan.iterations;
return defaultIterationCount;
}

function getWorstCaseCount(plan) {
if (plan.name in JetStreamParams.testWorstCaseCountMap)
return JetStreamParams.testWorstCaseCountMap[plan.name];
if (JetStreamParams.testWorstCaseCount)
return JetStreamParams.testWorstCaseCount;
if (plan.worstCaseCount !== undefined)
return plan.worstCaseCount;
return defaultWorstCaseCount;
}

if (isInBrowser) {
document.onkeydown = (keyboardEvent) => {
Expand Down Expand Up @@ -336,7 +317,7 @@ class Driver {
this.errors = [];
// Make benchmark list unique and sort it.
this.benchmarks = Array.from(new Set(benchmarks));
this.benchmarks.sort((a, b) => a.plan.name.toLowerCase() < b.plan.name.toLowerCase() ? 1 : -1);
this.benchmarks.sort((a, b) => a.name.toLowerCase() < b.name.toLowerCase() ? 1 : -1);
console.assert(this.benchmarks.length, "No benchmarks selected");
}

Expand Down Expand Up @@ -504,7 +485,7 @@ class Driver {
if (isInBrowser)
window.addEventListener("error", (e) => this.pushError("driver startup", e.error));
await this.prefetchResources();
this.benchmarks.sort((a, b) => a.plan.name.toLowerCase() < b.plan.name.toLowerCase() ? 1 : -1);
this.benchmarks.sort((a, b) => a.name.toLowerCase() < b.name.toLowerCase() ? 1 : -1);
if (isInBrowser)
this.prepareBrowserUI();
this.isReady = true;
Expand Down Expand Up @@ -855,21 +836,45 @@ class BrowserScripts extends Scripts {


class Benchmark {
constructor(plan)
{
this.plan = plan;
this.tags = this.processTags(plan.tags)
this.iterations = getIterationCount(plan);
this.isAsync = !!plan.isAsync;
this.allowUtf16 = !!plan.allowUtf16;
this.scripts = null;
this.preloads = [];
this.shellPrefetchedResources = null;
this.results = [];
constructor({
name,
files,
preload = {},
tags,
iterations,
deterministicRandom = false,
exposeBrowserTest = false,
allowUtf16 = false,
args = {} }) {
this._state = BenchmarkState.READY;
this.results = [];

this.name = name
this.tags = this._processTags(tags)
this._arguments = args;

this.iterations = this._processIterationCount(iterations);
this._deterministicRandom = deterministicRandom;
this._exposeBrowserTest = exposeBrowserTest;
this.allowUtf16 = !!allowUtf16;

// Resource handling:
this._scripts = null;
this._files = files;
this._preloadEntries = Object.entries(preload);
this._preloadBlobData = [];
this._shellPrefetchedResources = null;
}

processTags(rawTags) {
// Use getter so it can be overridden in subclasses (GroupedBenchmark).
get files() {
return this._files;
}
get preloadEntries() {
return this._preloadEntries;
}

_processTags(rawTags) {
const tags = new Set(rawTags.map(each => each.toLowerCase()));
if (tags.size != rawTags.length)
throw new Error(`${this.name} got duplicate tags: ${rawTags.join()}`);
Expand All @@ -879,9 +884,25 @@ class Benchmark {
return tags;
}

get name() { return this.plan.name; }
get files() { return this.plan.files; }
get preloadFiles() { return Object.values(this.plan.preload ?? {}); }
_processIterationCount(iterations) {
if (this.name in JetStreamParams.testIterationCountMap)
return JetStreamParams.testIterationCountMap[this.name];
if (JetStreamParams.testIterationCount)
return JetStreamParams.testIterationCount;
if (iterations)
return iterations;
return defaultIterationCount;
}

_processWorstCaseCount(worstCaseCount) {
if (this.name in JetStreamParams.testWorstCaseCountMap)
return JetStreamParams.testWorstCaseCountMap[plan.name];
if (JetStreamParams.testWorstCaseCount)
return JetStreamParams.testWorstCaseCount;
if (worstCaseCount !== undefined)
return worstCaseCount;
return defaultWorstCaseCount;
}

get isDone() {
return this._state == BenchmarkState.DONE || this._state == BenchmarkState.ERROR;
Expand All @@ -894,7 +915,7 @@ class Benchmark {

get benchmarkArguments() {
return {
...this.plan.arguments,
...this._arguments,
iterationCount: this.iterations,
};
}
Expand Down Expand Up @@ -971,7 +992,7 @@ class Benchmark {

get preIterationCode() {
let code = this.prepareForNextIterationCode ;
if (this.plan.deterministicRandom)
if (this._deterministicRandom)
code += `Math.random.__resetSeed();`;

if (JetStreamParams.customPreIterationCode)
Expand Down Expand Up @@ -1032,24 +1053,26 @@ class Benchmark {
globalThis?.gc();
}

const scripts = isInBrowser ? new BrowserScripts(this.preloads) : new ShellScripts(this.preloads);
const scripts = isInBrowser ?
new BrowserScripts(this._preloadBlobData) :
new ShellScripts(this._preloadBlobData);

if (!!this.plan.deterministicRandom)
if (this._deterministicRandom)
scripts.addDeterministicRandom()
if (!!this.plan.exposeBrowserTest)
if (this._exposeBrowserTest)
scripts.addBrowserTest();

if (this.shellPrefetchedResources) {
scripts.addPrefetchedResources(this.shellPrefetchedResources);
if (this._shellPrefetchedResources) {
scripts.addPrefetchedResources(this._shellPrefetchedResources);
}

const prerunCode = this.prerunCode;
if (prerunCode)
scripts.add(prerunCode);

if (!isInBrowser) {
console.assert(this.scripts && this.scripts.length === this.files.length);
for (const text of this.scripts)
console.assert(this._scripts && this._scripts.length === this.files.length);
for (const text of this._scripts)
scripts.add(text);
} else {
for (const file of this.files) {
Expand Down Expand Up @@ -1106,30 +1129,27 @@ class Benchmark {
async prefetchResourcesForBrowser() {
console.assert(isInBrowser);
const promises = this.files.map((file) => browserFileLoader.prefetchResourceFile(file));
for (const [name, resource] of Object.entries(this.plan.preload ?? {})) {
for (const [name, resource] of this.preloadEntries) {
promises.push(this.prefetchResourcePreload(name, resource));
}
await Promise.all(promises);
}

async prefetchResourcePreload(name, resource) {
const preloadData = await browserFileLoader.prefetchResourcePreload(name, resource);
this.preloads.push(preloadData);
this._preloadBlobData.push(preloadData);
}

prefetchResourcesForShell() {
// FIXME: move to ShellFileLoader.
console.assert(!isInBrowser);

console.assert(this.scripts === null, "This initialization should be called only once.");
this.scripts = this.files.map(file => shellFileLoader.load(file));
console.assert(this._scripts === null, "This initialization should be called only once.");
this._scripts = this.files.map(file => shellFileLoader.load(file));

console.assert(this.preloads.length === 0, "This initialization should be called only once.");
this.shellPrefetchedResources = Object.create(null);
if (!this.plan.preload) {
return;
}
for (let [name, resource] of Object.entries(this.plan.preload)) {
console.assert(this._preloadBlobData.length === 0, "This initialization should be called only once.");
this._shellPrefetchedResources = Object.create(null);
for (let [name, resource] of this.preloadEntries) {
const compressed = isCompressed(resource);
if (compressed && !JetStreamParams.prefetchResources) {
resource = uncompressedName(resource);
Expand All @@ -1140,10 +1160,10 @@ class Benchmark {
if (compressed) {
bytes = zlib.decompress(bytes);
}
this.shellPrefetchedResources[resource] = bytes;
this._shellPrefetchedResources[resource] = bytes;
}

this.preloads.push({ name, resource, blobURLOrPath: resource });
this._preloadBlobData.push({ name, resource, blobURLOrPath: resource });
}
}

Expand Down Expand Up @@ -1262,7 +1282,8 @@ class Benchmark {
tearDown() {
if (isInBrowser) {
browserFileLoader.free(this.files);
browserFileLoader.free(this.preloadFiles);
const preloadFiles = this.preloadEntries.map(([_, file]) => file);
browserFileLoader.free(preloadFiles);
}
}
};
Expand Down Expand Up @@ -1310,10 +1331,11 @@ class GroupedBenchmark extends Benchmark {
}

get files() {
let files = [];
for (const benchmark of this.benchmarks)
files = files.concat(benchmark.files);
return files;
return this.benchmarks.flatMap(benchmark => benchmark.files)
}

get preloadEntries() {
return this.benchmarks.flatMap(benchmark => benchmark.preloadEntries)
}

async run() {
Expand Down Expand Up @@ -1393,10 +1415,10 @@ class GroupedBenchmark extends Benchmark {
};

class DefaultBenchmark extends Benchmark {
constructor(...args) {
super(...args);
constructor({worstCaseCount, ...args}) {
super(args);

this.worstCaseCount = getWorstCaseCount(this.plan);
this.worstCaseCount = this._processWorstCaseCount(worstCaseCount);
this.firstIterationTime = null;
this.firstIterationScore = null;
this.worstTime = null;
Expand Down Expand Up @@ -1577,8 +1599,8 @@ class WasmEMCCBenchmark extends AsyncBenchmark {
};

class WSLBenchmark extends Benchmark {
constructor(...args) {
super(...args);
constructor(plan) {
super(plan);

this.stdlibTime = null;
this.stdlibScore = null;
Expand Down Expand Up @@ -1640,10 +1662,9 @@ class WSLBenchmark extends Benchmark {
}
};

class WasmLegacyBenchmark extends Benchmark {
constructor(...args) {
super(...args);

class AsyncWasmLegacyBenchmark extends Benchmark {
constructor(plan) {
super(plan);
this.startupTime = null;
this.startupScore = null;
this.runTime = null;
Expand Down Expand Up @@ -1746,19 +1767,15 @@ class WasmLegacyBenchmark extends Benchmark {

str += "};\n";
let preloadCount = 0;
for (const [name, resource] of Object.entries(this.plan.preload)) {
for (const [name, resource] of this.preloadEntries) {
preloadCount++;
str += `JetStream.loadBlob(${JSON.stringify(name)}, "${resource}", () => {\n`;
}
if (this.plan.async) {
str += `doRun().catch((e) => {
console.log("error running wasm:", e);
console.log(e.stack)
throw e;
});`;
} else {
str += `doRun();`
}
str += `doRun().catch((e) => {
console.log("error running wasm:", e);
console.log(e.stack)
throw e;
});`;
for (let i = 0; i < preloadCount; ++i) {
str += `})`;
}
Expand Down Expand Up @@ -2551,7 +2568,7 @@ let BENCHMARKS = [
allowUtf16: true,
tags: ["Wasm", "transformersjs"],
}),
new WasmLegacyBenchmark({
new AsyncWasmLegacyBenchmark({
name: "tfjs-wasm",
files: [
"./wasm/tfjs-model-helpers.js",
Expand All @@ -2567,13 +2584,12 @@ let BENCHMARKS = [
preload: {
tfjsBackendWasmBlob: "./wasm/tfjs-backend-wasm.wasm",
},
async: true,
deterministicRandom: true,
exposeBrowserTest: true,
allowUtf16: true,
tags: ["Wasm"],
}),
new WasmLegacyBenchmark({
new AsyncWasmLegacyBenchmark({
name: "tfjs-wasm-simd",
files: [
"./wasm/tfjs-model-helpers.js",
Expand All @@ -2589,7 +2605,6 @@ let BENCHMARKS = [
preload: {
tfjsBackendWasmSimdBlob: "./wasm/tfjs-backend-wasm-simd.wasm",
},
async: true,
deterministicRandom: true,
exposeBrowserTest: true,
allowUtf16: true,
Expand Down Expand Up @@ -2619,7 +2634,7 @@ let BENCHMARKS = [
preload: {
BUNDLE: "./babylonjs/dist/bundle.es5.min.js",
},
arguments: {
args: {
expectedCacheCommentCount: 23988,
},
tags: ["startup", "js", "class", "es5", "babylonjs"],
Expand All @@ -2634,7 +2649,7 @@ let BENCHMARKS = [
preload: {
BUNDLE: "./babylonjs/dist/bundle.es6.min.js",
},
arguments: {
args: {
expectedCacheCommentCount: 21222,
},
tags: ["Default", "js", "startup", "class", "es6", "babylonjs"],
Expand Down
6 changes: 2 additions & 4 deletions tests/unit-tests.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,10 +186,8 @@ function assertThrows(message, func) {
checkFile(benchmark.name, file, "file");
}

if (benchmark.plan.preload) {
for (const [name, file] of Object.entries(benchmark.plan.preload)) {
checkFile(benchmark.name, file, `preload.${name}`);
}
for (const [name, file] of benchmark.preloadEntries) {
checkFile(benchmark.name, file, `preload.${name}`);
}
}
})();
Expand Down
Loading