StartupBenchmark should create sources lazily

babylonjs/benchmark/startup.js

@@ -26,7 +26,6 @@
 // console.log = () => {};
 
 class Benchmark extends StartupBenchmark {
-  iteration = 0;
 
   constructor({iterationCount, expectedCacheCommentCount}) {
     super({
@@ -36,10 +35,11 @@ class Benchmark extends StartupBenchmark {
     });
   }
 
-  runIteration() {
-    const sourceCode = this.iterationSourceCodes[this.iteration];
+  runIteration(iteration) {
+    // super.prepareForNextIteration() sets this up for us.
+    const sourceCode = this.currentIterationSourceCode;
     if (!sourceCode)
-      throw new Error(`Could not find source for iteration ${this.iteration}`);
+      throw new Error(`Could not find source for iteration ${iteration}`);
     // Module in sourceCode it assigned to the ClassStartupTest variable.
     let BabylonJSBenchmark;
 
@@ -57,10 +57,9 @@ class Benchmark extends StartupBenchmark {
     // const loadTime = runStart - initStart;
     // const runTime = end - runStart;
     // For local debugging:
-    // print(`Iteration ${this.iteration}:`);
+    // print(`Iteration ${iteration}:`);
     // print(`  Load time: ${loadTime.toFixed(2)}ms`);
     // print(`  Render time: ${runTime.toFixed(2)}ms`);
-    this.iteration++;
   }
 
   validateIteration(lastResult) {

jsdom-d3-startup/benchmark.js

@@ -33,7 +33,6 @@ globalThis.clearTimeout = () => { };
 class Benchmark extends StartupBenchmark {
     lastResult;
     totalHash = 0xdeadbeef;
-    currentIteration = 0;
 
     constructor({iterationCount}) {
         super({
@@ -52,18 +51,18 @@ class Benchmark extends StartupBenchmark {
         this.usData = JSON.parse(this.usDataJsonString);
     }
 
-    runIteration() {
-        let iterationSourceCode = this.iterationSourceCodes[this.currentIteration];
+    runIteration(iteration) {
+        // super.prepareForNextIteration() sets this up for us.
+        let iterationSourceCode = this.currentIterationSourceCode;
         if (!iterationSourceCode)
-            throw new Error(`Could not find source for iteration ${this.currentIteration}`);
+            throw new Error(`Could not find source for iteration ${iteration}`);
         // Module in sourceCode it assigned to the ReactRenderTest variable.
         let D3Test;
         eval(iterationSourceCode);
         const html = D3Test.runTest(this.airportsCsvString, this.usData);
         const htmlHash = this.quickHash(html);
         this.lastResult = { html, htmlHash };
         this.totalHash ^= this.lastResult.htmlHash;
-        this.currentIteration++;
     }
 
     validate() {

mobx/benchmark.js

@@ -37,9 +37,10 @@ class Benchmark extends StartupBenchmark {
   }
 
   runIteration(iteration) {
-    // Module is loaded into PrismJSBenchmark
+    // Module is loaded into MobXBenchmark
     let MobXBenchmark;
-    eval(this.iterationSourceCodes[iteration]);
+    // super.prepareForNextIteration() sets this up for us.
+    eval(this.currentIterationSourceCode);
     this.lastResult = MobXBenchmark.runTest();
   }
 

prismjs/benchmark.js

@@ -65,7 +65,8 @@ class Benchmark extends StartupBenchmark {
   runIteration(iteration) {
     // Module is loaded into PrismJSBenchmark
     let PrismJSBenchmark;
-    eval(this.iterationSourceCodes[iteration]);
+    // super.prepareForNextIteration() sets this up for us.
+    eval(this.currentIterationSourceCode);
     this.lastResult = PrismJSBenchmark.runTest(this.samples);
 
     for (const result of this.lastResult) {

tests/unit-tests.js

@@ -201,6 +201,33 @@ function validateIterationSources(sources) {
   }
 }
 
+function generateAndValidateIterationSources(benchmark) {
+  const sources = [];
+  for (let i = 0; i < benchmark.iterationCount; i++) {
+    benchmark.prepareForNextIteration();
+    const source = benchmark.currentIterationSourceCode;
+    if (!source?.length) {
+      throw new Error(`Got invalid source at iteration ${i}`);
+    }
+    sources.push(source);
+  }
+
+  // Validate uniqueness based on sourceCodeReuseCount
+  let expectedSize = 1;
+  if (benchmark.sourceCodeReuseCount !== 0) {
+    expectedSize = Math.ceil(benchmark.iterationCount / benchmark.sourceCodeReuseCount);
+  }
+  const uniqueSources = new Set(sources);
+  if (uniqueSources.size != expectedSize) {
+    throw new Error(
+      `Expected ${expectedSize} unique sources, but got ${uniqueSources.size}.`
+    );
+  }
+
+  validateIterationSources(sources);
+  return sources;
+}
+
 (async function testStartupBenchmark() {
   try {
     JetStream.preload = { BUNDLE: "test-bundle.js" };
@@ -225,48 +252,34 @@ async function testStartupBenchmarkInnerTests() {
   });
   assertEquals(benchmark.iterationCount, 12);
   assertEquals(benchmark.expectedCacheCommentCount, 1);
-  assertEquals(benchmark.iterationSourceCodes.length, 0);
   assertEquals(benchmark.sourceCode, undefined);
   assertEquals(benchmark.sourceHash, 0);
   await benchmark.init();
   assertEquals(benchmark.sourceHash, 177573);
   assertEquals(benchmark.sourceCode.length, 68);
-  assertEquals(benchmark.iterationSourceCodes.length, 12);
-  assertEquals(new Set(benchmark.iterationSourceCodes).size, 12);
-  validateIterationSources(benchmark.iterationSourceCodes);
-
-  const noReuseBenchmark = new StartupBenchmark({
-    iterationCount: 12,
-    expectedCacheCommentCount: 1,
-    sourceCodeReuseCount: 0,
-  });
-  assertEquals(noReuseBenchmark.iterationSourceCodes.length, 0);
-  await noReuseBenchmark.init();
-  assertEquals(noReuseBenchmark.iterationSourceCodes.length, 12);
-  assertEquals(new Set(noReuseBenchmark.iterationSourceCodes).size, 1);
-  validateIterationSources(noReuseBenchmark.iterationSourceCodes);
+  const sources = generateAndValidateIterationSources(benchmark);
+  assertEquals(sources.length, 12);
+  assertEquals(new Set(sources).size, 12);
 
   const reuseBenchmark = new StartupBenchmark({
     iterationCount: 12,
     expectedCacheCommentCount: 1,
     sourceCodeReuseCount: 3,
   });
-  assertEquals(reuseBenchmark.iterationSourceCodes.length, 0);
   await reuseBenchmark.init();
-  assertEquals(reuseBenchmark.iterationSourceCodes.length, 12);
-  assertEquals(new Set(reuseBenchmark.iterationSourceCodes).size, 4);
-  validateIterationSources(reuseBenchmark.iterationSourceCodes);
+  const reuseSources = generateAndValidateIterationSources(reuseBenchmark);
+  assertEquals(reuseSources.length, 12);
+  assertEquals(new Set(reuseSources).size, 4);
 
   const reuseBenchmark2 = new StartupBenchmark({
     iterationCount: 12,
     expectedCacheCommentCount: 1,
     sourceCodeReuseCount: 5,
   });
-  assertEquals(reuseBenchmark2.iterationSourceCodes.length, 0);
   await reuseBenchmark2.init();
-  assertEquals(reuseBenchmark2.iterationSourceCodes.length, 12);
-  assertEquals(new Set(reuseBenchmark2.iterationSourceCodes).size, 3);
-  validateIterationSources(reuseBenchmark2.iterationSourceCodes);
+  const reuseSources2 = generateAndValidateIterationSources(reuseBenchmark2);
+  assertEquals(reuseSources2.length, 12);
+  assertEquals(new Set(reuseSources2).size, 3);
 }
 
 (function testStartupBenchmarkThrow() {

utils/StartupBenchmark.js

@@ -14,11 +14,11 @@ class StartupBenchmark {
   // Number of no-cache comments in the original #sourceCode.
   #expectedCacheCommentCount = 0;
   // How many times (separate iterations) should we reuse the source code.
-  // Use 0 to skip and only use a single #sourceCode string.
   #sourceCodeReuseCount = 1;
-  // #sourceCode for each iteration, number of unique sources is controlled
-  // by codeReuseCount;
-  #iterationSourceCodes = [];
+  // Current iteration being prepared
+  #currentIteration = 0;
+  // Source code for the current iteration
+  #currentIterationSourceCode = null;
 
   constructor({
     iterationCount,
@@ -36,8 +36,8 @@ class StartupBenchmark {
     );
     this.#expectedCacheCommentCount = expectedCacheCommentCount;
     console.assert(
-      sourceCodeReuseCount >= 0,
-      `Expected sourceCodeReuseCount to be non-negative, but got ${sourceCodeReuseCount}`
+      sourceCodeReuseCount > 0,
+      `Expected sourceCodeReuseCount to be positive, but got ${sourceCodeReuseCount}`
     );
     this.#sourceCodeReuseCount = sourceCodeReuseCount;
   }
@@ -62,8 +62,8 @@ class StartupBenchmark {
     return this.#sourceCodeReuseCount;
   }
 
-  get iterationSourceCodes() {
-    return this.#iterationSourceCodes;
+  get currentIterationSourceCode() {
+    return this.#currentIterationSourceCode;
   }
 
   async init() {
@@ -80,9 +80,6 @@ class StartupBenchmark {
     ).length;
     this.#sourceHash = this.quickHash(this.sourceCode);
     this.validateSourceCacheComments(cacheCommentCount);
-    for (let i = 0; i < this.iterationCount; i++)
-      this.#iterationSourceCodes[i] = this.createIterationSourceCode(i);
-    this.validateIterationSourceCodes();
   }
 
   validateSourceCacheComments(cacheCommentCount) {
@@ -92,27 +89,14 @@ class StartupBenchmark {
     );
   }
 
-  validateIterationSourceCodes() {
-    if (this.#iterationSourceCodes.some((each) => !each?.length))
-      throw new Error(`Got invalid iterationSourceCodes`);
-    let expectedSize = 1;
-    if (this.sourceCodeReuseCount !== 0)
-      expectedSize = Math.ceil(this.iterationCount / this.sourceCodeReuseCount);
-    const uniqueSources = new Set(this.iterationSourceCodes);
-    if (uniqueSources.size != expectedSize)
-      throw new Error(
-        `Expected ${expectedSize} unique sources, but got ${uniqueSources.size}.`
-      );
-  }
-
   createIterationSourceCode(iteration) {
+    if ((iteration % this.sourceCodeReuseCount) !== 0)
+      return this.#currentIterationSourceCode;
+
     // Alter the code per iteration to prevent caching.
     const cacheId =
       Math.floor(iteration / this.sourceCodeReuseCount) *
       this.sourceCodeReuseCount;
-    // Reuse existing sources if this.codeReuseCount > 1:
-    if (cacheId < this.iterationSourceCodes.length)
-      return this.iterationSourceCodes[cacheId];
 
     const sourceCode = this.sourceCode.replaceAll(
       CACHE_BUST_COMMENT_RE,
@@ -123,6 +107,11 @@ class StartupBenchmark {
     return sourceCode;
   }
 
+  prepareForNextIteration() {
+    this.#currentIterationSourceCode = this.createIterationSourceCode(this.#currentIteration);
+    this.#currentIteration++;
+  }
+
   quickHash(str) {
     let hash = 5381;
     let i = str.length;

web-ssr/benchmark.js

@@ -25,7 +25,8 @@ class Benchmark extends StartupBenchmark {
   }
 
   runIteration(iteration) {
-    let sourceCode = this.iterationSourceCodes[iteration];
+    // super.prepareForNextIteration() sets this up for us.
+    let sourceCode = this.currentIterationSourceCode;
     if (!sourceCode)
       throw new Error(`Could not find source for iteration ${iteration}`);
     // Module in sourceCode it assigned to the ReactRenderTest variable.
@@ -41,7 +42,7 @@ class Benchmark extends StartupBenchmark {
 
     const loadTime = runStart - initStart;
     const runTime = end - runStart;
-    // For local debugging: 
+    // For local debugging:
     // print(`Iteration ${iteration}:`);
     // print(`  Load time: ${loadTime.toFixed(2)}ms`);
     // print(`  Render time: ${runTime.toFixed(2)}ms`);