more

Author: camillobruni

.github/workflows/test.yml

@@ -10,31 +10,6 @@ on:
     workflow_dispatch:
 
 jobs:
-    linters:
-        name: Linters
-        runs-on: macos-latest
-        steps:
-            - name: Checkout Branch
-              uses: actions/checkout@v3
-            - name: Setup Node
-              uses: actions/setup-node@v3
-              with:
-                  node-version: 18.13.0
-            - name: Install
-              run: npm install
-            - name: Run linters
-              run: npm run format
-            - name: Check if anything changed
-              run: |
-                  git_status="`LC_ALL=C git status --porcelain --ignore-submodules -unormal 2>&1`"
-                  if [ -n "$git_status" ]; then
-                      printf "Some file(s) changed as the result of formatting, this means that you need to run the formatter on your patch.\n"
-                      printf "Here is what changed:\n"
-                      printf -- "$git_status\n\n"
-                      printf "And here is the diff:\n"
-                      git diff -U8
-                      exit 1
-                  fi
     build:
         name: Build
         runs-on: macos-latest

.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 .directory
+/node_modules
 
 # Ignore auto-generated files by VS & VSCode.
 /.vs/

JetStreamDriver.js

@@ -27,29 +27,44 @@
 
 const measureTotalTimeAsSubtest = false; // Once we move to preloading all resources, it would be good to turn this on.
 
+const defaultIterationCount = 120;
+const defaultWorstCaseCount = 4;
+
 globalThis.performance ??= Date;
 globalThis.RAMification ??= false;
 globalThis.testIterationCount ??= undefined;
 globalThis.testIterationCountMap ??= new Map();
+globalThis.testWorstCaseCount ??= undefined;
 globalThis.testWorstCaseCountMap ??= new Map();
 globalThis.dumpJSONResults ??= false;
 globalThis.customTestList ??= [];
-
 let shouldReport = false;
+
+function getIntParam(urlParams, key) {
+    if (!urlParams.has(key))
+        return undefined
+    const rawValue = urlParams.get(key);
+    const value = parseInt(rawValue);
+    if (value <= 0)
+        throw new Error(`Expected positive value for ${key}, but got ${rawValue}`)
+    return value
+}
+
 if (typeof(URLSearchParams) !== "undefined") {
     const urlParameters = new URLSearchParams(window.location.search);
     shouldReport = urlParameters.has('report') && urlParameters.get('report').toLowerCase() == 'true';
     if (urlParameters.has('test'))
         customTestList = urlParameters.getAll("test");
+    globalThis.testIterationCount = getIntParam(urlParameters, "iterationCount");
+    globalThis.testWorstCaseCount = getIntParam(urlParameters, "worstCaseCount");
 }
 
+
+
 // Used for the promise representing the current benchmark run.
 this.currentResolve = null;
 this.currentReject = null;
 
-const defaultIterationCount = 120;
-const defaultWorstCaseCount = 4;
-
 let showScoreDetails = false;
 let categoryScores = null;
 
@@ -77,6 +92,8 @@ function getIterationCount(plan) {
 function getWorstCaseCount(plan) {
     if (testWorstCaseCountMap.has(plan.name))
         return testWorstCaseCountMap.get(plan.name);
+    if (testWorstCaseCount)
+        return testWorstCaseCount;
     if (plan.worstCaseCount)
         return plan.worstCaseCount;
     return defaultWorstCaseCount;
@@ -214,6 +231,7 @@ const fileLoader = (function() {
 
 class Driver {
     constructor() {
+        this.isReady = false;
         this.benchmarks = [];
         this.blobDataCache = { };
         this.loadCache = { };
@@ -308,6 +326,11 @@ class Driver {
 
         this.reportScoreToRunBenchmarkRunner();
         this.dumpJSONResultsIfNeeded();
+        if (isInBrowser) {
+            globalThis.dispatchEvent(new CustomEvent("JetStreamDone", {
+                detail: this.resultsObject()
+            }));
+        }
     }
 
     runCode(string)
@@ -351,7 +374,6 @@ class Driver {
         var magicFrame = magic.contentDocument.getElementById("magicframe");
         magicFrame.contentDocument.open();
         magicFrame.contentDocument.write("<!DOCTYPE html><head><title>benchmark payload</title></head><body>\n" + string + "</body></html>");
-
         return magicFrame;
     }
 
@@ -414,8 +436,12 @@ class Driver {
         await this.prefetchResourcesForBrowser();
         await this.fetchResources();
         this.prepareToRun();
-        if (isInBrowser && shouldReport) {
-            setTimeout(() => this.start(), 4000);
+        this.isReady = true;
+        if (isInBrowser) {
+            globalThis.dispatchEvent(new Event("JetStreamReady"));
+            if (shouldReport) {
+                setTimeout(() => this.start(), 4000);
+            }
         }
     }
 
@@ -441,7 +467,7 @@ class Driver {
                 // If we've failed to prefetch resources even after a sequential 1 by 1 retry,
                 // then fail out early rather than letting subtests fail with a hang.
                 window.allIsGood = false;
-                throw new Error("Fetch failed"); 
+                throw new Error("Fetch failed");
             }
         }
 
@@ -467,7 +493,7 @@ class Driver {
         }
     }
 
-    resultsJSON()
+    resultsObject()
     {
         let results = {};
         for (let benchmark of this.benchmarks) {
@@ -486,8 +512,13 @@ class Driver {
         }
 
         results = {"JetStream3.0": {"metrics" : {"Score" : ["Geometric"]}, "tests" : results}};
+        return results;
 
-        return JSON.stringify(results);
+    }
+
+    resultsJSON()
+    {
+        return JSON.stringify(this.resultsObject());
     }
 
     dumpJSONResultsIfNeeded()
@@ -525,6 +556,7 @@ class Benchmark {
     {
         this.plan = plan;
         this.iterations = getIterationCount(plan);
+        console.log({name:this.name, iterations:this.iterations})
         this.isAsync = !!plan.isAsync;
 
         this.scripts = null;
@@ -547,11 +579,10 @@ class Benchmark {
                 let start = performance.now();
                 __benchmark.runIteration();
                 let end = performance.now();
-
                 results.push(Math.max(1, end - start));
             }
             if (__benchmark.validate)
-                __benchmark.validate();
+                __benchmark.validate(${this.iterations});
             top.currentResolve(results);`;
     }
 
@@ -804,7 +835,7 @@ class Benchmark {
 
         if (!blobData.blob) {
             window.allIsGood = false;
-            throw new Error("Fetch failed"); 
+            throw new Error("Fetch failed");
         }
 
         return !counter.failedPreloadResources && counter.loadedResources == counter.totalResources;
@@ -858,11 +889,10 @@ class Benchmark {
     scoreIdentifiers() { throw new Error("Must be implemented by subclasses"); }
 
     updateUIBeforeRun() {
-        if (!isInBrowser) {
-            if (!dumpJSONResults)
-                console.log(`Running ${this.name}:`);
+        if (!dumpJSONResults)
+            console.log(`Running ${this.name}:`);
+        if (!isInBrowser)
             return;
-        }
 
         let containerUI = document.getElementById("results");
         let resultsBenchmarkUI = document.getElementById(`benchmark-${this.name}`);
@@ -947,7 +977,6 @@ class DefaultBenchmark extends Benchmark {
             document.getElementById(worst4ID(this)).innerHTML = uiFriendlyNumber(this.worst4);
             document.getElementById(avgID(this)).innerHTML = uiFriendlyNumber(this.average);
             document.getElementById(scoreID(this)).innerHTML = uiFriendlyNumber(this.score);
-            return;
         }
 
         if (dumpJSONResults)
@@ -979,7 +1008,7 @@ class AsyncBenchmark extends DefaultBenchmark {
                 results.push(Math.max(1, end - start));
             }
             if (__benchmark.validate)
-                __benchmark.validate();
+                __benchmark.validate(${this.iterations});
             top.currentResolve(results);
         }
         doRun().catch((error) => { top.currentReject(error); });`
@@ -1045,7 +1074,6 @@ class WSLBenchmark extends Benchmark {
             document.getElementById("wsl-stdlib-score").innerHTML = uiFriendlyNumber(this.stdlib);
             document.getElementById("wsl-tests-score").innerHTML = uiFriendlyNumber(this.mainRun);
             document.getElementById("wsl-score-score").innerHTML = uiFriendlyNumber(this.score);
-            return;
         }
 
         if (dumpJSONResults)
@@ -1226,7 +1254,6 @@ class WasmBenchmark extends Benchmark {
             document.getElementById(this.startupID).innerHTML = uiFriendlyNumber(this.startupTime);
             document.getElementById(this.runID).innerHTML = uiFriendlyNumber(this.runTime);
             document.getElementById(this.scoreID).innerHTML = uiFriendlyNumber(this.score);
-            return;
         }
 
         if (dumpJSONResults)

RexBench/UniPoker/benchmark.js

@@ -39,8 +39,11 @@ class Benchmark {
         playHands(this._players);
     }
 
-    validate()
+    validate(iterations)
     {
+        if (!iterations)
+            throw "Invalid iterations"
+        const playerExpectations = getPlayerExpectations(iterations)
         if (this._players.length != playerExpectations.length)
             throw "Expect " + playerExpectations.length + ", but actually have " + this._players.length;
         if (isInBrowser) {

RexBench/UniPoker/expected.js

@@ -24,12 +24,15 @@
  */
 "use strict";
 
+const defaultIterationCount = 120;
+
 class PlayerExpectation
 {
-    constructor(wins, handTypeCounts)
+    constructor(iterations, wins, handTypeCounts)
     {
-        this._wins = wins;
-        this._handTypeCounts = handTypeCounts;
+        const factor =  iterations / defaultIterationCount;
+        this._wins = wins * factor;
+        this._handTypeCounts = handTypeCounts.map(each => each * factor);
     }
 
     validate(player)
@@ -62,10 +65,13 @@ PlayerExpectation._handTypes = [
     "Straight Flushes",
     "Royal Flushes"
 ];
-    
-var playerExpectations = [];
 
-playerExpectations.push(new PlayerExpectation(60120, [120600, 102000, 10080, 5040, 1440, 360, 480, 0, 0, 0]));
-playerExpectations.push(new PlayerExpectation(60480, [120360, 99600, 11760, 6120, 1200, 360, 480, 120, 0, 0]));
-playerExpectations.push(new PlayerExpectation(61440, [121200, 99720, 11040, 6120, 1080, 480, 360, 0, 0, 0]));
-playerExpectations.push(new PlayerExpectation(57960, [121320, 100440, 11040, 5760, 840, 480, 120, 0, 0, 0]));
+
+function getPlayerExpectations(iterations) {
+    const playerExpectations = [];
+    playerExpectations.push(new PlayerExpectation(iterations, 60120, [120600, 102000, 10080, 5040, 1440, 360, 480, 0, 0, 0]));
+    playerExpectations.push(new PlayerExpectation(iterations, 60480, [120360, 99600, 11760, 6120, 1200, 360, 480, 120, 0, 0]));
+    playerExpectations.push(new PlayerExpectation(iterations, 61440, [121200, 99720, 11040, 6120, 1080, 480, 360, 0, 0, 0]));
+    playerExpectations.push(new PlayerExpectation(iterations, 57960, [121320, 100440, 11040, 5760, 840, 480, 120, 0, 0, 0]));
+    return playerExpectations;
+}