Import all runs from a benchmark JSON file

When importing a file which is the result of a benchmark run with multiple iterations (which contains a `debugOutput` key), store all the runs in `RunData`.

Add static methods to `RunData` to handle both types of JSON, and move the data migration code there.

The score computation already correctly computed the arithmetic mean of the runs.

The code already correctly built the results table for multiple runs.

Author: smfr

MotionMark/resources/debug-runner/debug-runner.js

@@ -593,13 +593,16 @@ class DebugBenchmarkController extends BenchmarkController {
             var reader = new FileReader();
             reader.filename = file.name;
             reader.onload = (e) => {
-                var run = JSON.parse(e.target.result);
-                if (run.debugOutput instanceof Array)
-                    run = run.debugOutput[0];
+                const data = JSON.parse(e.target.result);
+                
+                let results;
+                if (data['debugOutput'] instanceof Array)
+                    results = RunData.resultsDataFromBenchmarkRunnerData(data['debugOutput']);
+                else
+                    results = RunData.resultsDataFromSingleRunData(data);
 
-                this.migrateImportedData(run);
                 this.ensureRunnerClient([ ], { });
-                this.runnerClient.scoreCalculator = new ScoreCalculator(new RunData(run.version, run.options, run.data));
+                this.runnerClient.scoreCalculator = new ScoreCalculator(results);
                 this.showResults();
             };
 
@@ -608,22 +611,6 @@ class DebugBenchmarkController extends BenchmarkController {
         }, false);
     }
 
-    migrateImportedData(runData)
-    {
-        if (!("version" in runData))
-            runData.version = "1.0";
-        
-        if (!("frame-rate" in runData.options)) {
-            runData.options["frame-rate"] = 60;
-            console.log("No frame-rate data; assuming 60fps")
-        }
-
-        if (!("system-frame-rate" in runData.options)) {
-            runData.options["system-frame-rate"] = 60;
-            console.log("No system-frame-rate data; assuming 60fps")
-        }
-    }
-
     frameRateDeterminationComplete(targetFrameRate)
     {
         let frameRateLabelContent = Strings.text.usingFrameRate.replace("%s", targetFrameRate);
@@ -704,7 +691,7 @@ class DebugBenchmarkController extends BenchmarkController {
             this.addedKeyEvent = true;
         }
 
-        var scoreCalculator = this.runnerClient.scoreCalculator;
+        const scoreCalculator = this.runnerClient.scoreCalculator;
         if (scoreCalculator.options["controller"] == "ramp")
             Headers.details[3].disabled = true;
         else {
@@ -717,10 +704,10 @@ class DebugBenchmarkController extends BenchmarkController {
             document.body.classList.add(scoreCalculator.options[Strings.json.configuration]);
         }
 
-        var score = scoreCalculator.score;
-        var confidence = ((scoreCalculator.scoreLowerBound / score - 1) * 100).toFixed(2) +
+        const score = scoreCalculator.score;
+        const confidence = ((scoreCalculator.scoreLowerBound / score - 1) * 100).toFixed(2) +
             "% / +" + ((scoreCalculator.scoreUpperBound / score - 1) * 100).toFixed(2) + "%";
-        var fps = scoreCalculator._systemFrameRate;
+        const fps = scoreCalculator._systemFrameRate;
         sectionsManager.setSectionVersion("results", scoreCalculator.version);
         sectionsManager.setSectionScore("results", score.toFixed(2), confidence, fps);
         sectionsManager.populateTable("results-header", Headers.testName, scoreCalculator);

MotionMark/resources/runner/motionmark.js

@@ -140,7 +140,7 @@ class BenchmarkController {
         try {
             targetFrameRate = await this.determineFrameRate(progressElement);
         } catch (e) {
-            console.log('Frame rate detection failed ' + e);
+            console.error('Frame rate detection failed ' + e);
         }
         this.frameRateDeterminationComplete(targetFrameRate);
     }

MotionMark/resources/runner/results.js

@@ -30,6 +30,56 @@ class RunData {
         this.options = options;
         this.runs = runs;
     }
+    
+    static resultsDataFromSingleRunData(singleRunData)
+    {
+        RunData.#migrateImportedData(singleRunData);
+        
+        if (!singleRunData.data instanceof Array) {
+            console.error('Imported singleRunData.data is not an array. Bailing');
+            return null;
+        }
+        
+        return new RunData(singleRunData.version, singleRunData.options, singleRunData.data);
+    }
+
+    static resultsDataFromBenchmarkRunnerData(benchmarkData)
+    {
+        if (!benchmarkData instanceof Array) {
+            console.log('Imported benchmarkData is not an array. Bailing');
+            return null;
+        }
+        
+        const runData = [];
+        for (const run of benchmarkData) {
+            RunData.#migrateImportedData(run);
+            if (run.data.length !== 1) {
+                console.error('Imported benchmarkData has a "data" array with an unexpected number of items. Bailing');
+                return null;
+            }
+        
+            runData.push(run.data[0]);
+        }
+        
+        // Version and options data should be these same for each run. Use the first run's information.
+        return new RunData(benchmarkData[0].version, benchmarkData[0].options, runData);
+    }
+    
+    static #migrateImportedData(options)
+    {
+        if (!("version" in options))
+            options.version = "1.0";
+        
+        if (!("frame-rate" in options)) {
+            options.options["frame-rate"] = 60;
+            console.log("No frame-rate data; assuming 60fps")
+        }
+
+        if (!("system-frame-rate" in options)) {
+            options.options["system-frame-rate"] = 60;
+            console.log("No system-frame-rate data; assuming 60fps")
+        }
+    }
 }
 
 class ScoreCalculator {
@@ -109,36 +159,37 @@ class ScoreCalculator {
 
     calculateScore(data)
     {
-        var result = {};
+        const result = {};
         data[Strings.json.result] = result;
-        var samples = data[Strings.json.samples];
+        const samples = data[Strings.json.samples];
         const desiredFrameLength = 1000 / this._targetFrameRate;
+        const complexityKey = Strings.json.complexity;
 
         function findRegression(series, profile) {
-            var minIndex = Math.round(.025 * series.length);
-            var maxIndex = Math.round(.975 * (series.length - 1));
-            var minComplexity = series.getFieldInDatum(minIndex, Strings.json.complexity);
-            var maxComplexity = series.getFieldInDatum(maxIndex, Strings.json.complexity);
+            const minIndex = Math.round(.025 * series.length);
+            const maxIndex = Math.round(.975 * (series.length - 1));
+            const minComplexity = series.getFieldInDatum(minIndex, complexityKey);
+            const maxComplexity = series.getFieldInDatum(maxIndex, complexityKey);
 
             if (Math.abs(maxComplexity - minComplexity) < 20 && maxIndex - minIndex < 20) {
                 minIndex = 0;
                 maxIndex = series.length - 1;
-                minComplexity = series.getFieldInDatum(minIndex, Strings.json.complexity);
-                maxComplexity = series.getFieldInDatum(maxIndex, Strings.json.complexity);
+                minComplexity = series.getFieldInDatum(minIndex, complexityKey);
+                maxComplexity = series.getFieldInDatum(maxIndex, complexityKey);
             }
 
-            var frameTypeIndex = series.fieldMap[Strings.json.frameType];
-            var complexityIndex = series.fieldMap[Strings.json.complexity];
-            var frameLengthIndex = series.fieldMap[Strings.json.frameLength];
-            var regressionOptions = { desiredFrameLength: desiredFrameLength };
+            const frameTypeIndex = series.fieldMap[Strings.json.frameType];
+            const complexityIndex = series.fieldMap[complexityKey];
+            const frameLengthIndex = series.fieldMap[Strings.json.frameLength];
+            const regressionOptions = { desiredFrameLength: desiredFrameLength };
             if (profile)
                 regressionOptions.preferredProfile = profile;
 
-            var regressionSamples = series.slice(minIndex, maxIndex + 1);
-            var animationSamples = regressionSamples.data.filter((sample) => sample[frameTypeIndex] == Strings.json.animationFrameType);
-            var regressionData = animationSamples.map((sample) => [ sample[complexityIndex], sample[frameLengthIndex] ]);
+            const regressionSamples = series.slice(minIndex, maxIndex + 1);
+            const animationSamples = regressionSamples.data.filter((sample) => sample[frameTypeIndex] == Strings.json.animationFrameType);
+            const regressionData = animationSamples.map((sample) => [ sample[complexityIndex], sample[frameLengthIndex] ]);
 
-            var regression = new Regression(regressionData, minIndex, maxIndex, regressionOptions);
+            const regression = new Regression(regressionData, minIndex, maxIndex, regressionOptions);
             return {
                 minComplexity: minComplexity,
                 maxComplexity: maxComplexity,
@@ -148,14 +199,14 @@ class ScoreCalculator {
         }
 
         // Convert these samples into SampleData objects if needed
-        [Strings.json.complexity, Strings.json.controller].forEach(function(seriesName) {
-            var series = samples[seriesName];
+        [complexityKey, Strings.json.controller].forEach(function(seriesName) {
+            const series = samples[seriesName];
             if (series && !(series instanceof SampleData))
                 samples[seriesName] = new SampleData(series.fieldMap, series.data);
         });
 
-        var isRampController = this._runData.options[Strings.json.controller] == "ramp";
-        var predominantProfile = "";
+        const isRampController = this._runData.options[Strings.json.controller] == "ramp";
+        let predominantProfile = "";
         if (isRampController) {
             var profiles = {};
             data[Strings.json.controller].forEach(function(regression) {
@@ -174,44 +225,44 @@ class ScoreCalculator {
             }
         }
 
-        var regressionResult = findRegression(samples[Strings.json.complexity], predominantProfile);
-        var calculation = regressionResult.regression;
-        result[Strings.json.complexity] = {};
-        result[Strings.json.complexity][Strings.json.regressions.segment1] = [
+        const regressionResult = findRegression(samples[complexityKey], predominantProfile);
+        const calculation = regressionResult.regression;
+        result[complexityKey] = {};
+        result[complexityKey][Strings.json.regressions.segment1] = [
             [regressionResult.minComplexity, calculation.s1 + calculation.t1 * regressionResult.minComplexity],
             [calculation.complexity, calculation.s1 + calculation.t1 * calculation.complexity]
         ];
-        result[Strings.json.complexity][Strings.json.regressions.segment2] = [
+        result[complexityKey][Strings.json.regressions.segment2] = [
             [calculation.complexity, calculation.s2 + calculation.t2 * calculation.complexity],
             [regressionResult.maxComplexity, calculation.s2 + calculation.t2 * regressionResult.maxComplexity]
         ];
-        result[Strings.json.complexity][Strings.json.complexity] = calculation.complexity;
-        result[Strings.json.complexity][Strings.json.measurements.stdev] = Math.sqrt(calculation.error / samples[Strings.json.complexity].length);
+        result[complexityKey][complexityKey] = calculation.complexity;
+        result[complexityKey][Strings.json.measurements.stdev] = Math.sqrt(calculation.error / samples[complexityKey].length);
 
         result[Strings.json.fps] = data.targetFPS;
 
         if (isRampController) {
-            var timeComplexity = new Experiment;
+            const timeComplexity = new Experiment;
             data[Strings.json.controller].forEach(function(regression) {
-                timeComplexity.sample(regression[Strings.json.complexity]);
+                timeComplexity.sample(regression[complexityKey]);
             });
 
-            var experimentResult = {};
+            const experimentResult = {};
             result[Strings.json.controller] = experimentResult;
             experimentResult[Strings.json.score] = timeComplexity.mean();
             experimentResult[Strings.json.measurements.average] = timeComplexity.mean();
             experimentResult[Strings.json.measurements.stdev] = timeComplexity.standardDeviation();
             experimentResult[Strings.json.measurements.percent] = timeComplexity.percentage();
 
             const bootstrapIterations = this._runData.options[Strings.json.bootstrapIterations];
-            var bootstrapResult = Regression.bootstrap(regressionResult.samples.data, bootstrapIterations, function(resampleData) {
-                var complexityIndex = regressionResult.samples.fieldMap[Strings.json.complexity];
+            const bootstrapResult = Regression.bootstrap(regressionResult.samples.data, bootstrapIterations, function(resampleData) {
+                const complexityIndex = regressionResult.samples.fieldMap[complexityKey];
                 resampleData.sort(function(a, b) {
                     return a[complexityIndex] - b[complexityIndex];
                 });
 
-                var resample = new SampleData(regressionResult.samples.fieldMap, resampleData);
-                var bootstrapRegressionResult = findRegression(resample, predominantProfile);
+                const resample = new SampleData(regressionResult.samples.fieldMap, resampleData);
+                const bootstrapRegressionResult = findRegression(resample, predominantProfile);
                 if (bootstrapRegressionResult.regression.t2 < 0) {
                   // A positive slope means the frame rate decreased with increased complexity (which is the expected
                   // benavior). OTOH, a negative slope means the framerate increased as the complexity increased. This
@@ -224,24 +275,24 @@ class ScoreCalculator {
                 return bootstrapRegressionResult.regression.complexity;
             }, .8);
 
-            result[Strings.json.complexity][Strings.json.bootstrap] = bootstrapResult;
+            result[complexityKey][Strings.json.bootstrap] = bootstrapResult;
             result[Strings.json.score] = bootstrapResult.median;
             result[Strings.json.scoreLowerBound] = bootstrapResult.confidenceLow;
             result[Strings.json.scoreUpperBound] = bootstrapResult.confidenceHigh;
         } else {
-            var marks = data[Strings.json.marks];
-            var samplingStartIndex = 0, samplingEndIndex = -1;
+            const marks = data[Strings.json.marks];
+            let samplingStartIndex = 0, samplingEndIndex = -1;
             if (Strings.json.samplingStartTimeOffset in marks)
                 samplingStartIndex = marks[Strings.json.samplingStartTimeOffset].index;
             if (Strings.json.samplingEndTimeOffset in marks)
                 samplingEndIndex = marks[Strings.json.samplingEndTimeOffset].index;
 
-            var averageComplexity = new Experiment;
-            var averageFrameLength = new Experiment;
-            var controllerSamples = samples[Strings.json.controller];
+            const averageComplexity = new Experiment;
+            const averageFrameLength = new Experiment;
+            const controllerSamples = samples[Strings.json.controller];
             controllerSamples.forEach(function (sample, i) {
                 if (i >= samplingStartIndex && (samplingEndIndex == -1 || i < samplingEndIndex)) {
-                    averageComplexity.sample(controllerSamples.getFieldInDatum(sample, Strings.json.complexity));
+                    averageComplexity.sample(controllerSamples.getFieldInDatum(sample, complexityKey));
                     var smoothedFrameLength = controllerSamples.getFieldInDatum(sample, Strings.json.smoothedFrameLength);
                     if (smoothedFrameLength && smoothedFrameLength != -1)
                         averageFrameLength.sample(smoothedFrameLength);
@@ -346,14 +397,14 @@ class ResultsTable {
 
     _addHeader()
     {
-        var thead = Utilities.createElement("thead", {}, this.element);
-        var row = Utilities.createElement("tr", {}, thead);
+        const thead = Utilities.createElement("thead", {}, this.element);
+        const row = Utilities.createElement("tr", {}, thead);
 
         this._headers.forEach(function (header) {
             if (header.disabled)
                 return;
 
-            var th = Utilities.createElement("th", {}, row);
+            const th = Utilities.createElement("th", {}, row);
             if (header.title != Strings.text.graph)
                 th.innerHTML = header.title;
             if (header.children)
@@ -368,18 +419,18 @@ class ResultsTable {
 
     _addEmptyRow()
     {
-        var row = Utilities.createElement("tr", {}, this.tbody);
+        const row = Utilities.createElement("tr", {}, this.tbody);
         this._flattenedHeaders.forEach(function (header) {
             return Utilities.createElement("td", { class: "suites-separator" }, row);
         });
     }
 
     _addTest(testName, testResult, options)
     {
-        var row = Utilities.createElement("tr", {}, this.tbody);
+        const row = Utilities.createElement("tr", {}, this.tbody);
 
         this._flattenedHeaders.forEach(function (header) {
-            var td = Utilities.createElement("td", {}, row);
+            const td = Utilities.createElement("td", {}, row);
             if (header.text == Strings.text.testName) {
                 td.textContent = testName;
             } else if (typeof header.text == "string") {
@@ -394,12 +445,12 @@ class ResultsTable {
 
     _addIteration(iterationResult, iterationData, options)
     {
-        var testsResults = iterationResult[Strings.json.results.tests];
-        for (var suiteName in testsResults) {
+        const testsResults = iterationResult[Strings.json.results.tests];
+        for (const suiteName in testsResults) {
             this._addEmptyRow();
-            var suiteResult = testsResults[suiteName];
-            var suiteData = iterationData[suiteName];
-            for (var testName in suiteResult)
+            const suiteResult = testsResults[suiteName];
+            const suiteData = iterationData[suiteName];
+            for (let testName in suiteResult)
                 this._addTest(testName, suiteResult[testName], options, suiteData[testName]);
         }
     }
@@ -410,7 +461,7 @@ class ResultsTable {
         this._addHeader();
         this._addBody();
 
-        var iterationsResults = scoreCalculator.results;
+        const iterationsResults = scoreCalculator.results;
         iterationsResults.forEach(function(iterationResult, index) {
             this._addIteration(iterationResult, scoreCalculator.data[index], scoreCalculator.options);
         }, this);