Add unit tests for data import, and minor fixes.

Add test-data-import.js which tests creating a `ResultsDashboard` with imported JSON,
which is the JSON produced by running the test via the browser. The test validates that
the results analysis produces output data in the correct format, but does not validate
the score calculation.

To make the unit test run faster, we make `bootstrapIterations` be configurable via
an option. The unit test sets this to a small value.

Fix a bug that prevented the JSON results dialog being shown by using arrow functions
in `BenchmarkController` so that `this` is correctly bound.

Author: smfr

MotionMark/resources/runner/motionmark.js

@@ -350,19 +350,19 @@ class BenchmarkController {
 
         var data = Utilities.createElement("div", {}, container);
         data.textContent = "Please wait...";
-        setTimeout(function() {
+        setTimeout(() => {
             var output = {
                 version: this.runnerClient.results.version,
                 options: this.runnerClient.results.options,
                 data: this.runnerClient.results.data
             };
-            data.textContent = JSON.stringify(output, function(key, value) {
+            data.textContent = JSON.stringify(output, (key, value) => {
                 if (typeof value === 'number')
                     return Utilities.toFixedNumber(value, 3);
                 return value;
             }, 1);
         }, 0);
-        data.onclick = function() {
+        data.onclick = () => {
             var selection = window.getSelection();
             selection.removeAllRanges();
             var range = document.createRange();
@@ -372,8 +372,8 @@ class BenchmarkController {
 
         var button = Utilities.createElement("button", {}, container);
         button.textContent = "Done";
-        button.onclick = function() {
-            benchmarkController.hideDebugInfo();
+        button.onclick = () => {
+            this.hideDebugInfo();
         };
     }
 

MotionMark/resources/runner/results.js

@@ -32,6 +32,11 @@ class ResultsDashboard {
         this._version = version;
         this._targetFrameRate = options["frame-rate"];
         this._systemFrameRate = options["system-frame-rate"];
+
+        const defaultBootstrapIterations = 2500;
+        if (!Object.hasOwn(this._options, Strings.json.bootstrapIterations))
+            this._options[Strings.json.bootstrapIterations] = defaultBootstrapIterations;
+        
         if (testData) {
             this._iterationsSamplers = testData;
             this._processData();
@@ -188,7 +193,7 @@ class ResultsDashboard {
             experimentResult[Strings.json.measurements.stdev] = timeComplexity.standardDeviation();
             experimentResult[Strings.json.measurements.percent] = timeComplexity.percentage();
 
-            const bootstrapIterations = 2500;
+            const bootstrapIterations = this._options[Strings.json.bootstrapIterations];
             var bootstrapResult = Regression.bootstrap(regressionResult.samples.data, bootstrapIterations, function(resampleData) {
                 var complexityIndex = regressionResult.samples.fieldMap[Strings.json.complexity];
                 resampleData.sort(function(a, b) {

MotionMark/resources/strings.js

@@ -60,6 +60,7 @@ var Strings = {
         scoreUpperBound: "scoreUpperBound",
         fps: "fps",
         bootstrap: "bootstrap",
+        bootstrapIterations: "bootstrapIterations",
         measurements: {
             average: "average",
             concern: "concern",

MotionMark/unit-tests/data/two-suite-single-iteration-test-data.json

@@ -10,6 +10,7 @@
     <script src="../resources/statistics.js"></script>
 
     <script src="../resources/runner/tests.js"></script>
+    <script src="../resources/runner/results.js"></script>
     <script src="../resources/debug-runner/tests.js"></script>
 
     <script src="../resources/runner/benchmark-runner.js"></script>
@@ -44,6 +45,7 @@ <h1>MotionMark Unit Tests</h1>
 
     <script src="tests/test-random.js"></script>
     <script src="tests/test-statistics.js"></script>
+    <script src="tests/test-data-import.js"></script>
 
     <script class="mocha-exec">
         mocha.run();

MotionMark/unit-tests/run-unit-tests.html

@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2024 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+describe('Data Import', function() {
+    it('Fetch test data', async function() {
+        this.timeout(200);
+
+        const url = "data/two-suite-single-iteration-test-data.json";
+        const response = await fetch(url);
+        expect(response.ok).to.be(true);
+
+        this.json = await response.json();
+        expect(this.json.version).to.be('1.4');
+    });
+
+    it('Validate imported data', function() {
+        expect(this.json.options instanceof Object).to.be(true);
+        expect(this.json.data instanceof Array).to.be(true);
+    });
+
+    it('Create results', function() {
+        this.timeout(500);
+
+        // Make the results analysis faster.
+        this.json.options[Strings.json.bootstrapIterations] = 10;
+
+        const dashboard = new ResultsDashboard(this.json.version, this.json.options, this.json.data);
+        this.results = dashboard.results;
+        expect(this.results instanceof Array).to.be(true);
+    });
+
+    it('Validate results', function() {
+
+        // Check that we have results, not their exact values.
+        const firstResults = this.results[0];
+
+        expect(typeof firstResults[Strings.json.score]).to.be('number');
+        expect(typeof firstResults[Strings.json.scoreLowerBound]).to.be('number');
+        expect(typeof firstResults[Strings.json.scoreUpperBound]).to.be('number');
+        
+        expect(firstResults[Strings.json.results.tests] instanceof Object).to.be(true);
+
+        const testsResults = firstResults[Strings.json.results.tests];
+        expect(testsResults['MotionMark'] instanceof Object).to.be(true);
+
+        const motionMarkResults = testsResults['MotionMark'];
+        expect(motionMarkResults['Multiply'] instanceof Object).to.be(true);
+        expect(motionMarkResults['Leaves'] instanceof Object).to.be(true);
+
+        const multiplyResults = motionMarkResults['Multiply'];
+        expect(typeof multiplyResults[Strings.json.score]).to.be('number');
+        expect(typeof multiplyResults[Strings.json.scoreLowerBound]).to.be('number');
+        expect(typeof multiplyResults[Strings.json.scoreUpperBound]).to.be('number');
+    });
+});