Windowed score functions

MotionMark/developer.html

@@ -99,6 +99,15 @@ <h3>Adjusting the test complexity:</h3>
                                         <li><label><input name="controller" type="radio" value="adaptive"> Maintain target FPS</label></li>
                                     </ul>
                                 </li>
+                                <li>
+                                    <h3>Score profile:</h3>
+                                    <ul>
+                                        <li><label><input name="score-profile" type="radio" value="slope" checked> Slope</label></li>
+                                        <li><label><input name="score-profile" type="radio" value="flat"> Flat</label></li>
+                                        <li><label><input name="score-profile" type="radio" value="window"> Windowed</label></li>
+                                        <li><label><input name="score-profile" type="radio" value="window-strict"> Windowed Strict</label></li>
+                                    </ul>
+                                </li>
                                 <li>
                                     <label>System frame rate: <input type="number" id="system-frame-rate" value="60"> FPS</label><br>
                                     <label>Target frame rate: <input type="number" id="frame-rate" value="60"> FPS</label>

MotionMark/resources/debug-runner/debug-runner.js

@@ -214,7 +214,7 @@ window.optionsManager = new class OptionsManager {
                 options[name] = formElement.checked;
             else if (type == "radio") {
                 var radios = formElements[name];
-                if (radios.constructor === HTMLCollection) {
+                if (radios.constructor === RadioNodeList || radios.constructor === HTMLCollection) {
                     for (var j = 0; j < radios.length; ++j) {
                         var radio = radios[j];
                         if (radio.checked) {
@@ -657,7 +657,8 @@ class DebugBenchmarkController extends BenchmarkController {
     startBenchmark()
     {
         benchmarkController.determineCanvasSize();
-        benchmarkController.options = Utilities.mergeObjects(this.benchmarkDefaultParameters, optionsManager.updateLocalStorageFromUI());
+        const optionsFromUI = optionsManager.updateLocalStorageFromUI();
+        benchmarkController.options = Utilities.mergeObjects(this.benchmarkDefaultParameters, optionsFromUI);
         benchmarkController.suites = suitesManager.updateLocalStorageFromUI();
         this._startBenchmark(benchmarkController.suites, benchmarkController.options, "running-test");
     }

MotionMark/resources/runner/results.js

@@ -180,14 +180,15 @@ class ScoreCalculator {
 
             const frameTypeIndex = series.fieldMap[Strings.json.frameType];
             const complexityIndex = series.fieldMap[complexityKey];
+            const frameTimeIndex = series.fieldMap[Strings.json.time];
             const frameLengthIndex = series.fieldMap[Strings.json.frameLength];
             const regressionOptions = { desiredFrameLength: desiredFrameLength };
             if (profile)
                 regressionOptions.preferredProfile = profile;
 
             const regressionSamples = series.slice(minIndex, maxIndex + 1);
             const animationSamples = regressionSamples.data.filter((sample) => sample[frameTypeIndex] == Strings.json.animationFrameType);
-            const regressionData = animationSamples.map((sample) => [ sample[complexityIndex], sample[frameLengthIndex] ]);
+            const regressionData = animationSamples.map((sample) => [ sample[complexityIndex], sample[frameLengthIndex], sample[frameTimeIndex] ]);
 
             const regression = new Regression(regressionData, minIndex, maxIndex, regressionOptions);
             return {
@@ -266,6 +267,7 @@ class ScoreCalculator {
 
                 const resample = new SampleData(regressionResult.samples.fieldMap, resampleData);
                 const bootstrapRegressionResult = findRegression(resample, predominantProfile);
+                //console.log('regression', bootstrapRegressionResult.regression);
                 if (bootstrapRegressionResult.regression.t2 < 0) {
                   // A positive slope means the frame rate decreased with increased complexity (which is the expected
                   // benavior). OTOH, a negative slope means the framerate increased as the complexity increased. This

MotionMark/resources/statistics.js

@@ -178,8 +178,10 @@ class Regression {
     constructor(samples, startIndex, endIndex, options)
     {
         const desiredFrameLength = options.desiredFrameLength;
-        var profile;
+        const kWindowSizeMultiple = 0.1;
 
+        var profile;
+
         if (!options.preferredProfile || options.preferredProfile == Strings.json.profiles.slope) {
             profile = this._calculateRegression(samples, {
                 shouldClip: true,
@@ -195,6 +197,11 @@ class Regression {
                 t2: 0
             });
             this.profile = Strings.json.profiles.flat;
+        } else if (options.preferredProfile == Strings.json.profiles.window || options.preferredProfile == Strings.json.profiles.windowStrict) {
+            const window_size = Math.max(1, Math.floor(samples.length * kWindowSizeMultiple));
+            const strict = options.preferredProfile == Strings.json.profiles.windowStrict;
+            profile = this._windowedFit(samples, desiredFrameLength, window_size, strict);
+            this.profile = options.preferredProfile;
         }
 
         this.startIndex = Math.min(startIndex, endIndex);
@@ -219,6 +226,111 @@ class Regression {
         return this.s1 + this.t1 * complexity;
     }
 
+    _windowedFit(samples, desiredFrameLength, windowSize, strict)
+    {
+        const kAllowedErrorFactor = 0.9;
+
+        const complexityIndex = 0;
+        const frameLengthIndex = 1;
+        const frameTimeIndex = 2;
+
+        const average = array => array.reduce((a, b) => a + b) / array.length;
+
+        var sortedSamples = samples.slice().sort((a, b) => {
+                if (a[complexityIndex] == b[complexityIndex])
+                    return b[frameTimeIndex] - a[frameTimeIndex];
+                return a[complexityIndex] - b[complexityIndex];
+            });
+
+        var cumFrameLength = 0.0;
+        var bestIndex = 0;
+        var bestComplexity = 0;
+        var runningFrameLengths = [];
+        var runningComplexities = [];
+
+        for (var i = 0; i < sortedSamples.length; ++i) {
+            runningFrameLengths.push(sortedSamples[i][frameLengthIndex]);
+            runningComplexities.push(sortedSamples[i][complexityIndex]);
+
+            if (runningFrameLengths.length < windowSize) {
+                continue
+            } else if (runningFrameLengths.length > windowSize) {
+                runningFrameLengths.shift();
+                runningComplexities.shift();
+            }
+
+            let averageFrameLength = average(runningFrameLengths);
+            let averageComplexity = average(runningComplexities);
+            let error = desiredFrameLength / averageFrameLength;
+            let adjustedComplexity = averageComplexity * Math.min(1.0, error);
+
+            if (error >= kAllowedErrorFactor) {
+                if (adjustedComplexity > bestComplexity) {
+                    bestComplexity = adjustedComplexity;
+                }
+            } else if (strict) {
+                break;
+            }
+        }
+
+        for (var i = 0; i < sortedSamples.length; ++i) {
+            if (sortedSamples[i][complexityIndex] <= bestComplexity)
+                bestIndex = i;
+        }
+
+        let complexity = bestComplexity;
+
+        // Calculate slope for remaining points
+        let t_nom = 0.0;
+        let t_denom = 0.0;
+        for (var i = bestIndex + 1; i < sortedSamples.length; i++) {
+            const tx = sortedSamples[i][complexityIndex] - complexity;
+            const ty = sortedSamples[i][frameLengthIndex] - desiredFrameLength;
+            t_nom += tx * ty;
+            t_denom += tx * tx;
+        }
+
+        var s1 = desiredFrameLength;
+        var t1 = 0;
+        var t2 = (t_nom / t_denom) || 0.0;
+        var s2 = desiredFrameLength - t2 * complexity;
+        var n1 = bestIndex + 1;
+        var n2 = sortedSamples.length - bestIndex - 1;
+
+        function getValueAt(at_complexity)
+        {
+            if (at_complexity > complexity)
+                return s2 + t2 * complexity;
+            return s1 + t1 * complexity;
+        }
+
+        let error1 = 0.0;
+        let error2 = 0.0;
+        for (var i = 0; i < n1; ++i) {
+            const frameLengthErr = sortedSamples[i][frameLengthIndex] - getValueAt(sortedSamples[i][complexityIndex]);
+            error1 += frameLengthErr * frameLengthErr;
+        }
+        for (var i = n1; i < sortedSamples.length; ++i) {
+            const frameLengthErr = sortedSamples[i][frameLengthIndex] - getValueAt(sortedSamples[i][complexityIndex]);
+            error2 += frameLengthErr * frameLengthErr;
+        }
+
+        return {
+            s1: s1,
+            t1: t1,
+            s2: s2,
+            t2: t2,
+            complexity: complexity,
+            // Number of samples included in the first segment, inclusive of bestIndex
+            n1: n1,
+            // Number of samples included in the second segment
+            n2: n2,
+            stdev1: Math.sqrt(error1 / n1),
+            stdev2: Math.sqrt(error2 / n2),
+            error: error1 + error2,
+        };        
+    }
+
     // A generic two-segment piecewise regression calculator. Based on Kundu/Ubhaya
     //
     // Minimize sum of (y - y')^2

MotionMark/resources/strings.js

@@ -78,7 +78,9 @@ var Strings = {
 
         profiles: {
             slope: "slope",
-            flat: "flat"
+            flat: "flat",
+            window: "window",
+            windowStrict: "window-strict",
         },
 
         results: {

MotionMark/tests/resources/controllers.js

@@ -170,10 +170,15 @@ class Controller {
     {
         return samples[sampleTimeIndex][i] - samples[sampleTimeIndex][i - 1];
     }
+
+    _getFrameTime(samples, i)
+    {
+        return samples[sampleTimeIndex][i];
+    }
 
     _previousFrameComplexity(samples, i)
     {
-        if (i > 0)
+        if (i > 1)
             return this._getComplexity(samples, i - 1);
 
         return 0;
@@ -399,6 +404,7 @@ class RampController extends Controller {
         super(benchmark, options);
 
         this.targetFPS = targetFPS;
+        this.preferredProfile = options["score-profile"];
 
         // Initially start with a tier test to find the bounds
         // The number of objects in a tier test is 10^|_tier|
@@ -586,10 +592,15 @@ class RampController extends Controller {
         for (var i = this._rampStartIndex; i < this._sampler.sampleCount; ++i) {
             if (this._getFrameType(this._sampler.samples, i) == Strings.json.mutationFrameType)
                 continue;
-            regressionData.push([ this._getComplexity(this._sampler.samples, i), this._getFrameLength(this._sampler.samples, i) ]);
+            regressionData.push(
+                [
+                    this._getComplexity(this._sampler.samples, i), 
+                    this._getFrameLength(this._sampler.samples, i),
+                    this._getFrameTime(this._sampler.samples, i)
+                ]);
         }
 
-        var regression = new Regression(regressionData, this._sampler.sampleCount - 1, this._rampStartIndex, { desiredFrameLength: this.frameLengthDesired });
+        var regression = new Regression(regressionData, this._sampler.sampleCount - 1, this._rampStartIndex, { desiredFrameLength: this.frameLengthDesired, preferredProfile: this.preferredProfile });
         this._rampRegressions.push(regression);
 
         var frameLengthAtMaxComplexity = regression.valueAt(this._maximumComplexity);