diff --git a/MotionMark/developer.html b/MotionMark/developer.html
index b2de742..d6974fa 100644
--- a/MotionMark/developer.html
+++ b/MotionMark/developer.html
@@ -99,6 +99,15 @@ Adjusting the test complexity:
+
+ Score profile:
+
+
diff --git a/MotionMark/resources/debug-runner/debug-runner.js b/MotionMark/resources/debug-runner/debug-runner.js
index 085f4d3..3cf9491 100644
--- a/MotionMark/resources/debug-runner/debug-runner.js
+++ b/MotionMark/resources/debug-runner/debug-runner.js
@@ -214,7 +214,7 @@ window.optionsManager = new class OptionsManager {
options[name] = formElement.checked;
else if (type == "radio") {
var radios = formElements[name];
- if (radios.constructor === HTMLCollection) {
+ if (radios.constructor === RadioNodeList || radios.constructor === HTMLCollection) {
for (var j = 0; j < radios.length; ++j) {
var radio = radios[j];
if (radio.checked) {
@@ -657,7 +657,8 @@ class DebugBenchmarkController extends BenchmarkController {
startBenchmark()
{
benchmarkController.determineCanvasSize();
- benchmarkController.options = Utilities.mergeObjects(this.benchmarkDefaultParameters, optionsManager.updateLocalStorageFromUI());
+ const optionsFromUI = optionsManager.updateLocalStorageFromUI();
+ benchmarkController.options = Utilities.mergeObjects(this.benchmarkDefaultParameters, optionsFromUI);
benchmarkController.suites = suitesManager.updateLocalStorageFromUI();
this._startBenchmark(benchmarkController.suites, benchmarkController.options, "running-test");
}
diff --git a/MotionMark/resources/runner/results.js b/MotionMark/resources/runner/results.js
index 291992a..6eea718 100644
--- a/MotionMark/resources/runner/results.js
+++ b/MotionMark/resources/runner/results.js
@@ -180,6 +180,7 @@ class ScoreCalculator {
const frameTypeIndex = series.fieldMap[Strings.json.frameType];
const complexityIndex = series.fieldMap[complexityKey];
+ const frameTimeIndex = series.fieldMap[Strings.json.time];
const frameLengthIndex = series.fieldMap[Strings.json.frameLength];
const regressionOptions = { desiredFrameLength: desiredFrameLength };
if (profile)
@@ -187,7 +188,7 @@ class ScoreCalculator {
const regressionSamples = series.slice(minIndex, maxIndex + 1);
const animationSamples = regressionSamples.data.filter((sample) => sample[frameTypeIndex] == Strings.json.animationFrameType);
- const regressionData = animationSamples.map((sample) => [ sample[complexityIndex], sample[frameLengthIndex] ]);
+ const regressionData = animationSamples.map((sample) => [ sample[complexityIndex], sample[frameLengthIndex], sample[frameTimeIndex] ]);
const regression = new Regression(regressionData, minIndex, maxIndex, regressionOptions);
return {
@@ -266,6 +267,7 @@ class ScoreCalculator {
const resample = new SampleData(regressionResult.samples.fieldMap, resampleData);
const bootstrapRegressionResult = findRegression(resample, predominantProfile);
+ //console.log('regression', bootstrapRegressionResult.regression);
if (bootstrapRegressionResult.regression.t2 < 0) {
// A positive slope means the frame rate decreased with increased complexity (which is the expected
// benavior). OTOH, a negative slope means the framerate increased as the complexity increased. This
diff --git a/MotionMark/resources/statistics.js b/MotionMark/resources/statistics.js
index bd0d7e3..d8e76ee 100644
--- a/MotionMark/resources/statistics.js
+++ b/MotionMark/resources/statistics.js
@@ -178,8 +178,10 @@ class Regression {
constructor(samples, startIndex, endIndex, options)
{
const desiredFrameLength = options.desiredFrameLength;
- var profile;
+ const kWindowSizeMultiple = 0.1;
+ var profile;
+
if (!options.preferredProfile || options.preferredProfile == Strings.json.profiles.slope) {
profile = this._calculateRegression(samples, {
shouldClip: true,
@@ -195,6 +197,11 @@ class Regression {
t2: 0
});
this.profile = Strings.json.profiles.flat;
+ } else if (options.preferredProfile == Strings.json.profiles.window || options.preferredProfile == Strings.json.profiles.windowStrict) {
+ const window_size = Math.max(1, Math.floor(samples.length * kWindowSizeMultiple));
+ const strict = options.preferredProfile == Strings.json.profiles.windowStrict;
+ profile = this._windowedFit(samples, desiredFrameLength, window_size, strict);
+ this.profile = options.preferredProfile;
}
this.startIndex = Math.min(startIndex, endIndex);
@@ -219,6 +226,111 @@ class Regression {
return this.s1 + this.t1 * complexity;
}
+ _windowedFit(samples, desiredFrameLength, windowSize, strict)
+ {
+ const kAllowedErrorFactor = 0.9;
+
+ const complexityIndex = 0;
+ const frameLengthIndex = 1;
+ const frameTimeIndex = 2;
+
+ const average = array => array.reduce((a, b) => a + b) / array.length;
+
+ var sortedSamples = samples.slice().sort((a, b) => {
+ if (a[complexityIndex] == b[complexityIndex])
+ return b[frameTimeIndex] - a[frameTimeIndex];
+ return a[complexityIndex] - b[complexityIndex];
+ });
+
+ var cumFrameLength = 0.0;
+ var bestIndex = 0;
+ var bestComplexity = 0;
+ var runningFrameLengths = [];
+ var runningComplexities = [];
+
+ for (var i = 0; i < sortedSamples.length; ++i) {
+ runningFrameLengths.push(sortedSamples[i][frameLengthIndex]);
+ runningComplexities.push(sortedSamples[i][complexityIndex]);
+
+ if (runningFrameLengths.length < windowSize) {
+ continue
+ } else if (runningFrameLengths.length > windowSize) {
+ runningFrameLengths.shift();
+ runningComplexities.shift();
+ }
+
+ let averageFrameLength = average(runningFrameLengths);
+ let averageComplexity = average(runningComplexities);
+ let error = desiredFrameLength / averageFrameLength;
+ let adjustedComplexity = averageComplexity * Math.min(1.0, error);
+
+ if (error >= kAllowedErrorFactor) {
+ if (adjustedComplexity > bestComplexity) {
+ bestComplexity = adjustedComplexity;
+ }
+ } else if (strict) {
+ break;
+ }
+ }
+
+ for (var i = 0; i < sortedSamples.length; ++i) {
+ if (sortedSamples[i][complexityIndex] <= bestComplexity)
+ bestIndex = i;
+ }
+
+ let complexity = bestComplexity;
+
+ // Calculate slope for remaining points
+ let t_nom = 0.0;
+ let t_denom = 0.0;
+ for (var i = bestIndex + 1; i < sortedSamples.length; i++) {
+ const tx = sortedSamples[i][complexityIndex] - complexity;
+ const ty = sortedSamples[i][frameLengthIndex] - desiredFrameLength;
+ t_nom += tx * ty;
+ t_denom += tx * tx;
+ }
+
+ var s1 = desiredFrameLength;
+ var t1 = 0;
+ var t2 = (t_nom / t_denom) || 0.0;
+ var s2 = desiredFrameLength - t2 * complexity;
+ var n1 = bestIndex + 1;
+ var n2 = sortedSamples.length - bestIndex - 1;
+
+ function getValueAt(at_complexity)
+ {
+ if (at_complexity > complexity)
+ return s2 + t2 * complexity;
+ return s1 + t1 * complexity;
+ }
+
+ let error1 = 0.0;
+ let error2 = 0.0;
+ for (var i = 0; i < n1; ++i) {
+ const frameLengthErr = sortedSamples[i][frameLengthIndex] - getValueAt(sortedSamples[i][complexityIndex]);
+ error1 += frameLengthErr * frameLengthErr;
+ }
+ for (var i = n1; i < sortedSamples.length; ++i) {
+ const frameLengthErr = sortedSamples[i][frameLengthIndex] - getValueAt(sortedSamples[i][complexityIndex]);
+ error2 += frameLengthErr * frameLengthErr;
+ }
+
+ return {
+ s1: s1,
+ t1: t1,
+ s2: s2,
+ t2: t2,
+ complexity: complexity,
+ // Number of samples included in the first segment, inclusive of bestIndex
+ n1: n1,
+ // Number of samples included in the second segment
+ n2: n2,
+ stdev1: Math.sqrt(error1 / n1),
+ stdev2: Math.sqrt(error2 / n2),
+ error: error1 + error2,
+ };
+ }
+
// A generic two-segment piecewise regression calculator. Based on Kundu/Ubhaya
//
// Minimize sum of (y - y')^2
diff --git a/MotionMark/resources/strings.js b/MotionMark/resources/strings.js
index 43e0510..735e901 100644
--- a/MotionMark/resources/strings.js
+++ b/MotionMark/resources/strings.js
@@ -78,7 +78,9 @@ var Strings = {
profiles: {
slope: "slope",
- flat: "flat"
+ flat: "flat",
+ window: "window",
+ windowStrict: "window-strict",
},
results: {
diff --git a/MotionMark/tests/resources/controllers.js b/MotionMark/tests/resources/controllers.js
index 25c2720..3788d52 100644
--- a/MotionMark/tests/resources/controllers.js
+++ b/MotionMark/tests/resources/controllers.js
@@ -170,10 +170,15 @@ class Controller {
{
return samples[sampleTimeIndex][i] - samples[sampleTimeIndex][i - 1];
}
+
+ _getFrameTime(samples, i)
+ {
+ return samples[sampleTimeIndex][i];
+ }
_previousFrameComplexity(samples, i)
{
- if (i > 0)
+ if (i > 1)
return this._getComplexity(samples, i - 1);
return 0;
@@ -399,6 +404,7 @@ class RampController extends Controller {
super(benchmark, options);
this.targetFPS = targetFPS;
+ this.preferredProfile = options["score-profile"];
// Initially start with a tier test to find the bounds
// The number of objects in a tier test is 10^|_tier|
@@ -586,10 +592,15 @@ class RampController extends Controller {
for (var i = this._rampStartIndex; i < this._sampler.sampleCount; ++i) {
if (this._getFrameType(this._sampler.samples, i) == Strings.json.mutationFrameType)
continue;
- regressionData.push([ this._getComplexity(this._sampler.samples, i), this._getFrameLength(this._sampler.samples, i) ]);
+ regressionData.push(
+ [
+ this._getComplexity(this._sampler.samples, i),
+ this._getFrameLength(this._sampler.samples, i),
+ this._getFrameTime(this._sampler.samples, i)
+ ]);
}
- var regression = new Regression(regressionData, this._sampler.sampleCount - 1, this._rampStartIndex, { desiredFrameLength: this.frameLengthDesired });
+ var regression = new Regression(regressionData, this._sampler.sampleCount - 1, this._rampStartIndex, { desiredFrameLength: this.frameLengthDesired, preferredProfile: this.preferredProfile });
this._rampRegressions.push(regression);
var frameLengthAtMaxComplexity = regression.valueAt(this._maximumComplexity);