Merge branch 'master' of github.com:krausest/js-framework-benchmark

Author: krausest

webdriver-ts-results/.eslintignore

@@ -1,2 +1,3 @@
 webpack.config.js
-.eslintrc.js
+.eslintrc.js
+src/results.ts

webdriver-ts-results/src/App.tsx

@@ -15,7 +15,7 @@ const App = (): JSX.Element => {
     return (
       <div>
         {disclaimer}
-        <p>The benchmark was run on a MacBook Pro 14 (32 GB RAM, 8/14 Cores, OSX 13.3), Chrome 112.0.5615.49 (arm64))
+        <p>The benchmark was run on a MacBook Pro 14 (32 GB RAM, 8/14 Cores, OSX 13.3), Chrome 113.0.5672.63 (arm64))
             using the puppeteer benchmark driver with reduced tracing.
         </p>
          <SelectionBar/>

webdriver-ts-results/src/Common.ts

@@ -5,6 +5,7 @@ const jStat = require('jstat').jStat;
 
 export enum StatisticResult {Slower, Undecided, Faster}
 export enum DisplayMode { DisplayMean, DisplayMedian, BoxPlot }
+export enum CpuDurationMode { Total='total', Script='script'}
 
 export enum FrameworkType { KEYED, NON_KEYED }
 
@@ -35,6 +36,8 @@ export const categories: Category[] = [
   {id:6, text:"[Issue]: Errors in the implementation", issues: [634], severity: Severity.Error},
 ]
 
+const DEFAULT_RESULTS_KEY = 'DEFAULT';
+
 interface IIssue {
     issue: number, 
     severity: Severity, 
@@ -64,19 +67,22 @@ export interface Benchmark {
 }
 
 export interface RawResult {
-    f: string;
-    b: string;
-    v: number[];
+    f: string,
+    b: string,
+    v: {[k:string]: number[]}
 }
 
-export interface Result {
-    framework: string;
-    benchmark: string;
+export interface ResultValues {
     values: number[];
     mean: number;
     median: number;
     standardDeviation: number;
 }
+export interface Result {
+    framework: string;
+    benchmark: string;
+    results: {[key:string]: ResultValues};
+}
 
 interface ResultData {
     benchmarks: Array<Benchmark>;
@@ -203,7 +209,8 @@ export class ResultTableData {
 
     constructor(public allFrameworks: Array<Framework>, public allBenchmarks: Array<Benchmark>, public results: ResultLookup,
         public selectedFrameworksInDropdown: Set<Framework>, public selectedBenchmarks: Set<Benchmark>, type: FrameworkType, sortKey: string,
-        public displayMode: DisplayMode, public compareWith: Framework|undefined, public selectedCategories: Set<number>) {     
+        public displayMode: DisplayMode, public compareWith: Framework|undefined, public selectedCategories: Set<number>,
+        public cpuDurationMode: string) {     
 
         this.selectedFameworks = new Set<Framework>();
 
@@ -341,13 +348,16 @@ export class ResultTableData {
     }
 
     computeFactors(benchmark: Benchmark): Array<TableResultValueEntry|null> {
+
+        const resultsKey = benchmark.type == BenchmarkType.CPU ? this.cpuDurationMode : DEFAULT_RESULTS_KEY;
+
         const benchmarkResults = this.frameworksForFactors.map(f => this.results(benchmark, f));
         const selectFn = (result: Result|null) => {
             if (result===null) return 0;
             if (this.displayMode === DisplayMode.DisplayMedian) {
-                return result.median;
+                return result.results[resultsKey].median;
             } else {
-                return result.mean;
+                return result.results[resultsKey].mean;
             }
         }
         const min = Math.max(benchmarkResults.reduce((min, result) => result===null ? min : Math.min(min, selectFn(result)), Number.POSITIVE_INFINITY));
@@ -357,32 +367,34 @@ export class ResultTableData {
         return this.frameworks.map(f => {
             const result = this.results(benchmark, f);
             if (result === null) return null;
+            const resultValues = result.results[resultsKey];
 
             const value = selectFn(result);
             const factor = value/min;
             // if (benchmark.type === BenchmarkType.CPU) {
             //     factor = Math.max(1, factor);
             // }    
-            const conficenceInterval = 1.959964 * (result.standardDeviation || 0) / Math.sqrt(result.values.length);
+            const conficenceInterval = 1.959964 * (resultValues.standardDeviation || 0) / Math.sqrt(resultValues.values.length);
             const conficenceIntervalStr = benchmark.type === BenchmarkType.MEM ? null : conficenceInterval.toFixed(1);
             const formattedValue = formatEn.format(value);
 
             if (!this.compareWith) {
                 return new TableResultValueEntry(f.name, value, formattedValue, conficenceIntervalStr, factor, factor.toFixed(2), computeColor(factor), '#000', StatisticResult.Undecided);
             } else {
                 const compareWithResults = this.results(benchmark, this.compareWith)!;
+                const compareWithResultsValues = compareWithResults.results[resultsKey]
                     // let meanStr = 'x'; //mean.toLocaleString('en-US', {minimumFractionDigits: 1, maximumFractionDigits: 1, useGrouping: true});
 
                 // X1,..,Xn: this Framework, Y1, ..., Ym: selected Framework
                 // https://de.wikipedia.org/wiki/Zweistichproben-t-Test
                 let statisticalResult = undefined;
                 let statisticalCol = undefined;
-                const compareWithMean = compareWithResults.mean;
-                const stdDev = result.standardDeviation || 0;
-                const compareWithResultsStdDev = compareWithResults.standardDeviation || 0;
+                const compareWithMean = compareWithResultsValues.mean;
+                const stdDev = compareWithResultsValues.standardDeviation || 0;
+                const compareWithResultsStdDev = compareWithResultsValues.standardDeviation || 0;
 
 
-                const x1 = result.mean;
+                const x1 = resultValues.mean;
                 const x2 = compareWithMean;
                 const s1_2 = stdDev*stdDev;
                 const s2_2 = compareWithResultsStdDev * compareWithResultsStdDev;

webdriver-ts-results/src/ResultTable.tsx

@@ -1,5 +1,5 @@
 import * as React from 'react';
-import {ResultTableData, DisplayMode, BenchmarkType, FrameworkType} from './Common';
+import {ResultTableData, DisplayMode, BenchmarkType, FrameworkType, CpuDurationMode} from './Common';
 import CpuResultsTable from './tables/CpuResultsTable'
 import MemResultsTable from './tables/MemResultsTable'
 import StartupResultsTable from './tables/StartupResultsTable'
@@ -24,6 +24,7 @@ const ResultTable = ({type}: Props): JSX.Element|null => {
   const data = useSelector<State, ResultTableData|undefined>((state) => state.resultTables[type]);
   const currentSortKey = useSelector<State, string>((state) => state.sortKey);
   const displayMode = useSelector<State, DisplayMode>((state) => state.displayMode);
+  const cpuDurationMode = useSelector<State, CpuDurationMode>((state) => state.cpuDurationMode);
   const sortBy = (sortKey: string) => dispatch(sort(sortKey))
 
   if (data === undefined || data.frameworks.length===0 || (data.getResult(BenchmarkType.CPU).benchmarks.length===0 && data.getResult(BenchmarkType.STARTUP).benchmarks.length===0 && data.getResult(BenchmarkType.MEM).benchmarks.length===0)) return null;
@@ -33,11 +34,12 @@ const ResultTable = ({type}: Props): JSX.Element|null => {
         <div key={texts[type].label}>
           <h1>{texts[type].label}</h1>
           <p>{texts[type].description}</p>
+          {cpuDurationMode==CpuDurationMode.Script && <h3>Warning: This is an experimental view. Don't rely on those values yet.</h3>}
             {
         displayMode === DisplayMode.BoxPlot ?
                 (
             <React.Suspense fallback={<div>Loading...</div>}>
-              <BoxPlotTable results={data.results} frameworks={data.frameworks} benchmarks={data.getResult(BenchmarkType.CPU).benchmarks} currentSortKey={currentSortKey} sortBy={sortBy}/>
+              <BoxPlotTable results={data.results} frameworks={data.frameworks} benchmarks={data.getResult(BenchmarkType.CPU).benchmarks} currentSortKey={currentSortKey} sortBy={sortBy} cpuDurationMode={cpuDurationMode}/>
             </React.Suspense>
                 )
         :

webdriver-ts-results/src/reducer.ts

@@ -1,4 +1,4 @@
-import { Benchmark, BenchmarkType, convertToMap, DisplayMode, Framework, FrameworkType, RawResult, Result, ResultTableData, SORT_BY_GEOMMEAN_CPU, categories, Severity } from "./Common"
+import { Benchmark, BenchmarkType, convertToMap, DisplayMode, Framework, FrameworkType, RawResult, Result, ResultTableData, SORT_BY_GEOMMEAN_CPU, categories, Severity, ResultValues, CpuDurationMode } from "./Common"
 import { benchmarks as benchmark_orig, frameworks, results as rawResults } from './results';
 
 // Temporarily disable script bootup time
@@ -9,12 +9,20 @@ const benchmarks = benchmark_orig.filter(b => b.id !== '32_startup-bt'
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const jStat = require('jstat').jStat;
 
-const results: Result[] = (rawResults as RawResult[]).map(res => Object.assign(({ framework: res.f, benchmark: res.b, values: res.v }),
-  {
-    mean: res.v ? jStat.mean(res.v) : Number.NaN,
-    median: res.v ? jStat.median(res.v) : Number.NaN,
-    standardDeviation: res.v ? jStat.stdev(res.v, true) : Number.NaN
-  }));
+const results: Result[] = (rawResults as RawResult[]).map(res => {
+  const values: {[k: string]: ResultValues} = {};
+  for (const key of Object.keys(res.v)) {
+    const r = res.v[key];
+    const vals = {
+      mean: r ? jStat.mean(r) : Number.NaN,
+      median: r ? jStat.median(r) : Number.NaN,
+      standardDeviation: r ? jStat.stdev(r, true) : Number.NaN,
+      values: r
+    }
+    values[key] = vals;
+  }
+  return { framework: res.f, benchmark: res.b, results: values}
+  });
 
 const removeKeyedSuffix = (value: string) => {
   if (value.endsWith('-non-keyed')) return value.substring(0, value.length - 10)
@@ -52,6 +60,7 @@ export interface State {
   displayMode: DisplayMode;
   compareWith: CompareWith;
   categories: Set<number>;
+  cpuDurationMode: CpuDurationMode;
 }
 
 export const areAllBenchmarksSelected = (state: State, type: BenchmarkType): boolean => state.benchmarkLists[type].every(b => state.selectedBenchmarks.has(b))
@@ -87,13 +96,14 @@ let preInitialState: State = {
     [FrameworkType.KEYED]: undefined,
     [FrameworkType.NON_KEYED]: undefined
   },
-  categories: new Set(categories.filter(c => c.severity != Severity.Error).map(c => c.id))
+  categories: new Set(categories.filter(c => c.severity != Severity.Error).map(c => c.id)),
+  cpuDurationMode : CpuDurationMode.Total
 }
 
-function updateResultTable({ frameworks, benchmarks, selectedFrameworksDropDown: selectedFrameworks, selectedBenchmarks, sortKey, displayMode, compareWith, categories }: State) {
+function updateResultTable({ frameworks, benchmarks, selectedFrameworksDropDown: selectedFrameworks, selectedBenchmarks, sortKey, displayMode, compareWith, categories, cpuDurationMode }: State) {
   return {
-    [FrameworkType.KEYED]: new ResultTableData(frameworks, benchmarks, resultLookup, selectedFrameworks, selectedBenchmarks, FrameworkType.KEYED, sortKey, displayMode, compareWith[FrameworkType.KEYED], categories),
-    [FrameworkType.NON_KEYED]: new ResultTableData(frameworks, benchmarks, resultLookup, selectedFrameworks, selectedBenchmarks, FrameworkType.NON_KEYED, sortKey, displayMode, compareWith[FrameworkType.NON_KEYED], categories)
+    [FrameworkType.KEYED]: new ResultTableData(frameworks, benchmarks, resultLookup, selectedFrameworks, selectedBenchmarks, FrameworkType.KEYED, sortKey, displayMode, compareWith[FrameworkType.KEYED], categories, cpuDurationMode),
+    [FrameworkType.NON_KEYED]: new ResultTableData(frameworks, benchmarks, resultLookup, selectedFrameworks, selectedBenchmarks, FrameworkType.NON_KEYED, sortKey, displayMode, compareWith[FrameworkType.NON_KEYED], categories, cpuDurationMode)
   }
 }
 
@@ -123,7 +133,7 @@ function extractState(state: any): Partial<State> {
   }
   if (state.categories!==undefined) {
     const newSelectedCategories = new Set<number>();
-    for (const f of state?.categories) {
+    for (const f of state?.categories ?? []) {
       for (const sc of categories) {
         if (f === sc.id) newSelectedCategories.add(sc.id);
       }
@@ -192,6 +202,10 @@ interface SelectDisplayModeAction { type: 'SELECT_DISPLAYMODE'; data: { displayM
 export const selectDisplayMode = (displayMode: DisplayMode): SelectDisplayModeAction => {
   return { type: 'SELECT_DISPLAYMODE', data: { displayMode } }
 }
+interface SelectCpuDurationModeAction { type: 'SELECT_CPUDURATIONMODE'; data: { cpuDurationMode: CpuDurationMode } }
+export const selectCpuDurationMode = (cpuDurationMode: CpuDurationMode): SelectCpuDurationModeAction => {
+  return { type: 'SELECT_CPUDURATIONMODE', data: { cpuDurationMode } }
+}
 
 interface CompareAction { type: 'COMPARE'; data: { framework: Framework } }
 export const compare = (framework: Framework): CompareAction => {
@@ -217,7 +231,7 @@ export const setStateFromClipboard = (state: any): SetStateFromClipboardAction =
 
 type Action = SelectFrameworkAction | SelectAllFrameworksAction | SelectBenchmarkAction | SelectAllBenchmarksAction
   | SelectDisplayModeAction | CompareAction | StopCompareAction | SortAction
-  | SelectCategoryAction | SelectAllCategoriesAction | SetStateFromClipboardAction;
+  | SelectCategoryAction | SelectAllCategoriesAction | SetStateFromClipboardAction | SelectCpuDurationModeAction;
 
 export const reducer = (state = initialState, action: Action): State => {
   console.log("reducer", action)
@@ -267,6 +281,10 @@ export const reducer = (state = initialState, action: Action): State => {
       const t = { ...state, displayMode: action.data.displayMode };
       return { ...t, resultTables: updateResultTable(t) };
     }
+    case 'SELECT_CPUDURATIONMODE': {
+      const t = { ...state, cpuDurationMode: action.data.cpuDurationMode };
+      return { ...t, resultTables: updateResultTable(t) };
+    }
     case 'COMPARE': {
       const compareWith = { ...state.compareWith };
       compareWith[action.data.framework.type] = action.data.framework;