add ability to run puppeteer CPU tests

Author: krausest

webdriver-ts/results.json

@@ -0,0 +1,22 @@
+import * as benchmarksPuppeteer from "./benchmarksPuppeteer";
+import * as benchmarksWebdriver from "./benchmarksWebdriver";
+
+export const benchmarks: Array<benchmarksPuppeteer.BenchmarkPuppeteer|benchmarksWebdriver.BenchmarkWebdriver> = [
+    benchmarksWebdriver.benchRun,
+    benchmarksWebdriver.benchReplaceAll,
+    benchmarksWebdriver.benchUpdate,
+    benchmarksWebdriver.benchSelect,
+    benchmarksWebdriver.benchSwapRows,
+    benchmarksWebdriver.benchRemove,
+    benchmarksWebdriver.benchRunBig,
+    benchmarksWebdriver.benchAppendToManyRows,
+    benchmarksWebdriver.benchClear,
+  
+    benchmarksPuppeteer.benchReadyMemory,
+    benchmarksPuppeteer.benchRunMemory,
+    benchmarksPuppeteer.benchUpdate5Memory,
+    benchmarksPuppeteer.benchReplace5Memory,
+    benchmarksPuppeteer.benchCreateClear5Memory,
+
+    benchmarksWebdriver.benchStartup,
+];

webdriver-ts/src/benchmarkConfiguration.ts

@@ -1,21 +1,27 @@
 import { fork } from "child_process";
 import * as fs from "fs";
 import * as yargs from "yargs";
-import { BenchmarkInfo, BenchmarkType } from "./benchmarksGeneric";
-import { BenchmarkPuppeteer, benchmarksPuppeteer } from "./benchmarksPuppeteer";
-import { benchmarksWebdriver, BenchmarkWebdriver, LighthouseData } from "./benchmarksWebdriver";
+import { BenchmarkInfo, BenchmarkType } from "./benchmarksCommon";
+import { BenchmarkPuppeteer } from "./benchmarksPuppeteer";
+import { BenchmarkWebdriver, LighthouseData } from "./benchmarksWebdriver";
 import { BenchmarkOptions, config, ErrorAndWarning, FrameworkData, initializeFrameworks } from "./common";
 import { writeResults } from "./writeResults";
+import {benchmarks} from "./benchmarkConfiguration";
 
 function forkAndCallBenchmark(
   framework: FrameworkData,
   benchmark: BenchmarkInfo,
   benchmarkOptions: BenchmarkOptions
 ): Promise<ErrorAndWarning> {
   return new Promise((resolve, reject) => {
-    const forked = fork(
-      benchmark.type == BenchmarkType.MEM ? "dist/forkedBenchmarkRunnerPuppeteer.js" : "dist/forkedBenchmarkRunnerWebdriver.js"
-    );
+    let forkedRunner = null;
+    if (benchmarks.some(b => b.benchmarkInfo.id == benchmark.id && b instanceof BenchmarkPuppeteer)) {
+      forkedRunner = "dist/forkedBenchmarkRunnerPuppeteer.js";
+    } else {
+      forkedRunner = "dist/forkedBenchmarkRunnerWebdriver.js";
+    }
+    console.log("forking ",forkedRunner);
+    const forked = fork(forkedRunner);
     if (config.LOG_DETAILS) console.log("FORKING:  forked child process");
     forked.send({
       config,
@@ -87,7 +93,7 @@ async function runBenchmakLoop(
       }
     }
     if (benchmark.type == BenchmarkType.CPU) {
-      // console.log("CPU results before: ", results);
+      console.log("CPU results before: ", results);
       (results as number[]).sort((a: number, b: number) => a - b);
       results = results.slice(0, config.NUM_ITERATIONS_FOR_BENCHMARK_CPU);
       // console.log("CPU results after: ", results)
@@ -109,13 +115,9 @@ async function runBench(runFrameworks: FrameworkData[], benchmarkNames: string[]
   let errors: String[] = [];
   let warnings: String[] = [];
 
-  let b1: Array<BenchmarkWebdriver | BenchmarkPuppeteer> = benchmarksWebdriver.filter((b) =>
+  let runBenchmarks: Array<BenchmarkWebdriver | BenchmarkPuppeteer> = benchmarks.filter((b) =>
     benchmarkNames.some((name) => b.id.toLowerCase().indexOf(name) > -1)
   );
-  let b2: Array<BenchmarkWebdriver | BenchmarkPuppeteer> = benchmarksPuppeteer.filter((b) =>
-    benchmarkNames.some((name) => b.id.toLowerCase().indexOf(name) > -1)
-  );
-  let runBenchmarks: Array<BenchmarkWebdriver | BenchmarkPuppeteer> = b1.concat(b2);
 
   let restart: string = undefined;
   let index = runFrameworks.findIndex((f) => f.fullNameWithKeyedAndVersion === restart);
@@ -132,6 +134,8 @@ async function runBench(runFrameworks: FrameworkData[], benchmarkNames: string[]
     runBenchmarks.map((b) => b.id)
   );
 
+    console.log("HEADLESS*** ", args.headless);
+
   let benchmarkOptions: BenchmarkOptions = {
     port: config.PORT.toFixed(),
     remoteDebuggingPort: config.REMOTE_DEBUGGING_PORT,
@@ -188,7 +192,7 @@ let args: any = yargs(process.argv)
     "$0 [--framework Framework1 Framework2 ...] [--benchmark Benchmark1 Benchmark2 ...] [--chromeBinary path] \n or: $0 [directory1] [directory2] .. [directory3]"
   )
   .help("help")
-  .boolean("headless")
+  .boolean("headless").default("headless", false)
   .boolean("smoketest")
   .array("framework")
   .array("benchmark")
@@ -218,6 +222,7 @@ async function main() {
   }
 
   if (!fs.existsSync(config.RESULTS_DIRECTORY)) fs.mkdirSync(config.RESULTS_DIRECTORY);
+  if (!fs.existsSync(config.TRACES_DIRECTORY)) fs.mkdirSync(config.TRACES_DIRECTORY);
 
   if (args.help) {
     yargs.showHelp();

webdriver-ts/src/benchmarkRunner.ts

@@ -0,0 +1,86 @@
+import { FrameworkData } from "./common";
+
+export enum BenchmarkType {
+  CPU,
+  MEM,
+  STARTUP,
+}
+
+export enum DurationMeasurementMode {
+  FIRST_PAINT_AFTER_LAYOUT,
+  LAST_PAINT
+}
+
+export interface BenchmarkInfo {
+  id: string;
+  type: BenchmarkType;
+  label: string;
+  description: string;
+  throttleCPU?: number;
+  allowBatching: boolean;
+  durationMeasurementMode: DurationMeasurementMode;
+}
+
+export function fileName(framework: FrameworkData, benchmark: BenchmarkInfo) {
+  return `${framework.fullNameWithKeyedAndVersion}_${benchmark.id}.json`;
+}
+
+
+export const BENCHMARK_01 = "01_run1k";
+export const BENCHMARK_02 = "02_replace1k";
+export const BENCHMARK_03 = "03_update10th1k_x16";
+export const BENCHMARK_04 = "04_select1k";
+export const BENCHMARK_05 = "05_swap1k";
+export const BENCHMARK_06 = "06_remove-one-1k";
+export const BENCHMARK_07 = "07_create10k";
+export const BENCHMARK_08 = "08_create1k-after1k_x2";
+export const BENCHMARK_09 = "09_clear1k_x8";
+
+export const BENCHMARK_21 = "21_ready-memory";
+export const BENCHMARK_22 = "22_run-memory";
+export const BENCHMARK_23 = "23_update5-memory";
+export const BENCHMARK_24 = "24_run5-memory";
+export const BENCHMARK_25 = "25_run-clear-memory";
+
+export const BENCHMARK_31 = "31_startup-ci";
+
+export type TBenchmarkID =
+  | typeof BENCHMARK_01
+  | typeof BENCHMARK_02
+  | typeof BENCHMARK_03
+  | typeof BENCHMARK_04
+  | typeof BENCHMARK_05
+  | typeof BENCHMARK_06
+  | typeof BENCHMARK_07
+  | typeof BENCHMARK_08
+  | typeof BENCHMARK_09
+  | typeof BENCHMARK_31;
+
+type ISlowDowns = {
+  [key in TBenchmarkID]?: number;
+};
+
+const slowDownsOSX: ISlowDowns = {
+  [BENCHMARK_03]: 2,
+  [BENCHMARK_04]: 4,
+  [BENCHMARK_05]: 2,
+  [BENCHMARK_09]: 2,
+};
+
+const slowDownsLinux: ISlowDowns = {
+  [BENCHMARK_03]: 16,
+  [BENCHMARK_04]: 16,
+  [BENCHMARK_05]: 4,
+  [BENCHMARK_08]: 2,
+  [BENCHMARK_09]: 8,
+};
+
+const slowDowns: ISlowDowns = process.platform == "darwin" ? slowDownsOSX : slowDownsLinux;
+
+export function slowDownNote(benchmark: TBenchmarkID): string {
+  return slowDowns[benchmark] ? " " + slowDowns[benchmark] + "x CPU slowdown." : "";
+}
+
+export function slowDownFactor(benchmark: TBenchmarkID): number | undefined {
+  return slowDowns[benchmark] ? slowDowns[benchmark] : undefined;
+}