add size benchmark

krausest · krausest · commit f2ff26d4624b · 2023-11-04T20:06:16.000+01:00
diff --git a/webdriver-ts/src/benchmarkRunner.ts b/webdriver-ts/src/benchmarkRunner.ts
@@ -17,21 +17,25 @@ import {
   cpuBenchmarkInfosArray,
   CPUBenchmarkResult,
   MemBenchmarkInfo,
+  SizeBenchmarkInfo,
   StartupBenchmarkInfo,
 } from "./benchmarksCommon.js";
 import { StartupBenchmarkResult } from "./benchmarksLighthouse.js";
 import { writeResults } from "./writeResults.js";
 import { PlausibilityCheck } from "./timeline.js";
+import { SizeBenchmarkResult } from "./benchmarksSize.js";
 
 function forkAndCallBenchmark(
   framework: FrameworkData,
   benchmarkInfo: BenchmarkInfo,
   benchmarkOptions: BenchmarkOptions
-): Promise<ErrorAndWarning<number | CPUBenchmarkResult | StartupBenchmarkResult>> {
+): Promise<ErrorAndWarning<number | CPUBenchmarkResult | StartupBenchmarkResult | SizeBenchmarkResult>> {
   return new Promise((resolve, reject) => {
     let forkedRunner = null;
     if (benchmarkInfo.type === BenchmarkType.STARTUP_MAIN) {
       forkedRunner = "dist/forkedBenchmarkRunnerLighthouse.js";
+    } else if (benchmarkInfo.type === BenchmarkType.SIZE_MAIN) {
+      forkedRunner = "dist/forkedBenchmarkRunnerSize.js";
     } else if (config.BENCHMARK_RUNNER == BenchmarkRunner.WEBDRIVER_CDP) {
       forkedRunner = "dist/forkedBenchmarkRunnerWebdriverCDP.js";
     } else if (config.BENCHMARK_RUNNER == BenchmarkRunner.PLAYWRIGHT) {
@@ -119,6 +123,51 @@ async function runBenchmakLoopStartup(
   //     return executeBenchmark(frameworks, keyed, frameworkName, benchmarkName, benchmarkOptions);
 }
 
+async function runBenchmakLoopSize(
+  framework: FrameworkData,
+  benchmarkInfo: SizeBenchmarkInfo,
+  benchmarkOptions: BenchmarkOptions
+): Promise<{ errors: string[]; warnings: string[] }> {
+  let warnings: string[] = [];
+  let errors: string[] = [];
+
+  let results: Array<SizeBenchmarkResult> = [];
+  let count = benchmarkOptions.numIterationsForSizeBenchmark;
+  benchmarkOptions.batchSize = 1;
+
+  let retries = 0;
+  let done = 0;
+
+  console.log("runBenchmakLoopSize", framework, benchmarkInfo);
+
+  while (done < count) {
+    console.log("FORKING:", benchmarkInfo.id, "BatchSize", benchmarkOptions.batchSize);
+    let res = await forkAndCallBenchmark(framework, benchmarkInfo, benchmarkOptions);
+    if (Array.isArray(res.result)) {
+      results = results.concat(res.result as SizeBenchmarkResult[]);
+    } else {
+      results.push(res.result);
+    }
+    warnings = warnings.concat(res.warnings);
+    if (res.error) {
+      errors.push(`Executing ${framework.uri} and benchmark ${benchmarkInfo.id} failed: ` + res.error);
+    }
+    done++;
+  }
+  console.log("******* result", results);
+  if (config.WRITE_RESULTS) {
+    await writeResults(benchmarkOptions.resultsDirectory, {
+      framework: framework,
+      benchmark: benchmarkInfo,
+      results: results,
+      type: BenchmarkType.SIZE,
+    });
+  }
+  return { errors, warnings };
+  // } else {
+  //     return executeBenchmark(frameworks, keyed, frameworkName, benchmarkName, benchmarkOptions);
+}
+
 async function runBenchmakLoop(
   framework: FrameworkData,
   benchmarkInfo: CPUBenchmarkInfo | MemBenchmarkInfo,
@@ -233,6 +282,12 @@ async function runBench(
             benchmarkInfos[j] as StartupBenchmarkInfo,
             benchmarkOptions
           );
+        } else if (benchmarkInfos[j].type == BenchmarkType.SIZE_MAIN) {
+          result = await runBenchmakLoopSize(
+            runFrameworks[i],
+            benchmarkInfos[j] as SizeBenchmarkInfo,
+            benchmarkOptions
+          );
         } else if (benchmarkInfos[j].type == BenchmarkType.CPU) {
           result = await runBenchmakLoop(
             runFrameworks[i],
@@ -345,6 +400,7 @@ async function main() {
       config.NUM_ITERATIONS_FOR_BENCHMARK_CPU + config.NUM_ITERATIONS_FOR_BENCHMARK_CPU_DROP_SLOWEST_COUNT,
     numIterationsForMemBenchmarks: config.NUM_ITERATIONS_FOR_BENCHMARK_MEM,
     numIterationsForStartupBenchmark: config.NUM_ITERATIONS_FOR_BENCHMARK_STARTUP,
+    numIterationsForSizeBenchmark: config.NUM_ITERATIONS_FOR_BENCHMARK_SIZE,
     batchSize: 1,
     resultsDirectory: "results",
     tracesDirectory: "traces",
diff --git a/webdriver-ts/src/benchmarksCommon.ts b/webdriver-ts/src/benchmarksCommon.ts
@@ -5,6 +5,8 @@ export enum BenchmarkType {
   MEM,
   STARTUP_MAIN,
   STARTUP,
+  SIZE_MAIN,
+  SIZE,
 }
 
 export interface BenchmarkInfoBase {
@@ -25,6 +27,19 @@ export interface MemBenchmarkInfo extends BenchmarkInfoBase {
   type: BenchmarkType.MEM;
 }
 
+export interface SizeInfoJSON {
+  size_uncompressed: number,
+  size_compressed: number,
+}
+
+export interface SizeBenchmarkInfo extends BenchmarkInfoBase {
+  type: BenchmarkType.SIZE;
+  fn(sizeInfo: SizeInfoJSON): number;
+}
+
+export interface SizeMainBenchmarkInfo extends BenchmarkInfoBase {
+  type: BenchmarkType.SIZE_MAIN;
+}
 export interface StartupMainBenchmarkInfo extends BenchmarkInfoBase {
   type: BenchmarkType.STARTUP_MAIN;
 }
@@ -35,7 +50,7 @@ export interface StartupBenchmarkInfo extends BenchmarkInfoBase {
   fn: (x: number) => number;
 }
 
-export type BenchmarkInfo = CPUBenchmarkInfo | MemBenchmarkInfo | StartupMainBenchmarkInfo | StartupBenchmarkInfo;
+export type BenchmarkInfo = CPUBenchmarkInfo | MemBenchmarkInfo | StartupMainBenchmarkInfo | StartupBenchmarkInfo | SizeBenchmarkInfo | SizeMainBenchmarkInfo;
 
 export interface BenchmarkImpl {
   benchmarkInfo: BenchmarkInfo;
@@ -68,6 +83,7 @@ export enum Benchmark {
   _25 = "25_run-clear-memory",
   _26 = "26_run-10k-memory",
   _30 = "30_startup",
+  _40 = "40_sizes",
 }
 
 export type BenchmarkId =
@@ -80,7 +96,8 @@ export type BenchmarkId =
   | typeof Benchmark._07
   | typeof Benchmark._08
   | typeof Benchmark._09
-  | typeof Benchmark._30;
+  | typeof Benchmark._30
+  | typeof Benchmark._40;
 
 const throttlingFactors: { [idx: string]: number } = {
   [Benchmark._03]: 4,
@@ -237,6 +254,15 @@ export const startupBenchmarkInfosArray: Array<StartupMainBenchmarkInfo> = [
     type: BenchmarkType.STARTUP_MAIN,
     label: "",
     description: () => "",
+  }
+];
+
+export const sizesBenchmarkInfosArray: Array<SizeMainBenchmarkInfo> = [
+  {
+    id: Benchmark._40,
+    type: BenchmarkType.SIZE_MAIN,
+    label: "",
+    description: () => "",
   },
 ];
 
@@ -255,4 +281,9 @@ for (let bi of startupBenchmarkInfosArray) {
   startupBenchmarkInfos[bi.id] = bi;
 }
 
-export const benchmarkInfos = [...cpuBenchmarkInfosArray, ...memBenchmarkInfosArray, ...startupBenchmarkInfosArray];
+export const sizeBenchmarkInfos: { [idx: string]: SizeMainBenchmarkInfo } = {};
+for (let bi of sizesBenchmarkInfosArray) {
+  sizeBenchmarkInfos[bi.id] = bi;
+}
+
+export const benchmarkInfos = [...cpuBenchmarkInfosArray, ...memBenchmarkInfosArray, ...startupBenchmarkInfosArray, ...sizesBenchmarkInfosArray];
diff --git a/webdriver-ts/src/benchmarksLighthouse.ts b/webdriver-ts/src/benchmarksLighthouse.ts
@@ -1,7 +1,7 @@
 import * as benchmarksCommon from "./benchmarksCommon.js";
 import { BenchmarkImpl, BenchmarkType, StartupBenchmarkInfo } from "./benchmarksCommon.js";
 
-export interface StartupBenchmarkResult extends BenchmarkImpl {
+export interface StartupBenchmarkResult {
   benchmark: StartupBenchmarkInfo;
   result: number;
 }
diff --git a/webdriver-ts/src/common.ts b/webdriver-ts/src/common.ts
@@ -38,6 +38,7 @@ export interface BenchmarkOptions {
   numIterationsForCPUBenchmarks: number;
   numIterationsForMemBenchmarks: number;
   numIterationsForStartupBenchmark: number;
+  numIterationsForSizeBenchmark: number;
 
   allowThrottling: boolean;
   resultsDirectory: string;
@@ -64,6 +65,7 @@ export let config = {
   NUM_ITERATIONS_FOR_BENCHMARK_CPU_DROP_SLOWEST_COUNT: 0, // drop the # of slowest results
   NUM_ITERATIONS_FOR_BENCHMARK_MEM: 1,
   NUM_ITERATIONS_FOR_BENCHMARK_STARTUP: 1,
+  NUM_ITERATIONS_FOR_BENCHMARK_SIZE: 1,
   WARMUP_COUNT: 5,
   TIMEOUT: 60 * 1000,
   LOG_PROGRESS: true,
diff --git a/webdriver-ts/src/createResultJS.ts b/webdriver-ts/src/createResultJS.ts
@@ -34,6 +34,7 @@ let benchmarkOptions: BenchmarkOptions = {
     config.NUM_ITERATIONS_FOR_BENCHMARK_CPU_DROP_SLOWEST_COUNT,
   numIterationsForMemBenchmarks: config.NUM_ITERATIONS_FOR_BENCHMARK_MEM,
   numIterationsForStartupBenchmark: config.NUM_ITERATIONS_FOR_BENCHMARK_STARTUP,
+  numIterationsForSizeBenchmark: config.NUM_ITERATIONS_FOR_BENCHMARK_SIZE,
   batchSize: 1,
   resultsDirectory: "results",
   tracesDirectory: "traces",
diff --git a/webdriver-ts/src/forkedBenchmarkRunnerLighthouse.ts b/webdriver-ts/src/forkedBenchmarkRunnerLighthouse.ts
@@ -61,12 +61,10 @@ async function runLighthouse(
     }
     if (config.LOG_DEBUG) console.log("lighthouse result", results);
 
-    return startupBenchmarks.map((bench) => {
-      return {
+    return startupBenchmarks.map((bench) => ({
         benchmark: bench,
         result: bench.fn(extractRawValue(results.lhr, bench.property)),
-      } as StartupBenchmarkResult;
-    });
+    }));
   } catch (error) {
     console.log("error running lighthouse", error);
     throw error;
diff --git a/webdriver-ts/src/isCSPCompliant.ts b/webdriver-ts/src/isCSPCompliant.ts
@@ -29,6 +29,7 @@ let benchmarkOptions: BenchmarkOptions = {
     config.NUM_ITERATIONS_FOR_BENCHMARK_CPU + config.NUM_ITERATIONS_FOR_BENCHMARK_CPU_DROP_SLOWEST_COUNT,
   numIterationsForMemBenchmarks: config.NUM_ITERATIONS_FOR_BENCHMARK_MEM,
   numIterationsForStartupBenchmark: config.NUM_ITERATIONS_FOR_BENCHMARK_STARTUP,
+  numIterationsForSizeBenchmark: config.NUM_ITERATIONS_FOR_BENCHMARK_SIZE,
   batchSize: 1,
   resultsDirectory: "results",
   tracesDirectory: "traces",
diff --git a/webdriver-ts/src/isKeyed.ts b/webdriver-ts/src/isKeyed.ts
@@ -32,6 +32,7 @@ let benchmarkOptions: BenchmarkOptions = {
     config.NUM_ITERATIONS_FOR_BENCHMARK_CPU + config.NUM_ITERATIONS_FOR_BENCHMARK_CPU_DROP_SLOWEST_COUNT,
   numIterationsForMemBenchmarks: config.NUM_ITERATIONS_FOR_BENCHMARK_MEM,
   numIterationsForStartupBenchmark: config.NUM_ITERATIONS_FOR_BENCHMARK_STARTUP,
+  numIterationsForSizeBenchmark: config.NUM_ITERATIONS_FOR_BENCHMARK_SIZE,
   batchSize: 1,
   resultsDirectory: "results",
   tracesDirectory: "traces",
diff --git a/webdriver-ts/src/parseTrace.ts b/webdriver-ts/src/parseTrace.ts
@@ -35,6 +35,7 @@ async function debugAll() {
       config.NUM_ITERATIONS_FOR_BENCHMARK_CPU + config.NUM_ITERATIONS_FOR_BENCHMARK_CPU_DROP_SLOWEST_COUNT,
     numIterationsForMemBenchmarks: config.NUM_ITERATIONS_FOR_BENCHMARK_MEM,
     numIterationsForStartupBenchmark: config.NUM_ITERATIONS_FOR_BENCHMARK_STARTUP,
+    numIterationsForSizeBenchmark: config.NUM_ITERATIONS_FOR_BENCHMARK_SIZE,
     batchSize: 1,
     resultsDirectory: "results",
     tracesDirectory: "traces",
diff --git a/webdriver-ts/src/writeResults.ts b/webdriver-ts/src/writeResults.ts
@@ -1,13 +1,14 @@
 import * as fs from "node:fs";
 import { BenchmarkInfo, BenchmarkType, CPUBenchmarkResult, fileName } from "./benchmarksCommon.js";
-import { StartupBenchmarkResult, subbenchmarks } from "./benchmarksLighthouse.js";
+import * as benchmarksLighthouse from "./benchmarksLighthouse.js";
+import * as benchmarksSize from "./benchmarksSize.js";
 import { FrameworkData, JsonResult, JsonResultData } from "./common.js";
 import { stats } from "./stats.js";
 
 export type ResultLightHouse = {
   framework: FrameworkData;
   benchmark: BenchmarkInfo;
-  results: StartupBenchmarkResult[];
+  results: benchmarksLighthouse.StartupBenchmarkResult[];
   type: BenchmarkType.STARTUP;
 };
 
@@ -25,10 +26,23 @@ export type ResultMem = {
   type: BenchmarkType.MEM;
 };
 
-export function writeResults(resultDir: string, res: ResultLightHouse | ResultCPU | ResultMem) {
+export type ResultSize = {
+  framework: FrameworkData;
+  benchmark: BenchmarkInfo;
+  results: benchmarksSize.SizeBenchmarkResult[];
+  type: BenchmarkType.SIZE;
+};
+
+export function writeResults(resultDir: string, res: ResultLightHouse | ResultCPU | ResultMem | ResultSize) {
   switch (res.type) {
     case BenchmarkType.STARTUP:
-      for (let subbench of subbenchmarks) {
+      for (let subbench of benchmarksLighthouse.subbenchmarks) {
+        let results = res.results.filter((r) => r.benchmark.id == subbench.id).map((r) => r.result);
+        createResultFile(resultDir, results, res.framework, subbench);
+      }
+      break;
+    case BenchmarkType.SIZE:
+      for (let subbench of benchmarksSize.subbenchmarks) {
         let results = res.results.filter((r) => r.benchmark.id == subbench.id).map((r) => r.result);
         createResultFile(resultDir, results, res.framework, subbench);
       }
@@ -64,6 +78,9 @@ function createResultFile(
     case BenchmarkType.STARTUP:
       type = "startup";
       break;
+    case BenchmarkType.SIZE:
+      type = "size";
+      break;
   }
   let convertResult = (label: string, data: number[]) => {
     let res = stats(data);

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`import * as benchmarksCommon from "./benchmarksCommon.js";`
`2`	`2`	`import { BenchmarkImpl, BenchmarkType, StartupBenchmarkInfo } from "./benchmarksCommon.js";`
`3`	`3`
`4`		`-export interface StartupBenchmarkResult extends BenchmarkImpl {`
	`4`	`+export interface StartupBenchmarkResult {`
`5`	`5`	`benchmark: StartupBenchmarkInfo;`
`6`	`6`	`result: number;`
`7`	`7`	`}`