Skip to content

Commit 8481752

Browse files
krausestkrausest
committed
use puppeteer for benchmarking
1 parent 76619be commit 8481752

18 files changed

+3587
-3428
lines changed

‎webdriver-ts-results/src/Common.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ export const knownIssues = [
4343
export function findIssue(issueNumber: number) {
4444
return knownIssues.find(i => i.issue === issueNumber)
4545
}
46-
export enum BenchmarkType { CPU, MEM, STARTUP }
46+
export enum BenchmarkType { CPU, MEM, DUMMY, STARTUP }
4747

4848
export interface Benchmark {
4949
id: string;

‎webdriver-ts-results/src/reducer.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ import { Benchmark, BenchmarkType, convertToMap, DisplayMode, Framework, Framewo
22
import {benchmarks as benchmark_orig, frameworks, results as rawResults} from './results';
33

44
// Temporarily disable script bootup time
5-
const benchmarks = benchmark_orig;
6-
// const benchmarks = benchmark_orig.filter(b => b.id!=='32_startup-bt');
5+
//const benchmarks = benchmark_orig;
6+
const benchmarks = benchmark_orig.filter(b => b.id!=='32_startup-bt');
77
// eslint-disable-next-line @typescript-eslint/no-explicit-any
88
// eslint-disable-next-line @typescript-eslint/no-var-requires
99
const jStat: any = require('jStat').jStat;

‎webdriver-ts-results/src/results.ts

Lines changed: 3331 additions & 3146 deletions
Large diffs are not rendered by default.

‎webdriver-ts-results/table.html

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎webdriver-ts/results.json

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎webdriver-ts/src/benchmarkConfiguration.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ import * as benchmarksPuppeteer from "./benchmarksPuppeteer";
22
import * as benchmarksWebdriver from "./benchmarksWebdriver";
33
import * as benchmarksLighthouse from "./benchmarksLighthouse";
44

5-
export const benchmarks: Array<benchmarksPuppeteer.BenchmarkPuppeteer|benchmarksWebdriver.BenchmarkWebdriver|benchmarksLighthouse.BenchmarkLighthouse> = [
5+
export type TBenchmark = benchmarksWebdriver.CPUBenchmarkWebdriver | benchmarksPuppeteer.TBenchmarkPuppeteer | benchmarksLighthouse.BenchmarkLighthouse;
6+
7+
export const benchmarks: Array<TBenchmark> = [
68
benchmarksPuppeteer.benchRun,
79
benchmarksPuppeteer.benchReplaceAll,
810
benchmarksPuppeteer.benchUpdate,
@@ -20,4 +22,4 @@ export const benchmarks: Array<benchmarksPuppeteer.BenchmarkPuppeteer|benchmarks
2022
benchmarksPuppeteer.benchCreateClear5Memory,
2123

2224
benchmarksLighthouse.benchLighthouse,
23-
];
25+
];

‎webdriver-ts/src/benchmarkRunner.ts

Lines changed: 100 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,23 @@ import { fork } from "child_process";
22
import * as fs from "fs";
33
import * as yargs from "yargs";
44
import { BenchmarkInfo, BenchmarkType } from "./benchmarksCommon";
5-
import { BenchmarkPuppeteer } from "./benchmarksPuppeteer";
6-
import { BenchmarkWebdriver } from "./benchmarksWebdriver";
5+
import { CPUBenchmarkPuppeteer, MemBenchmarkPuppeteer, TBenchmarkPuppeteer } from "./benchmarksPuppeteer";
6+
import { CPUBenchmarkWebdriver } from "./benchmarksWebdriver";
77
import { BenchmarkDriverOptions, BenchmarkOptions, config, ErrorAndWarning, FrameworkData, initializeFrameworks } from "./common";
88
import { writeResults } from "./writeResults";
99
import {benchmarks} from "./benchmarkConfiguration";
1010
import { BenchmarkLighthouse, StartupBenchmarkResult } from "./benchmarksLighthouse";
1111

1212
function forkAndCallBenchmark(
1313
framework: FrameworkData,
14-
benchmark: BenchmarkWebdriver|BenchmarkLighthouse|BenchmarkPuppeteer,
14+
benchmark: CPUBenchmarkWebdriver|TBenchmarkPuppeteer|BenchmarkLighthouse,
1515
benchmarkOptions: BenchmarkOptions
1616
): Promise<ErrorAndWarning> {
1717
return new Promise((resolve, reject) => {
1818
let forkedRunner = null;
1919
if (benchmark instanceof BenchmarkLighthouse) {
2020
forkedRunner = "dist/forkedBenchmarkRunnerLighthouse.js";
21-
} else if (benchmark instanceof BenchmarkPuppeteer) {
21+
} else if (benchmark instanceof CPUBenchmarkPuppeteer || benchmark instanceof MemBenchmarkPuppeteer) {
2222
forkedRunner = "dist/forkedBenchmarkRunnerPuppeteer.js";
2323
} else {
2424
forkedRunner = "dist/forkedBenchmarkRunnerWebdriver.js";
@@ -29,7 +29,7 @@ function forkAndCallBenchmark(
2929
forked.send({
3030
config,
3131
framework,
32-
benchmarkId: benchmark.id,
32+
benchmarkId: benchmark.benchmarkInfo.id,
3333
benchmarkOptions,
3434
});
3535
forked.on("message", async (msg: ErrorAndWarning) => {
@@ -54,116 +54,118 @@ async function runBenchmakLoopStartup(
5454
benchmark: BenchmarkLighthouse,
5555
benchmarkOptions: BenchmarkOptions
5656
): Promise<{ errors: String[]; warnings: String[] }> {
57-
if (config.FORK_CHROMEDRIVER) {
58-
let warnings: String[] = [];
59-
let errors: String[] = [];
57+
let warnings: String[] = [];
58+
let errors: String[] = [];
59+
60+
let results: Array<StartupBenchmarkResult> = [];
61+
let count = benchmarkOptions.numIterationsForStartupBenchmark;
62+
benchmarkOptions.batchSize = 1;
63+
64+
let retries = 0;
65+
let done = 0;
66+
67+
console.log("runBenchmakLoopStartup", framework, benchmark);
6068

61-
let results: Array<StartupBenchmarkResult> = [];
62-
let count = benchmarkOptions.numIterationsForStartupBenchmark;
63-
benchmarkOptions.batchSize = 1;
6469

65-
let retries = 0;
66-
let done = 0;
67-
68-
while (done < count) {
69-
console.log("FORKING: ", benchmark.id, " BatchSize ", benchmarkOptions.batchSize);
70-
let res = await forkAndCallBenchmark(framework, benchmark, benchmarkOptions);
71-
if (Array.isArray(res.result)) {
72-
results = results.concat(res.result as StartupBenchmarkResult[]);
73-
} else results.push(res.result);
74-
warnings = warnings.concat(res.warnings);
75-
if (res.error) {
76-
if (res.error.indexOf("Server terminated early with status 1") > -1) {
77-
console.log("******* STRANGE selenium error found - retry #", retries + 1);
78-
retries++;
79-
if (retries == 3) break;
80-
} else {
81-
errors.push(`Executing ${framework.uri} and benchmark ${benchmark.id} failed: ` + res.error);
82-
break;
83-
}
70+
while (done < count) {
71+
console.log("FORKING: ", benchmark.benchmarkInfo.id, " BatchSize ", benchmarkOptions.batchSize);
72+
let res = await forkAndCallBenchmark(framework, benchmark, benchmarkOptions);
73+
if (Array.isArray(res.result)) {
74+
results = results.concat(res.result as StartupBenchmarkResult[]);
75+
} else results.push(res.result);
76+
warnings = warnings.concat(res.warnings);
77+
if (res.error) {
78+
if (res.error.indexOf("Server terminated early with status 1") > -1) {
79+
console.log("******* STRANGE selenium error found - retry #", retries + 1);
80+
retries++;
81+
if (retries == 3) break;
82+
} else {
83+
errors.push(`Executing ${framework.uri} and benchmark ${benchmark.benchmarkInfo.id} failed: ` + res.error);
84+
break;
8485
}
85-
done++;
8686
}
87-
console.log("******* result ", results);
88-
await writeResults(config, {
89-
framework: framework,
90-
benchmark: benchmark,
91-
results: results,
92-
type: BenchmarkType.STARTUP
93-
});
94-
return { errors, warnings };
95-
// } else {
96-
// return executeBenchmark(frameworks, keyed, frameworkName, benchmarkName, benchmarkOptions);
87+
done++;
9788
}
89+
console.log("******* result ", results);
90+
await writeResults(config, {
91+
framework: framework,
92+
benchmark: benchmark,
93+
results: results,
94+
type: BenchmarkType.STARTUP
95+
});
96+
return { errors, warnings };
97+
// } else {
98+
// return executeBenchmark(frameworks, keyed, frameworkName, benchmarkName, benchmarkOptions);
9899
}
99100

100101
async function runBenchmakLoop(
101102
framework: FrameworkData,
102-
benchmark: BenchmarkWebdriver | BenchmarkPuppeteer,
103+
benchmark: CPUBenchmarkWebdriver | TBenchmarkPuppeteer,
103104
benchmarkOptions: BenchmarkOptions
104105
): Promise<{ errors: String[]; warnings: String[] }> {
105-
if (config.FORK_CHROMEDRIVER) {
106-
let warnings: String[] = [];
107-
let errors: String[] = [];
108-
109-
let results: Array<number> = [];
110-
let count = 0;
111-
112-
if (benchmark.type == BenchmarkType.CPU) {
113-
count = benchmarkOptions.numIterationsForCPUBenchmarks;
114-
benchmarkOptions.batchSize = config.ALLOW_BATCHING && benchmark.allowBatching ? count : 1;
115-
} else if (benchmark.type == BenchmarkType.MEM) {
116-
count = benchmarkOptions.numIterationsForMemBenchmarks;
117-
benchmarkOptions.batchSize = 1;
118-
}
106+
let warnings: String[] = [];
107+
let errors: String[] = [];
108+
109+
let results: Array<number> = [];
110+
let count = 0;
119111

120-
let retries = 0;
121-
122-
while (results.length < count) {
123-
benchmarkOptions.batchSize = Math.min(benchmarkOptions.batchSize, count - results.length);
124-
console.log("FORKING: ", benchmark.id, " BatchSize ", benchmarkOptions.batchSize);
125-
let res = await forkAndCallBenchmark(framework, benchmark, benchmarkOptions);
126-
if (Array.isArray(res.result)) {
127-
results = results.concat(res.result as number[]);
128-
} else results.push(res.result);
129-
warnings = warnings.concat(res.warnings);
130-
if (res.error) {
131-
if (res.error.indexOf("Server terminated early with status 1") > -1) {
132-
console.log("******* STRANGE selenium error found - retry #", retries + 1);
133-
retries++;
134-
if (retries == 3) break;
135-
} else {
136-
errors.push(`Executing ${framework.uri} and benchmark ${benchmark.id} failed: ` + res.error);
137-
break;
138-
}
112+
if (benchmark.type == BenchmarkType.CPU) {
113+
count = benchmarkOptions.numIterationsForCPUBenchmarks;
114+
// FIXME
115+
benchmarkOptions.batchSize = config.ALLOW_BATCHING && (benchmark.benchmarkInfo as any).allowBatching ? count : 1;
116+
} else if (benchmark.type == BenchmarkType.MEM) {
117+
count = benchmarkOptions.numIterationsForMemBenchmarks;
118+
benchmarkOptions.batchSize = 1;
119+
}
120+
121+
let retries = 0;
122+
123+
console.log("runBenchmakLoop", framework, benchmark);
124+
125+
while (results.length < count) {
126+
benchmarkOptions.batchSize = Math.min(benchmarkOptions.batchSize, count - results.length);
127+
console.log("FORKING: ", benchmark.benchmarkInfo.id, " BatchSize ", benchmarkOptions.batchSize);
128+
let res = await forkAndCallBenchmark(framework, benchmark, benchmarkOptions);
129+
if (Array.isArray(res.result)) {
130+
results = results.concat(res.result as number[]);
131+
} else results.push(res.result);
132+
warnings = warnings.concat(res.warnings);
133+
if (res.error) {
134+
if (res.error.indexOf("Server terminated early with status 1") > -1) {
135+
console.log("******* STRANGE selenium error found - retry #", retries + 1);
136+
retries++;
137+
if (retries == 3) break;
138+
} else {
139+
errors.push(`Executing ${framework.uri} and benchmark ${benchmark.benchmarkInfo.id} failed: ` + res.error);
140+
break;
139141
}
140142
}
141-
if (benchmark.type == BenchmarkType.CPU) {
142-
console.log("CPU results before: ", results);
143-
(results as number[]).sort((a: number, b: number) => a - b);
144-
results = results.slice(0, config.NUM_ITERATIONS_FOR_BENCHMARK_CPU);
145-
// console.log("CPU results after: ", results)
146-
}
147-
148-
console.log("******* result ", results);
149-
await writeResults(config, {
150-
framework: framework,
151-
benchmark: benchmark,
152-
results: results,
153-
type: benchmark.type as typeof BenchmarkType.CPU|BenchmarkType.MEM
154-
});
155-
return { errors, warnings };
156-
// } else {
157-
// return executeBenchmark(frameworks, keyed, frameworkName, benchmarkName, benchmarkOptions);
158143
}
144+
if (benchmark.type == BenchmarkType.CPU) {
145+
console.log("CPU results before: ", results);
146+
(results as number[]).sort((a: number, b: number) => a - b);
147+
results = results.slice(0, config.NUM_ITERATIONS_FOR_BENCHMARK_CPU);
148+
// console.log("CPU results after: ", results)
149+
}
150+
151+
console.log("******* result ", results);
152+
await writeResults(config, {
153+
framework: framework,
154+
benchmark: benchmark.benchmarkInfo,
155+
results: results,
156+
type: benchmark.type as typeof BenchmarkType.CPU|BenchmarkType.MEM
157+
});
158+
return { errors, warnings };
159+
// } else {
160+
// return executeBenchmark(frameworks, keyed, frameworkName, benchmarkName, benchmarkOptions);
159161
}
160162

161163
async function runBench(runFrameworks: FrameworkData[], benchmarkNames: string[]) {
162164
let errors: String[] = [];
163165
let warnings: String[] = [];
164166

165-
let runBenchmarks: Array<BenchmarkWebdriver | BenchmarkPuppeteer | BenchmarkLighthouse> = benchmarks.filter((b) =>
166-
benchmarkNames.some((name) => b.id.toLowerCase().indexOf(name) > -1)
167+
let runBenchmarks: Array<CPUBenchmarkWebdriver | TBenchmarkPuppeteer | BenchmarkLighthouse> = benchmarks.filter((b) =>
168+
benchmarkNames.some((name) => b.benchmarkInfo.id.toLowerCase().indexOf(name) > -1)
167169
);
168170

169171
let restart: string = undefined;
@@ -178,7 +180,7 @@ async function runBench(runFrameworks: FrameworkData[], benchmarkNames: string[]
178180
);
179181
console.log(
180182
"Benchmarks that will be run",
181-
runBenchmarks.map((b) => b.id)
183+
runBenchmarks.map((b) => b.benchmarkInfo.id)
182184
);
183185

184186
console.log("HEADLESS*** ", args.headless);
@@ -198,9 +200,10 @@ async function runBench(runFrameworks: FrameworkData[], benchmarkNames: string[]
198200
for (let i = 0; i < runFrameworks.length; i++) {
199201
for (let j = 0; j < runBenchmarks.length; j++) {
200202
try {
201-
let result = (runBenchmarks[j].type == BenchmarkType.STARTUP) ?
203+
console.log("****** runBenchmarks[j].type", runBenchmarks[j].type, runBenchmarks[j].type == BenchmarkType.STARTUP_MAIN)
204+
let result = (runBenchmarks[j].type == BenchmarkType.STARTUP_MAIN) ?
202205
await runBenchmakLoopStartup(runFrameworks[i], runBenchmarks[j] as BenchmarkLighthouse, benchmarkOptions)
203-
: await runBenchmakLoop(runFrameworks[i], runBenchmarks[j] as BenchmarkPuppeteer|BenchmarkWebdriver, benchmarkOptions);
206+
: await runBenchmakLoop(runFrameworks[i], runBenchmarks[j] as TBenchmarkPuppeteer|CPUBenchmarkWebdriver, benchmarkOptions);
204207
errors = errors.concat(result.errors);
205208
warnings = warnings.concat(result.warnings);
206209
} catch (e) {

‎webdriver-ts/src/benchmarksCommon.ts

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { FrameworkData } from "./common";
33
export enum BenchmarkType {
44
CPU,
55
MEM,
6+
STARTUP_MAIN,
67
STARTUP,
78
}
89

@@ -13,12 +14,37 @@ export enum DurationMeasurementMode {
1314

1415
export interface BenchmarkInfo {
1516
id: string;
16-
type: BenchmarkType;
1717
label: string;
1818
description: string;
19+
type: BenchmarkType;
20+
}
21+
22+
export interface CPUBenchmark extends BenchmarkInfo {
1923
throttleCPU?: number;
2024
allowBatching: boolean;
2125
durationMeasurementMode: DurationMeasurementMode;
26+
type: BenchmarkType.CPU;
27+
}
28+
29+
export interface MemBenchmark extends BenchmarkInfo {
30+
type: BenchmarkType.MEM;
31+
}
32+
33+
export interface StartupMainBenchmark extends BenchmarkInfo {
34+
type: BenchmarkType.STARTUP_MAIN;
35+
}
36+
37+
export interface StartupBenchmark extends BenchmarkInfo {
38+
type: BenchmarkType.STARTUP;
39+
property: string;
40+
fn: (x:number) => number;
41+
}
42+
43+
export type TBenchmark = CPUBenchmark | MemBenchmark | StartupMainBenchmark // |StartupBenchmark;
44+
45+
export interface BenchmarkImpl {
46+
benchmarkInfo: TBenchmark;
47+
type: BenchmarkType;
2248
}
2349

2450
export function fileName(framework: FrameworkData, benchmark: BenchmarkInfo) {

0 commit comments

Comments
 (0)