Skip to content

Commit 76619be

Browse files
krausestkrausest
committed
refactor to use puppeteer - step 1
1 parent 5f7e66c commit 76619be

15 files changed

+412
-281
lines changed

‎webdriver-ts-results/src/reducer.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ import { Benchmark, BenchmarkType, convertToMap, DisplayMode, Framework, Framewo
22
import {benchmarks as benchmark_orig, frameworks, results as rawResults} from './results';
33

44
// Temporarily disable script bootup time
5-
const benchmarks = benchmark_orig.filter(b => b.id!=='32_startup-bt');
5+
const benchmarks = benchmark_orig;
6+
// const benchmarks = benchmark_orig.filter(b => b.id!=='32_startup-bt');
67
// eslint-disable-next-line @typescript-eslint/no-explicit-any
78
// eslint-disable-next-line @typescript-eslint/no-var-requires
89
const jStat: any = require('jStat').jStat;

‎webdriver-ts/src/benchmarkConfiguration.ts

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,23 @@
11
import * as benchmarksPuppeteer from "./benchmarksPuppeteer";
22
import * as benchmarksWebdriver from "./benchmarksWebdriver";
3+
import * as benchmarksLighthouse from "./benchmarksLighthouse";
34

4-
export const benchmarks: Array<benchmarksPuppeteer.BenchmarkPuppeteer|benchmarksWebdriver.BenchmarkWebdriver> = [
5-
benchmarksWebdriver.benchRun,
6-
benchmarksWebdriver.benchReplaceAll,
7-
benchmarksWebdriver.benchUpdate,
8-
benchmarksWebdriver.benchSelect,
9-
benchmarksWebdriver.benchSwapRows,
10-
benchmarksWebdriver.benchRemove,
11-
benchmarksWebdriver.benchRunBig,
12-
benchmarksWebdriver.benchAppendToManyRows,
13-
benchmarksWebdriver.benchClear,
5+
export const benchmarks: Array<benchmarksPuppeteer.BenchmarkPuppeteer|benchmarksWebdriver.BenchmarkWebdriver|benchmarksLighthouse.BenchmarkLighthouse> = [
6+
benchmarksPuppeteer.benchRun,
7+
benchmarksPuppeteer.benchReplaceAll,
8+
benchmarksPuppeteer.benchUpdate,
9+
benchmarksPuppeteer.benchSelect,
10+
benchmarksPuppeteer.benchSwapRows,
11+
benchmarksPuppeteer.benchRemove,
12+
benchmarksPuppeteer.benchRunBig,
13+
benchmarksPuppeteer.benchAppendToManyRows,
14+
benchmarksPuppeteer.benchClear,
1415

1516
benchmarksPuppeteer.benchReadyMemory,
1617
benchmarksPuppeteer.benchRunMemory,
1718
benchmarksPuppeteer.benchUpdate5Memory,
1819
benchmarksPuppeteer.benchReplace5Memory,
1920
benchmarksPuppeteer.benchCreateClear5Memory,
2021

21-
benchmarksWebdriver.benchStartup,
22+
benchmarksLighthouse.benchLighthouse,
2223
];

‎webdriver-ts/src/benchmarkRunner.ts

Lines changed: 64 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,22 @@ import * as fs from "fs";
33
import * as yargs from "yargs";
44
import { BenchmarkInfo, BenchmarkType } from "./benchmarksCommon";
55
import { BenchmarkPuppeteer } from "./benchmarksPuppeteer";
6-
import { BenchmarkWebdriver, LighthouseData } from "./benchmarksWebdriver";
7-
import { BenchmarkOptions, config, ErrorAndWarning, FrameworkData, initializeFrameworks } from "./common";
6+
import { BenchmarkWebdriver } from "./benchmarksWebdriver";
7+
import { BenchmarkDriverOptions, BenchmarkOptions, config, ErrorAndWarning, FrameworkData, initializeFrameworks } from "./common";
88
import { writeResults } from "./writeResults";
99
import {benchmarks} from "./benchmarkConfiguration";
10+
import { BenchmarkLighthouse, StartupBenchmarkResult } from "./benchmarksLighthouse";
1011

1112
function forkAndCallBenchmark(
1213
framework: FrameworkData,
13-
benchmark: BenchmarkInfo,
14+
benchmark: BenchmarkWebdriver|BenchmarkLighthouse|BenchmarkPuppeteer,
1415
benchmarkOptions: BenchmarkOptions
1516
): Promise<ErrorAndWarning> {
1617
return new Promise((resolve, reject) => {
1718
let forkedRunner = null;
18-
if (benchmarks.some(b => b.benchmarkInfo.id == benchmark.id && b instanceof BenchmarkPuppeteer)) {
19+
if (benchmark instanceof BenchmarkLighthouse) {
20+
forkedRunner = "dist/forkedBenchmarkRunnerLighthouse.js";
21+
} else if (benchmark instanceof BenchmarkPuppeteer) {
1922
forkedRunner = "dist/forkedBenchmarkRunnerPuppeteer.js";
2023
} else {
2124
forkedRunner = "dist/forkedBenchmarkRunnerWebdriver.js";
@@ -46,6 +49,54 @@ function forkAndCallBenchmark(
4649
});
4750
}
4851

52+
async function runBenchmakLoopStartup(
53+
framework: FrameworkData,
54+
benchmark: BenchmarkLighthouse,
55+
benchmarkOptions: BenchmarkOptions
56+
): Promise<{ errors: String[]; warnings: String[] }> {
57+
if (config.FORK_CHROMEDRIVER) {
58+
let warnings: String[] = [];
59+
let errors: String[] = [];
60+
61+
let results: Array<StartupBenchmarkResult> = [];
62+
let count = benchmarkOptions.numIterationsForStartupBenchmark;
63+
benchmarkOptions.batchSize = 1;
64+
65+
let retries = 0;
66+
let done = 0;
67+
68+
while (done < count) {
69+
console.log("FORKING: ", benchmark.id, " BatchSize ", benchmarkOptions.batchSize);
70+
let res = await forkAndCallBenchmark(framework, benchmark, benchmarkOptions);
71+
if (Array.isArray(res.result)) {
72+
results = results.concat(res.result as StartupBenchmarkResult[]);
73+
} else results.push(res.result);
74+
warnings = warnings.concat(res.warnings);
75+
if (res.error) {
76+
if (res.error.indexOf("Server terminated early with status 1") > -1) {
77+
console.log("******* STRANGE selenium error found - retry #", retries + 1);
78+
retries++;
79+
if (retries == 3) break;
80+
} else {
81+
errors.push(`Executing ${framework.uri} and benchmark ${benchmark.id} failed: ` + res.error);
82+
break;
83+
}
84+
}
85+
done++;
86+
}
87+
console.log("******* result ", results);
88+
await writeResults(config, {
89+
framework: framework,
90+
benchmark: benchmark,
91+
results: results,
92+
type: BenchmarkType.STARTUP
93+
});
94+
return { errors, warnings };
95+
// } else {
96+
// return executeBenchmark(frameworks, keyed, frameworkName, benchmarkName, benchmarkOptions);
97+
}
98+
}
99+
49100
async function runBenchmakLoop(
50101
framework: FrameworkData,
51102
benchmark: BenchmarkWebdriver | BenchmarkPuppeteer,
@@ -55,7 +106,7 @@ async function runBenchmakLoop(
55106
let warnings: String[] = [];
56107
let errors: String[] = [];
57108

58-
let results: Array<number | LighthouseData> = [];
109+
let results: Array<number> = [];
59110
let count = 0;
60111

61112
if (benchmark.type == BenchmarkType.CPU) {
@@ -64,9 +115,6 @@ async function runBenchmakLoop(
64115
} else if (benchmark.type == BenchmarkType.MEM) {
65116
count = benchmarkOptions.numIterationsForMemBenchmarks;
66117
benchmarkOptions.batchSize = 1;
67-
} else {
68-
count = benchmarkOptions.numIterationsForStartupBenchmark;
69-
benchmarkOptions.batchSize = 1;
70118
}
71119

72120
let retries = 0;
@@ -75,11 +123,9 @@ async function runBenchmakLoop(
75123
benchmarkOptions.batchSize = Math.min(benchmarkOptions.batchSize, count - results.length);
76124
console.log("FORKING: ", benchmark.id, " BatchSize ", benchmarkOptions.batchSize);
77125
let res = await forkAndCallBenchmark(framework, benchmark, benchmarkOptions);
78-
if (res.result) {
79-
if (Array.isArray(res.result)) {
80-
results = results.concat(res.result);
81-
} else results.push(res.result);
82-
}
126+
if (Array.isArray(res.result)) {
127+
results = results.concat(res.result as number[]);
128+
} else results.push(res.result);
83129
warnings = warnings.concat(res.warnings);
84130
if (res.error) {
85131
if (res.error.indexOf("Server terminated early with status 1") > -1) {
@@ -104,6 +150,7 @@ async function runBenchmakLoop(
104150
framework: framework,
105151
benchmark: benchmark,
106152
results: results,
153+
type: benchmark.type as typeof BenchmarkType.CPU|BenchmarkType.MEM
107154
});
108155
return { errors, warnings };
109156
// } else {
@@ -115,7 +162,7 @@ async function runBench(runFrameworks: FrameworkData[], benchmarkNames: string[]
115162
let errors: String[] = [];
116163
let warnings: String[] = [];
117164

118-
let runBenchmarks: Array<BenchmarkWebdriver | BenchmarkPuppeteer> = benchmarks.filter((b) =>
165+
let runBenchmarks: Array<BenchmarkWebdriver | BenchmarkPuppeteer | BenchmarkLighthouse> = benchmarks.filter((b) =>
119166
benchmarkNames.some((name) => b.id.toLowerCase().indexOf(name) > -1)
120167
);
121168

@@ -151,7 +198,9 @@ async function runBench(runFrameworks: FrameworkData[], benchmarkNames: string[]
151198
for (let i = 0; i < runFrameworks.length; i++) {
152199
for (let j = 0; j < runBenchmarks.length; j++) {
153200
try {
154-
let result = await runBenchmakLoop(runFrameworks[i], runBenchmarks[j], benchmarkOptions);
201+
let result = (runBenchmarks[j].type == BenchmarkType.STARTUP) ?
202+
await runBenchmakLoopStartup(runFrameworks[i], runBenchmarks[j] as BenchmarkLighthouse, benchmarkOptions)
203+
: await runBenchmakLoop(runFrameworks[i], runBenchmarks[j] as BenchmarkPuppeteer|BenchmarkWebdriver, benchmarkOptions);
155204
errors = errors.concat(result.errors);
156205
warnings = warnings.concat(result.warnings);
157206
} catch (e) {

‎webdriver-ts/src/benchmarksCommon.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ export const BENCHMARK_23 = "23_update5-memory";
4242
export const BENCHMARK_24 = "24_run5-memory";
4343
export const BENCHMARK_25 = "25_run-clear-memory";
4444

45-
export const BENCHMARK_31 = "31_startup-ci";
45+
export const BENCHMARK_30 = "30_startup";
4646

4747
export type TBenchmarkID =
4848
| typeof BENCHMARK_01
@@ -54,7 +54,7 @@ export type TBenchmarkID =
5454
| typeof BENCHMARK_07
5555
| typeof BENCHMARK_08
5656
| typeof BENCHMARK_09
57-
| typeof BENCHMARK_31;
57+
| typeof BENCHMARK_30;
5858

5959
type ISlowDowns = {
6060
[key in TBenchmarkID]?: number;

‎webdriver-ts/src/benchmarksLighthouse.ts

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
// import { BenchmarkInfo, BenchmarkType } from "./benchmarksCommon";
2+
// import { config, FrameworkData } from "./common";
3+
// import * as benchmarksCommon from "./benchmarksCommon";
4+
// import {DurationMeasurementMode} from "./benchmarksCommon";
5+
6+
import { BenchmarkInfo, BenchmarkType, BENCHMARK_30, DurationMeasurementMode } from "./benchmarksCommon";
7+
8+
9+
// export interface LighthouseData {
10+
// TimeToConsistentlyInteractive: number;
11+
// ScriptBootUpTtime: number;
12+
// MainThreadWorkCost: number;
13+
// TotalKiloByteWeight: number;
14+
// [propName: string]: number;
15+
// }
16+
17+
// export interface StartupBenchmarkResult extends BenchmarkInfo {
18+
// property: keyof LighthouseData;
19+
// }
20+
21+
22+
export interface StartupBenchmark extends BenchmarkInfo {
23+
property: string;
24+
fn: (x:number) => number;
25+
}
26+
27+
export interface StartupBenchmarkResult extends StartupBenchmark {
28+
result: number;
29+
}
30+
31+
32+
let id = (x:number) => x;
33+
let toKb = (x:number) => x/1024;
34+
35+
export const benchStartupConsistentlyInteractive: StartupBenchmark = {
36+
id: "31_startup-ci",
37+
label: "consistently interactive",
38+
description: "a pessimistic TTI - when the CPU and network are both definitely very idle. (no more CPU tasks over 50ms)",
39+
property: "interactive",
40+
fn: id,
41+
type: BenchmarkType.STARTUP,
42+
allowBatching: false,
43+
durationMeasurementMode: DurationMeasurementMode.LAST_PAINT
44+
};
45+
46+
export const benchStartupBootup: StartupBenchmark = {
47+
id: "32_startup-bt",
48+
label: "script bootup time",
49+
description: "the total ms required to parse/compile/evaluate all the page's scripts",
50+
property: "bootup-time",
51+
fn: id,
52+
type: BenchmarkType.STARTUP,
53+
allowBatching: false,
54+
durationMeasurementMode: DurationMeasurementMode.LAST_PAINT
55+
};
56+
57+
export const benchStartupMainThreadWorkCost: StartupBenchmark = {
58+
id: "33_startup-mainthreadcost",
59+
label: "main thread work cost",
60+
description: "total amount of time spent doing work on the main thread. includes style/layout/etc.",
61+
property: "mainthread-work-breakdown",
62+
fn: id,
63+
type: BenchmarkType.STARTUP,
64+
allowBatching: false,
65+
durationMeasurementMode: DurationMeasurementMode.LAST_PAINT
66+
};
67+
68+
export const benchStartupTotalBytes: StartupBenchmark = {
69+
id: "34_startup-totalbytes",
70+
label: "total kilobyte weight",
71+
description: "network transfer cost (post-compression) of all the resources loaded into the page.",
72+
property: "total-byte-weight",
73+
fn: toKb,
74+
type: BenchmarkType.STARTUP,
75+
allowBatching: false,
76+
durationMeasurementMode: DurationMeasurementMode.LAST_PAINT
77+
};
78+
79+
export let startupBenchmarks = [benchStartupConsistentlyInteractive, benchStartupBootup, benchStartupMainThreadWorkCost, benchStartupTotalBytes];
80+
81+
export class BenchmarkLighthouse {
82+
id: string;
83+
type: BenchmarkType;
84+
label: string;
85+
description: string;
86+
throttleCPU?: number;
87+
allowBatching: boolean;
88+
durationMeasurementMode: DurationMeasurementMode;
89+
subbenchmarks = startupBenchmarks;
90+
91+
constructor(public benchmarkInfo: BenchmarkInfo) {
92+
this.id = benchmarkInfo.id;
93+
this.type = benchmarkInfo.type;
94+
this.label = benchmarkInfo.label;
95+
this.description = benchmarkInfo.description;
96+
this.throttleCPU = benchmarkInfo.throttleCPU;
97+
this.allowBatching = benchmarkInfo.allowBatching;
98+
this.durationMeasurementMode = benchmarkInfo.durationMeasurementMode;
99+
}
100+
}
101+
102+
export const benchLighthouse = new BenchmarkLighthouse({
103+
id: BENCHMARK_30,
104+
label: "Startup Metric",
105+
description: "Startup Metrics as reported from Lighthouse",
106+
type: BenchmarkType.STARTUP,
107+
allowBatching: false,
108+
durationMeasurementMode: DurationMeasurementMode.LAST_PAINT,
109+
});

‎webdriver-ts/src/benchmarksPuppeteer.ts

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,6 @@ export abstract class BenchmarkPuppeteer {
3131
after(page: Page, framework: FrameworkData): Promise<any> {
3232
return null;
3333
}
34-
// Good fit for a single result creating Benchmark
35-
resultKinds(): Array<BenchmarkInfo> {
36-
return [this.benchmarkInfo];
37-
}
38-
extractResult(results: any[], resultKind: BenchmarkInfo): number[] {
39-
return results;
40-
}
4134
}
4235

4336
export let benchRun = new class extends BenchmarkPuppeteer {
@@ -71,7 +64,7 @@ export const benchReplaceAll = new class extends BenchmarkPuppeteer {
7164
type: BenchmarkType.CPU,
7265
throttleCPU: slowDownFactor(benchmarksCommon.BENCHMARK_02),
7366
allowBatching: true,
74-
durationMeasurementMode: DurationMeasurementMode.FIRST_PAINT_AFTER_LAYOUT
67+
durationMeasurementMode: DurationMeasurementMode.LAST_PAINT
7568
});
7669
}
7770
async init(page: Page) {
@@ -97,7 +90,7 @@ export const benchUpdate = new class extends BenchmarkPuppeteer {
9790
type: BenchmarkType.CPU,
9891
throttleCPU: slowDownFactor(benchmarksCommon.BENCHMARK_03),
9992
allowBatching: true,
100-
durationMeasurementMode: DurationMeasurementMode.FIRST_PAINT_AFTER_LAYOUT
93+
durationMeasurementMode: DurationMeasurementMode.LAST_PAINT
10194
});
10295
}
10396
async init(page: Page) {
@@ -238,7 +231,7 @@ export const benchRunBig = new class extends BenchmarkPuppeteer {
238231
type: BenchmarkType.CPU,
239232
throttleCPU: slowDownFactor(benchmarksCommon.BENCHMARK_08),
240233
allowBatching: true,
241-
durationMeasurementMode: DurationMeasurementMode.FIRST_PAINT_AFTER_LAYOUT
234+
durationMeasurementMode: DurationMeasurementMode.LAST_PAINT
242235
});
243236
}
244237
async init(page: Page) {
@@ -261,7 +254,7 @@ export const benchClear = new class extends BenchmarkPuppeteer {
261254
type: BenchmarkType.CPU,
262255
throttleCPU: slowDownFactor(benchmarksCommon.BENCHMARK_09),
263256
allowBatching: true,
264-
durationMeasurementMode: DurationMeasurementMode.FIRST_PAINT_AFTER_LAYOUT
257+
durationMeasurementMode: DurationMeasurementMode.LAST_PAINT
265258
});
266259
}
267260
async init(page: Page) {

0 commit comments

Comments
 (0)