Skip to content

Commit 77b6f85

Browse files
krausestkrausest
committed
refactor for afterframe
1 parent 518cad2 commit 77b6f85

10 files changed

+57
-50
lines changed

‎webdriver-ts/src/benchmarkArgs.ts

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import yargs from 'yargs';
2+
3+
// FIXME: Clean up args.
4+
// What works: npm run bench keyed/react, npm run bench -- keyed/react, npm run bench -- keyed/react --count 1 --benchmark 01_
5+
// What doesn't work (keyed/react becomes an element of argument benchmark): npm run bench -- --count 1 --benchmark 01_ keyed/react
6+
7+
export let args: any = yargs(process.argv)
8+
.usage(
9+
"$0 [--framework Framework1 Framework2 ...] [--benchmark Benchmark1 Benchmark2 ...] [--chromeBinary path] \n or: $0 [directory1] [directory2] .. [directory3]"
10+
)
11+
.help("help")
12+
.boolean("headless").default("headless", false)
13+
.boolean("smoketest")
14+
.boolean("nothrottling").default("nothrottling", false)
15+
.string("runner").default("runner","puppeteer")
16+
.string("browser").default("browser","chrome")
17+
.array("framework")
18+
.array("benchmark")
19+
.string("chromeBinary").argv;
20+
21+
console.log("args", args);

‎webdriver-ts/src/benchmarkRunner.ts

Lines changed: 7 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
1+
import {args} from './benchmarkArgs.js';
2+
import { BenchmarkOptions, BENCHMARK_RUNNER, config, ErrorAndWarning, FrameworkData, initializeFrameworks } from "./common.js";
13
import { fork } from "child_process";
24
import * as fs from "fs";
3-
import yargs from 'yargs';
45
import { BenchmarkInfo, benchmarkInfos, BenchmarkType, CPUBenchmarkInfo, MemBenchmarkInfo, StartupBenchmarkInfo } from "./benchmarksCommon.js";
56
import { StartupBenchmarkResult } from "./benchmarksLighthouse.js";
6-
import { BenchmarkOptions, BENCHMARK_RUNNER, config, ErrorAndWarning, FrameworkData, initializeFrameworks } from "./common.js";
77
import { writeResults } from "./writeResults.js";
88

99
function forkAndCallBenchmark(
1010
framework: FrameworkData,
1111
benchmarkInfo: BenchmarkInfo,
1212
benchmarkOptions: BenchmarkOptions
13-
): Promise<ErrorAndWarning> {
13+
): Promise<ErrorAndWarning<number|StartupBenchmarkResult>> {
1414
return new Promise((resolve, reject) => {
1515
let forkedRunner = null;
1616
if (benchmarkInfo.type === BenchmarkType.STARTUP_MAIN) {
@@ -35,7 +35,7 @@ function forkAndCallBenchmark(
3535
benchmarkId: benchmarkInfo.id,
3636
benchmarkOptions,
3737
});
38-
forked.on("message", (msg: ErrorAndWarning) => {
38+
forked.on("message", (msg: ErrorAndWarning<number|StartupBenchmarkResult>) => {
3939
if (config.LOG_DETAILS) console.log("FORKING: main process got message from child", msg);
4040
resolve(msg);
4141
});
@@ -243,23 +243,6 @@ async function runBench(runFrameworks: FrameworkData[], benchmarkInfos: Benchmar
243243
}
244244
}
245245

246-
// FIXME: Clean up args.
247-
// What works: npm run bench keyed/react, npm run bench -- keyed/react, npm run bench -- keyed/react --count 1 --benchmark 01_
248-
// What doesn't work (keyed/react becomes an element of argument benchmark): npm run bench -- --count 1 --benchmark 01_ keyed/react
249-
250-
let args: any = yargs(process.argv)
251-
.usage(
252-
"$0 [--framework Framework1 Framework2 ...] [--benchmark Benchmark1 Benchmark2 ...] [--chromeBinary path] \n or: $0 [directory1] [directory2] .. [directory3]"
253-
)
254-
.help("help")
255-
.boolean("headless").default("headless", false)
256-
.boolean("smoketest")
257-
.string("runner").default("runner",config.BENCHMARK_RUNNER)
258-
.string("browser").default("browser",config.BROWSER)
259-
.array("framework")
260-
.array("benchmark")
261-
.string("chromeBinary").argv;
262-
263246
let runner = args.runner;
264247
if ([BENCHMARK_RUNNER.WEBDRIVER_CDP,BENCHMARK_RUNNER.WEBDRIVER,BENCHMARK_RUNNER.WEBDRIVER_AFTERFRAME,
265248
BENCHMARK_RUNNER.PLAYWRIGHT,
@@ -305,7 +288,9 @@ async function main() {
305288
config.EXIT_ON_ERROR = true;
306289
console.log('Using smoketest config ', JSON.stringify(config));
307290
}
308-
291+
if (config.BENCHMARK_RUNNER == BENCHMARK_RUNNER.WEBDRIVER_AFTERFRAME) {
292+
config.RESULTS_DIRECTORY = "results_client_"+config.BROWSER;
293+
}
309294
if (!fs.existsSync(config.RESULTS_DIRECTORY)) fs.mkdirSync(config.RESULTS_DIRECTORY);
310295
if (!fs.existsSync(config.TRACES_DIRECTORY)) fs.mkdirSync(config.TRACES_DIRECTORY);
311296

‎webdriver-ts/src/benchmarksCommon.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { FrameworkData, config } from "./common.js";
2+
import { args } from "./benchmarkArgs.js";
23

34
export enum BenchmarkType {
45
CPU,
@@ -96,7 +97,8 @@ const slowDownsOSX: ISlowDowns = {
9697

9798
const slowDownsLinux: ISlowDowns = slowDownsOSX;
9899

99-
const slowDowns: ISlowDowns = process.platform == "darwin" ? slowDownsOSX : slowDownsLinux;
100+
101+
const slowDowns: ISlowDowns = args.nothrottling ? {} : (process.platform == "darwin" ? slowDownsOSX : slowDownsLinux);
100102

101103
export function slowDownNote(benchmark: TBenchmarkID): string {
102104
return slowDowns[benchmark] ? " " + slowDowns[benchmark] + "x CPU slowdown." : "";

‎webdriver-ts/src/common.ts

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import axios from "axios";
2-
import { StartupBenchmarkResult } from "./benchmarksLighthouse.js";
32

43

54
export interface JSONResult {
@@ -18,10 +17,10 @@ export interface JSONResult {
1817

1918
export type TBenchmarkStatus = "OK" | "TEST_FAILED" | "TECHNICAL_ERROR";
2019

21-
export interface ErrorAndWarning {
20+
export interface ErrorAndWarning<T> {
2221
error: string;
2322
warnings: string[];
24-
result?: number[] | StartupBenchmarkResult[];
23+
result?: T[];
2524
}
2625

2726
export interface BenchmarkDriverOptions {

‎webdriver-ts/src/forkedBenchmarkRunnerLighthouse.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ async function runStartupBenchmark(
8989
framework: FrameworkData,
9090
benchmark: BenchmarkLighthouse,
9191
benchmarkOptions: BenchmarkOptions
92-
): Promise<ErrorAndWarning> {
92+
): Promise<ErrorAndWarning<StartupBenchmarkResult>> {
9393
console.log("benchmarking startup", framework, benchmark.benchmarkInfo.id);
9494

9595
let error: string = undefined;
@@ -106,13 +106,13 @@ export async function executeBenchmark(
106106
framework: FrameworkData,
107107
benchmarkId: string,
108108
benchmarkOptions: BenchmarkOptions
109-
): Promise<ErrorAndWarning> {
109+
): Promise<ErrorAndWarning<StartupBenchmarkResult>> {
110110
let runBenchmarks: Array<BenchmarkLighthouse> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && b instanceof BenchmarkLighthouse) as Array<BenchmarkLighthouse>;
111111
if (runBenchmarks.length != 1) throw `Benchmark name ${benchmarkId} is not unique (lighthouse)`;
112112

113113
let benchmark = runBenchmarks[0];
114114

115-
let errorAndWarnings: ErrorAndWarning;
115+
let errorAndWarnings: ErrorAndWarning<StartupBenchmarkResult>;
116116
errorAndWarnings = await runStartupBenchmark(framework, benchmark, benchmarkOptions);
117117
if (config.LOG_DEBUG) console.log("benchmark finished - got errors promise", errorAndWarnings);
118118
return errorAndWarnings;

‎webdriver-ts/src/forkedBenchmarkRunnerPlaywright.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ async function forceGC(page: Page, client: CDPSession) {
5353
}
5454
}
5555

56-
async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmarkPlaywright, benchmarkOptions: BenchmarkOptions): Promise<ErrorAndWarning>
56+
async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmarkPlaywright, benchmarkOptions: BenchmarkOptions): Promise<ErrorAndWarning<number>>
5757
{
5858
let error: string = undefined;
5959
let warnings: string[] = [];
@@ -153,7 +153,7 @@ async function runMemBenchmark(
153153
framework: FrameworkData,
154154
benchmark: MemBenchmarkPlaywright,
155155
benchmarkOptions: BenchmarkOptions
156-
): Promise<ErrorAndWarning> {
156+
): Promise<ErrorAndWarning<number>> {
157157
let error: string = undefined;
158158
let warnings: string[] = [];
159159
let results: number[] = [];
@@ -217,12 +217,12 @@ export async function executeBenchmark(
217217
framework: FrameworkData,
218218
benchmarkId: string,
219219
benchmarkOptions: BenchmarkOptions
220-
): Promise<ErrorAndWarning> {
220+
): Promise<ErrorAndWarning<number>> {
221221
let runBenchmarks: Array<TBenchmarkPlaywright> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && (b instanceof CPUBenchmarkPlaywright || b instanceof MemBenchmarkPlaywright) ) as Array<TBenchmarkPlaywright>;
222222

223223
let benchmark = runBenchmarks[0];
224224

225-
let errorAndWarnings: ErrorAndWarning;
225+
let errorAndWarnings: ErrorAndWarning<number>;
226226
if (benchmark.type == BenchmarkType.CPU) {
227227
errorAndWarnings = await runCPUBenchmark(framework, benchmark as CPUBenchmarkPlaywright, benchmarkOptions);
228228
} else {

‎webdriver-ts/src/forkedBenchmarkRunnerPuppeteer.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ async function forceGC(page: Page, client: CDPSession) {
5050
}
5151
}
5252

53-
async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmarkPuppeteer, benchmarkOptions: BenchmarkOptions): Promise<ErrorAndWarning>
53+
async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmarkPuppeteer, benchmarkOptions: BenchmarkOptions): Promise<ErrorAndWarning<number>>
5454
{
5555
let error: string = undefined;
5656
let warnings: string[] = [];
@@ -165,7 +165,7 @@ async function runMemBenchmark(
165165
framework: FrameworkData,
166166
benchmark: MemBenchmarkPuppeteer,
167167
benchmarkOptions: BenchmarkOptions
168-
): Promise<ErrorAndWarning> {
168+
): Promise<ErrorAndWarning<number>> {
169169
let error: string = undefined;
170170
let warnings: string[] = [];
171171
let results: number[] = [];
@@ -235,13 +235,13 @@ export async function executeBenchmark(
235235
framework: FrameworkData,
236236
benchmarkId: string,
237237
benchmarkOptions: BenchmarkOptions
238-
): Promise<ErrorAndWarning> {
238+
): Promise<ErrorAndWarning<number>> {
239239
let runBenchmarks: Array<TBenchmarkPuppeteer> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && (b instanceof CPUBenchmarkPuppeteer || b instanceof MemBenchmarkPuppeteer) ) as Array<TBenchmarkPuppeteer>;
240240
if (runBenchmarks.length != 1) throw `Benchmark name ${benchmarkId} is not unique (puppeteer)`;
241241

242242
let benchmark = runBenchmarks[0];
243243

244-
let errorAndWarnings: ErrorAndWarning;
244+
let errorAndWarnings: ErrorAndWarning<number>;
245245
if (benchmark.type == BenchmarkType.CPU) {
246246
errorAndWarnings = await runCPUBenchmark(framework, benchmark as CPUBenchmarkPuppeteer, benchmarkOptions);
247247
} else {

‎webdriver-ts/src/forkedBenchmarkRunnerWebdriver.ts

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,7 @@ async function runCPUBenchmark(
253253
framework: FrameworkData,
254254
benchmark: CPUBenchmarkWebdriver,
255255
benchmarkOptions: BenchmarkOptions
256-
): Promise<ErrorAndWarning> {
256+
): Promise<ErrorAndWarning<number>> {
257257
let error: string = undefined;
258258
let warnings: string[] = [];
259259

@@ -286,11 +286,11 @@ async function runCPUBenchmark(
286286
await driver.executeScript("console.timeStamp('runBenchmark')");
287287
await runBenchmark(driver, benchmark, framework);
288288
if (benchmark.benchmarkInfo.throttleCPU) {
289-
console.log("resetting CPU slowdown");
290-
await (driver as any).sendDevToolsCommand("Emulation.setCPUThrottlingRate", { rate: 1 });
289+
console.log("resetting CPU slowdown");
290+
await (driver as any).sendDevToolsCommand("Emulation.setCPUThrottlingRate", { rate: 1 });
291+
}
292+
await driver.executeScript("console.timeStamp('finishedBenchmark')");
291293
}
292-
await driver.executeScript("console.timeStamp('finishedBenchmark')");
293-
}
294294
let result = await computeResultsCPU(driver, benchmarkOptions, framework, benchmark, warnings, benchmarkOptions.batchSize);
295295
await driver.close();
296296
await driver.quit();
@@ -314,13 +314,13 @@ export async function executeBenchmark(
314314
framework: FrameworkData,
315315
benchmarkId: string,
316316
benchmarkOptions: BenchmarkOptions
317-
): Promise<ErrorAndWarning> {
317+
): Promise<ErrorAndWarning<number>> {
318318
let runBenchmarks: Array<CPUBenchmarkWebdriver> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && b instanceof CPUBenchmarkWebdriver) as Array<CPUBenchmarkWebdriver>;
319319
if (runBenchmarks.length != 1) throw `Benchmark name ${benchmarkId} is not unique (webdriver)`;
320320

321321
let benchmark = runBenchmarks[0];
322322

323-
let errorAndWarnings: ErrorAndWarning;
323+
let errorAndWarnings: ErrorAndWarning<number>;
324324
if (benchmark.benchmarkInfo.type == BenchmarkType.CPU) {
325325
errorAndWarnings = await runCPUBenchmark(framework, benchmark, benchmarkOptions);
326326
}

‎webdriver-ts/src/forkedBenchmarkRunnerWebdriverAfterframe.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ async function runCPUBenchmark(
4646
framework: FrameworkData,
4747
benchmark: CPUBenchmarkWebdriver,
4848
benchmarkOptions: BenchmarkOptions
49-
): Promise<ErrorAndWarning> {
49+
): Promise<ErrorAndWarning<number>> {
5050
let error: string = undefined;
5151
let warnings: string[] = [];
5252
let results: number[] = [];
@@ -102,13 +102,13 @@ export async function executeBenchmark(
102102
framework: FrameworkData,
103103
benchmarkId: string,
104104
benchmarkOptions: BenchmarkOptions
105-
): Promise<ErrorAndWarning> {
105+
): Promise<ErrorAndWarning<number>> {
106106
let runBenchmarks: Array<CPUBenchmarkWebdriver> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && b instanceof CPUBenchmarkWebdriver) as Array<CPUBenchmarkWebdriver>;
107107
if (runBenchmarks.length != 1) throw `Benchmark name ${benchmarkId} is not unique (webdriver)`;
108108

109109
let benchmark = runBenchmarks[0];
110110

111-
let errorAndWarnings: ErrorAndWarning;
111+
let errorAndWarnings: ErrorAndWarning<number>;
112112
if (benchmark.benchmarkInfo.type == BenchmarkType.CPU) {
113113
errorAndWarnings = await runCPUBenchmark(framework, benchmark, benchmarkOptions);
114114
}

‎webdriver-ts/src/forkedBenchmarkRunnerWebdriverCDP.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ async function runCPUBenchmark(
5858
framework: FrameworkData,
5959
benchmark: CPUBenchmarkWebdriverCDP,
6060
benchmarkOptions: BenchmarkOptions
61-
): Promise<ErrorAndWarning> {
61+
): Promise<ErrorAndWarning<number>> {
6262
let error: string = undefined;
6363
let warnings: string[] = [];
6464
let results: number[] = [];
@@ -161,13 +161,13 @@ export async function executeBenchmark(
161161
framework: FrameworkData,
162162
benchmarkId: string,
163163
benchmarkOptions: BenchmarkOptions
164-
): Promise<ErrorAndWarning> {
164+
): Promise<ErrorAndWarning<number>> {
165165
let runBenchmarks: Array<CPUBenchmarkWebdriverCDP> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && b instanceof CPUBenchmarkWebdriverCDP) as Array<CPUBenchmarkWebdriverCDP>;
166166
if (runBenchmarks.length != 1) throw `Benchmark name ${benchmarkId} is not unique (webdriver)`;
167167

168168
let benchmark = runBenchmarks[0];
169169

170-
let errorAndWarnings: ErrorAndWarning;
170+
let errorAndWarnings: ErrorAndWarning<number>;
171171
if (benchmark.benchmarkInfo.type == BenchmarkType.CPU) {
172172
errorAndWarnings = await runCPUBenchmark(framework, benchmark, benchmarkOptions);
173173
}

0 commit comments

Comments
 (0)