Skip to content

Commit ffd1bcf

Browse files
krausestkrausest
committed
cleanup benchmark driver
1 parent 17d0b0b commit ffd1bcf

10 files changed

+1458
-753
lines changed

‎webdriver-ts/package-lock.json

Lines changed: 1392 additions & 679 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎webdriver-ts/package.json

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -21,27 +21,27 @@
2121
"author": "",
2222
"license": "Apache-2.0",
2323
"devDependencies": {
24-
"@types/node": "18.16.1",
25-
"@types/ramda": "0.29.0",
26-
"@types/selenium-webdriver": "4.1.13",
27-
"@types/semver": "7.3.13",
28-
"@types/yargs": "17.0.24",
29-
"@vitest/coverage-v8": "^0.34.4",
30-
"ts-node": "^10.9.1",
31-
"typescript": "4.9.5",
32-
"vitest": "^0.34.4"
24+
"@types/node": "20.11.0",
25+
"@types/ramda": "0.29.9",
26+
"@types/selenium-webdriver": "4.1.21",
27+
"@types/semver": "7.5.6",
28+
"@types/yargs": "17.0.32",
29+
"@vitest/coverage-v8": "^1.2.0",
30+
"ts-node": "^10.9.2",
31+
"typescript": "5.3.3",
32+
"vitest": "^1.2.0"
3333
},
3434
"dependencies": {
35-
"chromedriver": "112.0.0",
35+
"chromedriver": "120.0.1",
3636
"cross-env": "7.0.3",
37-
"lighthouse": "10.1.1",
38-
"playwright": "1.39.0",
39-
"playwright-firefox": "1.39.0",
40-
"playwright-webkit": "1.39.0",
41-
"puppeteer-core": "21.5.0",
42-
"ramda": "0.29.0",
43-
"selenium-webdriver": "4.15.0",
44-
"semver": "7.5.0",
45-
"yargs": "17.7.1"
37+
"lighthouse": "11.4.0",
38+
"playwright": "1.40.1",
39+
"playwright-firefox": "1.40.1",
40+
"playwright-webkit": "1.40.1",
41+
"puppeteer-core": "21.7.0",
42+
"ramda": "0.29.1",
43+
"selenium-webdriver": "4.16.0",
44+
"semver": "7.5.4",
45+
"yargs": "17.7.2"
4646
}
4747
}

‎webdriver-ts/src/benchmarkRunner.ts

Lines changed: 13 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,6 @@ function forkAndCallBenchmark(
4040
forkedRunner = "dist/forkedBenchmarkRunnerWebdriverCDP.js";
4141
} else if (config.BENCHMARK_RUNNER == BenchmarkRunner.PLAYWRIGHT) {
4242
forkedRunner = "dist/forkedBenchmarkRunnerPlaywright.js";
43-
} else if (config.BENCHMARK_RUNNER == BenchmarkRunner.WEBDRIVER) {
44-
forkedRunner = "dist/forkedBenchmarkRunnerWebdriver.js";
4543
} else if (config.BENCHMARK_RUNNER == BenchmarkRunner.WEBDRIVER_AFTERFRAME) {
4644
forkedRunner = "dist/forkedBenchmarkRunnerWebdriverAfterframe.js";
4745
} else {
@@ -95,21 +93,19 @@ async function runBenchmakLoopStartup(
9593
let res = await forkAndCallBenchmark(framework, benchmarkInfo, benchmarkOptions);
9694
if (Array.isArray(res.result)) {
9795
results = results.concat(res.result as StartupBenchmarkResult[]);
98-
} else results.push(res.result);
99-
warnings = warnings.concat(res.warnings);
96+
}
97+
else {
98+
results.push(res.result);
99+
}
100+
if (res.warnings) {
101+
warnings = warnings.concat(res.warnings);
102+
}
100103
if (res.error) {
101-
if (res.error.includes("Server terminated early with status 1")) {
102-
console.log("******* STRANGE selenium error found - retry #", retries + 1);
103-
retries++;
104-
if (retries == 3) break;
105-
} else {
106-
errors.push(`Executing ${framework.uri} and benchmark ${benchmarkInfo.id} failed: ` + res.error);
107-
break;
108-
}
104+
errors.push(`Executing ${framework.uri} and benchmark ${benchmarkInfo.id} failed: ` + res.error);
105+
break;
109106
}
110107
done++;
111108
}
112-
console.log("******* result", results);
113109
if (config.WRITE_RESULTS) {
114110
await writeResults(benchmarkOptions.resultsDirectory, {
115111
framework: framework,
@@ -153,7 +149,6 @@ async function runBenchmakLoopSize(
153149
}
154150
done++;
155151
}
156-
console.log("******* result", results);
157152
if (config.WRITE_RESULTS) {
158153
await writeResults(benchmarkOptions.resultsDirectory, {
159154
framework: framework,
@@ -203,26 +198,11 @@ async function runBenchmakLoop(
203198
}
204199
warnings = warnings.concat(res.warnings);
205200
if (res.error) {
206-
if (res.error.includes("Server terminated early with status 1")) {
207-
console.log("******* STRANGE selenium error found - retry #", retries + 1);
208-
retries++;
209-
if (retries == 3) break;
210-
} else {
211-
errors.push(`Executing ${framework.uri} and benchmark ${benchmarkInfo.id} failed: ` + res.error);
212-
break;
213-
}
201+
console.log(`Executing ${framework.uri} and benchmark ${benchmarkInfo.id} failed: ` + res.error);
202+
errors.push(`Executing ${framework.uri} and benchmark ${benchmarkInfo.id} failed: ` + res.error);
203+
break;
214204
}
215205
}
216-
if (benchmarkInfo.type == BenchmarkType.CPU) {
217-
console.log("CPU results before:", results);
218-
// (results as CPUBenchmarkResult[]).sort(
219-
// (a: CPUBenchmarkResult, b: CPUBenchmarkResult) => a.total - b.total
220-
// );
221-
// results = results.slice(0, config.NUM_ITERATIONS_FOR_BENCHMARK_CPU);
222-
// console.log("CPU results after: ", results)
223-
}
224-
225-
console.log("******* result", results);
226206
if (config.WRITE_RESULTS) {
227207
if (benchmarkInfo.type == BenchmarkType.CPU) {
228208
await writeResults(benchmarkOptions.resultsDirectory, {
@@ -241,8 +221,6 @@ async function runBenchmakLoop(
241221
}
242222
}
243223
return { errors, warnings };
244-
// } else {
245-
// return executeBenchmark(frameworks, keyed, frameworkName, benchmarkName, benchmarkOptions);
246224
}
247225

248226
async function runBench(
@@ -368,7 +346,6 @@ async function main() {
368346
if (
369347
[
370348
BenchmarkRunner.WEBDRIVER_CDP,
371-
BenchmarkRunner.WEBDRIVER,
372349
BenchmarkRunner.WEBDRIVER_AFTERFRAME,
373350
BenchmarkRunner.PLAYWRIGHT,
374351
BenchmarkRunner.PUPPETEER,
@@ -379,7 +356,7 @@ async function main() {
379356
} else {
380357
console.log("ERROR: argument driver has illegal value " + runner, [
381358
BenchmarkRunner.WEBDRIVER_CDP,
382-
BenchmarkRunner.WEBDRIVER,
359+
BenchmarkRunner.WEBDRIVER_AFTERFRAME,
383360
BenchmarkRunner.PLAYWRIGHT,
384361
BenchmarkRunner.PUPPETEER,
385362
]);

‎webdriver-ts/src/common.ts

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ export interface JsonResult {
2121
export type BenchmarkStatus = "OK" | "TEST_FAILED" | "TECHNICAL_ERROR";
2222

2323
export interface ErrorAndWarning<T> {
24-
error: string;
25-
warnings: string[];
24+
error?: string;
25+
warnings?: string[];
2626
result?: T[];
2727
}
2828

@@ -56,7 +56,6 @@ export enum BenchmarkRunner {
5656
PUPPETEER = "puppeteer",
5757
PLAYWRIGHT = "playwright",
5858
WEBDRIVER_CDP = "webdrivercdp",
59-
WEBDRIVER = "webdriver",
6059
WEBDRIVER_AFTERFRAME = "webdriver-afterframe",
6160
}
6261

‎webdriver-ts/src/forkedBenchmarkRunnerLighthouse.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ process.on("message", (msg: any) => {
149149
})
150150
.catch((error) => {
151151
console.log("CATCH: Error in forkedBenchmarkRunnerLighthouse");
152-
process.send({ failure: convertError(error) });
152+
process.send({ error: convertError(error) });
153153
process.exit(0);
154154
});
155155
});

‎webdriver-ts/src/forkedBenchmarkRunnerPlaywright.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -263,7 +263,7 @@ process.on("message", (msg: any) => {
263263
})
264264
.catch((error) => {
265265
console.log("CATCH: Error in forkedBenchmarkRunner", error);
266-
process.send({ failure: convertError(error) });
266+
process.send({ error: convertError(error) });
267267
process.exit(0);
268268
});
269269
});

‎webdriver-ts/src/forkedBenchmarkRunnerPuppeteer.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -305,7 +305,7 @@ process.on("message", (msg: any) => {
305305
})
306306
.catch((error) => {
307307
console.log("CATCH: Error in forkedBenchmarkRunner");
308-
process.send({ failure: convertError(error) });
308+
process.send({ error: convertError(error) });
309309
process.exit(0);
310310
});
311311
});

‎webdriver-ts/src/forkedBenchmarkRunnerSize.ts

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -88,14 +88,11 @@ async function runSizeBenchmark(
8888
if (await disableCompressionResponse.text() !== "OK") console.log("ERROR - Could not disable compression - OK missing");
8989
try {
9090
if (browser) {
91-
console.log("*** browser close");
9291
await browser.close();
93-
console.log("*** browser closed");
9492
}
9593
} catch (error) {
9694
console.log("ERROR cleaning up driver", error);
9795
}
98-
console.log("*** browser has been shutting down");
9996
}
10097
}
10198

@@ -139,7 +136,7 @@ process.on("message", (msg: any) => {
139136
})
140137
.catch((error) => {
141138
console.log("CATCH: Error in forkedBenchmarkRunnerSize");
142-
process.send({ failure: convertError(error) });
139+
process.send({ error: convertError(error) });
143140
process.exit(0);
144141
});
145142
});

‎webdriver-ts/src/forkedBenchmarkRunnerWebdriverAfterframe.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ process.on("message", (msg: any) => {
145145
})
146146
.catch((error) => {
147147
console.log("CATCH: Error in forkedBenchmarkRunner");
148-
process.send({ failure: convertError(error) });
148+
process.send({ error: convertError(error) });
149149
process.exit(0);
150150
});
151151
});

‎webdriver-ts/src/forkedBenchmarkRunnerWebdriverCDP.ts

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { WebDriver } from "selenium-webdriver";
33
import { BenchmarkType, CPUBenchmarkResult, slowDownFactor } from "./benchmarksCommon.js";
44
import { benchmarks, CPUBenchmarkWebdriverCDP } from "./benchmarksWebdriverCDP.js";
55
import { BenchmarkOptions, config as defaultConfig, ErrorAndWarning, FrameworkData, Config } from "./common.js";
6-
import { computeResultsCPU, fileNameTrace } from "./timeline.js";
6+
import { computeResultsCPU, computeResultsJS, computeResultsPaint, fileNameTrace } from "./timeline.js";
77
import {
88
buildDriver,
99
setButtonsInShadowRoot,
@@ -15,7 +15,10 @@ import {
1515
let config: Config = defaultConfig;
1616

1717
// necessary to launch without specifiying a path
18-
require("chromedriver");
18+
import "chromedriver";
19+
20+
const wait = (delay = 1000) => new Promise((res) => setTimeout(res, delay));
21+
1922

2023
async function runBenchmark(
2124
driver: WebDriver,
@@ -80,8 +83,8 @@ async function runCPUBenchmark(
8083
console.log("benchmarking", framework, benchmark.benchmarkInfo.id, "with webdriver (tracing via CDP Connection)");
8184
let driver: WebDriver = null;
8285
try {
83-
driver = buildDriver(benchmarkOptions);
8486
for (let i = 0; i < benchmarkOptions.batchSize; i++) {
87+
driver = buildDriver(benchmarkOptions);
8588
let trace: any = { traceEvents: [] }; //await fs.open(fileNameTrace(framework, benchmark.benchmarkInfo, i), "w");
8689
setUseShadowRoot(framework.useShadowRoot);
8790
setUseRowShadowRoot(framework.useRowShadowRoot);
@@ -146,22 +149,38 @@ async function runCPUBenchmark(
146149
});
147150
});
148151

152+
await wait(100);
153+
149154
await runBenchmark(driver, benchmark, framework);
150155

151156
if (throttleCPU) {
152157
console.log("resetting CPU slowdown");
153158
await (driver as any).sendDevToolsCommand("Emulation.setCPUThrottlingRate", { rate: 1 });
154159
}
160+
await wait(100);
161+
155162
await cdpConnection.execute("Tracing.end", {});
156163
await p;
157164

158165
let result = await computeResultsCPU(fileNameTrace(framework, benchmark.benchmarkInfo, i, benchmarkOptions));
159-
results.push({ total: result.duration, script: 0, paint: 0 });
160-
console.log(`duration for ${framework.name} and ${benchmark.benchmarkInfo.id}: ${result}`);
166+
let resultScript = await computeResultsJS(
167+
result,
168+
config,
169+
fileNameTrace(framework, benchmark.benchmarkInfo, i, benchmarkOptions)
170+
);
171+
let resultPaint = await computeResultsPaint(
172+
result,
173+
config,
174+
fileNameTrace(framework, benchmark.benchmarkInfo, i, benchmarkOptions)
175+
);
176+
177+
let res = { total: result.duration, script: resultScript, paint: resultPaint };
178+
results.push(res);
179+
console.log(`duration for ${framework.name} and ${benchmark.benchmarkInfo.id}: ${JSON.stringify(res)}`);
161180
if (result.duration < 0) throw new Error(`duration ${result} < 0`);
181+
await driver.close();
182+
await driver.quit();
162183
}
163-
await driver.close();
164-
await driver.quit();
165184
return { error, warnings, result: results };
166185
} catch (error) {
167186
console.log("ERROR", error);
@@ -219,7 +238,7 @@ process.on("message", (msg: any) => {
219238
})
220239
.catch((error) => {
221240
console.log("CATCH: Error in forkedBenchmarkRunner");
222-
process.send({ failure: convertError(error) });
241+
process.send({ error: convertError(error) });
223242
process.exit(0);
224243
});
225244
});

0 commit comments

Comments
 (0)