Skip to content

Commit c84a2c8

Browse files
krausestkrausest
committed
add JS only duration for Playwright, add dummies for the others
1 parent 4a15da3 commit c84a2c8

7 files changed

+41
-29
lines changed

‎webdriver-ts/src/benchmarksWebdriverAfterframe.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ const SHORT_TIMEOUT = 20 * 1000;
1818
let durations: Array<number> = [];
1919

2020
export function getAfterframeDurations() {
21-
return durations;
21+
return durations.map(d => ({total:d, script:0}));
2222
}
2323

2424
export async function initMeasurement(driver: WebDriver) {

‎webdriver-ts/src/forkedBenchmarkRunnerPlaywright.ts

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { Browser, Page, CDPSession } from "playwright-core";
2-
import { BenchmarkType, slowDownFactor } from "./benchmarksCommon.js";
2+
import { BenchmarkType, CPUBenchmarkResult, slowDownFactor } from "./benchmarksCommon.js";
33
import { benchmarks, CPUBenchmarkPlaywright, fileNameTrace, MemBenchmarkPlaywright, TBenchmarkPlaywright } from "./benchmarksPlaywright.js";
44
import { BenchmarkOptions, config as defaultConfig, ErrorAndWarning, FrameworkData, TConfig } from "./common.js";
55
import { startBrowser } from "./playwrightAccess.js";
@@ -53,11 +53,11 @@ async function forceGC(page: Page, client: CDPSession) {
5353
}
5454
}
5555

56-
async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmarkPlaywright, benchmarkOptions: BenchmarkOptions): Promise<ErrorAndWarning<number>>
56+
async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmarkPlaywright, benchmarkOptions: BenchmarkOptions): Promise<ErrorAndWarning<CPUBenchmarkResult>>
5757
{
5858
let error: string = undefined;
5959
let warnings: string[] = [];
60-
let results: number[] = [];
60+
let results: CPUBenchmarkResult[] = [];
6161

6262
console.log("benchmarking ", framework, benchmark.benchmarkInfo.id);
6363
let browser : Browser = null;
@@ -72,6 +72,7 @@ async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmark
7272
});
7373
// }
7474
let client = await page.context().newCDPSession(page);
75+
await client.send('Performance.enable');
7576
for (let i = 0; i <benchmarkOptions.batchSize; i++) {
7677
await page.goto(`http://${benchmarkOptions.host}:${benchmarkOptions.port}/${framework.uri}/index.html`, {waitUntil: "networkidle"});
7778

@@ -112,16 +113,28 @@ async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmark
112113
screenshots: false,
113114
categories:categories
114115
});
116+
let m1 = (await client.send('Performance.getMetrics')).metrics;
117+
let m1_val = m1.find(m => m.name === 'ScriptDuration').value;
118+
let m1_Timestamp = m1.find(m => m.name === 'Timestamp').value;
119+
console.log("m1", m1, m1_val);
115120
console.log("runBenchmark Playwright");
116121
await runBenchmark(browser, page, benchmark, framework);
117122

118123
await wait(40);
124+
let m2 = (await client.send('Performance.getMetrics')).metrics;
125+
let m2_val = m2.find(m => m.name === 'ScriptDuration').value;
126+
let m2_Timestamp = m2.find(m => m.name === 'Timestamp').value;
127+
console.log("m2", m2, m2_val);
119128
await browser.stopTracing();
120129
if (throttleCPU) {
121130
await client.send('Emulation.setCPUThrottlingRate', { rate: 1 });
122131
}
123132
let result = await computeResultsCPU(config, fileNameTrace(framework, benchmark.benchmarkInfo, i, benchmarkOptions), benchmark.benchmarkInfo.durationMeasurementMode);
124-
results.push(result);
133+
let resultScript = (m2_val - m1_val)*1000.0;
134+
console.log("**** resultScript = ", resultScript);
135+
if (m2_Timestamp == m1_Timestamp) throw new Error("Page metrics timestamp didn't change");
136+
137+
results.push({total: result, script: resultScript});
125138
console.log(`duration for ${framework.name} and ${benchmark.benchmarkInfo.id}: ${result}`);
126139
if (result < 0)
127140
throw new Error(`duration ${result} < 0`);
@@ -218,12 +231,12 @@ export async function executeBenchmark(
218231
framework: FrameworkData,
219232
benchmarkId: string,
220233
benchmarkOptions: BenchmarkOptions
221-
): Promise<ErrorAndWarning<number>> {
234+
): Promise<ErrorAndWarning<number|CPUBenchmarkResult>> {
222235
let runBenchmarks: Array<TBenchmarkPlaywright> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && (b instanceof CPUBenchmarkPlaywright || b instanceof MemBenchmarkPlaywright) ) as Array<TBenchmarkPlaywright>;
223236

224237
let benchmark = runBenchmarks[0];
225238

226-
let errorAndWarnings: ErrorAndWarning<number>;
239+
let errorAndWarnings: ErrorAndWarning<number|CPUBenchmarkResult>;
227240
if (benchmark.type == BenchmarkType.CPU) {
228241
errorAndWarnings = await runCPUBenchmark(framework, benchmark as CPUBenchmarkPlaywright, benchmarkOptions);
229242
} else {

‎webdriver-ts/src/forkedBenchmarkRunnerPuppeteer.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,8 @@ async function runCPUBenchmark(framework: FrameworkData, benchmark: CPUBenchmark
139139
// let result = (m2.TaskDuration - m1.TaskDuration)*1000.0; //await computeResultsCPU(fileNameTrace(framework, benchmark, i), benchmarkOptions, framework, benchmark, warnings, benchmarkOptions.batchSize);
140140
let result = await computeResultsCPU(config, fileNameTrace(framework, benchmark.benchmarkInfo, i, benchmarkOptions), benchmark.benchmarkInfo.durationMeasurementMode);
141141
let resultScript = (m2.ScriptDuration - m1.ScriptDuration)*1000.0;
142-
console.log("**** resultJS = ", resultScript);
142+
console.log("**** resultScript = ", resultScript);
143+
if (m2.Timestamp == m1.Timestamp) throw new Error("Page metrics timestamp didn't change");
143144
results.push({total:result, script: resultScript});
144145
console.log(`duration for ${framework.name} and ${benchmark.benchmarkInfo.id}: ${result}`);
145146
if (result < 0)

‎webdriver-ts/src/forkedBenchmarkRunnerWebdriver.ts

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { setUseShadowRoot, buildDriver, setUseRowShadowRoot, setShadowRootName,
44

55
import { TConfig, config as defaultConfig, FrameworkData, ErrorAndWarning, BenchmarkOptions } from "./common.js";
66
import * as R from "ramda";
7-
import { BenchmarkType, DurationMeasurementMode, slowDownFactor } from "./benchmarksCommon.js";
7+
import { BenchmarkType, CPUBenchmarkResult, DurationMeasurementMode, slowDownFactor } from "./benchmarksCommon.js";
88

99
let config: TConfig = defaultConfig;
1010

@@ -126,7 +126,7 @@ async function computeResultsCPU(
126126
benchmark: CPUBenchmarkWebdriver,
127127
warnings: string[],
128128
expcectedResultCount: number
129-
): Promise<number[]> {
129+
): Promise<CPUBenchmarkResult[]> {
130130
let entriesBrowser = await driver.manage().logs().get(logging.Type.BROWSER);
131131
if (config.LOG_DEBUG) console.log("browser entries", entriesBrowser);
132132
const perfLogEvents = await fetchEventsFromPerformanceLog(driver);
@@ -135,7 +135,7 @@ async function computeResultsCPU(
135135
// if (config.LOG_DEBUG) console.log("filteredEvents ", asString(filteredEvents));
136136

137137
let remaining = R.dropWhile(type_eq("initBenchmark"))(filteredEvents);
138-
let results: number[] = [];
138+
let results: CPUBenchmarkResult[] = [];
139139

140140
while (remaining.length > 0) {
141141
let evts = R.splitWhen(type_eq("finishedBenchmark"))(remaining);
@@ -196,7 +196,8 @@ async function computeResultsCPU(
196196
console.log("soundness check failed. reported duration is bigger than whole benchmark duration", asString(eventsDuringBenchmark));
197197
throw "soundness check failed. reported duration is bigger than whole benchmark duration";
198198
}
199-
results.push(duration);
199+
// script is currently not implemented
200+
results.push({total:duration, script:0});
200201
}
201202
remaining = R.drop(1, evts[1]);
202203
}
@@ -253,7 +254,7 @@ async function runCPUBenchmark(
253254
framework: FrameworkData,
254255
benchmark: CPUBenchmarkWebdriver,
255256
benchmarkOptions: BenchmarkOptions
256-
): Promise<ErrorAndWarning<number>> {
257+
): Promise<ErrorAndWarning<number|CPUBenchmarkResult>> {
257258
let error: string = undefined;
258259
let warnings: string[] = [];
259260

@@ -315,13 +316,13 @@ export async function executeBenchmark(
315316
framework: FrameworkData,
316317
benchmarkId: string,
317318
benchmarkOptions: BenchmarkOptions
318-
): Promise<ErrorAndWarning<number>> {
319+
): Promise<ErrorAndWarning<number|CPUBenchmarkResult>> {
319320
let runBenchmarks: Array<CPUBenchmarkWebdriver> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && b instanceof CPUBenchmarkWebdriver) as Array<CPUBenchmarkWebdriver>;
320321
if (runBenchmarks.length != 1) throw `Benchmark name ${benchmarkId} is not unique (webdriver)`;
321322

322323
let benchmark = runBenchmarks[0];
323324

324-
let errorAndWarnings: ErrorAndWarning<number>;
325+
let errorAndWarnings: ErrorAndWarning<number|CPUBenchmarkResult>;
325326
if (benchmark.benchmarkInfo.type == BenchmarkType.CPU) {
326327
errorAndWarnings = await runCPUBenchmark(framework, benchmark, benchmarkOptions);
327328
}

‎webdriver-ts/src/forkedBenchmarkRunnerWebdriverAfterframe.ts

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { CPUBenchmarkWebdriver, benchmarks } from "./benchmarksWebdriverAfterfra
33
import { setUseShadowRoot, buildDriver, setUseRowShadowRoot, setShadowRootName, setButtonsInShadowRoot } from "./webdriverAccess.js";
44

55
import { TConfig, config as defaultConfig, FrameworkData, ErrorAndWarning, BenchmarkOptions } from "./common.js";
6-
import { BenchmarkType } from "./benchmarksCommon.js";
6+
import { BenchmarkType, CPUBenchmarkResult } from "./benchmarksCommon.js";
77
import { getAfterframeDurations, initMeasurement } from "./benchmarksWebdriverAfterframe.js";
88

99
let config: TConfig = defaultConfig;
@@ -46,10 +46,10 @@ async function runCPUBenchmark(
4646
framework: FrameworkData,
4747
benchmark: CPUBenchmarkWebdriver,
4848
benchmarkOptions: BenchmarkOptions
49-
): Promise<ErrorAndWarning<number>> {
49+
): Promise<ErrorAndWarning<CPUBenchmarkResult>> {
5050
let error: string = undefined;
5151
let warnings: string[] = [];
52-
let results: number[] = [];
52+
let results: CPUBenchmarkResult[] = [];
5353

5454
console.log("benchmarking ", framework, benchmark.benchmarkInfo.id);
5555
let driver: WebDriver = null;
@@ -102,13 +102,13 @@ export async function executeBenchmark(
102102
framework: FrameworkData,
103103
benchmarkId: string,
104104
benchmarkOptions: BenchmarkOptions
105-
): Promise<ErrorAndWarning<number>> {
105+
): Promise<ErrorAndWarning<number|CPUBenchmarkResult>> {
106106
let runBenchmarks: Array<CPUBenchmarkWebdriver> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && b instanceof CPUBenchmarkWebdriver) as Array<CPUBenchmarkWebdriver>;
107107
if (runBenchmarks.length != 1) throw `Benchmark name ${benchmarkId} is not unique (webdriver)`;
108108

109109
let benchmark = runBenchmarks[0];
110110

111-
let errorAndWarnings: ErrorAndWarning<number>;
111+
let errorAndWarnings: ErrorAndWarning<number|CPUBenchmarkResult>;
112112
if (benchmark.benchmarkInfo.type == BenchmarkType.CPU) {
113113
errorAndWarnings = await runCPUBenchmark(framework, benchmark, benchmarkOptions);
114114
}

‎webdriver-ts/src/forkedBenchmarkRunnerWebdriverCDP.ts

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import * as fs from "fs/promises";
22
import { WebDriver } from "selenium-webdriver";
3-
import { BenchmarkType, slowDownFactor } from "./benchmarksCommon.js";
3+
import { BenchmarkType, CPUBenchmarkResult, slowDownFactor } from "./benchmarksCommon.js";
44
import { benchmarks, CPUBenchmarkWebdriverCDP, fileNameTrace } from "./benchmarksWebdriverCDP.js";
55
import { BenchmarkOptions, config as defaultConfig, ErrorAndWarning, FrameworkData, TConfig } from "./common.js";
66
import { computeResultsCPU } from "./timeline.js";
@@ -58,10 +58,10 @@ async function runCPUBenchmark(
5858
framework: FrameworkData,
5959
benchmark: CPUBenchmarkWebdriverCDP,
6060
benchmarkOptions: BenchmarkOptions
61-
): Promise<ErrorAndWarning<number>> {
61+
): Promise<ErrorAndWarning<CPUBenchmarkResult>> {
6262
let error: string = undefined;
6363
let warnings: string[] = [];
64-
let results: number[] = [];
64+
let results: CPUBenchmarkResult[] = [];
6565

6666
console.log("benchmarking ", framework, benchmark.benchmarkInfo.id, "with webdriver (tracing via CDP Connection)");
6767
let driver: WebDriver = null;
@@ -134,7 +134,7 @@ async function runCPUBenchmark(
134134
await p;
135135

136136
let result = await computeResultsCPU(config, fileNameTrace(framework, benchmark.benchmarkInfo, i, benchmarkOptions), benchmark.benchmarkInfo.durationMeasurementMode);
137-
results.push(result);
137+
results.push({total:result, script: 0});
138138
console.log(`duration for ${framework.name} and ${benchmark.benchmarkInfo.id}: ${result}`);
139139
if (result < 0)
140140
throw new Error(`duration ${result} < 0`);
@@ -162,13 +162,13 @@ export async function executeBenchmark(
162162
framework: FrameworkData,
163163
benchmarkId: string,
164164
benchmarkOptions: BenchmarkOptions
165-
): Promise<ErrorAndWarning<number>> {
165+
): Promise<ErrorAndWarning<number|CPUBenchmarkResult>> {
166166
let runBenchmarks: Array<CPUBenchmarkWebdriverCDP> = benchmarks.filter(b => benchmarkId === b.benchmarkInfo.id && b instanceof CPUBenchmarkWebdriverCDP) as Array<CPUBenchmarkWebdriverCDP>;
167167
if (runBenchmarks.length != 1) throw `Benchmark name ${benchmarkId} is not unique (webdriver)`;
168168

169169
let benchmark = runBenchmarks[0];
170170

171-
let errorAndWarnings: ErrorAndWarning<number>;
171+
let errorAndWarnings: ErrorAndWarning<number|CPUBenchmarkResult>;
172172
if (benchmark.benchmarkInfo.type == BenchmarkType.CPU) {
173173
errorAndWarnings = await runCPUBenchmark(framework, benchmark, benchmarkOptions);
174174
}

‎webdriver-ts/src/writeResults.ts

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ export function writeResults(resultDir: string, res: ResultLightHouse|ResultCPU|
4141
}
4242
break;
4343
case BenchmarkType.CPU:
44-
console.log("writeResults", res);
4544
createResultFile(resultDir, ({total: res.results.map(r=>r.total), script: res.results.map(r=>r.script)}), res.framework, res.benchmark);
4645
break;
4746
case BenchmarkType.MEM:
@@ -51,7 +50,6 @@ export function writeResults(resultDir: string, res: ResultLightHouse|ResultCPU|
5150
}
5251

5352
function createResultFile(resultDir: string, data: number[]|{[key:string]: number[]}, framework: FrameworkData, benchmark: BenchmarkInfo) {
54-
console.log("createResultFile", data);
5553
let type = "";
5654
switch (benchmark.type) {
5755
case BenchmarkType.CPU:
@@ -91,7 +89,6 @@ function createResultFile(resultDir: string, data: number[]|{[key:string]: numbe
9189
} else {
9290
let values: {[k: string]: JSONResultData} = {};
9391
for (let key of Object.keys(data)) {
94-
console.log("converting", key, data[key], data);
9592
values[key] = convertResult(key, data[key]);
9693
}
9794
let result: JSONResult = {

0 commit comments

Comments
 (0)