Skip to content

Commit d006211

Browse files
krausestkrausest
committed
Merge branch 'twharmon-warmup-counts-cfg'
2 parents d3ab786 + 85044ad commit d006211

9 files changed

+119
-117
lines changed

‎webdriver-ts/src/benchmarksCommon.ts

Lines changed: 32 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { FrameworkData, config } from "./common.js";
1+
import { FrameworkData } from "./common.js";
22

33
export enum BenchmarkType {
44
CPU,
@@ -12,7 +12,7 @@ export enum BenchmarkType {
1212
export interface BenchmarkInfoBase {
1313
id: string;
1414
label: string;
15-
description(throttleCPU: number | undefined): string;
15+
description: string;
1616
type: BenchmarkType;
1717
}
1818

@@ -21,6 +21,7 @@ export interface CPUBenchmarkInfo extends BenchmarkInfoBase {
2121
type: BenchmarkType.CPU;
2222
layoutEventRequired: boolean;
2323
additionalNumberOfRuns: number;
24+
warmupCount: number
2425
}
2526

2627
export interface MemBenchmarkInfo extends BenchmarkInfoBase {
@@ -113,6 +114,10 @@ export function slowDownNote(throttleCPU: number | undefined): string {
113114
return throttleCPU ? ` ${throttleCPU} x CPU slowdown.` : "";
114115
}
115116

117+
export function warmupNote(b: BenchmarkInfo): string {
118+
return 'warmupCount' in b ? ` (${b.warmupCount} warmup runs).` : "";
119+
}
120+
116121
export function slowDownFactor(benchmarkId: string, allowThrottling: boolean): number | undefined {
117122
if (!allowThrottling) return undefined;
118123
return throttlingFactors[benchmarkId];
@@ -122,8 +127,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
122127
{
123128
id: Benchmark._01,
124129
label: "create rows",
125-
description: (throttleCPU: number | undefined) =>
126-
"creating 1,000 rows (" + config.WARMUP_COUNT + " warmup runs)." + slowDownNote(throttleCPU),
130+
warmupCount: 5,
131+
description: "creating 1,000 rows.",
127132
type: BenchmarkType.CPU,
128133
allowBatching: true,
129134
layoutEventRequired: true,
@@ -132,8 +137,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
132137
{
133138
id: Benchmark._02,
134139
label: "replace all rows",
135-
description: (throttleCPU: number | undefined) =>
136-
"updating all 1,000 rows (" + config.WARMUP_COUNT + " warmup runs)." + slowDownNote(throttleCPU),
140+
warmupCount: 5,
141+
description: "updating all 1,000 rows.",
137142
type: BenchmarkType.CPU,
138143
allowBatching: true,
139144
layoutEventRequired: true,
@@ -142,8 +147,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
142147
{
143148
id: Benchmark._03,
144149
label: "partial update",
145-
description: (throttleCPU: number | undefined) =>
146-
"updating every 10th row for 1,000 rows (3 warmup runs)." + slowDownNote(throttleCPU),
150+
warmupCount: 3,
151+
description: "updating every 10th row for 1,000 row.",
147152
type: BenchmarkType.CPU,
148153
allowBatching: true,
149154
layoutEventRequired: true,
@@ -152,8 +157,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
152157
{
153158
id: Benchmark._04,
154159
label: "select row",
155-
description: (throttleCPU: number | undefined) =>
156-
"highlighting a selected row. (" + config.WARMUP_COUNT + " warmup runs)." + slowDownNote(throttleCPU),
160+
warmupCount: 5,
161+
description: "highlighting a selected row.",
157162
type: BenchmarkType.CPU,
158163
allowBatching: true,
159164
layoutEventRequired: false,
@@ -162,8 +167,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
162167
{
163168
id: Benchmark._05,
164169
label: "swap rows",
165-
description: (throttleCPU: number | undefined) =>
166-
"swap 2 rows for table with 1,000 rows. (" + config.WARMUP_COUNT + " warmup runs)." + slowDownNote(throttleCPU),
170+
warmupCount: 5,
171+
description: "swap 2 rows for table with 1,000 rows.",
167172
type: BenchmarkType.CPU,
168173
allowBatching: true,
169174
layoutEventRequired: true,
@@ -172,8 +177,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
172177
{
173178
id: Benchmark._06,
174179
label: "remove row",
175-
description: (throttleCPU: number | undefined) =>
176-
"removing one row. (" + config.WARMUP_COUNT + " warmup runs)." + slowDownNote(throttleCPU),
180+
warmupCount: 5,
181+
description: "removing one row.",
177182
type: BenchmarkType.CPU,
178183
allowBatching: true,
179184
layoutEventRequired: true,
@@ -182,8 +187,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
182187
{
183188
id: Benchmark._07,
184189
label: "create many rows",
185-
description: (throttleCPU: number | undefined) =>
186-
"creating 10,000 rows. (" + config.WARMUP_COUNT + " warmup runs with 1k rows)." + slowDownNote(throttleCPU),
190+
warmupCount: 5,
191+
description: "creating 10,000 rows.",
187192
type: BenchmarkType.CPU,
188193
allowBatching: true,
189194
layoutEventRequired: true,
@@ -192,8 +197,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
192197
{
193198
id: Benchmark._08,
194199
label: "append rows to large table",
195-
description: (throttleCPU: number | undefined) =>
196-
"appending 1,000 to a table of 1,000 rows." + slowDownNote(throttleCPU),
200+
warmupCount: 5,
201+
description: "appending 1,000 to a table of 1,000 rows.",
197202
type: BenchmarkType.CPU,
198203
allowBatching: true,
199204
layoutEventRequired: true,
@@ -202,8 +207,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
202207
{
203208
id: Benchmark._09,
204209
label: "clear rows",
205-
description: (throttleCPU: number | undefined) =>
206-
"clearing a table with 1,000 rows." + slowDownNote(throttleCPU) + " (" + config.WARMUP_COUNT + " warmup runs).",
210+
warmupCount: 5,
211+
description: "clearing a table with 1,000 rows.",
207212
type: BenchmarkType.CPU,
208213
allowBatching: true,
209214
layoutEventRequired: true,
@@ -215,19 +220,19 @@ export const memBenchmarkInfosArray: Array<MemBenchmarkInfo> = [
215220
{
216221
id: Benchmark._21,
217222
label: "ready memory",
218-
description: () => "Memory usage after page load.",
223+
description: "Memory usage after page load.",
219224
type: BenchmarkType.MEM,
220225
},
221226
{
222227
id: Benchmark._22,
223228
label: "run memory",
224-
description: () => "Memory usage after adding 1,000 rows.",
229+
description: "Memory usage after adding 1,000 rows.",
225230
type: BenchmarkType.MEM,
226231
},
227232
{
228233
id: Benchmark._23,
229234
label: "update every 10th row for 1k rows (5 cycles)",
230-
description: () => "Memory usage after clicking update every 10th row 5 times",
235+
description: "Memory usage after clicking update every 10th row 5 times",
231236
type: BenchmarkType.MEM,
232237
},
233238
// {
@@ -239,13 +244,13 @@ export const memBenchmarkInfosArray: Array<MemBenchmarkInfo> = [
239244
{
240245
id: Benchmark._25,
241246
label: "creating/clearing 1k rows (5 cycles)",
242-
description: () => "Memory usage after creating and clearing 1000 rows 5 times",
247+
description: "Memory usage after creating and clearing 1000 rows 5 times",
243248
type: BenchmarkType.MEM,
244249
},
245250
{
246251
id: Benchmark._26,
247252
label: "run memory 10k",
248-
description: () => "Memory usage after adding 10,000 rows.",
253+
description: "Memory usage after adding 10,000 rows.",
249254
type: BenchmarkType.MEM,
250255
},
251256
];
@@ -255,7 +260,7 @@ export const startupBenchmarkInfosArray: Array<StartupMainBenchmarkInfo> = [
255260
id: Benchmark._30,
256261
type: BenchmarkType.STARTUP_MAIN,
257262
label: "",
258-
description: () => "",
263+
description: "",
259264
}
260265
];
261266

@@ -264,7 +269,7 @@ export const sizesBenchmarkInfosArray: Array<SizeMainBenchmarkInfo> = [
264269
id: Benchmark._40,
265270
type: BenchmarkType.SIZE_MAIN,
266271
label: "",
267-
description: () => "",
272+
description: "",
268273
},
269274
];
270275

‎webdriver-ts/src/benchmarksLighthouse.ts

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,7 @@ let toKb = (x: number) => x / 1024;
1212
export const benchStartupConsistentlyInteractive: StartupBenchmarkInfo = {
1313
id: "31_startup-ci",
1414
label: "consistently interactive",
15-
description: () =>
16-
"a pessimistic TTI - when the CPU and network are both definitely very idle. (no more CPU tasks over 50ms)",
15+
description: "a pessimistic TTI - when the CPU and network are both definitely very idle. (no more CPU tasks over 50ms)",
1716
property: "interactive",
1817
fn: id,
1918
type: BenchmarkType.STARTUP,
@@ -22,7 +21,7 @@ export const benchStartupConsistentlyInteractive: StartupBenchmarkInfo = {
2221
export const benchStartupBootup: StartupBenchmarkInfo = {
2322
id: "32_startup-bt",
2423
label: "script bootup time",
25-
description: () => "the total ms required to parse/compile/evaluate all the page's scripts",
24+
description: "the total ms required to parse/compile/evaluate all the page's scripts",
2625
property: "bootup-time",
2726
fn: id,
2827
type: BenchmarkType.STARTUP,
@@ -31,7 +30,7 @@ export const benchStartupBootup: StartupBenchmarkInfo = {
3130
export const benchStartupMainThreadWorkCost: StartupBenchmarkInfo = {
3231
id: "33_startup-mainthreadcost",
3332
label: "main thread work cost",
34-
description: () => "total amount of time spent doing work on the main thread. includes style/layout/etc.",
33+
description: "total amount of time spent doing work on the main thread. includes style/layout/etc.",
3534
property: "mainthread-work-breakdown",
3635
fn: id,
3736
type: BenchmarkType.STARTUP,
@@ -40,7 +39,7 @@ export const benchStartupMainThreadWorkCost: StartupBenchmarkInfo = {
4039
export const benchStartupMainInteractive: StartupBenchmarkInfo = {
4140
id: "34_startup-interactive",
4241
label: "interactive",
43-
description: () => "Time to Interactive is the amount of time it takes for the page to become fully interactive.",
42+
description: "Time to Interactive is the amount of time it takes for the page to become fully interactive.",
4443
property: "interactive",
4544
fn: id,
4645
type: BenchmarkType.STARTUP,

‎webdriver-ts/src/benchmarksPlaywright.ts

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ export let benchRun = new (class extends CPUBenchmarkPlaywright {
4242
}
4343
async init(browser: Browser, page: Page) {
4444
await checkElementExists(page, "#run");
45-
for (let i = 0; i < config.WARMUP_COUNT; i++) {
45+
for (let i = 0; i < this.benchmarkInfo.warmupCount; i++) {
4646
await clickElement(page, "#run");
4747
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", (i*1000+1).toFixed());
4848
await clickElement(page, "#clear");
@@ -51,7 +51,7 @@ export let benchRun = new (class extends CPUBenchmarkPlaywright {
5151
}
5252
async run(browser: Browser, page: Page) {
5353
await clickElement(page, "#run");
54-
await checkElementContainsText(page, "tbody>tr:nth-of-type(1000)>td:nth-of-type(1)", ((config.WARMUP_COUNT+1)*1000).toFixed());
54+
await checkElementContainsText(page, "tbody>tr:nth-of-type(1000)>td:nth-of-type(1)", ((this.benchmarkInfo.warmupCount+1)*1000).toFixed());
5555
}
5656
})();
5757

@@ -61,14 +61,14 @@ export const benchReplaceAll = new (class extends CPUBenchmarkPlaywright {
6161
}
6262
async init(browser: Browser, page: Page) {
6363
await checkElementExists(page, "#run");
64-
for (let i = 0; i < config.WARMUP_COUNT; i++) {
64+
for (let i = 0; i < this.benchmarkInfo.warmupCount; i++) {
6565
await clickElement(page, "#run");
6666
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", (i*1000+1).toFixed());
6767
}
6868
}
6969
async run(browser: Browser, page: Page) {
7070
await clickElement(page, "#run");
71-
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", `${config.WARMUP_COUNT * 1000 + 1}`);
71+
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", `${this.benchmarkInfo.warmupCount * 1000 + 1}`);
7272
}
7373
})();
7474

@@ -99,7 +99,7 @@ export const benchSelect = new (class extends CPUBenchmarkPlaywright {
9999
await checkElementExists(page, "#run");
100100
await clickElement(page, "#run");
101101
await checkElementContainsText(page, "tbody>tr:nth-of-type(1000)>td:nth-of-type(1)", "1000");
102-
for (let i = 0; i <= config.WARMUP_COUNT; i++) {
102+
for (let i = 0; i <= this.benchmarkInfo.warmupCount; i++) {
103103
await clickElement(page, `tbody>tr:nth-of-type(${i + 5})>td:nth-of-type(2)>a`);
104104
await checkElementHasClass(page, `tbody>tr:nth-of-type(${i + 5})`, "danger");
105105
await checkCountForSelector(page, "tbody>tr.danger", 1);
@@ -119,16 +119,16 @@ export const benchSwapRows = new (class extends CPUBenchmarkPlaywright {
119119
await checkElementExists(page, "#run");
120120
await clickElement(page, "#run");
121121
await checkElementExists(page, "tbody>tr:nth-of-type(1000)>td:nth-of-type(1)");
122-
for (let i = 0; i <= config.WARMUP_COUNT; i++) {
123-
let text = i % 2 == 0 ? "2" : "999";
124-
await clickElement(page, "#swaprows");
125-
await checkElementContainsText(page, "tbody>tr:nth-of-type(999)>td:nth-of-type(1)", text);
122+
for (let i = 0; i <= this.benchmarkInfo.warmupCount; i++) {
123+
let text = i % 2 == 0 ? "2" : "999";
124+
await clickElement(page, "#swaprows");
125+
await checkElementContainsText(page, "tbody>tr:nth-of-type(999)>td:nth-of-type(1)", text);
126126
}
127127
}
128128
async run(browser: Browser, page: Page) {
129129
await clickElement(page, "#swaprows");
130-
let text999 = config.WARMUP_COUNT % 2 == 0 ? "999" : "2";
131-
let text2 = config.WARMUP_COUNT % 2 == 0 ? "2" : "999";
130+
let text999 = this.benchmarkInfo.warmupCount % 2 == 0 ? "999" : "2";
131+
let text2 = this.benchmarkInfo.warmupCount % 2 == 0 ? "2" : "999";
132132
await checkElementContainsText(page, "tbody>tr:nth-of-type(999)>td:nth-of-type(1)", text999);
133133
await checkElementContainsText(page, "tbody>tr:nth-of-type(2)>td:nth-of-type(1)", text2);
134134
}
@@ -143,23 +143,23 @@ export const benchRemove = new (class extends CPUBenchmarkPlaywright {
143143
await checkElementExists(page, "#run");
144144
await clickElement(page, "#run");
145145
await checkElementExists(page, "tbody>tr:nth-of-type(1000)>td:nth-of-type(1)");
146-
for (let i = 0; i < config.WARMUP_COUNT; i++) {
147-
const rowToClick = config.WARMUP_COUNT - i + this.rowsToSkip;
146+
for (let i = 0; i < this.benchmarkInfo.warmupCount; i++) {
147+
const rowToClick = this.benchmarkInfo.warmupCount - i + this.rowsToSkip;
148148
await checkElementContainsText(page, `tbody>tr:nth-of-type(${rowToClick})>td:nth-of-type(1)`, rowToClick.toString());
149149
await clickElement(page, `tbody>tr:nth-of-type(${rowToClick})>td:nth-of-type(3)>a>span:nth-of-type(1)`);
150-
await checkElementContainsText(page, `tbody>tr:nth-of-type(${rowToClick})>td:nth-of-type(1)`, `${this.rowsToSkip + config.WARMUP_COUNT + 1}`);
150+
await checkElementContainsText(page, `tbody>tr:nth-of-type(${rowToClick})>td:nth-of-type(1)`, `${this.rowsToSkip + this.benchmarkInfo.warmupCount + 1}`);
151151
}
152-
await checkElementContainsText(page, `tbody>tr:nth-of-type(${this.rowsToSkip + 1})>td:nth-of-type(1)`, `${this.rowsToSkip + config.WARMUP_COUNT + 1}`);
152+
await checkElementContainsText(page, `tbody>tr:nth-of-type(${this.rowsToSkip + 1})>td:nth-of-type(1)`, `${this.rowsToSkip + this.benchmarkInfo.warmupCount + 1}`);
153153
await checkElementContainsText(page, `tbody>tr:nth-of-type(${this.rowsToSkip})>td:nth-of-type(1)`, `${this.rowsToSkip}`);
154154

155155
// Click on a row the second time
156-
await checkElementContainsText(page, `tbody>tr:nth-of-type(${this.rowsToSkip + 2})>td:nth-of-type(1)`, `${this.rowsToSkip + config.WARMUP_COUNT + 2}`);
156+
await checkElementContainsText(page, `tbody>tr:nth-of-type(${this.rowsToSkip + 2})>td:nth-of-type(1)`, `${this.rowsToSkip + this.benchmarkInfo.warmupCount + 2}`);
157157
await clickElement(page, `tbody>tr:nth-of-type(${this.rowsToSkip + 2})>td:nth-of-type(3)>a>span:nth-of-type(1)`);
158-
await checkElementContainsText(page, `tbody>tr:nth-of-type(${this.rowsToSkip + 2})>td:nth-of-type(1)`, `${this.rowsToSkip + config.WARMUP_COUNT + 3}`);
158+
await checkElementContainsText(page, `tbody>tr:nth-of-type(${this.rowsToSkip + 2})>td:nth-of-type(1)`, `${this.rowsToSkip + this.benchmarkInfo.warmupCount + 3}`);
159159
}
160160
async run(browser: Browser, page: Page) {
161161
await clickElement(page, `tbody>tr:nth-of-type(${this.rowsToSkip})>td:nth-of-type(3)>a>span:nth-of-type(1)`);
162-
await checkElementContainsText(page, `tbody>tr:nth-of-type(${this.rowsToSkip})>td:nth-of-type(1)`, `${this.rowsToSkip + config.WARMUP_COUNT + 1}`);
162+
await checkElementContainsText(page, `tbody>tr:nth-of-type(${this.rowsToSkip})>td:nth-of-type(1)`, `${this.rowsToSkip + this.benchmarkInfo.warmupCount + 1}`);
163163
}
164164
})();
165165

@@ -170,7 +170,7 @@ export const benchRunBig = new (class extends CPUBenchmarkPlaywright {
170170
}
171171
async init(browser: Browser, page: Page) {
172172
await checkElementExists(page, "#run");
173-
for (let i = 0; i < config.WARMUP_COUNT; i++) {
173+
for (let i = 0; i < this.benchmarkInfo.warmupCount; i++) {
174174
await clickElement(page, "#run");
175175
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", (i*1000+1).toFixed());
176176
await clickElement(page, "#clear");
@@ -189,7 +189,7 @@ export const benchAppendToManyRows = new (class extends CPUBenchmarkPlaywright {
189189
}
190190
async init(browser: Browser, page: Page) {
191191
await checkElementExists(page, "#run");
192-
for (let i = 0; i < config.WARMUP_COUNT; i++) {
192+
for (let i = 0; i < this.benchmarkInfo.warmupCount; i++) {
193193
await clickElement(page, "#run");
194194
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", (i*1000+1).toFixed());
195195
await clickElement(page, "#clear");
@@ -210,14 +210,14 @@ export const benchClear = new (class extends CPUBenchmarkPlaywright {
210210
}
211211
async init(browser: Browser, page: Page) {
212212
await checkElementExists(page, "#run");
213-
for (let i = 0; i < config.WARMUP_COUNT; i++) {
213+
for (let i = 0; i < this.benchmarkInfo.warmupCount; i++) {
214214
await clickElement(page, "#run");
215215
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", (i*1000+1).toFixed());
216216
await clickElement(page, "#clear");
217217
await checkElementNotExists(page, "tbody>tr:nth-of-type(1000)>td:nth-of-type(1)");
218218
}
219219
await clickElement(page, "#run");
220-
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", (config.WARMUP_COUNT*1000+1).toFixed());
220+
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", (this.benchmarkInfo.warmupCount*1000+1).toFixed());
221221
}
222222
async run(browser: Browser, page: Page) {
223223
await clickElement(page, "#clear");

0 commit comments

Comments
 (0)