Skip to content

Commit 4bf783f

Browse files
twharmontwharmon
committed
Make warmup count configurable for each benchmark
1 parent 1365139 commit 4bf783f

10 files changed

+112
-110
lines changed

‎webdriver-ts/src/benchmarksCommon.ts

Lines changed: 31 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ export enum BenchmarkType {
1212
export interface BenchmarkInfoBase {
1313
id: string;
1414
label: string;
15-
description(throttleCPU: number | undefined): string;
15+
description: string;
1616
type: BenchmarkType;
1717
}
1818

@@ -21,6 +21,7 @@ export interface CPUBenchmarkInfo extends BenchmarkInfoBase {
2121
type: BenchmarkType.CPU;
2222
layoutEventRequired: boolean;
2323
additionalNumberOfRuns: number;
24+
warmupCount: number
2425
}
2526

2627
export interface MemBenchmarkInfo extends BenchmarkInfoBase {
@@ -113,6 +114,10 @@ export function slowDownNote(throttleCPU: number | undefined): string {
113114
return throttleCPU ? ` ${throttleCPU} x CPU slowdown.` : "";
114115
}
115116

117+
export function warmupNote(b: BenchmarkInfo): string {
118+
return 'warmupCount' in b ? ` (${b.warmupCount} warmup runs).` : "";
119+
}
120+
116121
export function slowDownFactor(benchmarkId: string, allowThrottling: boolean): number | undefined {
117122
if (!allowThrottling) return undefined;
118123
return throttlingFactors[benchmarkId];
@@ -122,8 +127,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
122127
{
123128
id: Benchmark._01,
124129
label: "create rows",
125-
description: (throttleCPU: number | undefined) =>
126-
"creating 1,000 rows (" + config.WARMUP_COUNT + " warmup runs)." + slowDownNote(throttleCPU),
130+
warmupCount: 2,
131+
description: "creating 1,000 rows.",
127132
type: BenchmarkType.CPU,
128133
allowBatching: true,
129134
layoutEventRequired: true,
@@ -132,8 +137,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
132137
{
133138
id: Benchmark._02,
134139
label: "replace all rows",
135-
description: (throttleCPU: number | undefined) =>
136-
"updating all 1,000 rows (" + config.WARMUP_COUNT + " warmup runs)." + slowDownNote(throttleCPU),
140+
warmupCount: 2,
141+
description: "updating all 1,000 rows.",
137142
type: BenchmarkType.CPU,
138143
allowBatching: true,
139144
layoutEventRequired: true,
@@ -142,8 +147,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
142147
{
143148
id: Benchmark._03,
144149
label: "partial update",
145-
description: (throttleCPU: number | undefined) =>
146-
"updating every 10th row for 1,000 rows (3 warmup runs)." + slowDownNote(throttleCPU),
150+
warmupCount: 3,
151+
description: "updating every 10th row for 1,000 row.",
147152
type: BenchmarkType.CPU,
148153
allowBatching: true,
149154
layoutEventRequired: true,
@@ -152,8 +157,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
152157
{
153158
id: Benchmark._04,
154159
label: "select row",
155-
description: (throttleCPU: number | undefined) =>
156-
"highlighting a selected row. (" + config.WARMUP_COUNT + " warmup runs)." + slowDownNote(throttleCPU),
160+
warmupCount: 2,
161+
description: "highlighting a selected row.",
157162
type: BenchmarkType.CPU,
158163
allowBatching: true,
159164
layoutEventRequired: false,
@@ -162,8 +167,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
162167
{
163168
id: Benchmark._05,
164169
label: "swap rows",
165-
description: (throttleCPU: number | undefined) =>
166-
"swap 2 rows for table with 1,000 rows. (" + config.WARMUP_COUNT + " warmup runs)." + slowDownNote(throttleCPU),
170+
warmupCount: 2,
171+
description: "swap 2 rows for table with 1,000 rows.",
167172
type: BenchmarkType.CPU,
168173
allowBatching: true,
169174
layoutEventRequired: true,
@@ -172,8 +177,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
172177
{
173178
id: Benchmark._06,
174179
label: "remove row",
175-
description: (throttleCPU: number | undefined) =>
176-
"removing one row. (" + config.WARMUP_COUNT + " warmup runs)." + slowDownNote(throttleCPU),
180+
warmupCount: 2,
181+
description: "removing one row.",
177182
type: BenchmarkType.CPU,
178183
allowBatching: true,
179184
layoutEventRequired: true,
@@ -182,8 +187,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
182187
{
183188
id: Benchmark._07,
184189
label: "create many rows",
185-
description: (throttleCPU: number | undefined) =>
186-
"creating 10,000 rows. (" + config.WARMUP_COUNT + " warmup runs with 1k rows)." + slowDownNote(throttleCPU),
190+
warmupCount: 2,
191+
description: "creating 10,000 rows.",
187192
type: BenchmarkType.CPU,
188193
allowBatching: true,
189194
layoutEventRequired: true,
@@ -192,8 +197,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
192197
{
193198
id: Benchmark._08,
194199
label: "append rows to large table",
195-
description: (throttleCPU: number | undefined) =>
196-
"appending 1,000 to a table of 1,000 rows." + slowDownNote(throttleCPU),
200+
warmupCount: 2,
201+
description: "appending 1,000 to a table of 1,000 rows.",
197202
type: BenchmarkType.CPU,
198203
allowBatching: true,
199204
layoutEventRequired: true,
@@ -202,8 +207,8 @@ export const cpuBenchmarkInfosArray: Array<CPUBenchmarkInfo> = [
202207
{
203208
id: Benchmark._09,
204209
label: "clear rows",
205-
description: (throttleCPU: number | undefined) =>
206-
"clearing a table with 1,000 rows." + slowDownNote(throttleCPU) + " (" + config.WARMUP_COUNT + " warmup runs).",
210+
warmupCount: 2,
211+
description: "clearing a table with 1,000 rows.",
207212
type: BenchmarkType.CPU,
208213
allowBatching: true,
209214
layoutEventRequired: true,
@@ -215,19 +220,19 @@ export const memBenchmarkInfosArray: Array<MemBenchmarkInfo> = [
215220
{
216221
id: Benchmark._21,
217222
label: "ready memory",
218-
description: () => "Memory usage after page load.",
223+
description: "Memory usage after page load.",
219224
type: BenchmarkType.MEM,
220225
},
221226
{
222227
id: Benchmark._22,
223228
label: "run memory",
224-
description: () => "Memory usage after adding 1,000 rows.",
229+
description: "Memory usage after adding 1,000 rows.",
225230
type: BenchmarkType.MEM,
226231
},
227232
{
228233
id: Benchmark._23,
229234
label: "update every 10th row for 1k rows (5 cycles)",
230-
description: () => "Memory usage after clicking update every 10th row 5 times",
235+
description: "Memory usage after clicking update every 10th row 5 times",
231236
type: BenchmarkType.MEM,
232237
},
233238
// {
@@ -239,13 +244,13 @@ export const memBenchmarkInfosArray: Array<MemBenchmarkInfo> = [
239244
{
240245
id: Benchmark._25,
241246
label: "creating/clearing 1k rows (5 cycles)",
242-
description: () => "Memory usage after creating and clearing 1000 rows 5 times",
247+
description: "Memory usage after creating and clearing 1000 rows 5 times",
243248
type: BenchmarkType.MEM,
244249
},
245250
{
246251
id: Benchmark._26,
247252
label: "run memory 10k",
248-
description: () => "Memory usage after adding 10,000 rows.",
253+
description: "Memory usage after adding 10,000 rows.",
249254
type: BenchmarkType.MEM,
250255
},
251256
];
@@ -255,7 +260,7 @@ export const startupBenchmarkInfosArray: Array<StartupMainBenchmarkInfo> = [
255260
id: Benchmark._30,
256261
type: BenchmarkType.STARTUP_MAIN,
257262
label: "",
258-
description: () => "",
263+
description: "",
259264
}
260265
];
261266

@@ -264,7 +269,7 @@ export const sizesBenchmarkInfosArray: Array<SizeMainBenchmarkInfo> = [
264269
id: Benchmark._40,
265270
type: BenchmarkType.SIZE_MAIN,
266271
label: "",
267-
description: () => "",
272+
description: "",
268273
},
269274
];
270275

‎webdriver-ts/src/benchmarksLighthouse.ts

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,7 @@ let toKb = (x: number) => x / 1024;
1212
export const benchStartupConsistentlyInteractive: StartupBenchmarkInfo = {
1313
id: "31_startup-ci",
1414
label: "consistently interactive",
15-
description: () =>
16-
"a pessimistic TTI - when the CPU and network are both definitely very idle. (no more CPU tasks over 50ms)",
15+
description: "a pessimistic TTI - when the CPU and network are both definitely very idle. (no more CPU tasks over 50ms)",
1716
property: "interactive",
1817
fn: id,
1918
type: BenchmarkType.STARTUP,
@@ -22,7 +21,7 @@ export const benchStartupConsistentlyInteractive: StartupBenchmarkInfo = {
2221
export const benchStartupBootup: StartupBenchmarkInfo = {
2322
id: "32_startup-bt",
2423
label: "script bootup time",
25-
description: () => "the total ms required to parse/compile/evaluate all the page's scripts",
24+
description: "the total ms required to parse/compile/evaluate all the page's scripts",
2625
property: "bootup-time",
2726
fn: id,
2827
type: BenchmarkType.STARTUP,
@@ -31,7 +30,7 @@ export const benchStartupBootup: StartupBenchmarkInfo = {
3130
export const benchStartupMainThreadWorkCost: StartupBenchmarkInfo = {
3231
id: "33_startup-mainthreadcost",
3332
label: "main thread work cost",
34-
description: () => "total amount of time spent doing work on the main thread. includes style/layout/etc.",
33+
description: "total amount of time spent doing work on the main thread. includes style/layout/etc.",
3534
property: "mainthread-work-breakdown",
3635
fn: id,
3736
type: BenchmarkType.STARTUP,
@@ -40,7 +39,7 @@ export const benchStartupMainThreadWorkCost: StartupBenchmarkInfo = {
4039
export const benchStartupTotalBytes: StartupBenchmarkInfo = {
4140
id: "34_startup-totalbytes",
4241
label: "total kilobyte weight",
43-
description: () => "network transfer cost (post-compression) of all the resources loaded into the page.",
42+
description: "network transfer cost (post-compression) of all the resources loaded into the page.",
4443
property: "total-byte-weight",
4544
fn: toKb,
4645
type: BenchmarkType.STARTUP,

‎webdriver-ts/src/benchmarksPlaywright.ts

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ export let benchRun = new (class extends CPUBenchmarkPlaywright {
4242
}
4343
async init(browser: Browser, page: Page) {
4444
await checkElementExists(page, "#run");
45-
for (let i = 0; i < config.WARMUP_COUNT; i++) {
45+
for (let i = 0; i < this.benchmarkInfo.warmupCount; i++) {
4646
await clickElement(page, "#run");
4747
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", (i*1000+1).toFixed());
4848
await clickElement(page, "#clear");
@@ -51,7 +51,7 @@ export let benchRun = new (class extends CPUBenchmarkPlaywright {
5151
}
5252
async run(browser: Browser, page: Page) {
5353
await clickElement(page, "#run");
54-
await checkElementContainsText(page, "tbody>tr:nth-of-type(1000)>td:nth-of-type(1)", ((config.WARMUP_COUNT+1)*1000).toFixed());
54+
await checkElementContainsText(page, "tbody>tr:nth-of-type(1000)>td:nth-of-type(1)", ((this.benchmarkInfo.warmupCount+1)*1000).toFixed());
5555
}
5656
})();
5757

@@ -61,7 +61,7 @@ export const benchReplaceAll = new (class extends CPUBenchmarkPlaywright {
6161
}
6262
async init(browser: Browser, page: Page) {
6363
await checkElementExists(page, "#run");
64-
for (let i = 0; i < config.WARMUP_COUNT; i++) {
64+
for (let i = 0; i < this.benchmarkInfo.warmupCount; i++) {
6565
await clickElement(page, "#run");
6666
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", (i*1000+1).toFixed());
6767
}
@@ -99,7 +99,7 @@ export const benchSelect = new (class extends CPUBenchmarkPlaywright {
9999
await checkElementExists(page, "#run");
100100
await clickElement(page, "#run");
101101
await checkElementContainsText(page, "tbody>tr:nth-of-type(1000)>td:nth-of-type(1)", "1000");
102-
for (let i = 0; i <= config.WARMUP_COUNT; i++) {
102+
for (let i = 0; i <= this.benchmarkInfo.warmupCount; i++) {
103103
await clickElement(page, `tbody>tr:nth-of-type(${i + 5})>td:nth-of-type(2)>a`);
104104
await checkElementHasClass(page, `tbody>tr:nth-of-type(${i + 5})`, "danger");
105105
await checkCountForSelector(page, "tbody>tr.danger", 1);
@@ -119,7 +119,7 @@ export const benchSwapRows = new (class extends CPUBenchmarkPlaywright {
119119
await checkElementExists(page, "#run");
120120
await clickElement(page, "#run");
121121
await checkElementExists(page, "tbody>tr:nth-of-type(1000)>td:nth-of-type(1)");
122-
for (let i = 0; i <= config.WARMUP_COUNT; i++) {
122+
for (let i = 0; i <= this.benchmarkInfo.warmupCount; i++) {
123123
let text = i % 2 == 0 ? "2" : "999";
124124
await clickElement(page, "#swaprows");
125125
await checkElementContainsText(page, "tbody>tr:nth-of-type(999)>td:nth-of-type(1)", text);
@@ -140,10 +140,10 @@ export const benchRemove = new (class extends CPUBenchmarkPlaywright {
140140
await checkElementExists(page, "#run");
141141
await clickElement(page, "#run");
142142
await checkElementExists(page, "tbody>tr:nth-of-type(1000)>td:nth-of-type(1)");
143-
for (let i = 0; i < config.WARMUP_COUNT; i++) {
144-
await checkElementContainsText(page, `tbody>tr:nth-of-type(${config.WARMUP_COUNT - i + 4})>td:nth-of-type(1)`, (config.WARMUP_COUNT - i + 4).toString());
145-
await clickElement(page, `tbody>tr:nth-of-type(${config.WARMUP_COUNT - i + 4})>td:nth-of-type(3)>a>span:nth-of-type(1)`);
146-
await checkElementContainsText(page, `tbody>tr:nth-of-type(${config.WARMUP_COUNT - i + 4})>td:nth-of-type(1)`, "10");
143+
for (let i = 0; i < this.benchmarkInfo.warmupCount; i++) {
144+
await checkElementContainsText(page, `tbody>tr:nth-of-type(${this.benchmarkInfo.warmupCount - i + 4})>td:nth-of-type(1)`, (this.benchmarkInfo.warmupCount - i + 4).toString());
145+
await clickElement(page, `tbody>tr:nth-of-type(${this.benchmarkInfo.warmupCount - i + 4})>td:nth-of-type(3)>a>span:nth-of-type(1)`);
146+
await checkElementContainsText(page, `tbody>tr:nth-of-type(${this.benchmarkInfo.warmupCount - i + 4})>td:nth-of-type(1)`, "10");
147147
}
148148
await checkElementContainsText(page, `tbody>tr:nth-of-type(5)>td:nth-of-type(1)`, "10");
149149
await checkElementContainsText(page, `tbody>tr:nth-of-type(4)>td:nth-of-type(1)`, "4");
@@ -165,7 +165,7 @@ export const benchRunBig = new (class extends CPUBenchmarkPlaywright {
165165
}
166166
async init(browser: Browser, page: Page) {
167167
await checkElementExists(page, "#run");
168-
for (let i = 0; i < config.WARMUP_COUNT; i++) {
168+
for (let i = 0; i < this.benchmarkInfo.warmupCount; i++) {
169169
await clickElement(page, "#run");
170170
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", (i*1000+1).toFixed());
171171
await clickElement(page, "#clear");
@@ -184,7 +184,7 @@ export const benchAppendToManyRows = new (class extends CPUBenchmarkPlaywright {
184184
}
185185
async init(browser: Browser, page: Page) {
186186
await checkElementExists(page, "#run");
187-
for (let i = 0; i < config.WARMUP_COUNT; i++) {
187+
for (let i = 0; i < this.benchmarkInfo.warmupCount; i++) {
188188
await clickElement(page, "#run");
189189
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", (i*1000+1).toFixed());
190190
await clickElement(page, "#clear");
@@ -205,14 +205,14 @@ export const benchClear = new (class extends CPUBenchmarkPlaywright {
205205
}
206206
async init(browser: Browser, page: Page) {
207207
await checkElementExists(page, "#run");
208-
for (let i = 0; i < config.WARMUP_COUNT; i++) {
208+
for (let i = 0; i < this.benchmarkInfo.warmupCount; i++) {
209209
await clickElement(page, "#run");
210210
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", (i*1000+1).toFixed());
211211
await clickElement(page, "#clear");
212212
await checkElementNotExists(page, "tbody>tr:nth-of-type(1000)>td:nth-of-type(1)");
213213
}
214214
await clickElement(page, "#run");
215-
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", (config.WARMUP_COUNT*1000+1).toFixed());
215+
await checkElementContainsText(page, "tbody>tr:nth-of-type(1)>td:nth-of-type(1)", (this.benchmarkInfo.warmupCount*1000+1).toFixed());
216216
}
217217
async run(browser: Browser, page: Page) {
218218
await clickElement(page, "#clear");

0 commit comments

Comments
 (0)