Skip to content

Commit e05443c

Browse files
camillobrunicamillobruni
authored
Add startup benchmark helper (WebKit#158)
Add startup-helper folder: - StartupBenchmark handles source code creation - BabelCacheBuster.js is a babel plugin that injects a known comment into every function body - Add unit tests for StartupBenchmark
1 parent 4b8dff1 commit e05443c

File tree

3 files changed

+286
-23
lines changed

3 files changed

+286
-23
lines changed

‎startup-helper/BabelCacheBuster.js‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
// Babel plugin that adds CACHE_BUST_COMMENT to every function body.
2+
const CACHE_BUST_COMMENT = "ThouShaltNotCache";
3+
4+
module.exports = function ({ types: t }) {
5+
return {
6+
visitor: {
7+
Function(path) {
8+
const bodyPath = path.get("body");
9+
// Handle arrow functions: () => "value"
10+
// Convert them to block statements: () => { return "value"; }
11+
if (!bodyPath.isBlockStatement()) {
12+
const newBody = t.blockStatement([t.returnStatement(bodyPath.node)]);
13+
path.set("body", newBody);
14+
}
15+
16+
// Handle empty function bodies: function foo() {}
17+
// Add an empty statement so we have a first node to attach the comment to.
18+
if (path.get("body.body").length === 0) {
19+
path.get("body").pushContainer("body", t.emptyStatement());
20+
}
21+
22+
const firstNode = path.node.body.body[0];
23+
t.addComment(firstNode, "leading", CACHE_BUST_COMMENT);
24+
},
25+
},
26+
};
27+
};

‎startup-helper/StartupBenchmark.js‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
const CACHE_BUST_COMMENT = "/*ThouShaltNotCache*/";
2+
const CACHE_BUST_COMMENT_RE = new RegExp(
3+
`\n${RegExp.escape(CACHE_BUST_COMMENT)}\n`,
4+
"g"
5+
);
6+
7+
class StartupBenchmark {
8+
// Total iterations for this benchmark.
9+
#iterationCount = 0;
10+
// Original source code.
11+
#sourceCode;
12+
// quickHahs(this.#sourceCode) for use in custom validate() methods.
13+
#sourceHash = 0;
14+
// Number of no-cache comments in the original #sourceCode.
15+
#expectedCacheCommentCount = 0;
16+
// How many times (separate iterations) should we reuse the source code.
17+
// Use 0 to skip and only use a single #sourceCode string.
18+
#sourceCodeReuseCount = 1;
19+
// #sourceCode for each iteration, number of unique sources is controlled
20+
// by codeReuseCount;
21+
#iterationSourceCodes = [];
22+
23+
constructor({
24+
iterationCount,
25+
expectedCacheCommentCount,
26+
sourceCodeReuseCount = 1,
27+
} = {}) {
28+
console.assert(iterationCount > 0);
29+
this.#iterationCount = iterationCount;
30+
console.assert(expectedCacheCommentCount > 0);
31+
this.#expectedCacheCommentCount = expectedCacheCommentCount;
32+
console.assert(sourceCodeReuseCount >= 0);
33+
this.#sourceCodeReuseCount = sourceCodeReuseCount;
34+
}
35+
36+
get iterationCount() {
37+
return this.#iterationCount;
38+
}
39+
40+
get sourceCode() {
41+
return this.#sourceCode;
42+
}
43+
44+
get sourceHash() {
45+
return this.#sourceHash;
46+
}
47+
48+
get expectedCacheCommentCount() {
49+
return this.#expectedCacheCommentCount;
50+
}
51+
52+
get sourceCodeReuseCount() {
53+
return this.#sourceCodeReuseCount;
54+
}
55+
56+
get iterationSourceCodes() {
57+
return this.#iterationSourceCodes;
58+
}
59+
60+
async init() {
61+
this.#sourceCode = await JetStream.getString(JetStream.preload.BUNDLE);
62+
const cacheCommentCount = this.sourceCode.match(
63+
CACHE_BUST_COMMENT_RE
64+
).length;
65+
this.#sourceHash = this.quickHash(this.sourceCode);
66+
this.validateSourceCacheComments(cacheCommentCount);
67+
for (let i = 0; i < this.iterationCount; i++)
68+
this.#iterationSourceCodes[i] = this.createIterationSourceCode(i);
69+
this.validateIterationSourceCodes();
70+
}
71+
72+
validateSourceCacheComments(cacheCommentCount) {
73+
console.assert(
74+
cacheCommentCount === this.expectedCacheCommentCount,
75+
`Invalid cache comment count ${cacheCommentCount} expected ${this.expectedCacheCommentCount}.`
76+
);
77+
}
78+
79+
validateIterationSourceCodes() {
80+
if (this.#iterationSourceCodes.some((each) => !each?.length))
81+
throw new Error(`Got invalid iterationSourceCodes`);
82+
let expectedSize = 1;
83+
if (this.sourceCodeReuseCount !== 0)
84+
expectedSize = Math.ceil(this.iterationCount / this.sourceCodeReuseCount);
85+
const uniqueSources = new Set(this.iterationSourceCodes);
86+
if (uniqueSources.size != expectedSize)
87+
throw new Error(
88+
`Expected ${expectedSize} unique sources, but got ${uniqueSources.size}.`
89+
);
90+
}
91+
92+
createIterationSourceCode(iteration) {
93+
// Alter the code per iteration to prevent caching.
94+
const cacheId =
95+
Math.floor(iteration / this.sourceCodeReuseCount) *
96+
this.sourceCodeReuseCount;
97+
// Reuse existing sources if this.codeReuseCount > 1:
98+
if (cacheId < this.iterationSourceCodes.length)
99+
return this.iterationSourceCodes[cacheId];
100+
101+
const sourceCode = this.sourceCode.replaceAll(
102+
CACHE_BUST_COMMENT_RE,
103+
`/*${cacheId}*/`
104+
);
105+
// Warm up quickHash.
106+
this.quickHash(sourceCode);
107+
return sourceCode;
108+
}
109+
110+
quickHash(str) {
111+
let hash = 5381;
112+
let i = str.length;
113+
while (i > 0) {
114+
hash = (hash * 33) ^ (str.charCodeAt(i) | 0);
115+
i -= 919;
116+
}
117+
return hash | 0;
118+
}
119+
}

‎tests/unit-tests.js‎

Lines changed: 140 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
load("shell-config.js")
1+
load("shell-config.js");
2+
load("startup-helper/StartupBenchmark.js");
23
load("JetStreamDriver.js");
34

45
function assertTrue(condition, message) {
@@ -19,17 +20,25 @@ function assertEquals(actual, expected, message) {
1920
}
2021
}
2122

23+
function assertThrows(message, func) {
24+
let didThrow = false;
25+
try {
26+
func();
27+
} catch (e) {
28+
didThrow = true;
29+
}
30+
assertTrue(didThrow, message);
31+
}
32+
2233
(function testTagsAreLowerCaseStrings() {
2334
for (const benchmark of BENCHMARKS) {
24-
benchmark.tags.forEach(tag => {
25-
assertTrue(typeof(tag) == "string");
26-
assertTrue(tag == tag.toLowerCase());
27-
})
35+
benchmark.tags.forEach((tag) => {
36+
assertTrue(typeof tag == "string");
37+
assertTrue(tag == tag.toLowerCase());
38+
});
2839
}
2940
})();
3041

31-
32-
3342
(function testTagsAll() {
3443
for (const benchmark of BENCHMARKS) {
3544
const tags = benchmark.tags;
@@ -41,53 +50,56 @@ function assertEquals(actual, expected, message) {
4150
}
4251
})();
4352

44-
4553
(function testDriverBenchmarksOrder() {
4654
const benchmarks = findBenchmarksByTag("all");
4755
const driver = new Driver(benchmarks);
4856
assertEquals(driver.benchmarks.length, BENCHMARKS.length);
49-
const names = driver.benchmarks.map(b => b.name.toLowerCase()).sort().reverse();
57+
const names = driver.benchmarks
58+
.map((b) => b.name.toLowerCase())
59+
.sort()
60+
.reverse();
5061
for (let i = 0; i < names.length; i++) {
5162
assertEquals(driver.benchmarks[i].name.toLowerCase(), names[i]);
5263
}
5364
})();
5465

55-
5666
(function testEnableByTag() {
5767
const driverA = new Driver(findBenchmarksByTag("Default"));
5868
const driverB = new Driver(findBenchmarksByTag("default"));
5969
assertTrue(driverA.benchmarks.length > 0);
6070
assertEquals(driverA.benchmarks.length, driverB.benchmarks.length);
6171
const enabledBenchmarkNames = new Set(
62-
Array.from(driverA.benchmarks).map(b => b.name));
72+
Array.from(driverA.benchmarks).map((b) => b.name)
73+
);
6374
for (const benchmark of BENCHMARKS) {
6475
if (benchmark.tags.has("default"))
6576
assertTrue(enabledBenchmarkNames.has(benchmark.name));
6677
}
6778
})();
6879

69-
7080
(function testDriverEnableDuplicateAndSort() {
71-
const benchmarks = [...findBenchmarksByTag("wasm"), ...findBenchmarksByTag("wasm")];
72-
assertTrue(benchmarks.length > 0);
73-
const uniqueBenchmarks = new Set(benchmarks);
74-
assertFalse(uniqueBenchmarks.size == benchmarks.length);
75-
const driver = new Driver(benchmarks);
76-
assertEquals(driver.benchmarks.length, uniqueBenchmarks.size);
81+
const benchmarks = [
82+
...findBenchmarksByTag("wasm"),
83+
...findBenchmarksByTag("wasm"),
84+
];
85+
assertTrue(benchmarks.length > 0);
86+
const uniqueBenchmarks = new Set(benchmarks);
87+
assertFalse(uniqueBenchmarks.size == benchmarks.length);
88+
const driver = new Driver(benchmarks);
89+
assertEquals(driver.benchmarks.length, uniqueBenchmarks.size);
7790
})();
7891

79-
8092
(function testBenchmarkSubScores() {
8193
for (const benchmark of BENCHMARKS) {
8294
const subScores = benchmark.subScores();
8395
assertTrue(subScores instanceof Object);
8496
assertTrue(Object.keys(subScores).length > 0);
8597
for (const [name, value] of Object.entries(subScores)) {
86-
assertTrue(typeof(name) == "string");
98+
assertTrue(typeof name == "string");
8799
// "Score" can only be part of allScores().
88100
assertFalse(name == "Score");
89101
// Without running values should be either null (or 0 for GroupedBenchmark)
90-
assertFalse(value)
102+
assertFalse(value);
91103
}
92104
}
93105
})();
@@ -98,7 +110,112 @@ function assertEquals(actual, expected, message) {
98110
const allScores = benchmark.allScores();
99111
assertTrue("Score" in allScores);
100112
// All subScore items are part of allScores.
101-
for (const name of Object.keys(subScores))
102-
assertTrue(name in allScores);
113+
for (const name of Object.keys(subScores)) assertTrue(name in allScores);
103114
}
104115
})();
116+
117+
function validateIterationSources(sources) {
118+
for (const source of sources) {
119+
assertTrue(typeof source == "string");
120+
assertFalse(source.includes(CACHE_BUST_COMMENT));
121+
}
122+
}
123+
124+
(async function testStartupBenchmark() {
125+
try {
126+
JetStream.preload = { BUNDLE: "test-bundle.js" };
127+
JetStream.getString = (file) => {
128+
assertEquals(file, "test-bundle.js");
129+
return `function test() {
130+
${CACHE_BUST_COMMENT}
131+
return 1;
132+
}`;
133+
};
134+
await testStartupBenchmarkInnerTests();
135+
} finally {
136+
JetStream.preload = undefined;
137+
JetStream.getString = undefined;
138+
}
139+
})();
140+
141+
async function testStartupBenchmarkInnerTests() {
142+
const benchmark = new StartupBenchmark({
143+
iterationCount: 12,
144+
expectedCacheCommentCount: 1,
145+
});
146+
assertEquals(benchmark.iterationCount, 12);
147+
assertEquals(benchmark.expectedCacheCommentCount, 1);
148+
assertEquals(benchmark.iterationSourceCodes.length, 0);
149+
assertEquals(benchmark.sourceCode, undefined);
150+
assertEquals(benchmark.sourceHash, 0);
151+
await benchmark.init();
152+
assertEquals(benchmark.sourceHash, 177573);
153+
assertEquals(benchmark.sourceCode.length, 68);
154+
assertEquals(benchmark.iterationSourceCodes.length, 12);
155+
assertEquals(new Set(benchmark.iterationSourceCodes).size, 12);
156+
validateIterationSources(benchmark.iterationSourceCodes);
157+
158+
const noReuseBenchmark = new StartupBenchmark({
159+
iterationCount: 12,
160+
expectedCacheCommentCount: 1,
161+
sourceCodeReuseCount: 0,
162+
});
163+
assertEquals(noReuseBenchmark.iterationSourceCodes.length, 0);
164+
await noReuseBenchmark.init();
165+
assertEquals(noReuseBenchmark.iterationSourceCodes.length, 12);
166+
assertEquals(new Set(noReuseBenchmark.iterationSourceCodes).size, 1);
167+
validateIterationSources(noReuseBenchmark.iterationSourceCodes);
168+
169+
const reuseBenchmark = new StartupBenchmark({
170+
iterationCount: 12,
171+
expectedCacheCommentCount: 1,
172+
sourceCodeReuseCount: 3,
173+
});
174+
assertEquals(reuseBenchmark.iterationSourceCodes.length, 0);
175+
await reuseBenchmark.init();
176+
assertEquals(reuseBenchmark.iterationSourceCodes.length, 12);
177+
assertEquals(new Set(reuseBenchmark.iterationSourceCodes).size, 4);
178+
validateIterationSources(reuseBenchmark.iterationSourceCodes);
179+
180+
const reuseBenchmark2 = new StartupBenchmark({
181+
iterationCount: 12,
182+
expectedCacheCommentCount: 1,
183+
sourceCodeReuseCount: 5,
184+
});
185+
assertEquals(reuseBenchmark2.iterationSourceCodes.length, 0);
186+
await reuseBenchmark2.init();
187+
assertEquals(reuseBenchmark2.iterationSourceCodes.length, 12);
188+
assertEquals(new Set(reuseBenchmark2.iterationSourceCodes).size, 3);
189+
validateIterationSources(reuseBenchmark2.iterationSourceCodes);
190+
}
191+
192+
(function testStartupBenchmarkThrow() {
193+
assertThrows(
194+
"StartupBenchmark constructor should throw with no arguments.",
195+
() => new StartupBenchmark()
196+
);
197+
198+
assertThrows(
199+
"StartupBenchmark constructor should throw with missing expectedCacheCommentCount.",
200+
() => new StartupBenchmark({ iterationCount: 1 })
201+
);
202+
203+
assertThrows(
204+
"StartupBenchmark constructor should throw with missing iterationCount.",
205+
() => new StartupBenchmark({ expectedCacheCommentCount: 1 })
206+
);
207+
208+
assertThrows(
209+
"StartupBenchmark constructor should throw with iterationCount=0.",
210+
() => {
211+
new StartupBenchmark({ iterationCount: 0, expectedCacheCommentCount: 1 });
212+
}
213+
);
214+
215+
assertThrows(
216+
"StartupBenchmark constructor should throw with expectedCacheCommentCount=0.",
217+
() => {
218+
new StartupBenchmark({ iterationCount: 1, expectedCacheCommentCount: 0 });
219+
}
220+
);
221+
})();

0 commit comments

Comments
 (0)