Skip to content

Commit 7299a4c

Browse files
eqrioneqrion
committed
Decompress .z files in prefetch
1 parent 584ef4d commit 7299a4c

File tree

3 files changed

+186
-9
lines changed

3 files changed

+186
-9
lines changed

‎JetStreamDriver.js‎

Lines changed: 132 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,17 @@ const defaultIterationCount = 120;
3131
const defaultWorstCaseCount = 4;
3232

3333
if (!JetStreamParams.prefetchResources)
34-
console.warn("Disabling resource prefetching!");
34+
console.warn("Disabling resource prefetching! All compressed files must have been decompressed using `node utils/compress.mjs -d`");
35+
36+
if (!isInBrowser && JetStreamParams.prefetchResources) {
37+
// Use the wasm compiled zlib as a polyfill when decompression stream is
38+
// not available in JS shells.
39+
load("./wasm/zlib/shell.js");
40+
41+
// Load a polyfill for TextEncoder/TextDecoder in shells. Used when
42+
// decompressing a prefetched resource and converting it to text.
43+
load("./polyfills/fast-text-encoding/1.0.3/text.js");
44+
}
3545

3646
// Used for the promise representing the current benchmark run.
3747
this.currentResolve = null;
@@ -138,6 +148,21 @@ function uiFriendlyDuration(time) {
138148
return `${time.toFixed(3)} ms`;
139149
}
140150

151+
// Files can be zlib compressed to reduce the size of the JetStream source code.
152+
// We don't use http compression because we support running from the shell and
153+
// don't want to require a complicated server setup.
154+
//
155+
// zlib was chosen because we already have it in tree for the wasm-zlib test.
156+
function isCompressed(name) {
157+
return name.endsWith(".z");
158+
}
159+
160+
function uncompressedName(name) {
161+
if (name.endsWith(".z"))
162+
return name.slice(0, -2);
163+
return name;
164+
}
165+
141166
// TODO: Cleanup / remove / merge. This is only used for caching loads in the
142167
// non-browser setting. In the browser we use exclusively `loadCache`,
143168
// `loadBlob`, `doLoadBlob`, `prefetchResourcesForBrowser` etc., see below.
@@ -150,14 +175,28 @@ class ShellFileLoader {
150175
// share common code.
151176
load(url) {
152177
console.assert(!isInBrowser);
178+
179+
let compressed = isCompressed(url);
180+
if (compressed && !JetStreamParams.prefetchResources) {
181+
url = uncompressedName(url);
182+
}
183+
184+
// If we aren't supposed to prefetch this then return code snippet that will load the url on-demand.
153185
if (!JetStreamParams.prefetchResources)
154186
return `load("${url}");`
155187

156188
if (this.requests.has(url)) {
157189
return this.requests.get(url);
158190
}
159191

160-
const contents = readFile(url);
192+
let contents;
193+
if (compressed) {
194+
let bytes = new Int8Array(read(url, "binary"));
195+
bytes = zlib.decompress(bytes);
196+
contents = new TextDecoder().decode(bytes);
197+
} else {
198+
contents = readFile(url);
199+
}
161200
this.requests.set(url, contents);
162201
return contents;
163202
}
@@ -209,10 +248,14 @@ class Driver {
209248
performance.mark("update-ui");
210249
benchmark.updateUIAfterRun();
211250

212-
if (isInBrowser && JetStreamParams.prefetchResources) {
251+
if (isInBrowser) {
213252
const cache = JetStream.blobDataCache;
214253
for (const file of benchmark.files) {
215254
const blobData = cache[file];
255+
// If we didn't prefetch this resource, then no need to free it
256+
if (!blobData.blob) {
257+
continue
258+
}
216259
blobData.refCount--;
217260
if (!blobData.refCount)
218261
cache[file] = undefined;
@@ -361,6 +404,9 @@ class Driver {
361404

362405
async prefetchResources() {
363406
if (!isInBrowser) {
407+
if (JetStreamParams.prefetchResources) {
408+
await zlib.initialize();
409+
}
364410
for (const benchmark of this.benchmarks)
365411
benchmark.prefetchResourcesForShell();
366412
return;
@@ -576,6 +622,11 @@ class Scripts {
576622
}
577623

578624
class ShellScripts extends Scripts {
625+
constructor() {
626+
super();
627+
this.prefetchedResources = [];
628+
}
629+
579630
run() {
580631
let globalObject;
581632
let realm;
@@ -602,13 +653,33 @@ class ShellScripts extends Scripts {
602653
currentReject
603654
};
604655

656+
// Pass the prefetched resources to the benchmark global.
657+
if (JetStreamParams.prefetchResources) {
658+
// Pass the 'TextDecoder' polyfill into the benchmark global. Don't
659+
// use 'TextDecoder' as that will get picked up in the kotlin test
660+
// without full support.
661+
globalObject.ShellTextDecoder = TextDecoder;
662+
// Store shellPrefetchedResources on ShellPrefetchedResources so that
663+
// getBinary and getString can find them.
664+
globalObject.ShellPrefetchedResources = {};
665+
for (const [name, value] of this.prefetchedResources) {
666+
globalObject.ShellPrefetchedResources[name] = value;
667+
}
668+
} else {
669+
console.assert(this.prefetchedResources.length === 0, "Unexpected prefetched resources");
670+
}
671+
605672
globalObject.performance ??= performance;
606673
for (const script of this.scripts)
607674
globalObject.loadString(script);
608675

609676
return isD8 ? realm : globalObject;
610677
}
611678

679+
addPrefetchedResources(prefetchedResources) {
680+
this.prefetchedResources.push(...prefetchedResources);
681+
}
682+
612683
add(text) {
613684
this.scripts.push(text);
614685
}
@@ -641,7 +712,6 @@ class BrowserScripts extends Scripts {
641712
return magicFrame;
642713
}
643714

644-
645715
add(text) {
646716
this.scripts.push(`<script>${text}</script>`);
647717
}
@@ -661,6 +731,7 @@ class Benchmark {
661731
this.allowUtf16 = !!plan.allowUtf16;
662732
this.scripts = null;
663733
this.preloads = null;
734+
this.shellPrefetchedResources = null;
664735
this.results = [];
665736
this._state = BenchmarkState.READY;
666737
}
@@ -774,6 +845,9 @@ class Benchmark {
774845
if (!!this.plan.exposeBrowserTest)
775846
scripts.addBrowserTest();
776847

848+
if (this.shellPrefetchedResources) {
849+
scripts.addPrefetchedResources(this.shellPrefetchedResources);
850+
}
777851
if (this.plan.preload) {
778852
let preloadCode = "";
779853
for (let [ variableName, blobURLOrPath ] of this.preloads)
@@ -792,7 +866,7 @@ class Benchmark {
792866
} else {
793867
const cache = JetStream.blobDataCache;
794868
for (const file of this.plan.files) {
795-
scripts.addWithURL(JetStreamParams.prefetchResources ? cache[file].blobURL : file);
869+
scripts.addWithURL(cache[file].blobURL);
796870
}
797871
}
798872

@@ -843,10 +917,19 @@ class Benchmark {
843917

844918
async doLoadBlob(resource) {
845919
const blobData = JetStream.blobDataCache[resource];
920+
921+
const compressed = isCompressed(resource);
922+
if (compressed && !JetStreamParams.prefetchResources) {
923+
resource = uncompressedName(resource);
924+
}
925+
926+
// If we aren't supposed to prefetch this then set the blobURL to just
927+
// be the resource URL.
846928
if (!JetStreamParams.prefetchResources) {
847929
blobData.blobURL = resource;
848930
return blobData;
849931
}
932+
850933
let response;
851934
let tries = 3;
852935
while (tries--) {
@@ -862,7 +945,15 @@ class Benchmark {
862945
continue;
863946
throw new Error("Fetch failed");
864947
}
865-
const blob = await response.blob();
948+
949+
// If we need to decompress this, then run it through a decompression
950+
// stream.
951+
if (compressed) {
952+
const stream = response.body.pipeThrough(new DecompressionStream('deflate'))
953+
response = new Response(stream);
954+
}
955+
956+
let blob = await response.blob();
866957
blobData.blob = blob;
867958
blobData.blobURL = URL.createObjectURL(blob);
868959
return blobData;
@@ -998,7 +1089,28 @@ class Benchmark {
9981089
this.scripts = this.plan.files.map(file => shellFileLoader.load(file));
9991090

10001091
console.assert(this.preloads === null, "This initialization should be called only once.");
1001-
this.preloads = Object.entries(this.plan.preload ?? {});
1092+
this.preloads = [];
1093+
this.shellPrefetchedResources = [];
1094+
if (this.plan.preload) {
1095+
for (let name of Object.getOwnPropertyNames(this.plan.preload)) {
1096+
let file = this.plan.preload[name];
1097+
1098+
const compressed = isCompressed(file);
1099+
if (compressed && !JetStreamParams.prefetchResources) {
1100+
file = uncompressedName(file);
1101+
}
1102+
1103+
if (JetStreamParams.prefetchResources) {
1104+
let bytes = new Int8Array(read(file, "binary"));
1105+
if (compressed) {
1106+
bytes = zlib.decompress(bytes);
1107+
}
1108+
this.shellPrefetchedResources.push([file, bytes]);
1109+
}
1110+
1111+
this.preloads.push([name, file]);
1112+
}
1113+
}
10021114
}
10031115

10041116
scoreIdentifiers() {
@@ -1242,15 +1354,23 @@ class AsyncBenchmark extends DefaultBenchmark {
12421354
} else {
12431355
str += `
12441356
JetStream.getBinary = async function(path) {
1357+
if (ShellPrefetchedResources) {
1358+
return ShellPrefetchedResources[path];
1359+
}
12451360
return new Int8Array(read(path, "binary"));
12461361
};
12471362
12481363
JetStream.getString = async function(path) {
1364+
if (ShellPrefetchedResources) {
1365+
return new ShellTextDecoder().decode(ShellPrefetchedResources[path]);
1366+
}
12491367
return read(path);
12501368
};
12511369
12521370
JetStream.dynamicImport = async function(path) {
12531371
try {
1372+
// TODO: this skips the prefetched resources, but I'm
1373+
// not sure of a way around that.
12541374
return await import(path);
12551375
} catch (e) {
12561376
// In shells, relative imports require different paths, so try with and
@@ -1467,7 +1587,11 @@ class WasmLegacyBenchmark extends Benchmark {
14671587
`;
14681588
} else {
14691589
str += `
1470-
Module[key] = new Int8Array(read(path, "binary"));
1590+
if (ShellPrefetchedResources) {
1591+
Module[key] = ShellPrefetchedResources[path];
1592+
} else {
1593+
Module[key] = new Int8Array(read(path, "binary"));
1594+
}
14711595
if (andThen == doRun) {
14721596
globalObject.read = (...args) => {
14731597
console.log("should not be inside read: ", ...args);

‎wasm/zlib/shell.js‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// zlib-based utility for use in shells where CompressionStream and
2+
// DecompressionStream are not available.
3+
4+
function module() {
5+
'use strict';
6+
7+
let zlibPromise = null;
8+
let zlibModule = null;
9+
10+
async function initialize() {
11+
if (zlibPromise) {
12+
zlibModule = await zlibPromise;
13+
return zlibModule;
14+
}
15+
load('wasm/zlib/build/zlib.js');
16+
zlibPromise = setupModule({
17+
wasmBinary: new Int8Array(read('wasm/zlib/build/zlib.wasm', "binary")),
18+
});
19+
zlibModule = await zlibPromise;
20+
return zlibModule;
21+
}
22+
23+
function decompress(bytes) {
24+
zlibModule.FS.writeFile('in', bytes);
25+
const inputzStr = zlibModule.stringToNewUTF8('in');
26+
const inputzoutStr = zlibModule.stringToNewUTF8('out');
27+
if (zlibModule._decompressFile(inputzStr, inputzoutStr) !== 0) {
28+
throw new Error();
29+
}
30+
const output = zlibModule.FS.readFile('out');
31+
zlibModule._free(inputzStr);
32+
zlibModule._free(inputzoutStr);
33+
return output;
34+
}
35+
36+
return {
37+
initialize: initialize,
38+
decompress: decompress,
39+
};
40+
}
41+
42+
globalThis.zlib = module();
43+

‎web-tooling-benchmark/benchmark.js‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,18 @@ class Benchmark {
3838
async loadAllFiles(preload) {
3939
const loadPromises = Object.entries(preload).map(
4040
async ([name, url]) => {
41+
console.log(name);
42+
console.log(url);
4143
if (name.endsWith(".wasm")) {
42-
this.files[name] = (await JetStream.getBinary(url)).buffer;
44+
let buffer = (await JetStream.getBinary(url)).buffer;
45+
if (!(buffer instanceof ArrayBuffer)) {
46+
// The returned array buffer is from a different global when
47+
// prefetching resources and running in the shell. This is fine,
48+
// except for the source map code doing an instanceof
49+
// check that fails for the prototype being in a different realm.
50+
Object.setPrototypeOf(buffer, ArrayBuffer.prototype);
51+
}
52+
this.files[name] = buffer;
4353
} else {
4454
this.files[name] = await JetStream.getString(url);
4555
}

0 commit comments

Comments
 (0)