Skip to content

Commit 254d711

Browse files
eqrioneqrion
committed
Decompress .z files in prefetch
1 parent c432b1a commit 254d711

File tree

3 files changed

+186
-9
lines changed

3 files changed

+186
-9
lines changed

‎JetStreamDriver.js‎

Lines changed: 132 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,17 @@ const defaultIterationCount = 120;
3131
const defaultWorstCaseCount = 4;
3232

3333
if (!JetStreamParams.prefetchResources)
34-
console.warn("Disabling resource prefetching!");
34+
console.warn("Disabling resource prefetching! All compressed files must have been decompressed using `node utils/compress.mjs -d`");
35+
36+
if (!isInBrowser && JetStreamParams.prefetchResources) {
37+
// Use the wasm compiled zlib as a polyfill when decompression stream is
38+
// not available in JS shells.
39+
load("./wasm/zlib/shell.js");
40+
41+
// Load a polyfill for TextEncoder/TextDecoder in shells. Used when
42+
// decompressing a prefetched resource and converting it to text.
43+
load("./polyfills/fast-text-encoding/1.0.3/text.js");
44+
}
3545

3646
// Used for the promise representing the current benchmark run.
3747
this.currentResolve = null;
@@ -138,6 +148,21 @@ function uiFriendlyDuration(time) {
138148
return `${time.toFixed(3)} ms`;
139149
}
140150

151+
// Files can be zlib compressed to reduce the size of the JetStream source code.
152+
// We don't use http compression because we support running from the shell and
153+
// don't want to require a complicated server setup.
154+
//
155+
// zlib was chosen because we already have it in tree for the wasm-zlib test.
156+
function isCompressed(name) {
157+
return name.endsWith(".z");
158+
}
159+
160+
function uncompressedName(name) {
161+
if (name.endsWith(".z"))
162+
return name.slice(0, -2);
163+
return name;
164+
}
165+
141166
// TODO: Cleanup / remove / merge. This is only used for caching loads in the
142167
// non-browser setting. In the browser we use exclusively `loadCache`,
143168
// `loadBlob`, `doLoadBlob`, `prefetchResourcesForBrowser` etc., see below.
@@ -150,14 +175,28 @@ class ShellFileLoader {
150175
// share common code.
151176
load(url) {
152177
console.assert(!isInBrowser);
178+
179+
let compressed = isCompressed(url);
180+
if (compressed && !JetStreamParams.prefetchResources) {
181+
url = uncompressedName(url);
182+
}
183+
184+
// If we aren't supposed to prefetch this then return code snippet that will load the url on-demand.
153185
if (!JetStreamParams.prefetchResources)
154186
return `load("${url}");`
155187

156188
if (this.requests.has(url)) {
157189
return this.requests.get(url);
158190
}
159191

160-
const contents = readFile(url);
192+
let contents;
193+
if (compressed) {
194+
let bytes = new Int8Array(read(url, "binary"));
195+
bytes = zlib.decompress(bytes);
196+
contents = new TextDecoder().decode(bytes);
197+
} else {
198+
contents = readFile(url);
199+
}
161200
this.requests.set(url, contents);
162201
return contents;
163202
}
@@ -209,10 +248,14 @@ class Driver {
209248
performance.mark("update-ui");
210249
benchmark.updateUIAfterRun();
211250

212-
if (isInBrowser && JetStreamParams.prefetchResources) {
251+
if (isInBrowser) {
213252
const cache = JetStream.blobDataCache;
214253
for (const file of benchmark.files) {
215254
const blobData = cache[file];
255+
// If we didn't prefetch this resource, then no need to free it
256+
if (!blobData.blob) {
257+
continue
258+
}
216259
blobData.refCount--;
217260
if (!blobData.refCount)
218261
cache[file] = undefined;
@@ -338,6 +381,9 @@ class Driver {
338381

339382
async prefetchResources() {
340383
if (!isInBrowser) {
384+
if (JetStreamParams.prefetchResources) {
385+
await zlib.initialize();
386+
}
341387
for (const benchmark of this.benchmarks)
342388
benchmark.prefetchResourcesForShell();
343389
return;
@@ -553,6 +599,11 @@ class Scripts {
553599
}
554600

555601
class ShellScripts extends Scripts {
602+
constructor() {
603+
super();
604+
this.prefetchedResources = [];
605+
}
606+
556607
run() {
557608
let globalObject;
558609
let realm;
@@ -579,13 +630,33 @@ class ShellScripts extends Scripts {
579630
currentReject
580631
};
581632

633+
// Pass the prefetched resources to the benchmark global.
634+
if (JetStreamParams.prefetchResources) {
635+
// Pass the 'TextDecoder' polyfill into the benchmark global. Don't
636+
// use 'TextDecoder' as that will get picked up in the kotlin test
637+
// without full support.
638+
globalObject.ShellTextDecoder = TextDecoder;
639+
// Store shellPrefetchedResources on ShellPrefetchedResources so that
640+
// getBinary and getString can find them.
641+
globalObject.ShellPrefetchedResources = {};
642+
for (const [name, value] of this.prefetchedResources) {
643+
globalObject.ShellPrefetchedResources[name] = value;
644+
}
645+
} else {
646+
console.assert(this.prefetchedResources.length === 0, "Unexpected prefetched resources");
647+
}
648+
582649
globalObject.performance ??= performance;
583650
for (const script of this.scripts)
584651
globalObject.loadString(script);
585652

586653
return isD8 ? realm : globalObject;
587654
}
588655

656+
addPrefetchedResources(prefetchedResources) {
657+
this.prefetchedResources.push(...prefetchedResources);
658+
}
659+
589660
add(text) {
590661
this.scripts.push(text);
591662
}
@@ -618,7 +689,6 @@ class BrowserScripts extends Scripts {
618689
return magicFrame;
619690
}
620691

621-
622692
add(text) {
623693
this.scripts.push(`<script>${text}</script>`);
624694
}
@@ -638,6 +708,7 @@ class Benchmark {
638708
this.allowUtf16 = !!plan.allowUtf16;
639709
this.scripts = null;
640710
this.preloads = null;
711+
this.shellPrefetchedResources = null;
641712
this.results = [];
642713
this._state = BenchmarkState.READY;
643714
}
@@ -771,6 +842,9 @@ class Benchmark {
771842
if (!!this.plan.exposeBrowserTest)
772843
scripts.addBrowserTest();
773844

845+
if (this.shellPrefetchedResources) {
846+
scripts.addPrefetchedResources(this.shellPrefetchedResources);
847+
}
774848
if (this.plan.preload) {
775849
let preloadCode = "";
776850
for (let [ variableName, blobURLOrPath ] of this.preloads)
@@ -789,7 +863,7 @@ class Benchmark {
789863
} else {
790864
const cache = JetStream.blobDataCache;
791865
for (const file of this.plan.files) {
792-
scripts.addWithURL(JetStreamParams.prefetchResources ? cache[file].blobURL : file);
866+
scripts.addWithURL(cache[file].blobURL);
793867
}
794868
}
795869

@@ -840,10 +914,19 @@ class Benchmark {
840914

841915
async doLoadBlob(resource) {
842916
const blobData = JetStream.blobDataCache[resource];
917+
918+
const compressed = isCompressed(resource);
919+
if (compressed && !JetStreamParams.prefetchResources) {
920+
resource = uncompressedName(resource);
921+
}
922+
923+
// If we aren't supposed to prefetch this then set the blobURL to just
924+
// be the resource URL.
843925
if (!JetStreamParams.prefetchResources) {
844926
blobData.blobURL = resource;
845927
return blobData;
846928
}
929+
847930
let response;
848931
let tries = 3;
849932
while (tries--) {
@@ -859,7 +942,15 @@ class Benchmark {
859942
continue;
860943
throw new Error("Fetch failed");
861944
}
862-
const blob = await response.blob();
945+
946+
// If we need to decompress this, then run it through a decompression
947+
// stream.
948+
if (compressed) {
949+
const stream = response.body.pipeThrough(new DecompressionStream('deflate'))
950+
response = new Response(stream);
951+
}
952+
953+
let blob = await response.blob();
863954
blobData.blob = blob;
864955
blobData.blobURL = URL.createObjectURL(blob);
865956
return blobData;
@@ -995,7 +1086,28 @@ class Benchmark {
9951086
this.scripts = this.plan.files.map(file => shellFileLoader.load(file));
9961087

9971088
console.assert(this.preloads === null, "This initialization should be called only once.");
998-
this.preloads = Object.entries(this.plan.preload ?? {});
1089+
this.preloads = [];
1090+
this.shellPrefetchedResources = [];
1091+
if (this.plan.preload) {
1092+
for (let name of Object.getOwnPropertyNames(this.plan.preload)) {
1093+
let file = this.plan.preload[name];
1094+
1095+
const compressed = isCompressed(file);
1096+
if (compressed && !JetStreamParams.prefetchResources) {
1097+
file = uncompressedName(file);
1098+
}
1099+
1100+
if (JetStreamParams.prefetchResources) {
1101+
let bytes = new Int8Array(read(file, "binary"));
1102+
if (compressed) {
1103+
bytes = zlib.decompress(bytes);
1104+
}
1105+
this.shellPrefetchedResources.push([file, bytes]);
1106+
}
1107+
1108+
this.preloads.push([name, file]);
1109+
}
1110+
}
9991111
}
10001112

10011113
scoreIdentifiers() {
@@ -1268,15 +1380,23 @@ class AsyncBenchmark extends DefaultBenchmark {
12681380
} else {
12691381
str += `
12701382
JetStream.getBinary = async function(path) {
1383+
if (ShellPrefetchedResources) {
1384+
return ShellPrefetchedResources[path];
1385+
}
12711386
return new Int8Array(read(path, "binary"));
12721387
};
12731388
12741389
JetStream.getString = async function(path) {
1390+
if (ShellPrefetchedResources) {
1391+
return new ShellTextDecoder().decode(ShellPrefetchedResources[path]);
1392+
}
12751393
return read(path);
12761394
};
12771395
12781396
JetStream.dynamicImport = async function(path) {
12791397
try {
1398+
// TODO: this skips the prefetched resources, but I'm
1399+
// not sure of a way around that.
12801400
return await import(path);
12811401
} catch (e) {
12821402
// In shells, relative imports require different paths, so try with and
@@ -1493,7 +1613,11 @@ class WasmLegacyBenchmark extends Benchmark {
14931613
`;
14941614
} else {
14951615
str += `
1496-
Module[key] = new Int8Array(read(path, "binary"));
1616+
if (ShellPrefetchedResources) {
1617+
Module[key] = ShellPrefetchedResources[path];
1618+
} else {
1619+
Module[key] = new Int8Array(read(path, "binary"));
1620+
}
14971621
if (andThen == doRun) {
14981622
globalObject.read = (...args) => {
14991623
console.log("should not be inside read: ", ...args);

‎wasm/zlib/shell.js‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// zlib-based utility for use in shells where CompressionStream and
2+
// DecompressionStream are not available.
3+
4+
function module() {
5+
'use strict';
6+
7+
let zlibPromise = null;
8+
let zlibModule = null;
9+
10+
async function initialize() {
11+
if (zlibPromise) {
12+
zlibModule = await zlibPromise;
13+
return zlibModule;
14+
}
15+
load('wasm/zlib/build/zlib.js');
16+
zlibPromise = setupModule({
17+
wasmBinary: new Int8Array(read('wasm/zlib/build/zlib.wasm', "binary")),
18+
});
19+
zlibModule = await zlibPromise;
20+
return zlibModule;
21+
}
22+
23+
function decompress(bytes) {
24+
zlibModule.FS.writeFile('in', bytes);
25+
const inputzStr = zlibModule.stringToNewUTF8('in');
26+
const inputzoutStr = zlibModule.stringToNewUTF8('out');
27+
if (zlibModule._decompressFile(inputzStr, inputzoutStr) !== 0) {
28+
throw new Error();
29+
}
30+
const output = zlibModule.FS.readFile('out');
31+
zlibModule._free(inputzStr);
32+
zlibModule._free(inputzoutStr);
33+
return output;
34+
}
35+
36+
return {
37+
initialize: initialize,
38+
decompress: decompress,
39+
};
40+
}
41+
42+
globalThis.zlib = module();
43+

‎web-tooling-benchmark/benchmark.js‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,18 @@ class Benchmark {
3838
async loadAllFiles(preload) {
3939
const loadPromises = Object.entries(preload).map(
4040
async ([name, url]) => {
41+
console.log(name);
42+
console.log(url);
4143
if (name.endsWith(".wasm")) {
42-
this.files[name] = (await JetStream.getBinary(url)).buffer;
44+
let buffer = (await JetStream.getBinary(url)).buffer;
45+
if (!(buffer instanceof ArrayBuffer)) {
46+
// The returned array buffer is from a different global when
47+
// prefetching resources and running in the shell. This is fine,
48+
// except for the source map code doing an instanceof
49+
// check that fails for the prototype being in a different realm.
50+
Object.setPrototypeOf(buffer, ArrayBuffer.prototype);
51+
}
52+
this.files[name] = buffer;
4353
} else {
4454
this.files[name] = await JetStream.getString(url);
4555
}

0 commit comments

Comments
 (0)