Skip to content

Commit 57aeff1

Browse files
seanmorrisseanmorris
authored
[file-packager] Use fetch() for loading file packages. (emscripten-core#22016)
This PR switches AJAX requests for file packages to be loaded with fetch() rather than XmlHttpRequest(). This allows service workers to make AJAX requests, since XmlHttpRequest is not available in service workers. `Module['setStatus']` is now called with more granularity during package downloads. Previously, --embed-file was necessary to load file packages, which embeds the files into the binary and precludes caching and re-use of the package. Partially fixes emscripten-core#22003, only applies to file packages.
1 parent 16a0bf1 commit 57aeff1

File tree

2 files changed

+108
-53
lines changed

2 files changed

+108
-53
lines changed

‎test/test_browser.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5376,6 +5376,55 @@ def test_webpack(self, es6):
53765376
shutil.copyfile('webpack/src/hello.wasm', 'webpack/dist/hello.wasm')
53775377
self.run_browser('webpack/dist/index.html', '/report_result?exit:0')
53785378

5379+
def test_fetch_polyfill_preload(self):
5380+
create_file('hello.txt', 'hello, world!')
5381+
create_file('main.c', r'''
5382+
#include <stdio.h>
5383+
#include <string.h>
5384+
#include <emscripten.h>
5385+
int main() {
5386+
FILE *f = fopen("hello.txt", "r");
5387+
char buf[100];
5388+
fread(buf, 1, 20, f);
5389+
buf[20] = 0;
5390+
fclose(f);
5391+
printf("%s\n", buf);
5392+
return 0;
5393+
}''')
5394+
5395+
create_file('on_window_error_shell.html', r'''
5396+
<html>
5397+
<center><canvas id='canvas' width='256' height='256'></canvas></center>
5398+
<hr><div id='output'></div><hr>
5399+
<script type='text/javascript'>
5400+
window.addEventListener('error', event => {
5401+
const error = String(event.message);
5402+
window.disableErrorReporting = true;
5403+
window.onerror = null;
5404+
var xhr = new XMLHttpRequest();
5405+
xhr.open('GET', 'http://localhost:8888/report_result?exception:' + error.substr(-23), true);
5406+
xhr.send();
5407+
setTimeout(function() { window.close() }, 1000);
5408+
});
5409+
</script>
5410+
{{{ SCRIPT }}}
5411+
</body>
5412+
</html>''')
5413+
5414+
def test(args, expect_fail):
5415+
self.compile_btest('main.c', ['-sEXIT_RUNTIME', '--preload-file', 'hello.txt', '--shell-file', 'on_window_error_shell.html', '-o', 'a.out.html'] + args)
5416+
if expect_fail:
5417+
js = read_file('a.out.js')
5418+
create_file('a.out.js', 'let origFetch = fetch; fetch = undefined;\n' + js)
5419+
return self.run_browser('a.out.html', '/report_result?exception:fetch is not a function')
5420+
else:
5421+
return self.run_browser('a.out.html', '/report_result?exit:0')
5422+
5423+
test([], expect_fail=False)
5424+
test([], expect_fail=True)
5425+
test(['-sLEGACY_VM_SUPPORT'], expect_fail=False)
5426+
test(['-sLEGACY_VM_SUPPORT', '-sNO_POLYFILL'], expect_fail=True)
5427+
53795428
@no_wasm64('https://github.com/llvm/llvm-project/issues/98778')
53805429
def test_fetch_polyfill_shared_lib(self):
53815430
create_file('library.c', r'''

‎tools/file_packager.py‎

Lines changed: 59 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -359,7 +359,7 @@ def generate_object_file(data_files):
359359

360360
def main():
361361
if len(sys.argv) == 1:
362-
err('''Usage: file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins]
362+
err('''Usage: file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node]
363363
See the source for more details.''')
364364
return 1
365365

@@ -952,54 +952,62 @@ def generate_js(data_target, data_files, metadata):
952952
});
953953
return;
954954
}'''.strip()
955+
955956
ret += '''
956957
function fetchRemotePackage(packageName, packageSize, callback, errback) {
957958
%(node_support_code)s
958-
var xhr = new XMLHttpRequest();
959-
xhr.open('GET', packageName, true);
960-
xhr.responseType = 'arraybuffer';
961-
xhr.onprogress = (event) => {
962-
var url = packageName;
963-
var size = packageSize;
964-
if (event.total) size = event.total;
965-
if (event.loaded) {
966-
if (!xhr.addedTotal) {
967-
xhr.addedTotal = true;
968-
if (!Module['dataFileDownloads']) Module['dataFileDownloads'] = {};
969-
Module['dataFileDownloads'][url] = {
970-
loaded: event.loaded,
971-
total: size
972-
};
973-
} else {
974-
Module['dataFileDownloads'][url].loaded = event.loaded;
959+
Module['dataFileDownloads'] ??= {};
960+
fetch(packageName)
961+
.catch((cause) => Promise.reject(new Error(`Network Error: ${packageName}`, {cause}))) // If fetch fails, rewrite the error to include the failing URL & the cause.
962+
.then((response) => {
963+
if (!response.ok) {
964+
return Promise.reject(new Error(`${response.status}: ${response.url}`));
975965
}
976-
var total = 0;
977-
var loaded = 0;
978-
var num = 0;
979-
for (var download in Module['dataFileDownloads']) {
980-
var data = Module['dataFileDownloads'][download];
981-
total += data.total;
982-
loaded += data.loaded;
983-
num++;
966+
967+
if (!response.body && response.arrayBuffer) { // If we're using the polyfill, readers won't be available...
968+
return response.arrayBuffer().then(callback);
984969
}
985-
total = Math.ceil(total * Module['expectedDataFileDownloads']/num);
986-
Module['setStatus']?.(`Downloading data... (${loaded}/${total})`);
987-
} else if (!Module['dataFileDownloads']) {
970+
971+
const reader = response.body.getReader();
972+
const iterate = () => reader.read().then(handleChunk).catch((cause) => {
973+
return Promise.reject(new Error(`Unexpected error while handling : ${response.url} ${cause}`, {cause}));
974+
});
975+
976+
const chunks = [];
977+
const headers = response.headers;
978+
const total = Number(headers.get('Content-Length') ?? packageSize);
979+
let loaded = 0;
980+
981+
const handleChunk = ({done, value}) => {
982+
if (!done) {
983+
chunks.push(value);
984+
loaded += value.length;
985+
Module['dataFileDownloads'][packageName] = {loaded, total};
986+
987+
let totalLoaded = 0;
988+
let totalSize = 0;
989+
990+
for (const download of Object.values(Module['dataFileDownloads'])) {
991+
totalLoaded += download.loaded;
992+
totalSize += download.total;
993+
}
994+
995+
Module['setStatus']?.(`Downloading data... (${totalLoaded}/${totalSize})`);
996+
return iterate();
997+
} else {
998+
const packageData = new Uint8Array(chunks.map((c) => c.length).reduce((a, b) => a + b, 0));
999+
let offset = 0;
1000+
for (const chunk of chunks) {
1001+
packageData.set(chunk, offset);
1002+
offset += chunk.length;
1003+
}
1004+
callback(packageData.buffer);
1005+
}
1006+
};
1007+
9881008
Module['setStatus']?.('Downloading data...');
989-
}
990-
};
991-
xhr.onerror = (event) => {
992-
throw new Error("NetworkError for: " + packageName);
993-
}
994-
xhr.onload = (event) => {
995-
if (xhr.status == 200 || xhr.status == 304 || xhr.status == 206 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0
996-
var packageData = xhr.response;
997-
callback(packageData);
998-
} else {
999-
throw new Error(xhr.statusText + " : " + xhr.responseURL);
1000-
}
1001-
};
1002-
xhr.send(null);
1009+
return iterate();
1010+
});
10031011
};
10041012
10051013
function handleError(error) {
@@ -1097,15 +1105,14 @@ def generate_js(data_target, data_files, metadata):
10971105
function runMetaWithFS() {
10981106
Module['addRunDependency']('%(metadata_file)s');
10991107
var REMOTE_METADATA_NAME = Module['locateFile'] ? Module['locateFile']('%(metadata_file)s', '') : '%(metadata_file)s';
1100-
var xhr = new XMLHttpRequest();
1101-
xhr.onreadystatechange = () => {
1102-
if (xhr.readyState === 4 && xhr.status === 200) {
1103-
loadPackage(JSON.parse(xhr.responseText));
1104-
}
1105-
}
1106-
xhr.open('GET', REMOTE_METADATA_NAME, true);
1107-
xhr.overrideMimeType('application/json');
1108-
xhr.send(null);
1108+
fetch(REMOTE_METADATA_NAME)
1109+
.then((response) => {
1110+
if (response.ok) {
1111+
return response.json();
1112+
}
1113+
return Promise.reject(new Error(`${response.status}: ${response.url}`));
1114+
})
1115+
.then(loadPackage);
11091116
}
11101117
11111118
if (Module['calledRun']) {
@@ -1114,7 +1121,6 @@ def generate_js(data_target, data_files, metadata):
11141121
if (!Module['preRun']) Module['preRun'] = [];
11151122
Module["preRun"].push(runMetaWithFS);
11161123
}\n''' % {'metadata_file': os.path.basename(options.jsoutput + '.metadata')}
1117-
11181124
else:
11191125
_metadata_template = '''
11201126
}

0 commit comments

Comments
 (0)