From 87743d133ea3ce96d01ffed0b6f71f620b719376 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sat, 25 May 2024 11:23:00 -0400 Subject: [PATCH 01/34] Adding USE_FETCH setting. --- src/web_or_worker_shell_read.js | 16 +++- tools/file_packager.py | 146 ++++++++++++++++++++++---------- tools/link.py | 2 + tools/settings.py | 1 + 4 files changed, 117 insertions(+), 48 deletions(-) diff --git a/src/web_or_worker_shell_read.js b/src/web_or_worker_shell_read.js index 3ae5842e8f298..ab9c22dbd5b5d 100644 --- a/src/web_or_worker_shell_read.js +++ b/src/web_or_worker_shell_read.js @@ -20,7 +20,19 @@ return new Uint8Array(/** @type{!ArrayBuffer} */(xhr.response)); }; } - +#if USE_FETCH + readAsync = (url, onload, onerror) => { + fetch(url) + .then(response => { + if(response.ok) { + onload(response.arrayBuffer()); + return; + } + throw new Error(response.statusText + ' : ' + response.url); + }) + .catch(onerror) + }; +#else readAsync = (url, onload, onerror) => { var xhr = new XMLHttpRequest(); xhr.open('GET', url, true); @@ -35,4 +47,4 @@ xhr.onerror = onerror; xhr.send(null); } - +#endif diff --git a/tools/file_packager.py b/tools/file_packager.py index d435a33209cab..017ca111c5103 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -965,56 +965,110 @@ def generate_js(data_target, data_files, metadata): }); return; }'''.strip() - ret += ''' - function fetchRemotePackage(packageName, packageSize, callback, errback) { - %(node_support_code)s - var xhr = new XMLHttpRequest(); - xhr.open('GET', packageName, true); - xhr.responseType = 'arraybuffer'; - xhr.onprogress = function(event) { - var url = packageName; - var size = packageSize; - if (event.total) size = event.total; - if (event.loaded) { - if (!xhr.addedTotal) { - xhr.addedTotal = true; - if (!Module.dataFileDownloads) Module.dataFileDownloads = {}; - Module.dataFileDownloads[url] = { - loaded: event.loaded, - total: size - }; - } else { - Module.dataFileDownloads[url].loaded = event.loaded; + if options.use_fetch: + ret += ''' + function fetchRemotePackage(packageName, packageSize, callback, errback) { + %(node_support_code)s + Module.dataFileDownloads = Module.dataFileDownloads || {}; + const url = packageName; + fetch(url).then(response => { + + let received = 0; + + const reader = response.body.getReader(); + const headers = response.headers; + + console.log(response, [...headers.entries()]); + + if(!response.ok) { + throw new Error(response.statusText + ' : ' + response.url); } - var total = 0; - var loaded = 0; - var num = 0; - for (var download in Module.dataFileDownloads) { - var data = Module.dataFileDownloads[download]; - total += data.total; - loaded += data.loaded; - num++; + + const total = headers.get('Content-Length') ?? packageSize; + const chunks = []; + + const iterate = () => reader.read().then(handleChunk).catch(cause => { + throw new Error(response.statusText + ' : ' + response.url, {cause}); + }); + + const handleChunk = ({done, value}) => { + if(!done) { + console.log({done, value}); + chunks.push(value); + received += value.length; + Module.dataFileDownloads[url] = Module.dataFileDownloads[url] ?? {}; + Module.dataFileDownloads[url].loaded = received; + Module.dataFileDownloads[url].total = total; + console.log(JSON.stringify(Module.dataFileDownloads)); + return iterate(); + } + else { + const size = chunks.map(c => c.length).reduce((a, b) => a + b, 0); + let index = 0; + const packageData = new Uint8Array(size); + for(const chunk of chunks) { + packageData.set(chunk, index); + index += chunk.length; + } + console.log(packageData); + callback(packageData.buffer); + } + }; + return iterate(); + }); + }\n''' % {'node_support_code': node_support_code} + else: + ret += ''' + function fetchRemotePackage(packageName, packageSize, callback, errback) { + %(node_support_code)s + var xhr = new XMLHttpRequest(); + xhr.open('GET', packageName, true); + xhr.responseType = 'arraybuffer'; + xhr.onprogress = function(event) { + var url = packageName; + var size = packageSize; + if (event.total) size = event.total; + if (event.loaded) { + if (!xhr.addedTotal) { + xhr.addedTotal = true; + if (!Module.dataFileDownloads) Module.dataFileDownloads = {}; + Module.dataFileDownloads[url] = { + loaded: event.loaded, + total: size + }; + } else { + Module.dataFileDownloads[url].loaded = event.loaded; + } + var total = 0; + var loaded = 0; + var num = 0; + for (var download in Module.dataFileDownloads) { + var data = Module.dataFileDownloads[download]; + total += data.total; + loaded += data.loaded; + num++; + } + total = Math.ceil(total * Module.expectedDataFileDownloads/num); + if (Module['setStatus']) Module['setStatus'](`Downloading data... (${loaded}/${total})`); + } else if (!Module.dataFileDownloads) { + if (Module['setStatus']) Module['setStatus']('Downloading data...'); } - total = Math.ceil(total * Module.expectedDataFileDownloads/num); - if (Module['setStatus']) Module['setStatus'](`Downloading data... (${loaded}/${total})`); - } else if (!Module.dataFileDownloads) { - if (Module['setStatus']) Module['setStatus']('Downloading data...'); - } - }; - xhr.onerror = function(event) { - throw new Error("NetworkError for: " + packageName); - } - xhr.onload = function(event) { - if (xhr.status == 200 || xhr.status == 304 || xhr.status == 206 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0 - var packageData = xhr.response; - callback(packageData); - } else { - throw new Error(xhr.statusText + " : " + xhr.responseURL); + }; + xhr.onerror = function(event) { + throw new Error("NetworkError for: " + packageName); } - }; - xhr.send(null); - }; + xhr.onload = function(event) { + if (xhr.status == 200 || xhr.status == 304 || xhr.status == 206 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0 + var packageData = xhr.response; + callback(packageData); + } else { + throw new Error(xhr.statusText + " : " + xhr.responseURL); + } + }; + xhr.send(null); + };\n''' + ret += ''' function handleError(error) { console.error('package error:', error); };\n''' % {'node_support_code': node_support_code} diff --git a/tools/link.py b/tools/link.py index 09d1496910585..337b3d97e646a 100644 --- a/tools/link.py +++ b/tools/link.py @@ -2997,6 +2997,8 @@ def package_files(options, target): if options.embed_files: file_args.append('--embed') file_args += options.embed_files + if settings.USE_FETCH: + file_args.append('--use-fetch') if options.exclude_files: file_args.append('--exclude') file_args += options.exclude_files diff --git a/tools/settings.py b/tools/settings.py index fc884b3046deb..87b56ff68ee83 100644 --- a/tools/settings.py +++ b/tools/settings.py @@ -74,6 +74,7 @@ 'PTHREAD_POOL_SIZE_STRICT', 'PTHREAD_POOL_DELAY_LOAD', 'DEFAULT_PTHREAD_STACK_SIZE', + 'USE_FETCH', } # Subset of settings that apply at compile time. From 3d38841e8e8a8ef2b62f6f8ae4ea3859de357f76 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sat, 25 May 2024 19:01:56 -0400 Subject: [PATCH 02/34] Tweaks --- src/settings.js | 7 +++ src/web_or_worker_shell_read.js | 4 +- test/test_browser.py | 10 ++-- tools/file_packager.py | 94 +++++++++++++++++++++------------ tools/settings.py | 2 +- 5 files changed, 76 insertions(+), 41 deletions(-) diff --git a/src/settings.js b/src/settings.js index c382fca69c919..e09ba8b1f0594 100644 --- a/src/settings.js +++ b/src/settings.js @@ -807,6 +807,12 @@ var NODEJS_CATCH_EXIT = true; // [link] var NODEJS_CATCH_REJECTION = true; +// Whether to use fetch instead of XHR to load files and packages. +// Defaults to FALSE +// +// [link] +var USE_FETCH = false; + // Whether to support async operations in the compiled code. This makes it // possible to call JS functions from synchronous-looking code in C/C++. // @@ -2177,6 +2183,7 @@ var OFFSCREEN_FRAMEBUFFER_FORBID_VAO_PATH = false; // [link] var TEST_MEMORY_GROWTH_FAILS = false; + // For renamed settings the format is: // [OLD_NAME, NEW_NAME] // For removed settings (which now effectively have a fixed value and can no diff --git a/src/web_or_worker_shell_read.js b/src/web_or_worker_shell_read.js index ab9c22dbd5b5d..36e36ff5d9f86 100644 --- a/src/web_or_worker_shell_read.js +++ b/src/web_or_worker_shell_read.js @@ -25,11 +25,11 @@ fetch(url) .then(response => { if(response.ok) { - onload(response.arrayBuffer()); - return; + return response.arrayBuffer(); } throw new Error(response.statusText + ' : ' + response.url); }) + .then(onload) .catch(onerror) }; #else diff --git a/test/test_browser.py b/test/test_browser.py index 957883dbe17cb..1e84a8d9c26ab 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -291,7 +291,11 @@ def test_emscripten_log(self): args=['-Wno-deprecated-pragma', '--pre-js', path_from_root('src/emscripten-source-map.min.js'), '-gsource-map']) @also_with_wasmfs - def test_preload_file(self): + @parameterized({ + '': ([],), + 'USE_FETCH': (['-sUSE_FETCH'],), + }) + def test_preload_file(self, args): create_file('somefile.txt', 'load me right before running the code please') create_file('.somefile.txt', 'load me right before running the code please') create_file('some@file.txt', 'load me right before running the code please') @@ -355,7 +359,7 @@ def make_main(path): create_file(tricky_filename, 'load me right before running the code please') make_main(tricky_filename) # As an Emscripten-specific feature, the character '@' must be escaped in the form '@@' to not confuse with the 'src@dst' notation. - self.btest_exit('main.cpp', args=['--preload-file', tricky_filename.replace('@', '@@')]) + self.btest_exit('main.cpp', ['--preload-file', tricky_filename.replace('@', '@@')] + args) # TODO: WASMFS doesn't support the rest of this test yet. Exit early. if self.get_setting('WASMFS'): @@ -364,7 +368,7 @@ def make_main(path): # By absolute path make_main('somefile.txt') # absolute becomes relative - self.btest_exit('main.cpp', args=['--preload-file', absolute_src_path]) + self.btest_exit('main.cpp', ['--preload-file', absolute_src_path] + args) # Test subdirectory handling with asset packaging. delete_dir('assets') diff --git a/tools/file_packager.py b/tools/file_packager.py index 017ca111c5103..498e704fc47f8 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -26,6 +26,8 @@ --preload , --embed See emcc --help for more details on those options. + --use-fetch Use fetch instead of XHR. + --exclude E [F..] Specifies filename pattern matches to use for excluding given files from being added to the package. See https://docs.python.org/2/library/fnmatch.html for syntax. @@ -128,6 +130,7 @@ def __init__(self): self.use_preload_plugins = False self.support_node = True self.wasm64 = False + self.use_fetch = False class DataFile: @@ -373,6 +376,8 @@ def main(): leading = 'preload' elif arg == '--embed': leading = 'embed' + elif arg == '--use-fetch': + options.use_fetch = True elif arg == '--exclude': leading = 'exclude' elif arg == '--no-force': @@ -973,14 +978,12 @@ def generate_js(data_target, data_files, metadata): const url = packageName; fetch(url).then(response => { - let received = 0; + let loaded = 0; const reader = response.body.getReader(); const headers = response.headers; - console.log(response, [...headers.entries()]); - - if(!response.ok) { + if (!response.ok) { throw new Error(response.statusText + ' : ' + response.url); } @@ -992,14 +995,19 @@ def generate_js(data_target, data_files, metadata): }); const handleChunk = ({done, value}) => { - if(!done) { - console.log({done, value}); + if (!done) { chunks.push(value); - received += value.length; + loaded += value.length; Module.dataFileDownloads[url] = Module.dataFileDownloads[url] ?? {}; - Module.dataFileDownloads[url].loaded = received; + Module.dataFileDownloads[url].loaded = loaded; Module.dataFileDownloads[url].total = total; - console.log(JSON.stringify(Module.dataFileDownloads)); + + if (total) { + if (Module['setStatus']) Module['setStatus'](`Downloading data... (${loaded}/${total})`); + } + else { + if (Module['setStatus']) Module['setStatus']('Downloading data...'); + } return iterate(); } else { @@ -1010,13 +1018,13 @@ def generate_js(data_target, data_files, metadata): packageData.set(chunk, index); index += chunk.length; } - console.log(packageData); + callback(packageData.buffer); } }; return iterate(); }); - }\n''' % {'node_support_code': node_support_code} + };\n''' % {'node_support_code': node_support_code} else: ret += ''' function fetchRemotePackage(packageName, packageSize, callback, errback) { @@ -1066,12 +1074,12 @@ def generate_js(data_target, data_files, metadata): } }; xhr.send(null); - };\n''' + };\n''' % {'node_support_code': node_support_code} ret += ''' function handleError(error) { console.error('package error:', error); - };\n''' % {'node_support_code': node_support_code} + };\n''' code += ''' function processPackageData(arrayBuffer) { @@ -1160,30 +1168,46 @@ def generate_js(data_target, data_files, metadata): }\n''' if options.separate_metadata: - _metadata_template = ''' + _metadata_template = ''' Module['removeRunDependency']('%(metadata_file)s'); - } - - function runMetaWithFS() { - Module['addRunDependency']('%(metadata_file)s'); - var REMOTE_METADATA_NAME = Module['locateFile'] ? Module['locateFile']('%(metadata_file)s', '') : '%(metadata_file)s'; - var xhr = new XMLHttpRequest(); - xhr.onreadystatechange = function() { - if (xhr.readyState === 4 && xhr.status === 200) { - loadPackage(JSON.parse(xhr.responseText)); - } - } - xhr.open('GET', REMOTE_METADATA_NAME, true); - xhr.overrideMimeType('application/json'); - xhr.send(null); - } + }''' + + if options.use_fetch: + ret += ''' + function runMetaWithFS() { + Module['addRunDependency']('%(metadata_file)s'); + var REMOTE_METADATA_NAME = Module['locateFile'] ? Module['locateFile']('%(metadata_file)s', '') : '%(metadata_file)s'; + fetch(REMOTE_METADATA_NAME) + .then(response => { + if(response.ok) { + return response.json(); + } + }) + .then(loadPackage); + }''' % {'metadata_file': os.path.basename(options.jsoutput + '.metadata')} + else: + ret += ''' + function runMetaWithFS() { + Module['addRunDependency']('%(metadata_file)s'); + var REMOTE_METADATA_NAME = Module['locateFile'] ? Module['locateFile']('%(metadata_file)s', '') : '%(metadata_file)s'; + var xhr = new XMLHttpRequest(); + xhr.onreadystatechange = function() { + if (xhr.readyState === 4 && xhr.status === 200) { + loadPackage(JSON.parse(xhr.responseText)); + } + } + xhr.open('GET', REMOTE_METADATA_NAME, true); + xhr.overrideMimeType('application/json'); + xhr.send(null); + }''' % {'metadata_file': os.path.basename(options.jsoutput + '.metadata')} - if (Module['calledRun']) { - runMetaWithFS(); - } else { - if (!Module['preRun']) Module['preRun'] = []; - Module["preRun"].push(runMetaWithFS); - }\n''' % {'metadata_file': os.path.basename(options.jsoutput + '.metadata')} + ''' + if (Module['calledRun']) { + runMetaWithFS(); + } else { + if (!Module['preRun']) Module['preRun'] = []; + Module["preRun"].push(runMetaWithFS); + }\n''' % {'metadata_file': os.path.basename(options.jsoutput + '.metadata')} else: _metadata_template = ''' diff --git a/tools/settings.py b/tools/settings.py index 87b56ff68ee83..7dbf98e5b0f99 100644 --- a/tools/settings.py +++ b/tools/settings.py @@ -74,7 +74,6 @@ 'PTHREAD_POOL_SIZE_STRICT', 'PTHREAD_POOL_DELAY_LOAD', 'DEFAULT_PTHREAD_STACK_SIZE', - 'USE_FETCH', } # Subset of settings that apply at compile time. @@ -97,6 +96,7 @@ 'WASM_OBJECT_FILES', 'WASM_WORKERS', 'BULK_MEMORY', + 'USE_FETCH', # Internal settings used during compilation 'EXCEPTION_CATCHING_ALLOWED', From ffcaa3b16aabfd7bc2252b81c8f0834eb30cf470 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sat, 25 May 2024 21:42:49 -0400 Subject: [PATCH 03/34] Adding tests. --- test/test_browser.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/test/test_browser.py b/test/test_browser.py index 1e84a8d9c26ab..db48ced98a429 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -290,12 +290,13 @@ def test_emscripten_log(self): self.btest_exit('emscripten_log/emscripten_log.cpp', args=['-Wno-deprecated-pragma', '--pre-js', path_from_root('src/emscripten-source-map.min.js'), '-gsource-map']) - @also_with_wasmfs @parameterized({ '': ([],), - 'USE_FETCH': (['-sUSE_FETCH'],), + 'use_fetch': (['-sUSE_FETCH'],), }) + @also_with_wasmfs def test_preload_file(self, args): + print(repr(args)) create_file('somefile.txt', 'load me right before running the code please') create_file('.somefile.txt', 'load me right before running the code please') create_file('some@file.txt', 'load me right before running the code please') @@ -348,7 +349,7 @@ def make_main(path): for srcpath, dstpath in test_cases: print('Testing', srcpath, dstpath) make_main(dstpath) - self.btest_exit('main.cpp', args=['--preload-file', srcpath]) + self.btest_exit('main.cpp', args=['--preload-file', srcpath] + args) if WINDOWS: # On Windows, the following non-alphanumeric non-control code ASCII characters are supported. # The characters <, >, ", |, ?, * are not allowed, because the Windows filesystem doesn't support those. @@ -359,7 +360,7 @@ def make_main(path): create_file(tricky_filename, 'load me right before running the code please') make_main(tricky_filename) # As an Emscripten-specific feature, the character '@' must be escaped in the form '@@' to not confuse with the 'src@dst' notation. - self.btest_exit('main.cpp', ['--preload-file', tricky_filename.replace('@', '@@')] + args) + self.btest_exit('main.cpp', args=['--preload-file', tricky_filename.replace('@', '@@')] + args) # TODO: WASMFS doesn't support the rest of this test yet. Exit early. if self.get_setting('WASMFS'): @@ -368,7 +369,7 @@ def make_main(path): # By absolute path make_main('somefile.txt') # absolute becomes relative - self.btest_exit('main.cpp', ['--preload-file', absolute_src_path] + args) + self.btest_exit('main.cpp', args=['--preload-file', absolute_src_path] + args) # Test subdirectory handling with asset packaging. delete_dir('assets') @@ -1409,13 +1410,17 @@ def test_fs_workerfs_read(self): ''' % (secret, secret2)) self.btest_exit('fs/test_workerfs_read.c', args=['-lworkerfs.js', '--pre-js', 'pre.js', f'-DSECRET="{secret }"', f'-DSECRET2="{secret2}"', '--proxy-to-worker', '-lworkerfs.js']) - def test_fs_workerfs_package(self): + @parameterized({ + '': ([],), + 'use_fetch': (['-sUSE_FETCH'],), + }) + def test_fs_workerfs_package(self, args): self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', '$ccall') create_file('file1.txt', 'first') ensure_dir('sub') create_file('sub/file2.txt', 'second') self.run_process([FILE_PACKAGER, 'files.data', '--preload', 'file1.txt', Path('sub/file2.txt'), '--separate-metadata', '--js-output=files.js']) - self.btest(Path('fs/test_workerfs_package.cpp'), '1', args=['-lworkerfs.js', '--proxy-to-worker', '-lworkerfs.js']) + self.btest(Path('fs/test_workerfs_package.cpp'), '1', args=['-lworkerfs.js', '--proxy-to-worker', '-lworkerfs.js'] + args) def test_fs_lz4fs_package(self): # generate data @@ -2333,7 +2338,11 @@ def test_openal_capture_sanity(self): def test_openal_extensions(self): self.btest_exit('openal_extensions.c') - def test_runtimelink(self): + @parameterized({ + '': ([],), + 'use_fetch': (['-sUSE_FETCH'],) + }) + def test_runtimelink(self, args): create_file('header.h', r''' struct point { int x, y; @@ -2380,7 +2389,7 @@ def test_runtimelink(self): } ''') self.run_process([EMCC, 'supp.c', '-o', 'supp.wasm', '-sSIDE_MODULE', '-O2'] + self.get_emcc_args()) - self.btest_exit('main.c', args=['-sMAIN_MODULE=2', '-O2', 'supp.wasm']) + self.btest_exit('main.c', args=['-sMAIN_MODULE=2', '-O2', 'supp.wasm'] + args) @also_with_wasm2js def test_pre_run_deps(self): From de79ead3d5de9eae42ccd0ef48e8c12125d20da6 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sat, 25 May 2024 21:50:39 -0400 Subject: [PATCH 04/34] Tweak. --- src/settings.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/settings.js b/src/settings.js index e09ba8b1f0594..9c1bdce28cc0f 100644 --- a/src/settings.js +++ b/src/settings.js @@ -807,7 +807,7 @@ var NODEJS_CATCH_EXIT = true; // [link] var NODEJS_CATCH_REJECTION = true; -// Whether to use fetch instead of XHR to load files and packages. +// Whether to use fetch instead of XHR to load file packages and shared libraries. // Defaults to FALSE // // [link] From 139bcd40ddac835dfbc4ce72fefb45c72c9d25ee Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sat, 25 May 2024 21:54:35 -0400 Subject: [PATCH 05/34] Tweak. --- .../source/docs/tools_reference/settings_reference.rst | 10 ++++++++++ tools/file_packager.py | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/site/source/docs/tools_reference/settings_reference.rst b/site/source/docs/tools_reference/settings_reference.rst index 585ce6cdce562..a3be30a79685b 100644 --- a/site/source/docs/tools_reference/settings_reference.rst +++ b/site/source/docs/tools_reference/settings_reference.rst @@ -1202,6 +1202,16 @@ MIN_NODE_VERSION is 150000 or above. Default value: true +.. _use_fetch: + +USE_FETCH +========= + +Whether to use fetch instead of XHR to load file packages and shared libraries. +Defaults to FALSE + +Default value: false + .. _asyncify: ASYNCIFY diff --git a/tools/file_packager.py b/tools/file_packager.py index 498e704fc47f8..a6ce3b639d4af 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -1171,7 +1171,7 @@ def generate_js(data_target, data_files, metadata): _metadata_template = ''' Module['removeRunDependency']('%(metadata_file)s'); }''' - + if options.use_fetch: ret += ''' function runMetaWithFS() { @@ -1207,7 +1207,7 @@ def generate_js(data_target, data_files, metadata): } else { if (!Module['preRun']) Module['preRun'] = []; Module["preRun"].push(runMetaWithFS); - }\n''' % {'metadata_file': os.path.basename(options.jsoutput + '.metadata')} + }\n''' else: _metadata_template = ''' From 0407112eb97c8862cd893bda4e950f385e1f740f Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sat, 25 May 2024 21:57:31 -0400 Subject: [PATCH 06/34] Tweak. --- tools/file_packager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/file_packager.py b/tools/file_packager.py index a6ce3b639d4af..68ef5bdc55608 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -1018,7 +1018,7 @@ def generate_js(data_target, data_files, metadata): packageData.set(chunk, index); index += chunk.length; } - + callback(packageData.buffer); } }; From 3ead8de451a39811dcb1dd88f78a47fa8f1984cc Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sat, 25 May 2024 22:43:07 -0400 Subject: [PATCH 07/34] Rebasing generated sizes. --- test/code_size/hello_webgl2_wasm.json | 8 ++-- test/code_size/hello_webgl2_wasm2js.json | 8 ++-- test/code_size/hello_webgl_wasm.json | 8 ++-- test/code_size/hello_webgl_wasm2js.json | 8 ++-- test/core/test_em_asm_unicode.out | 2 +- test/core/test_env.out | 45 ++++++++++--------- test/core/test_environ.out | 4 +- test/core/test_longjmp.out | 2 +- test/other/test_unoptimized_code_size.js.size | 2 +- ...t_unoptimized_code_size_no_asserts.js.size | 2 +- .../test_unoptimized_code_size_strict.js.size | 2 +- 11 files changed, 46 insertions(+), 45 deletions(-) diff --git a/test/code_size/hello_webgl2_wasm.json b/test/code_size/hello_webgl2_wasm.json index 3cbe67f515809..6b3cc23cf3903 100644 --- a/test/code_size/hello_webgl2_wasm.json +++ b/test/code_size/hello_webgl2_wasm.json @@ -3,8 +3,8 @@ "a.html.gz": 379, "a.js": 4539, "a.js.gz": 2315, - "a.wasm": 10436, - "a.wasm.gz": 6725, - "total": 15544, - "total_gz": 9419 + "a.wasm": 10439, + "a.wasm.gz": 6728, + "total": 15547, + "total_gz": 9422 } diff --git a/test/code_size/hello_webgl2_wasm2js.json b/test/code_size/hello_webgl2_wasm2js.json index 6889500dd1e43..5779affbcfaa6 100644 --- a/test/code_size/hello_webgl2_wasm2js.json +++ b/test/code_size/hello_webgl2_wasm2js.json @@ -1,8 +1,8 @@ { "a.html": 354, "a.html.gz": 266, - "a.js": 22181, - "a.js.gz": 11582, - "total": 22535, - "total_gz": 11848 + "a.js": 22196, + "a.js.gz": 11586, + "total": 22550, + "total_gz": 11852 } diff --git a/test/code_size/hello_webgl_wasm.json b/test/code_size/hello_webgl_wasm.json index d1f596a3aa834..ed22032f89b92 100644 --- a/test/code_size/hello_webgl_wasm.json +++ b/test/code_size/hello_webgl_wasm.json @@ -3,8 +3,8 @@ "a.html.gz": 379, "a.js": 4056, "a.js.gz": 2152, - "a.wasm": 10436, - "a.wasm.gz": 6725, - "total": 15061, - "total_gz": 9256 + "a.wasm": 10439, + "a.wasm.gz": 6728, + "total": 15064, + "total_gz": 9259 } diff --git a/test/code_size/hello_webgl_wasm2js.json b/test/code_size/hello_webgl_wasm2js.json index f6ecebd21e1ab..bc112c36ee706 100644 --- a/test/code_size/hello_webgl_wasm2js.json +++ b/test/code_size/hello_webgl_wasm2js.json @@ -1,8 +1,8 @@ { "a.html": 354, "a.html.gz": 266, - "a.js": 21683, - "a.js.gz": 11407, - "total": 22037, - "total_gz": 11673 + "a.js": 21698, + "a.js.gz": 11410, + "total": 22052, + "total_gz": 11676 } diff --git a/test/core/test_em_asm_unicode.out b/test/core/test_em_asm_unicode.out index 4ef53eab9a770..31caff8090468 100644 --- a/test/core/test_em_asm_unicode.out +++ b/test/core/test_em_asm_unicode.out @@ -1 +1 @@ -hello world… \ No newline at end of file +hello world… diff --git a/test/core/test_env.out b/test/core/test_env.out index 993ec9530ee0f..b283000bdb636 100644 --- a/test/core/test_env.out +++ b/test/core/test_env.out @@ -4,26 +4,27 @@ LOGNAME=web_user PATH=/ PWD=/ HOME=/home/web_user -LANG=(C|en_US).UTF-8 -_=.*(/test_env.js|./this.program) +LANG=C.UTF-8 +_=/home/sean/projects/php-wasm/emscripten/out/test/test_env.js -getenv\(PATH\): / -getenv\(NONEXISTENT\): \(null\) -setenv/0\(PATH\) ret: 0 -getenv\(PATH\) after setenv/0: / -setenv/1\(PATH\) ret: 0 -getenv\(PATH\) after setenv/1: test2 -setenv\(SET_VALUE\) ret: 0 -setenv\(SET_VALUE2\) ret: 0 -getenv\(SET_VALUE\): test3 -getenv\(SET_VALUE2\): test4 -putenv\(PUT_VALUE\) ret: 0 -getenv\(PUT_VALUE\): test5 -getenv\(PUT_VALUE\) after alteration: Qest5 -unsetenv\(PUT_VALUE\) ret: 0 -getenv\(PUT_VALUE\) after unsetenv: \(null\) -setenv\(0\) ret: -1 -setenv\(''\) ret: -1 -setenv\(X=Y\) ret: -1 -unsetenv\(''\) ret: -1 -unsetenv\(X=Y\) ret: -1 +getenv(PATH): / +getenv(NONEXISTENT): (null) +setenv/0(PATH) ret: 0 +getenv(PATH) after setenv/0: / +setenv/1(PATH) ret: 0 +getenv(PATH) after setenv/1: test2 +setenv(SET_VALUE) ret: 0 +setenv(SET_VALUE2) ret: 0 +getenv(SET_VALUE): test3 +getenv(SET_VALUE2): test4 +putenv(PUT_VALUE) ret: 0 +getenv(PUT_VALUE): test5 +getenv(PUT_VALUE) after alteration: Qest5 +unsetenv(PUT_VALUE) ret: 0 +getenv(PUT_VALUE) after unsetenv: (null) +setenv(0) ret: -1 +setenv('') ret: -1 +setenv(X=Y) ret: -1 +unsetenv('') ret: -1 +unsetenv(X=Y) ret: -1 +clearenv -> 0 diff --git a/test/core/test_environ.out b/test/core/test_environ.out index 0ec4fa3b86303..1bb4746111860 100644 --- a/test/core/test_environ.out +++ b/test/core/test_environ.out @@ -3,5 +3,5 @@ LOGNAME=web_user PATH=/ PWD=/ HOME=/home/web_user -LANG=(C|en_US).UTF-8 -_=.*(/test_environ.js|/this.program) +LANG=C.UTF-8 +_=/home/sean/projects/php-wasm/emscripten/out/test/test_environ.js diff --git a/test/core/test_longjmp.out b/test/core/test_longjmp.out index 4a412fef09f74..e9cc75258d28d 100644 --- a/test/core/test_longjmp.out +++ b/test/core/test_longjmp.out @@ -1,4 +1,4 @@ first result: 1 1 second -result: 2 -1 \ No newline at end of file +result: 2 -1 diff --git a/test/other/test_unoptimized_code_size.js.size b/test/other/test_unoptimized_code_size.js.size index d004c80d94acc..59c3fc702689d 100644 --- a/test/other/test_unoptimized_code_size.js.size +++ b/test/other/test_unoptimized_code_size.js.size @@ -1 +1 @@ -55520 +55518 diff --git a/test/other/test_unoptimized_code_size_no_asserts.js.size b/test/other/test_unoptimized_code_size_no_asserts.js.size index 587cc797c2641..6a0a9c8c4c3ff 100644 --- a/test/other/test_unoptimized_code_size_no_asserts.js.size +++ b/test/other/test_unoptimized_code_size_no_asserts.js.size @@ -1 +1 @@ -31420 +31418 diff --git a/test/other/test_unoptimized_code_size_strict.js.size b/test/other/test_unoptimized_code_size_strict.js.size index b835a18890585..35f317d70a0b8 100644 --- a/test/other/test_unoptimized_code_size_strict.js.size +++ b/test/other/test_unoptimized_code_size_strict.js.size @@ -1 +1 @@ -54383 +54381 From 35b593f4b877e912f824fef1691a002f06cdb1d2 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sat, 25 May 2024 23:28:49 -0400 Subject: [PATCH 08/34] Reverting extraneous change. --- test/core/test_em_asm_unicode.out | 2 +- test/core/test_env.out | 45 +++++++++++++++---------------- test/core/test_environ.out | 4 +-- test/core/test_longjmp.out | 2 +- 4 files changed, 26 insertions(+), 27 deletions(-) diff --git a/test/core/test_em_asm_unicode.out b/test/core/test_em_asm_unicode.out index 31caff8090468..4ef53eab9a770 100644 --- a/test/core/test_em_asm_unicode.out +++ b/test/core/test_em_asm_unicode.out @@ -1 +1 @@ -hello world… +hello world… \ No newline at end of file diff --git a/test/core/test_env.out b/test/core/test_env.out index b283000bdb636..993ec9530ee0f 100644 --- a/test/core/test_env.out +++ b/test/core/test_env.out @@ -4,27 +4,26 @@ LOGNAME=web_user PATH=/ PWD=/ HOME=/home/web_user -LANG=C.UTF-8 -_=/home/sean/projects/php-wasm/emscripten/out/test/test_env.js +LANG=(C|en_US).UTF-8 +_=.*(/test_env.js|./this.program) -getenv(PATH): / -getenv(NONEXISTENT): (null) -setenv/0(PATH) ret: 0 -getenv(PATH) after setenv/0: / -setenv/1(PATH) ret: 0 -getenv(PATH) after setenv/1: test2 -setenv(SET_VALUE) ret: 0 -setenv(SET_VALUE2) ret: 0 -getenv(SET_VALUE): test3 -getenv(SET_VALUE2): test4 -putenv(PUT_VALUE) ret: 0 -getenv(PUT_VALUE): test5 -getenv(PUT_VALUE) after alteration: Qest5 -unsetenv(PUT_VALUE) ret: 0 -getenv(PUT_VALUE) after unsetenv: (null) -setenv(0) ret: -1 -setenv('') ret: -1 -setenv(X=Y) ret: -1 -unsetenv('') ret: -1 -unsetenv(X=Y) ret: -1 -clearenv -> 0 +getenv\(PATH\): / +getenv\(NONEXISTENT\): \(null\) +setenv/0\(PATH\) ret: 0 +getenv\(PATH\) after setenv/0: / +setenv/1\(PATH\) ret: 0 +getenv\(PATH\) after setenv/1: test2 +setenv\(SET_VALUE\) ret: 0 +setenv\(SET_VALUE2\) ret: 0 +getenv\(SET_VALUE\): test3 +getenv\(SET_VALUE2\): test4 +putenv\(PUT_VALUE\) ret: 0 +getenv\(PUT_VALUE\): test5 +getenv\(PUT_VALUE\) after alteration: Qest5 +unsetenv\(PUT_VALUE\) ret: 0 +getenv\(PUT_VALUE\) after unsetenv: \(null\) +setenv\(0\) ret: -1 +setenv\(''\) ret: -1 +setenv\(X=Y\) ret: -1 +unsetenv\(''\) ret: -1 +unsetenv\(X=Y\) ret: -1 diff --git a/test/core/test_environ.out b/test/core/test_environ.out index 1bb4746111860..0ec4fa3b86303 100644 --- a/test/core/test_environ.out +++ b/test/core/test_environ.out @@ -3,5 +3,5 @@ LOGNAME=web_user PATH=/ PWD=/ HOME=/home/web_user -LANG=C.UTF-8 -_=/home/sean/projects/php-wasm/emscripten/out/test/test_environ.js +LANG=(C|en_US).UTF-8 +_=.*(/test_environ.js|/this.program) diff --git a/test/core/test_longjmp.out b/test/core/test_longjmp.out index e9cc75258d28d..4a412fef09f74 100644 --- a/test/core/test_longjmp.out +++ b/test/core/test_longjmp.out @@ -1,4 +1,4 @@ first result: 1 1 second -result: 2 -1 +result: 2 -1 \ No newline at end of file From 5c2a7bb34074c1fff8482ad7799c390851e1d284 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sun, 26 May 2024 01:06:45 -0400 Subject: [PATCH 09/34] Readability. --- tools/file_packager.py | 82 ++++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/tools/file_packager.py b/tools/file_packager.py index 68ef5bdc55608..ad989691c0cca 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -1172,43 +1172,55 @@ def generate_js(data_target, data_files, metadata): Module['removeRunDependency']('%(metadata_file)s'); }''' - if options.use_fetch: - ret += ''' - function runMetaWithFS() { - Module['addRunDependency']('%(metadata_file)s'); - var REMOTE_METADATA_NAME = Module['locateFile'] ? Module['locateFile']('%(metadata_file)s', '') : '%(metadata_file)s'; - fetch(REMOTE_METADATA_NAME) - .then(response => { - if(response.ok) { - return response.json(); - } - }) - .then(loadPackage); - }''' % {'metadata_file': os.path.basename(options.jsoutput + '.metadata')} - else: - ret += ''' - function runMetaWithFS() { - Module['addRunDependency']('%(metadata_file)s'); - var REMOTE_METADATA_NAME = Module['locateFile'] ? Module['locateFile']('%(metadata_file)s', '') : '%(metadata_file)s'; - var xhr = new XMLHttpRequest(); - xhr.onreadystatechange = function() { - if (xhr.readyState === 4 && xhr.status === 200) { - loadPackage(JSON.parse(xhr.responseText)); - } - } - xhr.open('GET', REMOTE_METADATA_NAME, true); - xhr.overrideMimeType('application/json'); - xhr.send(null); - }''' % {'metadata_file': os.path.basename(options.jsoutput + '.metadata')} + if options.separate_metadata and not options.use_fetch: + _metadata_template = ''' + Module['removeRunDependency']('%(metadata_file)s'); + } - ''' - if (Module['calledRun']) { - runMetaWithFS(); - } else { - if (!Module['preRun']) Module['preRun'] = []; - Module["preRun"].push(runMetaWithFS); - }\n''' + function runMetaWithFS() { + Module['addRunDependency']('%(metadata_file)s'); + var REMOTE_METADATA_NAME = Module['locateFile'] ? Module['locateFile']('%(metadata_file)s', '') : '%(metadata_file)s'; + var xhr = new XMLHttpRequest(); + xhr.onreadystatechange = function() { + if (xhr.readyState === 4 && xhr.status === 200) { + loadPackage(JSON.parse(xhr.responseText)); + } + } + xhr.open('GET', REMOTE_METADATA_NAME, true); + xhr.overrideMimeType('application/json'); + xhr.send(null); + } + + if (Module['calledRun']) { + runMetaWithFS(); + } else { + if (!Module['preRun']) Module['preRun'] = []; + Module["preRun"].push(runMetaWithFS); + }\n''' % {'metadata_file': os.path.basename(options.jsoutput + '.metadata')} + + elif options.separate_metadata and optons.use_fetch: + _metadata_template = ''' + Module['removeRunDependency']('%(metadata_file)s'); + } + + function runMetaWithFS() { + Module['addRunDependency']('%(metadata_file)s'); + var REMOTE_METADATA_NAME = Module['locateFile'] ? Module['locateFile']('%(metadata_file)s', '') : '%(metadata_file)s'; + fetch(REMOTE_METADATA_NAME) + .then(response => { + if(response.ok) { + return response.json(); + } + }) + .then(loadPackage); + } + if (Module['calledRun']) { + runMetaWithFS(); + } else { + if (!Module['preRun']) Module['preRun'] = []; + Module["preRun"].push(runMetaWithFS); + }\n''' % {'metadata_file': os.path.basename(options.jsoutput + '.metadata')} else: _metadata_template = ''' } From 56319281e41e7231d816e78b5e9faf78ed61d2a3 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sun, 26 May 2024 01:07:50 -0400 Subject: [PATCH 10/34] Testing --- test/test_browser.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/test_browser.py b/test/test_browser.py index db48ced98a429..130239ba8bdd6 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -1479,14 +1479,17 @@ def test_fs_lz4fs_package(self): out = subprocess.check_output([FILE_PACKAGER, 'files.data', '--preload', 'files/file1.txt', 'files/file2.txt', 'files/file3.txt']) create_file('files.js', out, binary=True) self.btest_exit('fs/test_lz4fs.cpp', 2, args=['--pre-js', 'files.js'])''' - - def test_separate_metadata_later(self): + @parameterized({ + '': ([],), + 'use_fetch': (['-sUSE_FETCH'],), + }) + def test_separate_metadata_later(self, args): # see issue #6654 - we need to handle separate-metadata both when we run before # the main program, and when we are run later create_file('data.dat', ' ') self.run_process([FILE_PACKAGER, 'more.data', '--preload', 'data.dat', '--separate-metadata', '--js-output=more.js']) - self.btest(Path('browser/separate_metadata_later.cpp'), '1', args=['-sFORCE_FILESYSTEM']) + self.btest(Path('browser/separate_metadata_later.cpp'), '1', args=['-sFORCE_FILESYSTEM'] + args) def test_idbstore(self): secret = str(time.time()) From ff2d14265ca1575219f4164d413f859d161d7ba4 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sun, 26 May 2024 01:15:32 -0400 Subject: [PATCH 11/34] Removing debug. --- test/test_browser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_browser.py b/test/test_browser.py index 130239ba8bdd6..10b6a95d63275 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -296,7 +296,6 @@ def test_emscripten_log(self): }) @also_with_wasmfs def test_preload_file(self, args): - print(repr(args)) create_file('somefile.txt', 'load me right before running the code please') create_file('.somefile.txt', 'load me right before running the code please') create_file('some@file.txt', 'load me right before running the code please') From 9e06ef2763433358263e7b8b533f74db3726c162 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sun, 26 May 2024 01:17:48 -0400 Subject: [PATCH 12/34] Typo. --- tools/file_packager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/file_packager.py b/tools/file_packager.py index ad989691c0cca..df73891312c46 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -1198,7 +1198,7 @@ def generate_js(data_target, data_files, metadata): Module["preRun"].push(runMetaWithFS); }\n''' % {'metadata_file': os.path.basename(options.jsoutput + '.metadata')} - elif options.separate_metadata and optons.use_fetch: + elif options.separate_metadata and options.use_fetch: _metadata_template = ''' Module['removeRunDependency']('%(metadata_file)s'); } From 5899d3f180a4812636428e563e66a9c363b24ad3 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sun, 26 May 2024 01:25:25 -0400 Subject: [PATCH 13/34] Parameterizing test. --- test/test_browser.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_browser.py b/test/test_browser.py index 10b6a95d63275..f6220c2723d2d 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -1479,15 +1479,15 @@ def test_fs_lz4fs_package(self): create_file('files.js', out, binary=True) self.btest_exit('fs/test_lz4fs.cpp', 2, args=['--pre-js', 'files.js'])''' @parameterized({ - '': ([],), - 'use_fetch': (['-sUSE_FETCH'],), + '': ([],[]), + 'use_fetch': (['-sUSE_FETCH'],['--use-fetch']), }) - def test_separate_metadata_later(self, args): + def test_separate_metadata_later(self, args, packArgs): # see issue #6654 - we need to handle separate-metadata both when we run before # the main program, and when we are run later create_file('data.dat', ' ') - self.run_process([FILE_PACKAGER, 'more.data', '--preload', 'data.dat', '--separate-metadata', '--js-output=more.js']) + self.run_process([FILE_PACKAGER, 'more.data', '--preload', 'data.dat', '--separate-metadata', '--js-output=more.js'] + packArgs) self.btest(Path('browser/separate_metadata_later.cpp'), '1', args=['-sFORCE_FILESYSTEM'] + args) def test_idbstore(self): From c09f2c5356fd5f37c122431fe214ce2048976669 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sun, 26 May 2024 01:27:08 -0400 Subject: [PATCH 14/34] Spacing. --- test/test_browser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_browser.py b/test/test_browser.py index f6220c2723d2d..39a5dd5336933 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -1479,8 +1479,8 @@ def test_fs_lz4fs_package(self): create_file('files.js', out, binary=True) self.btest_exit('fs/test_lz4fs.cpp', 2, args=['--pre-js', 'files.js'])''' @parameterized({ - '': ([],[]), - 'use_fetch': (['-sUSE_FETCH'],['--use-fetch']), + '': ([], []), + 'use_fetch': (['-sUSE_FETCH'], ['--use-fetch']), }) def test_separate_metadata_later(self, args, packArgs): # see issue #6654 - we need to handle separate-metadata both when we run before From 2f43c20f3b6e8742b6c154463385ee74226aff33 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sun, 26 May 2024 10:10:30 -0400 Subject: [PATCH 15/34] Deduping default info in docs. --- site/source/docs/tools_reference/settings_reference.rst | 1 - src/settings.js | 1 - 2 files changed, 2 deletions(-) diff --git a/site/source/docs/tools_reference/settings_reference.rst b/site/source/docs/tools_reference/settings_reference.rst index a3be30a79685b..a98510e1b51f3 100644 --- a/site/source/docs/tools_reference/settings_reference.rst +++ b/site/source/docs/tools_reference/settings_reference.rst @@ -1208,7 +1208,6 @@ USE_FETCH ========= Whether to use fetch instead of XHR to load file packages and shared libraries. -Defaults to FALSE Default value: false diff --git a/src/settings.js b/src/settings.js index 9c1bdce28cc0f..b2357b1148efa 100644 --- a/src/settings.js +++ b/src/settings.js @@ -808,7 +808,6 @@ var NODEJS_CATCH_EXIT = true; var NODEJS_CATCH_REJECTION = true; // Whether to use fetch instead of XHR to load file packages and shared libraries. -// Defaults to FALSE // // [link] var USE_FETCH = false; From c5ecce51687181ca4fb0008e154bd2dda4011e29 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sun, 26 May 2024 15:09:46 -0400 Subject: [PATCH 16/34] Deduping if statement. --- tools/file_packager.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tools/file_packager.py b/tools/file_packager.py index df73891312c46..fb021dd8e81f7 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -1167,11 +1167,6 @@ def generate_js(data_target, data_files, metadata): Module["preRun"].push(runWithFS); // FS is not initialized yet, wait for it }\n''' - if options.separate_metadata: - _metadata_template = ''' - Module['removeRunDependency']('%(metadata_file)s'); - }''' - if options.separate_metadata and not options.use_fetch: _metadata_template = ''' Module['removeRunDependency']('%(metadata_file)s'); From 7e86b5bfac3702a913b9133be097c6b83b70423d Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Sun, 26 May 2024 21:39:47 -0400 Subject: [PATCH 17/34] Correcting indentation of JS output. --- tools/file_packager.py | 182 ++++++++++++++++++++--------------------- 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/tools/file_packager.py b/tools/file_packager.py index fb021dd8e81f7..5ddc9892a1ae4 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -21,7 +21,7 @@ Usage: - file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] + file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--use-fetch] --preload , --embed See emcc --help for more details on those options. @@ -362,7 +362,7 @@ def generate_object_file(data_files): def main(): if len(sys.argv) == 1: - err('''Usage: file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] + err('''Usage: file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--use-fetch] See the source for more details.''') return 1 @@ -780,7 +780,7 @@ def generate_js(data_target, data_files, metadata): } var REMOTE_PACKAGE_NAME = Module['locateFile'] ? Module['locateFile'](REMOTE_PACKAGE_BASE, '') : REMOTE_PACKAGE_BASE;\n''' % (js_manipulation.escape_for_js_string(data_target), js_manipulation.escape_for_js_string(remote_package_name)) metadata['remote_package_size'] = remote_package_size - ret += '''var REMOTE_PACKAGE_SIZE = metadata['remote_package_size'];\n''' + ret += ''' var REMOTE_PACKAGE_SIZE = metadata['remote_package_size'];\n''' if options.use_preload_cache: # Set the id to a hash of the preloaded data, so that caches survive over multiple builds @@ -972,109 +972,109 @@ def generate_js(data_target, data_files, metadata): }'''.strip() if options.use_fetch: ret += ''' - function fetchRemotePackage(packageName, packageSize, callback, errback) { - %(node_support_code)s - Module.dataFileDownloads = Module.dataFileDownloads || {}; - const url = packageName; - fetch(url).then(response => { + function fetchRemotePackage(packageName, packageSize, callback, errback) { + %(node_support_code)s + Module.dataFileDownloads = Module.dataFileDownloads || {}; + const url = packageName; + fetch(url).then(response => { - let loaded = 0; + let loaded = 0; - const reader = response.body.getReader(); - const headers = response.headers; + const reader = response.body.getReader(); + const headers = response.headers; - if (!response.ok) { - throw new Error(response.statusText + ' : ' + response.url); - } + if (!response.ok) { + throw new Error(response.statusText + ' : ' + response.url); + } - const total = headers.get('Content-Length') ?? packageSize; - const chunks = []; + const total = headers.get('Content-Length') ?? packageSize; + const chunks = []; - const iterate = () => reader.read().then(handleChunk).catch(cause => { - throw new Error(response.statusText + ' : ' + response.url, {cause}); - }); + const iterate = () => reader.read().then(handleChunk).catch(cause => { + throw new Error(response.statusText + ' : ' + response.url, {cause}); + }); - const handleChunk = ({done, value}) => { - if (!done) { - chunks.push(value); - loaded += value.length; - Module.dataFileDownloads[url] = Module.dataFileDownloads[url] ?? {}; - Module.dataFileDownloads[url].loaded = loaded; - Module.dataFileDownloads[url].total = total; + const handleChunk = ({done, value}) => { + if (!done) { + chunks.push(value); + loaded += value.length; + Module.dataFileDownloads[url] = Module.dataFileDownloads[url] ?? {}; + Module.dataFileDownloads[url].loaded = loaded; + Module.dataFileDownloads[url].total = total; - if (total) { - if (Module['setStatus']) Module['setStatus'](`Downloading data... (${loaded}/${total})`); - } - else { - if (Module['setStatus']) Module['setStatus']('Downloading data...'); - } - return iterate(); + if (total) { + if (Module['setStatus']) Module['setStatus'](`Downloading data... (${loaded}/${total})`); } else { - const size = chunks.map(c => c.length).reduce((a, b) => a + b, 0); - let index = 0; - const packageData = new Uint8Array(size); - for(const chunk of chunks) { - packageData.set(chunk, index); - index += chunk.length; - } - - callback(packageData.buffer); - } - }; - return iterate(); - }); - };\n''' % {'node_support_code': node_support_code} - else: - ret += ''' - function fetchRemotePackage(packageName, packageSize, callback, errback) { - %(node_support_code)s - var xhr = new XMLHttpRequest(); - xhr.open('GET', packageName, true); - xhr.responseType = 'arraybuffer'; - xhr.onprogress = function(event) { - var url = packageName; - var size = packageSize; - if (event.total) size = event.total; - if (event.loaded) { - if (!xhr.addedTotal) { - xhr.addedTotal = true; - if (!Module.dataFileDownloads) Module.dataFileDownloads = {}; - Module.dataFileDownloads[url] = { - loaded: event.loaded, - total: size - }; - } else { - Module.dataFileDownloads[url].loaded = event.loaded; + if (Module['setStatus']) Module['setStatus']('Downloading data...'); } - var total = 0; - var loaded = 0; - var num = 0; - for (var download in Module.dataFileDownloads) { - var data = Module.dataFileDownloads[download]; - total += data.total; - loaded += data.loaded; - num++; + return iterate(); + } + else { + const size = chunks.map(c => c.length).reduce((a, b) => a + b, 0); + let index = 0; + const packageData = new Uint8Array(size); + for(const chunk of chunks) { + packageData.set(chunk, index); + index += chunk.length; } - total = Math.ceil(total * Module.expectedDataFileDownloads/num); - if (Module['setStatus']) Module['setStatus'](`Downloading data... (${loaded}/${total})`); - } else if (!Module.dataFileDownloads) { - if (Module['setStatus']) Module['setStatus']('Downloading data...'); + + callback(packageData.buffer); } }; - xhr.onerror = function(event) { - throw new Error("NetworkError for: " + packageName); - } - xhr.onload = function(event) { - if (xhr.status == 200 || xhr.status == 304 || xhr.status == 206 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0 - var packageData = xhr.response; - callback(packageData); + return iterate(); + }); + };\n''' % {'node_support_code': node_support_code} + else: + ret += ''' + function fetchRemotePackage(packageName, packageSize, callback, errback) { + %(node_support_code)s + var xhr = new XMLHttpRequest(); + xhr.open('GET', packageName, true); + xhr.responseType = 'arraybuffer'; + xhr.onprogress = function(event) { + var url = packageName; + var size = packageSize; + if (event.total) size = event.total; + if (event.loaded) { + if (!xhr.addedTotal) { + xhr.addedTotal = true; + if (!Module.dataFileDownloads) Module.dataFileDownloads = {}; + Module.dataFileDownloads[url] = { + loaded: event.loaded, + total: size + }; } else { - throw new Error(xhr.statusText + " : " + xhr.responseURL); + Module.dataFileDownloads[url].loaded = event.loaded; } - }; - xhr.send(null); - };\n''' % {'node_support_code': node_support_code} + var total = 0; + var loaded = 0; + var num = 0; + for (var download in Module.dataFileDownloads) { + var data = Module.dataFileDownloads[download]; + total += data.total; + loaded += data.loaded; + num++; + } + total = Math.ceil(total * Module.expectedDataFileDownloads/num); + if (Module['setStatus']) Module['setStatus'](`Downloading data... (${loaded}/${total})`); + } else if (!Module.dataFileDownloads) { + if (Module['setStatus']) Module['setStatus']('Downloading data...'); + } + }; + xhr.onerror = function(event) { + throw new Error("NetworkError for: " + packageName); + } + xhr.onload = function(event) { + if (xhr.status == 200 || xhr.status == 304 || xhr.status == 206 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0 + var packageData = xhr.response; + callback(packageData); + } else { + throw new Error(xhr.statusText + " : " + xhr.responseURL); + } + }; + xhr.send(null); + };\n''' % {'node_support_code': node_support_code} ret += ''' function handleError(error) { From 684805591d44de63a49f3c802413fa5784148e74 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Wed, 29 May 2024 03:02:58 -0400 Subject: [PATCH 18/34] Removing use_fetch option and XmlHttpRequest. --- .../tools_reference/settings_reference.rst | 9 -- src/settings.js | 6 -- src/web_or_worker_shell_read.js | 18 +--- test/test_browser.py | 31 ++----- tools/file_packager.py | 91 +------------------ tools/link.py | 2 - tools/settings.py | 1 - 7 files changed, 14 insertions(+), 144 deletions(-) diff --git a/site/source/docs/tools_reference/settings_reference.rst b/site/source/docs/tools_reference/settings_reference.rst index a98510e1b51f3..585ce6cdce562 100644 --- a/site/source/docs/tools_reference/settings_reference.rst +++ b/site/source/docs/tools_reference/settings_reference.rst @@ -1202,15 +1202,6 @@ MIN_NODE_VERSION is 150000 or above. Default value: true -.. _use_fetch: - -USE_FETCH -========= - -Whether to use fetch instead of XHR to load file packages and shared libraries. - -Default value: false - .. _asyncify: ASYNCIFY diff --git a/src/settings.js b/src/settings.js index b2357b1148efa..c382fca69c919 100644 --- a/src/settings.js +++ b/src/settings.js @@ -807,11 +807,6 @@ var NODEJS_CATCH_EXIT = true; // [link] var NODEJS_CATCH_REJECTION = true; -// Whether to use fetch instead of XHR to load file packages and shared libraries. -// -// [link] -var USE_FETCH = false; - // Whether to support async operations in the compiled code. This makes it // possible to call JS functions from synchronous-looking code in C/C++. // @@ -2182,7 +2177,6 @@ var OFFSCREEN_FRAMEBUFFER_FORBID_VAO_PATH = false; // [link] var TEST_MEMORY_GROWTH_FAILS = false; - // For renamed settings the format is: // [OLD_NAME, NEW_NAME] // For removed settings (which now effectively have a fixed value and can no diff --git a/src/web_or_worker_shell_read.js b/src/web_or_worker_shell_read.js index 36e36ff5d9f86..8d7deb484272e 100644 --- a/src/web_or_worker_shell_read.js +++ b/src/web_or_worker_shell_read.js @@ -20,7 +20,7 @@ return new Uint8Array(/** @type{!ArrayBuffer} */(xhr.response)); }; } -#if USE_FETCH + readAsync = (url, onload, onerror) => { fetch(url) .then(response => { @@ -32,19 +32,3 @@ .then(onload) .catch(onerror) }; -#else - readAsync = (url, onload, onerror) => { - var xhr = new XMLHttpRequest(); - xhr.open('GET', url, true); - xhr.responseType = 'arraybuffer'; - xhr.onload = () => { - if (xhr.status == 200 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0 - onload(xhr.response); - return; - } - onerror(); - }; - xhr.onerror = onerror; - xhr.send(null); - } -#endif diff --git a/test/test_browser.py b/test/test_browser.py index 39a5dd5336933..6f147204226fe 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -290,12 +290,8 @@ def test_emscripten_log(self): self.btest_exit('emscripten_log/emscripten_log.cpp', args=['-Wno-deprecated-pragma', '--pre-js', path_from_root('src/emscripten-source-map.min.js'), '-gsource-map']) - @parameterized({ - '': ([],), - 'use_fetch': (['-sUSE_FETCH'],), - }) @also_with_wasmfs - def test_preload_file(self, args): + def test_preload_file(self): create_file('somefile.txt', 'load me right before running the code please') create_file('.somefile.txt', 'load me right before running the code please') create_file('some@file.txt', 'load me right before running the code please') @@ -1409,17 +1405,13 @@ def test_fs_workerfs_read(self): ''' % (secret, secret2)) self.btest_exit('fs/test_workerfs_read.c', args=['-lworkerfs.js', '--pre-js', 'pre.js', f'-DSECRET="{secret }"', f'-DSECRET2="{secret2}"', '--proxy-to-worker', '-lworkerfs.js']) - @parameterized({ - '': ([],), - 'use_fetch': (['-sUSE_FETCH'],), - }) - def test_fs_workerfs_package(self, args): + def test_fs_workerfs_package(self): self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', '$ccall') create_file('file1.txt', 'first') ensure_dir('sub') create_file('sub/file2.txt', 'second') self.run_process([FILE_PACKAGER, 'files.data', '--preload', 'file1.txt', Path('sub/file2.txt'), '--separate-metadata', '--js-output=files.js']) - self.btest(Path('fs/test_workerfs_package.cpp'), '1', args=['-lworkerfs.js', '--proxy-to-worker', '-lworkerfs.js'] + args) + self.btest(Path('fs/test_workerfs_package.cpp'), '1', args=['-lworkerfs.js', '--proxy-to-worker', '-lworkerfs.js']) def test_fs_lz4fs_package(self): # generate data @@ -1478,17 +1470,14 @@ def test_fs_lz4fs_package(self): out = subprocess.check_output([FILE_PACKAGER, 'files.data', '--preload', 'files/file1.txt', 'files/file2.txt', 'files/file3.txt']) create_file('files.js', out, binary=True) self.btest_exit('fs/test_lz4fs.cpp', 2, args=['--pre-js', 'files.js'])''' - @parameterized({ - '': ([], []), - 'use_fetch': (['-sUSE_FETCH'], ['--use-fetch']), - }) - def test_separate_metadata_later(self, args, packArgs): + + def test_separate_metadata_later(self): # see issue #6654 - we need to handle separate-metadata both when we run before # the main program, and when we are run later create_file('data.dat', ' ') - self.run_process([FILE_PACKAGER, 'more.data', '--preload', 'data.dat', '--separate-metadata', '--js-output=more.js'] + packArgs) - self.btest(Path('browser/separate_metadata_later.cpp'), '1', args=['-sFORCE_FILESYSTEM'] + args) + self.run_process([FILE_PACKAGER, 'more.data', '--preload', 'data.dat', '--separate-metadata', '--js-output=more.js']) + self.btest(Path('browser/separate_metadata_later.cpp'), '1', args=['-sFORCE_FILESYSTEM']) def test_idbstore(self): secret = str(time.time()) @@ -2340,11 +2329,7 @@ def test_openal_capture_sanity(self): def test_openal_extensions(self): self.btest_exit('openal_extensions.c') - @parameterized({ - '': ([],), - 'use_fetch': (['-sUSE_FETCH'],) - }) - def test_runtimelink(self, args): + def test_runtimelink(self): create_file('header.h', r''' struct point { int x, y; diff --git a/tools/file_packager.py b/tools/file_packager.py index 5ddc9892a1ae4..8de2aedec8aac 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -21,13 +21,11 @@ Usage: - file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--use-fetch] + file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] --preload , --embed See emcc --help for more details on those options. - --use-fetch Use fetch instead of XHR. - --exclude E [F..] Specifies filename pattern matches to use for excluding given files from being added to the package. See https://docs.python.org/2/library/fnmatch.html for syntax. @@ -130,7 +128,6 @@ def __init__(self): self.use_preload_plugins = False self.support_node = True self.wasm64 = False - self.use_fetch = False class DataFile: @@ -362,7 +359,7 @@ def generate_object_file(data_files): def main(): if len(sys.argv) == 1: - err('''Usage: file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--use-fetch] + err('''Usage: file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] See the source for more details.''') return 1 @@ -376,8 +373,6 @@ def main(): leading = 'preload' elif arg == '--embed': leading = 'embed' - elif arg == '--use-fetch': - options.use_fetch = True elif arg == '--exclude': leading = 'exclude' elif arg == '--no-force': @@ -970,8 +965,8 @@ def generate_js(data_target, data_files, metadata): }); return; }'''.strip() - if options.use_fetch: - ret += ''' + + ret += ''' function fetchRemotePackage(packageName, packageSize, callback, errback) { %(node_support_code)s Module.dataFileDownloads = Module.dataFileDownloads || {}; @@ -1025,56 +1020,6 @@ def generate_js(data_target, data_files, metadata): return iterate(); }); };\n''' % {'node_support_code': node_support_code} - else: - ret += ''' - function fetchRemotePackage(packageName, packageSize, callback, errback) { - %(node_support_code)s - var xhr = new XMLHttpRequest(); - xhr.open('GET', packageName, true); - xhr.responseType = 'arraybuffer'; - xhr.onprogress = function(event) { - var url = packageName; - var size = packageSize; - if (event.total) size = event.total; - if (event.loaded) { - if (!xhr.addedTotal) { - xhr.addedTotal = true; - if (!Module.dataFileDownloads) Module.dataFileDownloads = {}; - Module.dataFileDownloads[url] = { - loaded: event.loaded, - total: size - }; - } else { - Module.dataFileDownloads[url].loaded = event.loaded; - } - var total = 0; - var loaded = 0; - var num = 0; - for (var download in Module.dataFileDownloads) { - var data = Module.dataFileDownloads[download]; - total += data.total; - loaded += data.loaded; - num++; - } - total = Math.ceil(total * Module.expectedDataFileDownloads/num); - if (Module['setStatus']) Module['setStatus'](`Downloading data... (${loaded}/${total})`); - } else if (!Module.dataFileDownloads) { - if (Module['setStatus']) Module['setStatus']('Downloading data...'); - } - }; - xhr.onerror = function(event) { - throw new Error("NetworkError for: " + packageName); - } - xhr.onload = function(event) { - if (xhr.status == 200 || xhr.status == 304 || xhr.status == 206 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0 - var packageData = xhr.response; - callback(packageData); - } else { - throw new Error(xhr.statusText + " : " + xhr.responseURL); - } - }; - xhr.send(null); - };\n''' % {'node_support_code': node_support_code} ret += ''' function handleError(error) { @@ -1167,33 +1112,7 @@ def generate_js(data_target, data_files, metadata): Module["preRun"].push(runWithFS); // FS is not initialized yet, wait for it }\n''' - if options.separate_metadata and not options.use_fetch: - _metadata_template = ''' - Module['removeRunDependency']('%(metadata_file)s'); - } - - function runMetaWithFS() { - Module['addRunDependency']('%(metadata_file)s'); - var REMOTE_METADATA_NAME = Module['locateFile'] ? Module['locateFile']('%(metadata_file)s', '') : '%(metadata_file)s'; - var xhr = new XMLHttpRequest(); - xhr.onreadystatechange = function() { - if (xhr.readyState === 4 && xhr.status === 200) { - loadPackage(JSON.parse(xhr.responseText)); - } - } - xhr.open('GET', REMOTE_METADATA_NAME, true); - xhr.overrideMimeType('application/json'); - xhr.send(null); - } - - if (Module['calledRun']) { - runMetaWithFS(); - } else { - if (!Module['preRun']) Module['preRun'] = []; - Module["preRun"].push(runMetaWithFS); - }\n''' % {'metadata_file': os.path.basename(options.jsoutput + '.metadata')} - - elif options.separate_metadata and options.use_fetch: + if options.separate_metadata: _metadata_template = ''' Module['removeRunDependency']('%(metadata_file)s'); } diff --git a/tools/link.py b/tools/link.py index 337b3d97e646a..09d1496910585 100644 --- a/tools/link.py +++ b/tools/link.py @@ -2997,8 +2997,6 @@ def package_files(options, target): if options.embed_files: file_args.append('--embed') file_args += options.embed_files - if settings.USE_FETCH: - file_args.append('--use-fetch') if options.exclude_files: file_args.append('--exclude') file_args += options.exclude_files diff --git a/tools/settings.py b/tools/settings.py index 7dbf98e5b0f99..fc884b3046deb 100644 --- a/tools/settings.py +++ b/tools/settings.py @@ -96,7 +96,6 @@ 'WASM_OBJECT_FILES', 'WASM_WORKERS', 'BULK_MEMORY', - 'USE_FETCH', # Internal settings used during compilation 'EXCEPTION_CATCHING_ALLOWED', From 0bf9b719cc61c494f6c0358dfdc6b376e47a48b2 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Wed, 29 May 2024 03:10:54 -0400 Subject: [PATCH 19/34] Reverting extraneous changes. --- test/test_browser.py | 8 ++++---- test/test_sanity.py | 4 ++-- tools/shared.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_browser.py b/test/test_browser.py index 6f147204226fe..957883dbe17cb 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -344,7 +344,7 @@ def make_main(path): for srcpath, dstpath in test_cases: print('Testing', srcpath, dstpath) make_main(dstpath) - self.btest_exit('main.cpp', args=['--preload-file', srcpath] + args) + self.btest_exit('main.cpp', args=['--preload-file', srcpath]) if WINDOWS: # On Windows, the following non-alphanumeric non-control code ASCII characters are supported. # The characters <, >, ", |, ?, * are not allowed, because the Windows filesystem doesn't support those. @@ -355,7 +355,7 @@ def make_main(path): create_file(tricky_filename, 'load me right before running the code please') make_main(tricky_filename) # As an Emscripten-specific feature, the character '@' must be escaped in the form '@@' to not confuse with the 'src@dst' notation. - self.btest_exit('main.cpp', args=['--preload-file', tricky_filename.replace('@', '@@')] + args) + self.btest_exit('main.cpp', args=['--preload-file', tricky_filename.replace('@', '@@')]) # TODO: WASMFS doesn't support the rest of this test yet. Exit early. if self.get_setting('WASMFS'): @@ -364,7 +364,7 @@ def make_main(path): # By absolute path make_main('somefile.txt') # absolute becomes relative - self.btest_exit('main.cpp', args=['--preload-file', absolute_src_path] + args) + self.btest_exit('main.cpp', args=['--preload-file', absolute_src_path]) # Test subdirectory handling with asset packaging. delete_dir('assets') @@ -2376,7 +2376,7 @@ def test_runtimelink(self): } ''') self.run_process([EMCC, 'supp.c', '-o', 'supp.wasm', '-sSIDE_MODULE', '-O2'] + self.get_emcc_args()) - self.btest_exit('main.c', args=['-sMAIN_MODULE=2', '-O2', 'supp.wasm'] + args) + self.btest_exit('main.c', args=['-sMAIN_MODULE=2', '-O2', 'supp.wasm']) @also_with_wasm2js def test_pre_run_deps(self): diff --git a/test/test_sanity.py b/test/test_sanity.py index e1de89dd12711..dcbabb5d0b47d 100644 --- a/test/test_sanity.py +++ b/test/test_sanity.py @@ -284,8 +284,8 @@ def test_node(self): for version, succeed in [('v0.8.0', False), ('v4.1.0', False), ('v10.18.0', False), - ('v16.20.0', True), - ('v16.20.1-pre', True), + ('v19.20.3', True), + ('v19.20.4-pre', True), ('cheez', False)]: print(version, succeed) delete_file(SANITY_FILE) diff --git a/tools/shared.py b/tools/shared.py index 9eee4a374b8fd..9f8f834247ae7 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -58,7 +58,7 @@ # (settings.MIN_NODE_VERSION). # This version currently matches the node version that we ship with emsdk # which means that we can say for sure that this version is well supported. -MINIMUM_NODE_VERSION = (16, 20, 0) +MINIMUM_NODE_VERSION = (18, 20, 3) EXPECTED_LLVM_VERSION = 19 # These get set by setup_temp_dirs From 3f765bccfcedbec95daf79e691afe78ac4dd3536 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Wed, 29 May 2024 03:12:04 -0400 Subject: [PATCH 20/34] Revert "Reverting extraneous changes." This reverts commit 0bf9b719cc61c494f6c0358dfdc6b376e47a48b2. --- test/test_browser.py | 8 ++++---- test/test_sanity.py | 4 ++-- tools/shared.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_browser.py b/test/test_browser.py index 957883dbe17cb..6f147204226fe 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -344,7 +344,7 @@ def make_main(path): for srcpath, dstpath in test_cases: print('Testing', srcpath, dstpath) make_main(dstpath) - self.btest_exit('main.cpp', args=['--preload-file', srcpath]) + self.btest_exit('main.cpp', args=['--preload-file', srcpath] + args) if WINDOWS: # On Windows, the following non-alphanumeric non-control code ASCII characters are supported. # The characters <, >, ", |, ?, * are not allowed, because the Windows filesystem doesn't support those. @@ -355,7 +355,7 @@ def make_main(path): create_file(tricky_filename, 'load me right before running the code please') make_main(tricky_filename) # As an Emscripten-specific feature, the character '@' must be escaped in the form '@@' to not confuse with the 'src@dst' notation. - self.btest_exit('main.cpp', args=['--preload-file', tricky_filename.replace('@', '@@')]) + self.btest_exit('main.cpp', args=['--preload-file', tricky_filename.replace('@', '@@')] + args) # TODO: WASMFS doesn't support the rest of this test yet. Exit early. if self.get_setting('WASMFS'): @@ -364,7 +364,7 @@ def make_main(path): # By absolute path make_main('somefile.txt') # absolute becomes relative - self.btest_exit('main.cpp', args=['--preload-file', absolute_src_path]) + self.btest_exit('main.cpp', args=['--preload-file', absolute_src_path] + args) # Test subdirectory handling with asset packaging. delete_dir('assets') @@ -2376,7 +2376,7 @@ def test_runtimelink(self): } ''') self.run_process([EMCC, 'supp.c', '-o', 'supp.wasm', '-sSIDE_MODULE', '-O2'] + self.get_emcc_args()) - self.btest_exit('main.c', args=['-sMAIN_MODULE=2', '-O2', 'supp.wasm']) + self.btest_exit('main.c', args=['-sMAIN_MODULE=2', '-O2', 'supp.wasm'] + args) @also_with_wasm2js def test_pre_run_deps(self): diff --git a/test/test_sanity.py b/test/test_sanity.py index dcbabb5d0b47d..e1de89dd12711 100644 --- a/test/test_sanity.py +++ b/test/test_sanity.py @@ -284,8 +284,8 @@ def test_node(self): for version, succeed in [('v0.8.0', False), ('v4.1.0', False), ('v10.18.0', False), - ('v19.20.3', True), - ('v19.20.4-pre', True), + ('v16.20.0', True), + ('v16.20.1-pre', True), ('cheez', False)]: print(version, succeed) delete_file(SANITY_FILE) diff --git a/tools/shared.py b/tools/shared.py index 9f8f834247ae7..9eee4a374b8fd 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -58,7 +58,7 @@ # (settings.MIN_NODE_VERSION). # This version currently matches the node version that we ship with emsdk # which means that we can say for sure that this version is well supported. -MINIMUM_NODE_VERSION = (18, 20, 3) +MINIMUM_NODE_VERSION = (16, 20, 0) EXPECTED_LLVM_VERSION = 19 # These get set by setup_temp_dirs From 316aa8b7768e2107a3d59573e4d549418f1555fd Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Wed, 29 May 2024 03:13:13 -0400 Subject: [PATCH 21/34] Actually reverting extraneous change. --- test/test_browser.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_browser.py b/test/test_browser.py index 6f147204226fe..957883dbe17cb 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -344,7 +344,7 @@ def make_main(path): for srcpath, dstpath in test_cases: print('Testing', srcpath, dstpath) make_main(dstpath) - self.btest_exit('main.cpp', args=['--preload-file', srcpath] + args) + self.btest_exit('main.cpp', args=['--preload-file', srcpath]) if WINDOWS: # On Windows, the following non-alphanumeric non-control code ASCII characters are supported. # The characters <, >, ", |, ?, * are not allowed, because the Windows filesystem doesn't support those. @@ -355,7 +355,7 @@ def make_main(path): create_file(tricky_filename, 'load me right before running the code please') make_main(tricky_filename) # As an Emscripten-specific feature, the character '@' must be escaped in the form '@@' to not confuse with the 'src@dst' notation. - self.btest_exit('main.cpp', args=['--preload-file', tricky_filename.replace('@', '@@')] + args) + self.btest_exit('main.cpp', args=['--preload-file', tricky_filename.replace('@', '@@')]) # TODO: WASMFS doesn't support the rest of this test yet. Exit early. if self.get_setting('WASMFS'): @@ -364,7 +364,7 @@ def make_main(path): # By absolute path make_main('somefile.txt') # absolute becomes relative - self.btest_exit('main.cpp', args=['--preload-file', absolute_src_path] + args) + self.btest_exit('main.cpp', args=['--preload-file', absolute_src_path]) # Test subdirectory handling with asset packaging. delete_dir('assets') @@ -2376,7 +2376,7 @@ def test_runtimelink(self): } ''') self.run_process([EMCC, 'supp.c', '-o', 'supp.wasm', '-sSIDE_MODULE', '-O2'] + self.get_emcc_args()) - self.btest_exit('main.c', args=['-sMAIN_MODULE=2', '-O2', 'supp.wasm'] + args) + self.btest_exit('main.c', args=['-sMAIN_MODULE=2', '-O2', 'supp.wasm']) @also_with_wasm2js def test_pre_run_deps(self): From 5899e4208847b1b477e9def115b80d7ffaa11467 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Wed, 29 May 2024 04:03:32 -0400 Subject: [PATCH 22/34] Throw error on bad fetch in runMetaWithFS --- tools/file_packager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/file_packager.py b/tools/file_packager.py index 8de2aedec8aac..48a1f76251472 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -1125,6 +1125,7 @@ def generate_js(data_target, data_files, metadata): if(response.ok) { return response.json(); } + throw new Error(response.statusText + ' : ' + response.url); }) .then(loadPackage); } From 92ffd133ac2b30cfefde562d3402b93c72e0674e Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Wed, 29 May 2024 08:43:40 -0400 Subject: [PATCH 23/34] Tweaks. --- test/test_browser.py | 8 +++++--- tools/file_packager.py | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/test/test_browser.py b/test/test_browser.py index 957883dbe17cb..4ec8d010dc57b 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -699,15 +699,17 @@ def setup(assetLocalization):
+ {{{ SCRIPT }}} + + ''') + + self.run_process([EMCC, 'library.c', '-sSIDE_MODULE', '-O2', '-o', 'library.so']) + + def test(args, expect_fail): + self.compile_btest('main.c', ['library.so', '-sMAIN_MODULE', '--shell-file', 'on_window_error_shell.html', '-o', 'a.out.html']) + js = read_file('a.out.js') + if expect_fail: + create_file('a.out.js', 'fetch = undefined;\n' + js) + return self.run_browser('a.out.html', '/report_result?abort:TypeError') + else: + return self.run_browser('a.out.html', '/report_result?1') + + test([], expect_fail=True) + test(['-sLEGACY_VM_SUPPORT'], expect_fail=False) + test(['-sLEGACY_VM_SUPPORT', '-sNO_PROXY'], expect_fail=True) class emrun(RunnerCore): def test_emrun_info(self): @@ -5655,7 +5717,6 @@ def test_emrun(self): self.assertContained('Testing char sequences: %20%21 ä', stdout) self.assertContained('hello, error stream!', stderr) - class browser64(browser): def setUp(self): super().setUp() diff --git a/test/test_other.py b/test/test_other.py index b81ed3c99f393..6af047a7fdbd8 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -12028,11 +12028,10 @@ def test_standalone_syscalls(self): for engine in config.WASM_ENGINES: self.assertContained(expected, self.run_js('test.wasm', engine)) - def test_fetch_polyfill(self): + def test_fetch_polyfill_preload(self): path = 'hello-world.txt' create_file(path, 'hello, world!') create_file('main.cpp', r''' - #include #include #include #include @@ -12058,6 +12057,44 @@ def test(args, expect_fail): output = test(['-sLEGACY_VM_SUPPORT'], expect_fail=False) self.assertContained('|hello, world!|', output) + def test_fetch_polyfill_shared_lib(self): + create_file('library.c', r''' + #include + int library_func() { + return 42; + } + ''') + create_file('main.c', r''' + #include + #include + #include + #include + int main() { + int found = EM_ASM_INT( + return preloadedWasm['/library.so'] !== undefined; + ); + void *lib_handle = dlopen("/library.so", RTLD_NOW); + assert(lib_handle); + typedef int (*voidfunc)(); + voidfunc x = (voidfunc)dlsym(lib_handle, "library_func"); + printf("Got val: %d\n", x()); + assert(x); + assert(x() == 42); + return 0; + } + ''') + + self.run_process([EMCC, 'library.c', '-sSIDE_MODULE', '-O2', '-o', 'library.so']) + + def test(args, expect_fail): + self.run_process([EMCC, 'main.c', 'library.so', '-sMAIN_MODULE', '-o', 'a.out.js']) + js = read_file('a.out.js') + create_file('a.out.js', 'fetch = undefined;\n' + js) + return self.run_js('a.out.js', assert_returncode=NON_ZERO if expect_fail else 0) + + test([], expect_fail=True) + test(['-sLEGACY_VM_SUPPORT'], expect_fail=False) + @parameterized({ 'wasm2js': (['-sWASM=0'], ''), 'modularize': (['-sMODULARIZE'], 'Module()'), From 457ef2ce8d232b30db07edaed3d0ed3f40b10be3 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Wed, 29 May 2024 19:19:51 -0400 Subject: [PATCH 31/34] Adding fetch polyfill + test --- test/common.py | 2 +- test/test_browser.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/common.py b/test/common.py index 382991f6b3f41..a342524e42d30 100644 --- a/test/common.py +++ b/test/common.py @@ -2270,7 +2270,7 @@ def compile_btest(self, filename, args, reporting=Reporting.FULL): if reporting == Reporting.FULL: # If C reporting (i.e. the REPORT_RESULT macro) is required we # also include report_result.c and force-include report_result.h - self.run_process([EMCC, '-c', '-fPIC','-I' + TEST_ROOT, + self.run_process([EMCC, '-c', '-fPIC', '-I' + TEST_ROOT, '-DEMTEST_PORT_NUMBER=%d' % self.port, test_file('report_result.c')] + self.get_emcc_args(compile_only=True)) args += ['report_result.o', '-include', test_file('report_result.h')] diff --git a/test/test_browser.py b/test/test_browser.py index d13855dd0cecb..238c3d7cbf483 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -5613,6 +5613,7 @@ def test(args, expect_fail): test(['-sLEGACY_VM_SUPPORT'], expect_fail=False) test(['-sLEGACY_VM_SUPPORT', '-sNO_PROXY'], expect_fail=True) + class emrun(RunnerCore): def test_emrun_info(self): if not has_browser(): @@ -5717,6 +5718,7 @@ def test_emrun(self): self.assertContained('Testing char sequences: %20%21 ä', stdout) self.assertContained('hello, error stream!', stderr) + class browser64(browser): def setUp(self): super().setUp() From aeac94e5746f365f4c3fe8f51fbff933a53f6d58 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Wed, 29 May 2024 20:47:36 -0400 Subject: [PATCH 32/34] Polyfill tests for browser. --- src/polyfill/fetch.js | 3 ++ test/test_browser.py | 64 ++++++++++++++++++++++++++++++++++++++++ test/test_other.py | 67 ------------------------------------------ tools/file_packager.py | 12 ++++++-- 4 files changed, 76 insertions(+), 70 deletions(-) diff --git a/src/polyfill/fetch.js b/src/polyfill/fetch.js index 8ef81a3b6d400..979335699a84e 100644 --- a/src/polyfill/fetch.js +++ b/src/polyfill/fetch.js @@ -34,6 +34,8 @@ if (typeof globalThis.fetch == 'undefined') { const keys = []; const headers = {}; + request.responseType = 'arraybuffer'; + const response = () => ({ ok: ((request.status / 100) | 0) == 2, // 200-299 statusText: request.statusText, @@ -42,6 +44,7 @@ if (typeof globalThis.fetch == 'undefined') { text: () => Promise.resolve(request.responseText), json: () => Promise.resolve(request.responseText).then(JSON.parse), blob: () => Promise.resolve(new Blob([request.response])), + arrayBuffer: () => Promise.resolve(request.response), clone: response, headers: { keys: () => keys, diff --git a/test/test_browser.py b/test/test_browser.py index 238c3d7cbf483..fd637a10e1ba6 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -5550,6 +5550,70 @@ def test_webpack(self): shutil.copyfile('webpack/src/hello.wasm', 'webpack/dist/hello.wasm') self.run_browser('webpack/dist/index.html', '/report_result?exit:0') + def test_fetch_polyfill_preload(self): + path = 'hello-world.txt' + create_file(path, 'hello, world!') + create_file('main.cpp', r''' + #include + #include + #include + int main() { + FILE *f = fopen("%s", "r"); + char buf[100]; + fread(buf, 1, 20, f); + buf[20] = 0; + fclose(f); + printf("|%%s|\n", buf); + return 0; + } + ''' % path) + create_file('on_window_error_shell.html', r''' + +
+
+ + {{{ SCRIPT }}} + + ''') + + def test(args, expect_fail): + self.compile_btest('main.cpp', args + ['--preload-file', path, '--shell-file', 'on_window_error_shell.html', '-o', 'a.out.html']) + js = read_file('a.out.js') + if expect_fail: + create_file('a.out.js', 'fetch = undefined;\n' + js) + return self.run_browser('a.out.html', '/report_result?TypeError: fetch is not a function') + else: + return self.run_browser('a.out.html', '/report_result?1') + + test([], expect_fail=True) + test(['-sPOLYFILL'], expect_fail=False) + test(['-sLEGACY_VM_SUPPORT', '-sNO_POLYFILL'], expect_fail=True) + def test_fetch_polyfill_shared_lib(self): create_file('library.c', r''' #include diff --git a/test/test_other.py b/test/test_other.py index 6af047a7fdbd8..242ce132b3fe4 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -12028,73 +12028,6 @@ def test_standalone_syscalls(self): for engine in config.WASM_ENGINES: self.assertContained(expected, self.run_js('test.wasm', engine)) - def test_fetch_polyfill_preload(self): - path = 'hello-world.txt' - create_file(path, 'hello, world!') - create_file('main.cpp', r''' - #include - #include - #include - int main() { - FILE *f = fopen("%s", "r"); - char buf[100]; - fread(buf, 1, 20, f); - buf[20] = 0; - fclose(f); - printf("|%%s|\n", buf); - return 0; - } - ''' % path) - - def test(args, expect_fail): - self.run_process([EMCC, 'main.cpp'] + args + ['--preload-file', path]) - js = read_file('a.out.js') - create_file('a.out.js', 'fetch = undefined;\n' + js) - return self.run_js('a.out.js', assert_returncode=NON_ZERO if expect_fail else 0) - - test([], expect_fail=True) - - output = test(['-sLEGACY_VM_SUPPORT'], expect_fail=False) - self.assertContained('|hello, world!|', output) - - def test_fetch_polyfill_shared_lib(self): - create_file('library.c', r''' - #include - int library_func() { - return 42; - } - ''') - create_file('main.c', r''' - #include - #include - #include - #include - int main() { - int found = EM_ASM_INT( - return preloadedWasm['/library.so'] !== undefined; - ); - void *lib_handle = dlopen("/library.so", RTLD_NOW); - assert(lib_handle); - typedef int (*voidfunc)(); - voidfunc x = (voidfunc)dlsym(lib_handle, "library_func"); - printf("Got val: %d\n", x()); - assert(x); - assert(x() == 42); - return 0; - } - ''') - - self.run_process([EMCC, 'library.c', '-sSIDE_MODULE', '-O2', '-o', 'library.so']) - - def test(args, expect_fail): - self.run_process([EMCC, 'main.c', 'library.so', '-sMAIN_MODULE', '-o', 'a.out.js']) - js = read_file('a.out.js') - create_file('a.out.js', 'fetch = undefined;\n' + js) - return self.run_js('a.out.js', assert_returncode=NON_ZERO if expect_fail else 0) - - test([], expect_fail=True) - test(['-sLEGACY_VM_SUPPORT'], expect_fail=False) - @parameterized({ 'wasm2js': (['-sWASM=0'], ''), 'modularize': (['-sMODULARIZE'], 'Module()'), diff --git a/tools/file_packager.py b/tools/file_packager.py index 0f247b6e2a642..e247f5666b58e 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -977,13 +977,19 @@ def generate_js(data_target, data_files, metadata): let loaded = 0; - const reader = response.body.getReader(); - const headers = response.headers; - if (!response.ok) { return Promise.reject(new Error(response.statusText + ' : ' + response.url)); } + // If we're using the polyfill, readers won't be available... + if(!response.body && response.arrayBuffer) { + response.arrayBuffer().then(buffer => callback(buffer)); + return; + } + + const reader = response.body.getReader(); + const headers = response.headers; + const total = headers.get('Content-Length') ?? packageSize; const chunks = []; From 275388a6a9e696648a78f7e2373885a8fe76e94c Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Wed, 29 May 2024 21:03:09 -0400 Subject: [PATCH 33/34] Tweaks. --- test/test_browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_browser.py b/test/test_browser.py index fd637a10e1ba6..2189fe7b658fb 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -5675,7 +5675,7 @@ def test(args, expect_fail): test([], expect_fail=True) test(['-sLEGACY_VM_SUPPORT'], expect_fail=False) - test(['-sLEGACY_VM_SUPPORT', '-sNO_PROXY'], expect_fail=True) + test(['-sLEGACY_VM_SUPPORT', '-sNO_POLYFILL'], expect_fail=True) class emrun(RunnerCore): From d4a61622df7eda9a37af72b9c40d1a300229dd01 Mon Sep 17 00:00:00 2001 From: Sean Morris <640101+seanmorris@users.noreply.github.com> Date: Wed, 29 May 2024 21:06:56 -0400 Subject: [PATCH 34/34] Tweaks. --- test/test_browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_browser.py b/test/test_browser.py index 2189fe7b658fb..2164e15e821c9 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -5611,7 +5611,7 @@ def test(args, expect_fail): return self.run_browser('a.out.html', '/report_result?1') test([], expect_fail=True) - test(['-sPOLYFILL'], expect_fail=False) + test(['-sLEGACY_VM_SUPPORT'], expect_fail=False) test(['-sLEGACY_VM_SUPPORT', '-sNO_POLYFILL'], expect_fail=True) def test_fetch_polyfill_shared_lib(self):