Use fetch() for loading packages & shared libraries

site/source/docs/tools_reference/settings_reference.rst

@@ -1202,6 +1202,15 @@ MIN_NODE_VERSION is 150000 or above.
 
 Default value: true
 
+.. _use_fetch:
+
+USE_FETCH
+=========
+
+Whether to use fetch instead of XHR to load file packages and shared libraries.
+
+Default value: false
+
 .. _asyncify:
 
 ASYNCIFY

src/settings.js

@@ -807,6 +807,11 @@ var NODEJS_CATCH_EXIT = true;
 // [link]
 var NODEJS_CATCH_REJECTION = true;
 
+// Whether to use fetch instead of XHR to load file packages and shared libraries.
+//
+// [link]
+var USE_FETCH = false;
+
 // Whether to support async operations in the compiled code. This makes it
 // possible to call JS functions from synchronous-looking code in C/C++.
 //
@@ -2177,6 +2182,7 @@ var OFFSCREEN_FRAMEBUFFER_FORBID_VAO_PATH = false;
 // [link]
 var TEST_MEMORY_GROWTH_FAILS = false;
 
+
 // For renamed settings the format is:
 // [OLD_NAME, NEW_NAME]
 // For removed settings (which now effectively have a fixed value and can no

src/web_or_worker_shell_read.js

@@ -20,7 +20,19 @@
       return new Uint8Array(/** @type{!ArrayBuffer} */(xhr.response));
     };
   }
-
+#if USE_FETCH
+  readAsync = (url, onload, onerror) => {
+    fetch(url)
+    .then(response => {
+      if(response.ok) {
+        return response.arrayBuffer();
+      }
+      throw new Error(response.statusText + ' : ' + response.url);
+    })
+    .then(onload)
+    .catch(onerror)
+  };
+#else
   readAsync = (url, onload, onerror) => {
     var xhr = new XMLHttpRequest();
     xhr.open('GET', url, true);
@@ -35,4 +47,4 @@
     xhr.onerror = onerror;
     xhr.send(null);
   }
-
+#endif

test/code_size/hello_webgl2_wasm.json

@@ -3,8 +3,8 @@
   "a.html.gz": 379,
   "a.js": 4539,
   "a.js.gz": 2315,
-  "a.wasm": 10436,
-  "a.wasm.gz": 6725,
-  "total": 15544,
-  "total_gz": 9419
+  "a.wasm": 10439,
+  "a.wasm.gz": 6728,
+  "total": 15547,
+  "total_gz": 9422
 }

test/code_size/hello_webgl2_wasm2js.json

@@ -1,8 +1,8 @@
 {
   "a.html": 354,
   "a.html.gz": 266,
-  "a.js": 22181,
-  "a.js.gz": 11582,
-  "total": 22535,
-  "total_gz": 11848
+  "a.js": 22196,
+  "a.js.gz": 11586,
+  "total": 22550,
+  "total_gz": 11852
 }

test/code_size/hello_webgl_wasm.json

@@ -3,8 +3,8 @@
   "a.html.gz": 379,
   "a.js": 4056,
   "a.js.gz": 2152,
-  "a.wasm": 10436,
-  "a.wasm.gz": 6725,
-  "total": 15061,
-  "total_gz": 9256
+  "a.wasm": 10439,
+  "a.wasm.gz": 6728,
+  "total": 15064,
+  "total_gz": 9259
 }

test/code_size/hello_webgl_wasm2js.json

@@ -1,8 +1,8 @@
 {
   "a.html": 354,
   "a.html.gz": 266,
-  "a.js": 21683,
-  "a.js.gz": 11407,
-  "total": 22037,
-  "total_gz": 11673
+  "a.js": 21698,
+  "a.js.gz": 11410,
+  "total": 22052,
+  "total_gz": 11676
 }

test/other/test_unoptimized_code_size.js.size

@@ -1 +1 @@
-55520
+55518

test/other/test_unoptimized_code_size_no_asserts.js.size

@@ -1 +1 @@
-31420
+31418

test/other/test_unoptimized_code_size_strict.js.size

@@ -1 +1 @@
-54383
+54381

test/test_browser.py

@@ -290,8 +290,12 @@ def test_emscripten_log(self):
     self.btest_exit('emscripten_log/emscripten_log.cpp',
                     args=['-Wno-deprecated-pragma', '--pre-js', path_from_root('src/emscripten-source-map.min.js'), '-gsource-map'])
 
+  @parameterized({
+    '': ([],),
+    'use_fetch': (['-sUSE_FETCH'],),
+  })
   @also_with_wasmfs
-  def test_preload_file(self):
+  def test_preload_file(self, args):
     create_file('somefile.txt', 'load me right before running the code please')
     create_file('.somefile.txt', 'load me right before running the code please')
     create_file('[email protected]', 'load me right before running the code please')
@@ -344,7 +348,7 @@ def make_main(path):
     for srcpath, dstpath in test_cases:
       print('Testing', srcpath, dstpath)
       make_main(dstpath)
-      self.btest_exit('main.cpp', args=['--preload-file', srcpath])
+      self.btest_exit('main.cpp', args=['--preload-file', srcpath] + args)
     if WINDOWS:
       # On Windows, the following non-alphanumeric non-control code ASCII characters are supported.
       # The characters <, >, ", |, ?, * are not allowed, because the Windows filesystem doesn't support those.
@@ -355,7 +359,7 @@ def make_main(path):
     create_file(tricky_filename, 'load me right before running the code please')
     make_main(tricky_filename)
     # As an Emscripten-specific feature, the character '@' must be escaped in the form '@@' to not confuse with the 'src@dst' notation.
-    self.btest_exit('main.cpp', args=['--preload-file', tricky_filename.replace('@', '@@')])
+    self.btest_exit('main.cpp', args=['--preload-file', tricky_filename.replace('@', '@@')] + args)
 
     # TODO: WASMFS doesn't support the rest of this test yet. Exit early.
     if self.get_setting('WASMFS'):
@@ -364,7 +368,7 @@ def make_main(path):
     # By absolute path
 
     make_main('somefile.txt') # absolute becomes relative
-    self.btest_exit('main.cpp', args=['--preload-file', absolute_src_path])
+    self.btest_exit('main.cpp', args=['--preload-file', absolute_src_path] + args)
 
     # Test subdirectory handling with asset packaging.
     delete_dir('assets')
@@ -1405,13 +1409,17 @@ def test_fs_workerfs_read(self):
     ''' % (secret, secret2))
     self.btest_exit('fs/test_workerfs_read.c', args=['-lworkerfs.js', '--pre-js', 'pre.js', f'-DSECRET="{secret }"', f'-DSECRET2="{secret2}"', '--proxy-to-worker', '-lworkerfs.js'])
 
-  def test_fs_workerfs_package(self):
+  @parameterized({
+    '': ([],),
+    'use_fetch': (['-sUSE_FETCH'],),
+  })
+  def test_fs_workerfs_package(self, args):
     self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', '$ccall')
     create_file('file1.txt', 'first')
     ensure_dir('sub')
     create_file('sub/file2.txt', 'second')
     self.run_process([FILE_PACKAGER, 'files.data', '--preload', 'file1.txt', Path('sub/file2.txt'), '--separate-metadata', '--js-output=files.js'])
-    self.btest(Path('fs/test_workerfs_package.cpp'), '1', args=['-lworkerfs.js', '--proxy-to-worker', '-lworkerfs.js'])
+    self.btest(Path('fs/test_workerfs_package.cpp'), '1', args=['-lworkerfs.js', '--proxy-to-worker', '-lworkerfs.js'] + args)
 
   def test_fs_lz4fs_package(self):
     # generate data
@@ -1470,14 +1478,17 @@ def test_fs_lz4fs_package(self):
     out = subprocess.check_output([FILE_PACKAGER, 'files.data', '--preload', 'files/file1.txt', 'files/file2.txt', 'files/file3.txt'])
     create_file('files.js', out, binary=True)
     self.btest_exit('fs/test_lz4fs.cpp', 2, args=['--pre-js', 'files.js'])'''
-
-  def test_separate_metadata_later(self):
+  @parameterized({
+    '': ([], []),
+    'use_fetch': (['-sUSE_FETCH'], ['--use-fetch']),
+  })
+  def test_separate_metadata_later(self, args, packArgs):
     # see issue #6654 - we need to handle separate-metadata both when we run before
     # the main program, and when we are run later
 
     create_file('data.dat', ' ')
-    self.run_process([FILE_PACKAGER, 'more.data', '--preload', 'data.dat', '--separate-metadata', '--js-output=more.js'])
-    self.btest(Path('browser/separate_metadata_later.cpp'), '1', args=['-sFORCE_FILESYSTEM'])
+    self.run_process([FILE_PACKAGER, 'more.data', '--preload', 'data.dat', '--separate-metadata', '--js-output=more.js'] + packArgs)
+    self.btest(Path('browser/separate_metadata_later.cpp'), '1', args=['-sFORCE_FILESYSTEM'] + args)
 
   def test_idbstore(self):
     secret = str(time.time())
@@ -2329,7 +2340,11 @@ def test_openal_capture_sanity(self):
   def test_openal_extensions(self):
     self.btest_exit('openal_extensions.c')
 
-  def test_runtimelink(self):
+  @parameterized({
+    '': ([],),
+    'use_fetch': (['-sUSE_FETCH'],)
+  })
+  def test_runtimelink(self, args):
     create_file('header.h', r'''
       struct point {
         int x, y;
@@ -2376,7 +2391,7 @@ def test_runtimelink(self):
       }
     ''')
     self.run_process([EMCC, 'supp.c', '-o', 'supp.wasm', '-sSIDE_MODULE', '-O2'] + self.get_emcc_args())
-    self.btest_exit('main.c', args=['-sMAIN_MODULE=2', '-O2', 'supp.wasm'])
+    self.btest_exit('main.c', args=['-sMAIN_MODULE=2', '-O2', 'supp.wasm'] + args)
 
   @also_with_wasm2js
   def test_pre_run_deps(self):

tools/file_packager.py

@@ -21,11 +21,13 @@
 
 Usage:
 
-  file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node]
+  file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--use-fetch]
 
   --preload  ,
   --embed    See emcc --help for more details on those options.
 
+  --use-fetch Use fetch instead of XHR.
+
   --exclude E [F..] Specifies filename pattern matches to use for excluding given files from being added to the package.
                     See https://docs.python.org/2/library/fnmatch.html for syntax.
 
@@ -128,6 +130,7 @@ def __init__(self):
     self.use_preload_plugins = False
     self.support_node = True
     self.wasm64 = False
+    self.use_fetch = False
 
 
 class DataFile:
@@ -359,7 +362,7 @@ def generate_object_file(data_files):
 
 def main():
   if len(sys.argv) == 1:
-    err('''Usage: file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins]
+    err('''Usage: file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node] [--use-fetch]
   See the source for more details.''')
     return 1
 
@@ -373,6 +376,8 @@ def main():
       leading = 'preload'
     elif arg == '--embed':
       leading = 'embed'
+    elif arg == '--use-fetch':
+      options.use_fetch = True
     elif arg == '--exclude':
       leading = 'exclude'
     elif arg == '--no-force':
@@ -775,7 +780,7 @@ def generate_js(data_target, data_files, metadata):
       }
       var REMOTE_PACKAGE_NAME = Module['locateFile'] ? Module['locateFile'](REMOTE_PACKAGE_BASE, '') : REMOTE_PACKAGE_BASE;\n''' % (js_manipulation.escape_for_js_string(data_target), js_manipulation.escape_for_js_string(remote_package_name))
     metadata['remote_package_size'] = remote_package_size
-    ret += '''var REMOTE_PACKAGE_SIZE = metadata['remote_package_size'];\n'''
+    ret += '''      var REMOTE_PACKAGE_SIZE = metadata['remote_package_size'];\n'''
 
     if options.use_preload_cache:
       # Set the id to a hash of the preloaded data, so that caches survive over multiple builds
@@ -965,7 +970,63 @@ def generate_js(data_target, data_files, metadata):
           });
           return;
         }'''.strip()
-    ret += '''
+    if options.use_fetch:
+      ret += '''
+      function fetchRemotePackage(packageName, packageSize, callback, errback) {
+        %(node_support_code)s
+        Module.dataFileDownloads = Module.dataFileDownloads || {};
+        const url = packageName;
+        fetch(url).then(response => {
+
+          let loaded = 0;
+
+          const reader = response.body.getReader();
+          const headers = response.headers;
+
+          if (!response.ok) {
+            throw new Error(response.statusText + ' : ' + response.url);
+          }
+
+          const total = headers.get('Content-Length') ?? packageSize;
+          const chunks = [];
+
+          const iterate = () => reader.read().then(handleChunk).catch(cause => {
+            throw new Error(response.statusText + ' : ' + response.url, {cause});
+          });
+
+          const handleChunk = ({done, value}) => {
+            if (!done) {
+              chunks.push(value);
+              loaded += value.length;
+              Module.dataFileDownloads[url] = Module.dataFileDownloads[url] ?? {};
+              Module.dataFileDownloads[url].loaded = loaded;
+              Module.dataFileDownloads[url].total = total;
+
+              if (total) {
+                if (Module['setStatus']) Module['setStatus'](`Downloading data... (${loaded}/${total})`);
+              }
+              else {
+                if (Module['setStatus']) Module['setStatus']('Downloading data...');
+              }
+              return iterate();
+            }
+            else {
+              const size = chunks.map(c => c.length).reduce((a, b) => a + b, 0);
+              let index = 0;
+              const packageData = new Uint8Array(size);
+              for(const chunk of chunks) {
+                packageData.set(chunk, index);
+                index += chunk.length;
+              }
+
+              callback(packageData.buffer);
+            }
+          };
+          return iterate();
+        });
+      };\n''' % {'node_support_code': node_support_code}
+    else:
+      ret += '''
       function fetchRemotePackage(packageName, packageSize, callback, errback) {
         %(node_support_code)s
         var xhr = new XMLHttpRequest();
@@ -1013,11 +1074,12 @@ def generate_js(data_target, data_files, metadata):
           }
         };
         xhr.send(null);
-      };
+      };\n''' % {'node_support_code': node_support_code}
 
+    ret += '''
       function handleError(error) {
         console.error('package error:', error);
-      };\n''' % {'node_support_code': node_support_code}
+      };\n'''
 
     code += '''
       function processPackageData(arrayBuffer) {
@@ -1105,7 +1167,7 @@ def generate_js(data_target, data_files, metadata):
       Module["preRun"].push(runWithFS); // FS is not initialized yet, wait for it
     }\n'''
 
-  if options.separate_metadata:
+  if options.separate_metadata and not options.use_fetch:
       _metadata_template = '''
     Module['removeRunDependency']('%(metadata_file)s');
   }
@@ -1131,6 +1193,29 @@ def generate_js(data_target, data_files, metadata):
     Module["preRun"].push(runMetaWithFS);
   }\n''' % {'metadata_file': os.path.basename(options.jsoutput + '.metadata')}
 
+  elif options.separate_metadata and options.use_fetch:
+      _metadata_template = '''
+    Module['removeRunDependency']('%(metadata_file)s');
+  }
+
+  function runMetaWithFS() {
+    Module['addRunDependency']('%(metadata_file)s');
+    var REMOTE_METADATA_NAME = Module['locateFile'] ? Module['locateFile']('%(metadata_file)s', '') : '%(metadata_file)s';
+    fetch(REMOTE_METADATA_NAME)
+    .then(response => {
+      if(response.ok) {
+        return response.json();
+      }
+    })
+    .then(loadPackage);
+  }
+
+  if (Module['calledRun']) {
+    runMetaWithFS();
+  } else {
+    if (!Module['preRun']) Module['preRun'] = [];
+    Module["preRun"].push(runMetaWithFS);
+  }\n''' % {'metadata_file': os.path.basename(options.jsoutput + '.metadata')}
   else:
       _metadata_template = '''
     }