Use fetch() for loading packages & shared libraries

src/web_or_worker_shell_read.js

@@ -22,17 +22,13 @@
   }
 
   readAsync = (url, onload, onerror) => {
-    var xhr = new XMLHttpRequest();
-    xhr.open('GET', url, true);
-    xhr.responseType = 'arraybuffer';
-    xhr.onload = () => {
-      if (xhr.status == 200 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0
-        onload(xhr.response);
-        return;
+    fetch(url)
+    .then(response => {
+      if(response.ok) {
+        return response.arrayBuffer();
       }
-      onerror();
-    };
-    xhr.onerror = onerror;
-    xhr.send(null);
-  }
-
+      throw new Error(response.statusText + ' : ' + response.url);
+    })
+    .then(onload)
+    .catch(onerror)
+  };

test/other/test_unoptimized_code_size_no_asserts.js.size

@@ -1 +1 @@
-31420
+31418

test/test_browser.py

@@ -699,15 +699,17 @@ def setup(assetLocalization):
           <center><canvas id='canvas' width='256' height='256'></canvas></center>
           <hr><div id='output'></div><hr>
           <script type='text/javascript'>
-            window.onerror = (error) => {
+            const handler = event => {
+              const error = String(event.reason);
               window.disableErrorReporting = true;
               window.onerror = null;
               var result = error.includes("test.data") ? 1 : 0;
               var xhr = new XMLHttpRequest();
               xhr.open('GET', 'http://localhost:8888/report_result?' + result, true);
               xhr.send();
               setTimeout(function() { window.close() }, 1000);
-            }
+            };
+            window.addEventListener('unhandledrejection', handler);
             var Module = {
               locateFile: function (path, prefix) {if (path.endsWith(".wasm")) {return prefix + path;} else {return "''' + assetLocalization + r'''" + path;}},
               print: (function() {

tools/file_packager.py

@@ -359,7 +359,7 @@ def generate_object_file(data_files):
 
 def main():
   if len(sys.argv) == 1:
-    err('''Usage: file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins]
+    err('''Usage: file_packager TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--indexedDB-name=EM_PRELOAD_CACHE] [--separate-metadata] [--lz4] [--use-preload-plugins] [--no-node]
   See the source for more details.''')
     return 1
 
@@ -775,7 +775,7 @@ def generate_js(data_target, data_files, metadata):
       }
       var REMOTE_PACKAGE_NAME = Module['locateFile'] ? Module['locateFile'](REMOTE_PACKAGE_BASE, '') : REMOTE_PACKAGE_BASE;\n''' % (js_manipulation.escape_for_js_string(data_target), js_manipulation.escape_for_js_string(remote_package_name))
     metadata['remote_package_size'] = remote_package_size
-    ret += '''var REMOTE_PACKAGE_SIZE = metadata['remote_package_size'];\n'''
+    ret += '''      var REMOTE_PACKAGE_SIZE = metadata['remote_package_size'];\n'''
 
     if options.use_preload_cache:
       # Set the id to a hash of the preloaded data, so that caches survive over multiple builds
@@ -965,59 +965,68 @@ def generate_js(data_target, data_files, metadata):
           });
           return;
         }'''.strip()
+
     ret += '''
       function fetchRemotePackage(packageName, packageSize, callback, errback) {
         %(node_support_code)s
-        var xhr = new XMLHttpRequest();
-        xhr.open('GET', packageName, true);
-        xhr.responseType = 'arraybuffer';
-        xhr.onprogress = function(event) {
-          var url = packageName;
-          var size = packageSize;
-          if (event.total) size = event.total;
-          if (event.loaded) {
-            if (!xhr.addedTotal) {
-              xhr.addedTotal = true;
-              if (!Module.dataFileDownloads) Module.dataFileDownloads = {};
-              Module.dataFileDownloads[url] = {
-                loaded: event.loaded,
-                total: size
-              };
-            } else {
-              Module.dataFileDownloads[url].loaded = event.loaded;
+        Module.dataFileDownloads = Module.dataFileDownloads || {};
+        const url = packageName;
+        fetch(url)
+        .catch(error => { throw new Error(error + ' : ' + url) }) // Do this first so we don't catch errors from then()
+        .then(response => {
+
+          let loaded = 0;
+
+          const reader = response.body.getReader();
+          const headers = response.headers;
+
+          if (!response.ok) {
+            throw new Error(response.statusText + ' : ' + response.url);
+          }
+
+          const total = headers.get('Content-Length') ?? packageSize;
+          const chunks = [];
+
+          const iterate = () => reader.read().then(handleChunk).catch(cause => {
+            throw new Error(response.statusText + ' : ' + response.url, {cause});
+          });
+
+          const handleChunk = ({done, value}) => {
+            if (!done) {
+              chunks.push(value);
+              loaded += value.length;
+              Module.dataFileDownloads[url] = Module.dataFileDownloads[url] ?? {};
+              Module.dataFileDownloads[url].loaded = loaded;
+              Module.dataFileDownloads[url].total = total;
+
+              if (total) {
+                if (Module['setStatus']) Module['setStatus'](`Downloading data... (${loaded}/${total})`);
+              }
+              else {
+                if (Module['setStatus']) Module['setStatus']('Downloading data...');
+              }
+              return iterate();
             }
-            var total = 0;
-            var loaded = 0;
-            var num = 0;
-            for (var download in Module.dataFileDownloads) {
-            var data = Module.dataFileDownloads[download];
-              total += data.total;
-              loaded += data.loaded;
-              num++;
+            else {
+              const size = chunks.map(c => c.length).reduce((a, b) => a + b, 0);
+              let index = 0;
+              const packageData = new Uint8Array(size);
+              for(const chunk of chunks) {
+                packageData.set(chunk, index);
+                index += chunk.length;
+              }
+
+              callback(packageData.buffer);
             }
-            total = Math.ceil(total * Module.expectedDataFileDownloads/num);
-            if (Module['setStatus']) Module['setStatus'](`Downloading data... (${loaded}/${total})`);
-          } else if (!Module.dataFileDownloads) {
-            if (Module['setStatus']) Module['setStatus']('Downloading data...');
-          }
-        };
-        xhr.onerror = function(event) {
-          throw new Error("NetworkError for: " + packageName);
-        }
-        xhr.onload = function(event) {
-          if (xhr.status == 200 || xhr.status == 304 || xhr.status == 206 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0
-            var packageData = xhr.response;
-            callback(packageData);
-          } else {
-            throw new Error(xhr.statusText + " : " + xhr.responseURL);
-          }
-        };
-        xhr.send(null);
-      };
+          };
+          return iterate();
+        });
+      };\n''' % {'node_support_code': node_support_code}
 
+    ret += '''
       function handleError(error) {
         console.error('package error:', error);
-      };\n''' % {'node_support_code': node_support_code}
+      };\n'''
 
     code += '''
       function processPackageData(arrayBuffer) {
@@ -1113,15 +1122,14 @@ def generate_js(data_target, data_files, metadata):
   function runMetaWithFS() {
     Module['addRunDependency']('%(metadata_file)s');
     var REMOTE_METADATA_NAME = Module['locateFile'] ? Module['locateFile']('%(metadata_file)s', '') : '%(metadata_file)s';
-    var xhr = new XMLHttpRequest();
-    xhr.onreadystatechange = function() {
-     if (xhr.readyState === 4 && xhr.status === 200) {
-       loadPackage(JSON.parse(xhr.responseText));
-     }
-    }
-    xhr.open('GET', REMOTE_METADATA_NAME, true);
-    xhr.overrideMimeType('application/json');
-    xhr.send(null);
+    fetch(REMOTE_METADATA_NAME)
+    .then(response => {
+      if(response.ok) {
+        return response.json();
+      }
+      throw new Error(response.statusText + ' : ' + response.url);
+    })
+    .then(loadPackage);
   }
 
   if (Module['calledRun']) {
@@ -1130,7 +1138,6 @@ def generate_js(data_target, data_files, metadata):
     if (!Module['preRun']) Module['preRun'] = [];
     Module["preRun"].push(runMetaWithFS);
   }\n''' % {'metadata_file': os.path.basename(options.jsoutput + '.metadata')}
-
   else:
       _metadata_template = '''
     }