Add a way to monitor download progress.

Author: whitequark

README.md

@@ -13,7 +13,7 @@ Application API reference
 All of the other JavaScript YoWASP packages use the common runtime functionality implemented here. They export the function `runX` where `X` is the name of the application, which can be called as:
 
 ```js
-const filesOut = await runX(args, filesIn, { stdout, stderr, decodeASCII: true });
+const filesOut = await runX(args, filesIn, { stdout, stderr, decodeASCII: true, fetchProgress });
 ```
 
 Arguments and return value:
@@ -24,6 +24,7 @@ Arguments and return value:
 Options:
 - The `stdout` and `stderr` options are functions that are called with a sequence of bytes the application prints to the standard output and standard error streams respectively, or `null` to indicate that the stream is being flushed. If specified as `null`, the output on that stream is ignored. By default, each line of text from the combined streams is printed to the debugging console.
 - The `decodeASCII` option determines whether the values corresponding to files in `filesOut` are always instances of [Uint8Array][] (if `decodeASCII: false`), or whether the values corresponding to text files will be strings (if `decodeASCII: true`). A file is considered a text file if it contains only bytes `0x09`, `0x0a`, `0x0d`, or those in the range `0x20` to `0x7e` inclusive. The default is `decodeASCII: true`.
+- The `fetchProgress({ source, totalLength, doneLength })` option provides a callback to monitor download progress. The `source` is the `Application` object whose assets are being downloaded; `totalLength` and `doneLength` are in bytes. Note that if the server does not send the `Content-Length` header for *any* of the assets, `totalLength` will be `NaN`. By default, download progress is printed to the debugging console.
 
 If the application returns a non-zero exit code, the exception `Exit` (exported alongside the `runX` function) is raised. This exception has two properties:
 - The `code` property indicates the exit code. (Currently this is always 1 due to WebAssembly peculiarities.)

lib/api.d.ts

@@ -8,15 +8,21 @@ export type InputStream =
 export type OutputStream =
     (bytes: Uint8Array | null) => void;
 
+export type ProgressCallback =
+    ({ source: Application, totalLength: number, doneLength: number }) => void;
+
 export type RunOptions = {
     stdin?:  InputStream  | null;
     stdout?: OutputStream | null;
     stderr?: OutputStream | null;
     decodeASCII?: boolean;
     synchronously?: boolean;
+    fetchProgress?: ProgressCallback;
 };
 
 export class Application {
+    argv0: string;
+
     constructor(resources: () => Promise<any>, instantiate: any, argv0: string);
 
     run(args?: string[], files?: Tree, options?: RunOptions): Promise<Tree> | Tree | undefined;

lib/api.js

@@ -24,23 +24,58 @@ async function fetchObject(obj, fetchFn) {
     return obj;
 }
 
-function fetchWebAssembly(url) {
+function fetchWebAssembly(url, monitorProgress) {
+    const response = fetch(url).then(monitorProgress);
     if (WebAssembly.compileStreaming !== undefined) {
-        return fetch(url).then(WebAssembly.compileStreaming);
+        return response.then(WebAssembly.compileStreaming);
     } else {
         // Node doesn't have `{compile,instantiate}Streaming`.
-        return fetch(url).then((resp) => resp.arrayBuffer()).then(WebAssembly.compile);
+        return response.then((resp) => resp.arrayBuffer()).then(WebAssembly.compile);
     }
 }
 
-function fetchUint8Array(url) {
-    return fetch(url).then((resp) => resp.arrayBuffer()).then((buf) => new Uint8Array(buf));
+function fetchUint8Array(url, monitorProgress) {
+    const response = fetch(url).then(monitorProgress);
+    return response.then((resp) => resp.arrayBuffer()).then((buf) => new Uint8Array(buf));
 }
 
-function fetchResources({ modules, filesystem }) {
+function fetchResources({ modules, filesystem }, fetchProgress, application) {
+    /** @type {(response: Response) => Response} */
+    let monitorProgress = (response) => response;
+    if (fetchProgress !== undefined) {
+        const progress = new Map();
+        const notifyProgress = () => {
+            let cumTotalLength = 0, cumDoneLength = 0;
+            for (const { totalLength, doneLength } of progress.values()) {
+                cumTotalLength += totalLength;
+                cumDoneLength += doneLength;
+            }
+            fetchProgress({
+                source: application,
+                totalLength: cumTotalLength,
+                doneLength: cumDoneLength
+            });
+        };
+        monitorProgress = (response) => {
+            let totalLength = +response.headers.get('content-length');
+            let doneLength = 0;
+            progress.set(response, { totalLength, doneLength });
+            notifyProgress();
+            /** @type {TransformStream<Uint8Array, Uint8Array>} */
+            const monitorStream = new TransformStream({
+                transform(chunk, controller) {
+                    controller.enqueue(chunk);
+                    doneLength += chunk.length;
+                    progress.set(response, { totalLength, doneLength });
+                    notifyProgress();
+                }
+            });
+            return new Response(response.body.pipeThrough(monitorStream), response);
+        };
+    }
     return Promise.all([
-        fetchObject(modules, fetchWebAssembly),
-        fetchObject(filesystem, fetchUint8Array)
+        fetchObject(modules, (url) => fetchWebAssembly(url, monitorProgress)),
+        fetchObject(filesystem, (url) => fetchUint8Array(url, monitorProgress))
     ]).then(([modules, filesystem]) => {
         return { modules, filesystem };
     });
@@ -59,15 +94,26 @@ export class Application {
         this.#argv0 = argv0;
     }
 
+    get argv0() {
+        return this.#argv0;
+    }
+
     // The `printLine` option is deprecated and not documented but still accepted for compatibility.
     run(args = null, files = {}, options = {}) {
         if (this.#resourceData === null) {
             if (options.synchronously)
                 throw new Error("Cannot run application synchronously unless resources are " +
                                 "prefetched first; use `await run()` to do so");
 
+            /** @type {ProgressCallback} */
+            const defaultFetchProgress = ({ source, totalLength, doneLength }) => {
+                const percent = (100 * doneLength / totalLength).toFixed(0);
+                console.log(`${source.argv0}: fetched ${percent}% (${doneLength} / ${totalLength})`);
+            };
+            const fetchProgress = options.fetchProgress ?? defaultFetchProgress;
             return this.#resources()
-                .then((resourceObject) => fetchResources(resourceObject))
+                .then((resourceObject) =>
+                    fetchResources(resourceObject, fetchProgress, this))
                 .then((resourceData) => {
                     this.#resourceData = resourceData;
                     return this.run(args, files, options);

lib/fetch.js

@@ -4,10 +4,13 @@ if (typeof process === 'object' && process.release?.name === 'node') {
     fetch = async function(url, options) {
         if (url.protocol === 'file:') {
             const { readFile } = await import('node:fs/promises');
-            let contentType = 'application/octet-stream';
-            if (url.pathname.endsWith('.wasm'))
-                contentType = 'application/wasm';
-            return new Response(await readFile(url), { headers: { 'Content-Type': contentType } });
+            const data = await readFile(url);
+            const isWasm = url.pathname.endsWith('.wasm');
+            const headers = {
+                'content-length': data.length,
+                'content-type': (isWasm ? 'application/wasm' : 'application/octet-stream'),
+            };
+            return new Response(data, { headers });
         } else {
             return globalThis.fetch(url, options);
         }