[breaking-change] Calculate download progress correctly.

It turns out that `ReadableStream` is not (by itself) a way to compute
download progress: the `Content-Length` header is post-Zstd, causing
download progress to go above 100%, and our files are particularly in
need of being compressed, given the high ratios achievable.

This commit changes the interface to export total artifact size from
the resource module. Instead of `() => import('app-resources.js')`,
the import should now be used as simply `import('app-resources.js')`,
since the resource file itself is very small.

Author: whitequark

bin/pack-resources.js

@@ -3,88 +3,106 @@
 import { readdir, readFile, writeFile, stat } from 'fs/promises';
 import { createTar } from 'nanotar';
 
-async function packModules(root, urlRoot) {
-    const files =  await readdir(root, { withFileTypes: true });
-    const packedData = [`{\n`];
-    for (const file of files) {
-        if (file.isFile() && file.name.endsWith('.wasm')) {
-            packedData.push(`    ${JSON.stringify(file.name)}: `);
-            packedData.push(`new URL(${JSON.stringify(urlRoot + file.name)}, import.meta.url)`);
-            packedData.push(`,\n`);
-        }
-    }
-    packedData.push(`}`);
-    return packedData;
-}
-
-async function collectDirectory(root, dirPath = '', packedData = []) {
-    const files = await readdir(`${root}/${dirPath}`, { withFileTypes: true });
-    for (const file of files) {
-        const filePath = dirPath === '' ? file.name : `${dirPath}/${file.name}`;
-        const fileStats = await stat(`${root}/${filePath}`);
-        if (fileStats.isDirectory()) {
-            packedData.push({name: filePath});
-            await collectDirectory(root, filePath, packedData);
-        } else if (fileStats.isFile()) {
-            packedData.push({name: filePath, data: await readFile(`${root}/${filePath}`)});
-        } else {
-            console.error(`Unsupported '${filePath}'!`);
-            process.exit(2);
-        }
-    }
-    return packedData;
-}
-
 const args = process.argv.slice(2);
 if (!(args.length >= 2 && args.length <= 4)) {
-    console.error(`Usage: yowasp-pack-resources <resources.js> <gen-directory> [<share-directory>] [<share-root>]`);
+    console.error(`Usage: yowasp-pack-resources <resources.js> <wasm-directory> [<share-directory>] [<share-root>]`);
     process.exit(1);
 }
-
 const resourceFilePath = args[0];
-const genDirectory = args[1];
+const wasmDirectory = args[1];
 const shareDirectory = args[2];
 const shareRoot = args[3] || 'share';
 
+let totalSize = 0;
+
+async function fetchExpr(filePath, fileURL) {
+    const fileStats = await stat(filePath);
+    totalSize += fileStats.size;
+    return `fetch(new URL(${JSON.stringify(fileURL)}, import.meta.url))`;
+}
+
 let output = `\
 import { parseTar } from 'nanotar';
 
-function unpackResources(url) {
-    function defaultFetchFn(url) {
-        return fetch(url).then((resp) => resp.arrayBuffer());
+function compileWasmModule(response) {
+    if (WebAssembly.compileStreaming !== undefined) {
+        // Node.js does not have 'WebAssembly.{compile,instantiate}Streaming'.
+        return WebAssembly.compileStreaming(response);
+    } else {
+        return WebAssembly.compile(response.arrayBuffer());
     }
+}
 
-    return async (fetchFn = defaultFetchFn) => {
-        const root = {};
-        for (const tarEntry of parseTar(await fetchFn(url))) {
-            const nameParts = tarEntry.name.split('/');
-            const dirNames = nameParts.slice(0, -1);
-            const fileName = nameParts[nameParts.length - 1];
-            let dir = root;
-            for (const dirName of dirNames)
-                dir = dir[dirName];
-            if (tarEntry.type === 'directory') {
-                dir[fileName] = {};
-            } else {
-                dir[fileName] = tarEntry.data;
-            }
+function unpackTarFilesystem(buffer) {
+    const root = {};
+    for (const tarEntry of parseTar(buffer)) {
+        const nameParts = tarEntry.name.split('/');
+        const dirNames = nameParts.slice(0, -1);
+        const fileName = nameParts[nameParts.length - 1];
+        let dir = root;
+        for (const dirName of dirNames)
+            dir = dir[dirName];
+        if (tarEntry.type === 'directory') {
+            dir[fileName] = {};
+        } else {
+            dir[fileName] = tarEntry.data;
         }
-        return root;
-    };
+    }
+    return root;
 }
 
+const modules = async (fetch) => ({
+`;
+
+for (const dirent of await readdir(wasmDirectory, { withFileTypes: true })) {
+    if (dirent.isFile() && dirent.name.endsWith('.wasm')) {
+        const filePath = `${dirent.parentPath}/${dirent.name}`;
+        output += `\
+    ${JSON.stringify(dirent.name)}: await ${await fetchExpr(filePath, `./${dirent.name}`)}
+        .then(compileWasmModule),
 `;
-const moduleObject = (await packModules(genDirectory, './')).flat(Infinity).join('');
-output += `export const modules = ${moduleObject};\n\n`;
-if (shareDirectory) {
+    }
+}
+
+output += `\
+});
+
+const filesystem = async (fetch) => ({
+`;
+
+if (shareDirectory !== undefined) {
+    const tarEntries = [];
+    for (const dirent of await readdir(shareDirectory, { withFileTypes: true, recursive: true })) {
+        if (dirent.isDirectory()) {
+            tarEntries.push({name: dirent.name});
+        } else if (dirent.isFile()) {
+            const name = `${dirent.parentPath}/${dirent.name}`.replace(`${shareDirectory}/`, '');
+            const data = await readFile(`${dirent.parentPath}/${dirent.name}`);
+            tarEntries.push({name, data});
+        } else {
+            console.error(`Unsupported type of '${dirent.name}'!`);
+            process.exit(2);
+        }
+    }
+
+    const tarData = createTar(tarEntries);
     const tarFilePath = resourceFilePath.replace(/\.js$/, '.tar');
-    await writeFile(tarFilePath, createTar(await collectDirectory(shareDirectory)));
+    await writeFile(tarFilePath, tarData);
+
     const tarFileName = tarFilePath.replace(/^.+\//, '');
-    const resourceObject = `unpackResources(new URL('./${tarFileName}', import.meta.url))`;
-    output += `export const filesystem = {\n`;
-    output += `    ${shareRoot}: ${resourceObject},\n`;
-    output += `};\n`;
-} else {
-    output += `export const filesystem = {};\n`;
+    output += `\
+    ${JSON.stringify(shareRoot)}: await ${await fetchExpr(tarFilePath, `./${tarFileName}`)}
+        .then((resp) => resp.arrayBuffer())
+        .then(unpackTarFilesystem)
+`;
 }
+
+output += `\
+});
+
+const totalSize = ${totalSize};
+
+export { modules, filesystem, totalSize };
+`;
+
 await writeFile(resourceFilePath, output);

lib/api.js

@@ -4,91 +4,14 @@ import { lineBuffered } from './util.js';
 
 export { Exit } from './wasi-virt.js';
 
-async function fetchObject(obj, fetchFn) {
-    // Mutate the object being fetched, to avoid re-fetches within the same session.
-    // Do this in parallel to avoid head-of-line blocking.
-    const promises = [];
-    for (const [key, value] of Object.entries(obj)) {
-        if (typeof value === "string" || value instanceof Uint8Array) {
-            promises.push(Promise.resolve([key, value]));
-        } else if (value instanceof URL) {
-            promises.push(fetchFn(value).then((fetched) => [key, fetched]));
-        } else if (value instanceof Function) {
-            promises.push(await value(fetchFn).then((fetched) => [key, fetched]));
-        } else {
-            promises.push(fetchObject(value, fetchFn).then((fetched) => [key, fetched]));
-        }
-    }
-    for (const [key, value] of await Promise.all(promises))
-        obj[key] = value;
-    return obj;
-}
-
-function fetchWebAssembly(url, monitorProgress) {
-    const response = fetch(url).then(monitorProgress);
-    if (WebAssembly.compileStreaming !== undefined) {
-        return response.then(WebAssembly.compileStreaming);
-    } else {
-        // Node doesn't have `{compile,instantiate}Streaming`.
-        return response.then((resp) => resp.arrayBuffer()).then(WebAssembly.compile);
-    }
-}
-
-function fetchUint8Array(url, monitorProgress) {
-    const response = fetch(url).then(monitorProgress);
-    return response.then((resp) => resp.arrayBuffer()).then((buf) => new Uint8Array(buf));
-}
-
-function fetchResources({ modules, filesystem }, fetchProgress, application) {
-    /** @type {(response: Response) => Response} */
-    let monitorProgress = (response) => response;
-    if (fetchProgress !== undefined) {
-        const progress = new Map();
-        const notifyProgress = () => {
-            let cumTotalLength = 0, cumDoneLength = 0;
-            for (const { totalLength, doneLength } of progress.values()) {
-                cumTotalLength += totalLength;
-                cumDoneLength += doneLength;
-            }
-            fetchProgress({
-                source: application,
-                totalLength: cumTotalLength,
-                doneLength: cumDoneLength
-            });
-        };
-        monitorProgress = (response) => {
-            let totalLength = +response.headers.get('content-length');
-            let doneLength = 0;
-            progress.set(response, { totalLength, doneLength });
-            notifyProgress();
-            /** @type {TransformStream<Uint8Array, Uint8Array>} */
-            const monitorStream = new TransformStream({
-                transform(chunk, controller) {
-                    controller.enqueue(chunk);
-                    doneLength += chunk.length;
-                    progress.set(response, { totalLength, doneLength });
-                    notifyProgress();
-                }
-            });
-            return new Response(response.body.pipeThrough(monitorStream), response);
-        };
-    }
-    return Promise.all([
-        fetchObject(modules, (url) => fetchWebAssembly(url, monitorProgress)),
-        fetchObject(filesystem, (url) => fetchUint8Array(url, monitorProgress))
-    ]).then(([modules, filesystem]) => {
-        return { modules, filesystem };
-    });
-}
-
 export class Application {
-    #resources;
+    #resourceModule;
     #resourceData;
     #instantiate;
     #argv0;
 
-    constructor(resources, instantiate, argv0) {
-        this.#resources = resources;
+    constructor(resourceModule, instantiate, argv0) {
+        this.#resourceModule = resourceModule;
         this.#resourceData = null;
         this.#instantiate = instantiate;
         this.#argv0 = argv0;
@@ -98,6 +21,29 @@ export class Application {
         return this.#argv0;
     }
 
+    async #fetchResources(fetchProgress) {
+        const resourceModule = await this.#resourceModule;
+        let fetchFn = fetch;
+        if (fetchProgress !== undefined) {
+            const status = { source: this, totalLength: resourceModule.totalSize, doneLength: 0 };
+            fetchProgress(status);
+            fetchFn = (input, init) => fetch(input, init).then((response) => {
+                return new Response(response.body.pipeThrough(new TransformStream({
+                    transform(chunk, controller) {
+                        controller.enqueue(chunk);
+                        status.doneLength += chunk.length;
+                        fetchProgress(status);
+                    }
+                })), response);
+            });
+        }
+        const [modules, filesystem] = await Promise.all([
+            resourceModule.modules(fetchFn),
+            resourceModule.filesystem(fetchFn),
+        ]);
+        this.#resourceData = { modules, filesystem };
+    }
+
     // The `printLine` option is deprecated and not documented but still accepted for compatibility.
     run(args = null, files = {}, options = {}) {
         if (this.#resourceData === null) {
@@ -110,14 +56,9 @@ export class Application {
                 const percent = (100 * doneLength / totalLength).toFixed(0);
                 console.log(`${source.argv0}: fetched ${percent}% (${doneLength} / ${totalLength})`);
             };
-            const fetchProgress = options.fetchProgress ?? defaultFetchProgress;
-            return this.#resources()
-                .then((resourceObject) =>
-                    fetchResources(resourceObject, fetchProgress, this))
-                .then((resourceData) => {
-                    this.#resourceData = resourceData;
-                    return this.run(args, files, options);
-                });
+            return this.#fetchResources(options.fetchProgress ?? defaultFetchProgress).then(() => {
+                return this.run(args, files, options);
+            });
         }
 
         if (args === null)

package-in.json

@@ -1,6 +1,6 @@
 {
     "name": "@yowasp/runtime",
-    "version": "10.0",
+    "version": "11.0",
     "description": "Common runtime for YoWASP packages",
     "author": "Catherine <[email protected]>",
     "license": "ISC",

test/yowasp_runtime_test/index.js

@@ -1,13 +1,15 @@
 import { Application, Exit } from '@yowasp/runtime';
+import * as resources from './gen/resources.js';
 import { lineBuffered, chunked } from '@yowasp/runtime/util';
 import { instantiate } from './gen/copy.js';
 
-
-const yowaspRuntimeTest = new Application(() => import('./gen/resources.js'), instantiate, 'copy');
+const yowaspRuntimeTest = new Application(resources, instantiate, 'copy');
 
 
 if ((await yowaspRuntimeTest.run(['share/foo.txt', 'bar.txt'], {}))['bar.txt'] !== 'contents of foo')
-    throw 'test 1 failed';
+    throw 'test 1 failed (1)';
+if ((await yowaspRuntimeTest.run(['share/bar/baz.txt', 'bar.txt'], {}))['bar.txt'] !== 'meow\n')
+    throw 'test 1 failed (2)';
 
 if ((await yowaspRuntimeTest.run(['baz.txt', 'bar.txt'], {'baz.txt': 'contents of baz'}))['bar.txt'] !== 'contents of baz')
     throw 'test 2 failed';

test/yowasp_runtime_test/share/bar/baz.txt

@@ -0,0 +1 @@
+meow