Fix #2160 - Add m/py:progress events

esm/interpreter/_python.js

@@ -1,12 +1,31 @@
 import * as JSON from '@ungap/structured-clone/json';
 import IDBMapSync from '@webreflection/idb-map/sync';
+import { fetchFiles, fetchJSModules, fetchPaths } from './_utils.js';
 import { dedent } from '../utils.js';
 import { io } from './_io.js';
 
 export const loader = new WeakMap();
 
 // REQUIRES INTEGRATION TEST
 /* c8 ignore start */
+export const loadProgress = async (self, progress, interpreter, config, baseURL) => {
+    if (config.files) {
+        progress('Loading files');
+        await fetchFiles(self, interpreter, config.files, baseURL);
+        progress('Loaded files');
+    }
+    if (config.fetch) {
+        progress('Loading fetch');
+        await fetchPaths(self, interpreter, config.fetch, baseURL);
+        progress('Loaded fetch');
+    }
+    if (config.js_modules) {
+        progress('Loading JS modules');
+        await fetchJSModules(config.js_modules, baseURL);
+        progress('Loaded JS modules');
+    }
+};
+
 export const registerJSModule = (interpreter, name, value) => {
     if (name === 'polyscript') {
         value.lazy_py_modules = async (...packages) => {

esm/interpreter/_utils.js

@@ -167,4 +167,8 @@ export const fetchJSModules = ({ main, worker }, baseURL) => {
     }
     return all(promises);
 };
+
+export const createProgress = prefix => detail => {
+    dispatchEvent(new CustomEvent(`${prefix}:progress`, { detail }));
+};
 /* c8 ignore stop */

esm/interpreter/micropython.js

@@ -1,7 +1,7 @@
 import fetch from '@webreflection/fetch';
 
-import { fetchFiles, fetchJSModules, fetchPaths, writeFile } from './_utils.js';
-import { getFormat, loader, registerJSModule, run, runAsync, runEvent } from './_python.js';
+import { createProgress, writeFile } from './_utils.js';
+import { getFormat, loader, loadProgress, registerJSModule, run, runAsync, runEvent } from './_python.js';
 import { stdio, buffered } from './_io.js';
 import { absoluteURL } from '../utils.js';
 import mip from '../python/mip.js';
@@ -21,6 +21,8 @@ const mkdir = (FS, path) => {
     }
 };
 
+const progress = createProgress('mpy');
+
 export default {
     type,
     module: (version = '1.24.0-preview-114') =>
@@ -31,16 +33,19 @@ export default {
             stdout: buffered(console.log),
         });
         url = url.replace(/\.m?js$/, '.wasm');
+        progress('Loading MicroPython');
         const interpreter = await get(loadMicroPython({ linebuffer: false, stderr, stdout, url }));
         const py_imports = importPackages.bind(this, interpreter, baseURL);
         loader.set(interpreter, py_imports);
-        if (config.files) await fetchFiles(this, interpreter, config.files, baseURL);
-        if (config.fetch) await fetchPaths(this, interpreter, config.fetch, baseURL);
-        if (config.js_modules) await fetchJSModules(config.js_modules, baseURL);
-
+        await loadProgress(this, progress, interpreter, config, baseURL);
         // Install Micropython Package
         this.writeFile(interpreter, './mip.py', mip);
-        if (config.packages) await py_imports(config.packages);
+        if (config.packages) {
+            progress('Loading packages');
+            await py_imports(config.packages);
+            progress('Loaded packages');
+        }
+        progress('Loaded MicroPython');
         return interpreter;
     },
     registerJSModule,

esm/interpreter/pyodide.js

@@ -1,7 +1,7 @@
 import { create } from 'gc-hook';
 
-import { RUNNING_IN_WORKER, fetchFiles, fetchJSModules, fetchPaths, writeFile } from './_utils.js';
-import { getFormat, loader, registerJSModule, run, runAsync, runEvent } from './_python.js';
+import { RUNNING_IN_WORKER, createProgress, writeFile } from './_utils.js';
+import { getFormat, loader, loadProgress, registerJSModule, run, runAsync, runEvent } from './_python.js';
 import { stdio } from './_io.js';
 import { isArray } from '../utils.js';
 
@@ -73,6 +73,8 @@ const applyOverride = () => {
         }
     });
 };
+
+const progress = createProgress('py');
 /* c8 ignore stop */
 
 // REQUIRES INTEGRATION TEST
@@ -85,17 +87,17 @@ export default {
         // apply override ASAP then load foreign code
         if (!RUNNING_IN_WORKER && config.experimental_create_proxy === 'auto')
             applyOverride();
+        progress('Loading Pyodide');
         const { stderr, stdout, get } = stdio();
         const indexURL = url.slice(0, url.lastIndexOf('/'));
         const interpreter = await get(
             loadPyodide({ stderr, stdout, indexURL }),
         );
         const py_imports = importPackages.bind(interpreter);
         loader.set(interpreter, py_imports);
-        if (config.files) await fetchFiles(this, interpreter, config.files, baseURL);
-        if (config.fetch) await fetchPaths(this, interpreter, config.fetch, baseURL);
-        if (config.js_modules) await fetchJSModules(config.js_modules, baseURL);
+        await loadProgress(this, progress, interpreter, config, baseURL);
         if (config.packages) await py_imports(config.packages);
+        progress('Loaded Pyodide');
         return interpreter;
     },
     registerJSModule,
@@ -129,9 +131,21 @@ function transform(value) {
 
 // exposed utility to import packages via polyscript.lazy_py_modules
 async function importPackages(packages) {
+    // temporary patch/fix console.log which is used
+    // not only by Pyodide but by micropip too and there's
+    // no way to intercept those calls otherwise
+    const { log } = console;
+    const _log = (detail, ...rest) => {
+        log(detail, ...rest);
+        console.log = log;
+        progress(detail);
+        console.log = _log;
+    };
+    console.log = _log;
     await this.loadPackage('micropip');
     const micropip = this.pyimport('micropip');
     await micropip.install(packages, { keep_going: true });
+    console.log = log;
     micropip.destroy();
 }
 /* c8 ignore stop */

esm/interpreter/webr.js

@@ -21,7 +21,7 @@ const run = async (interpreter, code) => {
 export default {
     type,
     experimental: true,
-    module: (version = '0.4.1') =>
+    module: (version = '0.4.2') =>
         `https://cdn.jsdelivr.net/npm/webr@${version}/dist/webr.mjs`,
     async engine(module, config, _, baseURL) {
         const { get } = stdio();

esm/worker/_template.js

@@ -211,4 +211,9 @@ add('message', ({ data: { options, config: baseURL, configURL, code, hooks } })
     add('error');
     add('message');
     add('messageerror');
+    if (syncMainAndWorker) {
+        addEventListener('py:progress', ({ type, detail }) => {
+            window.dispatchEvent(new window.CustomEvent(type, { detail }));
+        });
+    }
 });

package.json

@@ -86,6 +86,6 @@
         "to-json-callback": "^0.1.1"
     },
     "worker": {
-        "blob": "sha256-n5bp7XbiMOC5xSM8r1BjWIl0efV5DHH41AoMDzRWNdg="
+        "blob": "sha256-HMkVAasELYptpWip1pbmxUKmFvQEwSa+FAMTOAnuhoY="
     }
 }

test/index.html

@@ -5,6 +5,7 @@
         <meta name="viewport" content="width=device-width,initial-scale=1.0" />
         <title>python</title>
         <link rel="stylesheet" href="style.css" />
+        <script>addEventListener('py:progress', ({ detail }) => console.log({ detail }));</script>
         <script type="module" src="../dist/index.js"></script>
     </head>
     <body>

test/index.js

@@ -36,6 +36,7 @@ globalThis.indexedDB = { open: () => ({}) };
 globalThis.document = document;
 globalThis.Element = window.Element;
 globalThis.CustomEvent = CustomEvent;
+globalThis.dispatchEvent = Object;
 globalThis.MutationObserver = window.MutationObserver;
 globalThis.Worker = class {};
 globalThis.XPathResult = {};

test/matplot.html

@@ -5,6 +5,7 @@
         <meta name="viewport" content="width=device-width,initial-scale=1.0" />
         <title>python</title>
         <link rel="stylesheet" href="style.css" />
+        <script>addEventListener('py:progress', ({ detail }) => console.log({ detail }));</script>
         <script defer src="./counter.js"></script>
         <script type="module" src="../dist/index.js"></script>
     </head>

test/matplot.json.html

@@ -5,6 +5,7 @@
         <meta name="viewport" content="width=device-width,initial-scale=1.0" />
         <title>python</title>
         <link rel="stylesheet" href="style.css" />
+        <script>addEventListener('py:progress', ({ detail }) => console.log({ detail }));</script>
         <script defer src="./counter.js"></script>
         <script type="module" src="../dist/index.js"></script>
     </head>

test/matplot.worker.html

@@ -5,6 +5,7 @@
         <meta name="viewport" content="width=device-width,initial-scale=1.0" />
         <title>python</title>
         <link rel="stylesheet" href="style.css" />
+        <script>addEventListener('py:progress', ({ detail }) => console.log({ detail }));</script>
         <script defer src="./counter.js"></script>
         <script type="module" src="../dist/index.js"></script>
     </head>

test/micropython.html

@@ -5,6 +5,7 @@
         <meta name="viewport" content="width=device-width,initial-scale=1.0" />
         <title>python</title>
         <link rel="stylesheet" href="style.css" />
+        <script>addEventListener('mpy:progress', ({ detail }) => console.log({ detail }));</script>
         <script defer src="./counter.js"></script>
         <script type="module" src="../dist/index.js"></script>
     </head>

test/micropython/random.html

@@ -3,6 +3,7 @@
 <head>
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <script>addEventListener('mpy:progress', ({ detail }) => console.log({ detail }));</script>
   <script type="module" src="../../dist/index.js"></script>
   <script type="micropython">
     import random, js

versions/webr

@@ -1 +1 @@
-0.4.1
+0.4.2