Merge pull request #69 from pyscript/js_modules

Added a way to register JS modules directly

Author: WebReflection

docs/README.md

@@ -254,6 +254,59 @@ Please read the [XWorker](#xworker) dedicated section to know more.
 </details>
 
 
+## Extra `config` features
+
+It is possible to land in either the *main* world or the *worker* one native *JS* modules (aka: *ESM*).
+
+In *polyscript*, this is possible by defining one or more `[js_modules.X]` fields in the config, where `X` is either *main* or *worker*:
+
+  * `[js_modules.main]` is a list of *source* -> *module name* pairs, similarly to how `[files]` field work, where the *module* name will then be reachable via `polyscript.js_modules.actual_name` in both *main* and *worker* world. As the *main* module lands on the main thread, where there is also likely some UI, it is also possible to define one or more related *CSS* to that module, as long as they target the very same name (see the example to better understand).
+  * `[js_modules.worker]` is a list of *source* -> *module name* pairs that actually land only in `<script type="x" worker>` cases. These modules are still reachable through the very same `polyscript.js_modules.actual_name` convention and this feature is meant to be used for modules that only works best, or work regardless, outside the *main* world. As example, if your *JS* module implies that `document` or `window` references, among other *DOM* related APIs, are globally available, it means that that module should be part of the `[js_modules.main]` list instead ... however, if the module works out of the box in a *Worker* environment, it is best for performance reasons to explicitly define such module under this field. Please note that *CSS* files are not accepted within this list because there's no way *CSS* can be useful or land in any meaningful way within a *Worker* environment.
+
+### js_modules config example
+
+```toml
+[js_modules.main]
+# this modules work best on main
+"https://cdn.jsdelivr.net/npm/[email protected]/dist/leaflet-src.esm.js" = "leaflet"
+"https://cdn.jsdelivr.net/npm/[email protected]/dist/leaflet.css" = "leaflet" # CSS
+# this works in both main and worker
+"https://cdn.jsdelivr.net/npm/html-escaper" = "html_escaper"
+
+[js_modules.worker]
+# this works out of the box in a worker too
+"https://cdn.jsdelivr.net/npm/html-escaper" = "html_escaper"
+# this works only in a worker
+"https://cdn.jsdelivr.net/npm/worker-only" = "worker_only"
+```
+
+```html
+<!-- main case -->
+<script type="pyodide" config="./that.toml">
+  # these both works
+  from polyscript.js_modules import leaflet as L
+  from polyscript.js_modules import html_escaper
+
+  # this fails as it's not reachable in main
+  from polyscript.js_modules import worker_only
+</script>
+
+<!-- worker case -->
+<script type="pyodide" config="./that.toml" worker>
+  # these works by proxying the main module and landing
+  # on main only when accessed, never before
+  # the CSS file also lands automatically on demand
+  from polyscript.js_modules import leaflet as L
+
+  # this works out of the box in the worker
+  from polyscript.js_modules import html_escaper
+
+  # this works only in a worker 👍
+  from polyscript.js_modules import worker_only
+</script>
+```
+
+
 ## How Events Work
 
 The event should contain the *interpreter* or *custom type* prefix, followed by the *event* type it'd like to handle.

docs/core.js

@@ -1,7 +1,7 @@
 import '@ungap/with-resolvers';
 import { $$ } from 'basic-devtools';
 
-import { assign, create, createOverload, createResolved, dedent, defineProperty, nodeInfo } from './utils.js';
+import { JSModules, assign, create, createOverload, createResolved, dedent, defineProperty, nodeInfo } from './utils.js';
 import { getDetails } from './script-handler.js';
 import { registry as defaultRegistry, prefixes, configs } from './interpreters.js';
 import { getRuntimeID } from './loader.js';
@@ -99,7 +99,10 @@ export const handleCustomType = (node) => {
                         XWorker,
                     };
 
-                    module.registerJSModule(interpreter, 'polyscript', { XWorker });
+                    module.registerJSModule(interpreter, 'polyscript', {
+                        js_modules: JSModules,
+                        XWorker,
+                    });
 
                     // patch methods accordingly to hooks (and only if needed)
                     for (const suffix of ['Run', 'RunAsync']) {

docs/core.js.map

@@ -1,7 +1,8 @@
 import '@ungap/with-resolvers';
+import { UNDEFINED } from 'proxy-target/types';
 
 import { getBuffer } from '../fetch-utils.js';
-import { absoluteURL } from '../utils.js';
+import { absoluteURL, all, entries, importCSS, importJS, isArray, isCSS } from '../utils.js';
 
 // REQUIRES INTEGRATION TEST
 /* c8 ignore start */
@@ -73,8 +74,6 @@ const resolve = (FS, path) => {
     return [FS.cwd()].concat(tree).join('/').replace(/^\/+/, '/');
 };
 
-import { all, isArray } from '../utils.js';
-
 const calculateFetchPaths = (config_fetch) => {
     // REQUIRES INTEGRATION TEST
     /* c8 ignore start */
@@ -142,7 +141,7 @@ const calculateFilesPaths = files => {
   const map = new Map;
   const targets = new Set;
   const sourceDest = [];
-  for (const [source, dest] of Object.entries(files)) {
+  for (const [source, dest] of entries(files)) {
     if (/^\{.+\}$/.test(source)) {
       if (map.has(source))
         throw new SyntaxError(`Duplicated template: ${source}`);
@@ -168,4 +167,21 @@ export const fetchFiles = (module, interpreter, config_files) =>
                 .then((buffer) => module.writeFile(interpreter, path, buffer)),
         ),
     );
+
+const RUNNING_IN_WORKER = typeof document === UNDEFINED;
+
+export const fetchJSModules = ({ main, worker }) => {
+    const promises = [];
+    if (worker && RUNNING_IN_WORKER) {
+        for (const [source, name] of entries(worker))
+            promises.push(importJS(source, name));
+    }
+    if (main && !RUNNING_IN_WORKER) {
+        for (const [source, name] of entries(main)) {
+            if (isCSS(source)) importCSS(source);
+            else promises.push(importJS(source, name));
+        }
+    }
+    return all(promises);
+};
 /* c8 ignore stop */

esm/custom.js

@@ -1,4 +1,4 @@
-import { fetchFiles, fetchPaths, stdio, writeFile } from './_utils.js';
+import { fetchFiles, fetchJSModules, fetchPaths, stdio, writeFile } from './_utils.js';
 import { registerJSModule, run, runAsync, runEvent } from './_python.js';
 
 const type = 'micropython';
@@ -15,6 +15,7 @@ export default {
         const interpreter = await get(loadMicroPython({ stderr, stdout, url }));
         if (config.files) await fetchFiles(this, interpreter, config.files);
         if (config.fetch) await fetchPaths(this, interpreter, config.fetch);
+        if (config.js_modules) await fetchJSModules(config.js_modules);
         return interpreter;
     },
     registerJSModule,

esm/interpreter/_utils.js

@@ -1,4 +1,4 @@
-import { fetchFiles, fetchPaths, stdio, writeFile } from './_utils.js';
+import { fetchFiles, fetchJSModules, fetchPaths, stdio, writeFile } from './_utils.js';
 import { registerJSModule, run, runAsync, runEvent } from './_python.js';
 
 const type = 'pyodide';
@@ -18,6 +18,7 @@ export default {
         );
         if (config.files) await fetchFiles(this, interpreter, config.files);
         if (config.fetch) await fetchPaths(this, interpreter, config.fetch);
+        if (config.js_modules) await fetchJSModules(config.js_modules);
         if (config.packages) {
             await interpreter.loadPackage('micropip');
             const micropip = await interpreter.pyimport('micropip');

esm/interpreter/micropython.js

@@ -1,5 +1,5 @@
 import { dedent } from '../utils.js';
-import { fetchFiles, fetchPaths } from './_utils.js';
+import { fetchFiles, fetchJSModules, fetchPaths } from './_utils.js';
 
 const type = 'ruby-wasm-wasi';
 const jsType = type.replace(/\W+/g, '_');
@@ -24,6 +24,7 @@ export default {
         const { vm: interpreter } = await DefaultRubyVM(module);
         if (config.files) await fetchFiles(this, interpreter, config.files);
         if (config.fetch) await fetchPaths(this, interpreter, config.fetch);
+        if (config.js_modules) await fetchJSModules(config.js_modules);
         return interpreter;
     },
     // Fallback to globally defined module fields (i.e. $xworker)

esm/interpreter/pyodide.js

@@ -1,5 +1,5 @@
 import { dedent } from '../utils.js';
-import { fetchFiles, fetchPaths, io, stdio, writeFileShim } from './_utils.js';
+import { fetchFiles, fetchJSModules, fetchPaths, io, stdio, writeFileShim } from './_utils.js';
 
 const type = 'wasmoon';
 
@@ -21,6 +21,7 @@ export default {
         });
         if (config.files) await fetchFiles(this, interpreter, config.files);
         if (config.fetch) await fetchPaths(this, interpreter, config.fetch);
+        if (config.js_modules) await fetchJSModules(config.js_modules);
         return interpreter;
     },
     // Fallback to globally defined module fields

esm/interpreter/ruby-wasm-wasi.js

@@ -4,7 +4,7 @@ import $xworker from './worker/class.js';
 import workerURL from './worker/url.js';
 import { getRuntime, getRuntimeID } from './loader.js';
 import { registry } from './interpreters.js';
-import { all, dispatch, resolve, defineProperty, nodeInfo } from './utils.js';
+import { JSModules, all, dispatch, resolve, defineProperty, nodeInfo } from './utils.js';
 import { getText } from './fetch-utils.js';
 
 const getRoot = (script) => {
@@ -60,7 +60,10 @@ const execute = async (script, source, XWorker, isAsync) => {
             configurable: true,
             get: () => script,
         });
-        module.registerJSModule(interpreter, 'polyscript', { XWorker });
+        module.registerJSModule(interpreter, 'polyscript', {
+            js_modules: JSModules,
+            XWorker,
+        });
         dispatch(script, type, 'ready');
         const result = module[isAsync ? 'runAsync' : 'run'](interpreter, content);
         const done = dispatch.bind(null, script, type, 'done');