Allow to install extra packages from extra indexes from jupyter-lite.json<F48>

This is in progress, but the idea is to be able to point to dev wheels
on rtd or similar without having to make a custom pyidide distribution.

For example:

```
// jupyter-lite.json
{
  "jupyter-config-data": {
    "litePluginSettings": {
       "extraPackagesAndIndexes": [
          {
            "indexes": [
              "https://cors.carreau.workers.dev/scientific-python-nightly-wheels/simple",
              "https://pypi.org/simple"
            ],
            "package": "xarray"
          }
        ]
      }
    }
  }
```

Package could also be a simple url to a whl (I need to check how it
works if it's a local URL), and indexes can be ommited.

WIP as I'm stuggling to have the typescript build process work.

Author: Carreau

packages/pyodide-kernel-extension/src/index.ts

@@ -58,6 +58,7 @@ const kernel: JupyterLiteServerPlugin<void> = {
     const pipliteUrls = rawPipUrls.map((pipUrl: string) => URLExt.parse(pipUrl).href);
     const disablePyPIFallback = !!config.disablePyPIFallback;
     const loadPyodideOptions = config.loadPyodideOptions || {};
+    const extraPackagesAndIndexes = config.extraPackagesAndIndexes || [];
 
     for (const [key, value] of Object.entries(loadPyodideOptions)) {
       if (key.endsWith('URL') && typeof value === 'string') {
@@ -99,6 +100,7 @@ const kernel: JupyterLiteServerPlugin<void> = {
           mountDrive,
           loadPyodideOptions,
           contentsManager,
+          extraPackagesAndIndexes,
         });
       },
     });

packages/pyodide-kernel/src/kernel.ts

@@ -120,6 +120,7 @@ export class PyodideKernel extends BaseKernel implements IKernel {
       location: this.location,
       mountDrive: options.mountDrive,
       loadPyodideOptions: options.loadPyodideOptions || {},
+      extraPackagesAndIndexes: options.extraPackagesAndIndexes || [],
     };
   }
 
@@ -394,5 +395,6 @@ export namespace PyodideKernel {
      * The Jupyterlite content manager
      */
     contentsManager: Contents.IManager;
+    extraPackagesAndIndexes: Array<{ package: string; indexes: string[] | null }>;
   }
 }

packages/pyodide-kernel/src/tokens.ts

@@ -90,6 +90,11 @@ export namespace IPyodideWorkerKernel {
      */
     mountDrive: boolean;
 
+    /**
+     * List of extra url/wheel paths to load, and extra associated indexes urls (if not a wheel url)
+     */
+    extraPackagesAndIndexes: Array<{ package: string; indexes: string[] | null }>;
+
     /**
      * additional options to provide to `loadPyodide`
      * @see https://pyodide.org/en/stable/usage/api/js-api.html#globalThis.loadPyodide

packages/pyodide-kernel/src/worker.ts

@@ -65,8 +65,13 @@ export class PyodideRemoteKernel {
       throw new Error('Uninitialized');
     }
 
-    const { pipliteWheelUrl, disablePyPIFallback, pipliteUrls, loadPyodideOptions } =
-      this._options;
+    const {
+      pipliteWheelUrl,
+      disablePyPIFallback,
+      pipliteUrls,
+      loadPyodideOptions,
+      extraPackagesAndIndexes,
+    } = this._options;
 
     const preloaded = (loadPyodideOptions || {}).packages || [];
 
@@ -80,6 +85,28 @@ export class PyodideRemoteKernel {
       await micropip.install('${pipliteWheelUrl}', keep_going=True)
     `);
     }
+    console.debug('extraPackagesAndIndexes', extraPackagesAndIndexes);
+
+    if (extraPackagesAndIndexes.length > 0) {
+      // note that here pkg can be a package name or a wheel url
+      for (let { package: pkg, indexes } of extraPackagesAndIndexes) {
+        let installCmd: string;
+        if (indexes === null) {
+          installCmd = `import micropip\nawait micropip.install('${pkg}', keep_going=True)`;
+        } else {
+          installCmd = `import micropip\nawait micropip.install('${pkg}', index_urls=${JSON.stringify(indexes)}, keep_going=True)`;
+        }
+        console.info('installCmd', installCmd);
+        try {
+          await this._pyodide.runPythonAsync(installCmd);
+          console.info(`Package ${pkg} Installed successfully`);
+        } catch (e) {
+          console.error('Error installing package', e);
+        }
+      }
+    } else {
+      console.info('no extra packages and indexes');
+    }
 
     // get piplite early enough to impact pyodide-kernel dependencies
     await this._pyodide.runPythonAsync(`