Add lockfileContents option to loadPyodide() (pyodide#5764)

And also `packageBaseUrl` option:
1. If `lockfileContents` is provided and not `packageBaseUrl`, then in the browser attempting to
   load a lock entry with a relative path fails.
2. In Node, `packageBaseUrl` is used in place of jsdelivr as the cdn url if passed.

See discussion in pyodide#5736.

Author: hoodmane

docs/expected_js_docs.txt

@@ -93,6 +93,21 @@ js:function
 js:attribute
   pyodide.ERRNO_CODES
   pyodide.FS
+  pyodide.Lockfile.info
+  pyodide.Lockfile.packages
+  pyodide.LockfileInfo.abi_version
+  pyodide.LockfileInfo.arch
+  pyodide.LockfileInfo.platform
+  pyodide.LockfileInfo.python
+  pyodide.LockfileInfo.version
+  pyodide.LockfilePackage.depends
+  pyodide.LockfilePackage.file_name
+  pyodide.LockfilePackage.imports
+  pyodide.LockfilePackage.install_dir
+  pyodide.LockfilePackage.name
+  pyodide.LockfilePackage.package_type
+  pyodide.LockfilePackage.sha256
+  pyodide.LockfilePackage.version
   pyodide.PATH
   pyodide.PackageData.fileName
   pyodide.PackageData.name
@@ -119,6 +134,9 @@ js:attribute
   pyodide.pyodide_py
   pyodide.version
 js:interface
+  pyodide.Lockfile
+  pyodide.LockfileInfo
+  pyodide.LockfilePackage
   pyodide.PackageData
 js:typealias
   pyodide.TypedArray

docs/project/changelog.md

@@ -24,6 +24,12 @@ myst:
 - {{ Fix }} Fixed a bug in Node.js which providing a relative path to `lockFileURL` parameter of `loadPyodide()` did not work.
   {pr}`5750`
 
+- {{ Enhancement }} Added `lockfileContents` and `packageBaseURL` options to
+  `loadPyodide`. This allows providing a lock file as a `Promise` for the
+  contents rather than a URL. If `lockfileContents` is provided, then
+  `packageBaseURL` must also be provided in order to resolve relative paths in
+  the lockfile.
+
 ## Version 0.28.0
 
 _July 4, 2025_

docs/sphinx_pyodide/sphinx_pyodide/jsdoc.py

@@ -1,7 +1,7 @@
 from collections.abc import Iterator
 
 from sphinx_js import ir
-from sphinx_js.ir import Class
+from sphinx_js.ir import Class, TypeXRefInternal
 from sphinx_js.typedoc import Analyzer as TsAnalyzer
 
 __all__ = ["ts_xref_formatter", "patch_sphinx_js"]
@@ -16,6 +16,10 @@ def ts_xref_formatter(_config, xref):
     from sphinx_pyodide.mdn_xrefs import JSDATA
 
     name = xref.name
+    if name == "Lockfile":
+        name = "~pyodide.Lockfile"
+    if name == "TypedArray":
+        name = "~pyodide.TypedArray"
     if name == "PyodideAPI":
         return ":ref:`PyodideAPI <js-api-pyodide>`"
     if name in JSDATA:
@@ -24,6 +28,8 @@ def ts_xref_formatter(_config, xref):
         return f":js:class:`~pyodide.ffi.{name}`"
     if name in ["ConcatArray", "IterableIterator", "unknown", "U"]:
         return f"``{name}``"
+    if isinstance(xref, TypeXRefInternal):
+        return f":js:{xref.kind}:`{name}`"
     return f":js:class:`{name}`"
 
 
@@ -118,6 +124,9 @@ def get_obj_mod(doclet: ir.TopLevel) -> str:
     filename = key[0]
     doclet.name = doclet.name.rpartition(".")[2]
 
+    if kind := doclet.block_tags.get("docgroup"):
+        return kind[0][0].text
+
     if filename == "pyodide.":
         return "globalThis"
 

src/js/api.ts

@@ -7,7 +7,7 @@ import { loadBinaryFile, nodeFSMod } from "./compat";
 import { version } from "./version";
 import { setStdin, setStdout, setStderr } from "./streams";
 import { scheduleCallback } from "./scheduler";
-import { TypedArray, PackageData, FSType } from "./types";
+import { TypedArray, PackageData, FSType, Lockfile } from "./types";
 import { IN_NODE, detectEnvironment } from "./environments";
 // @ts-ignore
 import LiteralMap from "./common/literal-map";
@@ -691,16 +691,16 @@ export class PyodideAPI {
    * The format of the lockfile is defined in the `pyodide/pyodide-lock
    * <https://github.com/pyodide/pyodide-lock>`_ repository.
    */
-  static get lockfile() {
+  static get lockfile(): Lockfile {
     return API.lockfile;
   }
 
   /**
-   * Returns the base URL of the lockfile, which is used to locate the packages
-   * distributed with the lockfile.
+   * Returns the URL or path with respect to which relative paths in the lock
+   * file are resolved, or undefined.
    */
-  static get lockfileBaseUrl() {
-    return API.lockfileBaseUrl;
+  static get lockfileBaseUrl(): string | undefined {
+    return API.config.packageCacheDir ?? API.config.packageBaseUrl;
   }
 }
 

src/js/compat.ts

@@ -25,6 +25,7 @@ declare var globalThis: {
   importScripts: (url: string) => void;
   document?: typeof document;
   fetch?: typeof fetch;
+  location?: URL;
 };
 
 /**
@@ -74,6 +75,10 @@ export async function initNodeModules() {
   };
 }
 
+export function isAbsolute(path: string): boolean {
+  return path.includes("://") || path.startsWith("/");
+}
+
 function node_resolvePath(path: string, base?: string): string {
   return nodePath.resolve(base || ".", path);
 }
@@ -322,10 +327,13 @@ export async function calculateDirname(): Promise<string> {
  * Ensure that the directory exists before trying to download files into it (Node.js only).
  * @param dir The directory to ensure exists
  */
-export async function ensureDirNode(dir: string) {
+export async function ensureDirNode(dir?: string) {
   if (!IN_NODE) {
     return;
   }
+  if (!dir) {
+    return;
+  }
 
   try {
     // Check if the `installBaseUrl` directory exists
@@ -337,3 +345,22 @@ export async function ensureDirNode(dir: string) {
     });
   }
 }
+
+/**
+ * Calculates the install base url for the package manager.
+ * exported for testing
+ * @param lockFileURL
+ * @returns the install base url
+ * @private
+ */
+export function calculateInstallBaseUrl(lockFileURL: string) {
+  // 1. If the lockfile URL includes a path with slash (file url in Node.js or http url in browser), use the directory of the lockfile URL
+  // 2. Otherwise, fallback to the current location
+  //    2.1. In the browser, use `location` to get the current location
+  //    2.2. In Node.js just use the pwd
+  return (
+    lockFileURL.substring(0, lockFileURL.lastIndexOf("/") + 1) ||
+    globalThis.location?.toString() ||
+    "."
+  );
+}

src/js/load-package.ts

@@ -2,7 +2,7 @@ import "./constants";
 import {
   Lockfile,
   PackageData,
-  InternalPackageData,
+  LockfilePackage,
   PackageLoadMetadata,
   PackageManagerAPI,
   PackageManagerModule,
@@ -23,6 +23,7 @@ import {
   resolvePath,
   initNodeModules,
   ensureDirNode,
+  isAbsolute,
 } from "./compat";
 import { Installer } from "./installer";
 import { createContextWrapper } from "./common/contextManager";
@@ -35,10 +36,12 @@ import { createContextWrapper } from "./common/contextManager";
  * @private
  */
 export async function initializePackageIndex(
-  lockFilePromise: Promise<Lockfile>,
+  lockFilePromise: Promise<Lockfile | string>,
 ) {
   await initNodeModules();
-  const lockfile = await lockFilePromise;
+  const lockfile_ = await lockFilePromise;
+  const lockfile: Lockfile =
+    typeof lockfile_ === "string" ? JSON.parse(lockfile_) : lockfile_;
   if (!lockfile.packages) {
     throw new Error(
       "Loaded pyodide lock file does not contain the expected key 'packages'.",
@@ -123,7 +126,7 @@ export class PackageManager {
 
   private _lock = createLock();
 
-  public installBaseUrl: string;
+  public installBaseUrl?: string;
 
   /**
    * The function to use for stdout and stderr, defaults to console.log and console.error
@@ -147,11 +150,15 @@ export class PackageManager {
     this.#module = pyodideModule;
     this.#installer = new Installer(api, pyodideModule);
 
-    const lockfileBase = calculateInstallBaseUrl(this.#api.config.lockFileURL);
     if (IN_NODE) {
-      this.installBaseUrl = this.#api.config.packageCacheDir ?? lockfileBase;
+      // In node, we'll try first to load from the packageCacheDir and then fall
+      // back to cdnURL
+      this.installBaseUrl =
+        this.#api.config.packageCacheDir ?? API.config.packageBaseUrl;
+      this.cdnURL = this.#api.config.cdnUrl;
     } else {
-      this.installBaseUrl = lockfileBase;
+      // use packageBaseUrl as the base URL for the packages
+      this.installBaseUrl = this.#api.config.packageBaseUrl;
     }
 
     this.stdout = (msg: string) => {
@@ -248,7 +255,7 @@ export class PackageManager {
       checkIntegrity: true,
     },
   ): Promise<Array<PackageData>> {
-    const loadedPackageData = new Set<InternalPackageData>();
+    const loadedPackageData = new Set<LockfilePackage>();
     const pkgNames = toStringArray(names);
 
     const toLoad = this.recursiveDependencies(pkgNames);
@@ -450,6 +457,13 @@ export class PackageManager {
       }
       const lockfilePackage = this.#api.lockfile_packages[pkg.normalizedName];
       fileName = lockfilePackage.file_name;
+      // TODO: Node caching logic assumes relative here...
+      if (!isAbsolute(fileName) && !this.installBaseUrl) {
+        throw new Error(
+          `Lock file file_name for package "${pkg.name}" is relative path "${fileName}" but no packageBaseUrl provided to loadPyodide.`,
+        );
+      }
+
       uri = resolvePath(fileName, this.installBaseUrl);
       fileSubResourceHash = "sha256-" + base16ToBase64(lockfilePackage.sha256);
     } else {
@@ -464,7 +478,12 @@ export class PackageManager {
       DEBUG && console.debug(`Downloading package ${pkg.name} from ${uri}`);
       return await loadBinaryFile(uri, fileSubResourceHash);
     } catch (e) {
-      if (!IN_NODE || pkg.channel !== this.defaultChannel) {
+      if (
+        !IN_NODE ||
+        pkg.channel !== this.defaultChannel ||
+        !fileName ||
+        fileName.startsWith("/")
+      ) {
         throw e;
       }
     }
@@ -538,7 +557,7 @@ export class PackageManager {
   private async downloadAndInstall(
     pkg: PackageLoadMetadata,
     toLoad: Map<string, PackageLoadMetadata>,
-    loaded: Set<InternalPackageData>,
+    loaded: Set<LockfilePackage>,
     failed: Map<string, Error>,
     checkIntegrity: boolean = true,
   ) {
@@ -572,10 +591,6 @@ export class PackageManager {
     }
   }
 
-  public setCdnUrl(url: string) {
-    this.cdnURL = url;
-  }
-
   /**
    * Flushes the stdout and stderr buffers, that were collected before the
    * stdout and stderr functions were set.
@@ -641,7 +656,7 @@ function filterPackageData({
   version,
   file_name,
   package_type,
-}: InternalPackageData): PackageData {
+}: LockfilePackage): PackageData {
   return { name, version, fileName: file_name, packageType: package_type };
 }
 
@@ -666,25 +681,6 @@ export function toStringArray(str: string | PyProxy | string[]): string[] {
   return str;
 }
 
-/**
- * Calculates the install base url for the package manager.
- * exported for testing
- * @param lockFileURL
- * @returns the install base url
- * @private
- */
-export function calculateInstallBaseUrl(lockFileURL: string) {
-  // 1. If the lockfile URL includes a path with slash (file url in Node.js or http url in browser), use the directory of the lockfile URL
-  // 2. Otherwise, fallback to the current location
-  //    2.1. In the browser, use `location` to get the current location
-  //    2.2. In Node.js just use the pwd
-  return (
-    lockFileURL.substring(0, lockFileURL.lastIndexOf("/") + 1) ||
-    globalThis.location?.toString() ||
-    "."
-  );
-}
-
 export let loadPackage: typeof PackageManager.prototype.loadPackage;
 /**
  * An object whose keys are the names of the loaded packages and whose values
@@ -710,13 +706,6 @@ if (typeof API !== "undefined" && typeof Module !== "undefined") {
    */
   loadedPackages = singletonPackageManager.loadedPackages;
 
-  // TODO: Find a better way to register these functions
-  API.setCdnUrl = singletonPackageManager.setCdnUrl.bind(
-    singletonPackageManager,
-  );
-
-  API.lockfileBaseUrl = singletonPackageManager.installBaseUrl;
-
   API.flushPackageManagerBuffers = singletonPackageManager.flushBuffers.bind(
     singletonPackageManager,
   );