Add support for loading dynamic libraries from python_modules

Author: hoodmane

build/python_metadata.bzl

@@ -109,8 +109,8 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([
         "pyodide_version": "0.26.0a2",
         "pyodide_date": "2024-03-01",
         "packages": PACKAGES_20240829_4,
-        "backport": "63",
-        "integrity": "sha256-xrG65VJvao9GYH07C73Uq2jA9DW7O1DP16fiZo36Xq0=",
+        "backport": "73",
+        "integrity": "sha256-Ju9/kVMOnhpJjONkmy81c5ZujrjMFWS5NqvMkYCNCuo=",
         "flag": "pythonWorkers",
         "enable_flag_name": "python_workers",
         "emscripten_version": "3.1.52",
@@ -134,15 +134,20 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([
                 "abi": "3.12",
                 "sha256": "5e6e21dbeda7c1eaadb99e6e52aa2ce45325b51e9a417198701e68e0cfd12a4c",
             },
+            {
+                "name": "scipy",
+                "abi": "3.12",
+                "sha256": "787e45be6969a5609093b3df9cc2dba2afec9e10bace977f5045697cc329aa7c",
+            },
         ],
     },
     {
         "name": "0.28.2",
         "pyodide_version": "0.28.2",
         "pyodide_date": "2025-01-16",
         "packages": PACKAGES_20250808,
-        "backport": "3",
-        "integrity": "sha256-SCMwCLKzdE65vBQmdeUPs1enbE8TzOu57LBupZzwJY4=",
+        "backport": "4",
+        "integrity": "sha256-bsCa4xEXjgtTDJu7h28GzjJEGqCVawPMqlQqkauCVp8=",
         "flag": "pythonWorkers20250116",
         "enable_flag_name": "python_workers_20250116",
         "emscripten_version": "4.0.9",
@@ -165,6 +170,11 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([
                 "abi": "3.13",
                 "sha256": "955091f1bd2eb33255ff2633df990bedc96e2f6294e78f2b416078777394f942",
             },
+            {
+                "name": "scipy",
+                "abi": "3.13",
+                "sha256": "4f1b6fc179bd5c6d3de68abc4aa9fca2aaecd09c5c8d357c2ecfedce7d621f3d",
+            },
         ],
     },
     {

src/pyodide/create_vendor_zip.py

@@ -49,8 +49,6 @@ def run_pywrangler_sync(work_dir: Path, python: str | None) -> Path:
     # TODO: Make pywrangler understand how to use Python 3.13 correctly and
     # remove these extra commands
     run(["uv", "venv"], cwd=work_dir, env=env)
-    run(["uv", "pip", "install", "pyodide-build"], cwd=work_dir, env=env)
-    run(["uv", "run", "pyodide", "xbuildenv", "install"], cwd=work_dir, env=env)
     run(["uv", "run", "pywrangler", "sync"], cwd=work_dir, env=env)
     python_modules_dir = work_dir / "python_modules"
     if not python_modules_dir.exists():

src/pyodide/internal/pool/builtin_wrappers.ts

@@ -48,8 +48,12 @@ export function patchDynlibLookup(Module: Module, libName: string): Uint8Array {
   try {
     return Module.FS.readFile('/usr/lib/' + libName);
   } catch (e) {
-    console.error('Failed to read ', libName, e);
-    throw e;
+    try {
+      return Module.FS.readFile('/session/metadata/lib/' + libName);
+    } catch (e) {
+      console.error('Failed to read ', libName, e);
+      throw e;
+    }
   }
 }
 

src/pyodide/internal/python.ts

@@ -28,7 +28,11 @@ import type { PyodideEntrypointHelper } from 'pyodide:python-entrypoint-helper';
 import { default as SetupEmscripten } from 'internal:setup-emscripten';
 
 import { default as UnsafeEval } from 'internal:unsafe-eval';
-import { PythonRuntimeError, reportError } from 'pyodide-internal:util';
+import {
+  PythonRuntimeError,
+  reportError,
+  simpleRunPython,
+} from 'pyodide-internal:util';
 import { loadPackages } from 'pyodide-internal:loadPackage';
 import { default as MetadataReader } from 'pyodide-internal:runtime-generated/metadata';
 import { TRANSITIVE_REQUIREMENTS } from 'pyodide-internal:metadata';
@@ -62,6 +66,10 @@ function prepareWasmLinearMemory(
   }
   if (Module.API.version !== '0.26.0a2') {
     finalizeBootstrap(Module, pyodide_entrypoint_helper);
+    simpleRunPython(
+      Module,
+      `import os; os.environ["LD_LIBRARY_PATH"] += ":/session/metadata/lib/"; del os`
+    );
   }
 }
 

src/workerd/server/tests/python/vendor_pkg_tests/BUILD

@@ -9,3 +9,8 @@ vendored_py_wd_test(
     "beautifulsoup4",
     skip_python_flags = ["0.28.2"],
 )
+
+vendored_py_wd_test(
+    "scipy",
+    skip_python_flags = ["0.28.2"],
+)

src/workerd/server/workerd-api.c++

@@ -226,7 +226,7 @@ class EmptyReadOnlyActorStorageImpl final: public rpc::ActorStorage::Stage::Serv
 jsg::Bundle::Reader retrievePyodideBundle(
     const api::pyodide::PythonConfig& pyConfig, kj::StringPtr version) {
   auto result = pyConfig.pyodideBundleManager.getPyodideBundle(version);
-  return KJ_ASSERT_NONNULL(result, "Failed to get Pyodide bundle");
+  return KJ_ASSERT_NONNULL(result, "Failed to get Pyodide bundle", version);
 }
 
 /**