Add support for loading dynamic libraries from python_modules (#5249)

Author: hoodmane

build/python_metadata.bzl

@@ -109,8 +109,8 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([
         "pyodide_version": "0.26.0a2",
         "pyodide_date": "2024-03-01",
         "packages": PACKAGES_20240829_4,
-        "backport": "63",
-        "integrity": "sha256-xrG65VJvao9GYH07C73Uq2jA9DW7O1DP16fiZo36Xq0=",
+        "backport": "75",
+        "integrity": "sha256-E1OCAgukWZU6gchqkxgYlvMSieJ8xz6aPFhN/KeqhTw=",
         "flag": "pythonWorkers",
         "enable_flag_name": "python_workers",
         "emscripten_version": "3.1.52",
@@ -139,15 +139,20 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([
                 "abi": None,
                 "sha256": "fc4fb50f73973c257277155b3cb113aa2cf68e9da8ef424ecb049b41bc463183",
             },
+            {
+                "name": "scipy",
+                "abi": "3.12",
+                "sha256": "787e45be6969a5609093b3df9cc2dba2afec9e10bace977f5045697cc329aa7c",
+            },
         ],
     },
     {
         "name": "0.28.2",
         "pyodide_version": "0.28.2",
         "pyodide_date": "2025-01-16",
         "packages": PACKAGES_20250808,
-        "backport": "3",
-        "integrity": "sha256-SCMwCLKzdE65vBQmdeUPs1enbE8TzOu57LBupZzwJY4=",
+        "backport": "6",
+        "integrity": "sha256-bsCa4xEXjgtTDJu7h28GzjJEGqCVawPMqlQqkauCVp8=",
         "flag": "pythonWorkers20250116",
         "enable_flag_name": "python_workers_20250116",
         "emscripten_version": "4.0.9",
@@ -175,6 +180,11 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([
                 "abi": None,
                 "sha256": "fc4fb50f73973c257277155b3cb113aa2cf68e9da8ef424ecb049b41bc463183",
             },
+            {
+                "name": "scipy",
+                "abi": "3.13",
+                "sha256": "4f1b6fc179bd5c6d3de68abc4aa9fca2aaecd09c5c8d357c2ecfedce7d621f3d",
+            },
         ],
     },
     {

src/pyodide/create_vendor_zip.py

@@ -49,8 +49,6 @@ def run_pywrangler_sync(work_dir: Path, python: str | None) -> Path:
     # TODO: Make pywrangler understand how to use Python 3.13 correctly and
     # remove these extra commands
     run(["uv", "venv"], cwd=work_dir, env=env)
-    run(["uv", "pip", "install", "pyodide-build"], cwd=work_dir, env=env)
-    run(["uv", "run", "pyodide", "xbuildenv", "install"], cwd=work_dir, env=env)
     run(["uv", "run", "pywrangler", "sync"], cwd=work_dir, env=env)
     python_modules_dir = work_dir / "python_modules"
     if not python_modules_dir.exists():

src/pyodide/internal/pool/builtin_wrappers.ts

@@ -45,11 +45,24 @@ export function reportUndefinedSymbolsPatched(Module: Module): void {
 }
 
 export function patchDynlibLookup(Module: Module, libName: string): Uint8Array {
+  // This function is for 0.26.0a2 only. In newer versions, we set LD_LIBRARY_PATH instead.
+  if (Module.API.version !== '0.26.0a2') {
+    throw new Error('Should not happen');
+  }
   try {
     return Module.FS.readFile('/usr/lib/' + libName);
   } catch (e) {
-    console.error('Failed to read ', libName, e);
-    throw e;
+    try {
+      // For scipy and similar libraries that depend on Pyodide's dynamic library deps, we may need
+      // extra "system libraries". These we'll put in python_modules/lib. So try loading system
+      // libraries from there too.
+      return Module.FS.readFile(
+        '/session/metadata/python_modules/lib/' + libName
+      );
+    } catch (e) {
+      console.error('Failed to read ', libName, e);
+      throw e;
+    }
   }
 }
 

src/pyodide/internal/pool/emscriptenSetup.ts

@@ -77,9 +77,12 @@ function getPrepareFileSystem(pythonStdlib: ArrayBuffer): PreRunHook {
     Module.API.pyVersionTuple = computeVersionTuple(Module);
     const [pymajor, pyminor] = Module.API.pyVersionTuple;
     Module.FS.sitePackages = `/lib/python${pymajor}.${pyminor}/site-packages`;
-    // finalizeBootstrap() will set LD_LIBRARY_PATH to this same value in a bit, but it's too late
-    // for us when we preload dynamic libraries.
-    Module.ENV.LD_LIBRARY_PATH = ['/usr/lib', Module.FS.sitePackages].join(':');
+    Module.LD_LIBRARY_PATH = [
+      '/usr/lib',
+      Module.FS.sitePackages,
+      '/session/metadata/python_modules/lib/',
+    ].join(':');
+    Module.ENV.LD_LIBRARY_PATH = Module.LD_LIBRARY_PATH;
     Module.FS.sessionSitePackages = '/session' + Module.FS.sitePackages;
     Module.FS.mkdirTree(Module.FS.sitePackages);
     Module.FS.writeFile(

src/pyodide/internal/snapshot.ts

@@ -810,6 +810,11 @@ export function finalizeBootstrap(
       customHiwireStateDeserializer
     );
   });
+  // finalizeBootstrap overrides LD_LIBRARY_PATH. Restore it.
+  simpleRunPython(
+    Module,
+    `import os; os.environ["LD_LIBRARY_PATH"] += ":/session/metadata/python_modules/lib/"; del os`
+  );
   if (IS_CREATING_SNAPSHOT) {
     return;
   }

src/pyodide/types/emscripten.d.ts

@@ -125,4 +125,5 @@ interface Module {
   snapshotDebug?: boolean;
   getEmptyTableSlot(): number;
   freeTableIndexes: number[];
+  LD_LIBRARY_PATH: string;
 }

src/workerd/server/tests/python/vendor_pkg_tests/BUILD

@@ -7,3 +7,5 @@ vendored_py_wd_test("beautifulsoup4")
 vendored_py_wd_test(
     "python-workers-runtime-sdk",
 )
+
+vendored_py_wd_test("scipy")

src/workerd/server/tests/python/vendor_pkg_tests/generate_modules.py

@@ -27,7 +27,7 @@
     # without `external/` prefix
     embed_path = ("../" * 7) + str(Path(*components[1:])).replace("\\", "\\\\")
     # without `external/fastapi_src/` prefix
-    module_path = str(Path(*components[2:])).replace("\\", "/")
+    module_path = str(Path("python_modules", *components[2:])).replace("\\", "/")
 
     # Format as a Cap'n Proto module definition.
     if f_path.endswith(".py"):

src/workerd/server/tests/python/vendor_pkg_tests/scipy.py

@@ -0,0 +1,10 @@
+import numpy as np
+from scipy import linalg
+
+
+async def test():
+    c = np.array([1, 3, 6, 10])
+    r = np.array([1, -1, -2, -3])
+    b = np.array([1, 2, 2, 5])
+    res = linalg.solve_toeplitz((c, r), b)
+    assert repr(res) == "array([ 1.66666667, -1.        , -2.66666667,  2.33333333])"

src/workerd/server/tests/python/vendor_pkg_tests/scipy_vendor.wd-test

@@ -0,0 +1,16 @@
+using Workerd = import "/workerd/workerd.capnp";
+
+const unitTests :Workerd.Config = (
+  services = [
+    ( name = "scipy-vendor-test",
+      worker = (
+        modules = [
+          (name = "main.py", pythonModule = embed "scipy.py"),
+          %PYTHON_VENDORED_MODULES%
+        ],
+        compatibilityDate = "2024-01-15",
+        compatibilityFlags = [%PYTHON_FEATURE_FLAGS],
+      )
+    ),
+  ],
+);