Don't preload dynamic libraries in newer Python compat dates

This fixes some but not all of the test failures with the new package
set and also should improve performance because it avoids loading
unneeded dynamic libraries.

Author: hoodmane

build/python/packages_20241218.bzl build/python/packages_20250324_1.bzlbuild/python/packages_20241218.bzl renamed to build/python/packages_20250324_1.bzl

@@ -1,10 +1,13 @@
 # This file is automatically generated by the Pyodide build script repo
 # (https://github.com/cloudflare/pyodide-build-scripts) and should not be manually modified.
 
-PACKAGES_20241218 = {
+PACKAGES_20250324_1 = {
     "PyJWT": [
         "jwt",
     ],
+    "aiohappyeyeballs": [
+        "aiohappyeyeballs",
+    ],
     "aiohttp": [
         "aiohttp",
     ],
@@ -125,11 +128,50 @@ PACKAGES_20241218 = {
     "packaging": [
         "packaging",
     ],
+    "propcache": [
+        "propcache",
+    ],
     "pycparser": [
         "pycparser",
     ],
     "pydantic": [
         "pydantic",
+        "pydantic.alias_generators",
+        "pydantic.aliases",
+        "pydantic.annotated_handlers",
+        "pydantic.class_validators",
+        "pydantic.color",
+        "pydantic.config",
+        "pydantic.dataclasses",
+        "pydantic.datetime_parse",
+        "pydantic.decorator",
+        "pydantic.deprecated",
+        "pydantic.env_settings",
+        "pydantic.error_wrappers",
+        "pydantic.errors",
+        "pydantic.experimental",
+        "pydantic.fields",
+        "pydantic.functional_serializers",
+        "pydantic.functional_validators",
+        "pydantic.generics",
+        "pydantic.json",
+        "pydantic.json_schema",
+        "pydantic.main",
+        "pydantic.networks",
+        "pydantic.parse",
+        "pydantic.plugin",
+        "pydantic.root_model",
+        "pydantic.schema",
+        "pydantic.tools",
+        "pydantic.type_adapter",
+        "pydantic.types",
+        "pydantic.typing",
+        "pydantic.utils",
+        "pydantic.v1",
+        "pydantic.validate_call_decorator",
+        "pydantic.validators",
+        "pydantic.version",
+        "pydantic.warnings",
     ],
     "pydantic_core": [
         "pydantic_core",

build/python_metadata.bzl

@@ -1,5 +1,5 @@
 load("//:build/python/packages_20240829_4.bzl", "PACKAGES_20240829_4")
-load("//:build/python/packages_20241218.bzl", "PACKAGES_20241218")
+load("//:build/python/packages_20250324_1.bzl", "PACKAGES_20250324_1")
 
 # The below is a list of pyodide-lock.json files for each package bundle version that we support.
 # Each of these gets embedded in the workerd and EW binary.
@@ -8,15 +8,15 @@ load("//:build/python/packages_20241218.bzl", "PACKAGES_20241218")
 # the lock file.
 PYTHON_LOCKFILES = {
     "20240829.4": "c2d9c67ea55a672b95a3beb8d66bfbe7df736edb4bb657383b263151e7e85ef4",
-    "20241218": "1421e9351baf24ec44d82f78b9ac26e8e0e6595bfe3f626dedb33147bfcd1998",
+    "20250324.1": "3e5a9317dc0cfcf63e556034bf0e87b958bd6debcfdccdfffc8ce477cc439626",
 }
 
 # This is a dictionary mapping a Python version with its packages version.
 #
 # NOTE: this needs to be kept in sync with compatibility-date.capnp.
 PYTHON_VERSION_TO_PACKAGES = {
     "0.26.0a2": "20240829.4",
-    "0.27.1": "20241218",
+    "0.27.1": "20250324.1",
     "development": "20240829.4",
 }
 
@@ -32,7 +32,7 @@ PYTHON_VERSION_TO_PACKAGES = {
 # first.
 PYTHON_IMPORTS_TO_TEST = {
     "20240829.4": PACKAGES_20240829_4,
-    "20241218": PACKAGES_20241218,
+    "20250324.1": PACKAGES_20250324_1,
 }
 
 # Each new package bundle should contain the same packages as the previous. We verify this
@@ -42,8 +42,8 @@ def verify_no_packages_were_removed():
     for i in range(0, len(package_dates) - 1):
         curr_pkgs = PYTHON_IMPORTS_TO_TEST[package_dates[i]]
         next_pkgs = PYTHON_IMPORTS_TO_TEST[package_dates[i + 1]]
-        for pkg in curr_pkgs:
-            if pkg not in next_pkgs:
-                fail(pkg + " from packages version ", package_dates[i], " not in ", package_dates[i + 1])
+        missing_pkgs = [pkg for pkg in curr_pkgs if pkg not in next_pkgs]
+        if missing_pkgs:
+            fail(str(missing_pkgs) + " from packages version ", package_dates[i], " not in ", package_dates[i + 1])
 
 verify_no_packages_were_removed()

src/pyodide/BUILD.bazel

@@ -113,8 +113,6 @@ const location = undefined;
 
 function addEventListener(){}
 
-function reportUndefinedSymbolsNoOp() {}
-
 if (typeof FinalizationRegistry === "undefined") {
     globalThis.FinalizationRegistry = class FinalizationRegistry {
         register(){}
@@ -155,10 +153,6 @@ REPLACEMENTS = [
         "Date.now",
         "monotonicDateNow",
     ],
-    [
-        "reportUndefinedSymbols()",
-        "reportUndefinedSymbolsNoOp()",
-    ],
     [
         "crypto.getRandomValues(",
         "getRandomValues(Module, ",
@@ -177,9 +171,16 @@ REPLACEMENTS = [
         "eval(UTF8ToString(ptr))",
         "(() => {throw new Error('Internal Emscripten code tried to eval, this should not happen, please file a bug report with your requirements.txt file\\'s contents')})()",
     ],
+    # Dynamic linking patches:
+    # library lookup
+    [
+        "!libData",
+        "!(libData ??= patchDynlibLookup(Module, libName))",
+    ],
+    # for ensuring memory base of dynlib is stable when restoring snapshots
     [
-        "!libData&&flags.fs",
-        "!(libData ??= patchDynlibLookup(Module, libName))&&flags.fs",
+        "getMemory(",
+        "Module.getMemoryPatched(Module, libName, ",
     ],
 ]
 

src/pyodide/internal/loadPackage.ts

@@ -18,10 +18,8 @@ import {
   USING_OLDEST_PACKAGES_VERSION,
 } from 'pyodide-internal:metadata';
 import {
-  DYNLIB_PATH,
   VIRTUALIZED_DIR,
   STDLIB_PACKAGES,
-  getSitePackagesPath,
 } from 'pyodide-internal:setupPackages';
 import { parseTarInfo } from 'pyodide-internal:tar';
 import { default as DiskCache } from 'pyodide-internal:disk_cache';

src/pyodide/internal/metadatafs.ts

@@ -56,7 +56,11 @@ export function createMetadataFS(Module: Module): object {
       if (parent.tree == undefined) {
         throw new Error('cannot lookup directory, tree is undefined');
       }
-      return parent.tree.get(name)!;
+      const res = parent.tree.get(name);
+      if (res === undefined) {
+        throw new Module.FS.ErrnoError(44);
+      }
+      return res;
     },
     read(stream, position, buffer) {
       return MetadataReader.read(stream.node.index!, position, buffer);

src/pyodide/internal/pool/emscriptenSetup.ts

@@ -62,7 +62,9 @@ function getPrepareFileSystem(pythonStdlib: ArrayBuffer): PreRunHook {
     try {
       const pymajor = Module._py_version_major();
       const pyminor = Module._py_version_minor();
-      Module.FS.mkdirTree(`/lib/python${pymajor}.${pyminor}/site-packages`);
+      Module.FS.sitePackages = `/lib/python${pymajor}.${pyminor}/site-packages`;
+      Module.FS.sessionSitePackages = '/session' + Module.FS.sitePackages;
+      Module.FS.mkdirTree(Module.FS.sitePackages);
       Module.FS.writeFile(
         `/lib/python${pymajor}${pyminor}.zip`,
         new Uint8Array(pythonStdlib),

src/pyodide/internal/setupPackages.ts

@@ -6,13 +6,13 @@ import {
   LOAD_WHEELS_FROM_R2,
   LOCKFILE,
   LOAD_WHEELS_FROM_ARTIFACT_BUNDLER,
-  USING_OLDEST_PACKAGES_VERSION,
 } from 'pyodide-internal:metadata';
 import { simpleRunPython } from 'pyodide-internal:util';
 import { default as EmbeddedPackagesTarReader } from 'pyodide-internal:packages_tar_reader';
 import { default as MetadataReader } from 'pyodide-internal:runtime-generated/metadata';
 
 const canonicalizeNameRegex = /[-_.]+/g;
+const DYNLIB_PATH = '/usr/lib';
 
 /**
  * Canonicalize a package name. Port of Python's packaging.utils.canonicalize_name.
@@ -153,8 +153,13 @@ class VirtualizedDir {
   }
 
   mount(Module: Module, tarFS: EmscriptenFS<TarFSInfo>) {
-    const path = getSitePackagesPath(Module);
-    Module.FS.mount(tarFS, { info: this.rootInfo }, path);
+    Module.FS.mkdirTree(Module.FS.sessionSitePackages);
+    Module.FS.mount(
+      tarFS,
+      { info: this.rootInfo },
+      Module.FS.sessionSitePackages
+    );
+    Module.FS.mkdirTree(DYNLIB_PATH);
     Module.FS.mount(tarFS, { info: this.dynlibTarFs }, DYNLIB_PATH);
   }
 }
@@ -215,14 +220,6 @@ function disabledLoadPackage(): never {
   );
 }
 
-export function getSitePackagesPath(Module: Module): string {
-  const pymajor = Module._py_version_major();
-  const pyminor = Module._py_version_minor();
-  return `/session/lib/python${pymajor}.${pyminor}/site-packages`;
-}
-
-export const DYNLIB_PATH = '/usr/lib';
-
 /**
  * This mounts a TarFS representing the site-packages directory (which contains the Python packages)
  * and another TarFS representing the dynlib directory (where dynlibs like libcrypto.so live).
@@ -233,9 +230,6 @@ export const DYNLIB_PATH = '/usr/lib';
  */
 export function mountSitePackages(Module: Module, pkgs: VirtualizedDir): void {
   const tarFS = createTarFS(Module);
-  const site_packages = getSitePackagesPath(Module);
-  Module.FS.mkdirTree(site_packages);
-  Module.FS.mkdirTree(DYNLIB_PATH);
   if (!LOAD_WHEELS_FROM_R2 && !LOAD_WHEELS_FROM_ARTIFACT_BUNDLER) {
     // if we are not loading additional wheels, then we're done
     // with site-packages and we can mount it here. Otherwise, we must mount it in
@@ -262,55 +256,13 @@ export function mountWorkerFiles(Module: Module) {
  * Has to run after the runtime is initialized.
  */
 export function adjustSysPath(Module: Module): void {
-  const site_packages = getSitePackagesPath(Module);
+  const site_packages = Module.FS.sessionSitePackages;
   simpleRunPython(
     Module,
     `import sys; sys.path.append("/session/metadata"); sys.path.append("${site_packages}"); del sys`
   );
 }
 
-function recursiveDependencies(
-  lockfile: PackageLock,
-  names: string[]
-): PackageDeclaration[] {
-  const toLoad = new Map();
-  for (const name of names) {
-    addPackageToLoad(lockfile, name, toLoad);
-  }
-  return Array.from(toLoad.values());
-}
-
-/**
- * Recursively add a package and its dependencies to toLoad.
- * A helper function for recursiveDependencies.
- * @param name The package to add
- * @param toLoad The set of names of packages to load
- * @private
- */
-function addPackageToLoad(
-  lockfile: PackageLock,
-  name: string,
-  toLoad: Map<string, PackageDeclaration>
-): void {
-  const normalizedName = canonicalizePackageName(name);
-  if (toLoad.has(normalizedName)) {
-    return;
-  }
-  const pkgInfo = lockfile.packages[normalizedName];
-  if (!pkgInfo) {
-    throw new Error(
-      `It appears that a package ("${name}") you requested is not available yet in workerd. \n` +
-        'If you would like this package to be included, please open an issue at https://github.com/cloudflare/workerd/discussions/new?category=python-packages.'
-    );
-  }
-
-  toLoad.set(normalizedName, pkgInfo);
-
-  for (let depName of pkgInfo.depends) {
-    addPackageToLoad(lockfile, depName, toLoad);
-  }
-}
-
 export { REQUIREMENTS };
 export const TRANSITIVE_REQUIREMENTS =
   MetadataReader.getTransitiveRequirements();