Merge pull request #3789 from cloudflare/hoodmane/fix-0.27.2-packages

Don't preload dynamic libraries in newer Python compat dates

Author: hoodmane

.github/CODEOWNERS

@@ -5,8 +5,8 @@ build-releases.sh @cloudflare/wrangler @cloudflare/workers-runtime-1
 RELEASE.md @cloudflare/wrangler @cloudflare/workers-runtime-1
 package.json @cloudflare/wrangler @cloudflare/workers-runtime-1
 pnpm-lock.yaml @cloudflare/wrangler @cloudflare/workers-runtime-1
-types/ @cloudflare/wrangler
-src/cloudflare/ @cloudflare/wrangler
+/types/ @cloudflare/wrangler
+/src/cloudflare/ @cloudflare/wrangler
 src/workerd/tools/ @cloudflare/wrangler @cloudflare/workers-runtime-1 @cloudflare/workers-durable-objects
 src/workerd/io/supported-compatibility-date.txt @cloudflare/wrangler @cloudflare/workers-runtime-1
 src/node/ @cloudflare/workers-runtime-1 @cloudflare/workers-durable-objects @cloudflare/workers-frameworks

build/python/packages_20241218.bzl renamed to build/python/packages_20250324_1.bzl

@@ -1,10 +1,13 @@
 # This file is automatically generated by the Pyodide build script repo
 # (https://github.com/cloudflare/pyodide-build-scripts) and should not be manually modified.
 
-PACKAGES_20241218 = {
+PACKAGES_20250324_1 = {
     "PyJWT": [
         "jwt",
     ],
+    "aiohappyeyeballs": [
+        "aiohappyeyeballs",
+    ],
     "aiohttp": [
         "aiohttp",
     ],
@@ -125,11 +128,50 @@ PACKAGES_20241218 = {
     "packaging": [
         "packaging",
     ],
+    "propcache": [
+        "propcache",
+    ],
     "pycparser": [
         "pycparser",
     ],
     "pydantic": [
         "pydantic",
+        "pydantic.alias_generators",
+        "pydantic.aliases",
+        "pydantic.annotated_handlers",
+        "pydantic.class_validators",
+        "pydantic.color",
+        "pydantic.config",
+        "pydantic.dataclasses",
+        "pydantic.datetime_parse",
+        "pydantic.decorator",
+        "pydantic.deprecated",
+        "pydantic.env_settings",
+        "pydantic.error_wrappers",
+        "pydantic.errors",
+        "pydantic.experimental",
+        "pydantic.fields",
+        "pydantic.functional_serializers",
+        "pydantic.functional_validators",
+        "pydantic.generics",
+        "pydantic.json",
+        "pydantic.json_schema",
+        "pydantic.main",
+        "pydantic.networks",
+        "pydantic.parse",
+        "pydantic.plugin",
+        "pydantic.root_model",
+        "pydantic.schema",
+        "pydantic.tools",
+        "pydantic.type_adapter",
+        "pydantic.types",
+        "pydantic.typing",
+        "pydantic.utils",
+        "pydantic.v1",
+        "pydantic.validate_call_decorator",
+        "pydantic.validators",
+        "pydantic.version",
+        "pydantic.warnings",
     ],
     "pydantic_core": [
         "pydantic_core",

build/python_metadata.bzl

@@ -1,5 +1,5 @@
 load("//:build/python/packages_20240829_4.bzl", "PACKAGES_20240829_4")
-load("//:build/python/packages_20241218.bzl", "PACKAGES_20241218")
+load("//:build/python/packages_20250324_1.bzl", "PACKAGES_20250324_1")
 
 # The below is a list of pyodide-lock.json files for each package bundle version that we support.
 # Each of these gets embedded in the workerd and EW binary.
@@ -8,15 +8,15 @@ load("//:build/python/packages_20241218.bzl", "PACKAGES_20241218")
 # the lock file.
 PYTHON_LOCKFILES = {
     "20240829.4": "c2d9c67ea55a672b95a3beb8d66bfbe7df736edb4bb657383b263151e7e85ef4",
-    "20241218": "1421e9351baf24ec44d82f78b9ac26e8e0e6595bfe3f626dedb33147bfcd1998",
+    "20250324.1": "3e5a9317dc0cfcf63e556034bf0e87b958bd6debcfdccdfffc8ce477cc439626",
 }
 
 # This is a dictionary mapping a Python version with its packages version.
 #
 # NOTE: this needs to be kept in sync with compatibility-date.capnp.
 PYTHON_VERSION_TO_PACKAGES = {
     "0.26.0a2": "20240829.4",
-    "0.27.1": "20241218",
+    "0.27.1": "20250324.1",
     "development": "20240829.4",
 }
 
@@ -32,7 +32,7 @@ PYTHON_VERSION_TO_PACKAGES = {
 # first.
 PYTHON_IMPORTS_TO_TEST = {
     "20240829.4": PACKAGES_20240829_4,
-    "20241218": PACKAGES_20241218,
+    "20250324.1": PACKAGES_20250324_1,
 }
 
 # Each new package bundle should contain the same packages as the previous. We verify this
@@ -42,9 +42,9 @@ def verify_no_packages_were_removed():
     for i in range(0, len(package_dates) - 1):
         curr_pkgs = PYTHON_IMPORTS_TO_TEST[package_dates[i]]
         next_pkgs = PYTHON_IMPORTS_TO_TEST[package_dates[i + 1]]
-        for pkg in curr_pkgs:
-            if pkg not in next_pkgs:
-                fail(pkg + " from packages version ", package_dates[i], " not in ", package_dates[i + 1])
+        missing_pkgs = [pkg for pkg in curr_pkgs if pkg not in next_pkgs]
+        if missing_pkgs:
+            fail(str(missing_pkgs) + " from packages version ", package_dates[i], " not in ", package_dates[i + 1])
 
 verify_no_packages_were_removed()
 
@@ -62,8 +62,8 @@ BUNDLE_VERSION_INFO = make_bundle_version_info([
         "name": "0.26.0a2",
         "pyodide_version": "0.26.0a2",
         "pyodide_date": "2024-03-01",
-        "backport": "21",
-        "integrity": "sha256-DTYjdnJ41I1Gq5BsUcgh4DmHcgq1CZlwvWYF/UbV2v4=",
+        "backport": "26",
+        "integrity": "sha256-PGbANe9AeoPAf2EO4BCrw0iUox+C7CgJLGR/79r04yM=",
         "feature_flags": [],
         "emscripten_version": "3.1.52",
         "python_version": "3.12.1",
@@ -73,8 +73,8 @@ BUNDLE_VERSION_INFO = make_bundle_version_info([
         "name": "0.27.1",
         "pyodide_version": "0.27.1",
         "pyodide_date": "2025-01-16",
-        "backport": "9",
-        "integrity": "sha256-4c+GXQ3lL83v7z2DR1HCNGHARAUWTu669GHmlt+xpB4=",
+        "backport": "14",
+        "integrity": "sha256-yJduukBGEXPOlWEZkg6gz0j31Kw1AHyy/t9+FYJcagw=",
         "feature_flags": ["pythonWorkers20250116"],
         "emscripten_version": "3.1.58",
         "python_version": "3.12.7",

src/pyodide/BUILD.bazel

@@ -113,7 +113,12 @@ const location = undefined;
 
 function addEventListener(){}
 
-function reportUndefinedSymbolsNoOp() {}
+function reportUndefinedSymbolsPatched(Module) {
+    if (Module.API.version === "0.26.0a2") {
+        return;
+    }
+    Module.reportUndefinedSymbols(undefined);
+}
 
 if (typeof FinalizationRegistry === "undefined") {
     globalThis.FinalizationRegistry = class FinalizationRegistry {
@@ -157,7 +162,7 @@ REPLACEMENTS = [
     ],
     [
         "reportUndefinedSymbols()",
-        "reportUndefinedSymbolsNoOp()",
+        "reportUndefinedSymbolsPatched(Module)",
     ],
     [
         "crypto.getRandomValues(",
@@ -177,9 +182,16 @@ REPLACEMENTS = [
         "eval(UTF8ToString(ptr))",
         "(() => {throw new Error('Internal Emscripten code tried to eval, this should not happen, please file a bug report with your requirements.txt file\\'s contents')})()",
     ],
+    # Dynamic linking patches:
+    # library lookup
+    [
+        "!libData",
+        "!(libData ??= patchDynlibLookup(Module, libName))",
+    ],
+    # for ensuring memory base of dynlib is stable when restoring snapshots
     [
-        "!libData&&flags.fs",
-        "!(libData ??= patchDynlibLookup(Module, libName))&&flags.fs",
+        "getMemory(",
+        "Module.getMemoryPatched(Module, libName, ",
     ],
 ]
 

src/pyodide/internal/loadPackage.ts

@@ -18,10 +18,8 @@ import {
   USING_OLDEST_PACKAGES_VERSION,
 } from 'pyodide-internal:metadata';
 import {
-  DYNLIB_PATH,
   VIRTUALIZED_DIR,
   STDLIB_PACKAGES,
-  getSitePackagesPath,
 } from 'pyodide-internal:setupPackages';
 import { parseTarInfo } from 'pyodide-internal:tar';
 import { default as DiskCache } from 'pyodide-internal:disk_cache';

src/pyodide/internal/metadatafs.ts

@@ -56,7 +56,11 @@ export function createMetadataFS(Module: Module): object {
       if (parent.tree == undefined) {
         throw new Error('cannot lookup directory, tree is undefined');
       }
-      return parent.tree.get(name)!;
+      const res = parent.tree.get(name);
+      if (res === undefined) {
+        throw new Module.FS.ErrnoError(44);
+      }
+      return res;
     },
     read(stream, position, buffer) {
       return MetadataReader.read(stream.node.index!, position, buffer);

src/pyodide/internal/pool/emscriptenSetup.ts

@@ -62,7 +62,9 @@ function getPrepareFileSystem(pythonStdlib: ArrayBuffer): PreRunHook {
     try {
       const pymajor = Module._py_version_major();
       const pyminor = Module._py_version_minor();
-      Module.FS.mkdirTree(`/lib/python${pymajor}.${pyminor}/site-packages`);
+      Module.FS.sitePackages = `/lib/python${pymajor}.${pyminor}/site-packages`;
+      Module.FS.sessionSitePackages = '/session' + Module.FS.sitePackages;
+      Module.FS.mkdirTree(Module.FS.sitePackages);
       Module.FS.writeFile(
         `/lib/python${pymajor}${pyminor}.zip`,
         new Uint8Array(pythonStdlib),

src/pyodide/internal/setupPackages.ts

@@ -6,13 +6,13 @@ import {
   LOAD_WHEELS_FROM_R2,
   LOCKFILE,
   LOAD_WHEELS_FROM_ARTIFACT_BUNDLER,
-  USING_OLDEST_PACKAGES_VERSION,
 } from 'pyodide-internal:metadata';
 import { simpleRunPython } from 'pyodide-internal:util';
 import { default as EmbeddedPackagesTarReader } from 'pyodide-internal:packages_tar_reader';
 import { default as MetadataReader } from 'pyodide-internal:runtime-generated/metadata';
 
 const canonicalizeNameRegex = /[-_.]+/g;
+const DYNLIB_PATH = '/usr/lib';
 
 /**
  * Canonicalize a package name. Port of Python's packaging.utils.canonicalize_name.
@@ -153,8 +153,13 @@ class VirtualizedDir {
   }
 
   mount(Module: Module, tarFS: EmscriptenFS<TarFSInfo>) {
-    const path = getSitePackagesPath(Module);
-    Module.FS.mount(tarFS, { info: this.rootInfo }, path);
+    Module.FS.mkdirTree(Module.FS.sessionSitePackages);
+    Module.FS.mount(
+      tarFS,
+      { info: this.rootInfo },
+      Module.FS.sessionSitePackages
+    );
+    Module.FS.mkdirTree(DYNLIB_PATH);
     Module.FS.mount(tarFS, { info: this.dynlibTarFs }, DYNLIB_PATH);
   }
 }
@@ -215,14 +220,6 @@ function disabledLoadPackage(): never {
   );
 }
 
-export function getSitePackagesPath(Module: Module): string {
-  const pymajor = Module._py_version_major();
-  const pyminor = Module._py_version_minor();
-  return `/session/lib/python${pymajor}.${pyminor}/site-packages`;
-}
-
-export const DYNLIB_PATH = '/usr/lib';
-
 /**
  * This mounts a TarFS representing the site-packages directory (which contains the Python packages)
  * and another TarFS representing the dynlib directory (where dynlibs like libcrypto.so live).
@@ -233,9 +230,6 @@ export const DYNLIB_PATH = '/usr/lib';
  */
 export function mountSitePackages(Module: Module, pkgs: VirtualizedDir): void {
   const tarFS = createTarFS(Module);
-  const site_packages = getSitePackagesPath(Module);
-  Module.FS.mkdirTree(site_packages);
-  Module.FS.mkdirTree(DYNLIB_PATH);
   if (!LOAD_WHEELS_FROM_R2 && !LOAD_WHEELS_FROM_ARTIFACT_BUNDLER) {
     // if we are not loading additional wheels, then we're done
     // with site-packages and we can mount it here. Otherwise, we must mount it in
@@ -262,55 +256,13 @@ export function mountWorkerFiles(Module: Module) {
  * Has to run after the runtime is initialized.
  */
 export function adjustSysPath(Module: Module): void {
-  const site_packages = getSitePackagesPath(Module);
+  const site_packages = Module.FS.sessionSitePackages;
   simpleRunPython(
     Module,
     `import sys; sys.path.append("/session/metadata"); sys.path.append("${site_packages}"); del sys`
   );
 }
 
-function recursiveDependencies(
-  lockfile: PackageLock,
-  names: string[]
-): PackageDeclaration[] {
-  const toLoad = new Map();
-  for (const name of names) {
-    addPackageToLoad(lockfile, name, toLoad);
-  }
-  return Array.from(toLoad.values());
-}
-
-/**
- * Recursively add a package and its dependencies to toLoad.
- * A helper function for recursiveDependencies.
- * @param name The package to add
- * @param toLoad The set of names of packages to load
- * @private
- */
-function addPackageToLoad(
-  lockfile: PackageLock,
-  name: string,
-  toLoad: Map<string, PackageDeclaration>
-): void {
-  const normalizedName = canonicalizePackageName(name);
-  if (toLoad.has(normalizedName)) {
-    return;
-  }
-  const pkgInfo = lockfile.packages[normalizedName];
-  if (!pkgInfo) {
-    throw new Error(
-      `It appears that a package ("${name}") you requested is not available yet in workerd. \n` +
-        'If you would like this package to be included, please open an issue at https://github.com/cloudflare/workerd/discussions/new?category=python-packages.'
-    );
-  }
-
-  toLoad.set(normalizedName, pkgInfo);
-
-  for (let depName of pkgInfo.depends) {
-    addPackageToLoad(lockfile, depName, toLoad);
-  }
-}
-
 export { REQUIREMENTS };
 export const TRANSITIVE_REQUIREMENTS =
   MetadataReader.getTransitiveRequirements();