feat(bazel): migrate pyodide to bzlmod

Author: alexeagle

WORKSPACE

@@ -1,5 +1 @@
 workspace(name = "workerd")
-
-load("//build/deps:dep_pyodide.bzl", "dep_pyodide")
-
-dep_pyodide()

build/deps/dep_pyodide.bzl

@@ -53,38 +53,6 @@ def _capnp_bundle(id = None, integrity = None, **_kwds):
         url = PYODIDE_CAPN_BIN + "pyodide_%s.capnp.bin" % id,
     )
 
-def _snapshot_http_file(bundle_name, folder, snapshot, integrity, hash, r2_base = PYODIDE_CAPN_BIN):
-    if not snapshot:
-        return
-    if not integrity:
-        fail("Snapshot %s from bundle %s has missing integrity" % (snapshot, bundle_name))
-    if folder == "baseline-snapshot/":
-        key = hash
-    else:
-        key = snapshot
-    http_file(
-        name = "pyodide-snapshot-" + snapshot,
-        integrity = integrity,
-        url = r2_base + folder + key,
-    )
-
-def _snapshot_http_files(
-        name,
-        baseline_snapshot = None,
-        baseline_snapshot_hash = None,
-        baseline_snapshot_integrity = None,
-        numpy_snapshot = None,
-        numpy_snapshot_integrity = None,
-        fastapi_snapshot = None,
-        fastapi_snapshot_integrity = None,
-        dedicated_fastapi_snapshot = None,
-        dedicated_fastapi_snapshot_integrity = None,
-        **_kwds):
-    _snapshot_http_file(name, "baseline-snapshot/", baseline_snapshot, baseline_snapshot_integrity, baseline_snapshot_hash)
-    _snapshot_http_file(name, "test-snapshot/", numpy_snapshot, numpy_snapshot_integrity, None)
-    _snapshot_http_file(name, "test-snapshot/", fastapi_snapshot, fastapi_snapshot_integrity, None)
-    _snapshot_http_file(name, "", dedicated_fastapi_snapshot, dedicated_fastapi_snapshot_integrity, None, VENDOR_R2)
-
 def dep_pyodide():
     for info in PYODIDE_VERSIONS:
         _pyodide_core(**info)
@@ -97,5 +65,50 @@ def dep_pyodide():
     for info in PYTHON_LOCKFILES:
         _pyodide_packages(**info)
 
+    # Accumulator to de-duplicate generated http_file rules.
+    gen_http_file = {}
+
+    def _snapshot_http_files(
+            name,
+            baseline_snapshot = None,
+            baseline_snapshot_hash = None,
+            baseline_snapshot_integrity = None,
+            numpy_snapshot = None,
+            numpy_snapshot_integrity = None,
+            fastapi_snapshot = None,
+            fastapi_snapshot_integrity = None,
+            dedicated_fastapi_snapshot = None,
+            dedicated_fastapi_snapshot_integrity = None,
+            **_kwds):
+        def _snapshot_http_file(bundle_name, folder, snapshot, integrity, hash, r2_base = PYODIDE_CAPN_BIN):
+            if not snapshot:
+                return
+            if not integrity:
+                fail("Snapshot %s from bundle %s has missing integrity" % (snapshot, bundle_name))
+            if folder == "baseline-snapshot/":
+                key = hash
+            else:
+                key = snapshot
+            gen_http_file[snapshot] = struct(
+                integrity = integrity,
+                url = r2_base + folder + key,
+            )
+
+        _snapshot_http_file(name, "baseline-snapshot/", baseline_snapshot, baseline_snapshot_integrity, baseline_snapshot_hash)
+        _snapshot_http_file(name, "test-snapshot/", numpy_snapshot, numpy_snapshot_integrity, None)
+        _snapshot_http_file(name, "test-snapshot/", fastapi_snapshot, fastapi_snapshot_integrity, None)
+        _snapshot_http_file(name, "", dedicated_fastapi_snapshot, dedicated_fastapi_snapshot_integrity, None, VENDOR_R2)
+
     for ver in BUNDLE_VERSION_INFO.values():
         _snapshot_http_files(**ver)
+    for snapshot, info in gen_http_file.items():
+        http_file(
+            name = "pyodide-snapshot-" + snapshot,
+            integrity = info.integrity,
+            url = info.url,
+        )
+
+def _impl(module_ctx):
+    dep_pyodide()
+
+pyodide = module_extension(implementation = _impl)

build/deps/python.MODULE.bazel

@@ -18,3 +18,57 @@ pip.parse(
     requirements_lock = "//build/deps:requirements.txt",
 )
 use_repo(pip, "py_deps", "v8_python_deps")
+
+pyodide = use_extension("//build/deps:dep_pyodide.bzl", "pyodide")
+
+[
+    use_repo(
+        pyodide,
+        repo.format(version = version),
+    )
+    for repo in [
+        "beautifulsoup4_src_{version}",
+        "fastapi_src_{version}",
+        "pyodide-{version}",
+        "python-workers-runtime-sdk_src_{version}",
+        "shapely_src_{version}",
+    ]
+    for version in [
+        "0.26.0a2",
+        "0.28.2",
+        "development",
+    ]
+    # Exclusions: some packages are absent in some versions
+    if repo.format(version = version) not in [
+        "pyodide-development",
+        "shapely_src_0.26.0a2",
+    ]
+]
+
+[
+    use_repo(
+        pyodide,
+        repo.format(version = version),
+    )
+    for repo in [
+        "all_pyodide_wheels_{version}",
+        "pyodide-lock_{version}.json",
+    ]
+    for version in [
+        "20240829.4",
+        "20250808",
+    ]
+]
+
+use_repo(
+    pyodide,
+    "pyodide-snapshot-baseline-4569679fb.bin",
+    "pyodide-snapshot-baseline-61eedf943.bin",
+    "pyodide-snapshot-ew-py-package-snapshot_fastapi-v2.bin",
+    "pyodide-snapshot-ew-py-package-snapshot_numpy-v2.bin",
+    "pyodide-snapshot-package_snapshot_fastapi-a6ccb56fe.bin",
+    "pyodide-snapshot-package_snapshot_numpy-60c9cb28e.bin",
+    "pyodide-snapshot-snapshot_a6b652a95810783f5078b9a5dbd4a07c30718acb4ff724e82c25db7353dd7f2d.bin",
+    "pyodide_0.26.0a2_2024-03-01_75.capnp.bin",
+    "pyodide_0.28.2_2025-01-16_6.capnp.bin",
+)

build/is_pyodide_bzlmod.bzl

@@ -0,0 +1,2 @@
+# workerd uses bzlmod for pyodide (and everything else)
+is_pyodide_bzlmod = True

src/workerd/server/tests/python/py_wd_test.bzl

@@ -1,3 +1,4 @@
+load("@//:build/is_pyodide_bzlmod.bzl", "is_pyodide_bzlmod")
 load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
 load("//:build/python_metadata.bzl", "BUNDLE_VERSION_INFO")
@@ -32,7 +33,15 @@ def _py_wd_test_helper(
 
     pkg_tag = BUNDLE_VERSION_INFO[python_flag]["packages"]
     data = data + ["@all_pyodide_wheels_%s//:whls" % pkg_tag]
-    args = args + ["--pyodide-package-disk-cache-dir", "../all_pyodide_wheels_%s" % pkg_tag]
+    args = args + ["--pyodide-package-disk-cache-dir"]
+
+    # TODO(cleanup): We support both mangled and non-mangled wheels paths for now, clean up once
+    # downstream repo fully uses bzlmod
+    if is_pyodide_bzlmod:
+        # +pyodide+ is a bzlmod canonical repository name
+        args.append("../+pyodide+all_pyodide_wheels_%s" % pkg_tag)
+    else:
+        args.append("../all_pyodide_wheels_%s" % pkg_tag)
 
     load_snapshot = None
     pyodide_version = BUNDLE_VERSION_INFO[python_flag]["real_pyodide_version"]