Python: Update bazel to be able to build both 0.27.1 and 0.26.0a2

Author: hoodmane

.github/workflows/release-python-runtime.yml

@@ -1,14 +1,6 @@
 name: Build Python Runtime
 
-on:
-  workflow_dispatch:
-    inputs:
-      pyodide:
-        description: The Pyodide version
-      pyodideRevision:
-        description: The Pyodide revision date
-      backport:
-        description: The Pyodide release backport number
+on: workflow_dispatch
 
 jobs:
   build:
@@ -36,17 +28,14 @@ jobs:
             # Strip comment in front of WORKERS_MIRROR_URL, then substitute secret to use it.
             sed -e '/WORKERS_MIRROR_URL/ { s@# *@@; s@WORKERS_MIRROR_URL@${{ secrets.WORKERS_MIRROR_URL }}@; }' -i.bak WORKSPACE
           fi
-      - name: Bazel build
-        run: |
-          bazel build //src/pyodide:pyodide.capnp.bin@rule --disk_cache=~/bazel-disk-cache --strip=always --remote_cache=https://bazel:${{ secrets.BAZEL_CACHE_KEY }}@bazel-remote-cache.devprod.cloudflare.dev --config=ci --config=release_linux
-          cp bazel-bin/src/pyodide/${{ inputs.pyodide }}/generated/pyodide.capnp.bin .
-      - name: Upload Pyodide capnproto bundle
+      - name: Build and upload Pyodide capnproto bundle
         env:
           R2_ACCOUNT_ID: ${{ secrets.PYODIDE_CAPNP_R2_ACCOUNT_ID }}
           R2_ACCESS_KEY_ID: ${{ secrets.PYODIDE_CAPNP_R2_ACCESS_KEY_ID }}
           R2_SECRET_ACCESS_KEY: ${{ secrets.PYODIDE_CAPNP_R2_SECRET_ACCESS_KEY }}
         run: |
+          bazel build @workerd//src/pyodide:python_bundles @workerd//src/pyodide:bundle_version_info --remote_cache=https://bazel:${{ secrets.BAZEL_CACHE_KEY }}@bazel-remote-cache.devprod.cloudflare.dev
           # boto3 v1.36.0 fails with:
           # NotImplemented error occurred in CreateMultipartUpload operation: Header 'x-amz-checksum-algorithm' with value 'CRC32' not implemented
           pip install 'boto3<1.36.0'
-          python3 src/pyodide/upload_bundle.py pyodide.capnp.bin pyodide_${{ inputs.pyodide }}_${{ inputs.pyodideRevision }}_${{ inputs.backport }}.capnp.bin
+          python3 src/pyodide/upload_bundles.py

build/deps/dep_pyodide.bzl

@@ -1,15 +1,18 @@
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
 load("//:build/pyodide_bucket.bzl", "PYODIDE_ALL_WHEELS_ZIP_SHA256", "PYODIDE_GITHUB_RELEASE_URL")
-load("//:build/python_metadata.bzl", "PYTHON_LOCKFILES")
+load("//:build/python_metadata.bzl", "PYODIDE_VERSIONS", "PYTHON_LOCKFILES")
 
 def dep_pyodide():
     # Use @workerd prefix on build_file so we can use this from edgeworker too
-    http_archive(
-        name = "pyodide",
-        build_file = "@workerd//:build/BUILD.pyodide",
-        sha256 = "fbda450a64093a8d246c872bb901ee172a57fe594c9f35bba61f36807c73300d",
-        urls = ["https://github.com/pyodide/pyodide/releases/download/0.26.0a2/pyodide-core-0.26.0a2.tar.bz2"],
-    )
+    for info in PYODIDE_VERSIONS:
+        version = info["version"]
+        sha256 = info["sha256"]
+        http_archive(
+            name = "pyodide-%s" % version,
+            build_file = "@workerd//:build/BUILD.pyodide",
+            sha256 = sha256,
+            urls = ["https://github.com/pyodide/pyodide/releases/download/%s/pyodide-core-%s.tar.bz2" % (version, version)],
+        )
 
     for package_date, package_lock_sha in PYTHON_LOCKFILES.items():
         http_file(

build/python_metadata.bzl

@@ -1,6 +1,17 @@
 load("//:build/python/packages_20240829_4.bzl", "PACKAGES_20240829_4")
 load("//:build/python/packages_20250324_1.bzl", "PACKAGES_20250324_1")
 
+PYODIDE_VERSIONS = [
+    {
+        "version": "0.26.0a2",
+        "sha256": "fbda450a64093a8d246c872bb901ee172a57fe594c9f35bba61f36807c73300d",
+    },
+    {
+        "version": "0.27.1",
+        "sha256": "6e45f93c71ed21bff4a06d6d9d8e27e815269dabf8ab34dc400939fd45edc665",
+    },
+]
+
 # This is the list of all the package metadata that we use.
 #
 # IMPORTANT: packages that are present here should never be removed after the package version is
@@ -36,12 +47,15 @@ def verify_no_packages_were_removed():
 
 verify_no_packages_were_removed()
 
+def _bundle_id(*, pyodide_version, pyodide_date, backport, **_kwds):
+    return "%s_%s_%s" % (pyodide_version, pyodide_date, backport)
+
 def make_bundle_version_info(versions):
     result = {}
     for entry in versions:
         name = entry["name"]
         if entry["name"] != "development":
-            entry["id"] = entry["pyodide_version"] + "_" + entry["pyodide_date"] + "_" + entry["backport"]
+            entry["id"] = _bundle_id(**entry)
         result[name] = entry
     return result
 
@@ -53,7 +67,7 @@ BUNDLE_VERSION_INFO = make_bundle_version_info([
         "pyodide_version": "0.26.0a2",
         "pyodide_date": "2024-03-01",
         "packages": "20240829.4",
-        "backport": "28",
+        "backport": 28,
         "integrity": "sha256-a2F/YpfjuGsVIsx5Z/WSbAaWSHL9vO13puPxX+wKwNI=",
         "feature_flags": [],
         "emscripten_version": "3.1.52",
@@ -65,7 +79,7 @@ BUNDLE_VERSION_INFO = make_bundle_version_info([
         "pyodide_version": "0.27.1",
         "pyodide_date": "2025-01-16",
         "packages": "20250324.1",
-        "backport": "16",
+        "backport": 16,
         "integrity": "sha256-znvAe5OAidJcQyeWeMFClJ5LPgP0QPztxZW/sogb1wI=",
         "feature_flags": ["pythonWorkers20250116"],
         "emscripten_version": "3.1.58",

src/pyodide/BUILD.bazel

@@ -1,8 +1,9 @@
 load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
+load("@bazel_skylib//rules:write_file.bzl", "write_file")
 load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library")
 load("//:build/capnp_embed.bzl", "capnp_embed")
-load("//:build/python_metadata.bzl", "PYTHON_LOCKFILES")
-load(":helpers.bzl", "copy_and_capnp_embed", "copy_to_generated", "python_bundle")
+load("//:build/python_metadata.bzl", "BUNDLE_VERSION_INFO", "PYTHON_LOCKFILES")
+load(":helpers.bzl", "copy_and_capnp_embed", "copy_to_generated", "python_bundles")
 
 copy_and_capnp_embed("python-entrypoint.js")
 
@@ -47,7 +48,7 @@ cc_capnp_library(
     ],
 )
 
-python_bundle("0.26.0a2")
+python_bundles()
 
 alias(
     name = "pyodide.capnp.bin",
@@ -66,3 +67,9 @@ alias(
     actual = "pyodide@0.26.0a2",
     visibility = ["//visibility:public"],
 )
+
+write_file(
+    name = "bundle_version_info",
+    out = "bundle_version_info.json",
+    content = [json.encode(BUNDLE_VERSION_INFO)],
+)

src/pyodide/helpers.bzl

@@ -4,6 +4,7 @@ load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
 load("@bazel_skylib//rules:write_file.bzl", "write_file")
 load("//:build/capnp_embed.bzl", "capnp_embed")
 load("//:build/js_file.bzl", "js_file")
+load("//:build/python_metadata.bzl", "PYODIDE_VERSIONS", "PYTHON_LOCKFILES")
 load("//:build/wd_ts_bundle.bzl", "wd_ts_bundle")
 
 def _out_name(src):
@@ -43,15 +44,28 @@ def copy_and_capnp_embed(src, version = None, out_name = None):
         deps = [out_name + "@copy"],
     )
 
-def python_bundle(version, pyodide_asm_wasm = None, pyodide_asm_js = None, python_stdlib_zip = None, emscripten_setup_override = None):
+def python_bundles(overrides = {}):
+    srcs = [_python_bundle_helper(info, overrides) for info in PYODIDE_VERSIONS]
+    native.filegroup(
+        name = "python_bundles",
+        srcs = srcs + [":bundle_version_info"],
+    )
+
+def _python_bundle_helper(info, overrides):
+    version = info["version"]
+    override = overrides.get(version, {})
+    return _python_bundle(version, **override)
+
+def _python_bundle(version, *, pyodide_asm_wasm = None, pyodide_asm_js = None, python_stdlib_zip = None, emscripten_setup_override = None):
+    pyodide_package = "@pyodide-%s//" % version
     if not pyodide_asm_wasm:
-        pyodide_asm_wasm = "@pyodide//:pyodide/pyodide.asm.wasm"
+        pyodide_asm_wasm = pyodide_package + ":pyodide/pyodide.asm.wasm"
 
     if not pyodide_asm_js:
-        pyodide_asm_js = "@pyodide//:pyodide/pyodide.asm.js"
+        pyodide_asm_js = pyodide_package + ":pyodide/pyodide.asm.js"
 
     if not python_stdlib_zip:
-        python_stdlib_zip = "@pyodide//:pyodide/python_stdlib.zip"
+        python_stdlib_zip = pyodide_package + ":pyodide/python_stdlib.zip"
 
     copy_and_capnp_embed("python-entrypoint.js", version = version)
 
@@ -243,8 +257,9 @@ def python_bundle(version, pyodide_asm_wasm = None, pyodide_asm_js = None, pytho
         out_dir = _out_path("", version),
     )
 
+    pyodide_cappn_bin_rule = "pyodide.capnp.bin@rule@" + version
     native.genrule(
-        name = "pyodide.capnp.bin@rule@" + version,
+        name = pyodide_cappn_bin_rule,
         srcs = [
             ":pyodide@%s.capnp" % version,
             "//src/workerd/jsg:modules.capnp",
@@ -287,3 +302,4 @@ def python_bundle(version, pyodide_asm_wasm = None, pyodide_asm_js = None, pytho
         tools = ["@capnp-cpp//src/capnp:capnp_tool"],
         visibility = ["//visibility:public"],
     )
+    return pyodide_cappn_bin_rule

src/pyodide/upload_bundle.py

@@ -0,0 +1,88 @@
+import json
+import subprocess
+import sys
+from copy import deepcopy
+from functools import cache
+from os import environ
+from pathlib import Path
+
+import requests
+from boto3 import client
+
+
+def cquery(rule):
+    res = subprocess.run(
+        [
+            "bazel",
+            "cquery",
+            rule,
+            "--output=files",
+        ],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if res.returncode:
+        print(res.stdout)
+        print(res.stderr)
+        sys.exit(res.returncode)
+    return res.stdout.strip()
+
+
+@cache
+def _bundle_version_info():
+    with Path(cquery("@workerd//src/pyodide:bundle_version_info")).open() as f:
+        return json.load(f)
+
+
+def bundle_version_info():
+    return deepcopy(_bundle_version_info())
+
+
+def get_pyodide_bin_path(ver):
+    return cquery(f"@workerd//src/pyodide:pyodide@{ver}.capnp")
+
+
+def bundle_key(*, pyodide_version, pyodide_date, backport, **_kwds):
+    return f"pyodide_{pyodide_version}_{pyodide_date}_{backport}.capnp.bin"
+
+
+def bundle_url(**kwds):
+    return "https://pyodide-capnp-bin.edgeworker.net/" + bundle_key(**kwds)
+
+
+def get_backport(ver):
+    info = bundle_version_info()[ver]
+    backport = int(info["backport"])
+    for b in range(backport + 1, backport + 10):
+        info["backport"] = b
+        url = bundle_url(**info)
+        res = requests.head(url)
+        if res.status_code == 404:
+            return b
+        res.raise_for_status()
+
+
+def main():
+    s3 = client(
+        "s3",
+        endpoint_url=f"https://{environ['R2_ACCOUNT_ID']}.r2.cloudflarestorage.com",
+        aws_access_key_id=environ["R2_ACCESS_KEY_ID"],
+        aws_secret_access_key=environ["R2_SECRET_ACCESS_KEY"],
+        region_name="auto",
+    )
+
+    for ver, info in bundle_version_info().items():
+        if ver.startswith("dev"):
+            continue
+        path = Path(get_pyodide_bin_path(ver)).resolve()
+        b = get_backport(ver)
+        info["backport"] = b
+        key = bundle_key(**info)
+        print(f"Uploading version {ver} backport {b}")
+        s3.upload_file(str(path), "pyodide-capnp-bin", key)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())