Python tests: load Pyodide bundle from disk cache

Instead of repeatedly downloading it from the network.

Author: hoodmane

build/deps/dep_pyodide.bzl

@@ -42,9 +42,18 @@ filegroup(
         url = VENDOR_R2 + name + pyver + "-vendored-for-ew-testing.zip",
     )
 
-SNAPSHOT_R2 = "https://pyodide-capnp-bin.edgeworker.net/"
+PYODIDE_CAPN_BIN = "https://pyodide-capnp-bin.edgeworker.net/"
+def _capnp_bundle(id = None, integrity = None, **_kwds):
+    if not id or not integrity:
+        return
+    http_file(
+        name = "pyodide_%s.capnp.bin" % id,
+        integrity = integrity,
+        url = PYODIDE_CAPN_BIN + "pyodide_%s.capnp.bin" % id,
+    )
+
 
-def _snapshot_http_file(bundle_name, folder, snapshot, integrity, hash, r2_base = SNAPSHOT_R2):
+def _snapshot_http_file(bundle_name, folder, snapshot, integrity, hash, r2_base = PYODIDE_CAPN_BIN):
     if not snapshot:
         return
     if not integrity:
@@ -81,6 +90,7 @@ def dep_pyodide():
         _pyodide_core(**info)
 
     for info in BUNDLE_VERSION_INFO.values():
+        _capnp_bundle(**info)
         for pkg in info["vendored_packages_for_tests"].values():
             _py_vendor_test_deps(version = info["name"], **pkg)
 

src/workerd/server/tests/python/py_wd_test.bzl

@@ -81,19 +81,45 @@ def _snapshot_file(snapshot):
     return [":" + snapshot]
 
 def _snapshot_files(
+        name,
         baseline_snapshot = None,
         numpy_snapshot = None,
         fastapi_snapshot = None,
         dedicated_fastapi_snapshot = None,
         **_kwds):
+    if name == "development":
+        return []
     result = []
     result += _snapshot_file(baseline_snapshot)
     result += _snapshot_file(numpy_snapshot)
     result += _snapshot_file(fastapi_snapshot)
     result += _snapshot_file(dedicated_fastapi_snapshot)
     return result
 
-def python_test_setup():
+def _snapshot_file_group():
+    snapshots = []
+    for x in BUNDLE_VERSION_INFO.values():
+        snapshots += _snapshot_files(**x)
+
+    native.filegroup(
+        name = "python_snapshots",
+        data = snapshots,
+        visibility = ["//visibility:public"],
+    )
+
+
+def _capnp_bundle(id, **_kwds):
+    if id == "dev":
+        return
+    name =  "pyodide_%s.capnp.bin" % id
+    copy_file(
+        name = name + "@rule",
+        src = "@%s//file" % name,
+        out = "pyodide-bundle-cache/" + name,
+        visibility = ["//visibility:public"],
+    )
+
+def _capnp_bundles_file_group():
     # pyodide_dev.capnp.bin represents a custom pyodide version "dev" that is generated
     # at build time using the latest contents of the src/pyodide directory.
     # This is used to run tests to ensure that they are always run against the latest build of
@@ -104,17 +130,15 @@ def python_test_setup():
         out = "pyodide-bundle-cache/pyodide_dev.capnp.bin",
         visibility = ["//visibility:public"],
     )
-    data = []
-    for x in BUNDLE_VERSION_INFO.values():
-        if x["name"] == "development":
-            continue
-        data += _snapshot_files(**x)
+    for info in BUNDLE_VERSION_INFO.values():
+        _capnp_bundle(**info)
 
-    native.filegroup(
-        name = "python_snapshots",
-        data = data,
-        visibility = ["//visibility:public"],
-    )
+def _capnp_rules():
+    return [":pyodide_%s.capnp.bin@rule" % info["id"] for info in BUNDLE_VERSION_INFO.values()]
+
+def python_test_setup():
+    _capnp_bundles_file_group()
+    _snapshot_file_group()
 
 def compute_python_flags(python_flags, skip_python_flags):
     if python_flags == "all":
@@ -154,7 +178,7 @@ def py_wd_test(
         name = directory
     elif name == None:
         name = src.removesuffix(".wd-test")
-    data += ["//src/workerd/server/tests/python:pyodide_dev.capnp.bin@rule"]
+    data += _capnp_rules()
     args = args + [
         "--pyodide-bundle-disk-cache-dir",
         "$(location //src/workerd/server/tests/python:pyodide_dev.capnp.bin@rule)/..",