Merge pull request #2 from cloudflare/ggu/generate_import_tests

Generate import tests and include them in pyodide_bucket.bzl

Author: garrettgu10

packages/import_tests.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+
+from typing import List
+import yaml
+
+def gen(packages: List[str], packages_dir = Path("packages")):
+    res = {}
+    for package in packages:
+        if package == "test": continue
+        try:
+            with open(packages_dir / package / "meta.yaml", "r") as file:
+                imports = set()
+                meta = yaml.load(file, Loader=yaml.FullLoader)
+                # add package -> top-level if it exists
+                if "package" in meta:
+                    if "top-level" in meta["package"]:
+                        imports.update(meta["package"]["top-level"])
+                # add test -> imports if it exists
+                if "test" in meta:
+                    if "imports" in meta["test"]:
+                        imports.update(meta["test"]["imports"])
+                
+                if len(imports) > 0:
+                    res[package] = list(imports)
+        except FileNotFoundError:
+            print(f"package {package}'s meta.yaml not found")
+    return res

packages/script.py

@@ -11,6 +11,8 @@
 import hashlib
 from datetime import datetime
 
+import import_tests
+
 def normalize(name):
     return re.sub(r"[-_.]+", "-", name).lower()
 
@@ -27,6 +29,11 @@ def gen_bzl_config(tag, dist):
     all_wheels_bytes = (dist / "all_wheels.zip").read_bytes()
     all_wheels_hash = hashlib.sha256(all_wheels_bytes).hexdigest()
 
+    with open(dist / "pyodide-lock.json", "r") as file:
+        lock = json.load(file)
+        packages = [package["name"] for package in lock["packages"].values()]
+        imports_to_test = import_tests.gen(packages)
+
     with open("pyodide_bucket.bzl", "w") as f:
         f.write("# Do not edit this file by hand. See docs/pyodide.md for info on how to generate it.\n")
         f.write("# These variables are factored out here because they are being shared by the WORKSPACE files in\n")
@@ -35,7 +42,9 @@ def gen_bzl_config(tag, dist):
         f.write("PYODIDE_GITHUB_RELEASE_URL = \"" + github_url + "\"\n")
         f.write("PYODIDE_LOCK_SHA256 = \"" + lock_hash + "\"\n")
         f.write("PYODIDE_PACKAGES_TAR_ZIP_SHA256 = \"" + zip_hash + "\"\n")
-        f.write("PYODIDE_ALL_WHEELS_ZIP_SHA256 = \"" + all_wheels_hash + "\"\n")
+        f.write("PYODIDE_ALL_WHEELS_ZIP_SHA256 = \"" + all_wheels_hash + "\"\n\n")
+        f.write("# IMPORTANT: when updating this file in git, check the diff to make sure none of the imports below are being removed unexpectedly\n")
+        f.write("PYODIDE_IMPORTS_TO_TEST = " + json.dumps(imports_to_test, indent=3, sort_keys=True) + "\n")
 
 # creates a package bundle .tar.zip file to be bundled in with edgeworker
 # the resulting bundle is written to dist/pyodide_packages.tar.zip