feat: support GraalPy (#1538)

* Properly close files used for testing

* Add support for GraalPy

* Help GraalPy discover build tools on Windows

* Expect manylinux-interpreters ensure graalpy* warning in pip

* Workaround GraalPy bugs on Windows

* Workaround oracle/graalpython#491 also when uv is not available

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update azure-pipelines.yml

* Update azure-pipelines.yml

* refacotor: use pathlib.write_text

Signed-off-by: Henry Schreiner <[email protected]>

* Include GraalPy in docker_warmup and remove workaround for installing it late

---------

Signed-off-by: Henry Schreiner <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Henry Schreiner <[email protected]>

Author: 3 people

.github/workflows/test.yml

@@ -80,7 +80,7 @@ jobs:
       env:
         CIBW_ARCHS_MACOS: x86_64 universal2 arm64
         CIBW_BUILD_FRONTEND: 'build[uv]'
-        CIBW_ENABLE: "cpython-prerelease cpython-freethreading pypy"
+        CIBW_ENABLE: "cpython-prerelease cpython-freethreading pypy graalpy"
 
     - name: Run a sample build (GitHub Action, only)
       uses: ./

README.md

@@ -36,14 +36,15 @@ While cibuildwheel itself requires a recent Python version to run (we support th
 | PyPy 3.9 v7.3  | ✅ | ✅  | ✅  | N/A | N/A | ✅¹ | ✅¹  | ✅¹ | N/A | N/A | N/A | N/A | N/A |
 | PyPy 3.10 v7.3 | ✅ | ✅  | ✅  | N/A | N/A | ✅¹ | ✅¹  | ✅¹ | N/A | N/A | N/A | N/A | N/A |
 | PyPy 3.11 v7.3 | ✅ | ✅  | ✅  | N/A | N/A | ✅¹ | ✅¹  | ✅¹ | N/A | N/A | N/A | N/A | N/A |
+| GraalPy 24.2   | ✅ | ✅  | ✅  | N/A | N/A | ✅¹ | N/A  | ✅¹ | N/A | N/A | N/A | N/A | N/A |
 
-<sup>¹ PyPy is only supported for manylinux wheels.</sup><br>
+<sup>¹ PyPy & GraalPy are only supported for manylinux wheels.</sup><br>
 <sup>² Windows arm64 support is experimental.</sup><br>
 <sup>³ Free-threaded mode requires opt-in using [`CIBW_ENABLE`](https://cibuildwheel.pypa.io/en/stable/options/#enable).</sup><br>
 <sup>⁴ Experimental, not yet supported on PyPI, but can be used directly in web deployment. Use `--platform pyodide` to build.</sup><br>
 <sup>⁵ manylinux armv7l support is experimental. As there are no RHEL based image for this architecture, it's using an Ubuntu based image instead.</sup><br>
 
-- Builds manylinux, musllinux, macOS 10.9+ (10.13+ for Python 3.12+), and Windows wheels for CPython and PyPy
+- Builds manylinux, musllinux, macOS 10.9+ (10.13+ for Python 3.12+), and Windows wheels for CPython, PyPy, and GraalPy
 - Works on GitHub Actions, Azure Pipelines, Travis CI, AppVeyor, CircleCI, GitLab CI, and Cirrus CI
 - Bundles shared library dependencies on Linux and macOS through [auditwheel](https://github.com/pypa/auditwheel) and [delocate](https://github.com/matthew-brett/delocate)
 - Runs your library's tests against the wheel-installed version of your library

azure-pipelines.yml

@@ -6,7 +6,7 @@ pr:
 
 jobs:
 - job: linux_311
-  timeoutInMinutes: 120
+  timeoutInMinutes: 180
   pool: {vmImage: 'Ubuntu-22.04'}
   steps:
     - task: UsePythonVersion@0
@@ -20,6 +20,7 @@ jobs:
 
 - job: macos_311
   pool: {vmImage: 'macOS-13'}
+  timeoutInMinutes: 120
   steps:
     - task: UsePythonVersion@0
       inputs:

bin/update_pythons.py

@@ -5,6 +5,7 @@
 import difflib
 import logging
 import operator
+import re
 import tomllib
 from collections.abc import Mapping, MutableMapping
 from pathlib import Path
@@ -44,13 +45,19 @@ class ConfigWinPP(TypedDict):
     url: str
 
 
+class ConfigWinGP(TypedDict):
+    identifier: str
+    version: str
+    url: str
+
+
 class ConfigApple(TypedDict):
     identifier: str
     version: str
     url: str
 
 
-AnyConfig = ConfigWinCP | ConfigWinPP | ConfigApple
+AnyConfig = ConfigWinCP | ConfigWinPP | ConfigWinGP | ConfigApple
 
 
 # The following set of "Versions" classes allow the initial call to the APIs to
@@ -106,6 +113,72 @@ def update_version_windows(self, spec: Specifier) -> ConfigWinCP | None:
         )
 
 
+class GraalPyVersions:
+    def __init__(self) -> None:
+        response = requests.get("https://api.github.com/repos/oracle/graalpython/releases")
+        response.raise_for_status()
+
+        releases = response.json()
+        gp_version_re = re.compile(r"-(\d+\.\d+\.\d+)$")
+        cp_version_re = re.compile(r"Python (\d+\.\d+(?:\.\d+)?)")
+        for release in releases:
+            m = gp_version_re.search(release["tag_name"])
+            if m:
+                release["graalpy_version"] = Version(m.group(1))
+            m = cp_version_re.search(release["body"])
+            if m:
+                release["python_version"] = Version(m.group(1))
+
+        self.releases = [r for r in releases if "graalpy_version" in r and "python_version" in r]
+
+    def update_version(self, identifier: str, spec: Specifier) -> AnyConfig:
+        if "x86_64" in identifier or "amd64" in identifier:
+            arch = "x86_64"
+        elif "arm64" in identifier or "aarch64" in identifier:
+            arch = "aarch64"
+        else:
+            msg = f"{identifier} not supported yet on GraalPy"
+            raise RuntimeError(msg)
+
+        releases = [r for r in self.releases if spec.contains(r["python_version"])]
+        releases = sorted(releases, key=lambda r: r["graalpy_version"])
+
+        if not releases:
+            msg = f"GraalPy {arch} not found for {spec}!"
+            raise RuntimeError(msg)
+
+        release = releases[-1]
+        version = release["python_version"]
+        gpversion = release["graalpy_version"]
+
+        if "macosx" in identifier:
+            arch = "x86_64" if "x86_64" in identifier else "arm64"
+            config = ConfigApple
+            platform = "macos"
+        elif "win" in identifier:
+            arch = "aarch64" if "arm64" in identifier else "x86_64"
+            config = ConfigWinGP
+            platform = "windows"
+        else:
+            msg = "GraalPy provides downloads for macOS and Windows and is included for manylinux"
+            raise RuntimeError(msg)
+
+        arch = "amd64" if arch == "x86_64" else "aarch64"
+        ext = "zip" if "win" in identifier else "tar.gz"
+        (url,) = (
+            rf["browser_download_url"]
+            for rf in release["assets"]
+            if rf["name"].endswith(f"{platform}-{arch}.{ext}")
+            and rf["name"].startswith(f"graalpy-{gpversion.major}")
+        )
+
+        return config(
+            identifier=identifier,
+            version=f"{version.major}.{version.minor}",
+            url=url,
+        )
+
+
 class PyPyVersions:
     def __init__(self, arch_str: ArchStr):
         response = requests.get("https://downloads.python.org/pypy/versions.json")
@@ -294,6 +367,8 @@ def __init__(self) -> None:
 
         self.ios_cpython = CPythonIOSVersions()
 
+        self.graalpy = GraalPyVersions()
+
     def update_config(self, config: MutableMapping[str, str]) -> None:
         identifier = config["identifier"]
         version = Version(config["version"])
@@ -311,6 +386,8 @@ def update_config(self, config: MutableMapping[str, str]) -> None:
                     config_update = self.macos_pypy.update_version_macos(spec)
                 elif "macosx_arm64" in identifier:
                     config_update = self.macos_pypy_arm64.update_version_macos(spec)
+            elif identifier.startswith("gp"):
+                config_update = self.graalpy.update_version(identifier, spec)
         elif "t-win32" in identifier and identifier.startswith("cp"):
             config_update = self.windows_t_32.update_version_windows(spec)
         elif "win32" in identifier and identifier.startswith("cp"):
@@ -322,6 +399,8 @@ def update_config(self, config: MutableMapping[str, str]) -> None:
                 config_update = self.windows_64.update_version_windows(spec)
             elif identifier.startswith("pp"):
                 config_update = self.windows_pypy_64.update_version_windows(spec)
+            elif identifier.startswith("gp"):
+                config_update = self.graalpy.update_version(identifier, spec)
         elif "t-win_arm64" in identifier and identifier.startswith("cp"):
             config_update = self.windows_t_arm64.update_version_windows(spec)
         elif "win_arm64" in identifier and identifier.startswith("cp"):

cibuildwheel/logger.py

@@ -243,6 +243,8 @@ def build_description_from_identifier(identifier: str) -> str:
         build_description += "CPython"
     elif python_interpreter == "pp":
         build_description += "PyPy"
+    elif python_interpreter == "gp":
+        build_description += "GraalPy"
     else:
         msg = f"unknown python {python_interpreter!r}"
         raise Exception(msg)

cibuildwheel/platforms/macos.py

@@ -103,15 +103,15 @@ def get_python_configurations(
     # skip builds as required by BUILD/SKIP
     python_configurations = [c for c in python_configurations if build_selector(c.identifier)]
 
-    # filter-out some cross-compilation configs with PyPy:
+    # filter-out some cross-compilation configs with PyPy and GraalPy:
     # can't build arm64 on x86_64
     # rosetta allows to build x86_64 on arm64
     if platform.machine() == "x86_64":
         python_configurations_before = set(python_configurations)
         python_configurations = [
             c
             for c in python_configurations
-            if not (c.identifier.startswith("pp") and c.identifier.endswith("arm64"))
+            if not (c.identifier.startswith(("pp", "gp")) and c.identifier.endswith("arm64"))
         ]
         removed_elements = python_configurations_before - set(python_configurations)
         if removed_elements:
@@ -191,6 +191,22 @@ def install_pypy(tmp: Path, url: str) -> Path:
     return installation_path / "bin" / "pypy3"
 
 
+def install_graalpy(tmp: Path, url: str) -> Path:
+    graalpy_archive = url.rsplit("/", 1)[-1]
+    extension = ".tar.gz"
+    assert graalpy_archive.endswith(extension)
+    installation_path = CIBW_CACHE_PATH / graalpy_archive[: -len(extension)]
+    with FileLock(str(installation_path) + ".lock"):
+        if not installation_path.exists():
+            downloaded_archive = tmp / graalpy_archive
+            download(url, downloaded_archive)
+            installation_path.mkdir(parents=True)
+            # GraalPy top-folder name is inconsistent with archive name
+            call("tar", "-C", installation_path, "--strip-components=1", "-xzf", downloaded_archive)
+            downloaded_archive.unlink()
+    return installation_path / "bin" / "graalpy"
+
+
 def setup_python(
     tmp: Path,
     python_configuration: PythonConfiguration,
@@ -212,6 +228,8 @@ def setup_python(
 
     elif implementation_id.startswith("pp"):
         base_python = install_pypy(tmp, python_configuration.url)
+    elif implementation_id.startswith("gp"):
+        base_python = install_graalpy(tmp, python_configuration.url)
     else:
         msg = "Unknown Python implementation"
         raise ValueError(msg)

cibuildwheel/platforms/windows.py

@@ -123,6 +123,20 @@ def install_pypy(tmp: Path, arch: str, url: str) -> Path:
     return installation_path / "python.exe"
 
 
+def install_graalpy(tmp: Path, url: str) -> Path:
+    zip_filename = url.rsplit("/", 1)[-1]
+    extension = ".zip"
+    assert zip_filename.endswith(extension)
+    installation_path = CIBW_CACHE_PATH / zip_filename[: -len(extension)]
+    with FileLock(str(installation_path) + ".lock"):
+        if not installation_path.exists():
+            graalpy_zip = tmp / zip_filename
+            download(url, graalpy_zip)
+            # Extract to the parent directory because the zip file still contains a directory
+            extract_zip(graalpy_zip, installation_path.parent)
+    return installation_path / "bin" / "graalpy.exe"
+
+
 def setup_setuptools_cross_compile(
     tmp: Path,
     python_configuration: PythonConfiguration,
@@ -239,6 +253,8 @@ def setup_python(
     elif implementation_id.startswith("pp"):
         assert python_configuration.url is not None
         base_python = install_pypy(tmp, python_configuration.arch, python_configuration.url)
+    elif implementation_id.startswith("gp"):
+        base_python = install_graalpy(tmp, python_configuration.url or "")
     else:
         msg = "Unknown Python implementation"
         raise ValueError(msg)
@@ -314,6 +330,49 @@ def setup_python(
         setup_setuptools_cross_compile(tmp, python_configuration, python_libs_base, env)
         setup_rust_cross_compile(tmp, python_configuration, python_libs_base, env)
 
+    if implementation_id.startswith("gp"):
+        # GraalPy fails to discover compilers, setup the relevant environment
+        # variables. Adapted from
+        # https://github.com/microsoft/vswhere/wiki/Start-Developer-Command-Prompt
+        # Remove when https://github.com/oracle/graalpython/issues/492 is fixed.
+        vcpath = subprocess.check_output(
+            [
+                Path(os.environ["PROGRAMFILES(X86)"])
+                / "Microsoft Visual Studio"
+                / "Installer"
+                / "vswhere.exe",
+                "-products",
+                "*",
+                "-latest",
+                "-property",
+                "installationPath",
+            ],
+            text=True,
+        ).strip()
+        log.notice(f"Discovering Visual Studio for GraalPy at {vcpath}")
+        env.update(
+            dict(
+                [
+                    envvar.strip().split("=", 1)
+                    for envvar in subprocess.check_output(
+                        [
+                            f"{vcpath}\\Common7\\Tools\\vsdevcmd.bat",
+                            "-no_logo",
+                            "-arch=amd64",
+                            "-host_arch=amd64",
+                            "&&",
+                            "set",
+                        ],
+                        shell=True,
+                        text=True,
+                        env=env,
+                    )
+                    .strip()
+                    .split("\n")
+                ]
+            )
+        )
+
     return base_python, env
 
 
@@ -342,6 +401,7 @@ def build(options: Options, tmp_path: Path) -> None:
         for config in python_configurations:
             build_options = options.build_options(config.identifier)
             build_frontend = build_options.build_frontend or BuildFrontendConfig("build")
+
             use_uv = build_frontend.name == "build[uv]" and can_use_uv(config)
             log.build_start(config.identifier)
 
@@ -390,6 +450,22 @@ def build(options: Options, tmp_path: Path) -> None:
                     build_frontend, build_options.build_verbosity, build_options.config_settings
                 )
 
+                if (
+                    config.identifier.startswith("gp")
+                    and build_frontend.name == "build"
+                    and "--no-isolation" not in extra_flags
+                    and "-n" not in extra_flags
+                ):
+                    # GraalPy fails to discover its standard library when a venv is created
+                    # from a virtualenv seeded executable. See
+                    # https://github.com/oracle/graalpython/issues/491 and remove this once
+                    # fixed upstream.
+                    log.notice(
+                        "Disabling build isolation to workaround GraalPy bug. If the build fails, consider using pip or build[uv] as build frontend."
+                    )
+                    shell("graalpy -m pip install setuptools wheel", env=env)
+                    extra_flags = [*extra_flags, "-n"]
+
                 build_env = env.copy()
                 if pip_version is not None:
                     build_env["VIRTUALENV_PIP"] = pip_version
@@ -414,6 +490,7 @@ def build(options: Options, tmp_path: Path) -> None:
                 elif build_frontend.name == "build" or build_frontend.name == "build[uv]":
                     if use_uv and "--no-isolation" not in extra_flags and "-n" not in extra_flags:
                         extra_flags.append("--installer=uv")
+
                     call(
                         "python",
                         "-m",

cibuildwheel/resources/build-platforms.toml

@@ -18,6 +18,7 @@ python_configurations = [
   { identifier = "pp39-manylinux_x86_64", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" },
   { identifier = "pp310-manylinux_x86_64", version = "3.10", path_str = "/opt/python/pp310-pypy310_pp73" },
   { identifier = "pp311-manylinux_x86_64", version = "3.11", path_str = "/opt/python/pp311-pypy311_pp73" },
+  { identifier = "gp242-manylinux_x86_64", version = "3.11", path_str = "/opt/python/graalpy311-graalpy242_311_native" },
   { identifier = "cp38-manylinux_aarch64", version = "3.8", path_str = "/opt/python/cp38-cp38" },
   { identifier = "cp39-manylinux_aarch64", version = "3.9", path_str = "/opt/python/cp39-cp39" },
   { identifier = "cp310-manylinux_aarch64", version = "3.10", path_str = "/opt/python/cp310-cp310" },
@@ -57,6 +58,7 @@ python_configurations = [
   { identifier = "pp39-manylinux_aarch64", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" },
   { identifier = "pp310-manylinux_aarch64", version = "3.10", path_str = "/opt/python/pp310-pypy310_pp73" },
   { identifier = "pp311-manylinux_aarch64", version = "3.11", path_str = "/opt/python/pp311-pypy311_pp73" },
+  { identifier = "gp242-manylinux_aarch64", version = "3.11", path_str = "/opt/python/graalpy311-graalpy242_311_native" },
   { identifier = "pp38-manylinux_i686", version = "3.8", path_str = "/opt/python/pp38-pypy38_pp73" },
   { identifier = "pp39-manylinux_i686", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" },
   { identifier = "pp310-manylinux_i686", version = "3.10", path_str = "/opt/python/pp310-pypy310_pp73" },
@@ -143,6 +145,8 @@ python_configurations = [
   { identifier = "pp310-macosx_arm64", version = "3.10", url = "https://downloads.python.org/pypy/pypy3.10-v7.3.19-macos_arm64.tar.bz2" },
   { identifier = "pp311-macosx_x86_64", version = "3.11", url = "https://downloads.python.org/pypy/pypy3.11-v7.3.19-macos_x86_64.tar.bz2" },
   { identifier = "pp311-macosx_arm64", version = "3.11", url = "https://downloads.python.org/pypy/pypy3.11-v7.3.19-macos_arm64.tar.bz2" },
+  { identifier = "gp242-macosx_x86_64", version = "3.11", url = "https://github.com/oracle/graalpython/releases/download/graal-24.2.0/graalpy-24.2.0-macos-amd64.tar.gz" },
+  { identifier = "gp242-macosx_arm64", version = "3.11", url = "https://github.com/oracle/graalpython/releases/download/graal-24.2.0/graalpy-24.2.0-macos-aarch64.tar.gz" },
 ]
 
 [windows]
@@ -171,6 +175,7 @@ python_configurations = [
   { identifier = "pp39-win_amd64", version = "3.9", arch = "64", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.16-win64.zip" },
   { identifier = "pp310-win_amd64", version = "3.10", arch = "64", url = "https://downloads.python.org/pypy/pypy3.10-v7.3.19-win64.zip" },
   { identifier = "pp311-win_amd64", version = "3.11", arch = "64", url = "https://downloads.python.org/pypy/pypy3.11-v7.3.19-win64.zip" },
+  { identifier = "gp242-win_amd64", version = "3.11", arch = "64", url = "https://github.com/oracle/graalpython/releases/download/graal-24.2.0/graalpy-24.2.0-windows-amd64.zip" },
 ]
 
 [pyodide]