GH-114809: Add support for macOS multi-arch builds with the JIT enabled (#131751)

Co-authored-by: Ronald Oussoren <[email protected]>
Co-authored-by: Brandt Bucher <[email protected]>

Author: 3 people

.github/workflows/jit.yml

@@ -113,7 +113,7 @@ jobs:
           find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete
           brew install llvm@${{ matrix.llvm }}
           export SDKROOT="$(xcrun --show-sdk-path)"
-          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }}
+          ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }}
           make all --jobs 4
           ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 

.gitignore

@@ -138,7 +138,7 @@ Tools/unicode/data/
 # hendrikmuhs/ccache-action@v1
 /.ccache
 /cross-build/
-/jit_stencils.h
+/jit_stencils*.h
 /platform
 /profile-clean-stamp
 /profile-run-stamp

Misc/NEWS.d/next/Core_and_Builtins/2025-03-26-04-55-25.gh-issue-114809.8rNyT7.rst

@@ -0,0 +1 @@
+Add support for macOS multi-arch builds with the JIT enabled

Tools/jit/_targets.py

@@ -25,6 +25,7 @@
 CPYTHON = TOOLS.parent
 PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h"
 TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c"
+ASYNCIO_RUNNER = asyncio.Runner()
 
 _S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection)
 _R = typing.TypeVar(
@@ -35,6 +36,7 @@
 @dataclasses.dataclass
 class _Target(typing.Generic[_S, _R]):
     triple: str
+    condition: str
     _: dataclasses.KW_ONLY
     alignment: int = 1
     args: typing.Sequence[str] = ()
@@ -188,7 +190,12 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]:
         return stencil_groups
 
     def build(
-        self, out: pathlib.Path, *, comment: str = "", force: bool = False
+        self,
+        out: pathlib.Path,
+        *,
+        comment: str = "",
+        force: bool = False,
+        stencils_h: str = "jit_stencils.h",
     ) -> None:
         """Build jit_stencils.h in the given directory."""
         if not self.stable:
@@ -197,14 +204,14 @@ def build(
             outline = "=" * len(warning)
             print("\n".join(["", outline, warning, request, outline, ""]))
         digest = f"// {self._compute_digest(out)}\n"
-        jit_stencils = out / "jit_stencils.h"
+        jit_stencils = out / stencils_h
         if (
             not force
             and jit_stencils.exists()
             and jit_stencils.read_text().startswith(digest)
         ):
             return
-        stencil_groups = asyncio.run(self._build_stencils())
+        stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils())
         jit_stencils_new = out / "jit_stencils.h.new"
         try:
             with jit_stencils_new.open("w") as file:
@@ -512,33 +519,40 @@ def get_target(host: str) -> _COFF | _ELF | _MachO:
     """Build a _Target for the given host "triple" and options."""
     target: _COFF | _ELF | _MachO
     if re.fullmatch(r"aarch64-apple-darwin.*", host):
-        target = _MachO(host, alignment=8, prefix="_")
+        condition = "defined(__aarch64__) && defined(__APPLE__)"
+        target = _MachO(host, condition, alignment=8, prefix="_")
     elif re.fullmatch(r"aarch64-pc-windows-msvc", host):
         args = ["-fms-runtime-lib=dll", "-fplt"]
-        target = _COFF(host, alignment=8, args=args)
+        condition = "defined(_M_ARM64)"
+        target = _COFF(host, condition, alignment=8, args=args)
     elif re.fullmatch(r"aarch64-.*-linux-gnu", host):
         args = [
             "-fpic",
             # On aarch64 Linux, intrinsics were being emitted and this flag
             # was required to disable them.
             "-mno-outline-atomics",
         ]
-        target = _ELF(host, alignment=8, args=args)
+        condition = "defined(__aarch64__) && defined(__linux__)"
+        target = _ELF(host, condition, alignment=8, args=args)
     elif re.fullmatch(r"i686-pc-windows-msvc", host):
         args = [
             "-DPy_NO_ENABLE_SHARED",
             # __attribute__((preserve_none)) is not supported
             "-Wno-ignored-attributes",
         ]
-        target = _COFF(host, args=args, prefix="_")
+        condition = "defined(_M_IX86)"
+        target = _COFF(host, condition, args=args, prefix="_")
     elif re.fullmatch(r"x86_64-apple-darwin.*", host):
-        target = _MachO(host, prefix="_")
+        condition = "defined(__x86_64__) && defined(__APPLE__)"
+        target = _MachO(host, condition, prefix="_")
     elif re.fullmatch(r"x86_64-pc-windows-msvc", host):
         args = ["-fms-runtime-lib=dll"]
-        target = _COFF(host, args=args)
+        condition = "defined(_M_X64)"
+        target = _COFF(host, condition, args=args)
     elif re.fullmatch(r"x86_64-.*-linux-gnu", host):
         args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0"]
-        target = _ELF(host, args=args)
+        condition = "defined(__x86_64__) && defined(__linux__)"
+        target = _ELF(host, condition, args=args)
     else:
         raise ValueError(host)
     return target

Tools/jit/build.py

@@ -11,7 +11,10 @@
     comment = f"$ {shlex.join([pathlib.Path(sys.executable).name] + sys.argv)}"
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
-        "target", type=_targets.get_target, help="a PEP 11 target triple to compile for"
+        "target",
+        nargs="+",
+        type=_targets.get_target,
+        help="a PEP 11 target triple to compile for",
     )
     parser.add_argument(
         "-d", "--debug", action="store_true", help="compile for a debug build of Python"
@@ -23,6 +26,22 @@
         "-v", "--verbose", action="store_true", help="echo commands as they are run"
     )
     args = parser.parse_args()
-    args.target.debug = args.debug
-    args.target.verbose = args.verbose
-    args.target.build(pathlib.Path.cwd(), comment=comment, force=args.force)
+    for target in args.target:
+        target.debug = args.debug
+        target.force = args.force
+        target.verbose = args.verbose
+        target.build(
+            pathlib.Path.cwd(),
+            comment=comment,
+            stencils_h=f"jit_stencils-{target.triple}.h",
+            force=args.force,
+        )
+
+    with open("jit_stencils.h", "w") as fp:
+        for idx, target in enumerate(args.target):
+            fp.write(f"#{'if' if idx == 0 else 'elif'} {target.condition}\n")
+            fp.write(f'#include "jit_stencils-{target.triple}.h"\n')
+
+        fp.write("#else\n")
+        fp.write('#error "unexpected target"\n')
+        fp.write("#endif\n")