GH-118306: Update JIT to use LLVM 18 (GH-118307)

Author: savannahostrowski

.github/workflows/jit.yml

@@ -41,7 +41,7 @@ jobs:
           - true
           - false
         llvm:
-          - 16
+          - 18
         include:
           - target: i686-pc-windows-msvc/msvc
             architecture: Win32
@@ -94,6 +94,7 @@ jobs:
       - name: Native Windows
         if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
         run: |
+          choco upgrade llvm -y
           choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
           ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
@@ -102,27 +103,31 @@ jobs:
       - name: Emulated Windows
         if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
         run: |
+          choco upgrade llvm -y
           choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
 
       - name: Native macOS
         if: runner.os == 'macOS'
         run: |
+          brew update
           brew install llvm@${{ matrix.llvm }}
           SDKROOT="$(xcrun --show-sdk-path)" \
             ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
           make all --jobs 4
           ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
+      # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
       - name: Native Linux
         if: runner.os == 'Linux' && matrix.architecture == 'x86_64'
         run: |
           sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
           export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
-          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
+          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }}
           make all --jobs 4
           ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
+      # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
       - name: Emulated Linux
         if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
         run: |
@@ -139,6 +144,6 @@ jobs:
           CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
             CPP="$CC --preprocess" \
             HOSTRUNNER=qemu-${{ matrix.architecture }} \
-            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
+            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
           make all --jobs 4
           ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3

Misc/NEWS.d/next/Core and Builtins/2024-04-26-05-38-18.gh-issue-118306.vRUEOU.rst

@@ -0,0 +1 @@
+Update JIT compilation to use LLVM 18

Python/jit.c

@@ -234,6 +234,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[])
                 assert((int64_t)value < (1LL << 31));
                 *loc32 = (uint32_t)value;
                 continue;
+            case HoleKind_ARM64_RELOC_BRANCH26:
             case HoleKind_IMAGE_REL_ARM64_BRANCH26:
             case HoleKind_R_AARCH64_CALL26:
             case HoleKind_R_AARCH64_JUMP26:
@@ -278,6 +279,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[])
             case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21:
             case HoleKind_IMAGE_REL_ARM64_PAGEBASE_REL21:
             case HoleKind_R_AARCH64_ADR_GOT_PAGE:
+            case HoleKind_R_AARCH64_ADR_PREL_PG_HI21:
                 // 21-bit count of pages between this page and an absolute address's
                 // page... I know, I know, it's weird. Pairs nicely with
                 // ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below).
@@ -341,6 +343,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[])
             case HoleKind_ARM64_RELOC_PAGEOFF12:
             case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12A:
             case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12L:
+            case HoleKind_R_AARCH64_ADD_ABS_LO12_NC:
             case HoleKind_R_AARCH64_LD64_GOT_LO12_NC:
                 // 12-bit low part of an absolute address. Pairs nicely with
                 // ARM64_RELOC_GOT_LOAD_PAGE21 (above).

Tools/jit/README.md

@@ -23,17 +23,21 @@ sudo ./llvm.sh 16
 
 ### macOS
 
-Install LLVM 16 with [Homebrew](https://brew.sh):
+Install LLVM 18 with [Homebrew](https://brew.sh):
 
 ```sh
-brew install llvm@16
+brew install llvm@18
 ```
 
 Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them.
 
 ### Windows
 
-Install LLVM 16 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=16), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".**
+Install LLVM 18 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=18), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".**
+
+### Dev Containers
+
+If you are working CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no need to install LLVM as the Fedora 40 base image includes LLVM 18 out of the box.
 
 ## Building
 

Tools/jit/_llvm.py

@@ -7,7 +7,7 @@
 import subprocess
 import typing
 
-_LLVM_VERSION = 16
+_LLVM_VERSION = 18
 _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\s+")
 
 _P = typing.ParamSpec("_P")

Tools/jit/_schema.py

@@ -2,6 +2,7 @@
 import typing
 
 HoleKind: typing.TypeAlias = typing.Literal[
+    "ARM64_RELOC_BRANCH26",
     "ARM64_RELOC_GOT_LOAD_PAGE21",
     "ARM64_RELOC_GOT_LOAD_PAGEOFF12",
     "ARM64_RELOC_PAGE21",
@@ -16,8 +17,10 @@
     "IMAGE_REL_I386_REL32",
     "R_AARCH64_ABS64",
     "R_AARCH64_ADR_GOT_PAGE",
+    "R_AARCH64_ADR_PREL_PG_HI21",
     "R_AARCH64_CALL26",
     "R_AARCH64_JUMP26",
+    "R_AARCH64_ADD_ABS_LO12_NC",
     "R_AARCH64_LD64_GOT_LO12_NC",
     "R_AARCH64_MOVW_UABS_G0_NC",
     "R_AARCH64_MOVW_UABS_G1_NC",
@@ -68,12 +71,12 @@ class _COFFSymbol(typing.TypedDict):
 
 
 class _ELFSymbol(typing.TypedDict):
-    Name: dict[typing.Literal["Value"], str]
+    Name: dict[typing.Literal["Name"], str]
     Value: int
 
 
 class _MachOSymbol(typing.TypedDict):
-    Name: dict[typing.Literal["Value"], str]
+    Name: dict[typing.Literal["Name"], str]
     Value: int
 
 
@@ -99,7 +102,7 @@ class ELFSection(typing.TypedDict):
     Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]]
     SectionData: dict[typing.Literal["Bytes"], list[int]]
     Symbols: list[dict[typing.Literal["Symbol"], _ELFSymbol]]
-    Type: dict[typing.Literal["Value"], str]
+    Type: dict[typing.Literal["Name"], str]
 
 
 class MachOSection(typing.TypedDict):

Tools/jit/_stencils.py

@@ -203,7 +203,9 @@ def process_relocations(self, *, alignment: int = 1) -> None:
         """Fix up all GOT and internal relocations for this stencil group."""
         for hole in self.code.holes.copy():
             if (
-                hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26"}
+                hole.kind in {
+                    "R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26"
+                }
                 and hole.value is HoleValue.ZERO
             ):
                 self.code.pad(alignment)

Tools/jit/_targets.py

@@ -221,7 +221,7 @@ def _handle_relocation(
             case {
                 "Offset": offset,
                 "Symbol": s,
-                "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind},
+                "Type": {"Name": "IMAGE_REL_I386_DIR32" as kind},
             }:
                 offset += base
                 value, symbol = self._unwrap_dllimport(s)
@@ -230,7 +230,7 @@ def _handle_relocation(
                 "Offset": offset,
                 "Symbol": s,
                 "Type": {
-                    "Value": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind
+                    "Name": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind
                 },
             }:
                 offset += base
@@ -242,7 +242,7 @@ def _handle_relocation(
                 "Offset": offset,
                 "Symbol": s,
                 "Type": {
-                    "Value": "IMAGE_REL_ARM64_BRANCH26"
+                    "Name": "IMAGE_REL_ARM64_BRANCH26"
                     | "IMAGE_REL_ARM64_PAGEBASE_REL21"
                     | "IMAGE_REL_ARM64_PAGEOFFSET_12A"
                     | "IMAGE_REL_ARM64_PAGEOFFSET_12L" as kind
@@ -262,7 +262,7 @@ class _ELF(
     def _handle_section(
         self, section: _schema.ELFSection, group: _stencils.StencilGroup
     ) -> None:
-        section_type = section["Type"]["Value"]
+        section_type = section["Type"]["Name"]
         flags = {flag["Name"] for flag in section["Flags"]["Flags"]}
         if section_type == "SHT_RELA":
             assert "SHF_INFO_LINK" in flags, flags
@@ -290,7 +290,7 @@ def _handle_section(
             for wrapped_symbol in section["Symbols"]:
                 symbol = wrapped_symbol["Symbol"]
                 offset = len(stencil.body) + symbol["Value"]
-                name = symbol["Name"]["Value"]
+                name = symbol["Name"]["Name"]
                 name = name.removeprefix(self.prefix)
                 group.symbols[name] = value, offset
             stencil.body.extend(section["SectionData"]["Bytes"])
@@ -312,9 +312,9 @@ def _handle_relocation(
             case {
                 "Addend": addend,
                 "Offset": offset,
-                "Symbol": {"Value": s},
+                "Symbol": {"Name": s},
                 "Type": {
-                    "Value": "R_AARCH64_ADR_GOT_PAGE"
+                    "Name": "R_AARCH64_ADR_GOT_PAGE"
                     | "R_AARCH64_LD64_GOT_LO12_NC"
                     | "R_X86_64_GOTPCREL"
                     | "R_X86_64_GOTPCRELX"
@@ -327,8 +327,8 @@ def _handle_relocation(
             case {
                 "Addend": addend,
                 "Offset": offset,
-                "Symbol": {"Value": s},
-                "Type": {"Value": kind},
+                "Symbol": {"Name": s},
+                "Type": {"Name": kind},
             }:
                 offset += base
                 s = s.removeprefix(self.prefix)
@@ -371,7 +371,7 @@ def _handle_section(
         for wrapped_symbol in section["Symbols"]:
             symbol = wrapped_symbol["Symbol"]
             offset = symbol["Value"] - start_address
-            name = symbol["Name"]["Value"]
+            name = symbol["Name"]["Name"]
             name = name.removeprefix(self.prefix)
             group.symbols[name] = value, offset
         assert "Relocations" in section
@@ -387,9 +387,9 @@ def _handle_relocation(
         match relocation:
             case {
                 "Offset": offset,
-                "Symbol": {"Value": s},
+                "Symbol": {"Name": s},
                 "Type": {
-                    "Value": "ARM64_RELOC_GOT_LOAD_PAGE21"
+                    "Name": "ARM64_RELOC_GOT_LOAD_PAGE21"
                     | "ARM64_RELOC_GOT_LOAD_PAGEOFF12" as kind
                 },
             }:
@@ -399,8 +399,8 @@ def _handle_relocation(
                 addend = 0
             case {
                 "Offset": offset,
-                "Symbol": {"Value": s},
-                "Type": {"Value": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind},
+                "Symbol": {"Name": s},
+                "Type": {"Name": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind},
             }:
                 offset += base
                 s = s.removeprefix(self.prefix)
@@ -410,13 +410,13 @@ def _handle_relocation(
                 )
             case {
                 "Offset": offset,
-                "Section": {"Value": s},
-                "Type": {"Value": "X86_64_RELOC_SIGNED" as kind},
+                "Section": {"Name": s},
+                "Type": {"Name": "X86_64_RELOC_SIGNED" as kind},
             } | {
                 "Offset": offset,
-                "Symbol": {"Value": s},
+                "Symbol": {"Name": s},
                 "Type": {
-                    "Value": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind
+                    "Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind
                 },
             }:
                 offset += base
@@ -427,12 +427,12 @@ def _handle_relocation(
                 )
             case {
                 "Offset": offset,
-                "Section": {"Value": s},
-                "Type": {"Value": kind},
+                "Section": {"Name": s},
+                "Type": {"Name": kind},
             } | {
                 "Offset": offset,
-                "Symbol": {"Value": s},
-                "Type": {"Value": kind},
+                "Symbol": {"Name": s},
+                "Type": {"Name": kind},
             }:
                 offset += base
                 s = s.removeprefix(self.prefix)
@@ -446,13 +446,12 @@ def _handle_relocation(
 def get_target(host: str) -> _COFF | _ELF | _MachO:
     """Build a _Target for the given host "triple" and options."""
     if re.fullmatch(r"aarch64-apple-darwin.*", host):
-        args = ["-mcmodel=large"]
-        return _MachO(host, alignment=8, args=args, prefix="_")
+        return _MachO(host, alignment=8, prefix="_")
     if re.fullmatch(r"aarch64-pc-windows-msvc", host):
         args = ["-fms-runtime-lib=dll"]
         return _COFF(host, alignment=8, args=args)
     if re.fullmatch(r"aarch64-.*-linux-gnu", host):
-        args = ["-mcmodel=large"]
+        args = ["-fpic"]
         return _ELF(host, alignment=8, args=args)
     if re.fullmatch(r"i686-pc-windows-msvc", host):
         args = ["-DPy_NO_ENABLE_SHARED"]
@@ -463,5 +462,6 @@ def get_target(host: str) -> _COFF | _ELF | _MachO:
         args = ["-fms-runtime-lib=dll"]
         return _COFF(host, args=args)
     if re.fullmatch(r"x86_64-.*-linux-gnu", host):
-        return _ELF(host)
+        args = ["-fpic"]
+        return _ELF(host, args=args)
     raise ValueError(host)