From d145747fa1c87004c65568014c3f81de24e7fd0a Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Thu, 18 Sep 2025 15:56:58 +0100 Subject: [PATCH 1/3] LLVM 20 bump --- .github/workflows/jit.yml | 8 ++++---- PCbuild/build.bat | 1 - PCbuild/get_externals.bat | 4 +--- Tools/jit/README.md | 18 +++++++++--------- Tools/jit/_llvm.py | 10 +--------- Tools/jit/_targets.py | 13 ++++--------- 6 files changed, 19 insertions(+), 35 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 52f7d0d2b3df95..2b0fcdc6b4964e 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -68,7 +68,7 @@ jobs: - true - false llvm: - - 19 + - 20 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -99,10 +99,10 @@ jobs: with: python-version: '3.11' - # PCbuild downloads LLVM automatically: - name: Windows if: runner.os == 'Windows' run: | + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.8 ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -143,7 +143,7 @@ jobs: fail-fast: false matrix: llvm: - - 19 + - 20 steps: - uses: actions/checkout@v4 with: @@ -171,7 +171,7 @@ jobs: # fail-fast: false # matrix: # llvm: - # - 19 + # - 20 # steps: # - uses: actions/checkout@v4 # with: diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 602357048867d6..6c8f1b31636c16 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -111,7 +111,6 @@ if "%IncludeExternals%"=="" set IncludeExternals=true if "%IncludeCTypes%"=="" set IncludeCTypes=true if "%IncludeSSL%"=="" set IncludeSSL=true if "%IncludeTkinter%"=="" set IncludeTkinter=true -if "%UseJIT%" NEQ "true" set IncludeLLVM=false if "%IncludeExternals%"=="true" call "%dir%get_externals.bat" diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index eff8d1ccd7f146..86cf98c0501f49 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -15,7 +15,6 @@ set IncludeSSLSrc=false if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts if "%~1"=="--no-libffi" (set IncludeLibffi=false) & shift & goto CheckOpts -if "%~1"=="--no-llvm" (set IncludeLLVM=false) & shift & goto CheckOpts if "%~1"=="--tkinter-src" (set IncludeTkinterSrc=true) & shift & goto CheckOpts if "%~1"=="--openssl-src" (set IncludeSSLSrc=true) & shift & goto CheckOpts if "%~1"=="--libffi-src" (set IncludeLibffiSrc=true) & shift & goto CheckOpts @@ -82,7 +81,6 @@ if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.16.2 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 -if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-19.1.7.0 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( @@ -101,7 +99,7 @@ goto end :usage echo.Valid options: -c, --clean, --clean-only, --organization, --python, -echo.--no-tkinter, --no-openssl, --no-llvm +echo.--no-tkinter, --no-openssl echo. echo.Pull all sources and binaries necessary for compiling optional extension echo.modules that rely on external libraries. diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 8e817574b4d72b..5afbd9f096d1e0 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,32 +9,32 @@ Python 3.11 or newer is required to build the JIT. The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 19 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 20 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-20`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. It's easy to install all of the required tools: ### Linux -Install LLVM 19 on Ubuntu/Debian: +Install LLVM 20 on Ubuntu/Debian: ```sh wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -sudo ./llvm.sh 19 +sudo ./llvm.sh 20 ``` -Install LLVM 19 on Fedora Linux 40 or newer: +Install LLVM 20 on Fedora Linux 40 or newer: ```sh -sudo dnf install 'clang(major) = 19' 'llvm(major) = 19' +sudo dnf install 'clang(major) = 20' 'llvm(major) = 20' ``` ### macOS -Install LLVM 19 with [Homebrew](https://brew.sh): +Install LLVM 20 with [Homebrew](https://brew.sh): ```sh -brew install llvm@19 +brew install llvm@20 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. @@ -43,12 +43,12 @@ Homebrew won't add any of the tools to your `$PATH`. That's okay; the build scri LLVM is downloaded automatically (along with other external binary dependencies) by `PCbuild\build.bat`. -Otherwise, you can install LLVM 19 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=19), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Otherwise, you can install LLVM 20 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=20), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** Alternatively, you can use [chocolatey](https://chocolatey.org): ```sh -choco install llvm --version=19.1.0 +choco install llvm --version=20.1.8 ``` diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index f09a8404871b24..7a18465fc5b1c0 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -8,11 +8,8 @@ import subprocess import typing -import _targets - -_LLVM_VERSION = 19 +_LLVM_VERSION = 20 _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") -_EXTERNALS_LLVM_TAG = "llvm-19.1.7.0" _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") @@ -75,11 +72,6 @@ async def _find_tool(tool: str, *, echo: bool = False) -> str | None: return path # Versioned executables: path = f"{tool}-{_LLVM_VERSION}" - if await _check_tool_version(path, echo=echo): - return path - # PCbuild externals: - externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS) - path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool) if await _check_tool_version(path, echo=echo): return path # Homebrew-installed executables: diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 2185d8190a8935..c2912594dd7732 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -25,7 +25,6 @@ TOOLS_JIT = TOOLS_JIT_BUILD.parent TOOLS = TOOLS_JIT.parent CPYTHON = TOOLS.parent -EXTERNALS = CPYTHON / "externals" PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" @@ -161,10 +160,6 @@ async def _compile( "-fno-asynchronous-unwind-tables", # Don't call built-in functions that we can't find or patch: "-fno-builtin", - # Emit relaxable 64-bit calls/jumps, so we don't have to worry about - # about emitting in-range trampolines for out-of-range targets. - # We can probably remove this and emit trampolines in the future: - "-fno-plt", # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", "-std=c11", @@ -561,13 +556,13 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: target = _COFF64(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"aarch64-.*-linux-gnu", host): # -mno-outline-atomics: Keep intrinsics from being emitted. - args = ["-fpic", "-mno-outline-atomics"] + args = ["-fpic", "-mno-outline-atomics", "-fno-plt"] condition = "defined(__aarch64__) && defined(__linux__)" optimizer = _optimizers.OptimizerAArch64 target = _ELF(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"i686-pc-windows-msvc", host): # -Wno-ignored-attributes: __attribute__((preserve_none)) is not supported here. - args = ["-DPy_NO_ENABLE_SHARED", "-Wno-ignored-attributes"] + args = ["-DPy_NO_ENABLE_SHARED", "-Wno-ignored-attributes", "-fno-plt"] optimizer = _optimizers.OptimizerX86 condition = "defined(_M_IX86)" target = _COFF32(host, condition, args=args, optimizer=optimizer) @@ -576,12 +571,12 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: optimizer = _optimizers.OptimizerX86 target = _MachO(host, condition, optimizer=optimizer) elif re.fullmatch(r"x86_64-pc-windows-msvc", host): - args = ["-fms-runtime-lib=dll"] + args = ["-fms-runtime-lib=dll", "-fno-plt"] condition = "defined(_M_X64)" optimizer = _optimizers.OptimizerX86 target = _COFF64(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"x86_64-.*-linux-gnu", host): - args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0"] + args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0", "-fno-plt"] condition = "defined(__x86_64__) && defined(__linux__)" optimizer = _optimizers.OptimizerX86 target = _ELF(host, condition, args=args, optimizer=optimizer) From 9324b14b590bcf30f09843bc16a95a2350616439 Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Fri, 19 Sep 2025 12:56:57 +0100 Subject: [PATCH 2/3] Add flags for testing --- Tools/jit/_targets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index c2ff3b46e75748..533827ee47ae6c 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -573,6 +573,7 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: elif re.fullmatch(r"x86_64-apple-darwin.*", host): host = "x86_64-apple-darwin" condition = "defined(__x86_64__) && defined(__APPLE__)" + args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0", "-fno-plt"] optimizer = _optimizers.OptimizerX86 target = _MachO(host, condition, optimizer=optimizer) elif re.fullmatch(r"x86_64-pc-windows-msvc", host): From e86764488fc29f4259e2bc9dbd877a02db3f5139 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sun, 5 Oct 2025 22:18:04 -0700 Subject: [PATCH 3/3] Fix windows --- PCbuild/build.bat | 1 + PCbuild/get_externals.bat | 6 ++++-- PCbuild/regen.targets | 2 +- Tools/jit/_llvm.py | 8 ++++++++ Tools/jit/_targets.py | 1 + 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 6c8f1b31636c16..602357048867d6 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -111,6 +111,7 @@ if "%IncludeExternals%"=="" set IncludeExternals=true if "%IncludeCTypes%"=="" set IncludeCTypes=true if "%IncludeSSL%"=="" set IncludeSSL=true if "%IncludeTkinter%"=="" set IncludeTkinter=true +if "%UseJIT%" NEQ "true" set IncludeLLVM=false if "%IncludeExternals%"=="true" call "%dir%get_externals.bat" diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index dcbfa7c7e32ce2..433e45a5e9f992 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -15,6 +15,7 @@ set IncludeSSLSrc=false if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts if "%~1"=="--no-libffi" (set IncludeLibffi=false) & shift & goto CheckOpts +if "%~1"=="--no-llvm" (set IncludeLLVM=false) & shift & goto CheckOpts if "%~1"=="--tkinter-src" (set IncludeTkinterSrc=true) & shift & goto CheckOpts if "%~1"=="--openssl-src" (set IncludeSSLSrc=true) & shift & goto CheckOpts if "%~1"=="--libffi-src" (set IncludeLibffiSrc=true) & shift & goto CheckOpts @@ -81,6 +82,7 @@ if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.18 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 +if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-20.1.8.0 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( @@ -99,7 +101,7 @@ goto end :usage echo.Valid options: -c, --clean, --clean-only, --organization, --python, -echo.--no-tkinter, --no-openssl +echo.--no-tkinter, --no-openssl, --no-llvm echo. echo.Pull all sources and binaries necessary for compiling optional extension echo.modules that rely on external libraries. @@ -115,4 +117,4 @@ echo.anything new. echo. exit /b -1 -:end +:end \ No newline at end of file diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 742597f5cb5ebd..e20627032ccd71 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -187,4 +187,4 @@ - + \ No newline at end of file diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 7a18465fc5b1c0..ac251698de1d98 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -8,8 +8,11 @@ import subprocess import typing +import _targets + _LLVM_VERSION = 20 _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") +_EXTERNALS_LLVM_TAG = "llvm-20.1.8.0" _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") @@ -72,6 +75,11 @@ async def _find_tool(tool: str, *, echo: bool = False) -> str | None: return path # Versioned executables: path = f"{tool}-{_LLVM_VERSION}" + if await _check_tool_version(path, echo=echo): + return path + # PCbuild externals: + externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS) + path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool) if await _check_tool_version(path, echo=echo): return path # Homebrew-installed executables: diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 533827ee47ae6c..b8fdbea7425439 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -25,6 +25,7 @@ TOOLS_JIT = TOOLS_JIT_BUILD.parent TOOLS = TOOLS_JIT.parent CPYTHON = TOOLS.parent +EXTERNALS = CPYTHON / "externals" PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c"