Upgrade to LLVM 20.1.0 (#553)

I just published an LLVM 20.1.0 toolchain with some computed gotos
patches to hopefully improve performance of CPython.


https://github.com/indygreg/toolchain-tools/releases/tag/toolchain-bootstrap%2F20250308

One of the patches changes codegen to fix a regression in LLVM 19.

The other enables bolt to work with computed gotos. To take full
advantage of this functionality we patch CPython configure to remove
some `--skip-funcs` which are no longer required.

Author: indygreg

cpython-unix/build-cpython.sh

@@ -270,6 +270,9 @@ if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then
     # Additional BOLT optimizations, being upstreamed in
     # https://github.com/python/cpython/issues/128514
     patch -p1 -i ${ROOT}/patch-configure-bolt-apply-flags-128514.patch
+
+    # Tweak --skip-funcs to work with our toolchain.
+    patch -p1 -i ${ROOT}/patch-configure-bolt-skip-funcs.patch
 fi
 
 # The optimization make targets are both phony and non-phony. This leads
@@ -437,10 +440,11 @@ if [ -n "${CPYTHON_OPTIMIZED}" ]; then
         if [[ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_13}" ]]; then
             # On 3.13, LLVM 18 is hard-coded into the configure script. Override it to our toolchain
             # version.
-            patch -p1 -i "${ROOT}/patch-jit-llvm-19.patch"
+            patch -p1 -i "${ROOT}/patch-jit-llvm-version-3.13.patch"
         fi
 
          if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then
+            patch -p1 -i "${ROOT}/patch-jit-llvm-version-3.14.patch"
             # On 3.14, we also use the tail calling interpreter which was incompatible with the JIT
             # until https://github.com/python/cpython/pull/129820 — backport that
             patch -p1 -i "${ROOT}/patch-jit-tail-call-compat-314-129820.patch"

cpython-unix/patch-configure-bolt-skip-funcs.patch

@@ -0,0 +1,17 @@
+diff --git a/configure.ac b/configure.ac
+index 3fcb18922c5..2df614a3063 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -2227,10 +2227,8 @@ then
+     [m4_normalize("
+       [-update-debug-sections]
+ 
+-      dnl At least LLVM 19.x doesn't support computed gotos in PIC compiled code.
+-      dnl Exclude functions containing computed gotos.
+-      dnl TODO this may be fixed in LLVM 20.x via https://github.com/llvm/llvm-project/pull/120267.
+-      [-skip-funcs=_PyEval_EvalFrameDefault,sre_ucs1_match/1,sre_ucs2_match/1,sre_ucs4_match/1]
++      dnl LLVM on at least 20.1.0 crashes on this symbol. Work around.
++      [-skip-funcs=RC4_options/1]
+     ")]
+   )
+ fi

cpython-unix/patch-jit-llvm-19.patch renamed to cpython-unix/patch-jit-llvm-version-3.13.patch

@@ -6,7 +6,7 @@ diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py
  import typing
 
 -_LLVM_VERSION = 18
-+_LLVM_VERSION = 19
++_LLVM_VERSION = 20
  _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+")
 
  _P = typing.ParamSpec("_P")

cpython-unix/patch-jit-llvm-version-3.14.patch

@@ -0,0 +1,12 @@
+diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py
+--- a/Tools/jit/_llvm.py
++++ b/Tools/jit/_llvm.py
+@@ -8,7 +8,7 @@
+ import subprocess
+ import typing
+ 
+-_LLVM_VERSION = 19
++_LLVM_VERSION = 20
+ _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+")
+ 
+ _P = typing.ParamSpec("_P")

pythonbuild/downloads.py

@@ -184,25 +184,25 @@
         "version": "14.0.3+20220508",
     },
     # Remember to update LLVM_URL in src/release.rs whenever upgrading.
-    "llvm-19-x86_64-linux": {
-        "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20241231/llvm-19.1.6+20241231-gnu_only-x86_64-unknown-linux-gnu.tar.zst",
-        "size": 251277644,
-        "sha256": "900bbb464d430e960ea97150e073395ae865332ce403e7c1e8f76640cab5e852",
-        "version": "19.1.6+20241231",
+    "llvm-20-x86_64-linux": {
+        "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250308/llvm-20.1.0+20250308-gnu_only-x86_64-unknown-linux-gnu.tar.zst",
+        "size": 283261860,
+        "sha256": "41d3d74e21e064e2e59a4e89feca74d58a5e9e95f73877f3c9ed82ca95607b47",
+        "version": "20.1.0+20240308",
     },
     # Remember to update LLVM_URL in src/release.rs whenever upgrading.
     "llvm-aarch64-macos": {
-        "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20241231/llvm-19.1.6+20241231-aarch64-apple-darwin.tar.zst",
-        "size": 143884729,
-        "sha256": "c0252c7ebe0b20125fe592a12d43a693fcab0bfa3bec9def6b96b97087e0f765",
-        "version": "19.1.6+20241231",
+        "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250308/llvm-20.1.0+20250308-aarch64-apple-darwin.tar.zst",
+        "size": 161006322,
+        "sha256": "9897bfaab16c930258f614250a1ade1a8f32df027181bd54a61d24b5a04e284c",
+        "version": "20.1.0+20240308",
     },
     # Remember to update LLVM_URL in src/release.rs whenever upgrading.
     "llvm-x86_64-macos": {
-        "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20241231/llvm-19.1.6+20241231-x86_64-apple-darwin.tar.zst",
-        "size": 143113277,
-        "sha256": "e990dea9ff89e4202d358b477a43f2bd128b244926534b7c8bb29feeaada994e",
-        "version": "19.1.6+20241231",
+        "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250308/llvm-20.1.0+20250308-x86_64-apple-darwin.tar.zst",
+        "size": 160133882,
+        "sha256": "95cfcb79d752e81735a7a7f7c4cb5a7c16360b6e9a75658a1bd50ce0547d4ad5",
+        "version": "20.1.0+20240308",
     },
     "m4": {
         "url": "https://ftp.gnu.org/gnu/m4/m4-1.4.19.tar.xz",

pythonbuild/utils.py

@@ -434,7 +434,7 @@ def clang_toolchain(host_platform: str, target_triple: str) -> str:
         if "musl" in target_triple:
             return "llvm-14-x86_64-linux"
         else:
-            return "llvm-19-x86_64-linux"
+            return "llvm-20-x86_64-linux"
     elif host_platform == "macos":
         if platform.mac_ver()[2] == "arm64":
             return "llvm-aarch64-macos"

src/release.rs

@@ -589,14 +589,14 @@ pub fn produce_install_only_stripped(tar_gz_path: &Path, llvm_dir: &Path) -> Res
 static LLVM_URL: Lazy<Url> = Lazy::new(|| {
     if cfg!(target_os = "macos") {
         if std::env::consts::ARCH == "aarch64" {
-            Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20241231/llvm-19.1.6+20241231-aarch64-apple-darwin.tar.zst").unwrap()
+            Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250308/llvm-20.1.0+20250308-aarch64-apple-darwin.tar.zst").unwrap()
         } else if std::env::consts::ARCH == "x86_64" {
-            Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20241231/llvm-19.1.6+20241231-x86_64-apple-darwin.tar.zst").unwrap()
+            Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250308/llvm-20.1.0+20250308-x86_64-apple-darwin.tar.zst").unwrap()
         } else {
             panic!("unsupported macOS architecture");
         }
     } else if cfg!(target_os = "linux") {
-        Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20241231/llvm-19.1.6+20241231-gnu_only-x86_64-unknown-linux-gnu.tar.zst").unwrap()
+        Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250308/llvm-20.1.0+20250308-gnu_only-x86_64-unknown-linux-gnu.tar.zst").unwrap()
     } else {
         panic!("unsupported platform");
     }