[pull] main from python:main

Doc/Makefile

@@ -184,7 +184,7 @@ venv:
 	fi
 
 .PHONY: dist-no-html
-dist-no-html: dist-text dist-pdf dist-epub dist-texinfo
+dist-no-html: dist-text dist-epub dist-texinfo
 
 .PHONY: dist
 dist:

Misc/NEWS.d/next/Build/2025-09-04-12-16-31.gh-issue-138497.Y_5YXh.rst

@@ -0,0 +1,4 @@
+The LLVM version used by the JIT at build time can now be modified using
+the ``LLVM_VERSION`` environment variable. Use this at your own risk, as
+there is only one officially supported LLVM version. For more information,
+please check ``Tools/jit/README.md``.

Python/ceval.c

@@ -438,31 +438,26 @@ int pthread_attr_destroy(pthread_attr_t *a)
 
 #endif
 
-
-void
-_Py_InitializeRecursionLimits(PyThreadState *tstate)
+static void
+hardware_stack_limits(uintptr_t *top, uintptr_t *base)
 {
-    _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
 #ifdef WIN32
     ULONG_PTR low, high;
     GetCurrentThreadStackLimits(&low, &high);
-    _tstate->c_stack_top = (uintptr_t)high;
+    *top = (uintptr_t)high;
     ULONG guarantee = 0;
     SetThreadStackGuarantee(&guarantee);
-    _tstate->c_stack_hard_limit = ((uintptr_t)low) + guarantee + _PyOS_STACK_MARGIN_BYTES;
-    _tstate->c_stack_soft_limit = _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES;
+    *base = (uintptr_t)low + guarantee;
 #elif defined(__APPLE__)
     pthread_t this_thread = pthread_self();
     void *stack_addr = pthread_get_stackaddr_np(this_thread); // top of the stack
     size_t stack_size = pthread_get_stacksize_np(this_thread);
-    _tstate->c_stack_top = (uintptr_t)stack_addr;
-    _tstate->c_stack_hard_limit = _tstate->c_stack_top - stack_size;
-    _tstate->c_stack_soft_limit = _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES;
+    *top = (uintptr_t)stack_addr;
+    *base = ((uintptr_t)stack_addr) - stack_size;
 #else
-    uintptr_t here_addr = _Py_get_machine_stack_pointer();
-/// XXX musl supports HAVE_PTHRED_GETATTR_NP, but the resulting stack size
-/// (on alpine at least) is much smaller than expected and imposes undue limits
-/// compared to the old stack size estimation.  (We assume musl is not glibc.)
+    /// XXX musl supports HAVE_PTHRED_GETATTR_NP, but the resulting stack size
+    /// (on alpine at least) is much smaller than expected and imposes undue limits
+    /// compared to the old stack size estimation.  (We assume musl is not glibc.)
 #  if defined(HAVE_PTHREAD_GETATTR_NP) && !defined(_AIX) && \
         !defined(__NetBSD__) && (defined(__GLIBC__) || !defined(__linux__))
     size_t stack_size, guard_size;
@@ -475,26 +470,35 @@ _Py_InitializeRecursionLimits(PyThreadState *tstate)
         err |= pthread_attr_destroy(&attr);
     }
     if (err == 0) {
-        uintptr_t base = ((uintptr_t)stack_addr) + guard_size;
-        _tstate->c_stack_top = base + stack_size;
-#ifdef _Py_THREAD_SANITIZER
-        // Thread sanitizer crashes if we use a bit more than half the stack.
-        _tstate->c_stack_soft_limit = base + (stack_size / 2);
-#else
-        _tstate->c_stack_soft_limit = base + _PyOS_STACK_MARGIN_BYTES * 2;
-#endif
-        _tstate->c_stack_hard_limit = base + _PyOS_STACK_MARGIN_BYTES;
-        assert(_tstate->c_stack_soft_limit < here_addr);
-        assert(here_addr < _tstate->c_stack_top);
+        *base = ((uintptr_t)stack_addr) + guard_size;
+        *top = (uintptr_t)stack_addr + stack_size;
         return;
     }
 #  endif
-    _tstate->c_stack_top = _Py_SIZE_ROUND_UP(here_addr, 4096);
-    _tstate->c_stack_soft_limit = _tstate->c_stack_top - Py_C_STACK_SIZE;
-    _tstate->c_stack_hard_limit = _tstate->c_stack_top - (Py_C_STACK_SIZE + _PyOS_STACK_MARGIN_BYTES);
+    uintptr_t here_addr = _Py_get_machine_stack_pointer();
+    uintptr_t top_addr = _Py_SIZE_ROUND_UP(here_addr, 4096);
+    *top = top_addr;
+    *base = top_addr - Py_C_STACK_SIZE;
 #endif
 }
 
+void
+_Py_InitializeRecursionLimits(PyThreadState *tstate)
+{
+    uintptr_t top;
+    uintptr_t base;
+    hardware_stack_limits(&top, &base);
+#ifdef _Py_THREAD_SANITIZER
+    // Thread sanitizer crashes if we use more than half the stack.
+    uintptr_t stacksize = top - base;
+    base += stacksize/2;
+#endif
+    _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
+    _tstate->c_stack_top = top;
+    _tstate->c_stack_hard_limit = base + _PyOS_STACK_MARGIN_BYTES;
+    _tstate->c_stack_soft_limit = base + _PyOS_STACK_MARGIN_BYTES * 2;
+}
+
 /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall()
    if the recursion_depth reaches recursion_limit. */
 int

Tools/jit/README.md

@@ -9,7 +9,7 @@ Python 3.11 or newer is required to build the JIT.
 
 The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon).
 
-LLVM version 19 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code.
+LLVM version 19 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code.
 
 It's easy to install all of the required tools:
 

Tools/jit/_llvm.py

@@ -10,8 +10,8 @@
 
 import _targets
 
-_LLVM_VERSION = 19
-_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+")
+
+_LLVM_VERSION = "19"
 _EXTERNALS_LLVM_TAG = "llvm-19.1.7.0"
 
 _P = typing.ParamSpec("_P")
@@ -56,53 +56,66 @@ async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str
 
 
 @_async_cache
-async def _check_tool_version(name: str, *, echo: bool = False) -> bool:
+async def _check_tool_version(
+    name: str, llvm_version: str, *, echo: bool = False
+) -> bool:
     output = await _run(name, ["--version"], echo=echo)
-    return bool(output and _LLVM_VERSION_PATTERN.search(output))
+    _llvm_version_pattern = re.compile(rf"version\s+{llvm_version}\.\d+\.\d+\S*\s+")
+    return bool(output and _llvm_version_pattern.search(output))
 
 
 @_async_cache
-async def _get_brew_llvm_prefix(*, echo: bool = False) -> str | None:
-    output = await _run("brew", ["--prefix", f"llvm@{_LLVM_VERSION}"], echo=echo)
+async def _get_brew_llvm_prefix(llvm_version: str, *, echo: bool = False) -> str | None:
+    output = await _run("brew", ["--prefix", f"llvm@{llvm_version}"], echo=echo)
     return output and output.removesuffix("\n")
 
 
 @_async_cache
-async def _find_tool(tool: str, *, echo: bool = False) -> str | None:
+async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None:
     # Unversioned executables:
     path = tool
-    if await _check_tool_version(path, echo=echo):
+    if await _check_tool_version(path, llvm_version, echo=echo):
         return path
     # Versioned executables:
-    path = f"{tool}-{_LLVM_VERSION}"
-    if await _check_tool_version(path, echo=echo):
+    path = f"{tool}-{llvm_version}"
+    if await _check_tool_version(path, llvm_version, echo=echo):
         return path
     # PCbuild externals:
     externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS)
     path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool)
-    if await _check_tool_version(path, echo=echo):
+    if await _check_tool_version(path, llvm_version, echo=echo):
         return path
     # Homebrew-installed executables:
-    prefix = await _get_brew_llvm_prefix(echo=echo)
+    prefix = await _get_brew_llvm_prefix(llvm_version, echo=echo)
     if prefix is not None:
         path = os.path.join(prefix, "bin", tool)
-        if await _check_tool_version(path, echo=echo):
+        if await _check_tool_version(path, llvm_version, echo=echo):
             return path
     # Nothing found:
     return None
 
 
 async def maybe_run(
-    tool: str, args: typing.Iterable[str], echo: bool = False
+    tool: str,
+    args: typing.Iterable[str],
+    echo: bool = False,
+    llvm_version: str = _LLVM_VERSION,
 ) -> str | None:
     """Run an LLVM tool if it can be found. Otherwise, return None."""
-    path = await _find_tool(tool, echo=echo)
+
+    path = await _find_tool(tool, llvm_version, echo=echo)
     return path and await _run(path, args, echo=echo)
 
 
-async def run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str:
+async def run(
+    tool: str,
+    args: typing.Iterable[str],
+    echo: bool = False,
+    llvm_version: str = _LLVM_VERSION,
+) -> str:
     """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError."""
-    output = await maybe_run(tool, args, echo=echo)
+
+    output = await maybe_run(tool, args, echo=echo, llvm_version=llvm_version)
     if output is None:
-        raise RuntimeError(f"Can't find {tool}-{_LLVM_VERSION}!")
+        raise RuntimeError(f"Can't find {tool}-{llvm_version}!")
     return output

Tools/jit/_targets.py

@@ -50,6 +50,7 @@ class _Target(typing.Generic[_S, _R]):
     debug: bool = False
     verbose: bool = False
     cflags: str = ""
+    llvm_version: str = _llvm._LLVM_VERSION
     known_symbols: dict[str, int] = dataclasses.field(default_factory=dict)
     pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve()
 
@@ -81,7 +82,9 @@ def _compute_digest(self) -> str:
     async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup:
         group = _stencils.StencilGroup()
         args = ["--disassemble", "--reloc", f"{path}"]
-        output = await _llvm.maybe_run("llvm-objdump", args, echo=self.verbose)
+        output = await _llvm.maybe_run(
+            "llvm-objdump", args, echo=self.verbose, llvm_version=self.llvm_version
+        )
         if output is not None:
             # Make sure that full paths don't leak out (for reproducibility):
             long, short = str(path), str(path.name)
@@ -99,7 +102,9 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup:
             "--sections",
             f"{path}",
         ]
-        output = await _llvm.run("llvm-readobj", args, echo=self.verbose)
+        output = await _llvm.run(
+            "llvm-readobj", args, echo=self.verbose, llvm_version=self.llvm_version
+        )
         # --elf-output-style=JSON is only *slightly* broken on Mach-O...
         output = output.replace("PrivateExtern\n", "\n")
         output = output.replace("Extern\n", "\n")
@@ -175,12 +180,16 @@ async def _compile(
             # Allow user-provided CFLAGS to override any defaults
             *shlex.split(self.cflags),
         ]
-        await _llvm.run("clang", args_s, echo=self.verbose)
+        await _llvm.run(
+            "clang", args_s, echo=self.verbose, llvm_version=self.llvm_version
+        )
         self.optimizer(
             s, label_prefix=self.label_prefix, symbol_prefix=self.symbol_prefix
         ).run()
         args_o = [f"--target={self.triple}", "-c", "-o", f"{o}", f"{s}"]
-        await _llvm.run("clang", args_o, echo=self.verbose)
+        await _llvm.run(
+            "clang", args_o, echo=self.verbose, llvm_version=self.llvm_version
+        )
         return await self._parse(o)
 
     async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]:
@@ -224,6 +233,8 @@ def build(
         if not self.stable:
             warning = f"JIT support for {self.triple} is still experimental!"
             request = "Please report any issues you encounter.".center(len(warning))
+            if self.llvm_version != _llvm._LLVM_VERSION:
+                request = f"Warning! Building with an LLVM version other than {_llvm._LLVM_VERSION} is not supported."
             outline = "=" * len(warning)
             print("\n".join(["", outline, warning, request, outline, ""]))
         digest = f"// {self._compute_digest()}\n"

Tools/jit/build.py

@@ -42,13 +42,16 @@
     parser.add_argument(
         "--cflags", help="additional flags to pass to the compiler", default=""
     )
+    parser.add_argument("--llvm-version", help="LLVM version to use")
     args = parser.parse_args()
     for target in args.target:
         target.debug = args.debug
         target.force = args.force
         target.verbose = args.verbose
         target.cflags = args.cflags
         target.pyconfig_dir = args.pyconfig_dir
+        if args.llvm_version:
+            target.llvm_version = args.llvm_version
         target.build(
             comment=comment,
             force=args.force,

configure.ac

@@ -2786,7 +2786,7 @@ AS_VAR_IF([jit_flags],
           [],
           [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"])
            AS_VAR_SET([REGEN_JIT_COMMAND],
-                      ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\""])
+                      ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\""])
            AS_VAR_IF([Py_DEBUG],
                      [true],
                      [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])],