GH-133779: Fix finding pyconfig.h on Windows JIT builds (GH-134349)

(cherry picked from commit 7ad9046)

Co-authored-by: Brandt Bucher <[email protected]>

Author: brandtbucher

PCbuild/regen.targets

@@ -125,8 +125,7 @@
       <JITArgs Condition="$(Platform) == 'x64'">x86_64-pc-windows-msvc</JITArgs>
       <JITArgs Condition="$(Configuration) == 'Debug'">$(JITArgs) --debug</JITArgs>
     </PropertyGroup>
-    <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py" $(JITArgs)'
-          WorkingDirectory="$(GeneratedJitStencilsDir)"/>
+    <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py" $(JITArgs) --output-dir "$(GeneratedJitStencilsDir)" --pyconfig-dir "$(PySourcePath)PC"'/>
   </Target>
   <Target Name="_CleanJIT" AfterTargets="Clean">
     <Delete Files="@(_JITOutputs)"/>

Tools/jit/_targets.py

@@ -47,6 +47,7 @@ class _Target(typing.Generic[_S, _R]):
     debug: bool = False
     verbose: bool = False
     known_symbols: dict[str, int] = dataclasses.field(default_factory=dict)
+    pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve()
 
     def _get_nop(self) -> bytes:
         if re.fullmatch(r"aarch64-.*", self.triple):
@@ -57,13 +58,13 @@ def _get_nop(self) -> bytes:
             raise ValueError(f"NOP not defined for {self.triple}")
         return nop
 
-    def _compute_digest(self, out: pathlib.Path) -> str:
+    def _compute_digest(self) -> str:
         hasher = hashlib.sha256()
         hasher.update(self.triple.encode())
         hasher.update(self.debug.to_bytes())
         # These dependencies are also reflected in _JITSources in regen.targets:
         hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes())
-        hasher.update((out / "pyconfig.h").read_bytes())
+        hasher.update((self.pyconfig_dir / "pyconfig.h").read_bytes())
         for dirpath, _, filenames in sorted(os.walk(TOOLS_JIT)):
             for filename in filenames:
                 hasher.update(pathlib.Path(dirpath, filename).read_bytes())
@@ -125,7 +126,7 @@ async def _compile(
             f"-D_JIT_OPCODE={opname}",
             "-D_PyJIT_ACTIVE",
             "-D_Py_JIT",
-            "-I.",
+            f"-I{self.pyconfig_dir}",
             f"-I{CPYTHON / 'Include'}",
             f"-I{CPYTHON / 'Include' / 'internal'}",
             f"-I{CPYTHON / 'Include' / 'internal' / 'mimalloc'}",
@@ -193,28 +194,27 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]:
 
     def build(
         self,
-        out: pathlib.Path,
         *,
         comment: str = "",
         force: bool = False,
-        stencils_h: str = "jit_stencils.h",
+        jit_stencils: pathlib.Path,
     ) -> None:
         """Build jit_stencils.h in the given directory."""
+        jit_stencils.parent.mkdir(parents=True, exist_ok=True)
         if not self.stable:
             warning = f"JIT support for {self.triple} is still experimental!"
             request = "Please report any issues you encounter.".center(len(warning))
             outline = "=" * len(warning)
             print("\n".join(["", outline, warning, request, outline, ""]))
-        digest = f"// {self._compute_digest(out)}\n"
-        jit_stencils = out / stencils_h
+        digest = f"// {self._compute_digest()}\n"
         if (
             not force
             and jit_stencils.exists()
             and jit_stencils.read_text().startswith(digest)
         ):
             return
         stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils())
-        jit_stencils_new = out / "jit_stencils.h.new"
+        jit_stencils_new = jit_stencils.parent / "jit_stencils.h.new"
         try:
             with jit_stencils_new.open("w") as file:
                 file.write(digest)

Tools/jit/build.py

@@ -8,7 +8,6 @@
 import _targets
 
 if __name__ == "__main__":
-    out = pathlib.Path.cwd().resolve()
     comment = f"$ {shlex.join([pathlib.Path(sys.executable).name] + sys.argv)}"
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
@@ -23,6 +22,20 @@
     parser.add_argument(
         "-f", "--force", action="store_true", help="force the entire JIT to be rebuilt"
     )
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        help="where to output generated files",
+        required=True,
+        type=lambda p: pathlib.Path(p).resolve(),
+    )
+    parser.add_argument(
+        "-p",
+        "--pyconfig-dir",
+        help="where to find pyconfig.h",
+        required=True,
+        type=lambda p: pathlib.Path(p).resolve(),
+    )
     parser.add_argument(
         "-v", "--verbose", action="store_true", help="echo commands as they are run"
     )
@@ -31,13 +44,13 @@
         target.debug = args.debug
         target.force = args.force
         target.verbose = args.verbose
+        target.pyconfig_dir = args.pyconfig_dir
         target.build(
-            out,
             comment=comment,
-            stencils_h=f"jit_stencils-{target.triple}.h",
             force=args.force,
+            jit_stencils=args.output_dir / f"jit_stencils-{target.triple}.h",
         )
-    jit_stencils_h = out / "jit_stencils.h"
+    jit_stencils_h = args.output_dir / "jit_stencils.h"
     lines = [f"// {comment}\n"]
     guard = "#if"
     for target in args.target:

configure

@@ -2776,7 +2776,7 @@ AS_VAR_IF([jit_flags],
           [],
           [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"])
            AS_VAR_SET([REGEN_JIT_COMMAND],
-                      ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host}"])
+                      ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir ."])
            AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"])
            AS_VAR_IF([Py_DEBUG],
                      [true],