Make the Unix build truly relocatable as CPython patch

Author: Chongkai Zhu

cpython-unix/build-cpython.sh

@@ -175,6 +175,12 @@ else
     patch -p1 -i ${ROOT}/patch-ctypes-callproc-legacy.patch
 fi
 
+if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then
+    patch -p1 -i ${ROOT}/patch-cpython-relocatable-sysconfig-3.13.patch
+elif [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_10}" ]; then
+    patch -p1 -i ${ROOT}/patch-cpython-relocatable-sysconfig-3.10.patch
+fi
+
 # On Windows, CPython looks for the Tcl/Tk libraries relative to the base prefix,
 # which we want. But on Unix, it doesn't. This patch applies similar behavior on Unix,
 # thereby ensuring that the Tcl/Tk libraries are found in the correct location.
@@ -883,6 +889,7 @@ fi
 # that a) it works on as many machines as possible b) doesn't leak details
 # about the build environment, which is non-portable.
 cat > ${ROOT}/hack_sysconfig.py << EOF
+import ast
 import json
 import os
 import sys
@@ -894,7 +901,7 @@ FREETHREADED = sysconfig.get_config_var("Py_GIL_DISABLED")
 MAJMIN = ".".join([str(sys.version_info[0]), str(sys.version_info[1])])
 LIB_SUFFIX = "t" if FREETHREADED else ""
 PYTHON_CONFIG = os.path.join(ROOT, "install", "bin", "python%s-config" % MAJMIN)
-PLATFORM_CONFIG = os.path.join(ROOT, sysconfig.get_config_var("LIBPL").lstrip("/"))
+PLATFORM_CONFIG = sysconfig.get_config_var("LIBPL")
 MAKEFILE = os.path.join(PLATFORM_CONFIG, "Makefile")
 SYSCONFIGDATA = os.path.join(
     ROOT,
@@ -925,51 +932,56 @@ def replace_in_all(search, replace):
     replace_in_file(SYSCONFIGDATA, search, replace)
 
 
-def replace_in_sysconfigdata(search, replace, keys):
-    """Replace a string in the sysconfigdata file for select keys."""
-    with open(SYSCONFIGDATA, "rb") as fh:
-        data = fh.read()
+def _find_build_time_vars_assign(module):
+    """Return the Assign node for 'build_time_vars = {...}' or None."""
+    for node in module.body:
+        if isinstance(node, ast.Assign):
+            for target in node.targets:
+                if isinstance(target, ast.Name) and target.id == "build_time_vars":
+                    return node
+    return None
 
-    globals_dict = {}
-    locals_dict = {}
-    exec(data, globals_dict, locals_dict)
-    build_time_vars = locals_dict['build_time_vars']
 
-    for key in keys:
-        if key in build_time_vars:
-            build_time_vars[key] = build_time_vars[key].replace(search, replace)
+def _patch_build_time_vars(assign_node, keys, search, replace):
+    """Patch plain string values for the given keys inside the dict literal."""
+    if not isinstance(assign_node.value, ast.Dict):
+        return
 
-    with open(SYSCONFIGDATA, "wb") as fh:
-        fh.write(b'# system configuration generated and used by the sysconfig module\n')
-        fh.write(('build_time_vars = %s' % json.dumps(build_time_vars, indent=4, sort_keys=True)).encode("utf-8"))
-        fh.close()
+    d: ast.Dict = assign_node.value
+    for key_node, val_node in zip(d.keys, d.values):
+        k = key_node.value
+        if k in keys and isinstance(val_node, ast.Constant) and isinstance(val_node.value, str):
+            val_node.value = val_node.value.replace(search, replace)
 
 
-def format_sysconfigdata():
-    """Reformat the sysconfigdata file to avoid implicit string concatenations.
+def replace_in_sysconfigdata(search, replace, keys):
+    """Replace a string in the sysconfigdata file for select keys."""
+    with open(SYSCONFIGDATA, "r", encoding="utf-8") as fh:
+        source = fh.read()
 
-    In some Python versions, the sysconfigdata file contains implicit string
-    concatenations that extend over multiple lines, which make string replacement
-    much harder. This function reformats the file to avoid this issue.
+    module = ast.parse(source)
+    assign = _find_build_time_vars_assign(module)
+    if assign is None:
+        # Nothing to do if build_time_vars isn't present.
+        return
 
-    See: https://github.com/python/cpython/blob/a03efb533a58fd13fb0cc7f4a5c02c8406a407bd/Mac/BuildScript/build-installer.py#L1360C1-L1385C15.
-    """
-    with open(SYSCONFIGDATA, "rb") as fh:
-        data = fh.read()
+    # Patch the dict
+    _patch_build_time_vars(assign, keys, search, replace)
 
-    globals_dict = {}
-    locals_dict = {}
-    exec(data, globals_dict, locals_dict)
-    build_time_vars = locals_dict['build_time_vars']
+    # Compute the textual prefix up to (but not including) the start of build_time_vars.
+    lines = source.splitlines()
+    start_line = assign.lineno  # 1-based
+    head_text = "\n".join(lines[: start_line - 1])
 
-    with open(SYSCONFIGDATA, "wb") as fh:
-        fh.write(b'# system configuration generated and used by the sysconfig module\n')
-        fh.write(('build_time_vars = %s' % json.dumps(build_time_vars, indent=4, sort_keys=True)).encode("utf-8"))
-        fh.close()
+    # Unparse the patched assignment
+    assign_src = ast.unparse(assign)
 
+    # Rewrite: preserved head + patched assignment + trailing newline
+    new_source = head_text + ("\n" if head_text and not head_text.endswith("\n") else "")
+    new_source += assign_src + "\n"
 
-# Format sysconfig to ensure that string replacements take effect.
-format_sysconfigdata()
+    with open(SYSCONFIGDATA, "w", encoding="utf-8") as fh:
+        fh.write(new_source)
 
 # Remove `-Werror=unguarded-availability-new` from `CFLAGS` and `CPPFLAGS`.
 # These flags are passed along when building extension modules. In that context,
@@ -1063,7 +1075,7 @@ metadata = {
     "python_paths_abstract": sysconfig.get_paths(expand=False),
     "python_exe": "install/bin/python%s%s" % (sysconfig.get_python_version(), sys.abiflags),
     "python_major_minor_version": sysconfig.get_python_version(),
-    "python_stdlib_platform_config": sysconfig.get_config_var("LIBPL").lstrip("/"),
+    "python_stdlib_platform_config": sysconfig.get_config_var("LIBPL"),
     "python_config_vars": {k: str(v) for k, v in sysconfig.get_config_vars().items()},
 }
 

cpython-unix/patch-cpython-relocatable-sysconfig-3.10.patch

@@ -0,0 +1,55 @@
+diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py
+index daf9f000060..ed70b771e56 100644
+--- a/Lib/sysconfig.py
++++ b/Lib/sysconfig.py
+@@ -1,6 +1,7 @@
+ """Access to Python's configuration information."""
+ 
+ import os
++import re
+ import sys
+ from os.path import pardir, realpath
+ 
+@@ -407,9 +408,29 @@ def _get_sysconfigdata_name():
+     )
+ 
+ 
++def _print_config_dict(d, stream):
++    # Build a regex that matches the prefix at the start of the
++    # string or following a whitespace character.
++    prefix = d.get("prefix", "/install")
++    install_prefix_pattern = re.compile(r"(^|\s)" + re.escape(prefix))
++    # The replacement string. Use a backreference \1 to keep the
++    # whitespace if it was present.
++    relocatable_path = r"\1{installed_base}"
++
++    print ("{", file=stream)
++    for k, v in sorted(d.items()):
++        if isinstance(v, str):
++            replaced, count = install_prefix_pattern.subn(relocatable_path, v)
++            if count:
++                value_literal = "f" + repr(replaced)
++                print(f"    {k!r}: {value_literal},", file=stream)
++                continue
++        print(f"    {k!r}: {v!r},", file=stream)
++    print ("}", file=stream)
++
++
+ def _generate_posix_vars():
+     """Generate the Python module containing build-time variables."""
+-    import pprint
+     vars = {}
+     # load the installed Makefile:
+     makefile = get_makefile_filename()
+@@ -463,8 +484,10 @@ def _generate_posix_vars():
+     with open(destfile, 'w', encoding='utf8') as f:
+         f.write('# system configuration generated and used by'
+                 ' the sysconfig module\n')
++        f.write('from pathlib import Path\n')
++        f.write('installed_base = str(Path(__file__).resolve().parent.parent.parent)\n')
+         f.write('build_time_vars = ')
+-        pprint.pprint(vars, stream=f)
++        _print_config_dict(vars, stream=f)
+ 
+     # Create file used for sys.path fixup -- see Modules/getpath.c
+     with open('pybuilddir.txt', 'w', encoding='utf8') as f:

cpython-unix/patch-cpython-relocatable-sysconfig-3.13.patch

@@ -0,0 +1,42 @@
+diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py
+index d7257b9d2d0..1209757b45b 100644
+--- a/Lib/sysconfig/__main__.py
++++ b/Lib/sysconfig/__main__.py
+@@ -1,4 +1,5 @@
+ import os
++import re
+ import sys
+ from sysconfig import (
+     _ALWAYS_STR,
+@@ -151,8 +152,22 @@ def _parse_makefile(filename, vars=None, keep_unresolved=True):
+ 
+ 
+ def _print_config_dict(d, stream):
++    # Build a regex that matches the prefix at the start of the
++    # string or following a whitespace character.
++    prefix = d.get("prefix", "/install")
++    install_prefix_pattern = re.compile(r"(^|\s)" + re.escape(prefix))
++    # The replacement string. Use a backreference \1 to keep the
++    # whitespace if it was present.
++    relocatable_path = r"\1{installed_base}"
++
+     print ("{", file=stream)
+     for k, v in sorted(d.items()):
++        if isinstance(v, str):
++            replaced, count = install_prefix_pattern.subn(relocatable_path, v)
++            if count:
++                value_literal = "f" + repr(replaced)
++                print(f"    {k!r}: {value_literal},", file=stream)
++                continue
+         print(f"    {k!r}: {v!r},", file=stream)
+     print ("}", file=stream)
+ 
+@@ -212,6 +227,8 @@ def _generate_posix_vars():
+     with open(destfile, 'w', encoding='utf8') as f:
+         f.write('# system configuration generated and used by'
+                 ' the sysconfig module\n')
++        f.write('from pathlib import Path\n')
++        f.write('installed_base = str(Path(__file__).resolve().parent.parent.parent)\n')
+         f.write('build_time_vars = ')
+         _print_config_dict(vars, stream=f)
+