[pull] main from python:main

.github/workflows/build.yml

@@ -527,6 +527,45 @@ jobs:
       config_hash: ${{ needs.check_source.outputs.config_hash }}
       free-threading: ${{ matrix.free-threading }}
 
+  cross-build-linux:
+    name: Cross build Linux
+    runs-on: ubuntu-latest
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Runner image version
+        run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV"
+      - name: Restore config.cache
+        uses: actions/cache@v4
+        with:
+          path: config.cache
+          key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
+      - name: Register gcc problem matcher
+        run: echo "::add-matcher::.github/problem-matchers/gcc.json"
+      - name: Set build dir
+        run:
+          # an absolute path outside of the working directoy
+          echo "BUILD_DIR=$(realpath ${{ github.workspace }}/../build)" >> "$GITHUB_ENV"
+      - name: Install Dependencies
+        run: sudo ./.github/workflows/posix-deps-apt.sh
+      - name: Configure host build
+        run: ./configure --prefix="$BUILD_DIR/host-python"
+      - name: Install host Python
+        run: make -j8 install
+      - name: Run test subset with host build
+        run: |
+          "$BUILD_DIR/host-python/bin/python3" -m test test_sysconfig test_site test_embed
+      - name: Configure cross build
+        run: ./configure --prefix="$BUILD_DIR/cross-python" --with-build-python="$BUILD_DIR/host-python/bin/python3"
+      - name: Install cross Python
+        run: make -j8 install
+      - name: Run test subset with host build
+        run: |
+          "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed
+
   # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
   cifuzz:
     name: CIFuzz

Lib/site.py

@@ -633,12 +633,9 @@ def venv(known_paths):
         # Doing this here ensures venv takes precedence over user-site
         addsitepackages(known_paths, [sys.prefix])
 
-        # addsitepackages will process site_prefix again if its in PREFIXES,
-        # but that's ok; known_paths will prevent anything being added twice
         if system_site == "true":
-            PREFIXES.insert(0, sys.prefix)
+            PREFIXES += [sys.base_prefix, sys.base_exec_prefix]
         else:
-            PREFIXES = [sys.prefix]
             ENABLE_USER_SITE = False
 
     return known_paths

Lib/sysconfig/__init__.py

@@ -116,8 +116,10 @@ def _getuserbase():
     if env_base:
         return env_base
 
-    # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories
-    if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}:
+    # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories.
+    # Use _PYTHON_HOST_PLATFORM to get the correct platform when cross-compiling.
+    system_name = os.environ.get('_PYTHON_HOST_PLATFORM', sys.platform).split('-')[0]
+    if system_name in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}:
         return None
 
     def joinuser(*args):
@@ -342,34 +344,53 @@ def get_makefile_filename():
     return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile')
 
 
+def _import_from_directory(path, name):
+    if name not in sys.modules:
+        import importlib.machinery
+        import importlib.util
+
+        spec = importlib.machinery.PathFinder.find_spec(name, [path])
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        sys.modules[name] = module
+    return sys.modules[name]
+
+
 def _get_sysconfigdata_name():
     multiarch = getattr(sys.implementation, '_multiarch', '')
     return os.environ.get(
         '_PYTHON_SYSCONFIGDATA_NAME',
         f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}',
     )
 
-def _init_posix(vars):
-    """Initialize the module as appropriate for POSIX systems."""
-    # _sysconfigdata is generated at build time, see _generate_posix_vars()
+
+def _get_sysconfigdata():
+    import importlib
+
     name = _get_sysconfigdata_name()
+    path = os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')
+    module = _import_from_directory(path, name) if path else importlib.import_module(name)
 
-    # For cross builds, the path to the target's sysconfigdata must be specified
-    # so it can be imported. It cannot be in PYTHONPATH, as foreign modules in
-    # sys.path can cause crashes when loaded by the host interpreter.
-    # Rely on truthiness as a valueless env variable is still an empty string.
-    # See OS X note in _generate_posix_vars re _sysconfigdata.
-    if (path := os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')):
-        from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES
-        from importlib.util import module_from_spec
-        spec = FileFinder(path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(name)
-        _temp = module_from_spec(spec)
-        spec.loader.exec_module(_temp)
-    else:
-        _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0)
-    build_time_vars = _temp.build_time_vars
+    return module.build_time_vars
+
+
+def _installation_is_relocated():
+    """Is the Python installation running from a different prefix than what was targetted when building?"""
+    if os.name != 'posix':
+        raise NotImplementedError('sysconfig._installation_is_relocated() is currently only supported on POSIX')
+
+    data = _get_sysconfigdata()
+    return (
+        data['prefix'] != getattr(sys, 'base_prefix', '')
+        or data['exec_prefix'] != getattr(sys, 'base_exec_prefix', '')
+    )
+
+
+def _init_posix(vars):
+    """Initialize the module as appropriate for POSIX systems."""
     # GH-126920: Make sure we don't overwrite any of the keys already set
-    vars.update(build_time_vars | vars)
+    vars.update(_get_sysconfigdata() | vars)
+
 
 def _init_non_posix(vars):
     """Initialize the module as appropriate for NT"""

Lib/sysconfig/__main__.py

@@ -232,10 +232,14 @@ def _generate_posix_vars():
 
     print(f'Written {destfile}')
 
+    install_vars = get_config_vars()
+    # Fix config vars to match the values after install (of the default environment)
+    install_vars['projectbase'] = install_vars['BINDIR']
+    install_vars['srcdir'] = install_vars['LIBPL']
     # Write a JSON file with the output of sysconfig.get_config_vars
     jsonfile = os.path.join(pybuilddir, _get_json_data_name())
     with open(jsonfile, 'w') as f:
-        json.dump(get_config_vars(), f, indent=2)
+        json.dump(install_vars, f, indent=2)
 
     print(f'Written {jsonfile}')
 

Lib/test/support/venv.py

@@ -6,6 +6,7 @@
 import sys
 import sysconfig
 import tempfile
+import unittest
 import venv
 
 
@@ -68,3 +69,14 @@ def run(self, *args, **subprocess_args):
             raise
         else:
             return result
+
+
+class VirtualEnvironmentMixin:
+    def venv(self, name=None, **venv_create_args):
+        venv_name = self.id()
+        if name:
+            venv_name += f'-{name}'
+        return VirtualEnvironment.from_tmpdir(
+            prefix=f'{venv_name}-venv-',
+            **venv_create_args,
+        )

Lib/test/test_getpath.py

@@ -1,7 +1,13 @@
 import copy
 import ntpath
+import os
 import pathlib
 import posixpath
+import shutil
+import subprocess
+import sys
+import sysconfig
+import tempfile
 import unittest
 
 from test.support import verbose
@@ -864,6 +870,37 @@ def test_PYTHONHOME_in_venv(self):
         actual = getpath(ns, expected)
         self.assertEqual(expected, actual)
 
+
+class RealGetPathTests(unittest.TestCase):
+    @unittest.skipUnless(
+        sysconfig.is_python_build(),
+        'Test only available when running from the buildir',
+    )
+    @unittest.skipUnless(
+        any(sys.platform.startswith(p) for p in ('linux', 'freebsd', 'centos')),
+        'Test only support on Linux-like OS-es (support LD_LIBRARY_PATH)',
+    )
+    @unittest.skipUnless(
+        sysconfig.get_config_var('LDLIBRARY') != sysconfig.get_config_var('LIBRARY'),
+        'Test only available when using a dynamic libpython',
+    )
+    def test_builddir_wrong_library_warning(self):
+        library_name = sysconfig.get_config_var('INSTSONAME')
+        with tempfile.TemporaryDirectory() as tmpdir:
+            shutil.copy2(
+                os.path.join(sysconfig.get_config_var('srcdir'), library_name),
+                os.path.join(tmpdir, library_name)
+            )
+            env = os.environ.copy()
+            env['LD_LIBRARY_PATH'] = tmpdir
+            process = subprocess.run(
+                [sys.executable, '-c', ''],
+                env=env, check=True, capture_output=True, text=True,
+            )
+        error_msg = 'The runtime library has been loaded from outside the build directory'
+        self.assertTrue(process.stderr.startswith(error_msg), process.stderr)
+
+
 # ******************************************************************************
 
 DEFAULT_NAMESPACE = dict(

Lib/test/test_sysconfig.py

@@ -20,7 +20,7 @@
 from test.support.import_helper import import_module
 from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink,
                                     change_cwd)
-from test.support.venv import VirtualEnvironment
+from test.support.venv import VirtualEnvironmentMixin
 
 import sysconfig
 from sysconfig import (get_paths, get_platform, get_config_vars,
@@ -37,7 +37,7 @@
 HAS_USER_BASE = sysconfig._HAS_USER_BASE
 
 
-class TestSysConfig(unittest.TestCase):
+class TestSysConfig(unittest.TestCase, VirtualEnvironmentMixin):
 
     def setUp(self):
         super(TestSysConfig, self).setUp()
@@ -111,13 +111,6 @@ def _cleanup_testfn(self):
         elif os.path.isdir(path):
             shutil.rmtree(path)
 
-    def venv(self, **venv_create_args):
-        return VirtualEnvironment.from_tmpdir(
-            prefix=f'{self.id()}-venv-',
-            **venv_create_args,
-        )
-
-
     def test_get_path_names(self):
         self.assertEqual(get_path_names(), sysconfig._SCHEME_KEYS)
 
@@ -650,8 +643,21 @@ def test_sysconfigdata_json(self):
 
         system_config_vars = get_config_vars()
 
-        # Ignore keys in the check
-        for key in ('projectbase', 'srcdir'):
+        ignore_keys = set()
+        # Keys dependent on Python being run outside the build directrory
+        if sysconfig.is_python_build():
+            ignore_keys |= {'srcdir'}
+        # Keys dependent on the executable location
+        if os.path.dirname(sys.executable) != system_config_vars['BINDIR']:
+            ignore_keys |= {'projectbase'}
+        # Keys dependent on the environment (different inside virtual environments)
+        if sys.prefix != sys.base_prefix:
+            ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase'}
+        # Keys dependent on Python being run from the prefix targetted when building (different on relocatable installs)
+        if sysconfig._installation_is_relocated():
+            ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase'}
+
+        for key in ignore_keys:
             json_config_vars.pop(key)
             system_config_vars.pop(key)
 

Modules/getpath.py

@@ -785,6 +785,19 @@ def search_up(prefix, *landmarks, test=isfile):
     base_exec_prefix = config.get('base_exec_prefix') or EXEC_PREFIX or base_prefix
 
 
+# ******************************************************************************
+# MISC. RUNTIME WARNINGS
+# ******************************************************************************
+
+# When running Python from the build directory, if libpython is dynamically
+# linked, the wrong library might be loaded.
+if build_prefix and library and not dirname(abspath(library)).startswith(build_prefix):
+    msg = f'The runtime library has been loaded from outside the build directory ({library})!'
+    if os_name == 'posix':
+        msg += ' Consider setting LD_LIBRARY_PATH=. to force it to be loaded from the build directory.'
+    warn(msg)
+
+
 # ******************************************************************************
 # SET pythonpath FROM _PTH FILE
 # ******************************************************************************