From 29b3ce8355b7d141ef61fa28757b7ff389189604 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Wed, 29 Jan 2025 22:35:55 +0000 Subject: [PATCH 1/5] GH-128469: warn when libpython was loaded from outside the build directory (#128645) Co-authored-by: Peter Bierma --- Lib/test/test_getpath.py | 37 +++++++++++++++++++++++++++++++++++++ Modules/getpath.py | 13 +++++++++++++ 2 files changed, 50 insertions(+) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index f86df9d0d03485..ca7cee0c39872a 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -1,7 +1,13 @@ import copy import ntpath +import os import pathlib import posixpath +import shutil +import subprocess +import sys +import sysconfig +import tempfile import unittest from test.support import verbose @@ -864,6 +870,37 @@ def test_PYTHONHOME_in_venv(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) + +class RealGetPathTests(unittest.TestCase): + @unittest.skipUnless( + sysconfig.is_python_build(), + 'Test only available when running from the buildir', + ) + @unittest.skipUnless( + any(sys.platform.startswith(p) for p in ('linux', 'freebsd', 'centos')), + 'Test only support on Linux-like OS-es (support LD_LIBRARY_PATH)', + ) + @unittest.skipUnless( + sysconfig.get_config_var('LDLIBRARY') != sysconfig.get_config_var('LIBRARY'), + 'Test only available when using a dynamic libpython', + ) + def test_builddir_wrong_library_warning(self): + library_name = sysconfig.get_config_var('INSTSONAME') + with tempfile.TemporaryDirectory() as tmpdir: + shutil.copy2( + os.path.join(sysconfig.get_config_var('srcdir'), library_name), + os.path.join(tmpdir, library_name) + ) + env = os.environ.copy() + env['LD_LIBRARY_PATH'] = tmpdir + process = subprocess.run( + [sys.executable, '-c', ''], + env=env, check=True, capture_output=True, text=True, + ) + error_msg = 'The runtime library has been loaded from outside the build directory' + self.assertTrue(process.stderr.startswith(error_msg), process.stderr) + + # ****************************************************************************** DEFAULT_NAMESPACE = dict( diff --git a/Modules/getpath.py b/Modules/getpath.py index be2210345afbda..9d531e29becbc8 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -785,6 +785,19 @@ def search_up(prefix, *landmarks, test=isfile): base_exec_prefix = config.get('base_exec_prefix') or EXEC_PREFIX or base_prefix +# ****************************************************************************** +# MISC. RUNTIME WARNINGS +# ****************************************************************************** + +# When running Python from the build directory, if libpython is dynamically +# linked, the wrong library might be loaded. +if build_prefix and library and not dirname(abspath(library)).startswith(build_prefix): + msg = f'The runtime library has been loaded from outside the build directory ({library})!' + if os_name == 'posix': + msg += ' Consider setting LD_LIBRARY_PATH=. to force it to be loaded from the build directory.' + warn(msg) + + # ****************************************************************************** # SET pythonpath FROM _PTH FILE # ****************************************************************************** From c931d75831e4176b4876875c01e4b989c8915275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Wed, 29 Jan 2025 22:47:20 +0000 Subject: [PATCH 2/5] GH-127178: improve compatibility in `_sysconfig_vars_(...).json` (#128558) --- Lib/sysconfig/__init__.py | 61 +++++++++++++++++++++++++------------- Lib/sysconfig/__main__.py | 6 +++- Lib/test/test_sysconfig.py | 17 +++++++++-- 3 files changed, 61 insertions(+), 23 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 3c3c9796ec3307..69f72452c4069a 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -116,8 +116,10 @@ def _getuserbase(): if env_base: return env_base - # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories - if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories. + # Use _PYTHON_HOST_PLATFORM to get the correct platform when cross-compiling. + system_name = os.environ.get('_PYTHON_HOST_PLATFORM', sys.platform).split('-')[0] + if system_name in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -342,6 +344,18 @@ def get_makefile_filename(): return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') +def _import_from_directory(path, name): + if name not in sys.modules: + import importlib.machinery + import importlib.util + + spec = importlib.machinery.PathFinder.find_spec(name, [path]) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[name] = module + return sys.modules[name] + + def _get_sysconfigdata_name(): multiarch = getattr(sys.implementation, '_multiarch', '') return os.environ.get( @@ -349,27 +363,34 @@ def _get_sysconfigdata_name(): f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', ) -def _init_posix(vars): - """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see _generate_posix_vars() + +def _get_sysconfigdata(): + import importlib + name = _get_sysconfigdata_name() + path = os.environ.get('_PYTHON_SYSCONFIGDATA_PATH') + module = _import_from_directory(path, name) if path else importlib.import_module(name) - # For cross builds, the path to the target's sysconfigdata must be specified - # so it can be imported. It cannot be in PYTHONPATH, as foreign modules in - # sys.path can cause crashes when loaded by the host interpreter. - # Rely on truthiness as a valueless env variable is still an empty string. - # See OS X note in _generate_posix_vars re _sysconfigdata. - if (path := os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')): - from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES - from importlib.util import module_from_spec - spec = FileFinder(path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(name) - _temp = module_from_spec(spec) - spec.loader.exec_module(_temp) - else: - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - build_time_vars = _temp.build_time_vars + return module.build_time_vars + + +def _installation_is_relocated(): + """Is the Python installation running from a different prefix than what was targetted when building?""" + if os.name != 'posix': + raise NotImplementedError('sysconfig._installation_is_relocated() is currently only supported on POSIX') + + data = _get_sysconfigdata() + return ( + data['prefix'] != getattr(sys, 'base_prefix', '') + or data['exec_prefix'] != getattr(sys, 'base_exec_prefix', '') + ) + + +def _init_posix(vars): + """Initialize the module as appropriate for POSIX systems.""" # GH-126920: Make sure we don't overwrite any of the keys already set - vars.update(build_time_vars | vars) + vars.update(_get_sysconfigdata() | vars) + def _init_non_posix(vars): """Initialize the module as appropriate for NT""" diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index 10728c709e1811..bc2197cfe79402 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -232,10 +232,14 @@ def _generate_posix_vars(): print(f'Written {destfile}') + install_vars = get_config_vars() + # Fix config vars to match the values after install (of the default environment) + install_vars['projectbase'] = install_vars['BINDIR'] + install_vars['srcdir'] = install_vars['LIBPL'] # Write a JSON file with the output of sysconfig.get_config_vars jsonfile = os.path.join(pybuilddir, _get_json_data_name()) with open(jsonfile, 'w') as f: - json.dump(get_config_vars(), f, indent=2) + json.dump(install_vars, f, indent=2) print(f'Written {jsonfile}') diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 1002d90074599a..3950d8888d4c8c 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -650,8 +650,21 @@ def test_sysconfigdata_json(self): system_config_vars = get_config_vars() - # Ignore keys in the check - for key in ('projectbase', 'srcdir'): + ignore_keys = set() + # Keys dependent on Python being run outside the build directrory + if sysconfig.is_python_build(): + ignore_keys |= {'srcdir'} + # Keys dependent on the executable location + if os.path.dirname(sys.executable) != system_config_vars['BINDIR']: + ignore_keys |= {'projectbase'} + # Keys dependent on the environment (different inside virtual environments) + if sys.prefix != sys.base_prefix: + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase'} + # Keys dependent on Python being run from the prefix targetted when building (different on relocatable installs) + if sysconfig._installation_is_relocated(): + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase'} + + for key in ignore_keys: json_config_vars.pop(key) system_config_vars.pop(key) From 99849ee0d3ebcddc97b6aeaf389f43a12f541068 Mon Sep 17 00:00:00 2001 From: Matthew Hughes Date: Wed, 29 Jan 2025 23:24:09 +0000 Subject: [PATCH 3/5] gh-127432: Add CI job to cross build Python (#128380) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Filipe Laíns 🇵🇸 --- .github/workflows/build.yml | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 72c1618982b146..c10c5b4aa46ffb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -527,6 +527,45 @@ jobs: config_hash: ${{ needs.check_source.outputs.config_hash }} free-threading: ${{ matrix.free-threading }} + cross-build-linux: + name: Cross build Linux + runs-on: ubuntu-latest + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Set build dir + run: + # an absolute path outside of the working directoy + echo "BUILD_DIR=$(realpath ${{ github.workspace }}/../build)" >> "$GITHUB_ENV" + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure host build + run: ./configure --prefix="$BUILD_DIR/host-python" + - name: Install host Python + run: make -j8 install + - name: Run test subset with host build + run: | + "$BUILD_DIR/host-python/bin/python3" -m test test_sysconfig test_site test_embed + - name: Configure cross build + run: ./configure --prefix="$BUILD_DIR/cross-python" --with-build-python="$BUILD_DIR/host-python/bin/python3" + - name: Install cross Python + run: make -j8 install + - name: Run test subset with host build + run: | + "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed + # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: name: CIFuzz From a549f439384b4509b25639337ffea21c2e55d452 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 30 Jan 2025 01:02:31 +0100 Subject: [PATCH 4/5] gh-128779: Fix site venv() for system site-packages (#129184) --- Lib/site.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index 92bd1ccdadd924..9da8b6724e1cec 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -633,12 +633,9 @@ def venv(known_paths): # Doing this here ensures venv takes precedence over user-site addsitepackages(known_paths, [sys.prefix]) - # addsitepackages will process site_prefix again if its in PREFIXES, - # but that's ok; known_paths will prevent anything being added twice if system_site == "true": - PREFIXES.insert(0, sys.prefix) + PREFIXES += [sys.base_prefix, sys.base_exec_prefix] else: - PREFIXES = [sys.prefix] ENABLE_USER_SITE = False return known_paths From a4722449caccc42ad644611d02fbdb5005f601eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Thu, 30 Jan 2025 03:32:24 +0000 Subject: [PATCH 5/5] tests: add test.support.venv.VirtualEnvironmentMixin (#129461) --- Lib/test/support/venv.py | 12 ++++++++++++ Lib/test/test_sysconfig.py | 11 ++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/Lib/test/support/venv.py b/Lib/test/support/venv.py index 78e6a51ec1815e..7bfb9e4f3c479f 100644 --- a/Lib/test/support/venv.py +++ b/Lib/test/support/venv.py @@ -6,6 +6,7 @@ import sys import sysconfig import tempfile +import unittest import venv @@ -68,3 +69,14 @@ def run(self, *args, **subprocess_args): raise else: return result + + +class VirtualEnvironmentMixin: + def venv(self, name=None, **venv_create_args): + venv_name = self.id() + if name: + venv_name += f'-{name}' + return VirtualEnvironment.from_tmpdir( + prefix=f'{venv_name}-venv-', + **venv_create_args, + ) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 3950d8888d4c8c..3738914cf17de8 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -20,7 +20,7 @@ from test.support.import_helper import import_module from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink, change_cwd) -from test.support.venv import VirtualEnvironment +from test.support.venv import VirtualEnvironmentMixin import sysconfig from sysconfig import (get_paths, get_platform, get_config_vars, @@ -37,7 +37,7 @@ HAS_USER_BASE = sysconfig._HAS_USER_BASE -class TestSysConfig(unittest.TestCase): +class TestSysConfig(unittest.TestCase, VirtualEnvironmentMixin): def setUp(self): super(TestSysConfig, self).setUp() @@ -111,13 +111,6 @@ def _cleanup_testfn(self): elif os.path.isdir(path): shutil.rmtree(path) - def venv(self, **venv_create_args): - return VirtualEnvironment.from_tmpdir( - prefix=f'{self.id()}-venv-', - **venv_create_args, - ) - - def test_get_path_names(self): self.assertEqual(get_path_names(), sysconfig._SCHEME_KEYS)