diff --git a/CHANGES.md b/CHANGES.md index efb683c1c..96625e48e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,13 @@ # Release Notes +## 2.91.1 + +This release partially fixes an interpreter caching bug for CPython interpreters that have the same +binary contents across patch versions with variance confined to `libpython`, other shared libraries +and stdlib code. + +* Fix identification of venv interpreters. (#3114) + ## 2.91.0 This release improves editable support by honoring editable requests when creating venvs from diff --git a/pex/interpreter.py b/pex/interpreter.py index 7d7e31737..326a0023c 100644 --- a/pex/interpreter.py +++ b/pex/interpreter.py @@ -1091,8 +1091,12 @@ def _resolve_pyenv_shim( return binary @classmethod - def _spawn_from_binary_external(cls, binary): - # type: (str) -> SpawnedJob[PythonInterpreter] + def _spawn_from_binary_external( + cls, + binary, # type: str + ignore_cached=False, # type: bool + ): + # type: (...) -> SpawnedJob[PythonInterpreter] def create_interpreter( stdout, # type: bytes @@ -1114,6 +1118,8 @@ def create_interpreter( return interpreter cache_dir = InterpreterDir.create(binary) + if ignore_cached: + safe_rmtree(cache_dir) if os.path.isfile(cache_dir.interp_info_file): try: with open(cache_dir.interp_info_file, "rb") as fp: @@ -1221,8 +1227,12 @@ def hashbang_matches(fn): return None @classmethod - def _spawn_from_binary(cls, binary): - # type: (str) -> SpawnedJob[PythonInterpreter] + def _spawn_from_binary( + cls, + binary, # type: str + ignore_cached=False, # type: bool + ): + # type: (...) -> SpawnedJob[PythonInterpreter] canonicalized_binary = cls.canonicalize_path(binary) if not os.path.exists(canonicalized_binary): raise cls.InterpreterNotFound( @@ -1231,9 +1241,9 @@ def _spawn_from_binary(cls, binary): # N.B.: The cache is written as the last step in PythonInterpreter instance initialization. cached_interpreter = cls._PYTHON_INTERPRETER_BY_NORMALIZED_PATH.get(canonicalized_binary) - if cached_interpreter is not None: + if cached_interpreter is not None and not ignore_cached: return SpawnedJob.completed(cached_interpreter) - return cls._spawn_from_binary_external(canonicalized_binary) + return cls._spawn_from_binary_external(canonicalized_binary, ignore_cached=ignore_cached) @classmethod def from_binary( @@ -1241,6 +1251,7 @@ def from_binary( binary, # type: str pyenv=None, # type: Optional[Pyenv] cwd=None, # type: Optional[str] + ignore_cached=False, # type: bool ): # type: (...) -> PythonInterpreter """Create an interpreter from the given `binary`. @@ -1250,6 +1261,8 @@ def from_binary( Auto-detected by default. :param cwd: The cwd to use as a base to look for python version files from. The process cwd by default. + :param ignore_cached: If the binary has already been identified, ignore the cached + identification and re-identify. :return: an interpreter created from the given `binary`. """ python = cls._resolve_pyenv_shim(binary, pyenv=pyenv, cwd=cwd) @@ -1257,7 +1270,10 @@ def from_binary( raise cls.IdentificationError("The pyenv shim at {} is not active.".format(binary)) try: - return cast(PythonInterpreter, cls._spawn_from_binary(python).await_result()) + return cast( + PythonInterpreter, + cls._spawn_from_binary(python, ignore_cached=ignore_cached).await_result(), + ) except Job.Error as e: raise cls.IdentificationError("Failed to identify {}: {}".format(binary, e)) @@ -1497,20 +1513,34 @@ def iter_base_candidate_binary_paths(interpreter): if is_exe(candidate_binary_path): yield candidate_binary_path - def is_same_interpreter(interpreter): - # type: (PythonInterpreter) -> bool - identity = interpreter._identity - return identity.version == version and identity.abi_tag == abi_tag - resolution_path = [] # type: List[str] base_interpreter = self while base_interpreter.is_venv: resolved = None # type: Optional[PythonInterpreter] + maybe_reinstalled_interpreters = [] # type: List[PythonInterpreter] for candidate_path in iter_base_candidate_binary_paths(base_interpreter): resolved_interpreter = self.from_binary(candidate_path) - if is_same_interpreter(resolved_interpreter): - resolved = resolved_interpreter - break + if resolved_interpreter.abi_tag == abi_tag: + if resolved_interpreter.version == version: + resolved = resolved_interpreter + break + else: + # N.B.: Different patch versions of Python can have the same `python` + # binary contents and only differ in shared libraries and the stdlib. We + # guard against that case here (i.e.: a CPython patch version upgrade or + # downgrade) by busting the cache as a last resort before failing to + # resolve a base interpreter. + # + # See: https://github.com/pex-tool/pex/issues/3113 + maybe_reinstalled_interpreters.append(resolved_interpreter) + if resolved is None: + for maybe_reinstalled_interpreter in maybe_reinstalled_interpreters: + re_resolved_interpreter = self.from_binary( + maybe_reinstalled_interpreter.binary, ignore_cached=True + ) + if re_resolved_interpreter.version == version: + resolved = re_resolved_interpreter + break if resolved is None: message = [ "Failed to resolve the base interpreter for the virtual environment at " @@ -1528,7 +1558,7 @@ def is_same_interpreter(interpreter): ) ) raise self.BaseInterpreterResolutionError("\n".join(message)) - base_interpreter = resolved_interpreter + base_interpreter = resolved resolution_path.append(base_interpreter.binary) return base_interpreter @@ -1551,6 +1581,11 @@ def free_threaded(self): def python(self): return self._identity.python + @property + def abi_tag(self): + # type: () -> str + return self._identity.abi_tag + @property def version(self): # type: () -> Tuple[int, int, int] diff --git a/pex/version.py b/pex/version.py index d2acc01c5..518e5899f 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.91.0" +__version__ = "2.91.1" diff --git a/testing/__init__.py b/testing/__init__.py index 5187397c3..7c92aa88b 100644 --- a/testing/__init__.py +++ b/testing/__init__.py @@ -641,9 +641,13 @@ def run_pyenv( ) -def ensure_python_distribution(version): - # type: (str) -> PyenvPythonDistribution - if version not in ALL_PY_VERSIONS: +def ensure_python_distribution( + version, # type: str + python_version=None, # type: Optional[str] + allow_adhoc_version=False, # type: bool +): + # type: (...) -> PyenvPythonDistribution + if not allow_adhoc_version and version not in ALL_PY_VERSIONS: raise ValueError("Please constrain version to one of {}".format(ALL_PY_VERSIONS)) if WINDOWS and _ALL_PY_VERSIONS_TO_VERSION_INFO[version][:2] < (3, 8): @@ -669,7 +673,7 @@ def ensure_python_distribution(version): if WINDOWS: python = os.path.join(interpreter_location, "python.exe") else: - major, minor = version.split(".")[:2] + major, minor = (python_version or version).split(".")[:2] python = os.path.join( interpreter_location, "bin", "python{major}.{minor}".format(major=major, minor=minor) ) diff --git a/tests/integration/test_issue_3113.py b/tests/integration/test_issue_3113.py new file mode 100644 index 000000000..77a7d7526 --- /dev/null +++ b/tests/integration/test_issue_3113.py @@ -0,0 +1,135 @@ +# Copyright 2026 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import glob +import os.path +import shutil +import subprocess +from typing import Tuple + +import pytest + +from pex.common import safe_delete +from pex.util import CacheHelper +from testing import ( + IS_LINUX_X86_64, + PyenvPythonDistribution, + ensure_python_distribution, + make_env, + run_pex_command, +) +from testing.pytest_utils.tmp import Tempdir + + +@pytest.fixture +def cowsay(tmpdir): + # type: (Tempdir) -> str + + pex_root = tmpdir.join("pex-root") + cowsay = tmpdir.join("cowsay.pex") + run_pex_command( + args=[ + "--pex-root", + pex_root, + "--runtime-pex-root", + pex_root, + "cowsay<6", + "-c", + "cowsay", + "-o", + cowsay, + ] + ).assert_success() + return cowsay + + +def pypy_dist( + pyenv_version, # type: str + python_version, # type: Tuple[int, int, int] +): + # type: (...) -> PyenvPythonDistribution + pypy_distribution = ensure_python_distribution( + pyenv_version, + python_version="{major}.{minor}".format(major=python_version[0], minor=python_version[1]), + allow_adhoc_version=True, + ) + assert ( + "{major}.{minor}.{patch}".format( + major=python_version[0], minor=python_version[1], patch=python_version[2] + ) + == subprocess.check_output( + args=[ + pypy_distribution.binary, + "-c", + "import sys; print('.'.join(map(str, sys.version_info[:3])))", + ] + ) + .decode("utf-8") + .strip() + ) + return pypy_distribution + + +@pytest.fixture +def pypy3_11_11_dist(): + # type: () -> PyenvPythonDistribution + return pypy_dist("pypy3.11-7.3.19", python_version=(3, 11, 11)) + + +@pytest.fixture +def pypy3_11_13_dist(): + # type: () -> PyenvPythonDistribution + return pypy_dist("pypy3.11-7.3.20", python_version=(3, 11, 13)) + + +@pytest.mark.skipif( + not IS_LINUX_X86_64, + reason=( + "We only need to test this for one known-good pair of interpreters with matching binary " + "hash and differing patch versions. The pypy3.11-7.3.19 / pypy3.11-7.3.20 pair is known to " + "meet this criteria for x86_64 Linux." + ), +) +def test_interpreter_upgrade_same_binary_hash( + tmpdir, # type: Tempdir + cowsay, # type: str + pypy3_11_11_dist, # type: PyenvPythonDistribution + pypy3_11_13_dist, # type: PyenvPythonDistribution +): + # type: (...) -> None + + assert pypy3_11_11_dist.binary != pypy3_11_13_dist.binary + assert CacheHelper.hash(pypy3_11_11_dist.binary) == CacheHelper.hash(pypy3_11_13_dist.binary) + + pypy_311_prefix = tmpdir.join("opt", "pypy") + + def install_pypy_311_and_create_venv( + pypy_distribution, # type: PyenvPythonDistribution + venv_dir, # type: str + ): + # type: (...) -> str + + if os.path.exists(pypy_311_prefix): + shutil.move(pypy_311_prefix, pypy_311_prefix + ".sav") + shutil.copytree(pypy_distribution.interpreter.prefix, pypy_311_prefix) + for binary in glob.glob(os.path.join(pypy_311_prefix, "bin", "*")): + if os.path.basename(binary) not in ("libpypy3.11-c.so", "pypy3.11"): + safe_delete(binary) + pypy_binary = os.path.join(pypy_311_prefix, "bin", "pypy3.11") + venv_path = tmpdir.join(venv_dir) + subprocess.check_call(args=[pypy_binary, "-m", "venv", venv_path]) + return os.path.join(venv_path, "bin", "python") + + pypy3_11_11_venv_binary = install_pypy_311_and_create_venv(pypy3_11_11_dist, "pypy3_11_11.venv") + assert b"| I am 3.11.11 |" in subprocess.check_output( + args=[pypy3_11_11_venv_binary, cowsay, "I am 3.11.11"], + env=make_env(PEX_PYTHON_PATH=pypy3_11_11_venv_binary), + ) + + pypy3_11_13_venv_binary = install_pypy_311_and_create_venv(pypy3_11_13_dist, "pypy3_11_13.venv") + assert b"| I am 3.11.13 |" in subprocess.check_output( + args=[pypy3_11_13_venv_binary, cowsay, "I am 3.11.13"], + env=make_env(PEX_PYTHON_PATH=pypy3_11_13_venv_binary), + )