Fix identification of venv interpreters. (#3114)

Fixes #3113

Author: jsirois

CHANGES.md

@@ -1,5 +1,13 @@
 # Release Notes
 
+## 2.91.1
+
+This release partially fixes an interpreter caching bug for CPython interpreters that have the same
+binary contents across patch versions with variance confined to `libpython`, other shared libraries
+and stdlib code.
+
+* Fix identification of venv interpreters. (#3114)
+
 ## 2.91.0
 
 This release improves editable support by honoring editable requests when creating venvs from

pex/interpreter.py

@@ -1091,8 +1091,12 @@ def _resolve_pyenv_shim(
         return binary
 
     @classmethod
-    def _spawn_from_binary_external(cls, binary):
-        # type: (str) -> SpawnedJob[PythonInterpreter]
+    def _spawn_from_binary_external(
+        cls,
+        binary,  # type: str
+        ignore_cached=False,  # type: bool
+    ):
+        # type: (...) -> SpawnedJob[PythonInterpreter]
 
         def create_interpreter(
             stdout,  # type: bytes
@@ -1114,6 +1118,8 @@ def create_interpreter(
             return interpreter
 
         cache_dir = InterpreterDir.create(binary)
+        if ignore_cached:
+            safe_rmtree(cache_dir)
         if os.path.isfile(cache_dir.interp_info_file):
             try:
                 with open(cache_dir.interp_info_file, "rb") as fp:
@@ -1221,8 +1227,12 @@ def hashbang_matches(fn):
         return None
 
     @classmethod
-    def _spawn_from_binary(cls, binary):
-        # type: (str) -> SpawnedJob[PythonInterpreter]
+    def _spawn_from_binary(
+        cls,
+        binary,  # type: str
+        ignore_cached=False,  # type: bool
+    ):
+        # type: (...) -> SpawnedJob[PythonInterpreter]
         canonicalized_binary = cls.canonicalize_path(binary)
         if not os.path.exists(canonicalized_binary):
             raise cls.InterpreterNotFound(
@@ -1231,16 +1241,17 @@ def _spawn_from_binary(cls, binary):
 
         # N.B.: The cache is written as the last step in PythonInterpreter instance initialization.
         cached_interpreter = cls._PYTHON_INTERPRETER_BY_NORMALIZED_PATH.get(canonicalized_binary)
-        if cached_interpreter is not None:
+        if cached_interpreter is not None and not ignore_cached:
             return SpawnedJob.completed(cached_interpreter)
-        return cls._spawn_from_binary_external(canonicalized_binary)
+        return cls._spawn_from_binary_external(canonicalized_binary, ignore_cached=ignore_cached)
 
     @classmethod
     def from_binary(
         cls,
         binary,  # type: str
         pyenv=None,  # type: Optional[Pyenv]
         cwd=None,  # type: Optional[str]
+        ignore_cached=False,  # type: bool
     ):
         # type: (...) -> PythonInterpreter
         """Create an interpreter from the given `binary`.
@@ -1250,14 +1261,19 @@ def from_binary(
                       Auto-detected by default.
         :param cwd: The cwd to use as a base to look for python version files from. The process cwd
                     by default.
+        :param ignore_cached: If the binary has already been identified, ignore the cached
+                              identification and re-identify.
         :return: an interpreter created from the given `binary`.
         """
         python = cls._resolve_pyenv_shim(binary, pyenv=pyenv, cwd=cwd)
         if python is None:
             raise cls.IdentificationError("The pyenv shim at {} is not active.".format(binary))
 
         try:
-            return cast(PythonInterpreter, cls._spawn_from_binary(python).await_result())
+            return cast(
+                PythonInterpreter,
+                cls._spawn_from_binary(python, ignore_cached=ignore_cached).await_result(),
+            )
         except Job.Error as e:
             raise cls.IdentificationError("Failed to identify {}: {}".format(binary, e))
 
@@ -1497,20 +1513,34 @@ def iter_base_candidate_binary_paths(interpreter):
                 if is_exe(candidate_binary_path):
                     yield candidate_binary_path
 
-        def is_same_interpreter(interpreter):
-            # type: (PythonInterpreter) -> bool
-            identity = interpreter._identity
-            return identity.version == version and identity.abi_tag == abi_tag
-
         resolution_path = []  # type: List[str]
         base_interpreter = self
         while base_interpreter.is_venv:
             resolved = None  # type: Optional[PythonInterpreter]
+            maybe_reinstalled_interpreters = []  # type: List[PythonInterpreter]
             for candidate_path in iter_base_candidate_binary_paths(base_interpreter):
                 resolved_interpreter = self.from_binary(candidate_path)
-                if is_same_interpreter(resolved_interpreter):
-                    resolved = resolved_interpreter
-                    break
+                if resolved_interpreter.abi_tag == abi_tag:
+                    if resolved_interpreter.version == version:
+                        resolved = resolved_interpreter
+                        break
+                    else:
+                        # N.B.: Different patch versions of Python can have the same `python`
+                        # binary contents and only differ in shared libraries and the stdlib. We
+                        # guard against that case here (i.e.: a CPython patch version upgrade or
+                        # downgrade) by busting the cache as a last resort before failing to
+                        # resolve a base interpreter.
+                        #
+                        # See: https://github.com/pex-tool/pex/issues/3113
+                        maybe_reinstalled_interpreters.append(resolved_interpreter)
+            if resolved is None:
+                for maybe_reinstalled_interpreter in maybe_reinstalled_interpreters:
+                    re_resolved_interpreter = self.from_binary(
+                        maybe_reinstalled_interpreter.binary, ignore_cached=True
+                    )
+                    if re_resolved_interpreter.version == version:
+                        resolved = re_resolved_interpreter
+                        break
             if resolved is None:
                 message = [
                     "Failed to resolve the base interpreter for the virtual environment at "
@@ -1528,7 +1558,7 @@ def is_same_interpreter(interpreter):
                     )
                 )
                 raise self.BaseInterpreterResolutionError("\n".join(message))
-            base_interpreter = resolved_interpreter
+            base_interpreter = resolved
             resolution_path.append(base_interpreter.binary)
         return base_interpreter
 
@@ -1551,6 +1581,11 @@ def free_threaded(self):
     def python(self):
         return self._identity.python
 
+    @property
+    def abi_tag(self):
+        # type: () -> str
+        return self._identity.abi_tag
+
     @property
     def version(self):
         # type: () -> Tuple[int, int, int]

pex/version.py

@@ -1,4 +1,4 @@
 # Copyright 2015 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = "2.91.0"
+__version__ = "2.91.1"

testing/__init__.py

@@ -641,9 +641,13 @@ def run_pyenv(
         )
 
 
-def ensure_python_distribution(version):
-    # type: (str) -> PyenvPythonDistribution
-    if version not in ALL_PY_VERSIONS:
+def ensure_python_distribution(
+    version,  # type: str
+    python_version=None,  # type: Optional[str]
+    allow_adhoc_version=False,  # type: bool
+):
+    # type: (...) -> PyenvPythonDistribution
+    if not allow_adhoc_version and version not in ALL_PY_VERSIONS:
         raise ValueError("Please constrain version to one of {}".format(ALL_PY_VERSIONS))
 
     if WINDOWS and _ALL_PY_VERSIONS_TO_VERSION_INFO[version][:2] < (3, 8):
@@ -669,7 +673,7 @@ def ensure_python_distribution(version):
     if WINDOWS:
         python = os.path.join(interpreter_location, "python.exe")
     else:
-        major, minor = version.split(".")[:2]
+        major, minor = (python_version or version).split(".")[:2]
         python = os.path.join(
             interpreter_location, "bin", "python{major}.{minor}".format(major=major, minor=minor)
         )

tests/integration/test_issue_3113.py

@@ -0,0 +1,135 @@
+# Copyright 2026 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import glob
+import os.path
+import shutil
+import subprocess
+from typing import Tuple
+
+import pytest
+
+from pex.common import safe_delete
+from pex.util import CacheHelper
+from testing import (
+    IS_LINUX_X86_64,
+    PyenvPythonDistribution,
+    ensure_python_distribution,
+    make_env,
+    run_pex_command,
+)
+from testing.pytest_utils.tmp import Tempdir
+
+
+@pytest.fixture
+def cowsay(tmpdir):
+    # type: (Tempdir) -> str
+
+    pex_root = tmpdir.join("pex-root")
+    cowsay = tmpdir.join("cowsay.pex")
+    run_pex_command(
+        args=[
+            "--pex-root",
+            pex_root,
+            "--runtime-pex-root",
+            pex_root,
+            "cowsay<6",
+            "-c",
+            "cowsay",
+            "-o",
+            cowsay,
+        ]
+    ).assert_success()
+    return cowsay
+
+
+def pypy_dist(
+    pyenv_version,  # type: str
+    python_version,  # type: Tuple[int, int, int]
+):
+    # type: (...) -> PyenvPythonDistribution
+    pypy_distribution = ensure_python_distribution(
+        pyenv_version,
+        python_version="{major}.{minor}".format(major=python_version[0], minor=python_version[1]),
+        allow_adhoc_version=True,
+    )
+    assert (
+        "{major}.{minor}.{patch}".format(
+            major=python_version[0], minor=python_version[1], patch=python_version[2]
+        )
+        == subprocess.check_output(
+            args=[
+                pypy_distribution.binary,
+                "-c",
+                "import sys; print('.'.join(map(str, sys.version_info[:3])))",
+            ]
+        )
+        .decode("utf-8")
+        .strip()
+    )
+    return pypy_distribution
+
+
+@pytest.fixture
+def pypy3_11_11_dist():
+    # type: () -> PyenvPythonDistribution
+    return pypy_dist("pypy3.11-7.3.19", python_version=(3, 11, 11))
+
+
+@pytest.fixture
+def pypy3_11_13_dist():
+    # type: () -> PyenvPythonDistribution
+    return pypy_dist("pypy3.11-7.3.20", python_version=(3, 11, 13))
+
+
+@pytest.mark.skipif(
+    not IS_LINUX_X86_64,
+    reason=(
+        "We only need to test this for one known-good pair of interpreters with matching binary "
+        "hash and differing patch versions. The pypy3.11-7.3.19 / pypy3.11-7.3.20 pair is known to "
+        "meet this criteria for x86_64 Linux."
+    ),
+)
+def test_interpreter_upgrade_same_binary_hash(
+    tmpdir,  # type: Tempdir
+    cowsay,  # type: str
+    pypy3_11_11_dist,  # type: PyenvPythonDistribution
+    pypy3_11_13_dist,  # type: PyenvPythonDistribution
+):
+    # type: (...) -> None
+
+    assert pypy3_11_11_dist.binary != pypy3_11_13_dist.binary
+    assert CacheHelper.hash(pypy3_11_11_dist.binary) == CacheHelper.hash(pypy3_11_13_dist.binary)
+
+    pypy_311_prefix = tmpdir.join("opt", "pypy")
+
+    def install_pypy_311_and_create_venv(
+        pypy_distribution,  # type: PyenvPythonDistribution
+        venv_dir,  # type: str
+    ):
+        # type: (...) -> str
+
+        if os.path.exists(pypy_311_prefix):
+            shutil.move(pypy_311_prefix, pypy_311_prefix + ".sav")
+        shutil.copytree(pypy_distribution.interpreter.prefix, pypy_311_prefix)
+        for binary in glob.glob(os.path.join(pypy_311_prefix, "bin", "*")):
+            if os.path.basename(binary) not in ("libpypy3.11-c.so", "pypy3.11"):
+                safe_delete(binary)
+        pypy_binary = os.path.join(pypy_311_prefix, "bin", "pypy3.11")
+        venv_path = tmpdir.join(venv_dir)
+        subprocess.check_call(args=[pypy_binary, "-m", "venv", venv_path])
+        return os.path.join(venv_path, "bin", "python")
+
+    pypy3_11_11_venv_binary = install_pypy_311_and_create_venv(pypy3_11_11_dist, "pypy3_11_11.venv")
+    assert b"| I am 3.11.11 |" in subprocess.check_output(
+        args=[pypy3_11_11_venv_binary, cowsay, "I am 3.11.11"],
+        env=make_env(PEX_PYTHON_PATH=pypy3_11_11_venv_binary),
+    )
+
+    pypy3_11_13_venv_binary = install_pypy_311_and_create_venv(pypy3_11_13_dist, "pypy3_11_13.venv")
+    assert b"| I am 3.11.13 |" in subprocess.check_output(
+        args=[pypy3_11_13_venv_binary, cowsay, "I am 3.11.13"],
+        env=make_env(PEX_PYTHON_PATH=pypy3_11_13_venv_binary),
+    )