Merge branch 'main' into locations-distutils-cfg-warning

Author: uranusjr

.github/workflows/ci.yml

@@ -30,7 +30,6 @@ jobs:
             tests:
               # Anything that's touching testable stuff
               - ".github/workflows/ci.yml"
-              - "tools/requirements/tests.txt"
               - "src/**"
               - "tests/**"
         if: github.event_name == 'pull_request'

.pre-commit-config.yaml

@@ -26,7 +26,6 @@ repos:
         ^src/pip/_internal/index|
         ^src/pip/_internal/models|
         ^src/pip/_internal/operations|
-        ^src/pip/_internal/req|
         ^src/pip/_internal/vcs|
         ^src/pip/_internal/\w+\.py$|
         # Tests

.readthedocs.yml

@@ -7,4 +7,4 @@ sphinx:
 python:
   version: 3.8
   install:
-    - requirements: tools/requirements/docs.txt
+    - requirements: docs/requirements.txt

MANIFEST.in

@@ -10,6 +10,7 @@ recursive-include  src/pip/_vendor *LICENSE*
 recursive-include  src/pip/_vendor *COPYING*
 
 include docs/docutils.conf
+include docs/requirements.txt
 
 exclude .coveragerc
 exclude .mailmap

tools/requirements/docs.txt renamed to docs/requirements.txt

@@ -0,0 +1,4 @@
+Add a ``feature_flag`` optional kwarg to the ``deprecated()`` function
+``pip._internal.utils.deprecation:deprecated``. Also formulate a corresponding canned
+message which suggests using the ``--use-feature={feature_flag}`` to test upcoming
+behavior.

news/10165.trivial.rst

@@ -24,9 +24,9 @@
     "protected-pip": "tools/tox_pip.py",
 }
 REQUIREMENTS = {
-    "docs": "tools/requirements/docs.txt",
-    "tests": "tools/requirements/tests.txt",
-    "common-wheels": "tools/requirements/tests-common_wheels.txt",
+    "docs": "docs/requirements.txt",
+    "tests": "tests/requirements.txt",
+    "common-wheels": "tests/requirements-common_wheels.txt",
 }
 
 AUTHORS_FILE = "AUTHORS.txt"

noxfile.py

@@ -4,9 +4,10 @@
 import pathlib
 import sys
 import sysconfig
-from typing import List, Optional
+from typing import Dict, Iterator, List, Optional, Tuple
 
 from pip._internal.models.scheme import SCHEME_KEYS, Scheme
+from pip._internal.utils.compat import WINDOWS
 from pip._internal.utils.deprecation import deprecated
 
 from . import _distutils, _sysconfig
@@ -42,6 +43,53 @@
     _MISMATCH_LEVEL = logging.WARNING
 
 
+def _looks_like_red_hat_patched_platlib_purelib(scheme: Dict[str, str]) -> bool:
+    platlib = scheme["platlib"]
+    if "/lib64/" not in platlib:
+        return False
+    unpatched = platlib.replace("/lib64/", "/lib/")
+    return unpatched.replace("$platbase/", "$base/") == scheme["purelib"]
+
+
+@functools.lru_cache(maxsize=None)
+def _looks_like_red_hat_patched() -> bool:
+    """Red Hat patches platlib in unix_prefix and unix_home, but not purelib.
+
+    This is the only way I can see to tell a Red Hat-patched Python.
+    """
+    from distutils.command.install import INSTALL_SCHEMES  # type: ignore
+
+    return all(
+        k in INSTALL_SCHEMES
+        and _looks_like_red_hat_patched_platlib_purelib(INSTALL_SCHEMES[k])
+        for k in ("unix_prefix", "unix_home")
+    )
+
+
+@functools.lru_cache(maxsize=None)
+def _looks_like_debian_patched() -> bool:
+    """Debian adds two additional schemes."""
+    from distutils.command.install import INSTALL_SCHEMES  # type: ignore
+
+    return "deb_system" in INSTALL_SCHEMES and "unix_local" in INSTALL_SCHEMES
+
+
+def _fix_abiflags(parts: Tuple[str]) -> Iterator[str]:
+    ldversion = sysconfig.get_config_var("LDVERSION")
+    abiflags: str = getattr(sys, "abiflags", None)
+
+    # LDVERSION does not end with sys.abiflags. Just return the path unchanged.
+    if not ldversion or not abiflags or not ldversion.endswith(abiflags):
+        yield from parts
+        return
+
+    # Strip sys.abiflags from LDVERSION-based path components.
+    for part in parts:
+        if part.endswith(ldversion):
+            part = part[: (0 - len(abiflags))]
+        yield part
+
+
 def _default_base(*, user: bool) -> str:
     if user:
         base = sysconfig.get_config_var("userbase")
@@ -145,12 +193,42 @@ def get_scheme(
             user
             and is_osx_framework()
             and k == "headers"
-            and old_v.parent == new_v
-            and old_v.name.startswith("python")
+            and old_v.parent.parent == new_v.parent
+            and old_v.parent.name.startswith("python")
         )
         if skip_osx_framework_user_special_case:
             continue
 
+        # On Red Hat and derived Linux distributions, distutils is patched to
+        # use "lib64" instead of "lib" for platlib.
+        if k == "platlib" and _looks_like_red_hat_patched():
+            continue
+
+        # Both Debian and Red Hat patch Python to place the system site under
+        # /usr/local instead of /usr. Debian also places lib in dist-packages
+        # instead of site-packages, but the /usr/local check should cover it.
+        skip_linux_system_special_case = (
+            not (user or home or prefix)
+            and old_v.parts[1:3] == ("usr", "local")
+            and len(new_v.parts) > 1
+            and new_v.parts[1] == "usr"
+            and (len(new_v.parts) < 3 or new_v.parts[2] != "local")
+            and (_looks_like_red_hat_patched() or _looks_like_debian_patched())
+        )
+        if skip_linux_system_special_case:
+            continue
+
+        # On Python 3.7 and earlier, sysconfig does not include sys.abiflags in
+        # the "pythonX.Y" part of the path, but distutils does.
+        skip_sysconfig_abiflag_bug = (
+            sys.version_info < (3, 8)
+            and not WINDOWS
+            and k in ("headers", "platlib", "purelib")
+            and tuple(_fix_abiflags(old_v.parts)) == new_v.parts
+        )
+        if skip_sysconfig_abiflag_bug:
+            continue
+
         warning_contexts.append((old_v, new_v, f"scheme.{k}"))
 
     if not warning_contexts:

src/pip/_internal/locations/__init__.py

@@ -1,3 +1,4 @@
+import functools
 import os
 import site
 import sys
@@ -46,5 +47,6 @@ def get_src_prefix() -> str:
     user_site = site.USER_SITE
 
 
+@functools.lru_cache(maxsize=None)
 def is_osx_framework() -> bool:
     return bool(sysconfig.get_config_var("PYTHONFRAMEWORK"))

src/pip/_internal/locations/base.py

@@ -9,8 +9,10 @@
 from .req_set import RequirementSet
 
 __all__ = [
-    "RequirementSet", "InstallRequirement",
-    "parse_requirements", "install_given_reqs",
+    "RequirementSet",
+    "InstallRequirement",
+    "parse_requirements",
+    "install_given_reqs",
 ]
 
 logger = logging.getLogger(__name__)
@@ -52,20 +54,18 @@ def install_given_reqs(
 
     if to_install:
         logger.info(
-            'Installing collected packages: %s',
-            ', '.join(to_install.keys()),
+            "Installing collected packages: %s",
+            ", ".join(to_install.keys()),
         )
 
     installed = []
 
     with indent_log():
         for req_name, requirement in to_install.items():
             if requirement.should_reinstall:
-                logger.info('Attempting uninstall: %s', req_name)
+                logger.info("Attempting uninstall: %s", req_name)
                 with indent_log():
-                    uninstalled_pathset = requirement.uninstall(
-                        auto_confirm=True
-                    )
+                    uninstalled_pathset = requirement.uninstall(auto_confirm=True)
             else:
                 uninstalled_pathset = None