Merge remote-tracking branch 'origin/main' into fix-direct-preference

Author: notatallshaw

.github/workflows/release.yml

@@ -36,7 +36,7 @@ jobs:
 
     steps:
       - name: Download all the dists
-        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
+        uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4
         with:
           name: python-package-distributions
           path: dist/

docs/html/topics/more-dependency-resolution.md

@@ -132,6 +132,8 @@ operations:
 * `get_preference` - this provides information to the resolver to help it choose
   which requirement to look at "next" when working through the resolution
   process.
+* `narrow_requirement_selection` - this provides a way to limit the number of
+  identifiers passed to `get_preference`.
 * `find_matches` - given a set of constraints, determine what candidates exist
   that satisfy them. This is essentially where the finder interacts with the
   resolver.
@@ -140,19 +142,26 @@ operations:
 * `get_dependencies` - get the dependency metadata for a candidate. This is
   the implementation of the process of getting and reading package metadata.
 
-Of these methods, the only non-trivial one is the `get_preference` method. This
-implements the heuristics used to guide the resolution, telling it which
-requirement to try to satisfy next. It's this method that is responsible for
-trying to guess which route through the dependency tree will be most productive.
-As noted above, it's doing this with limited information. See the following
-diagram
+Of these methods, the only non-trivial ones are the `get_preference` and
+`narrow_requirement_selection` methods. These implement heuristics used
+to guide the resolution, telling it which requirement to try to satisfy next.
+It's these methods that are responsible for trying to guess which route through
+the dependency tree will be most productive. As noted above, it's doing this
+with limited information. See the following diagram:
 
 ![](deps.png)
 
 When the provider is asked to choose between the red requirements (A->B and
 A->C) it doesn't know anything about the dependencies of B or C (i.e., the
 grey parts of the graph).
 
+Pip's current implementation of the provider implements
+`narrow_requirement_selection` as follows:
+
+* If Requires-Python is present only consider that
+* If there are causes of resolution conflict (backtrack causes) then
+  only consider them until there are no longer any resolution conflicts
+
 Pip's current implementation of the provider implements `get_preference` as
 follows:
 

news/13135.feature.rst

@@ -0,0 +1,3 @@
+Use :pep:`753` "Well-known Project URLs in Metadata" normalization rules when
+identifying an equivalent project URL to replace a missing ``Home-Page`` field
+in ``pip show``.

news/13229.bugfix.rst

@@ -0,0 +1,4 @@
+Parse wheel filenames according to `binary distribution format specification
+<https://packaging.python.org/en/latest/specifications/binary-distribution-format/#file-format>`_.
+When a filename doesn't match the spec a deprecation warning is emitted and the
+filename is parsed using the old method.

news/13253.feature.rst

@@ -0,0 +1,2 @@
+Speed up resolution by first only considering the preference of
+candidates that must be required to complete the resolution.

src/pip/_internal/commands/show.py

@@ -1,4 +1,5 @@
 import logging
+import string
 from optparse import Values
 from typing import Generator, Iterable, Iterator, List, NamedTuple, Optional
 
@@ -13,6 +14,13 @@
 logger = logging.getLogger(__name__)
 
 
+def normalize_project_url_label(label: str) -> str:
+    # This logic is from PEP 753 (Well-known Project URLs in Metadata).
+    chars_to_remove = string.punctuation + string.whitespace
+    removal_map = str.maketrans("", "", chars_to_remove)
+    return label.translate(removal_map).lower()
+
+
 class ShowCommand(Command):
     """
     Show information about one or more installed packages.
@@ -135,13 +143,9 @@ def _get_requiring_packages(current_dist: BaseDistribution) -> Iterator[str]:
         if not homepage:
             # It's common that there is a "homepage" Project-URL, but Home-page
             # remains unset (especially as PEP 621 doesn't surface the field).
-            #
-            # This logic was taken from PyPI's codebase.
             for url in project_urls:
                 url_label, url = url.split(",", maxsplit=1)
-                normalized_label = (
-                    url_label.casefold().replace("-", "").replace("_", "").strip()
-                )
+                normalized_label = normalize_project_url_label(url_label)
                 if normalized_label == "homepage":
                     homepage = url.strip()
                     break

src/pip/_internal/index/package_finder.py

@@ -514,11 +514,7 @@ def _sort_key(self, candidate: InstallationCandidate) -> CandidateSortingKey:
                 )
             if self._prefer_binary:
                 binary_preference = 1
-            if wheel.build_tag is not None:
-                match = re.match(r"^(\d+)(.*)$", wheel.build_tag)
-                assert match is not None, "guaranteed by filename validation"
-                build_tag_groups = match.groups()
-                build_tag = (int(build_tag_groups[0]), build_tag_groups[1])
+            build_tag = wheel.build_tag
         else:  # sdist
             pri = -(support_num)
         has_allowed_hash = int(link.is_hash_allowed(self._hashes))

src/pip/_internal/metadata/importlib/_envs.py

@@ -8,10 +8,14 @@
 import zipimport
 from typing import Iterator, List, Optional, Sequence, Set, Tuple
 
-from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
+from pip._vendor.packaging.utils import (
+    InvalidWheelFilename,
+    NormalizedName,
+    canonicalize_name,
+    parse_wheel_filename,
+)
 
 from pip._internal.metadata.base import BaseDistribution, BaseEnvironment
-from pip._internal.models.wheel import Wheel
 from pip._internal.utils.deprecation import deprecated
 from pip._internal.utils.filetypes import WHEEL_EXTENSION
 
@@ -26,7 +30,9 @@ def _looks_like_wheel(location: str) -> bool:
         return False
     if not os.path.isfile(location):
         return False
-    if not Wheel.wheel_file_re.match(os.path.basename(location)):
+    try:
+        parse_wheel_filename(os.path.basename(location))
+    except InvalidWheelFilename:
         return False
     return zipfile.is_zipfile(location)
 

src/pip/_internal/models/wheel.py

@@ -3,13 +3,13 @@
 """
 
 import re
-from typing import Dict, Iterable, List
+from typing import Dict, Iterable, List, Optional
 
 from pip._vendor.packaging.tags import Tag
+from pip._vendor.packaging.utils import BuildTag, parse_wheel_filename
 from pip._vendor.packaging.utils import (
-    InvalidWheelFilename as PackagingInvalidWheelName,
+    InvalidWheelFilename as _PackagingInvalidWheelFilename,
 )
-from pip._vendor.packaging.utils import parse_wheel_filename
 
 from pip._internal.exceptions import InvalidWheelFilename
 from pip._internal.utils.deprecation import deprecated
@@ -18,54 +18,75 @@
 class Wheel:
     """A wheel file"""
 
-    wheel_file_re = re.compile(
+    legacy_wheel_file_re = re.compile(
         r"""^(?P<namever>(?P<name>[^\s-]+?)-(?P<ver>[^\s-]*?))
         ((-(?P<build>\d[^-]*?))?-(?P<pyver>[^\s-]+?)-(?P<abi>[^\s-]+?)-(?P<plat>[^\s-]+?)
         \.whl|\.dist-info)$""",
         re.VERBOSE,
     )
 
     def __init__(self, filename: str) -> None:
-        """
-        :raises InvalidWheelFilename: when the filename is invalid for a wheel
-        """
-        wheel_info = self.wheel_file_re.match(filename)
-        if not wheel_info:
-            raise InvalidWheelFilename(f"{filename} is not a valid wheel filename.")
         self.filename = filename
-        self.name = wheel_info.group("name").replace("_", "-")
-        _version = wheel_info.group("ver")
-        if "_" in _version:
-            try:
-                parse_wheel_filename(filename)
-            except PackagingInvalidWheelName as e:
-                deprecated(
-                    reason=(
-                        f"Wheel filename {filename!r} is not correctly normalised. "
-                        "Future versions of pip will raise the following error:\n"
-                        f"{e.args[0]}\n\n"
-                    ),
-                    replacement=(
-                        "to rename the wheel to use a correctly normalised "
-                        "name (this may require updating the version in "
-                        "the project metadata)"
-                    ),
-                    gone_in="25.1",
-                    issue=12938,
-                )
-
-            _version = _version.replace("_", "-")
-
-        self.version = _version
-        self.build_tag = wheel_info.group("build")
-        self.pyversions = wheel_info.group("pyver").split(".")
-        self.abis = wheel_info.group("abi").split(".")
-        self.plats = wheel_info.group("plat").split(".")
-
-        # All the tag combinations from this file
-        self.file_tags = {
-            Tag(x, y, z) for x in self.pyversions for y in self.abis for z in self.plats
-        }
+
+        # To make mypy happy specify type hints that can come from either
+        # parse_wheel_filename or the legacy_wheel_file_re match.
+        self.name: str
+        self._build_tag: Optional[BuildTag] = None
+
+        try:
+            wheel_info = parse_wheel_filename(filename)
+            self.name, _version, self._build_tag, self.file_tags = wheel_info
+            self.version = str(_version)
+        except _PackagingInvalidWheelFilename as e:
+            # Check if the wheel filename is in the legacy format
+            legacy_wheel_info = self.legacy_wheel_file_re.match(filename)
+            if not legacy_wheel_info:
+                raise InvalidWheelFilename(e.args[0]) from None
+
+            deprecated(
+                reason=(
+                    f"Wheel filename {filename!r} is not correctly normalised. "
+                    "Future versions of pip will raise the following error:\n"
+                    f"{e.args[0]}\n\n"
+                ),
+                replacement=(
+                    "to rename the wheel to use a correctly normalised "
+                    "name (this may require updating the version in "
+                    "the project metadata)"
+                ),
+                gone_in="25.3",
+                issue=12938,
+            )
+
+            self.name = legacy_wheel_info.group("name").replace("_", "-")
+            self.version = legacy_wheel_info.group("ver").replace("_", "-")
+
+            # Generate the file tags from the legacy wheel filename
+            pyversions = legacy_wheel_info.group("pyver").split(".")
+            abis = legacy_wheel_info.group("abi").split(".")
+            plats = legacy_wheel_info.group("plat").split(".")
+            self.file_tags = frozenset(
+                Tag(interpreter=py, abi=abi, platform=plat)
+                for py in pyversions
+                for abi in abis
+                for plat in plats
+            )
+
+    @property
+    def build_tag(self) -> BuildTag:
+        if self._build_tag is not None:
+            return self._build_tag
+
+        # Parse the build tag from the legacy wheel filename
+        legacy_wheel_info = self.legacy_wheel_file_re.match(self.filename)
+        assert legacy_wheel_info is not None, "guaranteed by filename validation"
+        build_tag = legacy_wheel_info.group("build")
+        match = re.match(r"^(\d+)(.*)$", build_tag)
+        assert match is not None, "guaranteed by filename validation"
+        build_tag_groups = match.groups()
+        self._build_tag = (int(build_tag_groups[0]), build_tag_groups[1])
+
+        return self._build_tag
 
     def get_formatted_file_tags(self) -> List[str]:
         """Return the wheel's tags as a sorted list of strings."""

src/pip/_internal/resolution/resolvelib/provider.py

@@ -104,6 +104,49 @@ def __init__(
     def identify(self, requirement_or_candidate: Union[Requirement, Candidate]) -> str:
         return requirement_or_candidate.name
 
+    def narrow_requirement_selection(
+        self,
+        identifiers: Iterable[str],
+        resolutions: Mapping[str, Candidate],
+        candidates: Mapping[str, Iterator[Candidate]],
+        information: Mapping[str, Iterator["PreferenceInformation"]],
+        backtrack_causes: Sequence["PreferenceInformation"],
+    ) -> Iterable[str]:
+        """Produce a subset of identifiers that should be considered before others.
+
+        Currently pip narrows the following selection:
+            * Requires-Python, if present is always returned by itself
+            * Backtrack causes are considered next because they can be identified
+              in linear time here, whereas because get_preference() is called
+              for each identifier, it would be quadratic to check for them there.
+              Further, the current backtrack causes likely need to be resolved
+              before other requirements as a resolution can't be found while
+              there is a conflict.
+        """
+        backtrack_identifiers = set()
+        for info in backtrack_causes:
+            backtrack_identifiers.add(info.requirement.name)
+            if info.parent is not None:
+                backtrack_identifiers.add(info.parent.name)
+
+        current_backtrack_causes = []
+        for identifier in identifiers:
+            # Requires-Python has only one candidate and the check is basically
+            # free, so we always do it first to avoid needless work if it fails.
+            # This skips calling get_preference() for all other identifiers.
+            if identifier == REQUIRES_PYTHON_IDENTIFIER:
+                return [identifier]
+
+            # Check if this identifier is a backtrack cause
+            if identifier in backtrack_identifiers:
+                current_backtrack_causes.append(identifier)
+                continue
+
+        if current_backtrack_causes:
+            return current_backtrack_causes
+
+        return identifiers
+
     def get_preference(
         self,
         identifier: str,
@@ -156,20 +199,9 @@ def get_preference(
         unfree = bool(operators)
         requested_order = self._user_requested.get(identifier, math.inf)
 
-        # Requires-Python has only one candidate and the check is basically
-        # free, so we always do it first to avoid needless work if it fails.
-        requires_python = identifier == REQUIRES_PYTHON_IDENTIFIER
-
-        # Prefer the causes of backtracking on the assumption that the problem
-        # resolving the dependency tree is related to the failures that caused
-        # the backtracking
-        backtrack_cause = self.is_backtrack_cause(identifier, backtrack_causes)
-
         return (
-            not requires_python,
             not direct,
             not pinned,
-            not backtrack_cause,
             requested_order,
             not unfree,
             identifier,
@@ -224,14 +256,3 @@ def is_satisfied_by(self, requirement: Requirement, candidate: Candidate) -> boo
     def get_dependencies(self, candidate: Candidate) -> Sequence[Requirement]:
         with_requires = not self._ignore_dependencies
         return [r for r in candidate.iter_dependencies(with_requires) if r is not None]
-
-    @staticmethod
-    def is_backtrack_cause(
-        identifier: str, backtrack_causes: Sequence["PreferenceInformation"]
-    ) -> bool:
-        for backtrack_cause in backtrack_causes:
-            if identifier == backtrack_cause.requirement.name:
-                return True
-            if backtrack_cause.parent and identifier == backtrack_cause.parent.name:
-                return True
-        return False