Remove direct pkg_resource usages from resolver and preparer

Author: uranusjr

src/pip/_internal/distributions/base.py

@@ -1,9 +1,7 @@
 import abc
-from typing import Optional
-
-from pip._vendor.pkg_resources import Distribution
 
 from pip._internal.index.package_finder import PackageFinder
+from pip._internal.metadata.base import BaseDistribution
 from pip._internal.req import InstallRequirement
 
 
@@ -28,7 +26,7 @@ def __init__(self, req: InstallRequirement) -> None:
         self.req = req
 
     @abc.abstractmethod
-    def get_pkg_resources_distribution(self) -> Optional[Distribution]:
+    def get_metadata_distribution(self) -> BaseDistribution:
         raise NotImplementedError()
 
     @abc.abstractmethod

src/pip/_internal/distributions/installed.py

@@ -1,9 +1,6 @@
-from typing import Optional
-
-from pip._vendor.pkg_resources import Distribution
-
 from pip._internal.distributions.base import AbstractDistribution
 from pip._internal.index.package_finder import PackageFinder
+from pip._internal.metadata import BaseDistribution
 
 
 class InstalledDistribution(AbstractDistribution):
@@ -13,7 +10,8 @@ class InstalledDistribution(AbstractDistribution):
     been computed.
     """
 
-    def get_pkg_resources_distribution(self) -> Optional[Distribution]:
+    def get_metadata_distribution(self) -> BaseDistribution:
+        assert self.req.satisfied_by is not None, "not actually installed"
         return self.req.satisfied_by
 
     def prepare_distribution_metadata(

src/pip/_internal/distributions/sdist.py

@@ -1,12 +1,11 @@
 import logging
 from typing import Set, Tuple
 
-from pip._vendor.pkg_resources import Distribution
-
 from pip._internal.build_env import BuildEnvironment
 from pip._internal.distributions.base import AbstractDistribution
 from pip._internal.exceptions import InstallationError
 from pip._internal.index.package_finder import PackageFinder
+from pip._internal.metadata import BaseDistribution
 from pip._internal.utils.subprocess import runner_with_spinner_message
 
 logger = logging.getLogger(__name__)
@@ -19,8 +18,10 @@ class SourceDistribution(AbstractDistribution):
     generated, either using PEP 517 or using the legacy `setup.py egg_info`.
     """
 
-    def get_pkg_resources_distribution(self) -> Distribution:
-        return self.req.get_dist()
+    def get_metadata_distribution(self) -> BaseDistribution:
+        from pip._internal.metadata.pkg_resources import Distribution as _Dist
+
+        return _Dist(self.req.get_dist())
 
     def prepare_distribution_metadata(
         self, finder: PackageFinder, build_isolation: bool

src/pip/_internal/distributions/wheel.py

@@ -1,10 +1,12 @@
-from zipfile import ZipFile
-
-from pip._vendor.pkg_resources import Distribution
+from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.distributions.base import AbstractDistribution
 from pip._internal.index.package_finder import PackageFinder
-from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel
+from pip._internal.metadata import (
+    BaseDistribution,
+    FilesystemWheel,
+    get_wheel_distribution,
+)
 
 
 class WheelDistribution(AbstractDistribution):
@@ -13,20 +15,15 @@ class WheelDistribution(AbstractDistribution):
     This does not need any preparation as wheels can be directly unpacked.
     """
 
-    def get_pkg_resources_distribution(self) -> Distribution:
+    def get_metadata_distribution(self) -> BaseDistribution:
         """Loads the metadata from the wheel file into memory and returns a
         Distribution that uses it, not relying on the wheel file or
         requirement.
         """
-        # Set as part of preparation during download.
-        assert self.req.local_file_path
-        # Wheels are never unnamed.
-        assert self.req.name
-
-        with ZipFile(self.req.local_file_path, allowZip64=True) as z:
-            return pkg_resources_distribution_for_wheel(
-                z, self.req.name, self.req.local_file_path
-            )
+        assert self.req.local_file_path, "Set as part of preparation during download"
+        assert self.req.name, "Wheels are never unnamed"
+        wheel = FilesystemWheel(self.req.local_file_path)
+        return get_wheel_distribution(wheel, canonicalize_name(self.req.name))
 
     def prepare_distribution_metadata(
         self, finder: PackageFinder, build_isolation: bool

src/pip/_internal/exceptions.py

@@ -2,14 +2,15 @@
 
 import configparser
 from itertools import chain, groupby, repeat
-from typing import TYPE_CHECKING, Dict, List, Optional
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
 
 from pip._vendor.pkg_resources import Distribution
 from pip._vendor.requests.models import Request, Response
 
 if TYPE_CHECKING:
     from hashlib import _Hash
 
+    from pip._internal.metadata import BaseDistribution
     from pip._internal.req.req_install import InstallRequirement
 
 
@@ -38,7 +39,11 @@ class NoneMetadataError(PipError):
     "PKG-INFO").
     """
 
-    def __init__(self, dist: Distribution, metadata_name: str) -> None:
+    def __init__(
+        self,
+        dist: Union[Distribution, "BaseDistribution"],
+        metadata_name: str,
+    ) -> None:
         """
         :param dist: A Distribution object.
         :param metadata_name: The name of the metadata being accessed

src/pip/_internal/metadata/__init__.py

@@ -1,10 +1,13 @@
 from typing import List, Optional
 
-from .base import BaseDistribution, BaseEnvironment
+from .base import BaseDistribution, BaseEnvironment, FilesystemWheel, MemoryWheel, Wheel
 
 __all__ = [
     "BaseDistribution",
     "BaseEnvironment",
+    "FilesystemWheel",
+    "MemoryWheel",
+    "Wheel",
     "get_default_environment",
     "get_environment",
     "get_wheel_distribution",
@@ -35,7 +38,7 @@ def get_environment(paths: Optional[List[str]]) -> BaseEnvironment:
     return Environment.from_paths(paths)
 
 
-def get_wheel_distribution(wheel_path: str, canonical_name: str) -> BaseDistribution:
+def get_wheel_distribution(wheel: Wheel, canonical_name: str) -> BaseDistribution:
     """Get the representation of the specified wheel's distribution metadata.
 
     This returns a Distribution instance from the chosen backend based on
@@ -45,4 +48,4 @@ def get_wheel_distribution(wheel_path: str, canonical_name: str) -> BaseDistribu
     """
     from .pkg_resources import Distribution
 
-    return Distribution.from_wheel(wheel_path, canonical_name)
+    return Distribution.from_wheel(wheel, canonical_name)

src/pip/_internal/metadata/base.py

@@ -2,7 +2,9 @@
 import json
 import logging
 import re
+import zipfile
 from typing import (
+    IO,
     TYPE_CHECKING,
     Collection,
     Container,
@@ -14,6 +16,7 @@
 )
 
 from pip._vendor.packaging.requirements import Requirement
+from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet
 from pip._vendor.packaging.version import LegacyVersion, Version
 
 from pip._internal.models.direct_url import (
@@ -50,6 +53,12 @@ def group(self) -> str:
 
 
 class BaseDistribution(Protocol):
+    def __repr__(self) -> str:
+        return f"{self.raw_name} {self.version} ({self.location})"
+
+    def __str__(self) -> str:
+        return f"{self.raw_name} {self.version}"
+
     @property
     def location(self) -> Optional[str]:
         """Where the distribution is loaded from.
@@ -159,10 +168,42 @@ def metadata_version(self) -> Optional[str]:
     def raw_name(self) -> str:
         """Value of "Name:" in distribution metadata."""
         # The metadata should NEVER be missing the Name: key, but if it somehow
-        # does not, fall back to the known canonical name.
+        # does, fall back to the known canonical name.
         return self.metadata.get("Name", self.canonical_name)
 
+    @property
+    def requires_python(self) -> SpecifierSet:
+        """Value of "Requires-Python:" in distribution metadata.
+
+        If the key does not exist or contains an invalid value, an empty
+        SpecifierSet should be returned.
+        """
+        value = self.metadata.get("Requires-Python")
+        if value is None:
+            return SpecifierSet()
+        try:
+            # Convert to str to satisfy the type checker; this can be a Header object.
+            spec = SpecifierSet(str(value))
+        except InvalidSpecifier as e:
+            message = "Package %r has an invalid Requires-Python: %s"
+            logger.warning(message, self.raw_name, e)
+            return SpecifierSet()
+        return spec
+
     def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]:
+        """Dependencies of this distribution.
+
+        For modern .dist-info distributions, this is the collection of
+        "Requires-Dist:" entries in distribution metadata.
+        """
+        raise NotImplementedError()
+
+    def iter_provided_extras(self) -> Iterable[str]:
+        """Extras provided by this distribution.
+
+        For modern .dist-info distributions, this is the collection of
+        "Provides-Extra:" entries in distribution metadata.
+        """
         raise NotImplementedError()
 
 
@@ -240,3 +281,27 @@ def iter_installed_distributions(
         if user_only:
             it = (d for d in it if d.in_usersite)
         return (d for d in it if d.canonical_name not in skip)
+
+
+class Wheel(Protocol):
+    location: str
+
+    def as_zipfile(self) -> zipfile.ZipFile:
+        raise NotImplementedError()
+
+
+class FilesystemWheel(Wheel):
+    def __init__(self, location: str) -> None:
+        self.location = location
+
+    def as_zipfile(self) -> zipfile.ZipFile:
+        return zipfile.ZipFile(self.location, allowZip64=True)
+
+
+class MemoryWheel(Wheel):
+    def __init__(self, location: str, stream: IO[bytes]) -> None:
+        self.location = location
+        self.stream = stream
+
+    def as_zipfile(self) -> zipfile.ZipFile:
+        return zipfile.ZipFile(self.stream, allowZip64=True)

src/pip/_internal/metadata/pkg_resources.py

@@ -1,6 +1,5 @@
 import email.message
 import logging
-import zipfile
 from typing import (
     TYPE_CHECKING,
     Collection,
@@ -20,7 +19,13 @@
 from pip._internal.utils.packaging import get_installer, get_metadata
 from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel
 
-from .base import BaseDistribution, BaseEntryPoint, BaseEnvironment, DistributionVersion
+from .base import (
+    BaseDistribution,
+    BaseEntryPoint,
+    BaseEnvironment,
+    DistributionVersion,
+    Wheel,
+)
 
 if TYPE_CHECKING:
     from pip._vendor.packaging.utils import NormalizedName
@@ -39,9 +44,9 @@ def __init__(self, dist: pkg_resources.Distribution) -> None:
         self._dist = dist
 
     @classmethod
-    def from_wheel(cls, path: str, name: str) -> "Distribution":
-        with zipfile.ZipFile(path, allowZip64=True) as zf:
-            dist = pkg_resources_distribution_for_wheel(zf, name, path)
+    def from_wheel(cls, wheel: Wheel, name: str) -> "Distribution":
+        with wheel.as_zipfile() as zf:
+            dist = pkg_resources_distribution_for_wheel(zf, name, wheel.location)
         return cls(dist)
 
     @property
@@ -100,6 +105,9 @@ def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requiremen
             extras = frozenset(extras).intersection(self._dist.extras)
         return self._dist.requires(extras)
 
+    def iter_provided_extras(self) -> Iterable[str]:
+        return self._dist.extras
+
 
 class Environment(BaseEnvironment):
     def __init__(self, ws: pkg_resources.WorkingSet) -> None:

src/pip/_internal/network/lazy_wheel.py

@@ -8,33 +8,33 @@
 from typing import Any, Dict, Iterator, List, Optional, Tuple
 from zipfile import BadZipfile, ZipFile
 
-from pip._vendor.pkg_resources import Distribution
+from pip._vendor.packaging.utils import canonicalize_name
 from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
 
+from pip._internal.metadata import BaseDistribution, MemoryWheel, get_wheel_distribution
 from pip._internal.network.session import PipSession
 from pip._internal.network.utils import HEADERS, raise_for_status, response_chunks
-from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel
 
 
 class HTTPRangeRequestUnsupported(Exception):
     pass
 
 
-def dist_from_wheel_url(name: str, url: str, session: PipSession) -> Distribution:
-    """Return a pkg_resources.Distribution from the given wheel URL.
+def dist_from_wheel_url(name: str, url: str, session: PipSession) -> BaseDistribution:
+    """Return a distribution object from the given wheel URL.
 
     This uses HTTP range requests to only fetch the potion of the wheel
     containing metadata, just enough for the object to be constructed.
     If such requests are not supported, HTTPRangeRequestUnsupported
     is raised.
     """
-    with LazyZipOverHTTP(url, session) as wheel:
+    with LazyZipOverHTTP(url, session) as zf:
         # For read-only ZIP files, ZipFile only needs methods read,
         # seek, seekable and tell, not the whole IO protocol.
-        zip_file = ZipFile(wheel)  # type: ignore
+        wheel = MemoryWheel(zf.name, zf)  # type: ignore
         # After context manager exit, wheel.name
         # is an invalid file by intention.
-        return pkg_resources_distribution_for_wheel(zip_file, name, wheel.name)
+        return get_wheel_distribution(wheel, canonicalize_name(name))
 
 
 class LazyZipOverHTTP:

src/pip/_internal/operations/check.py

@@ -126,11 +126,9 @@ def _simulate_installation_of(
     # Modify it as installing requirement_set would (assuming no errors)
     for inst_req in to_install:
         abstract_dist = make_distribution_for_install_requirement(inst_req)
-        dist = abstract_dist.get_pkg_resources_distribution()
-
-        assert dist is not None
-        name = canonicalize_name(dist.project_name)
-        package_set[name] = PackageDetails(dist.parsed_version, dist.requires())
+        dist = abstract_dist.get_metadata_distribution()
+        name = dist.canonical_name
+        package_set[name] = PackageDetails(dist.version, list(dist.iter_dependencies()))
 
         installed.add(name)