importlib: Read distribution name/version from metadata directory names, if possible (#12656)

importlib does not cache metadata in-memory, so querying even simple
attributes like distribution names and versions can quickly become
expensive (as each access requires reading METADATA). Fortunately,
`Distribution.canonical_name` is optimized to parse the metadata
directory name to query the name if possible. This commit extends this
optimization to the finder implementation and version attribute.

.egg-info directory names tend to not include the version so they are
not considered for optimizing version lookup.

simplewheel-2.0-1-py2.py3-none-any.whl had to be modified to rename the
.dist-info directory which mistakenly included the wheel build tag (in
violation of the wheel specification).

    simplewheel/__init__.py
    simplewheel-2.0-1.dist-info/DESCRIPTION.rst
    simplewheel-2.0-1.dist-info/metadata.json
    simplewheel-2.0-1.dist-info/top_level.txt
    simplewheel-2.0-1.dist-info/WHEEL
    simplewheel-2.0-1.dist-info/METADATA
    simplewheel-2.0-1.dist-info/RECORD

Otherwise, it was mistaken for part of the version and led pip to think
the wheel was a post-release, breaking tests...

Co-authored-by: Tzu-ping Chung <[email protected]>

Author: ichard26

news/12656.feature.rst

@@ -0,0 +1,3 @@
+Improve discovery performance of installed packages when the
+``importlib.metadata`` backend is used to load distribution metadata
+(used by default under Python 3.11+).

news/aa82171b-1578-4128-8db3-9aa72b3a6a84.trivial.rst

@@ -1,5 +1,8 @@
 import importlib.metadata
-from typing import Any, Optional, Protocol, cast
+import os
+from typing import Any, Optional, Protocol, Tuple, cast
+
+from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
 
 
 class BadMetadata(ValueError):
@@ -43,13 +46,40 @@ def get_info_location(d: importlib.metadata.Distribution) -> Optional[BasePath]:
     return getattr(d, "_path", None)
 
 
-def get_dist_name(dist: importlib.metadata.Distribution) -> str:
-    """Get the distribution's project name.
+def parse_name_and_version_from_info_directory(
+    dist: importlib.metadata.Distribution,
+) -> Tuple[Optional[str], Optional[str]]:
+    """Get a name and version from the metadata directory name.
+
+    This is much faster than reading distribution metadata.
+    """
+    info_location = get_info_location(dist)
+    if info_location is None:
+        return None, None
+
+    stem, suffix = os.path.splitext(info_location.name)
+    if suffix == ".dist-info":
+        name, sep, version = stem.partition("-")
+        if sep:
+            return name, version
+
+    if suffix == ".egg-info":
+        name = stem.split("-", 1)[0]
+        return name, None
+
+    return None, None
+
+
+def get_dist_canonical_name(dist: importlib.metadata.Distribution) -> NormalizedName:
+    """Get the distribution's normalized name.
 
     The ``name`` attribute is only available in Python 3.10 or later. We are
     targeting exactly that, but Mypy does not know this.
     """
+    if name := parse_name_and_version_from_info_directory(dist)[0]:
+        return canonicalize_name(name)
+
     name = cast(Any, dist).name
     if not isinstance(name, str):
         raise BadMetadata(dist, reason="invalid metadata entry 'name'")
-    return name
+    return canonicalize_name(name)

src/pip/_internal/metadata/importlib/_compat.py

@@ -1,6 +1,5 @@
 import email.message
 import importlib.metadata
-import os
 import pathlib
 import zipfile
 from typing import (
@@ -31,7 +30,11 @@
 from pip._internal.utils.temp_dir import TempDirectory
 from pip._internal.utils.wheel import parse_wheel, read_wheel_metadata_file
 
-from ._compat import BasePath, get_dist_name
+from ._compat import (
+    BasePath,
+    get_dist_canonical_name,
+    parse_name_and_version_from_info_directory,
+)
 
 
 class WheelDistribution(importlib.metadata.Distribution):
@@ -154,25 +157,14 @@ def installed_location(self) -> Optional[str]:
             return None
         return normalize_path(str(self._installed_location))
 
-    def _get_dist_name_from_location(self) -> Optional[str]:
-        """Try to get the name from the metadata directory name.
-
-        This is much faster than reading metadata.
-        """
-        if self._info_location is None:
-            return None
-        stem, suffix = os.path.splitext(self._info_location.name)
-        if suffix not in (".dist-info", ".egg-info"):
-            return None
-        return stem.split("-", 1)[0]
-
     @property
     def canonical_name(self) -> NormalizedName:
-        name = self._get_dist_name_from_location() or get_dist_name(self._dist)
-        return canonicalize_name(name)
+        return get_dist_canonical_name(self._dist)
 
     @property
     def version(self) -> Version:
+        if version := parse_name_and_version_from_info_directory(self._dist)[1]:
+            return parse_version(version)
         return parse_version(self._dist.version)
 
     @property

src/pip/_internal/metadata/importlib/_dists.py

@@ -15,7 +15,7 @@
 from pip._internal.utils.deprecation import deprecated
 from pip._internal.utils.filetypes import WHEEL_EXTENSION
 
-from ._compat import BadMetadata, BasePath, get_dist_name, get_info_location
+from ._compat import BadMetadata, BasePath, get_dist_canonical_name, get_info_location
 from ._dists import Distribution
 
 logger = logging.getLogger(__name__)
@@ -61,14 +61,13 @@ def _find_impl(self, location: str) -> Iterator[FoundResult]:
         for dist in importlib.metadata.distributions(path=[location]):
             info_location = get_info_location(dist)
             try:
-                raw_name = get_dist_name(dist)
+                name = get_dist_canonical_name(dist)
             except BadMetadata as e:
                 logger.warning("Skipping %s due to %s", info_location, e.reason)
                 continue
-            normalized_name = canonicalize_name(raw_name)
-            if normalized_name in self._found_names:
+            if name in self._found_names:
                 continue
-            self._found_names.add(normalized_name)
+            self._found_names.add(name)
             yield dist, info_location
 
     def find(self, location: str) -> Iterator[BaseDistribution]:

src/pip/_internal/metadata/importlib/_envs.py

@@ -1195,7 +1195,7 @@ def test_install_nonlocal_compatible_wheel(
     )
     assert result.returncode == SUCCESS
 
-    distinfo = Path("scratch") / "target" / "simplewheel-2.0-1.dist-info"
+    distinfo = Path("scratch") / "target" / "simplewheel-2.0.dist-info"
     result.did_create(distinfo)
 
     # Test install without --target

tests/data/packages/simplewheel-2.0-1-py2.py3-none-any.whl

@@ -39,7 +39,7 @@ def test_install_report_basic(
     assert url.endswith("/packages/simplewheel-2.0-1-py2.py3-none-any.whl")
     assert (
         simplewheel_report["download_info"]["archive_info"]["hash"]
-        == "sha256=191d6520d0570b13580bf7642c97ddfbb46dd04da5dd2cf7bef9f32391dfe716"
+        == "sha256=71e1ca6b16ae3382a698c284013f66504f2581099b2ce4801f60e9536236ceee"
     )