Merge branch 'bugfix/95-perf' into 'master'

Performance optimization of distribution discovery

Closes #95

See merge request python-devs/importlib_metadata!106

Author: jaraco

importlib_metadata/__init__.py

@@ -10,6 +10,7 @@
 import operator
 import functools
 import itertools
+import posixpath
 import collections
 
 from ._compat import (
@@ -23,7 +24,6 @@
     NotADirectoryError,
     PermissionError,
     pathlib,
-    PYPY_OPEN_BUG,
     ModuleNotFoundError,
     MetaPathFinder,
     email_message_from_string,
@@ -389,10 +389,6 @@ def path(self):
             """
             return vars(self).get('path', sys.path)
 
-        @property
-        def pattern(self):
-            return '.*' if self.name is None else re.escape(self.name)
-
     @abc.abstractmethod
     def find_distributions(self, context=Context()):
         """
@@ -404,6 +400,73 @@ def find_distributions(self, context=Context()):
         """
 
 
+class FastPath:
+    """
+    Micro-optimized class for searching a path for
+    children.
+    """
+
+    def __init__(self, root):
+        self.root = root
+
+    def joinpath(self, child):
+        return pathlib.Path(self.root, child)
+
+    def children(self):
+        with suppress(Exception):
+            return os.listdir(self.root or '')
+        with suppress(Exception):
+            return self.zip_children()
+        return []
+
+    def zip_children(self):
+        zip_path = zipp.Path(self.root)
+        names = zip_path.root.namelist()
+        self.joinpath = zip_path.joinpath
+
+        return (
+            posixpath.split(child)[0]
+            for child in names
+            )
+
+    def is_egg(self, search):
+        root_n_low = os.path.split(self.root)[1].lower()
+
+        return (
+            root_n_low == search.normalized + '.egg'
+            or root_n_low.startswith(search.prefix)
+            and root_n_low.endswith('.egg'))
+
+    def search(self, name):
+        for child in self.children():
+            n_low = child.lower()
+            if (n_low in name.exact_matches
+                    or n_low.startswith(name.prefix)
+                    and n_low.endswith(name.suffixes)
+                    # legacy case:
+                    or self.is_egg(name) and n_low == 'egg-info'):
+                yield self.joinpath(child)
+
+
+class Prepared:
+    """
+    A prepared search for metadata on a possibly-named package.
+    """
+    normalized = ''
+    prefix = ''
+    suffixes = '.dist-info', '.egg-info'
+    exact_matches = [''][:0]
+
+    def __init__(self, name):
+        self.name = name
+        if name is None:
+            return
+        self.normalized = name.lower().replace('-', '_')
+        self.prefix = self.normalized + '-'
+        self.exact_matches = [
+            self.normalized + suffix for suffix in self.suffixes]
+
+
 @install
 class MetadataPathFinder(NullFinder, DistributionFinder):
     """A degenerate finder for distribution packages on the file system.
@@ -421,45 +484,17 @@ def find_distributions(self, context=DistributionFinder.Context()):
         (or all names if ``None`` indicated) along the paths in the list
         of directories ``context.path``.
         """
-        found = self._search_paths(context.pattern, context.path)
+        found = self._search_paths(context.name, context.path)
         return map(PathDistribution, found)
 
     @classmethod
-    def _search_paths(cls, pattern, paths):
+    def _search_paths(cls, name, paths):
         """Find metadata directories in paths heuristically."""
         return itertools.chain.from_iterable(
-            cls._search_path(path, pattern)
-            for path in map(cls._switch_path, paths)
+            path.search(Prepared(name))
+            for path in map(FastPath, paths)
             )
 
-    @staticmethod
-    def _switch_path(path):
-        if not PYPY_OPEN_BUG or os.path.isfile(path):  # pragma: no branch
-            with suppress(Exception):
-                return zipp.Path(path)
-        return pathlib.Path(path)
-
-    @classmethod
-    def _matches_info(cls, normalized, item):
-        template = r'{pattern}(-.*)?\.(dist|egg)-info'
-        manifest = template.format(pattern=normalized)
-        return re.match(manifest, item.name, flags=re.IGNORECASE)
-
-    @classmethod
-    def _matches_legacy(cls, normalized, item):
-        template = r'{pattern}-.*\.egg[\\/]EGG-INFO'
-        manifest = template.format(pattern=normalized)
-        return re.search(manifest, str(item), flags=re.IGNORECASE)
-
-    @classmethod
-    def _search_path(cls, root, pattern):
-        if not root.is_dir():
-            return ()
-        normalized = pattern.replace('-', '_')
-        return (item for item in root.iterdir()
-                if cls._matches_info(normalized, item)
-                or cls._matches_legacy(normalized, item))
-
 
 class PathDistribution(Distribution):
     def __init__(self, path):

importlib_metadata/_compat.py

@@ -111,9 +111,6 @@ def py2_message_from_string(text):  # nocoverpy3
     email.message_from_string
     )
 
-# https://bitbucket.org/pypy/pypy/issues/3021/ioopen-directory-leaks-a-file-descriptor
-PYPY_OPEN_BUG = getattr(sys, 'pypy_version_info', (9, 9, 9))[:3] <= (7, 1, 1)
-
 
 class PyPy_repr:
     """

importlib_metadata/docs/changelog.rst

@@ -2,6 +2,13 @@
  importlib_metadata NEWS
 =========================
 
+v1.4.0
+======
+
+* Through careful optimization, ``distribution()`` is
+  3-4x faster. Thanks to Antony Lee for the
+  contribution. Closes #95.
+
 v1.3.0
 ======