Use importlib instead of pkg_resources namespace package discovery (#1326)

Author: DanielNoord

astroid/const.py

@@ -1,6 +1,7 @@
 import enum
 import sys
 
+PY36 = sys.version_info[:2] == (3, 6)
 PY38 = sys.version_info[:2] == (3, 8)
 PY37_PLUS = sys.version_info >= (3, 7)
 PY38_PLUS = sys.version_info >= (3, 8)

astroid/interpreter/_import/util.py

@@ -3,15 +3,70 @@
 # Copyright (c) 2021 Daniël van Noord <[email protected]>
 # Copyright (c) 2021 Neil Girdhar <[email protected]>
 
-try:
-    import pkg_resources
-except ImportError:
-    pkg_resources = None  # type: ignore[assignment]
 
+from importlib import abc, util
+
+from astroid.const import PY36
+
+
+def _is_old_setuptools_namespace_package(modname: str) -> bool:
+    """Check for old types of setuptools namespace packages.
+
+    See https://setuptools.pypa.io/en/latest/pkg_resources.html and
+    https://packaging.python.org/en/latest/guides/packaging-namespace-packages/
+
+    Because pkg_resources is slow to import we only do so if explicitly necessary.
+    """
+    try:
+        import pkg_resources  # pylint: disable=import-outside-toplevel
+    except ImportError:
+        return False
 
-def is_namespace(modname):
     return (
-        pkg_resources is not None
-        and hasattr(pkg_resources, "_namespace_packages")
-        and modname in pkg_resources._namespace_packages
+        hasattr(pkg_resources, "_namespace_packages")
+        and modname in pkg_resources._namespace_packages  # type: ignore[attr-defined]
+    )
+
+
+def is_namespace(modname: str) -> bool:
+    """Determine whether we encounter a namespace package."""
+    if PY36:
+        # On Python 3.6 an AttributeError is raised when a package
+        # is lacking a __path__ attribute and thus is not a
+        # package.
+        try:
+            spec = util.find_spec(modname)
+        except (AttributeError, ValueError):
+            return _is_old_setuptools_namespace_package(modname)
+    else:
+        try:
+            spec = util.find_spec(modname)
+        except ValueError:
+            return _is_old_setuptools_namespace_package(modname)
+
+    # If there is no spec or origin this is a namespace package
+    # See: https://docs.python.org/3/library/importlib.html#importlib.machinery.ModuleSpec.origin
+    # We assume builtin packages are never namespace
+    if not spec or not spec.origin or spec.origin == "built-in":
+        return False
+
+    # If there is no loader the package is namespace
+    # See https://docs.python.org/3/library/importlib.html#importlib.abc.PathEntryFinder.find_loader
+    if not spec.loader:
+        return True
+    # This checks for _frozen_importlib.FrozenImporter, which does not inherit from InspectLoader
+    if hasattr(spec.loader, "_ORIGIN") and spec.loader._ORIGIN == "frozen":
+        return False
+    # Other loaders are namespace packages
+    if not isinstance(spec.loader, abc.InspectLoader):
+        return True
+
+    # Lastly we check if the package declares itself a namespace package
+    try:
+        source = spec.loader.get_source(spec.origin)
+    # If the loader can't handle the spec, we're dealing with a namespace package
+    except ImportError:
+        return False
+    return bool(
+        source and "pkg_resources" in source and "declare_namespace(__name__)" in source
     )