add code and tests to handle namespace pkgs

Author: Jonathon Belotti

src/BUILD

@@ -0,0 +1,13 @@
+py_library(
+    name = "src",
+    srcs = glob(["*.py"]),
+    srcs_version = "PY3",
+    deps = [],
+)
+
+py_test(
+    name = "test",
+    srcs = glob(["tests/*.py"]),
+    main = "tests/__main__.py",
+    deps = [":src"],
+)

src/namespace_pkgs.py

@@ -0,0 +1,69 @@
+import os
+import glob
+
+from typing import Set
+
+
+# See https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages
+PKGUTIL_STYLE_NS_PKG_INIT_CONTENTS = (
+    "# __path__ manipulation added by rules_python_external to support namespace pkgs.\n"
+    "__path__ = __import__('pkgutil').extend_path(__path__, __name__)\n"
+)
+
+
+def pkg_resources_style_namespace_packages(extracted_whl_directory) -> Set[str]:
+    """
+    "While this approach is no longer recommended, it is widely present in most existing namespace packages." - PyPA
+    See https://packaging.python.org/guides/packaging-namespace-packages/#pkg-resources-style-namespace-packages
+    """
+    namespace_pkg_dirs = set()
+
+    dist_info_dirs = glob.glob(os.path.join(extracted_whl_directory, "*.dist-info"))
+    if not dist_info_dirs:
+        raise ValueError(f"No *.dist-info directory found. {extracted_whl_directory} is not a valid Wheel.")
+    elif len(dist_info_dirs) > 1:
+        raise ValueError(f"Found more than 1 *.dist-info directory. {extracted_whl_directory} is not a valid Wheel.")
+    else:
+        dist_info = dist_info_dirs[0]
+    namespace_packages_record_file = os.path.join(dist_info, "namespace_packages.txt")
+    if os.path.exists(namespace_packages_record_file):
+        with open(namespace_packages_record_file) as nspkg:
+            for line in nspkg.readlines():
+                namespace = line.strip().replace(".", os.sep)
+                if namespace:
+                    namespace_pkg_dirs.add(
+                        os.path.join(extracted_whl_directory, namespace)
+                    )
+    return namespace_pkg_dirs
+
+
+def implicit_namespace_packages(directory, ignored_dirnames=None) -> Set[str]:
+    namespace_pkg_dirs = set()
+    for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
+        # We are only interested in dirs with no init file
+        if "__init__.py" in filenames:
+            dirnames[:] = []  # Remove dirnames from search
+            continue
+
+        for ignored_dir in (ignored_dirnames or []):
+            if ignored_dir in dirnames:
+                dirnames.remove(ignored_dir)
+
+        non_empty_directory = (dirnames or filenames)
+        if (
+            non_empty_directory and
+            # The root of the directory should never be an implicit namespace
+            dirpath != directory
+        ):
+            namespace_pkg_dirs.add(dirpath)
+    return namespace_pkg_dirs
+
+
+def add_pkgutil_style_namespace_pkg_init(dirpath) -> None:
+    """TODO"""
+    ns_pkg_init_filepath = os.path.join(dirpath, "__init__.py")
+
+    if os.path.isfile(ns_pkg_init_filepath):
+        raise ValueError(f"{dirpath} already contains an __init__.py file.")
+    with open(ns_pkg_init_filepath, "w") as ns_pkg_init_f:
+        ns_pkg_init_f.write(PKGUTIL_STYLE_NS_PKG_INIT_CONTENTS)

src/tests/__main__.py

@@ -0,0 +1,12 @@
+import unittest
+
+
+if __name__ == '__main__':
+    loader = unittest.TestLoader()
+    start_dir = '.'
+    suite = loader.discover(start_dir)
+
+    runner = unittest.TextTestRunner()
+    result = runner.run(suite)
+    if result.errors or result.failures:
+        exit(1)

src/tests/test_namespace_pkgs.py

@@ -0,0 +1,115 @@
+import os
+import pathlib
+import unittest
+import tempfile
+import shutil
+
+import src.namespace_pkgs
+
+
+class TempDir:
+    def __init__(self):
+        self.dir = tempfile.mkdtemp()
+
+    def root(self):
+        return self.dir
+
+    def add_dir(self, rel_path):
+        d = pathlib.Path(self.dir, rel_path)
+        d.mkdir(parents=True)
+
+    def add_file(self, rel_path, contents=None):
+        f = pathlib.Path(self.dir, rel_path)
+        f.parent.mkdir(parents=True, exist_ok=True)
+        if contents:
+            with open(f, "w") as writeable_f:
+                writeable_f.write(contents)
+        else:
+            f.touch()
+
+    def remove(self):
+        shutil.rmtree(self.dir)
+
+
+class TestPkgResourcesStyleNamespacePackages(unittest.TestCase):
+    def test_finds_correct_namespace_packages(self):
+        directory = TempDir()
+        directory.add_file("google/auth/__init__.py")
+        directory.add_file("google/auth/foo.py")
+        directory.add_file(
+            "google_auth-1.8.2.dist-info/namespace_packages.txt",
+            contents="google\n"
+        )
+
+        expected = {
+            f"{directory.root()}/google",
+        }
+        actual = src.namespace_pkgs.pkg_resources_style_namespace_packages(directory.root())
+        self.assertEqual(actual, expected)
+
+    def test_empty_case(self):
+        # Even though this directory contains directories with no __init__.py
+        # it has an empty namespace_packages.txt file so no namespace packages
+        # should be returned.
+        directory = TempDir()
+        directory.add_file("foo/bar/biz.py")
+        directory.add_file("foo/bee/boo.py")
+        directory.add_file("foo/buu/__init__.py")
+        directory.add_file("foo/buu/bii.py")
+        directory.add_file("foo-1.0.0.dist-info/namespace_packages.txt")
+
+        actual = src.namespace_pkgs.pkg_resources_style_namespace_packages(directory.root())
+        self.assertEqual(actual, set())
+
+    def test_missing_namespace_pkgs_record_file(self):
+        # Even though this directory contains directories with no __init__.py
+        # it has no namespace_packages.txt file, so no namespace packages should
+        # be found and returned.
+        directory = TempDir()
+        directory.add_file("foo/bar/biz.py")
+        directory.add_file("foo/bee/boo.py")
+        directory.add_file("foo/buu/__init__.py")
+        directory.add_file("foo/buu/bii.py")
+        directory.add_file("foo-1.0.0.dist-info/METADATA")
+        directory.add_file("foo-1.0.0.dist-info/RECORD")
+
+        actual = src.namespace_pkgs.pkg_resources_style_namespace_packages(directory.root())
+        self.assertEqual(actual, set())
+
+
+class TestImplicitNamespacePackages(unittest.TestCase):
+    def test_finds_correct_namespace_packages(self):
+        directory = TempDir()
+        directory.add_file("foo/bar/biz.py")
+        directory.add_file("foo/bee/boo.py")
+        directory.add_file("foo/buu/__init__.py")
+        directory.add_file("foo/buu/bii.py")
+
+        expected = {
+            f"{directory.root()}/foo",
+            f"{directory.root()}/foo/bar",
+            f"{directory.root()}/foo/bee",
+        }
+        actual = src.namespace_pkgs.implicit_namespace_packages(directory.root())
+        self.assertEqual(actual, expected)
+
+    def test_ignores_empty_directories(self):
+        directory = TempDir()
+        directory.add_file("foo/bar/biz.py")
+        directory.add_dir("foo/cat")
+
+        expected = {
+            f"{directory.root()}/foo",
+            f"{directory.root()}/foo/bar",
+        }
+        actual = src.namespace_pkgs.implicit_namespace_packages(directory.root())
+        self.assertEqual(actual, expected)
+
+    def test_empty_case(self):
+        directory = TempDir()
+        directory.add_file("foo/__init__.py")
+        directory.add_file("foo/bar/__init__.py")
+        directory.add_file("foo/bar/biz.py")
+
+        actual = src.namespace_pkgs.implicit_namespace_packages(directory.root())
+        self.assertEqual(actual, set())