Merge pull request #5 from dillon-giacoppo/jonathon/namespace-pkgs-support

Namespace pkgs support

Author: Jonathon Belotti

.gitignore

@@ -0,0 +1,136 @@
+# Intellij
+.ijwb/
+.idea/
+
+# Bazel
+bazel-*
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

README.md

@@ -42,3 +42,9 @@ py_binary(
     ],
 )
 ```
+
+## Development
+
+### Testing
+
+`bazel test //...`

src/BUILD

@@ -0,0 +1,13 @@
+py_library(
+    name = "src",
+    srcs = glob(["*.py"]),
+    srcs_version = "PY3",
+    deps = [],
+)
+
+py_test(
+    name = "test",
+    srcs = glob(["tests/*.py"]),
+    main = "tests/__main__.py",
+    deps = [":src"],
+)

src/extract_wheels.py

@@ -4,6 +4,8 @@
 import subprocess
 import sys
 
+
+from . import namespace_pkgs
 from .wheel import Wheel
 
 BUILD_TEMPLATE = """\
@@ -37,9 +39,37 @@ def sanitise_name(name):
     return "pypi__" + name.replace("-", "_").replace(".", "_").lower()
 
 
+def _setup_namespace_pkg_compatibility(extracted_whl_directory):
+    """
+    Namespace packages can be created in one of three ways. They are detailed here:
+    https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package
+
+    'pkgutil-style namespace packages' (2) works in Bazel, but 'native namespace packages' (1) and
+    'pkg_resources-style namespace packages' (3) do not.
+
+    We ensure compatibility with Bazel of methods 1 and 3 by converting them into method 2.
+    """
+    namespace_pkg_dirs = namespace_pkgs.pkg_resources_style_namespace_packages(
+        extracted_whl_directory
+    )
+    if (
+        not namespace_pkg_dirs and
+        namespace_pkgs.native_namespace_packages_supported()
+    ):
+        namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages(
+            extracted_whl_directory,
+            ignored_dirnames=[
+                f"{extracted_whl_directory}/bin",
+            ]
+        )
+
+    for ns_pkg_dir in namespace_pkg_dirs:
+        namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir)
+
+
 def extract_wheel(whl, directory, extras):
     """
-    Unzips a wheel into the Bazel repository and creates the BUILD file
+    Unzips a wheel into the Bazel repository and prepares it for use by Python rules.
 
     :param whl: the Wheel object we are unpacking
     :param directory: the subdirectory of the external repo to unzip to
@@ -48,6 +78,8 @@ def extract_wheel(whl, directory, extras):
 
     whl.unzip(directory)
 
+    _setup_namespace_pkg_compatibility(directory)
+
     with open(os.path.join(directory, "BUILD"), "w") as f:
         f.write(
             BUILD_TEMPLATE.format(

src/namespace_pkgs.py

@@ -0,0 +1,86 @@
+import os
+import glob
+import sys
+
+from typing import Set
+
+
+# See https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages
+PKGUTIL_STYLE_NS_PKG_INIT_CONTENTS = (
+    "# __path__ manipulation added by rules_python_external to support namespace pkgs.\n"
+    "__path__ = __import__('pkgutil').extend_path(__path__, __name__)\n"
+)
+
+
+def pkg_resources_style_namespace_packages(extracted_whl_directory) -> Set[str]:
+    """
+    Discovers namespace packages implemented using the 'pkg_resources-style namespace packages' method.
+
+    "While this approach is no longer recommended, it is widely present in most existing namespace packages." - PyPA
+    See https://packaging.python.org/guides/packaging-namespace-packages/#pkg-resources-style-namespace-packages
+    """
+    namespace_pkg_dirs = set()
+
+    dist_info_dirs = glob.glob(os.path.join(extracted_whl_directory, "*.dist-info"))
+    if not dist_info_dirs:
+        raise ValueError(f"No *.dist-info directory found. {extracted_whl_directory} is not a valid Wheel.")
+    elif len(dist_info_dirs) > 1:
+        raise ValueError(f"Found more than 1 *.dist-info directory. {extracted_whl_directory} is not a valid Wheel.")
+    else:
+        dist_info = dist_info_dirs[0]
+    namespace_packages_record_file = os.path.join(dist_info, "namespace_packages.txt")
+    if os.path.exists(namespace_packages_record_file):
+        with open(namespace_packages_record_file) as nspkg:
+            for line in nspkg.readlines():
+                namespace = line.strip().replace(".", os.sep)
+                if namespace:
+                    namespace_pkg_dirs.add(
+                        os.path.join(extracted_whl_directory, namespace)
+                    )
+    return namespace_pkg_dirs
+
+
+def native_namespace_packages_supported() -> bool:
+    return (sys.version_info.major, sys.version_info.minor) >= (3, 3)
+
+
+def implicit_namespace_packages(directory, ignored_dirnames=None) -> Set[str]:
+    """
+    Discovers namespace packages implemented using the 'native namespace packages' method,
+    AKA 'implicit namespace packages', which has been supported since Python 3.3.
+
+    See: https://packaging.python.org/guides/packaging-namespace-packages/#native-namespace-packages
+    """
+    namespace_pkg_dirs = set()
+    for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
+        # We are only interested in dirs with no __init__.py file
+        if "__init__.py" in filenames:
+            dirnames[:] = []  # Remove dirnames from search
+            continue
+
+        for ignored_dir in (ignored_dirnames or []):
+            if ignored_dir in dirnames:
+                dirnames.remove(ignored_dir)
+
+        non_empty_directory = (dirnames or filenames)
+        if (
+            non_empty_directory and
+            # The root of the directory should never be an implicit namespace
+            dirpath != directory
+        ):
+            namespace_pkg_dirs.add(dirpath)
+    return namespace_pkg_dirs
+
+
+def add_pkgutil_style_namespace_pkg_init(dir_path: str) -> None:
+    """
+    Used to implement the 'pkgutil-style namespace packages' method of
+    doing namespace packages.
+    See: https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages
+    """
+    ns_pkg_init_filepath = os.path.join(dir_path, "__init__.py")
+
+    if os.path.isfile(ns_pkg_init_filepath):
+        raise ValueError(f"{dir_path} already contains an __init__.py file.")
+    with open(ns_pkg_init_filepath, "w") as ns_pkg_init_f:
+        ns_pkg_init_f.write(PKGUTIL_STYLE_NS_PKG_INIT_CONTENTS)

src/tests/__main__.py

@@ -0,0 +1,12 @@
+import unittest
+
+
+if __name__ == '__main__':
+    loader = unittest.TestLoader()
+    start_dir = '.'
+    suite = loader.discover(start_dir)
+
+    runner = unittest.TextTestRunner()
+    result = runner.run(suite)
+    if result.errors or result.failures:
+        exit(1)