Fix unnecessary namespace directory creation (#54)

Author: dillon-giacoppo

example/BUILD

@@ -1,27 +1,29 @@
 load("@pip//:requirements.bzl", "requirement")
-load("@rules_python//python:defs.bzl", "py_runtime_pair")
 
 # Toolchain setup, this is optional.
 # Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE).
-py_runtime(
-    name = "python3_runtime",
-    files = ["@python_interpreter//:files"],
-    interpreter = "@python_interpreter//:python_bin",
-    python_version = "PY3",
-    visibility = ["//visibility:public"],
-)
-
-py_runtime_pair(
-    name = "my_py_runtime_pair",
-    py2_runtime = None,
-    py3_runtime = ":python3_runtime",
-)
-
-toolchain(
-    name = "my_py_toolchain",
-    toolchain = ":my_py_runtime_pair",
-    toolchain_type = "@bazel_tools//tools/python:toolchain_type",
-)
+#
+#load("@rules_python//python:defs.bzl", "py_runtime_pair")
+#
+#py_runtime(
+#    name = "python3_runtime",
+#    files = ["@python_interpreter//:files"],
+#    interpreter = "@python_interpreter//:python_bin",
+#    python_version = "PY3",
+#    visibility = ["//visibility:public"],
+#)
+#
+#py_runtime_pair(
+#    name = "my_py_runtime_pair",
+#    py2_runtime = None,
+#    py3_runtime = ":python3_runtime",
+#)
+#
+#toolchain(
+#    name = "my_py_toolchain",
+#    toolchain = ":my_py_runtime_pair",
+#    toolchain_type = "@bazel_tools//tools/python:toolchain_type",
+#)
 # End of toolchain setup.
 
 py_binary(

example/WORKSPACE

@@ -4,52 +4,15 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 http_archive(
     name = "rules_python",
-    sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161",
-    url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz",
+    url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.2/rules_python-0.0.2.tar.gz",
+    strip_prefix = "rules_python-0.0.2",
+    sha256 = "b5668cde8bb6e3515057ef465a35ad712214962f0b3a314e551204266c7be90c",
 )
 
 load("@rules_python//python:repositories.bzl", "py_repositories")
 
 py_repositories()
 
-# You could optionally use an in-build, compiled python interpreter as a toolchain,
-# and also use it to execute pip.
-
-# Special logic for building python interpreter with OpenSSL from homebrew.
-# See https://devguide.python.org/setup/#macos-and-os-x
-_py_configure = """
-if [[ "$OSTYPE" == "darwin"* ]]; then
-    ./configure --prefix=$(pwd)/bazel_install --with-openssl=$(brew --prefix openssl)
-else
-    ./configure --prefix=$(pwd)/bazel_install
-fi
-"""
-
-# NOTE: you need to have the SSL headers installed to build with openssl support (and use HTTPS).
-# E.g. on Ubuntu: `sudo apt install libssl-dev`
-http_archive(
-    name = "python_interpreter",
-    build_file_content = """
-exports_files(["python_bin"])
-filegroup(
-    name = "files",
-    srcs = glob(["bazel_install/**"], exclude = ["**/* *"]),
-    visibility = ["//visibility:public"],
-)
-""",
-    patch_cmds = [
-        "mkdir $(pwd)/bazel_install",
-        _py_configure,
-        "make",
-        "make install",
-        "ln -s bazel_install/bin/python3 python_bin",
-    ],
-    sha256 = "dfab5ec723c218082fe3d5d7ae17ecbdebffa9a1aea4d64aa3a2ecdd2e795864",
-    strip_prefix = "Python-3.8.3",
-    urls = ["https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tar.xz"],
-)
-# End of in-build Python interpreter setup.
-
 local_repository(
     name = "rules_python_external",
     path = "../",
@@ -75,7 +38,7 @@ pip_install(
     # 1. Python interpreter that you compile in the build file (as above in @python_interpreter).
     # 2. Pre-compiled python interpreter included with http_archive
     # 3. Wrapper script, like in the autodetecting python toolchain.
-    python_interpreter_target = "@python_interpreter//:python_bin",
+    #python_interpreter_target = "@python_interpreter//:python_bin",
 
     # (Optional) You can set quiet to False if you want to see pip output.
     #quiet = False,
@@ -84,6 +47,44 @@ pip_install(
     requirements = "//:requirements.txt",
 )
 
+# You could optionally use an in-build, compiled python interpreter as a toolchain,
+# and also use it to execute pip.
+#
+# Special logic for building python interpreter with OpenSSL from homebrew.
+# See https://devguide.python.org/setup/#macos-and-os-x
+#_py_configure = """
+#if [[ "$OSTYPE" == "darwin"* ]]; then
+#    ./configure --prefix=$(pwd)/bazel_install --with-openssl=$(brew --prefix openssl)
+#else
+#    ./configure --prefix=$(pwd)/bazel_install
+#fi
+#"""
+#
+# NOTE: you need to have the SSL headers installed to build with openssl support (and use HTTPS).
+# E.g. on Ubuntu: `sudo apt install libssl-dev`
+#http_archive(
+#    name = "python_interpreter",
+#    build_file_content = """
+#exports_files(["python_bin"])
+#filegroup(
+#    name = "files",
+#    srcs = glob(["bazel_install/**"], exclude = ["**/* *"]),
+#    visibility = ["//visibility:public"],
+#)
+#""",
+#    patch_cmds = [
+#        "mkdir $(pwd)/bazel_install",
+#        _py_configure,
+#        "make",
+#        "make install",
+#        "ln -s bazel_install/bin/python3 python_bin",
+#    ],
+#    sha256 = "dfab5ec723c218082fe3d5d7ae17ecbdebffa9a1aea4d64aa3a2ecdd2e795864",
+#    strip_prefix = "Python-3.8.3",
+#    urls = ["https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tar.xz"],
+#)
+
 # Optional:
 # Register the toolchain with the same python interpreter we used for pip in pip_install().
-register_toolchains("//:my_py_toolchain")
+#register_toolchains("//:my_py_toolchain")
+# End of in-build Python interpreter setup.

example/main.py

@@ -1,3 +1,4 @@
 import boto3
 
-print("Hello World")
+if __name__ == "__main__":
+    pass

extract_wheels/__init__.py

@@ -64,18 +64,22 @@ def main() -> None:
         required=True,
         help="The external repo name to install dependencies. In the format '@{REPO_NAME}'",
     )
-    parser.add_argument('--extra_pip_args', action='store',
-                        help=('Extra arguments to pass down to pip.'))
+    parser.add_argument(
+        "--extra_pip_args",
+        action="store",
+        help=("Extra arguments to pass down to pip."),
+    )
     parser.add_argument(
         "--pip_data_exclude",
-        action='store',
-        help='Additional data exclusion parameters to add to the pip packages BUILD file.'
+        action="store",
+        help="Additional data exclusion parameters to add to the pip packages BUILD file.",
     )
     args = parser.parse_args()
 
     pip_args = [sys.executable, "-m", "pip", "wheel", "-r", args.requirements]
     if args.extra_pip_args:
         pip_args += json.loads(args.extra_pip_args)["args"]
+
     # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
     subprocess.run(pip_args, check=True)
 

extract_wheels/lib/bazel.py

@@ -7,7 +7,9 @@
 from extract_wheels.lib import namespace_pkgs, wheel, purelib
 
 
-def generate_build_file_contents(name: str, dependencies: List[str], pip_data_exclude: List[str]) -> str:
+def generate_build_file_contents(
+    name: str, dependencies: List[str], pip_data_exclude: List[str]
+) -> str:
     """Generate a BUILD file for an unzipped Wheel
 
     Args:
@@ -39,7 +41,9 @@ def generate_build_file_contents(name: str, dependencies: List[str], pip_data_ex
             deps = [{dependencies}],
         )
         """.format(
-            name=name, dependencies=",".join(dependencies), data_exclude=json.dumps(data_exclude)
+            name=name,
+            dependencies=",".join(dependencies),
+            data_exclude=json.dumps(data_exclude),
         )
     )
 
@@ -93,33 +97,31 @@ def sanitise_name(name: str) -> str:
 
 
 def setup_namespace_pkg_compatibility(wheel_dir: str) -> None:
-    """Converts native namespace packages and pkg_resource-style packages to pkgutil-style packages
+    """Converts native namespace packages to pkgutil-style packages
 
     Namespace packages can be created in one of three ways. They are detailed here:
     https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package
 
-    'pkgutil-style namespace packages' (2) works in Bazel, but 'native namespace packages' (1) and
-    'pkg_resources-style namespace packages' (3) do not.
+    'pkgutil-style namespace packages' (2) and 'pkg_resources-style namespace packages' (3) works in Bazel, but
+    'native namespace packages' (1) do not.
 
-    We ensure compatibility with Bazel of methods 1 and 3 by converting them into method 2.
+    We ensure compatibility with Bazel of method 1 by converting them into method 2.
 
     Args:
         wheel_dir: the directory of the wheel to convert
     """
 
-    namespace_pkg_dirs = namespace_pkgs.pkg_resources_style_namespace_packages(
-        wheel_dir
+    namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages(
+        wheel_dir, ignored_dirnames=["%s/bin" % wheel_dir,],
     )
-    if not namespace_pkg_dirs and namespace_pkgs.native_namespace_packages_supported():
-        namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages(
-            wheel_dir, ignored_dirnames=["%s/bin" % wheel_dir,],
-        )
 
     for ns_pkg_dir in namespace_pkg_dirs:
         namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir)
 
 
-def extract_wheel(wheel_file: str, extras: Dict[str, Set[str]], pip_data_exclude: List[str]) -> str:
+def extract_wheel(
+    wheel_file: str, extras: Dict[str, Set[str]], pip_data_exclude: List[str]
+) -> str:
     """Extracts wheel into given directory and creates a py_library target.
 
     Args:

extract_wheels/lib/namespace_pkgs.py

@@ -1,36 +1,11 @@
 """Utility functions to discover python package types"""
 import os
-import sys
 import textwrap
 from typing import Set, List, Optional
 
 from extract_wheels.lib import wheel
 
 
-def pkg_resources_style_namespace_packages(wheel_dir: str) -> Set[str]:
-    """Discovers namespace packages implemented using the 'pkg_resources-style namespace packages' method.
-
-    "While this approach is no longer recommended, it is widely present in most existing namespace packages." - PyPA
-    See https://packaging.python.org/guides/packaging-namespace-packages/#pkg-resources-style-namespace-packages
-    """
-    namespace_pkg_dirs = set()
-
-    dist_info = wheel.get_dist_info(wheel_dir)
-    namespace_packages_record_file = os.path.join(dist_info, "namespace_packages.txt")
-    if os.path.exists(namespace_packages_record_file):
-        with open(namespace_packages_record_file) as nspkg:
-            for line in nspkg.readlines():
-                namespace = line.strip().replace(".", os.sep)
-                if namespace:
-                    namespace_pkg_dirs.add(os.path.join(wheel_dir, namespace))
-    return namespace_pkg_dirs
-
-
-def native_namespace_packages_supported() -> bool:
-    """Returns true if this version of Python supports native namespace packages."""
-    return (sys.version_info.major, sys.version_info.minor) >= (3, 3)
-
-
 def implicit_namespace_packages(
     directory: str, ignored_dirnames: Optional[List[str]] = None
 ) -> Set[str]:
@@ -84,8 +59,7 @@ def add_pkgutil_style_namespace_pkg_init(dir_path: str) -> None:
 
     if os.path.isfile(ns_pkg_init_filepath):
         raise ValueError("%s already contains an __init__.py file." % dir_path)
-    if not os.path.exists(dir_path):
-        os.makedirs(dir_path)
+
     with open(ns_pkg_init_filepath, "w") as ns_pkg_init_f:
         # See https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages
         ns_pkg_init_f.write(