Merge pull request #3 from dillon-giacoppo/DEPLOY-445-improvements

Author: dillon-giacoppo

.gitattributes

@@ -1 +1 @@
-third_party/** linguist-generated=true
+third_party/python/** linguist-generated=true

README.md

@@ -1,18 +1 @@
-workspace(name = "rules_pip")
-
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-
-http_archive(
-    name = "rules_python",
-    url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz",
-    sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161",
-)
-load("@rules_python//python:repositories.bzl", "py_repositories")
-py_repositories()
-
-load("//:defs.bzl", "pip_repository")
-
-pip_repository(
-    name = "py_deps",
-    requirements = "//:requirements.txt",
-)
+workspace(name = "rules_python_external")

WORKSPACE

@@ -1,12 +1,12 @@
 def _pip_repository_impl(rctx):
-    if not rctx.which("python3"):
-        fail("python not found")
+    if not rctx.which(rctx.attr.python_interpreter):
+        fail("python interpreter not found")
 
     rctx.file("BUILD", "")
 
     result = rctx.execute(
         [
-            rctx.which("python3"),
+            rctx.which(rctx.attr.python_interpreter),
             rctx.path(rctx.attr._script),
             "--requirements",
             rctx.path(rctx.attr.requirements),
@@ -26,12 +26,12 @@ pip_repository = repository_rule(
     attrs={
         "requirements": attr.label(allow_single_file=True, mandatory=True,),
         "wheel_env": attr.string_dict(),
+        "python_interpreter": attr.string(default="python3"),
         "_script": attr.label(
             executable=True,
             default=Label("//tools:wheel_wrapper.py"),
             cfg="host",
         ),
     },
-    local=False,
     implementation=_pip_repository_impl,
 )

defs.bzl

@@ -0,0 +1,101 @@
+import argparse
+import glob
+import os
+import subprocess
+import sys
+
+from .wheel import Wheel
+
+BUILD_TEMPLATE = """\
+package(default_visibility = ["//visibility:public"])
+
+load("@rules_python//python:defs.bzl", "py_library")
+
+py_library(
+    name = "{name}",
+    srcs = glob(["**/*.py"]),
+    data = glob(["**/*"], exclude=["**/*.py", "**/* *", "BUILD", "WORKSPACE"]),
+    # This makes this directory a top-level in the python import
+    # search path for anything that depends on this.
+    imports = ["."],
+    deps = [{dependencies}],
+)
+"""
+
+
+def sanitise_name(name):
+    return name.replace("-", "_").replace(".", "_").lower()
+
+
+def extract_wheel(whl, directory, extras):
+    """
+    Unzips a wheel into the Bazel repository and creates the BUILD file
+
+    :param whl: the Wheel object we are unpacking
+    :param directory: the subdirectory of the external repo to unzip to
+    :param extras: list of extras that we want to create targets for
+    """
+
+    whl.unzip(directory)
+
+    with open(os.path.join(directory, "BUILD"), "w") as f:
+        f.write(
+            BUILD_TEMPLATE.format(
+                name=sanitise_name(whl.name()),
+                dependencies=",".join(
+                    # Python libraries cannot have hyphen https://github.com/bazelbuild/bazel/issues/9171
+                    [
+                        '"//%s"' % sanitise_name(d)
+                        for d in whl.dependencies(extras_requested=extras)
+                    ]
+                ),
+            )
+        )
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Resolve and fetch artifacts transitively from PyPI"
+    )
+    parser.add_argument(
+        "--requirements",
+        action="store",
+        help="Path to requirements.txt from where to install dependencies",
+    )
+    parser.add_argument(
+        "--repo",
+        action="store",
+        help="The external repo name to install dependencies.",
+    )
+    args = parser.parse_args()
+
+    # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
+    subprocess.check_output(
+        [sys.executable, "-m", "pip", "wheel", "-r", args.requirements]
+    )
+
+    targets = set()
+
+    for wheel in [Wheel(whl) for whl in glob.glob("*.whl")]:
+        whl_label = sanitise_name(wheel.name())
+        os.mkdir(whl_label)
+        extract_wheel(wheel, whl_label, [])
+        targets.add('"{repo}//{name}"'.format(repo=args.repo, name=whl_label))
+        os.remove(wheel.path())
+
+    with open("requirements.bzl", "w") as f:
+        f.write(
+            """\
+all_requirements = [{requirement_labels}]
+
+def requirement(name):
+    name_key = name.replace("-", "_").replace(".", "_").lower()
+    return "{repo}//" + name_key
+""".format(
+                requirement_labels=",".join(targets), repo=args.repo
+            )
+        )
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,39 @@
+import pkginfo
+import zipfile
+import pkg_resources
+
+
+class Wheel(object):
+    def __init__(self, path):
+        self._path = path
+
+    def path(self):
+        return self._path
+
+    def name(self):
+        return self.metadata().name
+
+    def metadata(self):
+        return pkginfo.get_metadata(self.path())
+
+    def dependencies(self, extras_requested=None):
+        if not extras_requested:
+            # Provide an extra to safely evaluate the markers
+            # without matching any extra
+            extras_requested = [""]
+
+        dependency_set = set()
+
+        for req in self.metadata().requires_dist:
+            r = pkg_resources.Requirement(req)
+
+            if r.marker is None or any(
+                r.marker.evaluate({"extra": extra}) for extra in extras_requested
+            ):
+                dependency_set.add(r.name)
+
+        return dependency_set
+
+    def unzip(self, directory):
+        with zipfile.ZipFile(self.path(), "r") as whl:
+            whl.extractall(directory)