wip: fetch dependencies

Author: dillon-giacoppo

WORKSPACE

@@ -1 +1,15 @@
 workspace(name = "rules_python_external")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "rules_python",
+    url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz",
+    sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161",
+)
+
+load("@rules_python//python:repositories.bzl", "py_repositories")
+py_repositories()
+
+load("//:repositories.bzl", "rules_python_external_dependencies")
+rules_python_external_dependencies()

defs.bzl

@@ -1,3 +1,8 @@
+load("//:repositories.bzl", "all_requirements")
+
+DEFAULT_REPOSITORY_NAME = "pip"
+
+
 def _pip_repository_impl(rctx):
     if not rctx.which(rctx.attr.python_interpreter):
         fail("python interpreter not found")
@@ -7,17 +12,26 @@ def _pip_repository_impl(rctx):
     result = rctx.execute(
         [
             rctx.which(rctx.attr.python_interpreter),
-            rctx.path(rctx.attr._script),
+            rctx.path(rctx.attr._script).dirname,
             "--requirements",
             rctx.path(rctx.attr.requirements),
             "--repo",
             "@%s" % rctx.attr.name,
         ],
-
-        environment = rctx.attr.wheel_env,
+        environment={
+            # Manually construct the PYTHONPATH since we cannot use the toolchain here
+            "PYTHONPATH": ":".join(
+                # Includes the root of this repo and all the external dependencies from repositories.bzl
+                [str(rctx.path(rctx.attr._script).dirname.dirname)]
+                + [
+                    str(rctx.path(Label("@" + repo + "//:BUILD.bazel")).dirname)
+                    for repo in all_requirements
+                ]
+            )
+        },
     )
     if result.return_code:
-        fail("failed to create pip repository: %s (%s)" % (result.stdout, result.stderr))
+        fail("rules_python_external failed: %s (%s)" % (result.stdout, result.stderr))
 
     return
 
@@ -28,10 +42,14 @@ pip_repository = repository_rule(
         "wheel_env": attr.string_dict(),
         "python_interpreter": attr.string(default="python3"),
         "_script": attr.label(
-            executable=True,
-            default=Label("//tools:wheel_wrapper.py"),
-            cfg="host",
+            executable=True, default=Label("//src:__main__.py"), cfg="host",
         ),
     },
     implementation=_pip_repository_impl,
 )
+
+
+def pip_install(requirements, name=DEFAULT_REPOSITORY_NAME):
+    pip_repository(
+        name=name, requirements=requirements,
+    )

example/BUILD

@@ -0,0 +1,9 @@
+load("@pip//:requirements.bzl", "requirement")
+
+py_binary(
+    name = "main",
+    srcs = ["main.py"],
+    deps = [
+        requirement("boto3")
+    ],
+)

example/WORKSPACE

@@ -0,0 +1,24 @@
+workspace(name = "example_repo")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "rules_python",
+    url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz",
+    sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161",
+)
+load("@rules_python//python:repositories.bzl", "py_repositories")
+py_repositories()
+
+local_repository(
+    name = "rules_python_external",
+    path = "../",
+)
+
+load("@rules_python_external//:repositories.bzl", "rules_python_external_dependencies")
+rules_python_external_dependencies()
+
+load("@rules_python_external//:defs.bzl", "pip_install")
+pip_install(
+    requirements = "//:requirements.txt",
+)

example/main.py

@@ -0,0 +1,3 @@
+import boto3
+
+print("Hello World")

example/requirements.txt

@@ -0,0 +1 @@
+boto3

repositories.bzl

@@ -0,0 +1,55 @@
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+
+_RULE_DEPS = [
+    (
+        "pypi__pip",
+        "https://files.pythonhosted.org/packages/00/b6/9cfa56b4081ad13874b0c6f96af8ce16cfbc1cb06bedf8e9164ce5551ec1/pip-19.3.1-py2.py3-none-any.whl",
+        "6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7",
+    ),
+    (
+        "pypi__pkginfo",
+        "https://files.pythonhosted.org/packages/e6/d5/451b913307b478c49eb29084916639dc53a88489b993530fed0a66bab8b9/pkginfo-1.5.0.1-py2.py3-none-any.whl",
+        "a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32",
+    ),
+    (
+        "pypi__setuptools",
+        "https://files.pythonhosted.org/packages/54/28/c45d8b54c1339f9644b87663945e54a8503cfef59cf0f65b3ff5dd17cf64/setuptools-42.0.2-py2.py3-none-any.whl",
+        "c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6",
+    ),
+    (
+        "pypi__wheel",
+        "https://files.pythonhosted.org/packages/00/83/b4a77d044e78ad1a45610eb88f745be2fd2c6d658f9798a15e384b7d57c9/wheel-0.33.6-py2.py3-none-any.whl",
+        "f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28",
+    ),
+]
+
+_GENERIC_WHEEL = """\
+package(default_visibility = ["//visibility:public"])
+
+load("@rules_python//python:defs.bzl", "py_library")
+
+py_library(
+    name = "lib",
+    srcs = glob(["**/*.py"]),
+    data = glob(["**/*"], exclude=["**/*.py", "**/* *", "BUILD", "WORKSPACE"]),
+    # This makes this directory a top-level in the python import
+    # search path for anything that depends on this.
+    imports = ["."],
+)
+"""
+
+# Collate all the repository names so they can be easily consumed
+all_requirements = [name for (name, _, _) in _RULE_DEPS]
+
+
+def rules_python_external_dependencies():
+    for (name, url, sha256) in _RULE_DEPS:
+        maybe(
+            http_archive,
+            name,
+            url=url,
+            sha256=sha256,
+            type="zip",
+            build_file_content=_GENERIC_WHEEL,
+        )

src/BUILD

@@ -1,20 +1,19 @@
+load("//:repositories.bzl", "all_requirements")
+
 py_library(
     name = "src",
-    srcs = glob(["*.py"]),
     srcs_version = "PY3",
-    deps = [],
+    srcs = glob(["**/*.py"]),
+    data = glob(["**/*"], exclude=["**/*.py", "**/* *", "BUILD", "WORKSPACE"]),
+    deps = ["@"+ repo + "//:lib" for repo in all_requirements],
 )
 
 py_test(
     name = "test",
-    srcs = glob(["tests/*.py"]),
+    srcs = glob(["tests/**/*.py"]),
     main = "tests/__main__.py",
     deps = [
         ":src",
-        "//third_party/python:pkginfo",
     ],
     size="small",
-    imports = [
-        "../third_party/python/",
-    ]
 )

src/__init__.py

@@ -3,29 +3,8 @@
 reproducible wheels
 """
 import os
-import sys
 
-
-def configure_vendor():
-    """
-    The dependencies we vendor need to be added to be added to the PYTHONPATH. We add the root of our vendor tree
-    `third_party` which enables the modules to find their own dependencies as expected.
-    """
-    _this_file = __file__
-    if (_this_file is None) or not os.path.isfile(_this_file):
-        sys.exit("wheel_wrapper.py failed.  Cannot determine __file__")
-
-    _tool_dir = os.path.dirname(_this_file)
-    _root_dir = os.path.abspath(os.path.join(_tool_dir, ".."))
-
-    # This prepends the vendor directory to the sys.path so that we preference their use over any system modules.
-    sys.path[0:0] = [
-        os.path.join(_root_dir, "third_party/python"),
-        os.path.join(_root_dir, "."),
-    ]
-
-    # Pip creates python subprocesses, so we need to add the new path to the PYTHONPATH
-    os.environ["PYTHONPATH"] = os.pathsep.join(sys.path)
+import src.extract_wheels as whl
 
 
 def configure_reproducible_wheels():
@@ -53,15 +32,7 @@ def configure_reproducible_wheels():
         os.environ["PYTHONHASHSEED"] = "0"
 
 
-def main():
-    configure_vendor()
+if __name__ == "__main__":
     configure_reproducible_wheels()
 
-    # This must be imported after vendoring has been configured. As it tries to resolve its own dependencies
-    import src.extract_wheels as whl
-
     whl.main()
-
-
-if __name__ == "__main__":
-    main()