Incrementally download wheels at workspace time. (#432)

* Create support for lazily fetched repo's.
Refactor pip_repository rule to invoke different scripts based on the
value of the incremental attribute to the rule.
Create a new macro in repositories.bzl which will instantiate all the child repos representing
individual python packages.
Refactor code which is repeated between the parse_requirements_to_bzl
scripts and the extract_wheels script.

Author: hrfuller

.bazelrc

@@ -3,7 +3,7 @@
 # This lets us glob() up all the files inside the examples to make them inputs to tests
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh
-build --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install
-query --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install
+build --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse
+query --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse
 
 test --test_output=errors

.gitignore

@@ -37,3 +37,7 @@ bazel-bin
 bazel-genfiles
 bazel-out
 bazel-testlogs
+
+# vim swap files
+*.swp
+*.swo

README.md

@@ -105,7 +105,7 @@ target in the appropriate wheel repo.
 
 ### Importing `pip` dependencies
 
-To add pip dependencies to your `WORKSPACE` is you load
+To add pip dependencies to your `WORKSPACE` load
 the `pip_install` function, and call it to create the
 individual wheel repos.
 
@@ -136,6 +136,40 @@ re-executed in order to pick up a non-hermetic change to your environment (e.g.,
 updating your system `python` interpreter), you can completely flush out your
 repo cache with `bazel clean --expunge`.
 
+### Fetch `pip` dependencies lazily (experimental)
+
+One pain point with `pip_install` is the need to download all dependencies resolved by
+your requirements.txt before the bazel analysis phase can start. For large python monorepos
+this can take a long time, especially on slow connections.
+
+`pip_parse` provides a solution to this problem. If you can provide a lock
+file of all your python dependencies `pip_parse` will translate each requirement into its own external repository.
+Bazel will only fetch/build wheels for the requirements in the subgraph of your build target.
+
+There are API differences between `pip_parse` and `pip_install`:
+1. `pip_parse` requires a fully resolved lock file of your python dependencies. You can generate this using
+   `pip-compile`, or a virtualenv and `pip freeze`. `pip_parse` uses a label argument called `requirements_lock` instead of `requirements`
+   to make this distinction clear.
+2. `pip_parse` translates your requirements into a starlark macro called `install_deps`. You must call this macro in your WORKSPACE to
+   declare your dependencies.
+
+
+```python
+load("@rules_python//python:pip.bzl", "pip_parse")
+
+# Create a central repo that knows about the dependencies needed from
+# requirements_lock.txt.
+pip_parse(
+   name = "my_deps",
+   requirements_lock = "//path/to:requirements_lock.txt",
+)
+
+# Load the starlark macro which will define your dependencies.
+load("@my_deps//:requirements.bzl", "install_deps")
+# Call it to define repos for your requirements.
+install_deps()
+```
+
 ### Importing `pip` dependencies with `pip_import` (legacy)
 
 The deprecated `pip_import` can still be used if needed.

examples/BUILD

@@ -26,3 +26,8 @@ bazel_integration_test(
     name = "pip_install_example",
     timeout = "long",
 )
+
+bazel_integration_test(
+    name = "pip_parse_example",
+    timeout = "long",
+)

examples/pip_parse/BUILD

@@ -0,0 +1,42 @@
+load("@pip_parsed_deps//:requirements.bzl", "requirement")
+load("@rules_python//python:defs.bzl", "py_binary", "py_test")
+
+# Toolchain setup, this is optional.
+# Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE).
+#
+#load("@rules_python//python:defs.bzl", "py_runtime_pair")
+#
+#py_runtime(
+#    name = "python3_runtime",
+#    files = ["@python_interpreter//:files"],
+#    interpreter = "@python_interpreter//:python_bin",
+#    python_version = "PY3",
+#    visibility = ["//visibility:public"],
+#)
+#
+#py_runtime_pair(
+#    name = "my_py_runtime_pair",
+#    py2_runtime = None,
+#    py3_runtime = ":python3_runtime",
+#)
+#
+#toolchain(
+#    name = "my_py_toolchain",
+#    toolchain = ":my_py_runtime_pair",
+#    toolchain_type = "@bazel_tools//tools/python:toolchain_type",
+#)
+# End of toolchain setup.
+
+py_binary(
+    name = "main",
+    srcs = ["main.py"],
+    deps = [
+        requirement("requests"),
+    ],
+)
+
+py_test(
+    name = "test",
+    srcs = ["test.py"],
+    deps = [":main"],
+)

examples/pip_parse/WORKSPACE

@@ -0,0 +1,39 @@
+workspace(name = "example_repo")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "rules_python",
+    url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz",
+    sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0",
+)
+
+load("@rules_python//python:pip.bzl", "pip_parse")
+
+pip_parse(
+    # (Optional) You can provide extra parameters to pip.
+    # Here, make pip output verbose (this is usable with `quiet = False`).
+    # extra_pip_args = ["-v"],
+
+    # (Optional) You can exclude custom elements in the data section of the generated BUILD files for pip packages.
+    # Exclude directories with spaces in their names in this example (avoids build errors if there are such directories).
+    #pip_data_exclude = ["**/* */**"],
+
+    # (Optional) You can provide a python_interpreter (path) or a python_interpreter_target (a Bazel target, that
+    # acts as an executable). The latter can be anything that could be used as Python interpreter. E.g.:
+    # 1. Python interpreter that you compile in the build file (as above in @python_interpreter).
+    # 2. Pre-compiled python interpreter included with http_archive
+    # 3. Wrapper script, like in the autodetecting python toolchain.
+    #python_interpreter_target = "@python_interpreter//:python_bin",
+
+    # (Optional) You can set quiet to False if you want to see pip output.
+    #quiet = False,
+
+    # Uses the default repository name "pip_incremental"
+    requirements_lock = "//:requirements_lock.txt",
+)
+
+load("@pip_parsed_deps//:requirements.bzl", "install_deps")
+
+# Initialize repositories for all packages in requirements_lock.txt.
+install_deps()

examples/pip_parse/main.py

@@ -0,0 +1,5 @@
+import requests
+
+
+def version():
+    return requests.__version__

examples/pip_parse/requirements.txt

@@ -0,0 +1 @@
+requests==2.24.0

examples/pip_parse/requirements_lock.txt

@@ -0,0 +1,16 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file=requirements_lock.txt requirements.txt
+#
+certifi==2020.12.5
+    # via requests
+chardet==3.0.4
+    # via requests
+idna==2.10
+    # via requests
+requests==2.24.0
+    # via -r requirements.txt
+urllib3==1.25.11
+    # via requests

examples/pip_parse/test.py

@@ -0,0 +1,11 @@
+import unittest
+import main
+
+
+class ExampleTest(unittest.TestCase):
+    def test_main(self):
+        self.assertEqual("2.24.0", main.version())
+
+
+if __name__ == '__main__':
+    unittest.main()