Document how to vendor a pip_parse requirements.bzl file (#655)

* Document how to vendor a pip_parse requirements.bzl file

fixes #608

* code review feedback

Author: Alex Eagle

docs/pip.md

@@ -156,6 +156,7 @@ pip_parse(<a href="#pip_parse-requirements_lock">requirements_lock</a>, <a href=
 Accepts a locked/compiled requirements file and installs the dependencies listed within.
 
 Those dependencies become available in a generated `requirements.bzl` file.
+You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below.
 
 This macro runs a repository rule that invokes `pip`. In your WORKSPACE file:
 
@@ -218,6 +219,31 @@ alias(
 )
 ```
 
+## Vendoring the requirements.bzl file
+
+In some cases you may not want to generate the requirements.bzl file as a repository rule
+while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module
+such as a ruleset, you may want to include the requirements.bzl file rather than make your users
+install the WORKSPACE setup to generate it.
+See https://github.com/bazelbuild/rules_python/issues/608
+
+This is the same workflow as Gazelle, which creates `go_repository` rules with
+[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos)
+
+Simply run the same tool that the `pip_parse` repository rule calls.
+You can find the arguments in the generated BUILD file in the pip_parse repo,
+for example in `$(bazel info output_base)/external/pypi/BUILD.bazel` for a repo
+named `pypi`.
+
+The command will look like this:
+```shell
+bazel run -- @rules_python//python/pip_install/parse_requirements_to_bzl \
+  --requirements_lock ./requirements_lock.txt \
+  --quiet False --timeout 120 --repo pypi --repo-prefix pypi_ > requirements.bzl
+```
+
+Then load the requirements.bzl file directly, without using `pip_parse` in the WORKSPACE.
+
 
 **PARAMETERS**
 

python/pip.bzl

@@ -105,6 +105,7 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs):
     """Accepts a locked/compiled requirements file and installs the dependencies listed within.
 
     Those dependencies become available in a generated `requirements.bzl` file.
+    You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below.
 
     This macro runs a repository rule that invokes `pip`. In your WORKSPACE file:
 
@@ -167,6 +168,31 @@ def pip_parse(requirements_lock, name = "pip_parsed_deps", **kwargs):
     )
     ```
 
+    ## Vendoring the requirements.bzl file
+
+    In some cases you may not want to generate the requirements.bzl file as a repository rule
+    while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module
+    such as a ruleset, you may want to include the requirements.bzl file rather than make your users
+    install the WORKSPACE setup to generate it.
+    See https://github.com/bazelbuild/rules_python/issues/608
+
+    This is the same workflow as Gazelle, which creates `go_repository` rules with
+    [`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos)
+
+    Simply run the same tool that the `pip_parse` repository rule calls.
+    You can find the arguments in the generated BUILD file in the pip_parse repo,
+    for example in `$(bazel info output_base)/external/pypi/BUILD.bazel` for a repo
+    named `pypi`.
+
+    The command will look like this:
+    ```shell
+    bazel run -- @rules_python//python/pip_install/parse_requirements_to_bzl \\
+      --requirements_lock ./requirements_lock.txt \\
+      --quiet False --timeout 120 --repo pypi --repo-prefix pypi_ > requirements.bzl
+    ```
+
+    Then load the requirements.bzl file directly, without using `pip_parse` in the WORKSPACE.
+
     Args:
         requirements_lock (Label): A fully resolved 'requirements.txt' pip requirement file
             containing the transitive set of your dependencies. If this file is passed instead

python/pip_install/parse_requirements_to_bzl/__init__.py

@@ -4,7 +4,7 @@
 import sys
 import textwrap
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, TextIO, Tuple
 
 from pip._internal.network.session import PipSession
 from pip._internal.req import constructors
@@ -166,7 +166,11 @@ def coerce_to_bool(option):
     return str(option).lower() == "true"
 
 
-def main() -> None:
+def main(output: TextIO) -> None:
+    """Args:
+
+    output: where to write the resulting starlark, such as sys.stdout or an open file
+    """
     parser = argparse.ArgumentParser(
         description="Create rules to incrementally fetch needed \
 dependencies from a fully resolved requirements lock file."
@@ -216,7 +220,7 @@ def main() -> None:
         args.requirements_lock, whl_library_args["extra_pip_args"]
     )
     req_names = sorted([req.name for req, _ in install_requirements])
-    annotations = args.annotations.collect(req_names)
+    annotations = args.annotations.collect(req_names) if args.annotations else {}
 
     # Write all rendered annotation files and generate a list of the labels to write to the requirements file
     annotated_requirements = dict()
@@ -231,12 +235,11 @@ def main() -> None:
             }
         )
 
-    with open("requirements.bzl", "w") as requirement_file:
-        requirement_file.write(
-            generate_parsed_requirements_contents(
-                requirements_lock=args.requirements_lock,
-                repo_prefix=args.repo_prefix,
-                whl_library_args=whl_library_args,
-                annotations=annotated_requirements,
-            )
+    output.write(
+        generate_parsed_requirements_contents(
+            requirements_lock=args.requirements_lock,
+            repo_prefix=args.repo_prefix,
+            whl_library_args=whl_library_args,
+            annotations=annotated_requirements,
         )
+    )

python/pip_install/parse_requirements_to_bzl/__main__.py

@@ -1,5 +1,16 @@
 """Main entry point."""
+import os
+import sys
+
 from python.pip_install.parse_requirements_to_bzl import main
 
 if __name__ == "__main__":
-    main()
+    # Under `bazel run`, just print the generated starlark code.
+    # This allows users to check that into their repository rather than
+    # call pip_parse to generate as a repository rule.
+    if "BUILD_WORKING_DIRECTORY" in os.environ:
+        os.chdir(os.environ["BUILD_WORKING_DIRECTORY"])
+        main(sys.stdout)
+    else:
+        with open("requirements.bzl", "w") as requirement_file:
+            main(requirement_file)

python/pip_install/pip_repository.bzl

@@ -125,9 +125,6 @@ def _pip_repository_impl(rctx):
     if rctx.attr.incremental and not rctx.attr.requirements_lock:
         fail("Incremental mode requires a requirements_lock attribute be specified.")
 
-    # We need a BUILD file to load the generated requirements.bzl
-    rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS)
-
     # Write the annotations file to pass to the wheel maker
     annotations = {package: json.decode(data) for (package, data) in rctx.attr.annotations.items()}
     annotations_file = rctx.path("annotations.json")
@@ -152,7 +149,7 @@ def _pip_repository_impl(rctx):
         args += ["--python_interpreter", _get_python_interpreter_attr(rctx)]
         if rctx.attr.python_interpreter_target:
             args += ["--python_interpreter_target", str(rctx.attr.python_interpreter_target)]
-
+        progress_message = "Parsing requirements to starlark"
     else:
         args = [
             python_interpreter,
@@ -163,10 +160,13 @@ def _pip_repository_impl(rctx):
             "--annotations",
             annotations_file,
         ]
+        progress_message = "Extracting wheels"
 
     args += ["--repo", rctx.attr.name, "--repo-prefix", rctx.attr.repo_prefix]
     args = _parse_optional_attrs(rctx, args)
 
+    rctx.report_progress(progress_message)
+
     result = rctx.execute(
         args,
         # Manually construct the PYTHONPATH since we cannot use the toolchain here
@@ -178,6 +178,9 @@ def _pip_repository_impl(rctx):
     if result.return_code:
         fail("rules_python failed: %s (%s)" % (result.stdout, result.stderr))
 
+    # We need a BUILD file to load the generated requirements.bzl
+    rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS + "\n# The requirements.bzl file was generated by running:\n# " + " ".join([str(a) for a in args]))
+
     return
 
 common_env = [