Introduce compile_pip_requirements rule (#373)

This uses pip-tools to compile a requirements.in file to a requirements.txt file,
allowing transitive dependency versions to be pinned so that builds are reproducible.

Fixes #176

Author: alexeagle

examples/pip_install/.bazelrc

@@ -0,0 +1 @@
+test --test_output=errors

examples/pip_install/BUILD

@@ -1,5 +1,6 @@
 load("@pip//:requirements.bzl", "requirement")
 load("@rules_python//python:defs.bzl", "py_binary", "py_test")
+load("@rules_python//python/pip_install:requirements.bzl", "compile_pip_requirements")
 
 # Toolchain setup, this is optional.
 # Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE).
@@ -40,3 +41,8 @@ py_test(
     srcs = ["test.py"],
     deps = [":main"],
 )
+
+# Check that our compiled requirements are up-to-date
+compile_pip_requirements(
+    name = "requirements",
+)

examples/pip_install/requirements.in

@@ -0,0 +1 @@
+boto3==1.14.51

examples/pip_install/requirements.txt

@@ -1 +1,43 @@
-boto3==1.14.51
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    bazel run //:requirements.update
+#
+boto3==1.14.51 \
+    --hash=sha256:a6bdb808e948bd264af135af50efb76253e85732c451fa605b7a287faf022432 \
+    --hash=sha256:f9dbccbcec916051c6588adbccae86547308ac4cd154f1eb7cf6422f0e391a71
+    # via -r ./requirements.in
+botocore==1.17.63 \
+    --hash=sha256:40f13f6c9c29c307a9dc5982739e537ddce55b29787b90c3447b507e3283bcd6 \
+    --hash=sha256:aa88eafc6295132f4bc606f1df32b3248e0fa611724c0a216aceda767948ac75
+    # via
+    #   boto3
+    #   s3transfer
+docutils==0.15.2 \
+    --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \
+    --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \
+    --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99
+    # via botocore
+jmespath==0.10.0 \
+    --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \
+    --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f
+    # via
+    #   boto3
+    #   botocore
+python-dateutil==2.8.1 \
+    --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
+    --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a
+    # via botocore
+s3transfer==0.3.3 \
+    --hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \
+    --hash=sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db
+    # via boto3
+six==1.15.0 \
+    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
+    # via python-dateutil
+urllib3==1.25.11 \
+    --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \
+    --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e
+    # via botocore

python/BUILD

@@ -24,6 +24,8 @@ In an ideal renaming, we'd move the packaging rules to a different package so
 that @rules_python//python is only concerned with the core rules.
 """
 
+load("//python/pip_install:requirements.bzl", "compile_pip_requirements")
+
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # Apache 2.0
@@ -145,3 +147,8 @@ exports_files([
     "pip.bzl",
     "whl.bzl",
 ])
+
+compile_pip_requirements(
+    name = "requirements",
+    extra_args = ["--allow-unsafe"],
+)

python/pip_install/BUILD

@@ -1,7 +1,10 @@
+exports_files(["pip_compile.py"])
+
 filegroup(
     name = "distribution",
     srcs = glob(["*.bzl"]) + [
         "BUILD",
+        "pip_compile.py",
         "//python/pip_install/extract_wheels:distribution",
         "//python/pip_install/parse_requirements_to_bzl:distribution",
     ],

python/pip_install/pip_compile.py

@@ -0,0 +1,89 @@
+"Set defaults for the pip-compile command to run it under Bazel"
+
+import os
+import sys
+from shutil import copyfile
+
+from piptools.scripts.compile import cli
+
+if len(sys.argv) < 4:
+    print(
+        "Expected at least two arguments: requirements_in requirements_out",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+requirements_in = sys.argv.pop(1)
+requirements_txt = sys.argv.pop(1)
+update_target_name = sys.argv.pop(1)
+
+# Before loading click, set the locale for its parser.
+# If it leaks through to the system setting, it may fail:
+# RuntimeError: Click will abort further execution because Python 3 was configured to use ASCII
+# as encoding for the environment. Consult https://click.palletsprojects.com/python3/ for
+# mitigation steps.
+os.environ["LC_ALL"] = "C.UTF-8"
+os.environ["LANG"] = "C.UTF-8"
+
+UPDATE = True
+# Detect if we are running under `bazel test`
+if "TEST_TMPDIR" in os.environ:
+    UPDATE = False
+    # pip-compile wants the cache files to be writeable, but if we point
+    # to the real user cache, Bazel sandboxing makes the file read-only
+    # and we fail.
+    # In theory this makes the test more hermetic as well.
+    sys.argv.append("--cache-dir")
+    sys.argv.append(os.environ["TEST_TMPDIR"])
+    # Make a copy for pip-compile to read and mutate
+    requirements_out = os.path.join(
+        os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out"
+    )
+    copyfile(requirements_txt, requirements_out)
+
+elif "BUILD_WORKING_DIRECTORY" in os.environ:
+    os.chdir(os.environ['BUILD_WORKING_DIRECTORY'])
+else:
+    print(
+        "Expected to find BUILD_WORKING_DIRECTORY in environment",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+update_target_pkg = "/".join(requirements_in.split('/')[:-1])
+# $(rootpath) in the workspace root gives ./requirements.in
+if update_target_pkg == ".":
+    update_target_pkg = ""
+update_command = "bazel run //%s:%s" % (update_target_pkg, update_target_name)
+
+os.environ["CUSTOM_COMPILE_COMMAND"] = update_command
+
+sys.argv.append("--generate-hashes")
+sys.argv.append("--output-file")
+sys.argv.append(requirements_txt if UPDATE else requirements_out)
+sys.argv.append(requirements_in)
+
+if UPDATE:
+    print("Updating " + requirements_txt)
+    cli()
+else:
+    # cli will exit(0) on success
+    try:
+        print("Checking " + requirements_txt)
+        cli()
+        print("cli() should exit", file=sys.stderr)
+        sys.exit(1)
+    except SystemExit:
+        golden = open(requirements_txt).readlines()
+        out = open(requirements_out).readlines()
+        if golden != out:
+            import difflib
+
+            print(''.join(difflib.unified_diff(golden, out)), file=sys.stderr)
+            print(
+                "Lock file out of date. Run '"
+                + update_command
+                + "' to update.",
+                file=sys.stderr,
+            )
+            sys.exit(1)

python/pip_install/repositories.bzl

@@ -4,11 +4,21 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 
 _RULE_DEPS = [
+    (
+        "pypi__click",
+        "https://files.pythonhosted.org/packages/d2/3d/fa76db83bf75c4f8d338c2fd15c8d33fdd7ad23a9b5e57eb6c5de26b430e/click-7.1.2-py2.py3-none-any.whl",
+        "dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc",
+    ),
     (
         "pypi__pip",
         "https://files.pythonhosted.org/packages/fe/ef/60d7ba03b5c442309ef42e7d69959f73aacccd0d86008362a681c4698e83/pip-21.0.1-py3-none-any.whl",
         "37fd50e056e2aed635dec96594606f0286640489b0db0ce7607f7e51890372d5",
     ),
+    (
+        "pypi__pip_tools",
+        "https://files.pythonhosted.org/packages/6d/16/75d65bdccd48bb59a08e2bf167b01d8532f65604270d0a292f0f16b7b022/pip_tools-5.5.0-py2.py3-none-any.whl",
+        "10841c1e56c234d610d0466447685b9ea4ee4a2c274f858c0ef3c33d9bd0d985",
+    ),
     (
         "pypi__pkginfo",
         "https://files.pythonhosted.org/packages/4f/3c/535287349af1b117e082f8e77feca52fbe2fdf61ef1e6da6bcc2a72a3a79/pkginfo-1.6.1-py2.py3-none-any.whl",

python/pip_install/requirements.bzl

@@ -0,0 +1,94 @@
+"Rules to verify and update pip-compile locked requirements.txt"
+
+load("//python:defs.bzl", "py_binary", "py_test")
+load("//python/pip_install:repositories.bzl", "requirement")
+
+def compile_pip_requirements(
+        name,
+        extra_args = [],
+        visibility = ["//visibility:private"],
+        requirements_in = "requirements.in",
+        requirements_txt = "requirements.txt",
+        **kwargs):
+    """
+    Macro creating targets for running pip-compile
+
+    Produce a filegroup by default, named "[name]" which can be included in the data
+    of some other compile_pip_requirements rule that references these requirements
+    (e.g. with `-r ../other/requirements.txt`)
+
+    Produce two targets for checking pip-compile:
+
+    - validate with `bazel test <name>_test`
+    - update with   `bazel run <name>.update`
+
+    Args:
+        name: base name for generated targets, typically "requirements"
+        extra_args: passed to pip-compile
+        visibility: passed to both the _test and .update rules
+        requirements_in: file expressing desired dependencies
+        requirements_txt: result of "compiling" the requirements.in file
+        **kwargs: other bazel attributes passed to the "_test" rule
+    """
+    requirements_in = kwargs.pop("requirements_in", name + ".in")
+    requirements_txt = kwargs.pop("requirements_locked", name + ".txt")
+
+    # "Default" target produced by this macro
+    # Allow a compile_pip_requirements rule to include another one in the data
+    # for a requirements file that does `-r ../other/requirements.txt`
+    native.filegroup(
+        name = name,
+        srcs = kwargs.pop("data", []) + [requirements_txt],
+        visibility = visibility,
+    )
+
+    data = [name, requirements_in, requirements_txt]
+
+    # Use the Label constructor so this is expanded in the context of the file
+    # where it appears, which is to say, in @rules_python
+    pip_compile = Label("//python/pip_install:pip_compile.py")
+
+    loc = "$(rootpath %s)"
+
+    args = [
+        loc % requirements_in,
+        loc % requirements_txt,
+        name + ".update",
+    ] + extra_args
+
+    deps = [
+        requirement("click"),
+        requirement("pip"),
+        requirement("pip_tools"),
+        requirement("setuptools"),
+    ]
+
+    attrs = {
+        "args": args,
+        "data": data,
+        "deps": deps,
+        "main": pip_compile,
+        "srcs": [pip_compile],
+        "visibility": visibility,
+    }
+
+    # cheap way to detect the bazel version
+    _bazel_version_4_or_greater = "propeller_optimize" in dir(native)
+
+    # Bazel 4.0 added the "env" attribute to py_test/py_binary
+    if _bazel_version_4_or_greater:
+        attrs["env"] = kwargs.pop("env", {})
+
+    py_binary(
+        name = name + ".update",
+        **attrs
+    )
+
+    timeout = kwargs.pop("timeout", "short")
+
+    py_test(
+        name = name + "_test",
+        timeout = timeout,
+        # kwargs could contain test-specific attributes like size or timeout
+        **dict(attrs, **kwargs)
+    )

python/requirements.in

@@ -0,0 +1,10 @@
+pip==9.0.3
+setuptools==44.0.0
+wheel==0.30.0a0
+
+# For tests
+mock==2.0.0
+# This is a transitive dependency of mock, which only appears on some python versions
+# Right now our repo doesn't pin our Python interpreter, so our locked requirements
+# may differ between local dev and CI, for example.
+funcsigs==1.0.2