Parse requirements files for global pip flags. (#456)

Parse requirements files for global pip flags, then add them to the extra_pip_args array.

Pass a copy of each cleaned up line (comments and line-breaks removed) to
each whl_library repo verbatim. Then link based requirements or requirement
lines with requirement specific flags work.

Due to the pip restriction on using requirement specific flags in
requirements.txt files only, write each requirement line to a temp file
before invoking pip in each whl_library repo.

Fixes #438 #447

Author: hrfuller

examples/pip_parse/WORKSPACE

@@ -29,7 +29,7 @@ pip_parse(
     # (Optional) You can set quiet to False if you want to see pip output.
     #quiet = False,
 
-    # Uses the default repository name "pip_incremental"
+    # Uses the default repository name "pip_parsed_deps"
     requirements_lock = "//:requirements_lock.txt",
 )
 

examples/pip_parse/requirements_lock.txt

@@ -2,15 +2,27 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file=requirements_lock.txt requirements.txt
+#    pip-compile --generate-hashes  --output-file=requirements_lock.txt requirements.txt
 #
-certifi==2020.12.5
+--extra-index-url https://pypi.org/simple
+
+certifi==2020.12.5 \
+    --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \
+    --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 \
     # via requests
-chardet==3.0.4
+chardet==3.0.4 \
+    --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
+    --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
     # via requests
-idna==2.10
+idna==2.10 \
+    --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \
+    --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 \
     # via requests
-requests==2.24.0
+requests==2.24.0 \
+    --hash=sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b \
+    --hash=sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898 \
     # via -r requirements.txt
-urllib3==1.25.11
+urllib3==1.25.11 \
+    --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \
+    --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e \
     # via requests

python/pip_install/extract_wheels/lib/arguments_test.py

@@ -17,7 +17,7 @@ def test_arguments(self) -> None:
         args_dict = deserialize_structured_args(args_dict)
         self.assertIn("repo", args_dict)
         self.assertIn("extra_pip_args", args_dict)
-        self.assertEqual(args_dict["pip_data_exclude"], None)
+        self.assertEqual(args_dict["pip_data_exclude"], [])
         self.assertEqual(args_dict["enable_implicit_namespace_pkgs"], False)
         self.assertEqual(args_dict["repo"], repo_name)
         self.assertEqual(args_dict["extra_pip_args"], index_url)

python/pip_install/parse_requirements_to_bzl/__init__.py

@@ -2,28 +2,47 @@
 import json
 import textwrap
 import sys
+import shlex
 from typing import List, Tuple
 
 from python.pip_install.extract_wheels.lib import bazel, arguments
 from pip._internal.req import parse_requirements, constructors
 from pip._internal.req.req_install import InstallRequirement
+from pip._internal.req.req_file import get_file_content, preprocess, handle_line, get_line_parser, RequirementsFileParser
 from pip._internal.network.session import PipSession
 
 
-def parse_install_requirements(requirements_lock: str) -> List[InstallRequirement]:
-    return [
-        constructors.install_req_from_parsed_requirement(pr)
-        for pr in parse_requirements(requirements_lock, session=PipSession())
-    ]
+def parse_install_requirements(requirements_lock: str, extra_pip_args: List[str]) -> List[Tuple[InstallRequirement, str]]:
+    ps = PipSession()
+    # This is roughly taken from pip._internal.req.req_file.parse_requirements
+    # (https://github.com/pypa/pip/blob/21.0.1/src/pip/_internal/req/req_file.py#L127) in order to keep
+    # the original line (sort-of, its preprocessed) from the requirements_lock file around, to pass to sub repos
+    # as the requirement.
+    line_parser = get_line_parser(finder=None)
+    parser = RequirementsFileParser(ps, line_parser)
+    install_req_and_lines: List[Tuple[InstallRequirement, str]] = []
+    _, content = get_file_content(requirements_lock, ps)
+    for parsed_line, (_, line) in zip(parser.parse(requirements_lock, constraint=False), preprocess(content)):
+        if parsed_line.is_requirement:
+            install_req_and_lines.append(
+                (
+                    constructors.install_req_from_line(parsed_line.requirement),
+                    line
+                )
+            )
+
+        else:
+            extra_pip_args.extend(shlex.split(line))
+    return install_req_and_lines
 
 
-def repo_names_and_requirements(install_reqs: List[InstallRequirement], repo_prefix: str) -> List[Tuple[str, str]]:
+def repo_names_and_requirements(install_reqs: List[Tuple[InstallRequirement, str]], repo_prefix: str) -> List[Tuple[str, str]]:
     return [
         (
             bazel.sanitise_name(ir.name, prefix=repo_prefix),
-            str(ir.req)
+            line,
         )
-        for ir in install_reqs
+        for ir, line in install_reqs
     ]
 
 def deserialize_structured_args(args):
@@ -35,6 +54,8 @@ def deserialize_structured_args(args):
     for arg_name in structured_args:
         if args.get(arg_name) is not None:
             args[arg_name] = json.loads(args[arg_name])["args"]
+        else:
+            args[arg_name] = []
     return args
 
 
@@ -54,13 +75,13 @@ def generate_parsed_requirements_contents(all_args: argparse.Namespace) -> str:
     requirements_lock = args.pop("requirements_lock")
     repo_prefix = bazel.whl_library_repo_prefix(args["repo"])
 
-    install_reqs = parse_install_requirements(requirements_lock)
-    repo_names_and_reqs = repo_names_and_requirements(install_reqs, repo_prefix)
+    install_req_and_lines = parse_install_requirements(requirements_lock, args["extra_pip_args"])
+    repo_names_and_reqs = repo_names_and_requirements(install_req_and_lines, repo_prefix)
     all_requirements = ", ".join(
-        [bazel.sanitised_repo_library_label(ir.name, repo_prefix=repo_prefix) for ir in install_reqs]
+        [bazel.sanitised_repo_library_label(ir.name, repo_prefix=repo_prefix) for ir, _ in install_req_and_lines]
     )
     all_whl_requirements = ", ".join(
-        [bazel.sanitised_repo_file_label(ir.name, repo_prefix=repo_prefix) for ir in install_reqs]
+        [bazel.sanitised_repo_file_label(ir.name, repo_prefix=repo_prefix) for ir, _ in install_req_and_lines]
     )
     return textwrap.dedent("""\
         load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")

python/pip_install/parse_requirements_to_bzl/extract_single_wheel/__init__.py

@@ -4,6 +4,8 @@
 import subprocess
 import json
 
+from tempfile import NamedTemporaryFile
+
 from python.pip_install.extract_wheels.lib import bazel, requirements, arguments
 from python.pip_install.extract_wheels import configure_reproducible_wheels
 
@@ -27,10 +29,15 @@ def main() -> None:
     if args.extra_pip_args:
         pip_args += json.loads(args.extra_pip_args)["args"]
 
-    pip_args.append(args.requirement)
+    with NamedTemporaryFile(mode='wb') as requirement_file:
+        requirement_file.write(args.requirement.encode("utf-8"))
+        requirement_file.flush()
+        # Requirement specific args like --hash can only be passed in a requirements file,
+        # so write our single requirement into a temp file in case it has any of those flags.
+        pip_args.extend(["-r", requirement_file.name])
 
-    # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
-    subprocess.run(pip_args, check=True)
+        # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
+        subprocess.run(pip_args, check=True)
 
     name, extras_for_pkg = requirements._parse_requirement_for_extra(args.requirement)
     extras = {name: extras_for_pkg} if extras_for_pkg and name else dict()

python/pip_install/parse_requirements_to_bzl/parse_requirements_to_bzl_test.py

@@ -15,8 +15,9 @@ class TestParseRequirementsToBzl(unittest.TestCase):
 
     def test_generated_requirements_bzl(self) -> None:
         with NamedTemporaryFile() as requirements_lock:
-            requirement_string = "foo==0.0.0"
-            requirements_lock.write(bytes(requirement_string, encoding="utf-8"))
+            comments_and_flags = "#comment\n--require-hashes True\n"
+            requirement_string = "foo==0.0.0 --hash=sha256:hashofFoowhl"
+            requirements_lock.write(bytes(comments_and_flags + requirement_string, encoding="utf-8"))
             requirements_lock.flush()
             args = argparse.Namespace()
             args.requirements_lock = requirements_lock.name
@@ -32,7 +33,8 @@ def test_generated_requirements_bzl(self) -> None:
             self.assertIn(all_whl_requirements, contents, contents)
             self.assertIn(requirement_string, contents, contents)
             self.assertIn(requirement_string, contents, contents)
-            self.assertIn("'extra_pip_args': {}".format(repr(extra_pip_args)), contents, contents)
+            all_flags = extra_pip_args + ["--require-hashes", "True"]
+            self.assertIn("'extra_pip_args': {}".format(repr(all_flags)), contents, contents)
 
 
 if __name__ == "__main__":