move the logic to the index_sources function

Author: aignas

examples/pip_parse_vendored/requirements.bzl

@@ -33,11 +33,11 @@ all_data_requirements = [
 ]
 
 _packages = [
-    ("my_project_pip_deps_vendored_certifi", "certifi==2023.7.22     --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082     --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"),
-    ("my_project_pip_deps_vendored_charset_normalizer", "charset-normalizer==2.1.1     --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845     --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"),
-    ("my_project_pip_deps_vendored_idna", "idna==3.4     --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4     --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"),
-    ("my_project_pip_deps_vendored_requests", "requests==2.28.1     --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983     --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"),
-    ("my_project_pip_deps_vendored_urllib3", "urllib3==1.26.13     --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc     --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"),
+    ("my_project_pip_deps_vendored_certifi", "certifi==2023.7.22 --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"),
+    ("my_project_pip_deps_vendored_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"),
+    ("my_project_pip_deps_vendored_idna", "idna==3.4 --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"),
+    ("my_project_pip_deps_vendored_requests", "requests==2.28.1 --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"),
+    ("my_project_pip_deps_vendored_urllib3", "urllib3==1.26.13 --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"),
 ]
 _config = {
     "download_only": False,

python/private/pypi/extension.bzl

@@ -292,7 +292,7 @@ def _create_whl_repos(
                     args.pop("download_only", None)
 
                     repo_name = whl_repo_name(pip_name, distribution.filename, distribution.sha256)
-                    args["requirement"] = _requirement_without_marker(requirement.srcs.requirement)
+                    args["requirement"] = requirement.srcs.requirement
                     args["urls"] = [distribution.url]
                     args["sha256"] = distribution.sha256
                     args["filename"] = distribution.filename
@@ -324,7 +324,7 @@ def _create_whl_repos(
                 logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line))
 
             args = dict(whl_library_args)  # make a copy
-            args["requirement"] = _requirement_without_marker(requirement.requirement_line)
+            args["requirement"] = requirement.srcs.requirement_line
             if requirement.extra_pip_args:
                 args["extra_pip_args"] = requirement.extra_pip_args
 
@@ -535,25 +535,6 @@ You cannot use both the additive_build_content and additive_build_content_file a
         is_reproducible = is_reproducible,
     )
 
-def _requirement_without_marker(requirement):
-    requirement, _, maybe_hashes = requirement.partition(";")
-    ret = requirement.strip(" ")
-    _, marker, maybe_hashes = maybe_hashes.partition("--hash=")
-    if not maybe_hashes:
-        ret, marker, maybe_hashes = ret.partition("--hash=")
-
-    if maybe_hashes:
-        return "{} {}{}".format(
-            ret.strip(" "),
-            marker.strip(" "),
-            " --hash=".join([
-                h.strip(" ")
-                for h in maybe_hashes.split("--hash=")
-            ]),
-        )
-    else:
-        return ret
-
 def _pip_impl(module_ctx):
     """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories.
 

python/private/pypi/index_sources.bzl

@@ -26,28 +26,43 @@ def index_sources(line):
         line(str): The requirements.txt entry.
 
     Returns:
-        A struct with shas attribute containing a list of shas to download from pypi_index.
+        A struct with shas attribute containing:
+            * `shas` - a list of shas to download from pypi_index.
+            * `version` - the version of the package.
+            * `marker` - the marker as per PEP508 spec.
+            * `requirement` - a requirement line without the marker. This can
+                be given to `pip` to install a package.
     """
+    line = line.replace("\\", " ")
     head, _, maybe_hashes = line.partition(";")
     _, _, version = head.partition("==")
     version = version.partition(" ")[0].strip()
 
-    if "@" in head:
-        shas = []
-    else:
-        maybe_hashes = maybe_hashes or line
-        shas = [
-            sha.strip()
-            for sha in maybe_hashes.split("--hash=sha256:")[1:]
-        ]
+    marker, _, _ = maybe_hashes.partition("--hash=")
+    maybe_hashes = maybe_hashes or line
+    shas = [
+        sha.strip()
+        for sha in maybe_hashes.split("--hash=sha256:")[1:]
+    ]
 
+    marker = marker.strip()
     if head == line:
-        head = line.partition("--hash=")[0].strip()
+        requirement = line.partition("--hash=")[0].strip()
     else:
-        head = head + ";" + maybe_hashes.partition("--hash=")[0].strip()
+        requirement = head.strip()
+
+    requirement_line = "{} {}".format(
+        requirement,
+        " ".join(["--hash=sha256:{}".format(sha) for sha in shas]),
+    ).strip()
+    if "@" in head:
+        requirement = requirement_line
+        shas = []
 
     return struct(
-        requirement = line if not shas else head,
+        requirement = requirement,
+        requirement_line = requirement_line,
         version = version,
         shas = sorted(shas),
+        marker = marker,
     )

python/private/pypi/pip_repository.bzl

@@ -101,7 +101,7 @@ def _pip_repository_impl(rctx):
         if not r:
             continue
         options = options or r.extra_pip_args
-        selected_requirements[name] = r.requirement_line
+        selected_requirements[name] = r.srcs.requirement_line
 
     bzl_packages = sorted(selected_requirements.keys())
 

tests/pypi/index_sources/index_sources_tests.bzl

@@ -20,34 +20,62 @@ load("//python/private/pypi:index_sources.bzl", "index_sources")  # buildifier:
 _tests = []
 
 def _test_no_simple_api_sources(env):
-    inputs = [
-        "foo==0.0.1",
-        "foo==0.0.1 @ https://someurl.org",
-        "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef",
-        "foo==0.0.1 @ https://someurl.org; python_version < 2.7 --hash=sha256:deadbeef",
-    ]
-    for input in inputs:
+    inputs = {
+        "foo==0.0.1": struct(
+            requirement = "foo==0.0.1",
+            marker = "",
+        ),
+        "foo==0.0.1 @ https://someurl.org": struct(
+            requirement = "foo==0.0.1 @ https://someurl.org",
+            marker = "",
+        ),
+        "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef": struct(
+            requirement = "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef",
+            marker = "",
+        ),
+        "foo==0.0.1 @ https://someurl.org; python_version < \"2.7\"\\    --hash=sha256:deadbeef": struct(
+            requirement = "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef",
+            marker = "python_version < \"2.7\"",
+        ),
+    }
+    for input, want in inputs.items():
         got = index_sources(input)
         env.expect.that_collection(got.shas).contains_exactly([])
         env.expect.that_str(got.version).equals("0.0.1")
+        env.expect.that_str(got.requirement).equals(want.requirement)
+        env.expect.that_str(got.requirement_line).equals(got.requirement)
+        env.expect.that_str(got.marker).equals(want.marker)
 
 _tests.append(_test_no_simple_api_sources)
 
 def _test_simple_api_sources(env):
     tests = {
-        "foo==0.0.2 --hash=sha256:deafbeef    --hash=sha256:deadbeef": [
-            "deadbeef",
-            "deafbeef",
-        ],
-        "foo[extra]==0.0.2; (python_version < 2.7 or something_else == \"@\") --hash=sha256:deafbeef    --hash=sha256:deadbeef": [
-            "deadbeef",
-            "deafbeef",
-        ],
+        "foo==0.0.2 --hash=sha256:deafbeef    --hash=sha256:deadbeef": struct(
+            shas = [
+                "deadbeef",
+                "deafbeef",
+            ],
+            marker = "",
+            requirement = "foo==0.0.2",
+            requirement_line = "foo==0.0.2 --hash=sha256:deafbeef --hash=sha256:deadbeef",
+        ),
+        "foo[extra]==0.0.2; (python_version < 2.7 or extra == \"@\") --hash=sha256:deafbeef    --hash=sha256:deadbeef": struct(
+            shas = [
+                "deadbeef",
+                "deafbeef",
+            ],
+            marker = "(python_version < 2.7 or extra == \"@\")",
+            requirement = "foo[extra]==0.0.2",
+            requirement_line = "foo[extra]==0.0.2 --hash=sha256:deafbeef --hash=sha256:deadbeef",
+        ),
     }
-    for input, want_shas in tests.items():
+    for input, want in tests.items():
         got = index_sources(input)
-        env.expect.that_collection(got.shas).contains_exactly(want_shas)
+        env.expect.that_collection(got.shas).contains_exactly(want.shas)
         env.expect.that_str(got.version).equals("0.0.2")
+        env.expect.that_str(got.requirement).equals(want.requirement)
+        env.expect.that_str(got.requirement_line).equals(want.requirement_line)
+        env.expect.that_str(got.marker).equals(want.marker)
 
 _tests.append(_test_simple_api_sources)