feat(pypi): support direct urls for wheels in bazel downloader

CHANGELOG.md

@@ -79,6 +79,9 @@ Unreleased changes template.
   binary whose interpreter to use.
 * (pypi) An extra argument to add the interpreter lib dir to `LDFLAGS` when
   building wheels from `sdist`.
+* (pypi) Direct HTTP urls for wheels and sdists are now supported when using
+  {obj}`experimental_index_url` (bazel downloader).
+  Partially fixes [#2363](https://github.com/bazelbuild/rules_python/issues/2363).
 
 {#v0-0-0-removed}
 ### Removed

python/private/pypi/index_sources.bzl

@@ -32,6 +32,7 @@ def index_sources(line):
             * `marker` - str; the marker expression, as per PEP508 spec.
             * `requirement` - str; a requirement line without the marker. This can
                 be given to `pip` to install a package.
+            * `url` - str; URL if the requirement specifies a direct URL, empty string otherwise.
     """
     line = line.replace("\\", " ")
     head, _, maybe_hashes = line.partition(";")
@@ -55,14 +56,18 @@ def index_sources(line):
         requirement,
         " ".join(["--hash=sha256:{}".format(sha) for sha in shas]),
     ).strip()
+
+    url = ""
     if "@" in head:
         requirement = requirement_line
-        shas = []
+        _, _, url_and_rest = requirement.partition("@")
+        url = url_and_rest.strip().partition(" ")[0].strip()
 
     return struct(
         requirement = requirement,
         requirement_line = requirement_line,
         version = version,
         shas = sorted(shas),
         marker = marker,
+        url = url,
     )

python/private/pypi/parse_requirements.bzl

@@ -292,6 +292,23 @@ def _add_dists(*, requirement, index_urls, logger = None):
         index_urls: The result of simpleapi_download.
         logger: A logger for printing diagnostic info.
     """
+
+    # Handle direct URLs in requirements
+    if requirement.srcs.url:
+        url = requirement.srcs.url
+        _, _, filename = url.rpartition("/")
+        direct_url_dist = struct(
+            url = url,
+            filename = filename,
+            sha256 = requirement.srcs.shas[0] if requirement.srcs.shas else "",
+            yanked = False,
+        )
+
+        if filename.endswith(".whl"):
+            return [direct_url_dist], None
+        else:
+            return [], direct_url_dist
+
     if not index_urls:
         return [], None
 

tests/pypi/index_sources/index_sources_tests.bzl

@@ -24,27 +24,39 @@ def _test_no_simple_api_sources(env):
         "foo==0.0.1": struct(
             requirement = "foo==0.0.1",
             marker = "",
+            url = "",
         ),
         "foo==0.0.1 @ https://someurl.org": struct(
             requirement = "foo==0.0.1 @ https://someurl.org",
             marker = "",
+            url = "https://someurl.org",
         ),
-        "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef": struct(
-            requirement = "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef",
+        "foo==0.0.1 @ https://someurl.org/package.whl": struct(
+            requirement = "foo==0.0.1 @ https://someurl.org/package.whl",
             marker = "",
+            url = "https://someurl.org/package.whl",
         ),
-        "foo==0.0.1 @ https://someurl.org; python_version < \"2.7\"\\    --hash=sha256:deadbeef": struct(
-            requirement = "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef",
+        "foo==0.0.1 @ https://someurl.org/package.whl --hash=sha256:deadbeef": struct(
+            requirement = "foo==0.0.1 @ https://someurl.org/package.whl --hash=sha256:deadbeef",
+            marker = "",
+            url = "https://someurl.org/package.whl",
+            shas = ["deadbeef"],
+        ),
+        "foo==0.0.1 @ https://someurl.org/package.whl; python_version < \"2.7\"\\    --hash=sha256:deadbeef": struct(
+            requirement = "foo==0.0.1 @ https://someurl.org/package.whl --hash=sha256:deadbeef",
             marker = "python_version < \"2.7\"",
+            url = "https://someurl.org/package.whl",
+            shas = ["deadbeef"],
         ),
     }
     for input, want in inputs.items():
         got = index_sources(input)
-        env.expect.that_collection(got.shas).contains_exactly([])
+        env.expect.that_collection(got.shas).contains_exactly(want.shas if hasattr(want, "shas") else [])
         env.expect.that_str(got.version).equals("0.0.1")
         env.expect.that_str(got.requirement).equals(want.requirement)
         env.expect.that_str(got.requirement_line).equals(got.requirement)
         env.expect.that_str(got.marker).equals(want.marker)
+        env.expect.that_str(got.url).equals(want.url)
 
 _tests.append(_test_no_simple_api_sources)
 
@@ -58,6 +70,7 @@ def _test_simple_api_sources(env):
             marker = "",
             requirement = "foo==0.0.2",
             requirement_line = "foo==0.0.2 --hash=sha256:deafbeef --hash=sha256:deadbeef",
+            url = "",
         ),
         "foo[extra]==0.0.2; (python_version < 2.7 or extra == \"@\") --hash=sha256:deafbeef    --hash=sha256:deadbeef": struct(
             shas = [
@@ -67,6 +80,7 @@ def _test_simple_api_sources(env):
             marker = "(python_version < 2.7 or extra == \"@\")",
             requirement = "foo[extra]==0.0.2",
             requirement_line = "foo[extra]==0.0.2 --hash=sha256:deafbeef --hash=sha256:deadbeef",
+            url = "",
         ),
     }
     for input, want in tests.items():
@@ -76,6 +90,7 @@ def _test_simple_api_sources(env):
         env.expect.that_str(got.requirement).equals(want.requirement)
         env.expect.that_str(got.requirement_line).equals(want.requirement_line)
         env.expect.that_str(got.marker).equals(want.marker)
+        env.expect.that_str(got.url).equals(want.url)
 
 _tests.append(_test_simple_api_sources)
 

tests/pypi/parse_requirements/parse_requirements_tests.bzl

@@ -26,7 +26,10 @@ foo==0.0.1 \
     --hash=sha256:deadb00f
 """,
         "requirements_direct": """\
-foo[extra] @ https://some-url
+foo[extra] @ https://some-url/package.whl
+bar @ https://example.org/bar-1.0.whl --hash=sha256:deadbeef
+baz @ https://test.com/baz-2.0.whl; python_version < "3.8" --hash=sha256:deadb00f
+qux @ https://example.org/qux-1.0.tar.gz --hash=sha256:deadbe0f
 """,
         "requirements_extra_args": """\
 --index-url=example.org
@@ -106,6 +109,7 @@ def _test_simple(env):
                     requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef",
                     shas = ["deadbeef"],
                     version = "0.0.1",
+                    url = "",
                 ),
                 target_platforms = [
                     "linux_x86_64",
@@ -124,6 +128,110 @@ def _test_simple(env):
 
 _tests.append(_test_simple)
 
+def _test_direct_urls(env):
+    got = parse_requirements(
+        ctx = _mock_ctx(),
+        requirements_by_platform = {
+            "requirements_direct": ["linux_x86_64"],
+        },
+    )
+    env.expect.that_dict(got).contains_exactly({
+        "bar": [
+            struct(
+                distribution = "bar",
+                extra_pip_args = [],
+                sdist = None,
+                is_exposed = True,
+                srcs = struct(
+                    marker = "",
+                    requirement = "bar @ https://example.org/bar-1.0.whl --hash=sha256:deadbeef",
+                    requirement_line = "bar @ https://example.org/bar-1.0.whl --hash=sha256:deadbeef",
+                    shas = ["deadbeef"],
+                    version = "",
+                    url = "https://example.org/bar-1.0.whl",
+                ),
+                target_platforms = ["linux_x86_64"],
+                whls = [struct(
+                    url = "https://example.org/bar-1.0.whl",
+                    filename = "bar-1.0.whl",
+                    sha256 = "deadbeef",
+                    yanked = False,
+                )],
+            ),
+        ],
+        "baz": [
+            struct(
+                distribution = "baz",
+                extra_pip_args = [],
+                sdist = None,
+                is_exposed = True,
+                srcs = struct(
+                    marker = "python_version < \"3.8\"",
+                    requirement = "baz @ https://test.com/baz-2.0.whl --hash=sha256:deadb00f",
+                    requirement_line = "baz @ https://test.com/baz-2.0.whl --hash=sha256:deadb00f",
+                    shas = ["deadb00f"],
+                    version = "",
+                    url = "https://test.com/baz-2.0.whl",
+                ),
+                target_platforms = ["linux_x86_64"],
+                whls = [struct(
+                    url = "https://test.com/baz-2.0.whl",
+                    filename = "baz-2.0.whl",
+                    sha256 = "deadb00f",
+                    yanked = False,
+                )],
+            ),
+        ],
+        "foo": [
+            struct(
+                distribution = "foo",
+                extra_pip_args = [],
+                sdist = None,
+                is_exposed = True,
+                srcs = struct(
+                    marker = "",
+                    requirement = "foo[extra] @ https://some-url/package.whl",
+                    requirement_line = "foo[extra] @ https://some-url/package.whl",
+                    shas = [],
+                    version = "",
+                    url = "https://some-url/package.whl",
+                ),
+                target_platforms = ["linux_x86_64"],
+                whls = [struct(
+                    url = "https://some-url/package.whl",
+                    filename = "package.whl",
+                    sha256 = "",
+                    yanked = False,
+                )],
+            ),
+        ],
+        "qux": [
+            struct(
+                distribution = "qux",
+                extra_pip_args = [],
+                sdist = struct(
+                    url = "https://example.org/qux-1.0.tar.gz",
+                    filename = "qux-1.0.tar.gz",
+                    sha256 = "deadbe0f",
+                    yanked = False,
+                ),
+                is_exposed = True,
+                srcs = struct(
+                    marker = "",
+                    requirement = "qux @ https://example.org/qux-1.0.tar.gz --hash=sha256:deadbe0f",
+                    requirement_line = "qux @ https://example.org/qux-1.0.tar.gz --hash=sha256:deadbe0f",
+                    shas = ["deadbe0f"],
+                    version = "",
+                    url = "https://example.org/qux-1.0.tar.gz",
+                ),
+                target_platforms = ["linux_x86_64"],
+                whls = [],
+            ),
+        ],
+    })
+
+_tests.append(_test_direct_urls)
+
 def _test_extra_pip_args(env):
     got = parse_requirements(
         ctx = _mock_ctx(),
@@ -145,6 +253,7 @@ def _test_extra_pip_args(env):
                     requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef",
                     shas = ["deadbeef"],
                     version = "0.0.1",
+                    url = "",
                 ),
                 target_platforms = [
                     "linux_x86_64",
@@ -182,6 +291,7 @@ def _test_dupe_requirements(env):
                     requirement_line = "foo[extra,extra_2]==0.0.1 --hash=sha256:deadbeef",
                     shas = ["deadbeef"],
                     version = "0.0.1",
+                    url = "",
                 ),
                 target_platforms = ["linux_x86_64"],
                 whls = [],
@@ -211,6 +321,7 @@ def _test_multi_os(env):
                     requirement_line = "bar==0.0.1 --hash=sha256:deadb00f",
                     shas = ["deadb00f"],
                     version = "0.0.1",
+                    url = "",
                 ),
                 target_platforms = ["windows_x86_64"],
                 whls = [],
@@ -228,6 +339,7 @@ def _test_multi_os(env):
                     requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf",
                     shas = ["deadbaaf"],
                     version = "0.0.3",
+                    url = "",
                 ),
                 target_platforms = ["linux_x86_64"],
                 whls = [],
@@ -243,6 +355,7 @@ def _test_multi_os(env):
                     requirement_line = "foo[extra]==0.0.2 --hash=sha256:deadbeef",
                     shas = ["deadbeef"],
                     version = "0.0.2",
+                    url = "",
                 ),
                 target_platforms = ["windows_x86_64"],
                 whls = [],
@@ -282,6 +395,7 @@ def _test_multi_os_legacy(env):
                     requirement_line = "bar==0.0.1 --hash=sha256:deadb00f",
                     shas = ["deadb00f"],
                     version = "0.0.1",
+                    url = "",
                 ),
                 target_platforms = ["cp39_linux_x86_64"],
                 whls = [],
@@ -299,6 +413,7 @@ def _test_multi_os_legacy(env):
                     requirement_line = "foo==0.0.1 --hash=sha256:deadbeef",
                     shas = ["deadbeef"],
                     version = "0.0.1",
+                    url = "",
                 ),
                 target_platforms = ["cp39_linux_x86_64"],
                 whls = [],
@@ -314,6 +429,7 @@ def _test_multi_os_legacy(env):
                     requirement = "foo==0.0.3",
                     shas = ["deadbaaf"],
                     version = "0.0.3",
+                    url = "",
                 ),
                 target_platforms = ["cp39_osx_aarch64"],
                 whls = [],
@@ -367,6 +483,7 @@ def _test_env_marker_resolution(env):
                     requirement_line = "bar==0.0.1 --hash=sha256:deadbeef",
                     shas = ["deadbeef"],
                     version = "0.0.1",
+                    url = "",
                 ),
                 target_platforms = ["cp311_linux_super_exotic", "cp311_windows_x86_64"],
                 whls = [],
@@ -384,6 +501,7 @@ def _test_env_marker_resolution(env):
                     requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef",
                     shas = ["deadbeef"],
                     version = "0.0.1",
+                    url = "",
                 ),
                 target_platforms = ["cp311_windows_x86_64"],
                 whls = [],
@@ -419,6 +537,7 @@ def _test_different_package_version(env):
                     requirement_line = "foo==0.0.1 --hash=sha256:deadb00f",
                     shas = ["deadb00f"],
                     version = "0.0.1",
+                    url = "",
                 ),
                 target_platforms = ["linux_x86_64"],
                 whls = [],
@@ -434,6 +553,7 @@ def _test_different_package_version(env):
                     requirement_line = "foo==0.0.1+local --hash=sha256:deadbeef",
                     shas = ["deadbeef"],
                     version = "0.0.1+local",
+                    url = "",
                 ),
                 target_platforms = ["linux_x86_64"],
                 whls = [],