diff --git a/CHANGELOG.md b/CHANGELOG.md index e59d225189..947d5fcd32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,9 @@ Unreleased changes template. binary whose interpreter to use. * (pypi) An extra argument to add the interpreter lib dir to `LDFLAGS` when building wheels from `sdist`. +* (pypi) Direct HTTP urls for wheels and sdists are now supported when using + {obj}`experimental_index_url` (bazel downloader). + Partially fixes [#2363](https://github.com/bazelbuild/rules_python/issues/2363). {#v0-0-0-removed} ### Removed diff --git a/python/private/pypi/index_sources.bzl b/python/private/pypi/index_sources.bzl index 8b3c300946..e3762d2a48 100644 --- a/python/private/pypi/index_sources.bzl +++ b/python/private/pypi/index_sources.bzl @@ -32,6 +32,7 @@ def index_sources(line): * `marker` - str; the marker expression, as per PEP508 spec. * `requirement` - str; a requirement line without the marker. This can be given to `pip` to install a package. + * `url` - str; URL if the requirement specifies a direct URL, empty string otherwise. """ line = line.replace("\\", " ") head, _, maybe_hashes = line.partition(";") @@ -55,9 +56,12 @@ def index_sources(line): requirement, " ".join(["--hash=sha256:{}".format(sha) for sha in shas]), ).strip() + + url = "" if "@" in head: requirement = requirement_line - shas = [] + _, _, url_and_rest = requirement.partition("@") + url = url_and_rest.strip().partition(" ")[0].strip() return struct( requirement = requirement, @@ -65,4 +69,5 @@ def index_sources(line): version = version, shas = sorted(shas), marker = marker, + url = url, ) diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 2bca8d8621..dbff44ecb3 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -292,6 +292,23 @@ def _add_dists(*, requirement, index_urls, logger = None): index_urls: The result of simpleapi_download. logger: A logger for printing diagnostic info. """ + + # Handle direct URLs in requirements + if requirement.srcs.url: + url = requirement.srcs.url + _, _, filename = url.rpartition("/") + direct_url_dist = struct( + url = url, + filename = filename, + sha256 = requirement.srcs.shas[0] if requirement.srcs.shas else "", + yanked = False, + ) + + if filename.endswith(".whl"): + return [direct_url_dist], None + else: + return [], direct_url_dist + if not index_urls: return [], None diff --git a/tests/pypi/index_sources/index_sources_tests.bzl b/tests/pypi/index_sources/index_sources_tests.bzl index 440957e2f0..ffeed87a7b 100644 --- a/tests/pypi/index_sources/index_sources_tests.bzl +++ b/tests/pypi/index_sources/index_sources_tests.bzl @@ -24,27 +24,39 @@ def _test_no_simple_api_sources(env): "foo==0.0.1": struct( requirement = "foo==0.0.1", marker = "", + url = "", ), "foo==0.0.1 @ https://someurl.org": struct( requirement = "foo==0.0.1 @ https://someurl.org", marker = "", + url = "https://someurl.org", ), - "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef": struct( - requirement = "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef", + "foo==0.0.1 @ https://someurl.org/package.whl": struct( + requirement = "foo==0.0.1 @ https://someurl.org/package.whl", marker = "", + url = "https://someurl.org/package.whl", ), - "foo==0.0.1 @ https://someurl.org; python_version < \"2.7\"\\ --hash=sha256:deadbeef": struct( - requirement = "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef", + "foo==0.0.1 @ https://someurl.org/package.whl --hash=sha256:deadbeef": struct( + requirement = "foo==0.0.1 @ https://someurl.org/package.whl --hash=sha256:deadbeef", + marker = "", + url = "https://someurl.org/package.whl", + shas = ["deadbeef"], + ), + "foo==0.0.1 @ https://someurl.org/package.whl; python_version < \"2.7\"\\ --hash=sha256:deadbeef": struct( + requirement = "foo==0.0.1 @ https://someurl.org/package.whl --hash=sha256:deadbeef", marker = "python_version < \"2.7\"", + url = "https://someurl.org/package.whl", + shas = ["deadbeef"], ), } for input, want in inputs.items(): got = index_sources(input) - env.expect.that_collection(got.shas).contains_exactly([]) + env.expect.that_collection(got.shas).contains_exactly(want.shas if hasattr(want, "shas") else []) env.expect.that_str(got.version).equals("0.0.1") env.expect.that_str(got.requirement).equals(want.requirement) env.expect.that_str(got.requirement_line).equals(got.requirement) env.expect.that_str(got.marker).equals(want.marker) + env.expect.that_str(got.url).equals(want.url) _tests.append(_test_no_simple_api_sources) @@ -58,6 +70,7 @@ def _test_simple_api_sources(env): marker = "", requirement = "foo==0.0.2", requirement_line = "foo==0.0.2 --hash=sha256:deafbeef --hash=sha256:deadbeef", + url = "", ), "foo[extra]==0.0.2; (python_version < 2.7 or extra == \"@\") --hash=sha256:deafbeef --hash=sha256:deadbeef": struct( shas = [ @@ -67,6 +80,7 @@ def _test_simple_api_sources(env): marker = "(python_version < 2.7 or extra == \"@\")", requirement = "foo[extra]==0.0.2", requirement_line = "foo[extra]==0.0.2 --hash=sha256:deafbeef --hash=sha256:deadbeef", + url = "", ), } for input, want in tests.items(): @@ -76,6 +90,7 @@ def _test_simple_api_sources(env): env.expect.that_str(got.requirement).equals(want.requirement) env.expect.that_str(got.requirement_line).equals(want.requirement_line) env.expect.that_str(got.marker).equals(want.marker) + env.expect.that_str(got.url).equals(want.url) _tests.append(_test_simple_api_sources) diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 77e22b825a..8edc2689bf 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -26,7 +26,10 @@ foo==0.0.1 \ --hash=sha256:deadb00f """, "requirements_direct": """\ -foo[extra] @ https://some-url +foo[extra] @ https://some-url/package.whl +bar @ https://example.org/bar-1.0.whl --hash=sha256:deadbeef +baz @ https://test.com/baz-2.0.whl; python_version < "3.8" --hash=sha256:deadb00f +qux @ https://example.org/qux-1.0.tar.gz --hash=sha256:deadbe0f """, "requirements_extra_args": """\ --index-url=example.org @@ -106,6 +109,7 @@ def _test_simple(env): requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.1", + url = "", ), target_platforms = [ "linux_x86_64", @@ -124,6 +128,110 @@ def _test_simple(env): _tests.append(_test_simple) +def _test_direct_urls(env): + got = parse_requirements( + ctx = _mock_ctx(), + requirements_by_platform = { + "requirements_direct": ["linux_x86_64"], + }, + ) + env.expect.that_dict(got).contains_exactly({ + "bar": [ + struct( + distribution = "bar", + extra_pip_args = [], + sdist = None, + is_exposed = True, + srcs = struct( + marker = "", + requirement = "bar @ https://example.org/bar-1.0.whl --hash=sha256:deadbeef", + requirement_line = "bar @ https://example.org/bar-1.0.whl --hash=sha256:deadbeef", + shas = ["deadbeef"], + version = "", + url = "https://example.org/bar-1.0.whl", + ), + target_platforms = ["linux_x86_64"], + whls = [struct( + url = "https://example.org/bar-1.0.whl", + filename = "bar-1.0.whl", + sha256 = "deadbeef", + yanked = False, + )], + ), + ], + "baz": [ + struct( + distribution = "baz", + extra_pip_args = [], + sdist = None, + is_exposed = True, + srcs = struct( + marker = "python_version < \"3.8\"", + requirement = "baz @ https://test.com/baz-2.0.whl --hash=sha256:deadb00f", + requirement_line = "baz @ https://test.com/baz-2.0.whl --hash=sha256:deadb00f", + shas = ["deadb00f"], + version = "", + url = "https://test.com/baz-2.0.whl", + ), + target_platforms = ["linux_x86_64"], + whls = [struct( + url = "https://test.com/baz-2.0.whl", + filename = "baz-2.0.whl", + sha256 = "deadb00f", + yanked = False, + )], + ), + ], + "foo": [ + struct( + distribution = "foo", + extra_pip_args = [], + sdist = None, + is_exposed = True, + srcs = struct( + marker = "", + requirement = "foo[extra] @ https://some-url/package.whl", + requirement_line = "foo[extra] @ https://some-url/package.whl", + shas = [], + version = "", + url = "https://some-url/package.whl", + ), + target_platforms = ["linux_x86_64"], + whls = [struct( + url = "https://some-url/package.whl", + filename = "package.whl", + sha256 = "", + yanked = False, + )], + ), + ], + "qux": [ + struct( + distribution = "qux", + extra_pip_args = [], + sdist = struct( + url = "https://example.org/qux-1.0.tar.gz", + filename = "qux-1.0.tar.gz", + sha256 = "deadbe0f", + yanked = False, + ), + is_exposed = True, + srcs = struct( + marker = "", + requirement = "qux @ https://example.org/qux-1.0.tar.gz --hash=sha256:deadbe0f", + requirement_line = "qux @ https://example.org/qux-1.0.tar.gz --hash=sha256:deadbe0f", + shas = ["deadbe0f"], + version = "", + url = "https://example.org/qux-1.0.tar.gz", + ), + target_platforms = ["linux_x86_64"], + whls = [], + ), + ], + }) + +_tests.append(_test_direct_urls) + def _test_extra_pip_args(env): got = parse_requirements( ctx = _mock_ctx(), @@ -145,6 +253,7 @@ def _test_extra_pip_args(env): requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.1", + url = "", ), target_platforms = [ "linux_x86_64", @@ -182,6 +291,7 @@ def _test_dupe_requirements(env): requirement_line = "foo[extra,extra_2]==0.0.1 --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.1", + url = "", ), target_platforms = ["linux_x86_64"], whls = [], @@ -211,6 +321,7 @@ def _test_multi_os(env): requirement_line = "bar==0.0.1 --hash=sha256:deadb00f", shas = ["deadb00f"], version = "0.0.1", + url = "", ), target_platforms = ["windows_x86_64"], whls = [], @@ -228,6 +339,7 @@ def _test_multi_os(env): requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf", shas = ["deadbaaf"], version = "0.0.3", + url = "", ), target_platforms = ["linux_x86_64"], whls = [], @@ -243,6 +355,7 @@ def _test_multi_os(env): requirement_line = "foo[extra]==0.0.2 --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.2", + url = "", ), target_platforms = ["windows_x86_64"], whls = [], @@ -282,6 +395,7 @@ def _test_multi_os_legacy(env): requirement_line = "bar==0.0.1 --hash=sha256:deadb00f", shas = ["deadb00f"], version = "0.0.1", + url = "", ), target_platforms = ["cp39_linux_x86_64"], whls = [], @@ -299,6 +413,7 @@ def _test_multi_os_legacy(env): requirement_line = "foo==0.0.1 --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.1", + url = "", ), target_platforms = ["cp39_linux_x86_64"], whls = [], @@ -314,6 +429,7 @@ def _test_multi_os_legacy(env): requirement = "foo==0.0.3", shas = ["deadbaaf"], version = "0.0.3", + url = "", ), target_platforms = ["cp39_osx_aarch64"], whls = [], @@ -367,6 +483,7 @@ def _test_env_marker_resolution(env): requirement_line = "bar==0.0.1 --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.1", + url = "", ), target_platforms = ["cp311_linux_super_exotic", "cp311_windows_x86_64"], whls = [], @@ -384,6 +501,7 @@ def _test_env_marker_resolution(env): requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.1", + url = "", ), target_platforms = ["cp311_windows_x86_64"], whls = [], @@ -419,6 +537,7 @@ def _test_different_package_version(env): requirement_line = "foo==0.0.1 --hash=sha256:deadb00f", shas = ["deadb00f"], version = "0.0.1", + url = "", ), target_platforms = ["linux_x86_64"], whls = [], @@ -434,6 +553,7 @@ def _test_different_package_version(env): requirement_line = "foo==0.0.1+local --hash=sha256:deadbeef", shas = ["deadbeef"], version = "0.0.1+local", + url = "", ), target_platforms = ["linux_x86_64"], whls = [],