From c79f566430d67638b9eee092d2564f905d218b7e Mon Sep 17 00:00:00 2001 From: aignas <240938+aignas@users.noreply.github.com> Date: Fri, 11 Oct 2024 12:03:38 +0900 Subject: [PATCH] feat(bzlmod): enable download_only for experimental_index_url With this change we can support ibazel for building our docs again because we will just not have any sdists that are causing issues. This limits the scope to only supporting whls at this time, but for the time being it is the best solution. Fixes #2223 Work towards #260 --- .bazelrc | 4 ++-- CHANGELOG.md | 2 ++ MODULE.bazel | 1 + examples/bzlmod/MODULE.bazel.lock | 4 ++-- python/private/pypi/extension.bzl | 16 +++++++++------- tests/pypi/integration/BUILD.bazel | 2 +- 6 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.bazelrc b/.bazelrc index 1ca469cd75..b484751c3c 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered test --test_output=errors diff --git a/CHANGELOG.md b/CHANGELOG.md index 67c3477443..44333a5eb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,8 @@ A brief description of the categories of changes: * `3.12 -> 3.12.7` [20241008]: https://github.com/indygreg/python-build-standalone/releases/tag/20241008 * (coverage) Add support for python 3.13 and bump `coverage.py` to 7.6.1. +* (bzlmod) Add support for `download_only` flag to disable usage of `sdists` + when {bzl:attr}`pip.parse.experimental_index_url` is set. ### Removed diff --git a/MODULE.bazel b/MODULE.bazel index 58c7ae229b..780d7f8666 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -91,6 +91,7 @@ dev_pip = use_extension( dev_dependency = True, ) dev_pip.parse( + download_only = True, # this will not add the `sdist` values to the transitive closures at all. hub_name = "dev_pip", python_version = "3.11", requirements_lock = "//docs:requirements.txt", diff --git a/examples/bzlmod/MODULE.bazel.lock b/examples/bzlmod/MODULE.bazel.lock index 2ddbe40bc3..8ef6e97588 100644 --- a/examples/bzlmod/MODULE.bazel.lock +++ b/examples/bzlmod/MODULE.bazel.lock @@ -1231,7 +1231,7 @@ }, "@@rules_python~//python/extensions:pip.bzl%pip": { "general": { - "bzlTransitiveDigest": "kPvx0u6SR68H+8BqqEfxd3NpVZjuMOyzI9Xml01rdrQ=", + "bzlTransitiveDigest": "Rc0r+OwZ3f57lwygvIXLejgTsJKr8ezIcd136SpMyDo=", "usagesDigest": "MChlcSw99EuW3K7OOoMcXQIdcJnEh6YmfyjJm+9mxIg=", "recordedFileInputs": { "@@other_module~//requirements_lock_3_11.txt": "a7d0061366569043d5efcf80e34a32c732679367cb3c831c4cdc606adc36d314", @@ -6138,7 +6138,7 @@ }, "@@rules_python~//python/private/pypi:pip.bzl%pip_internal": { "general": { - "bzlTransitiveDigest": "c3OA6iewVGq8nz0o3iI2AtIQhsRZIg/E/PDq2vuAQTw=", + "bzlTransitiveDigest": "s4J1aKQE20A+qzGkeB4XwzitVphP4H3vYmeLg6pMZxA=", "usagesDigest": "Y8ihY+R57BAFhalrVLVdJFrpwlbsiKz9JPJ99ljF7HA=", "recordedFileInputs": { "@@rules_python~//tools/publish/requirements.txt": "031e35d03dde03ae6305fe4b3d1f58ad7bdad857379752deede0f93649991b8a", diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 200aa4327e..36fb20e030 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -151,9 +151,6 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s get_index_urls = None if pip_attr.experimental_index_url: - if pip_attr.download_only: - fail("Currently unsupported to use `download_only` and `experimental_index_url`") - get_index_urls = lambda ctx, distributions: simpleapi_download( ctx, attr = struct( @@ -263,11 +260,11 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s is_exposed = False for requirement in requirements: is_exposed = is_exposed or requirement.is_exposed - for distribution in requirement.whls + [requirement.sdist]: - if not distribution: - # sdist may be None - continue + dists = requirement.whls + if not pip_attr.download_only and requirement.sdist: + dists = dists + [requirement.sdist] + for distribution in dists: found_something = True is_hub_reproducible = False @@ -564,6 +561,11 @@ In the future this could be defaulted to `https://pypi.org` when this feature be stable. This is equivalent to `--index-url` `pip` option. + +:::{versionchanged} 0.37.0 +If {attr}`download_only` is set, then `sdist` archives will be discarded and `pip.parse` will +operate in wheel-only mode. +::: """, ), "experimental_index_url_overrides": attr.string_dict( diff --git a/tests/pypi/integration/BUILD.bazel b/tests/pypi/integration/BUILD.bazel index f846bfb124..9ea8dcebe4 100644 --- a/tests/pypi/integration/BUILD.bazel +++ b/tests/pypi/integration/BUILD.bazel @@ -1,5 +1,5 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") -load("@dev_pip//:requirements.bzl", "all_requirements") +load("@rules_python_publish_deps//:requirements.bzl", "all_requirements") load(":transitions.bzl", "transition_rule") build_test(