From 1c768ca541a0b66d249f5e7db41a36049b2d91b4 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 31 Aug 2025 00:06:40 +0900 Subject: [PATCH 01/28] feat(pypi): enable pipstar by default on bzlmod Extra fixes: - add more logging to the failure - better METADATA files in tests - fix tests --- CHANGELOG.md | 2 ++ python/private/internal_config_repo.bzl | 3 ++- python/private/internal_dev_deps.bzl | 3 +++ python/private/pypi/whl_metadata.bzl | 6 +++++- .../testdata/ns-sub1/ns-sub1-1.0.dist-info/METADATA | 2 ++ .../testdata/ns-sub2/ns_sub2-1.0.dist-info/METADATA | 2 ++ tests/support/support.bzl | 3 +++ .../testdata/somepkg-1.0.dist-info/METADATA | 2 ++ 8 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55d0d3fa2f..652398a4fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,8 @@ END_UNRELEASED_TEMPLATE * (bootstrap) For {obj}`--bootstrap_impl=system_python`, the sys.path order has changed from `[app paths, stdlib, runtime site-packages]` to `[stdlib, app paths, runtime site-packages]`. +* (pypi) `pipstar` flag has been flipped to be enabled by default for bzlmod + ([#2949](https://github.com/bazel-contrib/rules_python/issues/2949)). {#v0-0-0-fixed} ### Fixed diff --git a/python/private/internal_config_repo.bzl b/python/private/internal_config_repo.bzl index cfe2fdfd77..aa9fdf661f 100644 --- a/python/private/internal_config_repo.bzl +++ b/python/private/internal_config_repo.bzl @@ -20,8 +20,9 @@ settings for rules to later use. load(":repo_utils.bzl", "repo_utils") +_BZLMOD_ENABLED = "@@" in str(Label("//:unused")) _ENABLE_PIPSTAR_ENVVAR_NAME = "RULES_PYTHON_ENABLE_PIPSTAR" -_ENABLE_PIPSTAR_DEFAULT = "0" +_ENABLE_PIPSTAR_DEFAULT = "1" if _BZLMOD_ENABLED else "0" _ENABLE_PYSTAR_ENVVAR_NAME = "RULES_PYTHON_ENABLE_PYSTAR" _ENABLE_PYSTAR_DEFAULT = "1" _ENABLE_DEPRECATION_WARNINGS_ENVVAR_NAME = "RULES_PYTHON_DEPRECATION_WARNINGS" diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index d621a5d941..551dea6478 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -40,6 +40,7 @@ def _internal_dev_deps_impl(mctx): name = "somepkg_with_build_files", whl_file = "@whl_with_build_files//:somepkg-1.0-any-none-any.whl", requirement = "somepkg", + dep_template = "@rules_python//tests/support:support.bzl", ) # Setup for //tests/implicit_namespace_packages @@ -53,6 +54,7 @@ def _internal_dev_deps_impl(mctx): whl_file = "@implicit_namespace_ns_sub1_whl//:ns_sub1-1.0-any-none-any.whl", requirement = "ns-sub1", enable_implicit_namespace_pkgs = False, + dep_template = "@rules_python//tests/support:support.bzl", ) whl_from_dir_repo( @@ -65,6 +67,7 @@ def _internal_dev_deps_impl(mctx): whl_file = "@implicit_namespace_ns_sub2_whl//:ns_sub2-1.0-any-none-any.whl", requirement = "ns-sub2", enable_implicit_namespace_pkgs = False, + dep_template = "@rules_python//tests/support:support.bzl", ) internal_dev_deps = module_extension( diff --git a/python/private/pypi/whl_metadata.bzl b/python/private/pypi/whl_metadata.bzl index cf2d51afda..b0202ee3f4 100644 --- a/python/private/pypi/whl_metadata.bzl +++ b/python/private/pypi/whl_metadata.bzl @@ -26,7 +26,11 @@ def whl_metadata(*, install_dir, read_fn, logger): result = parse_whl_metadata(contents) if not (result.name and result.version): - logger.fail("Failed to parsed the wheel METADATA file:\n{}".format(contents)) + logger.fail("Failed to parsed the wheel METADATA file:\n{}\n{}\n{}".format( + 80 * "=", + contents.rstrip("\n"), + 80 * "=", + )) return None return result diff --git a/tests/implicit_namespace_packages/testdata/ns-sub1/ns-sub1-1.0.dist-info/METADATA b/tests/implicit_namespace_packages/testdata/ns-sub1/ns-sub1-1.0.dist-info/METADATA index e69de29bb2..ecec6086ba 100644 --- a/tests/implicit_namespace_packages/testdata/ns-sub1/ns-sub1-1.0.dist-info/METADATA +++ b/tests/implicit_namespace_packages/testdata/ns-sub1/ns-sub1-1.0.dist-info/METADATA @@ -0,0 +1,2 @@ +Name: ns-sub1 +Version: 1.0 diff --git a/tests/implicit_namespace_packages/testdata/ns-sub2/ns_sub2-1.0.dist-info/METADATA b/tests/implicit_namespace_packages/testdata/ns-sub2/ns_sub2-1.0.dist-info/METADATA index e69de29bb2..92cbb8ec2e 100644 --- a/tests/implicit_namespace_packages/testdata/ns-sub2/ns_sub2-1.0.dist-info/METADATA +++ b/tests/implicit_namespace_packages/testdata/ns-sub2/ns_sub2-1.0.dist-info/METADATA @@ -0,0 +1,2 @@ +Name: ns-sub2 +Version: 1.0 diff --git a/tests/support/support.bzl b/tests/support/support.bzl index f8694629c1..ab2b7a8083 100644 --- a/tests/support/support.bzl +++ b/tests/support/support.bzl @@ -22,6 +22,9 @@ load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER") # buildifier: disable=bzl-visibility +# this is used in integration tests with `whl_library` +whl_map = [] + MAC = Label("//tests/support:mac") MAC_X86_64 = Label("//tests/support:mac_x86_64") LINUX = Label("//tests/support:linux") diff --git a/tests/whl_with_build_files/testdata/somepkg-1.0.dist-info/METADATA b/tests/whl_with_build_files/testdata/somepkg-1.0.dist-info/METADATA index e69de29bb2..0829d1f756 100644 --- a/tests/whl_with_build_files/testdata/somepkg-1.0.dist-info/METADATA +++ b/tests/whl_with_build_files/testdata/somepkg-1.0.dist-info/METADATA @@ -0,0 +1,2 @@ +Name: somepkg +Version: 1.0 From 10692cebd520c7b2d4d01347790726c9e69f8360 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 7 Sep 2025 22:40:54 +0900 Subject: [PATCH 02/28] chore: cleanup bzlmod codebase after pipstar enabling --- python/private/pypi/extension.bzl | 8 --- python/private/pypi/hub_builder.bzl | 67 ++--------------- tests/pypi/hub_builder/hub_builder_tests.bzl | 76 ++------------------ 3 files changed, 10 insertions(+), 141 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 4708c8e53a..dbc6c15603 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -21,7 +21,6 @@ load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config") load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:repo_utils.bzl", "repo_utils") -load(":evaluate_markers.bzl", EVALUATE_MARKERS_SRCS = "SRCS") load(":hub_builder.bzl", "hub_builder") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_whl_name.bzl", "parse_whl_name") @@ -672,13 +671,6 @@ EXPERIMENTAL: this may be removed without notice. doc = """\ A dict of labels to wheel names that is typically generated by the whl_modifications. The labels are JSON config files describing the modifications. -""", - ), - "_evaluate_markers_srcs": attr.label_list( - default = EVALUATE_MARKERS_SRCS, - doc = """\ -The list of labels to use as SRCS for the marker evaluation code. This ensures that the -code will be re-evaluated when any of files in the default changes. """, ), }, **ATTRS) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index b6088e4ded..6e5ca72cb9 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -5,7 +5,7 @@ load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:version.bzl", "version") load("//python/private:version_label.bzl", "version_label") load(":attrs.bzl", "use_isolated") -load(":evaluate_markers.bzl", "evaluate_markers_py", evaluate_markers_star = "evaluate_markers") +load(":evaluate_markers.bzl", "evaluate_markers") load(":parse_requirements.bzl", "parse_requirements") load(":pep508_env.bzl", "env") load(":pep508_evaluate.bzl", "evaluate") @@ -175,8 +175,6 @@ def _add_whl_library(self, *, python_version, whl, repo): # disallow building from sdist. return - platforms = self._platforms[python_version] - # TODO @aignas 2025-06-29: we should not need the version in the repo_name if # we are using pipstar and we are downloading the wheel using the downloader repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) @@ -188,17 +186,6 @@ def _add_whl_library(self, *, python_version, whl, repo): )) self._whl_libraries[repo_name] = repo.args - if not self._config.enable_pipstar and "experimental_target_platforms" in repo.args: - self._whl_libraries[repo_name] |= { - "experimental_target_platforms": sorted({ - # TODO @aignas 2025-07-07: this should be solved in a better way - platforms[candidate].triple.partition("_")[-1]: None - for p in repo.args["experimental_target_platforms"] - for candidate in platforms - if candidate.endswith(p) - }), - } - mapping = self._whl_map.setdefault(whl.name, {}) if repo.config_setting in mapping and mapping[repo.config_setting] != repo_name: fail( @@ -329,41 +316,9 @@ def _evaluate_markers(self, pip_attr): if self._evaluate_markers_fn: return self._evaluate_markers_fn - if self._config.enable_pipstar: - return lambda _, requirements: evaluate_markers_star( - requirements = requirements, - platforms = self._platforms[pip_attr.python_version], - ) - - interpreter = _detect_interpreter(self, pip_attr) - - # NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either - # in the PATH or if specified as a label. We will configure the env - # markers when evaluating the requirement lines based on the output - # from the `requirements_files_by_platform` which should have something - # similar to: - # { - # "//:requirements.txt": ["cp311_linux_x86_64", ...] - # } - # - # We know the target python versions that we need to evaluate the - # markers for and thus we don't need to use multiple python interpreter - # instances to perform this manipulation. This function should be executed - # only once by the underlying code to minimize the overhead needed to - # spin up a Python interpreter. - return lambda module_ctx, requirements: evaluate_markers_py( - module_ctx, - requirements = { - k: { - p: self._platforms[pip_attr.python_version][p].triple - for p in plats - } - for k, plats in requirements.items() - }, - python_interpreter = interpreter.path, - python_interpreter_target = interpreter.target, - srcs = pip_attr._evaluate_markers_srcs, - logger = self._logger, + return lambda _, requirements: evaluate_markers( + requirements = requirements, + platforms = self._platforms[pip_attr.python_version], ) def _create_whl_repos( @@ -435,7 +390,6 @@ def _create_whl_repos( auth_patterns = self._config.auth_patterns or pip_attr.auth_patterns, python_version = _major_minor_version(pip_attr.python_version), is_multiple_versions = whl.is_multiple_versions, - enable_pipstar = self._config.enable_pipstar, ) _add_whl_library( self, @@ -463,8 +417,6 @@ def _common_args(self, module_ctx, *, pip_attr): python_interpreter = interpreter.path, python_interpreter_target = interpreter.target, ) - if not self._config.enable_pipstar: - maybe_args["experimental_target_platforms"] = pip_attr.experimental_target_platforms whl_library_args.update({k: v for k, v in maybe_args.items() if v}) maybe_args_with_default = dict( @@ -512,8 +464,7 @@ def _whl_repo( netrc, auth_patterns, python_version, - use_downloader, - enable_pipstar = False): + use_downloader): args = dict(whl_library_args) args["requirement"] = src.requirement_line is_whl = src.filename.endswith(".whl") @@ -556,14 +507,6 @@ def _whl_repo( args["urls"] = [src.url] args["sha256"] = src.sha256 args["filename"] = src.filename - if not enable_pipstar: - args["experimental_target_platforms"] = [ - # Get rid of the version for the target platforms because we are - # passing the interpreter any way. Ideally we should search of ways - # how to pass the target platforms through the hub repo. - p.partition("_")[2] - for p in src.target_platforms - ] return struct( repo_name = whl_repo_name(src.filename, src.sha256), diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 9f6ee6720d..31f9e93511 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -40,11 +40,9 @@ simple==0.0.1 \ def hub_builder( env, - enable_pipstar = False, debug = False, config = None, minor_mapping = {}, - evaluate_markers_fn = None, simpleapi_download_fn = None, available_interpreters = {}): builder = _hub_builder( @@ -52,7 +50,6 @@ def hub_builder( module_name = "unit_test", config = config or struct( # no need to evaluate the markers with the interpreter - enable_pipstar = enable_pipstar, platforms = { "{}_{}{}".format(os, cpu, freethreaded): _plat( name = "{}_{}{}".format(os, cpu, freethreaded), @@ -76,13 +73,13 @@ def hub_builder( netrc = None, auth_patterns = None, ), + evaluate_markers_fn = None, whl_overrides = {}, minor_mapping = minor_mapping or {"3.15": "3.15.19"}, available_interpreters = available_interpreters or { "python_3_15_host": "unit_test_interpreter_target", }, simpleapi_download_fn = simpleapi_download_fn or (lambda *a, **k: {}), - evaluate_markers_fn = evaluate_markers_fn, logger = repo_utils.logger( struct( os = struct( @@ -303,14 +300,6 @@ _tests.append(_test_simple_multiple_python_versions) def _test_simple_with_markers(env): builder = hub_builder( env, - evaluate_markers_fn = lambda _, requirements, **__: { - key: [ - platform - for platform in platforms - if ("x86_64" in platform and "platform_machine ==" in key) or ("x86_64" not in platform and "platform_machine !=" in key) - ] - for key, platforms in requirements.items() - }, ) builder.pip_parse( _mock_mctx( @@ -405,7 +394,6 @@ def _test_torch_experimental_index_url(env): env, config = struct( netrc = None, - enable_pipstar = False, auth_patterns = {}, platforms = { "{}_{}".format(os, cpu): _plat( @@ -431,15 +419,6 @@ def _test_torch_experimental_index_url(env): "python_3_12_host": "unit_test_interpreter_target", }, minor_mapping = {"3.12": "3.12.19"}, - evaluate_markers_fn = lambda _, requirements, **__: { - # todo once 2692 is merged, this is going to be easier to test. - key: [ - platform - for platform in platforms - if ("x86_64" in platform and "platform_machine ==" in key) or ("x86_64" not in platform and "platform_machine !=" in key) - ] - for key, platforms in requirements.items() - }, simpleapi_download_fn = mocksimpleapi_download, ) builder.pip_parse( @@ -516,7 +495,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ pypi.whl_libraries().contains_exactly({ "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": ["linux_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -525,7 +503,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": ["linux_aarch64"], "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", @@ -534,7 +511,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": ["windows_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -543,7 +519,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": ["osx_aarch64"], "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", @@ -621,8 +596,6 @@ simple==0.0.3 \ "pypi_315_extra": { "dep_template": "@pypi//{name}:{target}", "download_only": True, - # TODO @aignas 2025-04-20: ensure that this is in the hub repo - # "experimental_target_platforms": ["cp315_linux_x86_64"], "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"], "python_interpreter_target": "unit_test_interpreter_target", "requirement": "extra==0.0.1 --hash=sha256:deadb00f", @@ -821,12 +794,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef pypi.whl_libraries().contains_exactly({ "pypi_315_any_name": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "windows_aarch64", - ], "extra_pip_args": ["--extra-args-for-sdist-building"], "filename": "any-name.tar.gz", "python_interpreter_target": "unit_test_interpreter_target", @@ -836,12 +803,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef }, "pypi_315_direct_without_sha_0_0_1_py3_none_any": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "windows_aarch64", - ], "filename": "direct_without_sha-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "direct_without_sha==0.0.1", @@ -862,12 +823,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef }, "pypi_315_simple_py3_none_any_deadb00f": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "windows_aarch64", - ], "filename": "simple-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple==0.0.1", @@ -876,12 +831,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef }, "pypi_315_some_pkg_py3_none_any_deadbaaf": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "windows_aarch64", - ], "filename": "some_pkg-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "some_pkg==0.0.1", @@ -890,12 +839,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef }, "pypi_315_some_py3_none_any_deadb33f": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "windows_aarch64", - ], "filename": "some-other-pkg-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "some_other_pkg==0.0.1", @@ -925,14 +868,6 @@ _tests.append(_test_simple_get_index) def _test_optimum_sys_platform_extra(env): builder = hub_builder( env, - evaluate_markers_fn = lambda _, requirements, **__: { - key: [ - platform - for platform in platforms - if ("darwin" in key and "osx" in platform) or ("linux" in key and "linux" in platform) - ] - for key, platforms in requirements.items() - }, ) builder.pip_parse( _mock_mctx( @@ -946,13 +881,14 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' _parse( hub_name = "pypi", python_version = "3.15", - requirements_lock = "universal.txt", + requirements_by_platform = { + "universal.txt": "linux_*,osx_*", + }, ), ) pypi = builder.build() - # FIXME @aignas 2025-09-07: we should expose the `optimum` package - pypi.exposed_packages().contains_exactly([]) + pypi.exposed_packages().contains_exactly(["optimum"]) pypi.group_map().contains_exactly({}) pypi.whl_map().contains_exactly({ "optimum": { @@ -995,9 +931,7 @@ _tests.append(_test_optimum_sys_platform_extra) def _test_pipstar_platforms(env): builder = hub_builder( env, - enable_pipstar = True, config = struct( - enable_pipstar = True, netrc = None, auth_patterns = {}, platforms = { From 6177306c7fdc0b1b4bdf36654167a5c4966ba505 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 8 Sep 2025 21:50:19 +0900 Subject: [PATCH 03/28] Revert "chore: cleanup bzlmod codebase after pipstar enabling" This reverts commit 10692cebd520c7b2d4d01347790726c9e69f8360. --- python/private/pypi/extension.bzl | 8 +++ python/private/pypi/hub_builder.bzl | 67 +++++++++++++++-- tests/pypi/hub_builder/hub_builder_tests.bzl | 76 ++++++++++++++++++-- 3 files changed, 141 insertions(+), 10 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index dbc6c15603..4708c8e53a 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -21,6 +21,7 @@ load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config") load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:repo_utils.bzl", "repo_utils") +load(":evaluate_markers.bzl", EVALUATE_MARKERS_SRCS = "SRCS") load(":hub_builder.bzl", "hub_builder") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_whl_name.bzl", "parse_whl_name") @@ -671,6 +672,13 @@ EXPERIMENTAL: this may be removed without notice. doc = """\ A dict of labels to wheel names that is typically generated by the whl_modifications. The labels are JSON config files describing the modifications. +""", + ), + "_evaluate_markers_srcs": attr.label_list( + default = EVALUATE_MARKERS_SRCS, + doc = """\ +The list of labels to use as SRCS for the marker evaluation code. This ensures that the +code will be re-evaluated when any of files in the default changes. """, ), }, **ATTRS) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 6e5ca72cb9..b6088e4ded 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -5,7 +5,7 @@ load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:version.bzl", "version") load("//python/private:version_label.bzl", "version_label") load(":attrs.bzl", "use_isolated") -load(":evaluate_markers.bzl", "evaluate_markers") +load(":evaluate_markers.bzl", "evaluate_markers_py", evaluate_markers_star = "evaluate_markers") load(":parse_requirements.bzl", "parse_requirements") load(":pep508_env.bzl", "env") load(":pep508_evaluate.bzl", "evaluate") @@ -175,6 +175,8 @@ def _add_whl_library(self, *, python_version, whl, repo): # disallow building from sdist. return + platforms = self._platforms[python_version] + # TODO @aignas 2025-06-29: we should not need the version in the repo_name if # we are using pipstar and we are downloading the wheel using the downloader repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) @@ -186,6 +188,17 @@ def _add_whl_library(self, *, python_version, whl, repo): )) self._whl_libraries[repo_name] = repo.args + if not self._config.enable_pipstar and "experimental_target_platforms" in repo.args: + self._whl_libraries[repo_name] |= { + "experimental_target_platforms": sorted({ + # TODO @aignas 2025-07-07: this should be solved in a better way + platforms[candidate].triple.partition("_")[-1]: None + for p in repo.args["experimental_target_platforms"] + for candidate in platforms + if candidate.endswith(p) + }), + } + mapping = self._whl_map.setdefault(whl.name, {}) if repo.config_setting in mapping and mapping[repo.config_setting] != repo_name: fail( @@ -316,9 +329,41 @@ def _evaluate_markers(self, pip_attr): if self._evaluate_markers_fn: return self._evaluate_markers_fn - return lambda _, requirements: evaluate_markers( - requirements = requirements, - platforms = self._platforms[pip_attr.python_version], + if self._config.enable_pipstar: + return lambda _, requirements: evaluate_markers_star( + requirements = requirements, + platforms = self._platforms[pip_attr.python_version], + ) + + interpreter = _detect_interpreter(self, pip_attr) + + # NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either + # in the PATH or if specified as a label. We will configure the env + # markers when evaluating the requirement lines based on the output + # from the `requirements_files_by_platform` which should have something + # similar to: + # { + # "//:requirements.txt": ["cp311_linux_x86_64", ...] + # } + # + # We know the target python versions that we need to evaluate the + # markers for and thus we don't need to use multiple python interpreter + # instances to perform this manipulation. This function should be executed + # only once by the underlying code to minimize the overhead needed to + # spin up a Python interpreter. + return lambda module_ctx, requirements: evaluate_markers_py( + module_ctx, + requirements = { + k: { + p: self._platforms[pip_attr.python_version][p].triple + for p in plats + } + for k, plats in requirements.items() + }, + python_interpreter = interpreter.path, + python_interpreter_target = interpreter.target, + srcs = pip_attr._evaluate_markers_srcs, + logger = self._logger, ) def _create_whl_repos( @@ -390,6 +435,7 @@ def _create_whl_repos( auth_patterns = self._config.auth_patterns or pip_attr.auth_patterns, python_version = _major_minor_version(pip_attr.python_version), is_multiple_versions = whl.is_multiple_versions, + enable_pipstar = self._config.enable_pipstar, ) _add_whl_library( self, @@ -417,6 +463,8 @@ def _common_args(self, module_ctx, *, pip_attr): python_interpreter = interpreter.path, python_interpreter_target = interpreter.target, ) + if not self._config.enable_pipstar: + maybe_args["experimental_target_platforms"] = pip_attr.experimental_target_platforms whl_library_args.update({k: v for k, v in maybe_args.items() if v}) maybe_args_with_default = dict( @@ -464,7 +512,8 @@ def _whl_repo( netrc, auth_patterns, python_version, - use_downloader): + use_downloader, + enable_pipstar = False): args = dict(whl_library_args) args["requirement"] = src.requirement_line is_whl = src.filename.endswith(".whl") @@ -507,6 +556,14 @@ def _whl_repo( args["urls"] = [src.url] args["sha256"] = src.sha256 args["filename"] = src.filename + if not enable_pipstar: + args["experimental_target_platforms"] = [ + # Get rid of the version for the target platforms because we are + # passing the interpreter any way. Ideally we should search of ways + # how to pass the target platforms through the hub repo. + p.partition("_")[2] + for p in src.target_platforms + ] return struct( repo_name = whl_repo_name(src.filename, src.sha256), diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 31f9e93511..9f6ee6720d 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -40,9 +40,11 @@ simple==0.0.1 \ def hub_builder( env, + enable_pipstar = False, debug = False, config = None, minor_mapping = {}, + evaluate_markers_fn = None, simpleapi_download_fn = None, available_interpreters = {}): builder = _hub_builder( @@ -50,6 +52,7 @@ def hub_builder( module_name = "unit_test", config = config or struct( # no need to evaluate the markers with the interpreter + enable_pipstar = enable_pipstar, platforms = { "{}_{}{}".format(os, cpu, freethreaded): _plat( name = "{}_{}{}".format(os, cpu, freethreaded), @@ -73,13 +76,13 @@ def hub_builder( netrc = None, auth_patterns = None, ), - evaluate_markers_fn = None, whl_overrides = {}, minor_mapping = minor_mapping or {"3.15": "3.15.19"}, available_interpreters = available_interpreters or { "python_3_15_host": "unit_test_interpreter_target", }, simpleapi_download_fn = simpleapi_download_fn or (lambda *a, **k: {}), + evaluate_markers_fn = evaluate_markers_fn, logger = repo_utils.logger( struct( os = struct( @@ -300,6 +303,14 @@ _tests.append(_test_simple_multiple_python_versions) def _test_simple_with_markers(env): builder = hub_builder( env, + evaluate_markers_fn = lambda _, requirements, **__: { + key: [ + platform + for platform in platforms + if ("x86_64" in platform and "platform_machine ==" in key) or ("x86_64" not in platform and "platform_machine !=" in key) + ] + for key, platforms in requirements.items() + }, ) builder.pip_parse( _mock_mctx( @@ -394,6 +405,7 @@ def _test_torch_experimental_index_url(env): env, config = struct( netrc = None, + enable_pipstar = False, auth_patterns = {}, platforms = { "{}_{}".format(os, cpu): _plat( @@ -419,6 +431,15 @@ def _test_torch_experimental_index_url(env): "python_3_12_host": "unit_test_interpreter_target", }, minor_mapping = {"3.12": "3.12.19"}, + evaluate_markers_fn = lambda _, requirements, **__: { + # todo once 2692 is merged, this is going to be easier to test. + key: [ + platform + for platform in platforms + if ("x86_64" in platform and "platform_machine ==" in key) or ("x86_64" not in platform and "platform_machine !=" in key) + ] + for key, platforms in requirements.items() + }, simpleapi_download_fn = mocksimpleapi_download, ) builder.pip_parse( @@ -495,6 +516,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ pypi.whl_libraries().contains_exactly({ "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": { "dep_template": "@pypi//{name}:{target}", + "experimental_target_platforms": ["linux_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -503,6 +525,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": { "dep_template": "@pypi//{name}:{target}", + "experimental_target_platforms": ["linux_aarch64"], "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", @@ -511,6 +534,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": { "dep_template": "@pypi//{name}:{target}", + "experimental_target_platforms": ["windows_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -519,6 +543,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": { "dep_template": "@pypi//{name}:{target}", + "experimental_target_platforms": ["osx_aarch64"], "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", @@ -596,6 +621,8 @@ simple==0.0.3 \ "pypi_315_extra": { "dep_template": "@pypi//{name}:{target}", "download_only": True, + # TODO @aignas 2025-04-20: ensure that this is in the hub repo + # "experimental_target_platforms": ["cp315_linux_x86_64"], "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"], "python_interpreter_target": "unit_test_interpreter_target", "requirement": "extra==0.0.1 --hash=sha256:deadb00f", @@ -794,6 +821,12 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef pypi.whl_libraries().contains_exactly({ "pypi_315_any_name": { "dep_template": "@pypi//{name}:{target}", + "experimental_target_platforms": [ + "linux_aarch64", + "linux_x86_64", + "osx_aarch64", + "windows_aarch64", + ], "extra_pip_args": ["--extra-args-for-sdist-building"], "filename": "any-name.tar.gz", "python_interpreter_target": "unit_test_interpreter_target", @@ -803,6 +836,12 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef }, "pypi_315_direct_without_sha_0_0_1_py3_none_any": { "dep_template": "@pypi//{name}:{target}", + "experimental_target_platforms": [ + "linux_aarch64", + "linux_x86_64", + "osx_aarch64", + "windows_aarch64", + ], "filename": "direct_without_sha-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "direct_without_sha==0.0.1", @@ -823,6 +862,12 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef }, "pypi_315_simple_py3_none_any_deadb00f": { "dep_template": "@pypi//{name}:{target}", + "experimental_target_platforms": [ + "linux_aarch64", + "linux_x86_64", + "osx_aarch64", + "windows_aarch64", + ], "filename": "simple-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple==0.0.1", @@ -831,6 +876,12 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef }, "pypi_315_some_pkg_py3_none_any_deadbaaf": { "dep_template": "@pypi//{name}:{target}", + "experimental_target_platforms": [ + "linux_aarch64", + "linux_x86_64", + "osx_aarch64", + "windows_aarch64", + ], "filename": "some_pkg-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "some_pkg==0.0.1", @@ -839,6 +890,12 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef }, "pypi_315_some_py3_none_any_deadb33f": { "dep_template": "@pypi//{name}:{target}", + "experimental_target_platforms": [ + "linux_aarch64", + "linux_x86_64", + "osx_aarch64", + "windows_aarch64", + ], "filename": "some-other-pkg-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "some_other_pkg==0.0.1", @@ -868,6 +925,14 @@ _tests.append(_test_simple_get_index) def _test_optimum_sys_platform_extra(env): builder = hub_builder( env, + evaluate_markers_fn = lambda _, requirements, **__: { + key: [ + platform + for platform in platforms + if ("darwin" in key and "osx" in platform) or ("linux" in key and "linux" in platform) + ] + for key, platforms in requirements.items() + }, ) builder.pip_parse( _mock_mctx( @@ -881,14 +946,13 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' _parse( hub_name = "pypi", python_version = "3.15", - requirements_by_platform = { - "universal.txt": "linux_*,osx_*", - }, + requirements_lock = "universal.txt", ), ) pypi = builder.build() - pypi.exposed_packages().contains_exactly(["optimum"]) + # FIXME @aignas 2025-09-07: we should expose the `optimum` package + pypi.exposed_packages().contains_exactly([]) pypi.group_map().contains_exactly({}) pypi.whl_map().contains_exactly({ "optimum": { @@ -931,7 +995,9 @@ _tests.append(_test_optimum_sys_platform_extra) def _test_pipstar_platforms(env): builder = hub_builder( env, + enable_pipstar = True, config = struct( + enable_pipstar = True, netrc = None, auth_patterns = {}, platforms = { From b8bb5fa3e32a19bc366dcd498dc34c4a37bf985a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 8 Sep 2025 21:52:49 +0900 Subject: [PATCH 04/28] enable pipstar in tests --- tests/pypi/hub_builder/hub_builder_tests.bzl | 40 +------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 9f6ee6720d..12fb1eaba4 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -40,7 +40,7 @@ simple==0.0.1 \ def hub_builder( env, - enable_pipstar = False, + enable_pipstar = True, debug = False, config = None, minor_mapping = {}, @@ -405,7 +405,7 @@ def _test_torch_experimental_index_url(env): env, config = struct( netrc = None, - enable_pipstar = False, + enable_pipstar = True, auth_patterns = {}, platforms = { "{}_{}".format(os, cpu): _plat( @@ -516,7 +516,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ pypi.whl_libraries().contains_exactly({ "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": ["linux_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -525,7 +524,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": ["linux_aarch64"], "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", @@ -534,7 +532,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": ["windows_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -543,7 +540,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": ["osx_aarch64"], "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", @@ -621,8 +617,6 @@ simple==0.0.3 \ "pypi_315_extra": { "dep_template": "@pypi//{name}:{target}", "download_only": True, - # TODO @aignas 2025-04-20: ensure that this is in the hub repo - # "experimental_target_platforms": ["cp315_linux_x86_64"], "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"], "python_interpreter_target": "unit_test_interpreter_target", "requirement": "extra==0.0.1 --hash=sha256:deadb00f", @@ -821,12 +815,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef pypi.whl_libraries().contains_exactly({ "pypi_315_any_name": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "windows_aarch64", - ], "extra_pip_args": ["--extra-args-for-sdist-building"], "filename": "any-name.tar.gz", "python_interpreter_target": "unit_test_interpreter_target", @@ -836,12 +824,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef }, "pypi_315_direct_without_sha_0_0_1_py3_none_any": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "windows_aarch64", - ], "filename": "direct_without_sha-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "direct_without_sha==0.0.1", @@ -862,12 +844,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef }, "pypi_315_simple_py3_none_any_deadb00f": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "windows_aarch64", - ], "filename": "simple-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple==0.0.1", @@ -876,12 +852,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef }, "pypi_315_some_pkg_py3_none_any_deadbaaf": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "windows_aarch64", - ], "filename": "some_pkg-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "some_pkg==0.0.1", @@ -890,12 +860,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef }, "pypi_315_some_py3_none_any_deadb33f": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "windows_aarch64", - ], "filename": "some-other-pkg-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "some_other_pkg==0.0.1", From 2ab8e3c42b956b75072e6475e540f6a65a8ed8ae Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 28 Sep 2025 12:14:31 +0900 Subject: [PATCH 05/28] chore: also enable pipstar on WORKSPACE --- CHANGELOG.md | 6 +++-- examples/pip_parse_vendored/requirements.bzl | 3 +++ python/private/pypi/config.bzl.tmpl.bzlmod | 9 -------- .../pypi/generate_whl_library_build_bazel.bzl | 23 +++++++++++-------- python/private/pypi/hub_repository.bzl | 10 -------- .../private/pypi/requirements.bzl.tmpl.bzlmod | 2 ++ .../pypi/requirements.bzl.tmpl.workspace | 3 +++ python/private/pypi/whl_library.bzl | 4 ++++ ...generate_whl_library_build_bazel_tests.bzl | 8 +++---- 9 files changed, 34 insertions(+), 34 deletions(-) delete mode 100644 python/private/pypi/config.bzl.tmpl.bzlmod diff --git a/CHANGELOG.md b/CHANGELOG.md index b3c0a1c527..bd43dfaa66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,8 +69,10 @@ END_UNRELEASED_TEMPLATE * (bootstrap) For {obj}`--bootstrap_impl=system_python`, the sys.path order has changed from `[app paths, stdlib, runtime site-packages]` to `[stdlib, app paths, runtime site-packages]`. -* (pypi) `pipstar` flag has been flipped to be enabled by default for bzlmod - ([#2949](https://github.com/bazel-contrib/rules_python/issues/2949)). +* (pypi) `pipstar` flag has been flipped to be enabled by default, to turn it + off use `RULES_PYTHON_ENABLE_PIPSTAR=0` environment variable. If you do, please + add a comment to + [#2949](https://github.com/bazel-contrib/rules_python/issues/2949). {#v0-0-0-fixed} ### Fixed diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index ead5c49b26..9774fea1bb 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -32,6 +32,8 @@ all_data_requirements = [ "@my_project_pip_deps_vendored_urllib3//:data", ] +packages = sorted(all_whl_requirements_by_package) + _packages = [ ("my_project_pip_deps_vendored_certifi", "certifi==2023.7.22 --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), ("my_project_pip_deps_vendored_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), @@ -112,5 +114,6 @@ def install_deps(**whl_library_kwargs): group_name = group_name, group_deps = group_deps, annotation = _get_annotation(requirement), + packages = packages, **whl_config ) diff --git a/python/private/pypi/config.bzl.tmpl.bzlmod b/python/private/pypi/config.bzl.tmpl.bzlmod deleted file mode 100644 index c3ada70d27..0000000000 --- a/python/private/pypi/config.bzl.tmpl.bzlmod +++ /dev/null @@ -1,9 +0,0 @@ -"""Extra configuration values that are exposed from the hub repository for spoke repositories to access. - -NOTE: This is internal `rules_python` API and if you would like to depend on it, please raise an issue -with your usecase. This may change in between rules_python versions without any notice. - -@generated by rules_python pip.parse bzlmod extension. -""" - -whl_map = %%WHL_MAP%% diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 3764e720c0..0813769646 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -72,6 +72,7 @@ def generate_whl_library_build_bazel( "requires", "metadata_name", "metadata_version", + "packages", "include", ] else: @@ -83,16 +84,20 @@ def generate_whl_library_build_bazel( "default_python_version", ] dep_template = kwargs.get("dep_template") - loads.append( - """load("{}", "{}")""".format( - dep_template.format( - name = "", - target = "config.bzl", + packages = kwargs.pop("packages", []) + if packages: + kwargs["include"] = render.list(packages) + else: + loads.append( + """load("{}", "{}")""".format( + dep_template.format( + name = "", + target = "requirements.bzl", + ), + "packages", ), - "whl_map", - ), - ) - kwargs["include"] = "whl_map" + ) + kwargs["include"] = "packages" for arg in unsupported_args: if kwargs.get(arg): diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 1d572d09e2..aef35f7be7 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -46,13 +46,6 @@ def _impl(rctx): macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name) rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) - rctx.template( - "config.bzl", - rctx.attr._config_template, - substitutions = { - "%%WHL_MAP%%": render.dict(rctx.attr.whl_map, value_repr = lambda x: "None"), - }, - ) rctx.template("requirements.bzl", rctx.attr._requirements_bzl_template, substitutions = { "%%ALL_DATA_REQUIREMENTS%%": render.list([ macro_tmpl.format(p, "data") @@ -99,9 +92,6 @@ The wheel map where values are json.encoded strings of the whl_map constructed in the pip.parse tag class. """, ), - "_config_template": attr.label( - default = ":config.bzl.tmpl.bzlmod", - ), "_requirements_bzl_template": attr.label( default = ":requirements.bzl.tmpl.bzlmod", ), diff --git a/python/private/pypi/requirements.bzl.tmpl.bzlmod b/python/private/pypi/requirements.bzl.tmpl.bzlmod index ba227aeb2d..c4352b4232 100644 --- a/python/private/pypi/requirements.bzl.tmpl.bzlmod +++ b/python/private/pypi/requirements.bzl.tmpl.bzlmod @@ -13,6 +13,8 @@ all_whl_requirements = all_whl_requirements_by_package.values() all_data_requirements = %%ALL_DATA_REQUIREMENTS%% +packages = sorted(all_whl_requirements_by_package) + def requirement(name): return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg") diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace index 2f4bcd6916..5a5184848f 100644 --- a/python/private/pypi/requirements.bzl.tmpl.workspace +++ b/python/private/pypi/requirements.bzl.tmpl.workspace @@ -13,6 +13,8 @@ all_whl_requirements = all_whl_requirements_by_package.values() all_data_requirements = %%ALL_DATA_REQUIREMENTS%% +packages = sorted(all_whl_requirements_by_package) + _packages = %%PACKAGES%% _config = %%CONFIG%% _annotations = %%ANNOTATIONS%% @@ -68,5 +70,6 @@ def install_deps(**whl_library_kwargs): group_name = group_name, group_deps = group_deps, annotation = _get_annotation(requirement), + packages = packages, **whl_config ) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 5cc53d84c6..9e71e9a05a 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -423,6 +423,7 @@ def _whl_library_impl(rctx): name = whl_path.basename, sdist_filename = sdist_filename, dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix), + packages = rctx.attr.packages, entry_points = entry_points, metadata_name = metadata.name, metadata_version = metadata.version, @@ -592,6 +593,9 @@ For example if your whl depends on `numpy` and your Python package repo is named "group_name": attr.string( doc = "Name of the group, if any.", ), + "packages": attr.string_list( + doc = "The list of packages to include in the transitive dependencies. This may be used in workspace where one may vendor the requirements.bzl file.", + ), "repo": attr.string( doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.", ), diff --git a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl index 225b296ebf..f3d759c02a 100644 --- a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl +++ b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl @@ -85,7 +85,7 @@ _tests.append(_test_all_legacy) def _test_all(env): want = """\ -load("@pypi//:config.bzl", "whl_map") +load("@pypi//:requirements.bzl", "packages") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets_from_requires") package(default_visibility = ["//visibility:public"]) @@ -112,7 +112,7 @@ whl_library_targets_from_requires( "qux", ], group_name = "qux", - include = whl_map, + include = packages, name = "foo.whl", requires_dist = [ "foo", @@ -149,7 +149,7 @@ _tests.append(_test_all) def _test_all_with_loads(env): want = """\ -load("@pypi//:config.bzl", "whl_map") +load("@pypi//:requirements.bzl", "packages") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets_from_requires") package(default_visibility = ["//visibility:public"]) @@ -176,7 +176,7 @@ whl_library_targets_from_requires( "qux", ], group_name = "qux", - include = whl_map, + include = packages, name = "foo.whl", requires_dist = [ "foo", From 320b65fc7563b6952cf7c049b8399bddbc9afee5 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 28 Sep 2025 12:21:59 +0900 Subject: [PATCH 06/28] pass a dummy list instead of passing dep_template --- python/private/internal_dev_deps.bzl | 6 +++--- tests/support/support.bzl | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index 551dea6478..e9581f32fa 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -40,7 +40,7 @@ def _internal_dev_deps_impl(mctx): name = "somepkg_with_build_files", whl_file = "@whl_with_build_files//:somepkg-1.0-any-none-any.whl", requirement = "somepkg", - dep_template = "@rules_python//tests/support:support.bzl", + packages = ["dummy"], ) # Setup for //tests/implicit_namespace_packages @@ -54,7 +54,7 @@ def _internal_dev_deps_impl(mctx): whl_file = "@implicit_namespace_ns_sub1_whl//:ns_sub1-1.0-any-none-any.whl", requirement = "ns-sub1", enable_implicit_namespace_pkgs = False, - dep_template = "@rules_python//tests/support:support.bzl", + packages = ["dummy"], ) whl_from_dir_repo( @@ -67,7 +67,7 @@ def _internal_dev_deps_impl(mctx): whl_file = "@implicit_namespace_ns_sub2_whl//:ns_sub2-1.0-any-none-any.whl", requirement = "ns-sub2", enable_implicit_namespace_pkgs = False, - dep_template = "@rules_python//tests/support:support.bzl", + packages = ["dummy"], ) internal_dev_deps = module_extension( diff --git a/tests/support/support.bzl b/tests/support/support.bzl index bdd610b10e..96c6ad902a 100644 --- a/tests/support/support.bzl +++ b/tests/support/support.bzl @@ -21,9 +21,6 @@ load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility -# this is used in integration tests with `whl_library` -whl_map = [] - MAC = Label("//tests/support:mac") MAC_X86_64 = Label("//tests/support:mac_x86_64") LINUX = Label("//tests/support:linux") From 8fc665ed25d7c9eda29e077233027be3f8a22282 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 28 Sep 2025 12:25:16 +0900 Subject: [PATCH 07/28] a better included packages passing --- python/private/internal_dev_deps.bzl | 3 --- .../private/pypi/generate_whl_library_build_bazel.bzl | 6 +++--- python/private/pypi/whl_library.bzl | 10 +++++++++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index e9581f32fa..d621a5d941 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -40,7 +40,6 @@ def _internal_dev_deps_impl(mctx): name = "somepkg_with_build_files", whl_file = "@whl_with_build_files//:somepkg-1.0-any-none-any.whl", requirement = "somepkg", - packages = ["dummy"], ) # Setup for //tests/implicit_namespace_packages @@ -54,7 +53,6 @@ def _internal_dev_deps_impl(mctx): whl_file = "@implicit_namespace_ns_sub1_whl//:ns_sub1-1.0-any-none-any.whl", requirement = "ns-sub1", enable_implicit_namespace_pkgs = False, - packages = ["dummy"], ) whl_from_dir_repo( @@ -67,7 +65,6 @@ def _internal_dev_deps_impl(mctx): whl_file = "@implicit_namespace_ns_sub2_whl//:ns_sub2-1.0-any-none-any.whl", requirement = "ns-sub2", enable_implicit_namespace_pkgs = False, - packages = ["dummy"], ) internal_dev_deps = module_extension( diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 0813769646..cb95c93052 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -85,9 +85,7 @@ def generate_whl_library_build_bazel( ] dep_template = kwargs.get("dep_template") packages = kwargs.pop("packages", []) - if packages: - kwargs["include"] = render.list(packages) - else: + if dep_template: loads.append( """load("{}", "{}")""".format( dep_template.format( @@ -98,6 +96,8 @@ def generate_whl_library_build_bazel( ), ) kwargs["include"] = "packages" + elif packages: + kwargs["include"] = render.list(packages) for arg in unsupported_args: if kwargs.get(arg): diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 9e71e9a05a..6e4d5ec9de 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -419,10 +419,18 @@ def _whl_library_impl(rctx): logger = logger, ) + dep_template = rctx.attr.dep_template + + # packages will be set in tests or WORKSPACE + if rctx.attr.packages: + dep_template = dep_template or "@{}{{name}}//:{{target}}".format( + rctx.attr.repo_prefix, + ) + build_file_contents = generate_whl_library_build_bazel( name = whl_path.basename, sdist_filename = sdist_filename, - dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix), + dep_template = dep_template, packages = rctx.attr.packages, entry_points = entry_points, metadata_name = metadata.name, From 8ceeee6bc97aa182172a94cd4e84c9ed3fe01c14 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 28 Sep 2025 12:28:29 +0900 Subject: [PATCH 08/28] actually turn the switch on for WORKSPACE --- python/private/internal_config_repo.bzl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/private/internal_config_repo.bzl b/python/private/internal_config_repo.bzl index a878b4c55a..0c6210696e 100644 --- a/python/private/internal_config_repo.bzl +++ b/python/private/internal_config_repo.bzl @@ -21,9 +21,8 @@ settings for rules to later use. load("//python/private:text_util.bzl", "render") load(":repo_utils.bzl", "repo_utils") -_BZLMOD_ENABLED = "@@" in str(Label("//:unused")) _ENABLE_PIPSTAR_ENVVAR_NAME = "RULES_PYTHON_ENABLE_PIPSTAR" -_ENABLE_PIPSTAR_DEFAULT = "1" if _BZLMOD_ENABLED else "0" +_ENABLE_PIPSTAR_DEFAULT = "1" _ENABLE_DEPRECATION_WARNINGS_ENVVAR_NAME = "RULES_PYTHON_DEPRECATION_WARNINGS" _ENABLE_DEPRECATION_WARNINGS_DEFAULT = "0" From 881e6bfa3cf1614b9e837f987c2f4ce66b9999e1 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 28 Sep 2025 12:38:50 +0900 Subject: [PATCH 09/28] fix the dep wiring in WORKSPACE --- .../pypi/generate_whl_library_build_bazel.bzl | 20 +++++++------------ python/private/pypi/whl_library.bzl | 10 +++------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index cb95c93052..21af402a1e 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -85,19 +85,13 @@ def generate_whl_library_build_bazel( ] dep_template = kwargs.get("dep_template") packages = kwargs.pop("packages", []) - if dep_template: - loads.append( - """load("{}", "{}")""".format( - dep_template.format( - name = "", - target = "requirements.bzl", - ), - "packages", - ), - ) - kwargs["include"] = "packages" - elif packages: - kwargs["include"] = render.list(packages) + if kwargs.get("requires_dist"): + packages_load = dep_template.format(name = "", target = "requirements.bzl") + if "_//" in packages_load: + kwargs["include"] = render.list(packages) + else: + loads.append("""load("{}", "{}")""".format(packages_load, "packages")) + kwargs["include"] = "packages" for arg in unsupported_args: if kwargs.get(arg): diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 6e4d5ec9de..d88d624e7c 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -421,16 +421,12 @@ def _whl_library_impl(rctx): dep_template = rctx.attr.dep_template - # packages will be set in tests or WORKSPACE - if rctx.attr.packages: - dep_template = dep_template or "@{}{{name}}//:{{target}}".format( - rctx.attr.repo_prefix, - ) - build_file_contents = generate_whl_library_build_bazel( name = whl_path.basename, sdist_filename = sdist_filename, - dep_template = dep_template, + dep_template = dep_template or "@{}{{name}}//:{{target}}".format( + rctx.attr.repo_prefix, + ), packages = rctx.attr.packages, entry_points = entry_points, metadata_name = metadata.name, From a9e4e83f07ea6ee846ce9c176024c96afe070681 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 28 Sep 2025 12:42:00 +0900 Subject: [PATCH 10/28] add a note --- python/private/pypi/whl_library.bzl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index d88d624e7c..9d936f4810 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -369,7 +369,10 @@ def _whl_library_impl(rctx): timeout = rctx.attr.timeout, ) - if rp_config.enable_pipstar: + # NOTE @aignas 2025-09-28: if someone has an old vendored file that does not have the + # dep_template set or the packages is not set either, we should still not break, best to + # disable pipstar for that particular case. + if rp_config.enable_pipstar and (rctx.dep_template or rctx.packages): pypi_repo_utils.execute_checked( rctx, op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path), From e0962449acf26dd8985bba14053463d477f3f353 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 28 Sep 2025 15:57:30 +0900 Subject: [PATCH 11/28] fixup --- python/private/pypi/whl_library.bzl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 9d936f4810..6c4ef4d730 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -372,7 +372,7 @@ def _whl_library_impl(rctx): # NOTE @aignas 2025-09-28: if someone has an old vendored file that does not have the # dep_template set or the packages is not set either, we should still not break, best to # disable pipstar for that particular case. - if rp_config.enable_pipstar and (rctx.dep_template or rctx.packages): + if rp_config.enable_pipstar and (rctx.attr.dep_template or rctx.attr.packages): pypi_repo_utils.execute_checked( rctx, op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path), @@ -422,12 +422,10 @@ def _whl_library_impl(rctx): logger = logger, ) - dep_template = rctx.attr.dep_template - build_file_contents = generate_whl_library_build_bazel( name = whl_path.basename, sdist_filename = sdist_filename, - dep_template = dep_template or "@{}{{name}}//:{{target}}".format( + dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format( rctx.attr.repo_prefix, ), packages = rctx.attr.packages, From 47b80c40321b99e4c78c622f0428b98aad70e0b7 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 30 Sep 2025 23:22:40 +0900 Subject: [PATCH 12/28] add extra docs to the changelog and adjust a bit --- CHANGELOG.md | 6 +++++- .../pypi/generate_whl_library_build_bazel.bzl | 14 ++++++++------ python/private/pypi/whl_library.bzl | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd43dfaa66..908deab5ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,7 +72,11 @@ END_UNRELEASED_TEMPLATE * (pypi) `pipstar` flag has been flipped to be enabled by default, to turn it off use `RULES_PYTHON_ENABLE_PIPSTAR=0` environment variable. If you do, please add a comment to - [#2949](https://github.com/bazel-contrib/rules_python/issues/2949). + [#2949](https://github.com/bazel-contrib/rules_python/issues/2949). On `WORKSPACE` + you will see `whl_library` re-fetching if the list of packages in the + `requirements.txt` file changes, to workaround this in cases where you are not + vendoring the requirements you can set + {obj}`pip_repository.use_hub_alias_dependencies` to `True`. {#v0-0-0-fixed} ### Fixed diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 21af402a1e..1d676ac7c7 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -85,13 +85,15 @@ def generate_whl_library_build_bazel( ] dep_template = kwargs.get("dep_template") packages = kwargs.pop("packages", []) - if kwargs.get("requires_dist"): + if not kwargs.get("requires_dist"): + # no deps, we can leave the extra loads out + pass + elif packages: + kwargs["include"] = render.list(packages) + else: packages_load = dep_template.format(name = "", target = "requirements.bzl") - if "_//" in packages_load: - kwargs["include"] = render.list(packages) - else: - loads.append("""load("{}", "{}")""".format(packages_load, "packages")) - kwargs["include"] = "packages" + loads.append("""load("{}", "{}")""".format(packages_load, "packages")) + kwargs["include"] = "packages" for arg in unsupported_args: if kwargs.get(arg): diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 6c4ef4d730..da615a7b52 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -428,7 +428,7 @@ def _whl_library_impl(rctx): dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format( rctx.attr.repo_prefix, ), - packages = rctx.attr.packages, + packages = rctx.attr.packages if rctx.attr.repo_prefix else [], entry_points = entry_points, metadata_name = metadata.name, metadata_version = metadata.version, From 162c2f4c2f3be4b3548fd64dca0e3866668b9502 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 30 Sep 2025 23:25:00 +0900 Subject: [PATCH 13/28] also test what we suggest in CHANGELOG --- WORKSPACE | 1 + 1 file changed, 1 insertion(+) diff --git a/WORKSPACE b/WORKSPACE index 077ddb5e68..4ddc9b86a2 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -129,6 +129,7 @@ pip_parse( requirements_darwin = "//tools/publish:requirements_darwin.txt", requirements_lock = "//tools/publish:requirements_linux.txt", requirements_windows = "//tools/publish:requirements_windows.txt", + use_hub_alias_dependencies = True, ) load("@rules_python_publish_deps//:requirements.bzl", "install_deps") From c8bd48481c10ef47e30014aa9b484b9f02a3be4d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 1 Oct 2025 18:29:37 +0900 Subject: [PATCH 14/28] Apply suggestion from @gemini-code-assist[bot] Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- python/private/pypi/whl_metadata.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/pypi/whl_metadata.bzl b/python/private/pypi/whl_metadata.bzl index b0202ee3f4..a56aac5782 100644 --- a/python/private/pypi/whl_metadata.bzl +++ b/python/private/pypi/whl_metadata.bzl @@ -26,7 +26,7 @@ def whl_metadata(*, install_dir, read_fn, logger): result = parse_whl_metadata(contents) if not (result.name and result.version): - logger.fail("Failed to parsed the wheel METADATA file:\n{}\n{}\n{}".format( + logger.fail("Failed to parse the wheel METADATA file:\n{}\n{}\n{}".format( 80 * "=", contents.rstrip("\n"), 80 * "=", From e6e60ac37d7c6efceb2bce5228c5974821c00a0f Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 1 Oct 2025 18:30:00 +0900 Subject: [PATCH 15/28] Update CHANGELOG.md Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 908deab5ad..00b899321c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,9 +73,9 @@ END_UNRELEASED_TEMPLATE off use `RULES_PYTHON_ENABLE_PIPSTAR=0` environment variable. If you do, please add a comment to [#2949](https://github.com/bazel-contrib/rules_python/issues/2949). On `WORKSPACE` - you will see `whl_library` re-fetching if the list of packages in the - `requirements.txt` file changes, to workaround this in cases where you are not - vendoring the requirements you can set + you will see `whl_library` re-fetching if the list of packages in the + `requirements.txt` file changes. To work around this in cases where you are not + vendoring the requirements, you can set {obj}`pip_repository.use_hub_alias_dependencies` to `True`. {#v0-0-0-fixed} From a91a14711e6c9d058857dfa6252a59539ba0a975 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 2 Oct 2025 20:49:13 +0900 Subject: [PATCH 16/28] wip --- examples/pip_parse_vendored/requirements.bzl | 15 +++++++--- python/pip_install/pip_repository.bzl | 2 ++ python/private/pypi/config.bzl.tmpl.bzlmod | 9 ++++++ .../pypi/generate_whl_library_build_bazel.bzl | 6 +--- python/private/pypi/hub_builder.bzl | 1 + python/private/pypi/hub_repository.bzl | 10 +++++++ python/private/pypi/pip_repository.bzl | 9 +++++- .../private/pypi/requirements.bzl.tmpl.bzlmod | 2 -- .../pypi/requirements.bzl.tmpl.workspace | 13 ++++++-- python/private/pypi/whl_config_repository.bzl | 30 +++++++++++++++++++ python/private/pypi/whl_library.bzl | 10 +++---- 11 files changed, 87 insertions(+), 20 deletions(-) create mode 100644 python/private/pypi/config.bzl.tmpl.bzlmod create mode 100644 python/private/pypi/whl_config_repository.bzl diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index 9774fea1bb..891c742745 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -4,7 +4,7 @@ """ load("@rules_python//python:pip.bzl", "pip_utils") -load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library") +load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_config_repository", "whl_library") all_requirements = [ "@my_project_pip_deps_vendored_certifi//:pkg", @@ -32,8 +32,6 @@ all_data_requirements = [ "@my_project_pip_deps_vendored_urllib3//:data", ] -packages = sorted(all_whl_requirements_by_package) - _packages = [ ("my_project_pip_deps_vendored_certifi", "certifi==2023.7.22 --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), ("my_project_pip_deps_vendored_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), @@ -100,6 +98,15 @@ def install_deps(**whl_library_kwargs): groups = all_requirement_groups, ) + config_repo = "my_project_pip_deps_vendored__config" + whl_config_repository( + name = config_repo, + whl_map = { + p: "" + for p in all_whl_requirements_by_package + }, + ) + # Install wheels which may be participants in a group whl_config = dict(_config) whl_config.update(whl_library_kwargs) @@ -114,6 +121,6 @@ def install_deps(**whl_library_kwargs): group_name = group_name, group_deps = group_deps, annotation = _get_annotation(requirement), - packages = packages, + config_load = "@{}//:config.bzl".format(config_repo), **whl_config ) diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 18deee1993..dd2412d5e7 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -17,10 +17,12 @@ load("//python/private/pypi:group_library.bzl", _group_library = "group_library") load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation") load("//python/private/pypi:pip_repository.bzl", _pip_repository = "pip_repository") +load("//python/private/pypi:whl_config_repository.bzl", _whl_config_repository = "whl_config_repository") load("//python/private/pypi:whl_library.bzl", _whl_library = "whl_library") # Re-exports for backwards compatibility group_library = _group_library pip_repository = _pip_repository whl_library = _whl_library +whl_config_repository = _whl_config_repository package_annotation = _package_annotation diff --git a/python/private/pypi/config.bzl.tmpl.bzlmod b/python/private/pypi/config.bzl.tmpl.bzlmod new file mode 100644 index 0000000000..f460745f88 --- /dev/null +++ b/python/private/pypi/config.bzl.tmpl.bzlmod @@ -0,0 +1,9 @@ +"""Extra configuration values that are exposed from the hub repository for spoke repositories to access. + +NOTE: This is internal `rules_python` API and if you would like to depend on it, please raise an issue +with your usecase. This may change in between rules_python versions without any notice. + +@generated by rules_python pip.parse bzlmod extension. +""" + +packages = %%PACKAGES%% diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 1d676ac7c7..e207f6d2f5 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -83,15 +83,11 @@ def generate_whl_library_build_bazel( "target_platforms", "default_python_version", ] - dep_template = kwargs.get("dep_template") - packages = kwargs.pop("packages", []) + packages_load = kwargs.pop("config_load") if not kwargs.get("requires_dist"): # no deps, we can leave the extra loads out pass - elif packages: - kwargs["include"] = render.list(packages) else: - packages_load = dep_template.format(name = "", target = "requirements.bzl") loads.append("""load("{}", "{}")""".format(packages_load, "packages")) kwargs["include"] = "packages" diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index b6088e4ded..58d35f2681 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -451,6 +451,7 @@ def _common_args(self, module_ctx, *, pip_attr): # attrs. whl_library_args = dict( dep_template = "@{}//{{name}}:{{target}}".format(self.name), + config_load = "@{}//:config.bzl".format(self.name), ) maybe_args = dict( # The following values are safe to omit if they have false like values diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index aef35f7be7..0762c7b122 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -46,6 +46,13 @@ def _impl(rctx): macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name) rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) + rctx.template( + "config.bzl", + rctx.attr._config_template, + substitutions = { + "%%PACKAGES%%": render.dict(rctx.attr.whl_map, value_repr = lambda x: "None"), + }, + ) rctx.template("requirements.bzl", rctx.attr._requirements_bzl_template, substitutions = { "%%ALL_DATA_REQUIREMENTS%%": render.list([ macro_tmpl.format(p, "data") @@ -92,6 +99,9 @@ The wheel map where values are json.encoded strings of the whl_map constructed in the pip.parse tag class. """, ), + "_config_template": attr.label( + default = ":config.bzl.tmpl.bzlmod", + ), "_requirements_bzl_template": attr.label( default = ":requirements.bzl.tmpl.bzlmod", ), diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 2cf20cd5a7..44da7f7a59 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -156,7 +156,7 @@ def _pip_repository_impl(rctx): imports = [ # NOTE: Maintain the order consistent with `buildifier` 'load("@rules_python//python:pip.bzl", "pip_utils")', - 'load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")', + 'load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_config_repository", "whl_library")', ] annotations = {} @@ -203,6 +203,13 @@ def _pip_repository_impl(rctx): rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { + " # %%CONFIG_LIBRARY%%": """\ + whl_config_library( + name = config_repo, + whl_map = { + p: "" for p in all_whl_requirements_by_package + }, + )""", " # %%GROUP_LIBRARY%%": """\ group_repo = "{name}__groups" group_library( diff --git a/python/private/pypi/requirements.bzl.tmpl.bzlmod b/python/private/pypi/requirements.bzl.tmpl.bzlmod index c4352b4232..ba227aeb2d 100644 --- a/python/private/pypi/requirements.bzl.tmpl.bzlmod +++ b/python/private/pypi/requirements.bzl.tmpl.bzlmod @@ -13,8 +13,6 @@ all_whl_requirements = all_whl_requirements_by_package.values() all_data_requirements = %%ALL_DATA_REQUIREMENTS%% -packages = sorted(all_whl_requirements_by_package) - def requirement(name): return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg") diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace index 5a5184848f..60b0cf7c60 100644 --- a/python/private/pypi/requirements.bzl.tmpl.workspace +++ b/python/private/pypi/requirements.bzl.tmpl.workspace @@ -13,8 +13,6 @@ all_whl_requirements = all_whl_requirements_by_package.values() all_data_requirements = %%ALL_DATA_REQUIREMENTS%% -packages = sorted(all_whl_requirements_by_package) - _packages = %%PACKAGES%% _config = %%CONFIG%% _annotations = %%ANNOTATIONS%% @@ -56,6 +54,15 @@ def install_deps(**whl_library_kwargs): # %%GROUP_LIBRARY%% + config_repo = "%%NAME%%__config" + whl_config_repository( + name = config_repo, + whl_map = { + p: "" + for p in all_whl_requirements_by_package + }, + ) + # Install wheels which may be participants in a group whl_config = dict(_config) whl_config.update(whl_library_kwargs) @@ -70,6 +77,6 @@ def install_deps(**whl_library_kwargs): group_name = group_name, group_deps = group_deps, annotation = _get_annotation(requirement), - packages = packages, + config_load = "@{}//:config.bzl".format(config_repo), **whl_config ) diff --git a/python/private/pypi/whl_config_repository.bzl b/python/private/pypi/whl_config_repository.bzl new file mode 100644 index 0000000000..f925fdba8e --- /dev/null +++ b/python/private/pypi/whl_config_repository.bzl @@ -0,0 +1,30 @@ +"" + +load("//python/private:text_util.bzl", "render") + +def _impl(rctx): + rctx.file("BUILD.bazel", "") + rctx.template( + "config.bzl", + rctx.attr._config_template, + substitutions = { + "%%PACKAGES%%": render.dict(rctx.attr.whl_map, value_repr = lambda x: "None"), + }, + ) + +whl_config_repository = repository_rule( + attrs = { + "whl_map": attr.string_dict( + mandatory = True, + doc = """\ +The wheel map where values are json.encoded strings of the whl_map constructed +in the pip.parse tag class. +""", + ), + "_config_template": attr.label( + default = ":config.bzl.tmpl.bzlmod", + ), + }, + doc = """A rule for WORKSPACE to ensure correct whl_repository configuration. PRIVATE USE ONLY.""", + implementation = _impl, +) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index da615a7b52..70a32046b9 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -372,7 +372,7 @@ def _whl_library_impl(rctx): # NOTE @aignas 2025-09-28: if someone has an old vendored file that does not have the # dep_template set or the packages is not set either, we should still not break, best to # disable pipstar for that particular case. - if rp_config.enable_pipstar and (rctx.attr.dep_template or rctx.attr.packages): + if rp_config.enable_pipstar and rctx.attr.config_load: pypi_repo_utils.execute_checked( rctx, op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path), @@ -428,7 +428,7 @@ def _whl_library_impl(rctx): dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format( rctx.attr.repo_prefix, ), - packages = rctx.attr.packages if rctx.attr.repo_prefix else [], + config_load = rctx.attr.config_load, entry_points = entry_points, metadata_name = metadata.name, metadata_version = metadata.version, @@ -578,6 +578,9 @@ whl_library_attrs = dict({ ), allow_files = True, ), + "config_load": attr.string( + doc = "The load location for configuration for pipstar.", + ), "dep_template": attr.string( doc = """ The dep template to use for referencing the dependencies. It should have `{name}` @@ -598,9 +601,6 @@ For example if your whl depends on `numpy` and your Python package repo is named "group_name": attr.string( doc = "Name of the group, if any.", ), - "packages": attr.string_list( - doc = "The list of packages to include in the transitive dependencies. This may be used in workspace where one may vendor the requirements.bzl file.", - ), "repo": attr.string( doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.", ), From 52d65ddaaf71b108cef6f2854f694f421431dfdf Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 2 Oct 2025 20:50:25 +0900 Subject: [PATCH 17/28] remove the CHANGELOG note --- CHANGELOG.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00b899321c..bd43dfaa66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,11 +72,7 @@ END_UNRELEASED_TEMPLATE * (pypi) `pipstar` flag has been flipped to be enabled by default, to turn it off use `RULES_PYTHON_ENABLE_PIPSTAR=0` environment variable. If you do, please add a comment to - [#2949](https://github.com/bazel-contrib/rules_python/issues/2949). On `WORKSPACE` - you will see `whl_library` re-fetching if the list of packages in the - `requirements.txt` file changes. To work around this in cases where you are not - vendoring the requirements, you can set - {obj}`pip_repository.use_hub_alias_dependencies` to `True`. + [#2949](https://github.com/bazel-contrib/rules_python/issues/2949). {#v0-0-0-fixed} ### Fixed From e5916e7bfd48982174f8b7a63413ee9116d0838c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 2 Oct 2025 20:56:02 +0900 Subject: [PATCH 18/28] use hub aliases repo if available: wip finish multi_pip_parse if needed --- WORKSPACE | 1 - python/private/pypi/pip_repository.bzl | 14 +++++++++----- .../private/pypi/requirements.bzl.tmpl.workspace | 9 +-------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 4ddc9b86a2..077ddb5e68 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -129,7 +129,6 @@ pip_parse( requirements_darwin = "//tools/publish:requirements_darwin.txt", requirements_lock = "//tools/publish:requirements_linux.txt", requirements_windows = "//tools/publish:requirements_windows.txt", - use_hub_alias_dependencies = True, ) load("@rules_python_publish_deps//:requirements.bzl", "install_deps") diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 44da7f7a59..7a9652ce08 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -204,12 +204,16 @@ def _pip_repository_impl(rctx): rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { " # %%CONFIG_LIBRARY%%": """\ - whl_config_library( + config_repo = "{name}__config" + whl_config_repository( name = config_repo, - whl_map = { - p: "" for p in all_whl_requirements_by_package - }, - )""", + whl_map = {{ + p: "" + for p in all_whl_requirements_by_package + }}, + )""".format(name = rctx.attr.name) if not rctx.attr.use_hub_alias_dependencies else """\ + config_repo = "{name}" +""".format(name = rctx.attr.name), " # %%GROUP_LIBRARY%%": """\ group_repo = "{name}__groups" group_library( diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace index 60b0cf7c60..b5b77ae85d 100644 --- a/python/private/pypi/requirements.bzl.tmpl.workspace +++ b/python/private/pypi/requirements.bzl.tmpl.workspace @@ -54,14 +54,7 @@ def install_deps(**whl_library_kwargs): # %%GROUP_LIBRARY%% - config_repo = "%%NAME%%__config" - whl_config_repository( - name = config_repo, - whl_map = { - p: "" - for p in all_whl_requirements_by_package - }, - ) + # %%CONFIG_LIBRARY%% # Install wheels which may be participants in a group whl_config = dict(_config) From af68cce3732d8f0c68b07c3b3c3196b430ba2cea Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Oct 2025 14:57:30 +0900 Subject: [PATCH 19/28] consolidate the config and group libraries --- examples/pip_parse_vendored/requirements.bzl | 2 +- python/pip_install/pip_repository.bzl | 3 +- python/private/pypi/group_library.bzl | 40 ---------- python/private/pypi/pip_repository.bzl | 11 +-- .../pypi/requirements.bzl.tmpl.workspace | 4 +- python/private/pypi/whl_config_repository.bzl | 23 +++++- python/private/pypi/whl_library_targets.bzl | 10 ++- tests/pypi/extension/extension_tests.bzl | 1 + ...generate_whl_library_build_bazel_tests.bzl | 75 ++++++++++++++++++- tests/pypi/hub_builder/hub_builder_tests.bzl | 27 +++++++ .../whl_library_targets_tests.bzl | 6 +- 11 files changed, 136 insertions(+), 66 deletions(-) delete mode 100644 python/private/pypi/group_library.bzl diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index 891c742745..11456f0600 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -91,7 +91,7 @@ def install_deps(**whl_library_kwargs): for requirement in group_requirements } - group_repo = "my_project_pip_deps_vendored__groups" + group_repo = "my_project_pip_deps_vendored__config" group_library( name = group_repo, repo_prefix = "my_project_pip_deps_vendored_", diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index dd2412d5e7..8b40d9883b 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -14,14 +14,13 @@ "" -load("//python/private/pypi:group_library.bzl", _group_library = "group_library") load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation") load("//python/private/pypi:pip_repository.bzl", _pip_repository = "pip_repository") load("//python/private/pypi:whl_config_repository.bzl", _whl_config_repository = "whl_config_repository") load("//python/private/pypi:whl_library.bzl", _whl_library = "whl_library") # Re-exports for backwards compatibility -group_library = _group_library +group_library = _whl_config_repository pip_repository = _pip_repository whl_library = _whl_library whl_config_repository = _whl_config_repository diff --git a/python/private/pypi/group_library.bzl b/python/private/pypi/group_library.bzl deleted file mode 100644 index ff800e2f18..0000000000 --- a/python/private/pypi/group_library.bzl +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2024 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""group_library implementation for WORKSPACE setups.""" - -load(":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") - -def _group_library_impl(rctx): - build_file_contents = generate_group_library_build_bazel( - repo_prefix = rctx.attr.repo_prefix, - groups = rctx.attr.groups, - ) - rctx.file("BUILD.bazel", build_file_contents) - -group_library = repository_rule( - attrs = { - "groups": attr.string_list_dict( - doc = "A mapping of group names to requirements within that group.", - ), - "repo_prefix": attr.string( - doc = "Prefix used for the whl_library created components of each group", - ), - }, - implementation = _group_library_impl, - doc = """ -Create a package containing only wrapper py_library and whl_library rules for implementing dependency groups. -This is an implementation detail of dependency groups and should not be used alone. - """, -) diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 7a9652ce08..8b424f8282 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -203,10 +203,12 @@ def _pip_repository_impl(rctx): rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { - " # %%CONFIG_LIBRARY%%": """\ + " # %%CONFIG_REPO%%": """\ config_repo = "{name}__config" whl_config_repository( name = config_repo, + repo_prefix = "{name}_", + groups = all_requirement_groups, whl_map = {{ p: "" for p in all_whl_requirements_by_package @@ -214,13 +216,6 @@ def _pip_repository_impl(rctx): )""".format(name = rctx.attr.name) if not rctx.attr.use_hub_alias_dependencies else """\ config_repo = "{name}" """.format(name = rctx.attr.name), - " # %%GROUP_LIBRARY%%": """\ - group_repo = "{name}__groups" - group_library( - name = group_repo, - repo_prefix = "{name}_", - groups = all_requirement_groups, - )""".format(name = rctx.attr.name) if not rctx.attr.use_hub_alias_dependencies else "", "%%ALL_DATA_REQUIREMENTS%%": render.list([ macro_tmpl.format(p, "data") for p in bzl_packages diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace index b5b77ae85d..7fcf9c7530 100644 --- a/python/private/pypi/requirements.bzl.tmpl.workspace +++ b/python/private/pypi/requirements.bzl.tmpl.workspace @@ -52,9 +52,7 @@ def install_deps(**whl_library_kwargs): for requirement in group_requirements } - # %%GROUP_LIBRARY%% - - # %%CONFIG_LIBRARY%% + # %%CONFIG_REPO%% # Install wheels which may be participants in a group whl_config = dict(_config) diff --git a/python/private/pypi/whl_config_repository.bzl b/python/private/pypi/whl_config_repository.bzl index f925fdba8e..e98d2ab6c3 100644 --- a/python/private/pypi/whl_config_repository.bzl +++ b/python/private/pypi/whl_config_repository.bzl @@ -1,19 +1,31 @@ -"" +"""whl_config_library implementation for WORKSPACE setups.""" load("//python/private:text_util.bzl", "render") +load(":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") def _impl(rctx): + build_file_contents = generate_group_library_build_bazel( + repo_prefix = rctx.attr.repo_prefix, + groups = rctx.attr.groups, + ) + rctx.file("_groups/BUILD.bazel", build_file_contents) rctx.file("BUILD.bazel", "") rctx.template( "config.bzl", rctx.attr._config_template, substitutions = { - "%%PACKAGES%%": render.dict(rctx.attr.whl_map, value_repr = lambda x: "None"), + "%%PACKAGES%%": render.dict(rctx.attr.whl_map or {}, value_repr = lambda x: "None"), }, ) whl_config_repository = repository_rule( attrs = { + "groups": attr.string_list_dict( + doc = "A mapping of group names to requirements within that group.", + ), + "repo_prefix": attr.string( + doc = "Prefix used for the whl_library created components of each group", + ), "whl_map": attr.string_dict( mandatory = True, doc = """\ @@ -25,6 +37,11 @@ in the pip.parse tag class. default = ":config.bzl.tmpl.bzlmod", ), }, - doc = """A rule for WORKSPACE to ensure correct whl_repository configuration. PRIVATE USE ONLY.""", + doc = """ +Create a package containing only wrapper py_library and whl_library rules for implementing dependency groups. +This is an implementation detail of dependency groups and should not be used alone. + +PRIVATE USE ONLY, only used in WORKSPACE. + """, implementation = _impl, ) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 89c1d348b3..50ade2f3a8 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -274,12 +274,18 @@ def whl_library_targets( if group_name and "//:" in dep_template: # This is the legacy behaviour where the group library is outside the hub repo label_tmpl = dep_template.format( - name = "_groups", + name = "_config", target = normalize_name(group_name) + "_{}", + ).replace( + "//:", + "//_groups:", ) impl_vis = [dep_template.format( - name = "_groups", + name = "_config", target = "__pkg__", + ).replace( + "//:", + "//_groups:", )] native.alias( diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 0514e1d95b..b1e363bc7b 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -168,6 +168,7 @@ def _test_simple(env): }}) pypi.whl_libraries().contains_exactly({ "pypi_315_simple": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf", diff --git a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl index f3d759c02a..39c2eb4379 100644 --- a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl +++ b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl @@ -37,7 +37,7 @@ whl_library_targets( "exclude_via_attr", "data_exclude_all", ], - dep_template = "@pypi//{name}:{target}", + dep_template = "@pypi_{name}//:{target}", dependencies = ["foo"], dependencies_by_platform = { "baz": ["bar"], @@ -59,7 +59,7 @@ whl_library_targets( # SOMETHING SPECIAL AT THE END """ actual = generate_whl_library_build_bazel( - dep_template = "@pypi//{name}:{target}", + dep_template = "@pypi_{name}//:{target}", name = "foo.whl", dependencies = ["foo"], dependencies_by_platform = {"baz": ["bar"]}, @@ -83,9 +83,74 @@ whl_library_targets( _tests.append(_test_all_legacy) +def _test_all_workspace(env): + want = """\ +load("@pypi//:config.bzl", "packages") +load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets_from_requires") + +package(default_visibility = ["//visibility:public"]) + +whl_library_targets_from_requires( + copy_executables = { + "exec_src": "exec_dest", + }, + copy_files = { + "file_src": "file_dest", + }, + data = ["extra_target"], + data_exclude = [ + "exclude_via_attr", + "data_exclude_all", + ], + dep_template = "@pypi//{name}:{target}", + entry_points = { + "foo": "bar.py", + }, + group_deps = [ + "foo", + "fox", + "qux", + ], + group_name = "qux", + include = packages, + name = "foo.whl", + requires_dist = [ + "foo", + "bar-baz", + "qux", + ], + srcs_exclude = ["srcs_exclude_all"], +) + +# SOMETHING SPECIAL AT THE END +""" + actual = generate_whl_library_build_bazel( + dep_template = "@pypi//{name}:{target}", + name = "foo.whl", + requires_dist = ["foo", "bar-baz", "qux"], + entry_points = { + "foo": "bar.py", + }, + data_exclude = ["exclude_via_attr"], + annotation = struct( + copy_files = {"file_src": "file_dest"}, + copy_executables = {"exec_src": "exec_dest"}, + data = ["extra_target"], + data_exclude_glob = ["data_exclude_all"], + srcs_exclude_glob = ["srcs_exclude_all"], + additive_build_content = """# SOMETHING SPECIAL AT THE END""", + ), + config_load = "@pypi//:config.bzl", + group_name = "qux", + group_deps = ["foo", "fox", "qux"], + ) + env.expect.that_str(actual.replace("@@", "@")).equals(want) + +_tests.append(_test_all_workspace) + def _test_all(env): want = """\ -load("@pypi//:requirements.bzl", "packages") +load("@pypi//:config.bzl", "packages") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets_from_requires") package(default_visibility = ["//visibility:public"]) @@ -140,6 +205,7 @@ whl_library_targets_from_requires( srcs_exclude_glob = ["srcs_exclude_all"], additive_build_content = """# SOMETHING SPECIAL AT THE END""", ), + config_load = "@pypi//:config.bzl", group_name = "qux", group_deps = ["foo", "fox", "qux"], ) @@ -149,7 +215,7 @@ _tests.append(_test_all) def _test_all_with_loads(env): want = """\ -load("@pypi//:requirements.bzl", "packages") +load("@pypi//:config.bzl", "packages") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets_from_requires") package(default_visibility = ["//visibility:public"]) @@ -205,6 +271,7 @@ whl_library_targets_from_requires( additive_build_content = """# SOMETHING SPECIAL AT THE END""", ), group_name = "qux", + config_load = "@pypi//:config.bzl", group_deps = ["foo", "fox", "qux"], ) env.expect.that_str(actual.replace("@@", "@")).equals(want) diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 12fb1eaba4..ee6200a70a 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -135,6 +135,7 @@ def _test_simple(env): }) pypi.whl_libraries().contains_exactly({ "pypi_315_simple": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf", @@ -186,11 +187,13 @@ def _test_simple_multiple_requirements(env): }) pypi.whl_libraries().contains_exactly({ "pypi_315_simple_osx_aarch64": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple==0.0.2 --hash=sha256:deadb00f", }, "pypi_315_simple_windows_aarch64": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple==0.0.1 --hash=sha256:deadbeef", @@ -276,21 +279,25 @@ new-package==0.0.1 --hash=sha256:deadb00f2 }) pypi.whl_libraries().contains_exactly({ "pypi_315_old_package": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "old-package==0.0.1 --hash=sha256:deadbaaf", }, "pypi_315_simple": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple==0.0.1 --hash=sha256:deadbeef", }, "pypi_316_new_package": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "new-package==0.0.1 --hash=sha256:deadb00f2", }, "pypi_316_simple": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple==0.0.2 --hash=sha256:deadb00f", @@ -357,11 +364,13 @@ torch==2.4.1 ; platform_machine != 'x86_64' \ }) pypi.whl_libraries().contains_exactly({ "pypi_315_torch_linux_aarch64_osx_aarch64_windows_aarch64": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1 --hash=sha256:deadbeef", }, "pypi_315_torch_linux_x86_64_linux_x86_64_freethreaded": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -515,6 +524,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }) pypi.whl_libraries().contains_exactly({ "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -523,6 +533,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "urls": ["https://torch.index/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl"], }, "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -531,6 +542,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "urls": ["https://torch.index/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"], }, "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -539,6 +551,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "urls": ["https://torch.index/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl"], }, "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -615,6 +628,7 @@ simple==0.0.3 \ }) pypi.whl_libraries().contains_exactly({ "pypi_315_extra": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "download_only": True, "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"], @@ -622,6 +636,7 @@ simple==0.0.3 \ "requirement": "extra==0.0.1 --hash=sha256:deadb00f", }, "pypi_315_simple_linux_x86_64": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "download_only": True, "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"], @@ -629,6 +644,7 @@ simple==0.0.3 \ "requirement": "simple==0.0.1 --hash=sha256:deadbeef", }, "pypi_315_simple_osx_aarch64": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "download_only": True, "extra_pip_args": ["--platform=macosx_10_9_arm64", "--python-version=315", "--implementation=cp", "--abi=cp315"], @@ -814,6 +830,7 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef }) pypi.whl_libraries().contains_exactly({ "pypi_315_any_name": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "extra_pip_args": ["--extra-args-for-sdist-building"], "filename": "any-name.tar.gz", @@ -823,6 +840,7 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "urls": ["some-archive/any-name.tar.gz"], }, "pypi_315_direct_without_sha_0_0_1_py3_none_any": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "direct_without_sha-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -831,18 +849,21 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "urls": ["example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl"], }, "pypi_315_git_dep": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "extra_pip_args": ["--extra-args-for-sdist-building"], "python_interpreter_target": "unit_test_interpreter_target", "requirement": "git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef", }, "pypi_315_pip_fallback": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "extra_pip_args": ["--extra-args-for-sdist-building"], "python_interpreter_target": "unit_test_interpreter_target", "requirement": "pip_fallback==0.0.1", }, "pypi_315_simple_py3_none_any_deadb00f": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "simple-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -851,6 +872,7 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "urls": ["example2.org"], }, "pypi_315_some_pkg_py3_none_any_deadbaaf": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "some_pkg-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -859,6 +881,7 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "urls": ["example-direct.org/some_pkg-0.0.1-py3-none-any.whl"], }, "pypi_315_some_py3_none_any_deadb33f": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "some-other-pkg-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -942,11 +965,13 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' }) pypi.whl_libraries().contains_exactly({ "pypi_315_optimum_linux_aarch64_linux_x86_64_linux_x86_64_freethreaded": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime-gpu]==1.17.1", }, "pypi_315_optimum_osx_aarch64": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime]==1.17.1", @@ -1023,11 +1048,13 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' }) pypi.whl_libraries().contains_exactly({ "pypi_315_optimum_mylinuxx86_64": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime-gpu]==1.17.1", }, "pypi_315_optimum_myosxaarch64": { + "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime]==1.17.1", diff --git a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl index 615358f35d..b7fb9094d7 100644 --- a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl +++ b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl @@ -420,8 +420,8 @@ def _test_group(env): ) env.expect.that_collection(alias_calls).contains_exactly([ - {"name": "pkg", "actual": "@pypi__groups//:qux_pkg", "visibility": ["//visibility:public"]}, - {"name": "whl", "actual": "@pypi__groups//:qux_whl", "visibility": ["//visibility:public"]}, + {"name": "pkg", "actual": "@pypi__config//_groups:qux_pkg", "visibility": ["//visibility:public"]}, + {"name": "whl", "actual": "@pypi__config//_groups:qux_whl", "visibility": ["//visibility:public"]}, ]) # buildifier: @unsorted-dict-items env.expect.that_collection(py_library_calls).has_size(1) @@ -446,7 +446,7 @@ def _test_group(env): "//conditions:default": [], }), "tags": [], - "visibility": ["@pypi__groups//:__pkg__"], + "visibility": ["@pypi__config//_groups:__pkg__"], "experimental_venvs_site_packages": Label("//python/config_settings:venvs_site_packages"), }) # buildifier: @unsorted-dict-items From a61e67663d58a9e7c6896736daf29aa44390b7d5 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Oct 2025 14:58:05 +0900 Subject: [PATCH 20/28] fixup --- examples/pip_parse_vendored/requirements.bzl | 11 +++-------- python/pip_install/BUILD.bazel | 2 +- python/private/pypi/pip_repository.bzl | 2 +- python/private/pypi/whl_config_repository.bzl | 1 - 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index 11456f0600..08c6ac216a 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -4,7 +4,7 @@ """ load("@rules_python//python:pip.bzl", "pip_utils") -load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_config_repository", "whl_library") +load("@rules_python//python/pip_install:pip_repository.bzl", "whl_config_repository", "whl_library") all_requirements = [ "@my_project_pip_deps_vendored_certifi//:pkg", @@ -91,16 +91,11 @@ def install_deps(**whl_library_kwargs): for requirement in group_requirements } - group_repo = "my_project_pip_deps_vendored__config" - group_library( - name = group_repo, - repo_prefix = "my_project_pip_deps_vendored_", - groups = all_requirement_groups, - ) - config_repo = "my_project_pip_deps_vendored__config" whl_config_repository( name = config_repo, + repo_prefix = "my_project_pip_deps_vendored_", + groups = all_requirement_groups, whl_map = { p: "" for p in all_whl_requirements_by_package diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel index 09bc46eea7..7a7746f56a 100644 --- a/python/pip_install/BUILD.bazel +++ b/python/pip_install/BUILD.bazel @@ -22,9 +22,9 @@ bzl_library( name = "pip_repository_bzl", srcs = ["pip_repository.bzl"], deps = [ - "//python/private/pypi:group_library_bzl", "//python/private/pypi:package_annotation_bzl", "//python/private/pypi:pip_repository_bzl", + "//python/private/pypi:whl_config_library_bzl", "//python/private/pypi:whl_library_bzl", ], ) diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 8b424f8282..31de157a85 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -156,7 +156,7 @@ def _pip_repository_impl(rctx): imports = [ # NOTE: Maintain the order consistent with `buildifier` 'load("@rules_python//python:pip.bzl", "pip_utils")', - 'load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_config_repository", "whl_library")', + 'load("@rules_python//python/pip_install:pip_repository.bzl", "whl_config_repository", "whl_library")', ] annotations = {} diff --git a/python/private/pypi/whl_config_repository.bzl b/python/private/pypi/whl_config_repository.bzl index e98d2ab6c3..4c275e4805 100644 --- a/python/private/pypi/whl_config_repository.bzl +++ b/python/private/pypi/whl_config_repository.bzl @@ -27,7 +27,6 @@ whl_config_repository = repository_rule( doc = "Prefix used for the whl_library created components of each group", ), "whl_map": attr.string_dict( - mandatory = True, doc = """\ The wheel map where values are json.encoded strings of the whl_map constructed in the pip.parse tag class. From e6f09b504e437899bcd672e09177884a8c385d88 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Oct 2025 22:15:22 +0900 Subject: [PATCH 21/28] further cleanup to prepare for final review --- .../{config.bzl.tmpl.bzlmod => config.bzl.tmpl} | 2 -- python/private/pypi/hub_repository.bzl | 2 +- python/private/pypi/pip_repository.bzl | 16 +++++++++++++++- python/private/pypi/whl_config_repository.bzl | 2 +- 4 files changed, 17 insertions(+), 5 deletions(-) rename python/private/pypi/{config.bzl.tmpl.bzlmod => config.bzl.tmpl} (85%) diff --git a/python/private/pypi/config.bzl.tmpl.bzlmod b/python/private/pypi/config.bzl.tmpl similarity index 85% rename from python/private/pypi/config.bzl.tmpl.bzlmod rename to python/private/pypi/config.bzl.tmpl index f460745f88..1037153cac 100644 --- a/python/private/pypi/config.bzl.tmpl.bzlmod +++ b/python/private/pypi/config.bzl.tmpl @@ -2,8 +2,6 @@ NOTE: This is internal `rules_python` API and if you would like to depend on it, please raise an issue with your usecase. This may change in between rules_python versions without any notice. - -@generated by rules_python pip.parse bzlmod extension. """ packages = %%PACKAGES%% diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 0762c7b122..f915aa1c77 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -100,7 +100,7 @@ in the pip.parse tag class. """, ), "_config_template": attr.label( - default = ":config.bzl.tmpl.bzlmod", + default = ":config.bzl.tmpl", ), "_requirements_bzl_template": attr.label( default = ":requirements.bzl.tmpl.bzlmod", diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 31de157a85..90c8e3bee3 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -193,7 +193,7 @@ def _pip_repository_impl(rctx): aliases = render_pkg_aliases( aliases = { pkg: rctx.attr.name + "_" + pkg - for pkg in bzl_packages or [] + for pkg in bzl_packages }, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = requirement_cycles, @@ -202,6 +202,17 @@ def _pip_repository_impl(rctx): rctx.file(path, contents) rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) + if rctx.attr.use_hub_alias_dependencies: + rctx.template( + "config.bzl", + rctx.attr._config_template, + substitutions = { + "%%PACKAGES%%": render.dict({ + pkg: None + for pkg in bzl_packages + }, value_repr = lambda x: "None"), + }, + ) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { " # %%CONFIG_REPO%%": """\ config_repo = "{name}__config" @@ -255,6 +266,9 @@ generated using the `package_name` macro. For example usage, see [this WORKSPACE file](https://github.com/bazel-contrib/rules_python/blob/main/examples/pip_repository_annotations/WORKSPACE). """, ), + _config_template = attr.label( + default = ":config.bzl.tmpl", + ), _template = attr.label( default = ":requirements.bzl.tmpl.workspace", ), diff --git a/python/private/pypi/whl_config_repository.bzl b/python/private/pypi/whl_config_repository.bzl index 4c275e4805..9e9b9553be 100644 --- a/python/private/pypi/whl_config_repository.bzl +++ b/python/private/pypi/whl_config_repository.bzl @@ -33,7 +33,7 @@ in the pip.parse tag class. """, ), "_config_template": attr.label( - default = ":config.bzl.tmpl.bzlmod", + default = ":config.bzl.tmpl", ), }, doc = """ From 323cb1387941cc62b5dcd345030a030108b04a3c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Oct 2025 22:38:59 +0900 Subject: [PATCH 22/28] extra tidyup and fixes --- CHANGELOG.md | 2 ++ python/pip_install/BUILD.bazel | 2 +- python/private/pypi/BUILD.bazel | 9 +++++++++ python/private/pypi/attrs.bzl | 1 + 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd43dfaa66..b9e4ecc977 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,8 @@ END_UNRELEASED_TEMPLATE off use `RULES_PYTHON_ENABLE_PIPSTAR=0` environment variable. If you do, please add a comment to [#2949](https://github.com/bazel-contrib/rules_python/issues/2949). + With this PR we are deprecating {obj}`pip.parse.experimental_target_platforms` and + {obj}`pip_repository.experimental_target_platforms`. It will be removed in the version. {#v0-0-0-fixed} ### Fixed diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel index 7a7746f56a..ea48505669 100644 --- a/python/pip_install/BUILD.bazel +++ b/python/pip_install/BUILD.bazel @@ -24,7 +24,7 @@ bzl_library( deps = [ "//python/private/pypi:package_annotation_bzl", "//python/private/pypi:pip_repository_bzl", - "//python/private/pypi:whl_config_library_bzl", + "//python/private/pypi:whl_config_repository_bzl", "//python/private/pypi:whl_library_bzl", ], ) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index b9650c8152..96a2702372 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -408,6 +408,15 @@ bzl_library( ], ) +bzl_library( + name = "whl_config_repository_bzl", + srcs = ["whl_config_repository.bzl"], + deps = [ + ":generate_group_library_build_bazel_bzl", + "//python/private:text_util_bzl", + ], +) + bzl_library( name = "whl_config_setting_bzl", srcs = ["whl_config_setting.bzl"], diff --git a/python/private/pypi/attrs.bzl b/python/private/pypi/attrs.bzl index a122fc8479..19bc60d503 100644 --- a/python/private/pypi/attrs.bzl +++ b/python/private/pypi/attrs.bzl @@ -152,6 +152,7 @@ Special values: `host` (for generating deps for the host platform only) and NOTE: this is not for cross-compiling Python wheels but rather for parsing the `whl` METADATA correctly. """, + deprecated = "Use pipstar instead of this feature", ), "extra_hub_aliases": attr.string_list_dict( doc = """\ From 51576147308188449735ef4cca370ba0eac2532d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Oct 2025 22:39:38 +0900 Subject: [PATCH 23/28] deprecated is not an attr kwarg --- python/private/pypi/attrs.bzl | 1 - 1 file changed, 1 deletion(-) diff --git a/python/private/pypi/attrs.bzl b/python/private/pypi/attrs.bzl index 19bc60d503..a122fc8479 100644 --- a/python/private/pypi/attrs.bzl +++ b/python/private/pypi/attrs.bzl @@ -152,7 +152,6 @@ Special values: `host` (for generating deps for the host platform only) and NOTE: this is not for cross-compiling Python wheels but rather for parsing the `whl` METADATA correctly. """, - deprecated = "Use pipstar instead of this feature", ), "extra_hub_aliases": attr.string_list_dict( doc = """\ From 4b184168bfc84d6e9cf69ec9821c6e35bffbeed0 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Oct 2025 22:52:53 +0900 Subject: [PATCH 24/28] fixup --- python/private/pypi/BUILD.bazel | 8 -------- 1 file changed, 8 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 96a2702372..601894b10e 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -157,14 +157,6 @@ bzl_library( ], ) -bzl_library( - name = "group_library_bzl", - srcs = ["group_library.bzl"], - deps = [ - ":generate_group_library_build_bazel_bzl", - ], -) - bzl_library( name = "hub_builder_bzl", srcs = ["hub_builder.bzl"], From 42fef585a59640a590df9a8afef4fc36567982b7 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Oct 2025 22:55:09 +0900 Subject: [PATCH 25/28] clarify wording --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f08a01e20e..2f4afd7d9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,8 +78,8 @@ END_UNRELEASED_TEMPLATE off use `RULES_PYTHON_ENABLE_PIPSTAR=0` environment variable. If you do, please add a comment to [#2949](https://github.com/bazel-contrib/rules_python/issues/2949). - With this PR we are deprecating {obj}`pip.parse.experimental_target_platforms` and - {obj}`pip_repository.experimental_target_platforms`. It will be removed in the version. + With this release we are deprecating {obj}`pip.parse.experimental_target_platforms` and + {obj}`pip_repository.experimental_target_platforms`. {#v0-0-0-fixed} ### Fixed From ff28b1bb17715ce3585c9d1c16d4143c083fbb70 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 5 Oct 2025 12:57:17 +0900 Subject: [PATCH 26/28] comments --- examples/pip_parse_vendored/requirements.bzl | 4 +-- python/pip_install/BUILD.bazel | 2 +- python/pip_install/pip_repository.bzl | 6 ++-- python/private/pypi/BUILD.bazel | 4 +-- python/private/pypi/pip_repository.bzl | 35 +++++++++++-------- ...fig_repository.bzl => whl_config_repo.bzl} | 6 ++-- python/private/pypi/whl_library_targets.bzl | 4 +++ 7 files changed, 36 insertions(+), 25 deletions(-) rename python/private/pypi/{whl_config_repository.bzl => whl_config_repo.bzl} (92%) diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index 08c6ac216a..317858a31a 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -4,7 +4,7 @@ """ load("@rules_python//python:pip.bzl", "pip_utils") -load("@rules_python//python/pip_install:pip_repository.bzl", "whl_config_repository", "whl_library") +load("@rules_python//python/pip_install:pip_repository.bzl", "whl_config_repo", "whl_library") all_requirements = [ "@my_project_pip_deps_vendored_certifi//:pkg", @@ -92,7 +92,7 @@ def install_deps(**whl_library_kwargs): } config_repo = "my_project_pip_deps_vendored__config" - whl_config_repository( + whl_config_repo( name = config_repo, repo_prefix = "my_project_pip_deps_vendored_", groups = all_requirement_groups, diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel index ea48505669..665375cc5b 100644 --- a/python/pip_install/BUILD.bazel +++ b/python/pip_install/BUILD.bazel @@ -24,7 +24,7 @@ bzl_library( deps = [ "//python/private/pypi:package_annotation_bzl", "//python/private/pypi:pip_repository_bzl", - "//python/private/pypi:whl_config_repository_bzl", + "//python/private/pypi:whl_config_repo_bzl", "//python/private/pypi:whl_library_bzl", ], ) diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 8b40d9883b..f9c3c9fb56 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -16,12 +16,12 @@ load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation") load("//python/private/pypi:pip_repository.bzl", _pip_repository = "pip_repository") -load("//python/private/pypi:whl_config_repository.bzl", _whl_config_repository = "whl_config_repository") +load("//python/private/pypi:whl_config_repo.bzl", _whl_config_repo = "whl_config_repo") load("//python/private/pypi:whl_library.bzl", _whl_library = "whl_library") # Re-exports for backwards compatibility -group_library = _whl_config_repository +group_library = _whl_config_repo pip_repository = _pip_repository whl_library = _whl_library -whl_config_repository = _whl_config_repository +whl_config_repo = _whl_config_repo package_annotation = _package_annotation diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 601894b10e..7d5314dd62 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -401,8 +401,8 @@ bzl_library( ) bzl_library( - name = "whl_config_repository_bzl", - srcs = ["whl_config_repository.bzl"], + name = "whl_config_repo_bzl", + srcs = ["whl_config_repo.bzl"], deps = [ ":generate_group_library_build_bazel_bzl", "//python/private:text_util_bzl", diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 90c8e3bee3..bfdc1830b1 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -24,6 +24,21 @@ load(":pip_repository_attrs.bzl", "ATTRS") load(":render_pkg_aliases.bzl", "render_pkg_aliases") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") +_HUB_CONFIG_REPO_TEMPLATE = """\ + config_repo = "{name}" + """ +_LEGACY_CONFIG_REPO_TEMPLATE = """\ + config_repo = "{name}__config" + whl_config_repo( + name = config_repo, + repo_prefix = "{name}_", + groups = all_requirement_groups, + whl_map = {{ + p: "" + for p in all_whl_requirements_by_package + }}, + )""" + def _get_python_interpreter_attr(rctx): """A helper function for getting the `python_interpreter` attribute or it's default @@ -156,7 +171,7 @@ def _pip_repository_impl(rctx): imports = [ # NOTE: Maintain the order consistent with `buildifier` 'load("@rules_python//python:pip.bzl", "pip_utils")', - 'load("@rules_python//python/pip_install:pip_repository.bzl", "whl_config_repository", "whl_library")', + 'load("@rules_python//python/pip_install:pip_repository.bzl", "whl_config_repo", "whl_library")', ] annotations = {} @@ -213,20 +228,12 @@ def _pip_repository_impl(rctx): }, value_repr = lambda x: "None"), }, ) + config_repo_template = _HUB_CONFIG_REPO_TEMPLATE + else: + config_repo_template = _LEGACY_CONFIG_REPO_TEMPLATE + rctx.template("requirements.bzl", rctx.attr._template, substitutions = { - " # %%CONFIG_REPO%%": """\ - config_repo = "{name}__config" - whl_config_repository( - name = config_repo, - repo_prefix = "{name}_", - groups = all_requirement_groups, - whl_map = {{ - p: "" - for p in all_whl_requirements_by_package - }}, - )""".format(name = rctx.attr.name) if not rctx.attr.use_hub_alias_dependencies else """\ - config_repo = "{name}" -""".format(name = rctx.attr.name), + " # %%CONFIG_REPO%%": config_repo_template.format(name = rctx.attr.name), "%%ALL_DATA_REQUIREMENTS%%": render.list([ macro_tmpl.format(p, "data") for p in bzl_packages diff --git a/python/private/pypi/whl_config_repository.bzl b/python/private/pypi/whl_config_repo.bzl similarity index 92% rename from python/private/pypi/whl_config_repository.bzl rename to python/private/pypi/whl_config_repo.bzl index 9e9b9553be..b7cea5a8b7 100644 --- a/python/private/pypi/whl_config_repository.bzl +++ b/python/private/pypi/whl_config_repo.bzl @@ -3,7 +3,7 @@ load("//python/private:text_util.bzl", "render") load(":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") -def _impl(rctx): +def _whl_config_repo_impl(rctx): build_file_contents = generate_group_library_build_bazel( repo_prefix = rctx.attr.repo_prefix, groups = rctx.attr.groups, @@ -18,7 +18,7 @@ def _impl(rctx): }, ) -whl_config_repository = repository_rule( +whl_config_repo = repository_rule( attrs = { "groups": attr.string_list_dict( doc = "A mapping of group names to requirements within that group.", @@ -42,5 +42,5 @@ This is an implementation detail of dependency groups and should not be used alo PRIVATE USE ONLY, only used in WORKSPACE. """, - implementation = _impl, + implementation = _whl_config_repo_impl, ) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 50ade2f3a8..a2d77daf4c 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -273,6 +273,10 @@ def whl_library_targets( # implementation. if group_name and "//:" in dep_template: # This is the legacy behaviour where the group library is outside the hub repo + # + # It is expected to disappear when we drop WORKSPACE or drop the vendoring of + # pip_parse `requirements.bzl` in WORKSPACE. The alternative would be to add + # another argument to the macro, but it is already full of arguments. label_tmpl = dep_template.format( name = "_config", target = normalize_name(group_name) + "_{}", From a0e4b4791aa3a6373d04b696823eb5eae65ba80d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 5 Oct 2025 13:10:34 +0900 Subject: [PATCH 27/28] fixup! comments --- examples/pip_parse_vendored/requirements.bzl | 5 +++-- python/private/pypi/pip_repository.bzl | 14 +++++--------- .../private/pypi/requirements.bzl.tmpl.workspace | 5 +++-- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index 317858a31a..f5551573fb 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -93,7 +93,7 @@ def install_deps(**whl_library_kwargs): config_repo = "my_project_pip_deps_vendored__config" whl_config_repo( - name = config_repo, + name = "my_project_pip_deps_vendored__config", repo_prefix = "my_project_pip_deps_vendored_", groups = all_requirement_groups, whl_map = { @@ -101,6 +101,7 @@ def install_deps(**whl_library_kwargs): for p in all_whl_requirements_by_package }, ) + config_load = "@{}//:config.bzl".format(config_repo) # Install wheels which may be participants in a group whl_config = dict(_config) @@ -116,6 +117,6 @@ def install_deps(**whl_library_kwargs): group_name = group_name, group_deps = group_deps, annotation = _get_annotation(requirement), - config_load = "@{}//:config.bzl".format(config_repo), + config_load = config_load, **whl_config ) diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index bfdc1830b1..e9a4c44da3 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -24,13 +24,9 @@ load(":pip_repository_attrs.bzl", "ATTRS") load(":render_pkg_aliases.bzl", "render_pkg_aliases") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") -_HUB_CONFIG_REPO_TEMPLATE = """\ - config_repo = "{name}" - """ -_LEGACY_CONFIG_REPO_TEMPLATE = """\ - config_repo = "{name}__config" +_CONFIG_REPO_TEMPLATE = """"{name}__config" whl_config_repo( - name = config_repo, + name = "{name}__config", repo_prefix = "{name}_", groups = all_requirement_groups, whl_map = {{ @@ -228,12 +224,11 @@ def _pip_repository_impl(rctx): }, value_repr = lambda x: "None"), }, ) - config_repo_template = _HUB_CONFIG_REPO_TEMPLATE + config_repo_template = repr(rctx.attr.name) else: - config_repo_template = _LEGACY_CONFIG_REPO_TEMPLATE + config_repo_template = _CONFIG_REPO_TEMPLATE.format(name = rctx.attr.name) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { - " # %%CONFIG_REPO%%": config_repo_template.format(name = rctx.attr.name), "%%ALL_DATA_REQUIREMENTS%%": render.list([ macro_tmpl.format(p, "data") for p in bzl_packages @@ -249,6 +244,7 @@ def _pip_repository_impl(rctx): }), "%%ANNOTATIONS%%": render.dict(dict(sorted(annotations.items()))), "%%CONFIG%%": render.dict(dict(sorted(config.items()))), + "%%CONFIG_REPO%%": config_repo_template, "%%EXTRA_PIP_ARGS%%": json.encode(options), "%%IMPORTS%%": "\n".join(imports), "%%MACRO_TMPL%%": macro_tmpl, diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace index 7fcf9c7530..f61a5271f6 100644 --- a/python/private/pypi/requirements.bzl.tmpl.workspace +++ b/python/private/pypi/requirements.bzl.tmpl.workspace @@ -52,7 +52,8 @@ def install_deps(**whl_library_kwargs): for requirement in group_requirements } - # %%CONFIG_REPO%% + config_repo = %%CONFIG_REPO%% + config_load = "@{}//:config.bzl".format(config_repo) # Install wheels which may be participants in a group whl_config = dict(_config) @@ -68,6 +69,6 @@ def install_deps(**whl_library_kwargs): group_name = group_name, group_deps = group_deps, annotation = _get_annotation(requirement), - config_load = "@{}//:config.bzl".format(config_repo), + config_load = config_load, **whl_config ) From f308d071ed666886b5f1d1c9754b4ebb4a393ba2 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 5 Oct 2025 13:17:51 +0900 Subject: [PATCH 28/28] add notes about vendoring --- CHANGELOG.md | 4 +++- python/private/pypi/whl_library.bzl | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f4afd7d9b..cc59e387ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,7 +79,9 @@ END_UNRELEASED_TEMPLATE add a comment to [#2949](https://github.com/bazel-contrib/rules_python/issues/2949). With this release we are deprecating {obj}`pip.parse.experimental_target_platforms` and - {obj}`pip_repository.experimental_target_platforms`. + {obj}`pip_repository.experimental_target_platforms`. For users using `WORKSPACE` and + vendoring the `requirements.bzl` file, please re-vendor so that downstream is unaffected + when the APIs get removed. {#v0-0-0-fixed} ### Fixed diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 70a32046b9..fe3308ad3c 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -372,6 +372,8 @@ def _whl_library_impl(rctx): # NOTE @aignas 2025-09-28: if someone has an old vendored file that does not have the # dep_template set or the packages is not set either, we should still not break, best to # disable pipstar for that particular case. + # + # Remove non-pipstar and config_load check when we release rules_python 2. if rp_config.enable_pipstar and rctx.attr.config_load: pypi_repo_utils.execute_checked( rctx,