From ba62630b227856499fb0fd9bcde49cc0e9e27c8f Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 17 Jun 2025 12:38:52 -0700 Subject: [PATCH 01/11] feat: generate filegroup with all extracted wheel files --- python/private/pypi/labels.bzl | 1 + python/private/pypi/whl_library_targets.bzl | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/python/private/pypi/labels.bzl b/python/private/pypi/labels.bzl index 73df07b2d2..22161b1496 100644 --- a/python/private/pypi/labels.bzl +++ b/python/private/pypi/labels.bzl @@ -14,6 +14,7 @@ """Constants used by parts of pip_repository for naming libraries and wheels.""" +EXTRACTED_WHEEL_FILES = "extracted_whl_files" WHEEL_FILE_PUBLIC_LABEL = "whl" WHEEL_FILE_IMPL_LABEL = "_whl" PY_LIBRARY_PUBLIC_LABEL = "pkg" diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 518d17163f..f28ee12d42 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -24,6 +24,7 @@ load( ":labels.bzl", "DATA_LABEL", "DIST_INFO_LABEL", + "EXTRACTED_WHEEL_FILES", "PY_LIBRARY_IMPL_LABEL", "PY_LIBRARY_PUBLIC_LABEL", "WHEEL_ENTRY_POINT_PREFIX", @@ -101,8 +102,16 @@ def whl_library_targets( srcs_exclude = [], tags = [], filegroups = { - DIST_INFO_LABEL: ["site-packages/*.dist-info/**"], - DATA_LABEL: ["data/**"], + EXTRACTED_WHEEL_FILES: dict( + include = ["**"], + exclude = ["*.whl"], + ), + DIST_INFO_LABEL: dict( + include = ["site-packages/*.dist-info/**"], + ), + DATA_LABEL: dict( + include = ["data/**"], + ), }, dependencies = [], dependencies_by_platform = {}, @@ -168,10 +177,11 @@ def whl_library_targets( tags = sorted(tags) data = [] + data - for filegroup_name, glob in filegroups.items(): + for filegroup_name, glob_kwargs in filegroups.items(): + glob_kwargs = {"allow_empty": True} | glob_kwargs native.filegroup( name = filegroup_name, - srcs = native.glob(glob, allow_empty = True), + srcs = native.glob(**glob_kwargs), visibility = ["//visibility:public"], ) From aac9bf7cafa38dca0cf7ec4dc6ab94b18fa16d18 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Fri, 20 Jun 2025 05:12:55 +0000 Subject: [PATCH 02/11] feat(pypi): Add filegroup for all extracted wheel files Addresses review comments from PR #3011: - Adds `EXTRACTED_WHEEL_FILES` to `pkg_aliases` in `python/private/pypi/pkg_aliases.bzl`. - Changes the `exclude` attribute for `EXTRACTED_WHEEL_FILES` in `python/private/pypi/whl_library_targets.bzl` to use the specific wheel filename. - Adds a TODO comment in `python/private/pypi/whl_library_targets.bzl` to consider parsing the `RECORD` file in the future. - Updates `CHANGELOG.md` to include the new feature. Note: Encountered a persistent internal Bazel error while attempting to run tests. The changes could not be verified with automated tests. --- CHANGELOG.md | 1 + python/private/pypi/pkg_aliases.bzl | 2 ++ python/private/pypi/whl_library_targets.bzl | 29 ++++++++++++--------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9897dc9ec8..f82fb83f5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,7 @@ END_UNRELEASED_TEMPLATE * (pypi) To configure the environment for `requirements.txt` evaluation, use the newly added developer preview of the `pip.default` tag class. Only `rules_python` and root modules can use this feature. +* (pypi) A new filegroup target `{name}_extracted_whl_files` is now available in `whl_library`, providing access to all files extracted from a wheel. {#v0-0-0-removed} ### Removed diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index d71c37cb4b..4d3cc61590 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -79,6 +79,7 @@ load( ":labels.bzl", "DATA_LABEL", "DIST_INFO_LABEL", + "EXTRACTED_WHEEL_FILES", "PY_LIBRARY_IMPL_LABEL", "PY_LIBRARY_PUBLIC_LABEL", "WHEEL_FILE_IMPL_LABEL", @@ -151,6 +152,7 @@ def pkg_aliases( WHEEL_FILE_PUBLIC_LABEL: WHEEL_FILE_IMPL_LABEL if group_name else WHEEL_FILE_PUBLIC_LABEL, DATA_LABEL: DATA_LABEL, DIST_INFO_LABEL: DIST_INFO_LABEL, + EXTRACTED_WHEEL_FILES: EXTRACTED_WHEEL_FILES, } | { x: x for x in extra_aliases or [] diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index f28ee12d42..2fff2e02f3 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -101,18 +101,6 @@ def whl_library_targets( data_exclude = [], srcs_exclude = [], tags = [], - filegroups = { - EXTRACTED_WHEEL_FILES: dict( - include = ["**"], - exclude = ["*.whl"], - ), - DIST_INFO_LABEL: dict( - include = ["site-packages/*.dist-info/**"], - ), - DATA_LABEL: dict( - include = ["data/**"], - ), - }, dependencies = [], dependencies_by_platform = {}, dependencies_with_markers = {}, @@ -177,7 +165,22 @@ def whl_library_targets( tags = sorted(tags) data = [] + data - for filegroup_name, glob_kwargs in filegroups.items(): + _filegroups = { + # TODO(#3011): Consider parsing the RECORD file to get a precise list of files + # instead of relying on a glob with excludes. + EXTRACTED_WHEEL_FILES: dict( + include = ["**"], + exclude = [name], + ), + DIST_INFO_LABEL: dict( + include = ["site-packages/*.dist-info/**"], + ), + DATA_LABEL: dict( + include = ["data/**"], + ), + } + + for filegroup_name, glob_kwargs in _filegroups.items(): glob_kwargs = {"allow_empty": True} | glob_kwargs native.filegroup( name = filegroup_name, From 80ac1f7dd5a8c98b73ab321bacbc1d1a6a9a2d5b Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 19 Jun 2025 22:23:30 -0700 Subject: [PATCH 03/11] fix up filegroup arg --- python/private/pypi/whl_library_targets.bzl | 36 +++++++++++---------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 2fff2e02f3..0d1f42afc9 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -102,6 +102,7 @@ def whl_library_targets( srcs_exclude = [], tags = [], dependencies = [], + filegroups = None, dependencies_by_platform = {}, dependencies_with_markers = {}, group_deps = [], @@ -131,8 +132,8 @@ def whl_library_targets( dependencies by platform key. dependencies_with_markers: {type}`dict[str, str]` A marker to evaluate in order for the dep to be included. - filegroups: {type}`dict[str, list[str]]` A dictionary of the target - names and the glob matches. + filegroups: {type}`dict[str, list[str]] | None` A dictionary of the target + names and the glob matches. If `None`, defaults will be used. group_name: {type}`str` name of the dependency group (if any) which contains this library. If set, this library will behave as a shim to group implementation rules which will provide simultaneously @@ -165,22 +166,23 @@ def whl_library_targets( tags = sorted(tags) data = [] + data - _filegroups = { - # TODO(#3011): Consider parsing the RECORD file to get a precise list of files - # instead of relying on a glob with excludes. - EXTRACTED_WHEEL_FILES: dict( - include = ["**"], - exclude = [name], - ), - DIST_INFO_LABEL: dict( - include = ["site-packages/*.dist-info/**"], - ), - DATA_LABEL: dict( - include = ["data/**"], - ), - } + if filesgroups == None: + filegroups = { + # TODO(#3011): Consider parsing the RECORD file to get a precise list of files + # instead of relying on a glob with excludes. + EXTRACTED_WHEEL_FILES: dict( + include = ["**"], + exclude = [name], + ), + DIST_INFO_LABEL: dict( + include = ["site-packages/*.dist-info/**"], + ), + DATA_LABEL: dict( + include = ["data/**"], + ), + } - for filegroup_name, glob_kwargs in _filegroups.items(): + for filegroup_name, glob_kwargs in filegroups.items(): glob_kwargs = {"allow_empty": True} | glob_kwargs native.filegroup( name = filegroup_name, From ee51ad244a66443c85388cccedbd503824e788bc Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 19 Jun 2025 22:26:35 -0700 Subject: [PATCH 04/11] add version note --- docs/pypi/use.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/pypi/use.md b/docs/pypi/use.md index cab07ea4fd..c98f5c5a32 100644 --- a/docs/pypi/use.md +++ b/docs/pypi/use.md @@ -45,6 +45,11 @@ Note, that the hub repo contains the following targets for each package: * `@pypi//numpy:whl` - the {obj}`filegroup` that is the `.whl` file itself which includes all of the transitive dependencies via the {attr}`filegroup.data` attribute. +:::{versionadded} VERSION_NEXT_FEATURE + +The `:extracted_whl_files` target was added +::: + ## Entry points If you would like to access [entry points][whl_ep], see the `py_console_script_binary` rule documentation, From 021746b2ac430fccac63f5d5f16a8079207049fa Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 19 Jun 2025 22:27:48 -0700 Subject: [PATCH 05/11] remove defunct todo --- python/private/pypi/whl_library_targets.bzl | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 0d1f42afc9..ebfb324ca8 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -168,8 +168,6 @@ def whl_library_targets( if filesgroups == None: filegroups = { - # TODO(#3011): Consider parsing the RECORD file to get a precise list of files - # instead of relying on a glob with excludes. EXTRACTED_WHEEL_FILES: dict( include = ["**"], exclude = [name], From c2a377a621e27ccf932254a4dfc90b8aa3175a9a Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 19 Jun 2025 23:07:31 -0700 Subject: [PATCH 06/11] fix var typo --- python/private/pypi/whl_library_targets.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index ebfb324ca8..a737a51750 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -166,7 +166,7 @@ def whl_library_targets( tags = sorted(tags) data = [] + data - if filesgroups == None: + if filegroups == None: filegroups = { EXTRACTED_WHEEL_FILES: dict( include = ["**"], From bc99905e39403977947d15d0108cbed6ceb83516 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 24 Jun 2025 21:17:26 -0700 Subject: [PATCH 07/11] pass sdist filename along --- python/private/pypi/whl_library.bzl | 5 +++++ python/private/pypi/whl_library_targets.bzl | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index c271449b3d..4a6b17b281 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -248,6 +248,7 @@ def _whl_library_impl(rctx): environment = _create_repository_execution_environment(rctx, python_interpreter, logger = logger) whl_path = None + sdist_filename = None if rctx.attr.whl_file: whl_path = rctx.path(rctx.attr.whl_file) @@ -276,6 +277,8 @@ def _whl_library_impl(rctx): if filename.endswith(".whl"): whl_path = rctx.path(filename) else: + sdist_filename = filename + # It is an sdist and we need to tell PyPI to use a file in this directory # and, allow getting build dependencies from PYTHONPATH, which we # setup in this repository rule, but still download any necessary @@ -381,6 +384,7 @@ def _whl_library_impl(rctx): build_file_contents = generate_whl_library_build_bazel( name = whl_path.basename, + sdist_filename = sdist_filename, dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix), entry_points = entry_points, metadata_name = metadata.name, @@ -454,6 +458,7 @@ def _whl_library_impl(rctx): build_file_contents = generate_whl_library_build_bazel( name = whl_path.basename, + sdist_filename = sdist_filename, dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix), entry_points = entry_points, # TODO @aignas 2025-05-17: maybe have a build flag for this instead diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index ebfb324ca8..10c381a8fd 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -124,6 +124,8 @@ def whl_library_targets( Args: name: {type}`str` The file to match for including it into the `whl` filegroup. This may be also parsed to generate extra metadata. + sdist_filename: {type}`str | None` If the wheel was built from an sdist, + the filename of the sdist. dep_template: {type}`str` The dep_template to use for dependency interpolation. tags: {type}`list[str]` The tags set on the `py_library`. @@ -170,7 +172,7 @@ def whl_library_targets( filegroups = { EXTRACTED_WHEEL_FILES: dict( include = ["**"], - exclude = [name], + exclude = [name] + [sdist_filename] if sdist_filename else [], ), DIST_INFO_LABEL: dict( include = ["site-packages/*.dist-info/**"], From 1f03a3fcb87d2903be42bbef7fc0a93993d4d515 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 25 Jun 2025 08:21:03 -0700 Subject: [PATCH 08/11] exclude build file from all files target --- python/private/pypi/whl_library_targets.bzl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index e1ccd1455f..e818ad3b2c 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -172,7 +172,10 @@ def whl_library_targets( filegroups = { EXTRACTED_WHEEL_FILES: dict( include = ["**"], - exclude = [name] + [sdist_filename] if sdist_filename else [], + exclude = [ + "BUILD.bazel", + name, + ] + [sdist_filename] if sdist_filename else [], ), DIST_INFO_LABEL: dict( include = ["site-packages/*.dist-info/**"], From 8b820e13e5914e1404eaf482073b00b3536ffc9b Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 25 Jun 2025 12:48:56 -0700 Subject: [PATCH 09/11] exclude more special bazel files --- python/private/pypi/whl_library_targets.bzl | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index e818ad3b2c..a546f65d8b 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -34,6 +34,16 @@ load( load(":namespace_pkgs.bzl", "create_inits") load(":pep508_deps.bzl", "deps") +# Files that are special to the Bazel processing of things. +_BAZEL_REPO_FILE_GLOBS = [ + "BUILD", + "BUILD.bazel", + "REPO.bazel", + "WORKSPACE", + "WORKSPACE", + "WORKSPACE.bazel", +] + def whl_library_targets_from_requires( *, name, @@ -172,10 +182,10 @@ def whl_library_targets( filegroups = { EXTRACTED_WHEEL_FILES: dict( include = ["**"], - exclude = [ - "BUILD.bazel", - name, - ] + [sdist_filename] if sdist_filename else [], + exclude = ( + _BAZEL_REPO_FILE_GLOBS + + [sdist_filename] if sdist_filename else [] + ), ), DIST_INFO_LABEL: dict( include = ["site-packages/*.dist-info/**"], From 11f41c507a12d59ab5a43fd585116609893c5d0a Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 6 Jul 2025 20:43:52 -0700 Subject: [PATCH 10/11] fix tests --- python/private/pypi/whl_library_targets.bzl | 5 +++-- tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 5 +++++ .../whl_library_targets/whl_library_targets_tests.bzl | 11 ++++++++--- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index e96b2099f8..aed5bc74f5 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -108,6 +108,7 @@ def whl_library_targets( *, name, dep_template, + sdist_filename = None, data_exclude = [], srcs_exclude = [], tags = [], @@ -135,10 +136,10 @@ def whl_library_targets( Args: name: {type}`str` The file to match for including it into the `whl` filegroup. This may be also parsed to generate extra metadata. - sdist_filename: {type}`str | None` If the wheel was built from an sdist, - the filename of the sdist. dep_template: {type}`str` The dep_template to use for dependency interpolation. + sdist_filename: {type}`str | None` If the wheel was built from an sdist, + the filename of the sdist. tags: {type}`list[str]` The tags set on the `py_library`. dependencies: {type}`list[str]` A list of dependencies. dependencies_by_platform: {type}`dict[str, list[str]]` A list of diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 123ee725f8..3fd08c393c 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -43,6 +43,7 @@ def _test_legacy_aliases(env): "whl": "@repo//:whl", "data": "@repo//:data", "dist_info": "@repo//:dist_info", + "extracted_whl_files": "@repo//:extracted_whl_files", "my_special": "@repo//:my_special", } @@ -242,6 +243,10 @@ def _test_group_aliases(env): "name": "dist_info", "actual": "@repo//:dist_info", }, + { + "name": "extracted_whl_files", + "actual": "@repo//:extracted_whl_files", + }, { "name": "pkg", "actual": "//_groups:my_group_pkg", diff --git a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl index bc58be9698..3a99b27bf2 100644 --- a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl +++ b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl @@ -27,9 +27,9 @@ _tests = [] def _test_filegroups(env): calls = [] - def glob(match, *, allow_empty): + def glob(include, *, exclude = [], allow_empty): env.expect.that_bool(allow_empty).equals(True) - return match + return include whl_library_targets( name = "", @@ -41,7 +41,7 @@ def _test_filegroups(env): rules = struct(), ) - env.expect.that_collection(calls).contains_exactly([ + env.expect.that_collection(calls, expr = "filegroup calls").contains_exactly([ { "name": "dist_info", "srcs": ["site-packages/*.dist-info/**"], @@ -52,6 +52,11 @@ def _test_filegroups(env): "srcs": ["data/**"], "visibility": ["//visibility:public"], }, + { + "name": "extracted_whl_files", + "srcs": ["**"], + "visibility": ["//visibility:public"], + }, { "name": "whl", "srcs": [""], From 5688f160b93c60b3f1047f2816755ed64c6b0a18 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 6 Jul 2025 20:46:20 -0700 Subject: [PATCH 11/11] lint --- tests/pypi/whl_library_targets/whl_library_targets_tests.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl index 3a99b27bf2..ec7ca63832 100644 --- a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl +++ b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl @@ -28,6 +28,7 @@ def _test_filegroups(env): calls = [] def glob(include, *, exclude = [], allow_empty): + _ = exclude # @unused env.expect.that_bool(allow_empty).equals(True) return include