Skip to content

Commit 42b72db

Browse files
chrislovecnmchrislovecnm
authored
feat(bzlmod): Implementing wheel annotations via whl_mods (#1278)
This commit implements a bzlmod extension that allows users to create "annotations" for wheel builds. The wheel_builder.py accepts a JSON file via a parameter called annotations; this extension creates those JSON files. The pip extension accepts a Label -> String dict argument of the JSON files. This feature is renamed to `whl_mods` because the JSON files are handled differently and the name "annotations" is uninformative. This modifies the creation of the BUILD files and their content, and is much more than just adding some notes about a whl. The whl_mod extension wheel names and the wheel names in pip must match. Closes: #1213
1 parent b8f1645 commit 42b72db

13 files changed

+435
-5
lines changed

‎.bazelrc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
# This lets us glob() up all the files inside the examples to make them inputs to tests
44
# (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
55
# To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh
6-
build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_point,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
7-
query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_point,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
6+
build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
7+
query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
88

99
test --test_output=errors
1010

‎examples/bzlmod/MODULE.bazel‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,50 @@ python.toolchain(
3737
# rules based on the `python_version` arg values.
3838
use_repo(python, "python_3_10", "python_3_9", "python_aliases")
3939

40+
# This extension allows a user to create modifications to how rules_python
41+
# creates different wheel repositories. Different attributes allow the user
42+
# to modify the BUILD file, and copy files.
43+
# See @rules_python//python/extensions:whl_mods.bzl attributes for more information
44+
# on each of the attributes.
45+
# You are able to set a hub name, so that you can have different modifications of the same
46+
# wheel in different pip hubs.
4047
pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
4148

49+
# Call whl_mods.create for the requests package.
50+
pip.whl_mods(
51+
# we are using the appended_build_content.BUILD file
52+
# to add content to the request wheel BUILD file.
53+
additive_build_content_file = "//whl_mods:appended_build_content.BUILD",
54+
data = [":generated_file"],
55+
hub_name = "whl_mods_hub",
56+
whl_name = "requests",
57+
)
58+
59+
ADDITIVE_BUILD_CONTENT = """\
60+
load("@bazel_skylib//rules:write_file.bzl", "write_file")
61+
write_file(
62+
name = "generated_file",
63+
out = "generated_file.txt",
64+
content = ["Hello world from build content file"],
65+
)
66+
"""
67+
68+
# Call whl_mods.create for the wheel package.
69+
pip.whl_mods(
70+
additive_build_content = ADDITIVE_BUILD_CONTENT,
71+
copy_executables = {
72+
"@@//whl_mods:data/copy_executable.py": "copied_content/executable.py",
73+
},
74+
copy_files = {
75+
"@@//whl_mods:data/copy_file.txt": "copied_content/file.txt",
76+
},
77+
data = [":generated_file"],
78+
data_exclude_glob = ["site-packages/*.dist-info/WHEEL"],
79+
hub_name = "whl_mods_hub",
80+
whl_name = "wheel",
81+
)
82+
use_repo(pip, "whl_mods_hub")
83+
4284
# To fetch pip dependencies, use pip.parse. We can pass in various options,
4385
# but typically we pass requirements and the Python version. The Python
4486
# version must have been configured by a corresponding `python.toolchain()`
@@ -50,12 +92,26 @@ pip.parse(
5092
python_version = "3.9",
5193
requirements_lock = "//:requirements_lock_3_9.txt",
5294
requirements_windows = "//:requirements_windows_3_9.txt",
95+
# These modifications were created above and we
96+
# are providing pip.parse with the label of the mod
97+
# and the name of the wheel.
98+
whl_modifications = {
99+
"@whl_mods_hub//:requests.json": "requests",
100+
"@whl_mods_hub//:wheel.json": "wheel",
101+
},
53102
)
54103
pip.parse(
55104
hub_name = "pip",
56105
python_version = "3.10",
57106
requirements_lock = "//:requirements_lock_3_10.txt",
58107
requirements_windows = "//:requirements_windows_3_10.txt",
108+
# These modifications were created above and we
109+
# are providing pip.parse with the label of the mod
110+
# and the name of the wheel.
111+
whl_modifications = {
112+
"@whl_mods_hub//:requests.json": "requests",
113+
"@whl_mods_hub//:wheel.json": "wheel",
114+
},
59115
)
60116

61117
# NOTE: The pip_39 repo is only used because the plain `@pip` repo doesn't

‎examples/bzlmod/requirements.in‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
--extra-index-url https://pypi.python.org/simple/
2+
3+
wheel
14
websockets
25
requests~=2.25.1
36
s3cmd~=2.1.0

‎examples/bzlmod/requirements_lock_3_10.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
#
55
# bazel run //:requirements_3_10.update
66
#
7+
--extra-index-url https://pypi.python.org/simple/
8+
79
astroid==2.13.5 \
810
--hash=sha256:6891f444625b6edb2ac798829b689e95297e100ddf89dbed5a8c610e34901501 \
911
--hash=sha256:df164d5ac811b9f44105a72b8f9d5edfb7b5b2d7e979b04ea377a77b3229114a
@@ -238,6 +240,10 @@ websockets==11.0.3 \
238240
--hash=sha256:fb06eea71a00a7af0ae6aefbb932fb8a7df3cb390cc217d51a9ad7343de1b8d0 \
239241
--hash=sha256:ffd7dcaf744f25f82190856bc26ed81721508fc5cbf2a330751e135ff1283564
240242
# via -r requirements.in
243+
wheel==0.40.0 \
244+
--hash=sha256:cd1196f3faee2b31968d626e1731c94f99cbdb67cf5a46e4f5656cbee7738873 \
245+
--hash=sha256:d236b20e7cb522daf2390fa84c55eea81c5c30190f90f29ae2ca1ad8355bf247
246+
# via -r requirements.in
241247
wrapt==1.15.0 \
242248
--hash=sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0 \
243249
--hash=sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420 \

‎examples/bzlmod/requirements_lock_3_9.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
#
55
# bazel run //:requirements_3_9.update
66
#
7+
--extra-index-url https://pypi.python.org/simple/
8+
79
astroid==2.12.13 \
810
--hash=sha256:10e0ad5f7b79c435179d0d0f0df69998c4eef4597534aae44910db060baeb907 \
911
--hash=sha256:1493fe8bd3dfd73dc35bd53c9d5b6e49ead98497c47b2307662556a5692d29d7
@@ -227,6 +229,10 @@ websockets==11.0.3 \
227229
--hash=sha256:fb06eea71a00a7af0ae6aefbb932fb8a7df3cb390cc217d51a9ad7343de1b8d0 \
228230
--hash=sha256:ffd7dcaf744f25f82190856bc26ed81721508fc5cbf2a330751e135ff1283564
229231
# via -r requirements.in
232+
wheel==0.40.0 \
233+
--hash=sha256:cd1196f3faee2b31968d626e1731c94f99cbdb67cf5a46e4f5656cbee7738873 \
234+
--hash=sha256:d236b20e7cb522daf2390fa84c55eea81c5c30190f90f29ae2ca1ad8355bf247
235+
# via -r requirements.in
230236
wrapt==1.14.1 \
231237
--hash=sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3 \
232238
--hash=sha256:01c205616a89d09827986bc4e859bcabd64f5a0662a7fe95e0d359424e0e071b \

‎examples/bzlmod/requirements_windows_3_10.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
#
55
# bazel run //:requirements_3_10.update
66
#
7+
--extra-index-url https://pypi.python.org/simple/
8+
79
astroid==2.13.5 \
810
--hash=sha256:6891f444625b6edb2ac798829b689e95297e100ddf89dbed5a8c610e34901501 \
911
--hash=sha256:df164d5ac811b9f44105a72b8f9d5edfb7b5b2d7e979b04ea377a77b3229114a
@@ -242,6 +244,10 @@ websockets==11.0.3 \
242244
--hash=sha256:fb06eea71a00a7af0ae6aefbb932fb8a7df3cb390cc217d51a9ad7343de1b8d0 \
243245
--hash=sha256:ffd7dcaf744f25f82190856bc26ed81721508fc5cbf2a330751e135ff1283564
244246
# via -r requirements.in
247+
wheel==0.40.0 \
248+
--hash=sha256:cd1196f3faee2b31968d626e1731c94f99cbdb67cf5a46e4f5656cbee7738873 \
249+
--hash=sha256:d236b20e7cb522daf2390fa84c55eea81c5c30190f90f29ae2ca1ad8355bf247
250+
# via -r requirements.in
245251
wrapt==1.15.0 \
246252
--hash=sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0 \
247253
--hash=sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420 \

‎examples/bzlmod/requirements_windows_3_9.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
#
55
# bazel run //:requirements_3_9.update
66
#
7+
--extra-index-url https://pypi.python.org/simple/
8+
79
astroid==2.12.13 \
810
--hash=sha256:10e0ad5f7b79c435179d0d0f0df69998c4eef4597534aae44910db060baeb907 \
911
--hash=sha256:1493fe8bd3dfd73dc35bd53c9d5b6e49ead98497c47b2307662556a5692d29d7
@@ -231,6 +233,10 @@ websockets==11.0.3 \
231233
--hash=sha256:fb06eea71a00a7af0ae6aefbb932fb8a7df3cb390cc217d51a9ad7343de1b8d0 \
232234
--hash=sha256:ffd7dcaf744f25f82190856bc26ed81721508fc5cbf2a330751e135ff1283564
233235
# via -r requirements.in
236+
wheel==0.40.0 \
237+
--hash=sha256:cd1196f3faee2b31968d626e1731c94f99cbdb67cf5a46e4f5656cbee7738873 \
238+
--hash=sha256:d236b20e7cb522daf2390fa84c55eea81c5c30190f90f29ae2ca1ad8355bf247
239+
# via -r requirements.in
234240
wrapt==1.14.1 \
235241
--hash=sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3 \
236242
--hash=sha256:01c205616a89d09827986bc4e859bcabd64f5a0662a7fe95e0d359424e0e071b \

‎examples/bzlmod/whl_mods/BUILD.bazel‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
load("@rules_python//python:defs.bzl", "py_test")
2+
3+
exports_files(
4+
glob(["data/**"]),
5+
visibility = ["//visibility:public"],
6+
)
7+
8+
py_test(
9+
name = "pip_whl_mods_test",
10+
srcs = ["pip_whl_mods_test.py"],
11+
env = {
12+
"REQUESTS_PKG_DIR": "pip_39_requests",
13+
"WHEEL_PKG_DIR": "pip_39_wheel",
14+
},
15+
main = "pip_whl_mods_test.py",
16+
deps = [
17+
"@pip//requests:pkg",
18+
"@pip//wheel:pkg",
19+
"@rules_python//python/runfiles",
20+
],
21+
)

‎examples/bzlmod/whl_mods/appended_build_content.BUILD‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
load("@bazel_skylib//rules:write_file.bzl", "write_file")
2+
3+
write_file(
4+
name = "generated_file",
5+
out = "generated_file.txt",
6+
content = ["Hello world from requests"],
7+
)

‎examples/bzlmod/whl_mods/data/copy_executable.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env python
2+
# Copyright 2023 The Bazel Authors. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
17+
if __name__ == "__main__":
18+
print("Hello world from copied executable")

0 commit comments

Comments
 (0)