Skip to content

Commit 78fc4de

Browse files
authored
feat(pip): provide pypi -> whl target mapping in requirements.bzl (#1532)
Currently a BUILD file can load `all_whl_requirements` but then can't determine which one is associated with a given package installed by the user. This makes it impossible to build rules where the user can choose a given package and then override the wheel used, for example. @mattem and I are working at a client where we need this ability. This PR makes a small, non-breaking refactoring to the generated `requirements.bzl` file so that this information is available in a new `all_whl_requirements_by_package` symbol. Users can then do something like this: ``` load("@pip//:requirements.bzl", "all_whl_requirements_by_package") some_rule( wheels = dict(all_whl_requirements_by_package, **{ "charset-normalizer": "@charset_1_2_3//:file" }), ) ```
1 parent dd4f904 commit 78fc4de

File tree

6 files changed

+24
-14
lines changed

6 files changed

+24
-14
lines changed

CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@ A brief description of the categories of changes:
3232
* (pip_parse) The installation of `pip_parse` repository rule toolchain
3333
dependencies is now done as part of `py_repositories` call.
3434

35+
* (pip_parse) The generated `requirements.bzl` file now has an additional symbol
36+
`all_whl_requirements_by_package` which provides a map from the original package name
37+
(as it appears in requirements.txt) to the target that provides the built wheel file.
38+
3539
* (pip_parse) The flag `incompatible_generate_aliases` has been flipped to
3640
`True` by default on `non-bzlmod` setups allowing users to use the same label
3741
strings during the transition period. For example, instead of

examples/pip_parse_vendored/requirements.bzl

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")
99

1010
all_requirements = ["@pip//certifi:pkg", "@pip//charset_normalizer:pkg", "@pip//idna:pkg", "@pip//requests:pkg", "@pip//urllib3:pkg"]
1111

12-
all_whl_requirements = ["@pip//certifi:whl", "@pip//charset_normalizer:whl", "@pip//idna:whl", "@pip//requests:whl", "@pip//urllib3:whl"]
12+
all_whl_requirements_by_package = {"certifi": "@pip//certifi:whl", "charset-normalizer": "@pip//charset_normalizer:whl", "idna": "@pip//idna:whl", "requests": "@pip//requests:whl", "urllib3": "@pip//urllib3:whl"}
13+
14+
all_whl_requirements = all_whl_requirements_by_package.values()
1315

1416
all_data_requirements = ["@pip//certifi:data", "@pip//charset_normalizer:data", "@pip//idna:data", "@pip//requests:data", "@pip//urllib3:data"]
1517

python/pip_install/pip_repository.bzl

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ def _pip_repository_impl(rctx):
276276

277277
packages = [(normalize_name(name), requirement) for name, requirement in parsed_requirements_txt.requirements]
278278

279-
bzl_packages = sorted([name for name, _ in packages])
279+
bzl_packages = dict(sorted([[name, normalize_name(name)] for name, _ in parsed_requirements_txt.requirements]))
280280

281281
imports = [
282282
'load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")',
@@ -314,7 +314,7 @@ def _pip_repository_impl(rctx):
314314

315315
if rctx.attr.incompatible_generate_aliases:
316316
macro_tmpl = "@%s//{}:{}" % rctx.attr.name
317-
aliases = render_pkg_aliases(repo_name = rctx.attr.name, bzl_packages = bzl_packages)
317+
aliases = render_pkg_aliases(repo_name = rctx.attr.name, bzl_packages = bzl_packages.values())
318318
for path, contents in aliases.items():
319319
rctx.file(path, contents)
320320
else:
@@ -324,16 +324,16 @@ def _pip_repository_impl(rctx):
324324
rctx.template("requirements.bzl", rctx.attr._template, substitutions = {
325325
"%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([
326326
macro_tmpl.format(p, "data")
327-
for p in bzl_packages
327+
for p in bzl_packages.values()
328328
]),
329329
"%%ALL_REQUIREMENTS%%": _format_repr_list([
330330
macro_tmpl.format(p, "pkg")
331-
for p in bzl_packages
332-
]),
333-
"%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([
334-
macro_tmpl.format(p, "whl")
335-
for p in bzl_packages
331+
for p in bzl_packages.values()
336332
]),
333+
"%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": _format_dict(_repr_dict({
334+
name: macro_tmpl.format(p, "whl")
335+
for name, p in bzl_packages.items()
336+
})),
337337
"%%ANNOTATIONS%%": _format_dict(_repr_dict(annotations)),
338338
"%%CONFIG%%": _format_dict(_repr_dict(config)),
339339
"%%EXTRA_PIP_ARGS%%": json.encode(options),

python/pip_install/pip_repository_requirements.bzl.tmpl

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@ from %%REQUIREMENTS_LOCK%%
88

99
all_requirements = %%ALL_REQUIREMENTS%%
1010

11-
all_whl_requirements = %%ALL_WHL_REQUIREMENTS%%
11+
all_whl_requirements_by_package = %%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%
12+
13+
all_whl_requirements = all_whl_requirements_by_package.values()
1214

1315
all_data_requirements = %%ALL_DATA_REQUIREMENTS%%
1416

python/private/bzlmod/pip_repository.bzl

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,10 @@ def _pip_repository_impl(rctx):
5151
macro_tmpl.format(p, p)
5252
for p in bzl_packages
5353
]),
54-
"%%ALL_WHL_REQUIREMENTS%%": render.list([
55-
macro_tmpl.format(p, "whl")
54+
"%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": render.dict({
55+
p: macro_tmpl.format(p, "whl")
5656
for p in bzl_packages
57-
]),
57+
}),
5858
"%%MACRO_TMPL%%": macro_tmpl,
5959
"%%NAME%%": rctx.attr.name,
6060
})

python/private/bzlmod/requirements.bzl.tmpl

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55

66
all_requirements = %%ALL_REQUIREMENTS%%
77

8-
all_whl_requirements = %%ALL_WHL_REQUIREMENTS%%
8+
all_whl_requirements_by_package = %%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%
9+
10+
all_whl_requirements = all_whl_requirements_by_package.values()
911

1012
all_data_requirements = %%ALL_DATA_REQUIREMENTS%%
1113

0 commit comments

Comments
 (0)