Skip to content

Commit 67e233f

Browse files
aignasaignas
authored
fix(pypi): output only necessary target_platforms (#2710)
This change reduces the number of lines we are going to write to the MODULE.bazel.lock file by not writing `experimental_target_platforms` to the lock file that eventually get discarded in the `whl_library` if the wheel is platform specific [1]. This means that the tests will become more easy to understand, but technically this is a no-op change, only resulting in a smaller lock file: ``` $ wc -l MODULE.bazel.lock 6536 MODULE.bazel.lock $ bazel mod deps --lockfile_mode=refresh ... $ wc -l MODULE.bazel.lock 6154 MODULE.bazel.lock ``` Work related to #2622 [1]: https://github.com/bazel-contrib/rules_python/blob/09145b9f628d482246eaa70421bf0cbae9acb096/python/private/pypi/whl_library.bzl#L337
1 parent 5d6827e commit 67e233f

File tree

4 files changed

+190
-3
lines changed

4 files changed

+190
-3
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ Unreleased changes template.
6464
* 3.12.9
6565
* 3.13.2
6666
* (pypi) Use `xcrun xcodebuild --showsdks` to find XCode root.
67+
* (pypi) The `bzlmod` extension will now generate smaller lock files for when
68+
using `experimental_index_url`.
6769
* (toolchains) Remove all but `3.8.20` versions of the Python `3.8` interpreter who has
6870
reached EOL. If users still need other versions of the `3.8` interpreter, please supply
6971
the URLs manually {bzl:ob}`python.toolchain` or {bzl:obj}`python_register_toolchains` calls.

‎python/private/pypi/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ bzl_library(
9393
":whl_config_setting_bzl",
9494
":whl_library_bzl",
9595
":whl_repo_name_bzl",
96+
":whl_target_platforms_bzl",
9697
"//python/private:full_version_bzl",
9798
"//python/private:normalize_name_bzl",
9899
"//python/private:semver_bzl",

‎python/private/pypi/extension.bzl‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ load(":simpleapi_download.bzl", "simpleapi_download")
3232
load(":whl_config_setting.bzl", "whl_config_setting")
3333
load(":whl_library.bzl", "whl_library")
3434
load(":whl_repo_name.bzl", "pypi_repo_name", "whl_repo_name")
35+
load(":whl_target_platforms.bzl", "whl_target_platforms")
3536

3637
def _major_minor_version(version):
3738
version = semver(version)
@@ -296,9 +297,18 @@ def _whl_repos(*, requirement, whl_library_args, download_only, netrc, auth_patt
296297

297298
# Pure python wheels or sdists may need to have a platform here
298299
target_platforms = None
299-
if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"):
300-
if multiple_requirements_for_whl:
301-
target_platforms = requirement.target_platforms
300+
if distribution.filename.endswith(".whl") and not distribution.filename.endswith("-any.whl"):
301+
parsed_whl = parse_whl_name(distribution.filename)
302+
whl_platforms = whl_target_platforms(
303+
platform_tag = parsed_whl.platform_tag,
304+
)
305+
args["experimental_target_platforms"] = [
306+
p
307+
for p in requirement.target_platforms
308+
if [None for wp in whl_platforms if p.endswith(wp.target_platform)]
309+
]
310+
elif multiple_requirements_for_whl:
311+
target_platforms = requirement.target_platforms
302312

303313
repo_name = whl_repo_name(
304314
distribution.filename,

‎tests/pypi/extension/extension_tests.bzl‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
load("@rules_testing//lib:test_suite.bzl", "test_suite")
1818
load("@rules_testing//lib:truth.bzl", "subjects")
1919
load("//python/private/pypi:extension.bzl", "parse_modules") # buildifier: disable=bzl-visibility
20+
load("//python/private/pypi:parse_simpleapi_html.bzl", "parse_simpleapi_html") # buildifier: disable=bzl-visibility
2021
load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility
2122

2223
_tests = []
@@ -332,6 +333,179 @@ torch==2.4.1 ; platform_machine != 'x86_64' \
332333

333334
_tests.append(_test_simple_with_markers)
334335

336+
def _test_torch_experimental_index_url(env):
337+
def mocksimpleapi_download(*_, **__):
338+
return {
339+
"torch": parse_simpleapi_html(
340+
url = "https://torch.index",
341+
content = """\
342+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp310-cp310-linux_x86_64.whl#sha256=833490a28ac156762ed6adaa7c695879564fa2fd0dc51bcf3fdb2c7b47dc55e6">torch-2.4.1+cpu-cp310-cp310-linux_x86_64.whl</a><br/>
343+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp310-cp310-win_amd64.whl#sha256=1dd062d296fb78aa7cfab8690bf03704995a821b5ef69cfc807af5c0831b4202">torch-2.4.1+cpu-cp310-cp310-win_amd64.whl</a><br/>
344+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp311-cp311-linux_x86_64.whl#sha256=2b03e20f37557d211d14e3fb3f71709325336402db132a1e0dd8b47392185baf">torch-2.4.1+cpu-cp311-cp311-linux_x86_64.whl</a><br/>
345+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp311-cp311-win_amd64.whl#sha256=76a6fe7b10491b650c630bc9ae328df40f79a948296b41d3b087b29a8a63cbad">torch-2.4.1+cpu-cp311-cp311-win_amd64.whl</a><br/>
346+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl#sha256=8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364">torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl</a><br/>
347+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl#sha256=3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97">torch-2.4.1+cpu-cp312-cp312-win_amd64.whl</a><br/>
348+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp38-cp38-linux_x86_64.whl#sha256=0c0a7cc4f7c74ff024d5a5e21230a01289b65346b27a626f6c815d94b4b8c955">torch-2.4.1+cpu-cp38-cp38-linux_x86_64.whl</a><br/>
349+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp38-cp38-win_amd64.whl#sha256=330e780f478707478f797fdc82c2a96e9b8c5f60b6f1f57bb6ad1dd5b1e7e97e">torch-2.4.1+cpu-cp38-cp38-win_amd64.whl</a><br/>
350+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp39-cp39-linux_x86_64.whl#sha256=3c99506980a2fb4b634008ccb758f42dd82f93ae2830c1e41f64536e310bf562">torch-2.4.1+cpu-cp39-cp39-linux_x86_64.whl</a><br/>
351+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp39-cp39-win_amd64.whl#sha256=c4f2c3c026e876d4dad7629170ec14fff48c076d6c2ae0e354ab3fdc09024f00">torch-2.4.1+cpu-cp39-cp39-win_amd64.whl</a><br/>
352+
<a href="/whl/cpu/torch-2.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=fa27b048d32198cda6e9cff0bf768e8683d98743903b7e5d2b1f5098ded1d343">torch-2.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl</a><br/>
353+
<a href="/whl/cpu/torch-2.4.1-cp310-none-macosx_11_0_arm64.whl#sha256=d36a8ef100f5bff3e9c3cea934b9e0d7ea277cb8210c7152d34a9a6c5830eadd">torch-2.4.1-cp310-none-macosx_11_0_arm64.whl</a><br/>
354+
<a href="/whl/cpu/torch-2.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=30be2844d0c939161a11073bfbaf645f1c7cb43f62f46cc6e4df1c119fb2a798">torch-2.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl</a><br/>
355+
<a href="/whl/cpu/torch-2.4.1-cp311-none-macosx_11_0_arm64.whl#sha256=ddddbd8b066e743934a4200b3d54267a46db02106876d21cf31f7da7a96f98ea">torch-2.4.1-cp311-none-macosx_11_0_arm64.whl</a><br/>
356+
<a href="/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a">torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl</a><br/>
357+
<a href="/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl#sha256=72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d">torch-2.4.1-cp312-none-macosx_11_0_arm64.whl</a><br/>
358+
<a href="/whl/cpu/torch-2.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=56ad2a760b7a7882725a1eebf5657abbb3b5144eb26bcb47b52059357463c548">torch-2.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl</a><br/>
359+
<a href="/whl/cpu/torch-2.4.1-cp38-none-macosx_11_0_arm64.whl#sha256=5fc1d4d7ed265ef853579caf272686d1ed87cebdcd04f2a498f800ffc53dab71">torch-2.4.1-cp38-none-macosx_11_0_arm64.whl</a><br/>
360+
<a href="/whl/cpu/torch-2.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=1495132f30f722af1a091950088baea383fe39903db06b20e6936fd99402803e">torch-2.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl</a><br/>
361+
<a href="/whl/cpu/torch-2.4.1-cp39-none-macosx_11_0_arm64.whl#sha256=a38de2803ee6050309aac032676536c3d3b6a9804248537e38e098d0e14817ec">torch-2.4.1-cp39-none-macosx_11_0_arm64.whl</a><br/>
362+
""",
363+
),
364+
}
365+
366+
pypi = _parse_modules(
367+
env,
368+
module_ctx = _mock_mctx(
369+
_mod(
370+
name = "rules_python",
371+
parse = [
372+
_parse(
373+
hub_name = "pypi",
374+
python_version = "3.12",
375+
experimental_index_url = "https://torch.index",
376+
requirements_lock = "universal.txt",
377+
),
378+
],
379+
),
380+
read = lambda x: {
381+
"universal.txt": """\
382+
torch==2.4.1 ; platform_machine != 'x86_64' \
383+
--hash=sha256:1495132f30f722af1a091950088baea383fe39903db06b20e6936fd99402803e \
384+
--hash=sha256:30be2844d0c939161a11073bfbaf645f1c7cb43f62f46cc6e4df1c119fb2a798 \
385+
--hash=sha256:36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a \
386+
--hash=sha256:56ad2a760b7a7882725a1eebf5657abbb3b5144eb26bcb47b52059357463c548 \
387+
--hash=sha256:5fc1d4d7ed265ef853579caf272686d1ed87cebdcd04f2a498f800ffc53dab71 \
388+
--hash=sha256:72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d \
389+
--hash=sha256:a38de2803ee6050309aac032676536c3d3b6a9804248537e38e098d0e14817ec \
390+
--hash=sha256:d36a8ef100f5bff3e9c3cea934b9e0d7ea277cb8210c7152d34a9a6c5830eadd \
391+
--hash=sha256:ddddbd8b066e743934a4200b3d54267a46db02106876d21cf31f7da7a96f98ea \
392+
--hash=sha256:fa27b048d32198cda6e9cff0bf768e8683d98743903b7e5d2b1f5098ded1d343
393+
# via -r requirements.in
394+
torch==2.4.1+cpu ; platform_machine == 'x86_64' \
395+
--hash=sha256:0c0a7cc4f7c74ff024d5a5e21230a01289b65346b27a626f6c815d94b4b8c955 \
396+
--hash=sha256:1dd062d296fb78aa7cfab8690bf03704995a821b5ef69cfc807af5c0831b4202 \
397+
--hash=sha256:2b03e20f37557d211d14e3fb3f71709325336402db132a1e0dd8b47392185baf \
398+
--hash=sha256:330e780f478707478f797fdc82c2a96e9b8c5f60b6f1f57bb6ad1dd5b1e7e97e \
399+
--hash=sha256:3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97 \
400+
--hash=sha256:3c99506980a2fb4b634008ccb758f42dd82f93ae2830c1e41f64536e310bf562 \
401+
--hash=sha256:76a6fe7b10491b650c630bc9ae328df40f79a948296b41d3b087b29a8a63cbad \
402+
--hash=sha256:833490a28ac156762ed6adaa7c695879564fa2fd0dc51bcf3fdb2c7b47dc55e6 \
403+
--hash=sha256:8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364 \
404+
--hash=sha256:c4f2c3c026e876d4dad7629170ec14fff48c076d6c2ae0e354ab3fdc09024f00
405+
# via -r requirements.in
406+
""",
407+
}[x],
408+
),
409+
available_interpreters = {
410+
"python_3_12_host": "unit_test_interpreter_target",
411+
},
412+
evaluate_markers = lambda _, requirements, **__: {
413+
# todo once 2692 is merged, this is going to be easier to test.
414+
key: [
415+
platform
416+
for platform in platforms
417+
if ("x86_64" in platform and "platform_machine ==" in key) or ("x86_64" not in platform and "platform_machine !=" in key)
418+
]
419+
for key, platforms in requirements.items()
420+
},
421+
simpleapi_download = mocksimpleapi_download,
422+
)
423+
424+
pypi.is_reproducible().equals(False)
425+
pypi.exposed_packages().contains_exactly({"pypi": ["torch"]})
426+
pypi.hub_group_map().contains_exactly({"pypi": {}})
427+
pypi.hub_whl_map().contains_exactly({"pypi": {
428+
"torch": {
429+
"pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [
430+
struct(
431+
config_setting = None,
432+
filename = "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl",
433+
target_platforms = None,
434+
version = "3.12",
435+
),
436+
],
437+
"pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [
438+
struct(
439+
config_setting = None,
440+
filename = "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
441+
target_platforms = None,
442+
version = "3.12",
443+
),
444+
],
445+
"pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [
446+
struct(
447+
config_setting = None,
448+
filename = "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl",
449+
target_platforms = None,
450+
version = "3.12",
451+
),
452+
],
453+
"pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [
454+
struct(
455+
config_setting = None,
456+
filename = "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl",
457+
target_platforms = None,
458+
version = "3.12",
459+
),
460+
],
461+
},
462+
}})
463+
pypi.whl_libraries().contains_exactly({
464+
"pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": {
465+
"dep_template": "@pypi//{name}:{target}",
466+
"experimental_target_platforms": ["cp312_linux_x86_64"],
467+
"filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl",
468+
"python_interpreter_target": "unit_test_interpreter_target",
469+
"repo": "pypi_312",
470+
"requirement": "torch==2.4.1+cpu",
471+
"sha256": "8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364",
472+
"urls": ["https://torch.index/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl"],
473+
},
474+
"pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": {
475+
"dep_template": "@pypi//{name}:{target}",
476+
"experimental_target_platforms": ["cp312_linux_aarch64"],
477+
"filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
478+
"python_interpreter_target": "unit_test_interpreter_target",
479+
"repo": "pypi_312",
480+
"requirement": "torch==2.4.1",
481+
"sha256": "36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a",
482+
"urls": ["https://torch.index/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"],
483+
},
484+
"pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": {
485+
"dep_template": "@pypi//{name}:{target}",
486+
"experimental_target_platforms": ["cp312_windows_x86_64"],
487+
"filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl",
488+
"python_interpreter_target": "unit_test_interpreter_target",
489+
"repo": "pypi_312",
490+
"requirement": "torch==2.4.1+cpu",
491+
"sha256": "3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97",
492+
"urls": ["https://torch.index/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl"],
493+
},
494+
"pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": {
495+
"dep_template": "@pypi//{name}:{target}",
496+
"experimental_target_platforms": ["cp312_osx_aarch64"],
497+
"filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl",
498+
"python_interpreter_target": "unit_test_interpreter_target",
499+
"repo": "pypi_312",
500+
"requirement": "torch==2.4.1",
501+
"sha256": "72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d",
502+
"urls": ["https://torch.index/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl"],
503+
},
504+
})
505+
pypi.whl_mods().contains_exactly({})
506+
507+
_tests.append(_test_torch_experimental_index_url)
508+
335509
def _test_download_only_multiple(env):
336510
pypi = _parse_modules(
337511
env,

0 commit comments

Comments
 (0)