diff --git a/CHANGELOG.md b/CHANGELOG.md
index 80466fc3f9..9dedf6fca2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -64,6 +64,8 @@ Unreleased changes template.
* 3.12.9
* 3.13.2
* (pypi) Use `xcrun xcodebuild --showsdks` to find XCode root.
+* (pypi) The `bzlmod` extension will now generate smaller lock files for when
+ using `experimental_index_url`.
* (toolchains) Remove all but `3.8.20` versions of the Python `3.8` interpreter who has
reached EOL. If users still need other versions of the `3.8` interpreter, please supply
the URLs manually {bzl:ob}`python.toolchain` or {bzl:obj}`python_register_toolchains` calls.
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
index 6f80272af6..79eb4dba46 100644
--- a/python/private/pypi/BUILD.bazel
+++ b/python/private/pypi/BUILD.bazel
@@ -93,6 +93,7 @@ bzl_library(
":whl_config_setting_bzl",
":whl_library_bzl",
":whl_repo_name_bzl",
+ ":whl_target_platforms_bzl",
"//python/private:full_version_bzl",
"//python/private:normalize_name_bzl",
"//python/private:semver_bzl",
diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl
index be00bf8ab3..be3067d04a 100644
--- a/python/private/pypi/extension.bzl
+++ b/python/private/pypi/extension.bzl
@@ -32,6 +32,7 @@ load(":simpleapi_download.bzl", "simpleapi_download")
load(":whl_config_setting.bzl", "whl_config_setting")
load(":whl_library.bzl", "whl_library")
load(":whl_repo_name.bzl", "pypi_repo_name", "whl_repo_name")
+load(":whl_target_platforms.bzl", "whl_target_platforms")
def _major_minor_version(version):
version = semver(version)
@@ -296,9 +297,18 @@ def _whl_repos(*, requirement, whl_library_args, download_only, netrc, auth_patt
# Pure python wheels or sdists may need to have a platform here
target_platforms = None
- if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"):
- if multiple_requirements_for_whl:
- target_platforms = requirement.target_platforms
+ if distribution.filename.endswith(".whl") and not distribution.filename.endswith("-any.whl"):
+ parsed_whl = parse_whl_name(distribution.filename)
+ whl_platforms = whl_target_platforms(
+ platform_tag = parsed_whl.platform_tag,
+ )
+ args["experimental_target_platforms"] = [
+ p
+ for p in requirement.target_platforms
+ if [None for wp in whl_platforms if p.endswith(wp.target_platform)]
+ ]
+ elif multiple_requirements_for_whl:
+ target_platforms = requirement.target_platforms
repo_name = whl_repo_name(
distribution.filename,
diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl
index 8c01a02271..1b18d2a339 100644
--- a/tests/pypi/extension/extension_tests.bzl
+++ b/tests/pypi/extension/extension_tests.bzl
@@ -17,6 +17,7 @@
load("@rules_testing//lib:test_suite.bzl", "test_suite")
load("@rules_testing//lib:truth.bzl", "subjects")
load("//python/private/pypi:extension.bzl", "parse_modules") # buildifier: disable=bzl-visibility
+load("//python/private/pypi:parse_simpleapi_html.bzl", "parse_simpleapi_html") # buildifier: disable=bzl-visibility
load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility
_tests = []
@@ -332,6 +333,179 @@ torch==2.4.1 ; platform_machine != 'x86_64' \
_tests.append(_test_simple_with_markers)
+def _test_torch_experimental_index_url(env):
+ def mocksimpleapi_download(*_, **__):
+ return {
+ "torch": parse_simpleapi_html(
+ url = "https://torch.index",
+ content = """\
+ torch-2.4.1+cpu-cp310-cp310-linux_x86_64.whl
+ torch-2.4.1+cpu-cp310-cp310-win_amd64.whl
+ torch-2.4.1+cpu-cp311-cp311-linux_x86_64.whl
+ torch-2.4.1+cpu-cp311-cp311-win_amd64.whl
+ torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl
+ torch-2.4.1+cpu-cp312-cp312-win_amd64.whl
+ torch-2.4.1+cpu-cp38-cp38-linux_x86_64.whl
+ torch-2.4.1+cpu-cp38-cp38-win_amd64.whl
+ torch-2.4.1+cpu-cp39-cp39-linux_x86_64.whl
+ torch-2.4.1+cpu-cp39-cp39-win_amd64.whl
+ torch-2.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
+ torch-2.4.1-cp310-none-macosx_11_0_arm64.whl
+ torch-2.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
+ torch-2.4.1-cp311-none-macosx_11_0_arm64.whl
+ torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
+ torch-2.4.1-cp312-none-macosx_11_0_arm64.whl
+ torch-2.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
+ torch-2.4.1-cp38-none-macosx_11_0_arm64.whl
+ torch-2.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
+ torch-2.4.1-cp39-none-macosx_11_0_arm64.whl
+""",
+ ),
+ }
+
+ pypi = _parse_modules(
+ env,
+ module_ctx = _mock_mctx(
+ _mod(
+ name = "rules_python",
+ parse = [
+ _parse(
+ hub_name = "pypi",
+ python_version = "3.12",
+ experimental_index_url = "https://torch.index",
+ requirements_lock = "universal.txt",
+ ),
+ ],
+ ),
+ read = lambda x: {
+ "universal.txt": """\
+torch==2.4.1 ; platform_machine != 'x86_64' \
+ --hash=sha256:1495132f30f722af1a091950088baea383fe39903db06b20e6936fd99402803e \
+ --hash=sha256:30be2844d0c939161a11073bfbaf645f1c7cb43f62f46cc6e4df1c119fb2a798 \
+ --hash=sha256:36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a \
+ --hash=sha256:56ad2a760b7a7882725a1eebf5657abbb3b5144eb26bcb47b52059357463c548 \
+ --hash=sha256:5fc1d4d7ed265ef853579caf272686d1ed87cebdcd04f2a498f800ffc53dab71 \
+ --hash=sha256:72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d \
+ --hash=sha256:a38de2803ee6050309aac032676536c3d3b6a9804248537e38e098d0e14817ec \
+ --hash=sha256:d36a8ef100f5bff3e9c3cea934b9e0d7ea277cb8210c7152d34a9a6c5830eadd \
+ --hash=sha256:ddddbd8b066e743934a4200b3d54267a46db02106876d21cf31f7da7a96f98ea \
+ --hash=sha256:fa27b048d32198cda6e9cff0bf768e8683d98743903b7e5d2b1f5098ded1d343
+ # via -r requirements.in
+torch==2.4.1+cpu ; platform_machine == 'x86_64' \
+ --hash=sha256:0c0a7cc4f7c74ff024d5a5e21230a01289b65346b27a626f6c815d94b4b8c955 \
+ --hash=sha256:1dd062d296fb78aa7cfab8690bf03704995a821b5ef69cfc807af5c0831b4202 \
+ --hash=sha256:2b03e20f37557d211d14e3fb3f71709325336402db132a1e0dd8b47392185baf \
+ --hash=sha256:330e780f478707478f797fdc82c2a96e9b8c5f60b6f1f57bb6ad1dd5b1e7e97e \
+ --hash=sha256:3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97 \
+ --hash=sha256:3c99506980a2fb4b634008ccb758f42dd82f93ae2830c1e41f64536e310bf562 \
+ --hash=sha256:76a6fe7b10491b650c630bc9ae328df40f79a948296b41d3b087b29a8a63cbad \
+ --hash=sha256:833490a28ac156762ed6adaa7c695879564fa2fd0dc51bcf3fdb2c7b47dc55e6 \
+ --hash=sha256:8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364 \
+ --hash=sha256:c4f2c3c026e876d4dad7629170ec14fff48c076d6c2ae0e354ab3fdc09024f00
+ # via -r requirements.in
+""",
+ }[x],
+ ),
+ available_interpreters = {
+ "python_3_12_host": "unit_test_interpreter_target",
+ },
+ evaluate_markers = lambda _, requirements, **__: {
+ # todo once 2692 is merged, this is going to be easier to test.
+ key: [
+ platform
+ for platform in platforms
+ if ("x86_64" in platform and "platform_machine ==" in key) or ("x86_64" not in platform and "platform_machine !=" in key)
+ ]
+ for key, platforms in requirements.items()
+ },
+ simpleapi_download = mocksimpleapi_download,
+ )
+
+ pypi.is_reproducible().equals(False)
+ pypi.exposed_packages().contains_exactly({"pypi": ["torch"]})
+ pypi.hub_group_map().contains_exactly({"pypi": {}})
+ pypi.hub_whl_map().contains_exactly({"pypi": {
+ "torch": {
+ "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [
+ struct(
+ config_setting = None,
+ filename = "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl",
+ target_platforms = None,
+ version = "3.12",
+ ),
+ ],
+ "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [
+ struct(
+ config_setting = None,
+ filename = "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
+ target_platforms = None,
+ version = "3.12",
+ ),
+ ],
+ "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [
+ struct(
+ config_setting = None,
+ filename = "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl",
+ target_platforms = None,
+ version = "3.12",
+ ),
+ ],
+ "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [
+ struct(
+ config_setting = None,
+ filename = "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl",
+ target_platforms = None,
+ version = "3.12",
+ ),
+ ],
+ },
+ }})
+ pypi.whl_libraries().contains_exactly({
+ "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": {
+ "dep_template": "@pypi//{name}:{target}",
+ "experimental_target_platforms": ["cp312_linux_x86_64"],
+ "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_312",
+ "requirement": "torch==2.4.1+cpu",
+ "sha256": "8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364",
+ "urls": ["https://torch.index/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl"],
+ },
+ "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": {
+ "dep_template": "@pypi//{name}:{target}",
+ "experimental_target_platforms": ["cp312_linux_aarch64"],
+ "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_312",
+ "requirement": "torch==2.4.1",
+ "sha256": "36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a",
+ "urls": ["https://torch.index/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"],
+ },
+ "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": {
+ "dep_template": "@pypi//{name}:{target}",
+ "experimental_target_platforms": ["cp312_windows_x86_64"],
+ "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_312",
+ "requirement": "torch==2.4.1+cpu",
+ "sha256": "3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97",
+ "urls": ["https://torch.index/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl"],
+ },
+ "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": {
+ "dep_template": "@pypi//{name}:{target}",
+ "experimental_target_platforms": ["cp312_osx_aarch64"],
+ "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_312",
+ "requirement": "torch==2.4.1",
+ "sha256": "72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d",
+ "urls": ["https://torch.index/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl"],
+ },
+ })
+ pypi.whl_mods().contains_exactly({})
+
+_tests.append(_test_torch_experimental_index_url)
+
def _test_download_only_multiple(env):
pypi = _parse_modules(
env,