fix(pypi): output only necessary target_platforms

CHANGELOG.md

@@ -64,6 +64,8 @@ Unreleased changes template.
     * 3.12.9
     * 3.13.2
 * (pypi) Use `xcrun xcodebuild --showsdks` to find XCode root.
+* (pypi) The `bzlmod` extension will now generate smaller lock files for  when
+  using `experimental_index_url`.
 * (toolchains) Remove all but `3.8.20` versions of the Python `3.8` interpreter who has
   reached EOL. If users still need other versions of the `3.8` interpreter, please supply
   the URLs manually {bzl:ob}`python.toolchain` or {bzl:obj}`python_register_toolchains` calls.

python/private/pypi/BUILD.bazel

@@ -93,6 +93,7 @@ bzl_library(
         ":whl_config_setting_bzl",
         ":whl_library_bzl",
         ":whl_repo_name_bzl",
+        ":whl_target_platforms_bzl",
         "//python/private:full_version_bzl",
         "//python/private:normalize_name_bzl",
         "//python/private:semver_bzl",

python/private/pypi/extension.bzl

@@ -32,6 +32,7 @@ load(":simpleapi_download.bzl", "simpleapi_download")
 load(":whl_config_setting.bzl", "whl_config_setting")
 load(":whl_library.bzl", "whl_library")
 load(":whl_repo_name.bzl", "pypi_repo_name", "whl_repo_name")
+load(":whl_target_platforms.bzl", "whl_target_platforms")
 
 def _major_minor_version(version):
     version = semver(version)
@@ -296,9 +297,18 @@ def _whl_repos(*, requirement, whl_library_args, download_only, netrc, auth_patt
 
         # Pure python wheels or sdists may need to have a platform here
         target_platforms = None
-        if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"):
-            if multiple_requirements_for_whl:
-                target_platforms = requirement.target_platforms
+        if distribution.filename.endswith(".whl") and not distribution.filename.endswith("-any.whl"):
+            parsed_whl = parse_whl_name(distribution.filename)
+            whl_platforms = whl_target_platforms(
+                platform_tag = parsed_whl.platform_tag,
+            )
+            args["experimental_target_platforms"] = [
+                p
+                for p in requirement.target_platforms
+                if [None for wp in whl_platforms if p.endswith(wp.target_platform)]
+            ]
+        elif multiple_requirements_for_whl:
+            target_platforms = requirement.target_platforms
 
         repo_name = whl_repo_name(
             distribution.filename,

tests/pypi/extension/extension_tests.bzl

@@ -17,6 +17,7 @@
 load("@rules_testing//lib:test_suite.bzl", "test_suite")
 load("@rules_testing//lib:truth.bzl", "subjects")
 load("//python/private/pypi:extension.bzl", "parse_modules")  # buildifier: disable=bzl-visibility
+load("//python/private/pypi:parse_simpleapi_html.bzl", "parse_simpleapi_html")  # buildifier: disable=bzl-visibility
 load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting")  # buildifier: disable=bzl-visibility
 
 _tests = []
@@ -332,6 +333,179 @@ torch==2.4.1 ; platform_machine != 'x86_64' \
 
 _tests.append(_test_simple_with_markers)
 
+def _test_torch_experimental_index_url(env):
+    def mocksimpleapi_download(*_, **__):
+        return {
+            "torch": parse_simpleapi_html(
+                url = "https://torch.index",
+                content = """\
+    <a href="/whl/cpu/torch-2.4.1%2Bcpu-cp310-cp310-linux_x86_64.whl#sha256=833490a28ac156762ed6adaa7c695879564fa2fd0dc51bcf3fdb2c7b47dc55e6">torch-2.4.1+cpu-cp310-cp310-linux_x86_64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1%2Bcpu-cp310-cp310-win_amd64.whl#sha256=1dd062d296fb78aa7cfab8690bf03704995a821b5ef69cfc807af5c0831b4202">torch-2.4.1+cpu-cp310-cp310-win_amd64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1%2Bcpu-cp311-cp311-linux_x86_64.whl#sha256=2b03e20f37557d211d14e3fb3f71709325336402db132a1e0dd8b47392185baf">torch-2.4.1+cpu-cp311-cp311-linux_x86_64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1%2Bcpu-cp311-cp311-win_amd64.whl#sha256=76a6fe7b10491b650c630bc9ae328df40f79a948296b41d3b087b29a8a63cbad">torch-2.4.1+cpu-cp311-cp311-win_amd64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl#sha256=8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364">torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl#sha256=3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97">torch-2.4.1+cpu-cp312-cp312-win_amd64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1%2Bcpu-cp38-cp38-linux_x86_64.whl#sha256=0c0a7cc4f7c74ff024d5a5e21230a01289b65346b27a626f6c815d94b4b8c955">torch-2.4.1+cpu-cp38-cp38-linux_x86_64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1%2Bcpu-cp38-cp38-win_amd64.whl#sha256=330e780f478707478f797fdc82c2a96e9b8c5f60b6f1f57bb6ad1dd5b1e7e97e">torch-2.4.1+cpu-cp38-cp38-win_amd64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1%2Bcpu-cp39-cp39-linux_x86_64.whl#sha256=3c99506980a2fb4b634008ccb758f42dd82f93ae2830c1e41f64536e310bf562">torch-2.4.1+cpu-cp39-cp39-linux_x86_64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1%2Bcpu-cp39-cp39-win_amd64.whl#sha256=c4f2c3c026e876d4dad7629170ec14fff48c076d6c2ae0e354ab3fdc09024f00">torch-2.4.1+cpu-cp39-cp39-win_amd64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=fa27b048d32198cda6e9cff0bf768e8683d98743903b7e5d2b1f5098ded1d343">torch-2.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1-cp310-none-macosx_11_0_arm64.whl#sha256=d36a8ef100f5bff3e9c3cea934b9e0d7ea277cb8210c7152d34a9a6c5830eadd">torch-2.4.1-cp310-none-macosx_11_0_arm64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=30be2844d0c939161a11073bfbaf645f1c7cb43f62f46cc6e4df1c119fb2a798">torch-2.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1-cp311-none-macosx_11_0_arm64.whl#sha256=ddddbd8b066e743934a4200b3d54267a46db02106876d21cf31f7da7a96f98ea">torch-2.4.1-cp311-none-macosx_11_0_arm64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a">torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl#sha256=72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d">torch-2.4.1-cp312-none-macosx_11_0_arm64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=56ad2a760b7a7882725a1eebf5657abbb3b5144eb26bcb47b52059357463c548">torch-2.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1-cp38-none-macosx_11_0_arm64.whl#sha256=5fc1d4d7ed265ef853579caf272686d1ed87cebdcd04f2a498f800ffc53dab71">torch-2.4.1-cp38-none-macosx_11_0_arm64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=1495132f30f722af1a091950088baea383fe39903db06b20e6936fd99402803e">torch-2.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl</a><br/>
+    <a href="/whl/cpu/torch-2.4.1-cp39-none-macosx_11_0_arm64.whl#sha256=a38de2803ee6050309aac032676536c3d3b6a9804248537e38e098d0e14817ec">torch-2.4.1-cp39-none-macosx_11_0_arm64.whl</a><br/>
+""",
+            ),
+        }
+
+    pypi = _parse_modules(
+        env,
+        module_ctx = _mock_mctx(
+            _mod(
+                name = "rules_python",
+                parse = [
+                    _parse(
+                        hub_name = "pypi",
+                        python_version = "3.12",
+                        experimental_index_url = "https://torch.index",
+                        requirements_lock = "universal.txt",
+                    ),
+                ],
+            ),
+            read = lambda x: {
+                "universal.txt": """\
+torch==2.4.1 ; platform_machine != 'x86_64' \
+    --hash=sha256:1495132f30f722af1a091950088baea383fe39903db06b20e6936fd99402803e \
+    --hash=sha256:30be2844d0c939161a11073bfbaf645f1c7cb43f62f46cc6e4df1c119fb2a798 \
+    --hash=sha256:36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a \
+    --hash=sha256:56ad2a760b7a7882725a1eebf5657abbb3b5144eb26bcb47b52059357463c548 \
+    --hash=sha256:5fc1d4d7ed265ef853579caf272686d1ed87cebdcd04f2a498f800ffc53dab71 \
+    --hash=sha256:72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d \
+    --hash=sha256:a38de2803ee6050309aac032676536c3d3b6a9804248537e38e098d0e14817ec \
+    --hash=sha256:d36a8ef100f5bff3e9c3cea934b9e0d7ea277cb8210c7152d34a9a6c5830eadd \
+    --hash=sha256:ddddbd8b066e743934a4200b3d54267a46db02106876d21cf31f7da7a96f98ea \
+    --hash=sha256:fa27b048d32198cda6e9cff0bf768e8683d98743903b7e5d2b1f5098ded1d343
+    # via -r requirements.in
+torch==2.4.1+cpu ; platform_machine == 'x86_64' \
+    --hash=sha256:0c0a7cc4f7c74ff024d5a5e21230a01289b65346b27a626f6c815d94b4b8c955 \
+    --hash=sha256:1dd062d296fb78aa7cfab8690bf03704995a821b5ef69cfc807af5c0831b4202 \
+    --hash=sha256:2b03e20f37557d211d14e3fb3f71709325336402db132a1e0dd8b47392185baf \
+    --hash=sha256:330e780f478707478f797fdc82c2a96e9b8c5f60b6f1f57bb6ad1dd5b1e7e97e \
+    --hash=sha256:3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97 \
+    --hash=sha256:3c99506980a2fb4b634008ccb758f42dd82f93ae2830c1e41f64536e310bf562 \
+    --hash=sha256:76a6fe7b10491b650c630bc9ae328df40f79a948296b41d3b087b29a8a63cbad \
+    --hash=sha256:833490a28ac156762ed6adaa7c695879564fa2fd0dc51bcf3fdb2c7b47dc55e6 \
+    --hash=sha256:8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364 \
+    --hash=sha256:c4f2c3c026e876d4dad7629170ec14fff48c076d6c2ae0e354ab3fdc09024f00
+    # via -r requirements.in
+""",
+            }[x],
+        ),
+        available_interpreters = {
+            "python_3_12_host": "unit_test_interpreter_target",
+        },
+        evaluate_markers = lambda _, requirements, **__: {
+            # todo once 2692 is merged, this is going to be easier to test.
+            key: [
+                platform
+                for platform in platforms
+                if ("x86_64" in platform and "platform_machine ==" in key) or ("x86_64" not in platform and "platform_machine !=" in key)
+            ]
+            for key, platforms in requirements.items()
+        },
+        simpleapi_download = mocksimpleapi_download,
+    )
+
+    pypi.is_reproducible().equals(False)
+    pypi.exposed_packages().contains_exactly({"pypi": ["torch"]})
+    pypi.hub_group_map().contains_exactly({"pypi": {}})
+    pypi.hub_whl_map().contains_exactly({"pypi": {
+        "torch": {
+            "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [
+                struct(
+                    config_setting = None,
+                    filename = "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl",
+                    target_platforms = None,
+                    version = "3.12",
+                ),
+            ],
+            "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [
+                struct(
+                    config_setting = None,
+                    filename = "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
+                    target_platforms = None,
+                    version = "3.12",
+                ),
+            ],
+            "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [
+                struct(
+                    config_setting = None,
+                    filename = "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl",
+                    target_platforms = None,
+                    version = "3.12",
+                ),
+            ],
+            "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [
+                struct(
+                    config_setting = None,
+                    filename = "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl",
+                    target_platforms = None,
+                    version = "3.12",
+                ),
+            ],
+        },
+    }})
+    pypi.whl_libraries().contains_exactly({
+        "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": {
+            "dep_template": "@pypi//{name}:{target}",
+            "experimental_target_platforms": ["cp312_linux_x86_64"],
+            "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl",
+            "python_interpreter_target": "unit_test_interpreter_target",
+            "repo": "pypi_312",
+            "requirement": "torch==2.4.1+cpu",
+            "sha256": "8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364",
+            "urls": ["https://torch.index/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl"],
+        },
+        "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": {
+            "dep_template": "@pypi//{name}:{target}",
+            "experimental_target_platforms": ["cp312_linux_aarch64"],
+            "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
+            "python_interpreter_target": "unit_test_interpreter_target",
+            "repo": "pypi_312",
+            "requirement": "torch==2.4.1",
+            "sha256": "36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a",
+            "urls": ["https://torch.index/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"],
+        },
+        "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": {
+            "dep_template": "@pypi//{name}:{target}",
+            "experimental_target_platforms": ["cp312_windows_x86_64"],
+            "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl",
+            "python_interpreter_target": "unit_test_interpreter_target",
+            "repo": "pypi_312",
+            "requirement": "torch==2.4.1+cpu",
+            "sha256": "3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97",
+            "urls": ["https://torch.index/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl"],
+        },
+        "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": {
+            "dep_template": "@pypi//{name}:{target}",
+            "experimental_target_platforms": ["cp312_osx_aarch64"],
+            "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl",
+            "python_interpreter_target": "unit_test_interpreter_target",
+            "repo": "pypi_312",
+            "requirement": "torch==2.4.1",
+            "sha256": "72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d",
+            "urls": ["https://torch.index/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl"],
+        },
+    })
+    pypi.whl_mods().contains_exactly({})
+
+_tests.append(_test_torch_experimental_index_url)
+
 def _test_download_only_multiple(env):
     pypi = _parse_modules(
         env,