fix(pypi): pull fewer wheels with experimental_index_url (#3058)

Before this we would pull all of the wheels that the user target
configuration
would be compatible with and that meant that it was not customizable.
This also
meant that there were a lot of footguns in the configuration where the
select
statements were not really foolproof.

With this PR we select only those sources that need to be for the
declared
configurations.

Freethreaded support should be done by defining extra freethreaded
platforms
using the new builder API. It is done as a followup in #3063.

This is also changing the default platforms to be only the fully
supported
platforms. This makes the testing easier and avoids us running into
compatibility issues during the rollout.

Work towards #2747
Fixes #2759
Fixes #2849

Author: aignas

CHANGELOG.md

@@ -69,6 +69,10 @@ END_UNRELEASED_TEMPLATE
 * (toolchain) Python 3.13 now references 3.13.6
 * (gazelle) Switched back to smacker/go-tree-sitter, fixing
   [#2630](https://github.com/bazel-contrib/rules_python/issues/2630)
+* (pypi) From now on the list of default platforms only includes `linux_x86_64`, `linux_aarch64`,
+  `osx_x86_64`, `osx_aarch64` and `windows_x86_64`. If you are on other platforms, you need to
+  use the `pip.default` to configure it yourself. If you are interested in graduating the
+  platform, consider helping set us up CI for them and update the documentation.
 * (ci) We are now testing on Ubuntu 22.04 for RBE and non-RBE configurations.
 * (core) `#!/usr/bin/env bash` is now used as a shebang in the stage1 bootstrap template.
 * (gazelle:docs) The Gazelle docs have been migrated from {gh-path}`gazelle/README.md` to
@@ -90,6 +94,10 @@ END_UNRELEASED_TEMPLATE
   ([#2503](https://github.com/bazel-contrib/rules_python/issues/2503)).
 * (pypi) The pipstar `defaults` configuration now supports any custom platform
   name.
+* (pypi) The selection of the whls has been changed and should no longer result
+  in ambiguous select matches ({gh-issue}`2759`) and should be much more efficient
+  when running `bazel query` due to fewer repositories being included 
+  ({gh-issue}`2849`).
 * Multi-line python imports (e.g. with escaped newlines) are now correctly processed by Gazelle.
 * (toolchains) `local_runtime_repo` works with multiarch Debian with Python 3.8
   ([#3099](https://github.com/bazel-contrib/rules_python/issues/3099)).

MODULE.bazel

@@ -74,16 +74,18 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
         env = {"platform_version": "0"},
         os_name = "linux",
         platform = "linux_{}".format(cpu),
+        whl_abi_tags = [
+            "abi3",
+            "cp{major}{minor}",
+        ],
+        whl_platform_tags = [
+            "linux_{}".format(cpu),
+            "manylinux_*_{}".format(cpu),
+        ],
     )
     for cpu in [
         "x86_64",
         "aarch64",
-        # TODO @aignas 2025-05-19: only leave tier 0-1 cpus when stabilizing the
-        # `pip.default` extension. i.e. drop the below values - users will have to
-        # define themselves if they need them.
-        "arm",
-        "ppc",
-        "s390x",
     ]
 ]
 
@@ -99,26 +101,53 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
         env = {"platform_version": "14.0"},
         os_name = "osx",
         platform = "osx_{}".format(cpu),
+        whl_abi_tags = [
+            "abi3",
+            "cp{major}{minor}",
+        ],
+        whl_platform_tags = [
+            "macosx_*_{}".format(suffix)
+            for suffix in platform_tag_cpus
+        ],
     )
-    for cpu in [
-        "aarch64",
-        "x86_64",
-    ]
+    for cpu, platform_tag_cpus in {
+        "aarch64": [
+            "universal2",
+            "arm64",
+        ],
+        "x86_64": [
+            "universal2",
+            "x86_64",
+        ],
+    }.items()
+]
+
+[
+    pip.default(
+        arch_name = cpu,
+        config_settings = [
+            "@platforms//cpu:{}".format(cpu),
+            "@platforms//os:windows",
+        ],
+        env = {"platform_version": "0"},
+        os_name = "windows",
+        platform = "windows_{}".format(cpu),
+        whl_abi_tags = [
+            "abi3",
+            "cp{major}{minor}",
+        ],
+        whl_platform_tags = whl_platform_tags,
+    )
+    for cpu, whl_platform_tags in {
+        "x86_64": ["win_amd64"],
+    }.items()
 ]
 
-pip.default(
-    arch_name = "x86_64",
-    config_settings = [
-        "@platforms//cpu:x86_64",
-        "@platforms//os:windows",
-    ],
-    env = {"platform_version": "0"},
-    os_name = "windows",
-    platform = "windows_x86_64",
-)
 pip.parse(
     # NOTE @aignas 2024-10-26: We have an integration test that depends on us
     # being able to build sdists for this hub, so explicitly set this to False.
+    #
+    # how do we test sdists? Maybe just worth adding a single sdist somewhere?
     download_only = False,
     experimental_index_url = "https://pypi.org/simple",
     hub_name = "rules_python_publish_deps",

examples/bzlmod/MODULE.bazel

@@ -158,6 +158,23 @@ pip.whl_mods(
 )
 use_repo(pip, "whl_mods_hub")
 
+# Because below we are using `windows_aarch64` platform, we have to define various
+# properties for it.
+pip.default(
+    arch_name = "aarch64",
+    config_settings = [
+        "@platforms//os:windows",
+        "@platforms//cpu:aarch64",
+    ],
+    env = {
+        "platform_version": "0",
+    },
+    os_name = "windows",
+    platform = "windows_aarch64",
+    whl_abi_tags = [],  # default to all ABIs
+    whl_platform_tags = ["win_amd64"],
+)
+
 # To fetch pip dependencies, use pip.parse. We can pass in various options,
 # but typically we pass requirements and the Python version. The Python
 # version must have been configured by a corresponding `python.toolchain()`

python/private/pypi/BUILD.bazel

@@ -116,11 +116,11 @@ bzl_library(
         ":parse_whl_name_bzl",
         ":pep508_env_bzl",
         ":pip_repository_attrs_bzl",
+        ":python_tag_bzl",
         ":simpleapi_download_bzl",
         ":whl_config_setting_bzl",
         ":whl_library_bzl",
         ":whl_repo_name_bzl",
-        ":whl_target_platforms_bzl",
         "//python/private:full_version_bzl",
         "//python/private:normalize_name_bzl",
         "//python/private:version_bzl",
@@ -209,7 +209,7 @@ bzl_library(
         ":parse_requirements_txt_bzl",
         ":pypi_repo_utils_bzl",
         ":requirements_files_by_platform_bzl",
-        ":whl_target_platforms_bzl",
+        ":select_whl_bzl",
         "//python/private:normalize_name_bzl",
         "//python/private:repo_utils_bzl",
     ],
@@ -438,5 +438,4 @@ bzl_library(
 bzl_library(
     name = "whl_target_platforms_bzl",
     srcs = ["whl_target_platforms.bzl"],
-    deps = [":parse_whl_name_bzl"],
 )

python/private/pypi/evaluate_markers.bzl

@@ -43,11 +43,11 @@ def evaluate_markers(*, requirements, platforms):
     for req_string, platform_strings in requirements.items():
         req = requirement(req_string)
         for platform_str in platform_strings:
-            env = platforms.get(platform_str)
-            if not env:
+            plat = platforms.get(platform_str)
+            if not plat:
                 fail("Please define platform: '{}'".format(platform_str))
 
-            if evaluate(req.marker, env = env):
+            if evaluate(req.marker, env = plat.env):
                 ret.setdefault(req_string, []).append(platform_str)
 
     return ret