Merge branch 'main' into fix.site.packages.overlap

Author: rickeylev

CHANGELOG.md

@@ -79,6 +79,13 @@ END_UNRELEASED_TEMPLATE
   length errors due to too long environment variables.
 * (bootstrap) {obj}`--bootstrap_impl=script` now supports the `-S` interpreter
   setting.
+* (pypi) We now use the Minimal Version Selection (MVS) algorithm to select
+  the right wheel when there are multiple wheels for the target platform
+  (e.g. `musllinux_1_1_x86_64` and `musllinux_1_2_x86_64`). If the user
+  wants to set the minimum version for the selection algorithm, use the
+  {attr}`pip.defaults.whl_platform_tags` attribute to configure that. If
+  `musllinux_*_x86_64` is specified, we will chose the lowest available
+  wheel version. Fixes [#3250](https://github.com/bazel-contrib/rules_python/issues/3250).
 * (venvs) {obj}`--vens_site_packages=yes` no longer errors when packages with
   overlapping files or directories are used together.
   ([#3204](https://github.com/bazel-contrib/rules_python/issues/3204)).

python/private/pypi/extension.bzl

@@ -492,35 +492,37 @@ preference.
 Will always  include `"any"` even if it is not specified.
 
 The items in this list can contain a single `*` character that is equivalent to matching the
-latest available version component in the platform_tag. Note, if the wheel platform tag does not
-have a version component, e.g. `linux_x86_64` or `win_amd64`, then `*` will act as a regular
-character.
-
-We will always select the highest available `platform_tag` version that is compatible with the
-target platform.
+lowest available version component in the platform_tag. If the wheel platform tag does not
+have a version component, e.g. `linux_x86_64` or `win_amd64`, then `*` will act as a regular character.
 
 :::{note}
+Normally, the `*` in the matcher means that we will target the lowest platform version that we can
+and will give preference to whls built targeting the older versions of the platform. If you
+specify the version, then we will use the MVS (Minimal Version Selection) algorithm to select the
+compatible wheel. As such, you need to keep in mind how to configure the target platforms to
+select a particular wheel of your preference.
+
 We select a single wheel and the last match will take precedence, if the platform_tag that we
 match has a version component (e.g. `android_x_arch`, then the version `x` will be used in the
-matching algorithm).
-
-If the matcher you provide has `*`, then we will match a wheel with the highest available target platform, i.e. if `musllinux_1_1_arch` and `musllinux_1_2_arch` are both present, then we will select `musllinux_1_2_arch`.
-Otherwise we will select the highest available version that is equal or lower to the specifier, i.e. if `manylinux_2_12` and `manylinux_2_17` wheels are present and the matcher is `manylinux_2_15`, then we will match `manylinux_2_12` but not `manylinux_2_17`.
-:::
-
-:::{note}
-The following tag prefixes should be used instead of the legacy equivalents:
-* `manylinux_2_5` instead of `manylinux1`
-* `manylinux_2_12` instead of `manylinux2010`
-* `manylinux_2_17` instead of `manylinux2014`
-
-When parsing the whl filenames `rules_python` will automatically transform wheel filenames to the
-latest format.
+MVS matching algorithm).
+
+Common patterns:
+* To select any versioned wheel for an `<os>`, `<arch>`, use `<os>_*_<arch>`, e.g.
+  `manylinux_2_17_x86_64`.
+* To exclude versions up to `X.Y` - **submit a PR supporting this feature**.
+* To exclude versions above `X.Y`, provide the full platform tag specifier, e.g.
+  `musllinux_1_2_x86_64`, which will ensure that no wheels with `musllinux_1_3_x86_64` or higher
+  are selected.
 :::
 
 :::{seealso}
 See official [docs](https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#platform-tag) for more information.
 :::
+:::{versionchanged} VERSION_NEXT_FEATURE
+The matching of versioned platforms have been switched to MVS (Minimal Version Selection)
+algorithm for easier evaluation logic and fewer surprises. The legacy platform tags are
+supported from this version without extra handling from the user.
+:::
 """,
     ),
 } | AUTH_ATTRS

python/private/pypi/select_whl.bzl

@@ -10,6 +10,21 @@ _MANYLINUX = "manylinux"
 _MACOSX = "macosx"
 _MUSLLINUX = "musllinux"
 
+# Taken from https://peps.python.org/pep-0600/
+_LEGACY_ALIASES = {
+    "manylinux1_i686": "manylinux_2_5_i686",
+    "manylinux1_x86_64": "manylinux_2_5_x86_64",
+    "manylinux2010_i686": "manylinux_2_12_i686",
+    "manylinux2010_x86_64": "manylinux_2_12_x86_64",
+    "manylinux2014_aarch64": "manylinux_2_17_aarch64",
+    "manylinux2014_armv7l": "manylinux_2_17_armv7l",
+    "manylinux2014_i686": "manylinux_2_17_i686",
+    "manylinux2014_ppc64": "manylinux_2_17_ppc64",
+    "manylinux2014_ppc64le": "manylinux_2_17_ppc64le",
+    "manylinux2014_s390x": "manylinux_2_17_s390x",
+    "manylinux2014_x86_64": "manylinux_2_17_x86_64",
+}
+
 def _value_priority(*, tag, values):
     keys = []
     for priority, wp in enumerate(values):
@@ -18,17 +33,61 @@ def _value_priority(*, tag, values):
 
     return max(keys) if keys else None
 
-def _platform_tag_priority(*, tag, values):
-    # Implements matching platform tag
-    # https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/
-
-    if not (
+def _is_platform_tag_versioned(tag):
+    return (
         tag.startswith(_ANDROID) or
         tag.startswith(_IOS) or
         tag.startswith(_MACOSX) or
         tag.startswith(_MANYLINUX) or
         tag.startswith(_MUSLLINUX)
-    ):
+    )
+
+def _parse_platform_tags(tags):
+    """A helper function that parses all of the platform tags.
+
+    The main idea is to make this more robust and have better debug messages about which will
+    is compatible and which is not with the target platform.
+    """
+    ret = []
+    replacements = {}
+    for tag in tags:
+        tag = _LEGACY_ALIASES.get(tag, tag)
+
+        if not _is_platform_tag_versioned(tag):
+            ret.append(tag)
+            continue
+
+        want_os, sep, tail = tag.partition("_")
+        if not sep:
+            fail("could not parse the tag: {}".format(tag))
+
+        want_major, _, tail = tail.partition("_")
+        if want_major == "*":
+            # the expected match is any version
+            want_arch = tail
+        elif want_os.startswith(_ANDROID):
+            want_arch = tail
+        else:
+            # drop the minor version segment
+            _, _, want_arch = tail.partition("_")
+
+        placeholder = "{}_*_{}".format(want_os, want_arch)
+        replacements[placeholder] = tag
+        if placeholder in ret:
+            ret.remove(placeholder)
+
+        ret.append(placeholder)
+
+    return [
+        replacements.get(p, p)
+        for p in ret
+    ]
+
+def _platform_tag_priority(*, tag, values):
+    # Implements matching platform tag
+    # https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/
+
+    if not _is_platform_tag_versioned(tag):
         res = _value_priority(tag = tag, values = values)
         if res == None:
             return res
@@ -39,7 +98,7 @@ def _platform_tag_priority(*, tag, values):
 
     os, _, tail = tag.partition("_")
     major, _, tail = tail.partition("_")
-    if not os.startswith(_ANDROID):
+    if not tag.startswith(_ANDROID):
         minor, _, arch = tail.partition("_")
     else:
         minor = "0"
@@ -65,7 +124,7 @@ def _platform_tag_priority(*, tag, values):
             want_major = ""
             want_minor = ""
             want_arch = tail
-        elif os.startswith(_ANDROID):
+        elif tag.startswith(_ANDROID):
             # we set it to `0` above, so setting the `want_minor` her to `0` will make things
             # consistent.
             want_minor = "0"
@@ -81,7 +140,7 @@ def _platform_tag_priority(*, tag, values):
         # if want_major is defined, then we know that we don't have a `*` in the matcher.
         want_version = (int(want_major), int(want_minor)) if want_major else None
         if not want_version or version <= want_version:
-            keys.append((priority, version))
+            keys.append((priority, (-version[0], -version[1])))
 
     return max(keys) if keys else None
 
@@ -222,7 +281,7 @@ def select_whl(
         implementation_name = implementation_name,
         python_version = python_version,
         whl_abi_tags = whl_abi_tags,
-        whl_platform_tags = whl_platform_tags,
+        whl_platform_tags = _parse_platform_tags(whl_platform_tags),
         logger = logger,
     )
 

tests/pypi/select_whl/select_whl_tests.bzl

@@ -131,7 +131,6 @@ def _test_not_select_abi3(env):
         whl_abi_tags = ["none"],
         python_version = "3.13",
         limit = 2,
-        debug = True,
     )
     _match(
         env,
@@ -232,6 +231,34 @@ def _test_select_by_supported_cp_version(env):
 
 _tests.append(_test_select_by_supported_cp_version)
 
+def _test_legacy_manylinux(env):
+    for legacy, replacement in {
+        "manylinux1": "manylinux_2_5",
+        "manylinux2010": "manylinux_2_12",
+        "manylinux2014": "manylinux_2_17",
+    }.items():
+        for plat in [legacy, replacement]:
+            whls = [
+                "pkg-0.0.1-py3-none-{}_x86_64.whl".format(plat),
+                "pkg-0.0.1-py3-none-any.whl",
+            ]
+
+            got = _select_whl(
+                whls = whls,
+                whl_platform_tags = ["{}_x86_64".format(legacy)],
+                whl_abi_tags = ["none"],
+                python_version = "3.10",
+            )
+            want = _select_whl(
+                whls = whls,
+                whl_platform_tags = ["{}_x86_64".format(replacement)],
+                whl_abi_tags = ["none"],
+                python_version = "3.10",
+            )
+            _match(env, [got], want.filename)
+
+_tests.append(_test_legacy_manylinux)
+
 def _test_supported_cp_version_manylinux(env):
     whls = [
         "pkg-0.0.1-py2.py3-none-manylinux_1_1_x86_64.whl",
@@ -406,9 +433,10 @@ def _test_multiple_musllinux(env):
     _match(
         env,
         got,
-        # select the one with the highest version that is matching
-        "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl",
+        # select the one with the lowest version that is matching because we want to
+        # increase the compatibility
         "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl",
+        "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl",
     )
 
 _tests.append(_test_multiple_musllinux)
@@ -423,17 +451,85 @@ def _test_multiple_musllinux_exact_params(env):
         whl_abi_tags = ["none"],
         python_version = "3.12",
         limit = 2,
+        debug = True,
     )
     _match(
         env,
         got,
-        # select the one with the lowest version, because of the input to the function
-        "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl",
+        # 1.2 is not within the candidates because it is not compatible
         "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl",
     )
 
 _tests.append(_test_multiple_musllinux_exact_params)
 
+def _test_multiple_mvs_match(env):
+    got = _select_whl(
+        whls = [
+            "pkg-0.0.1-py3-none-musllinux_1_4_x86_64.whl",
+            "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl",
+            "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl",
+        ],
+        whl_platform_tags = ["musllinux_1_3_x86_64"],
+        whl_abi_tags = ["none"],
+        python_version = "3.12",
+        limit = 2,
+    )
+    _match(
+        env,
+        got,
+        # select the one with the lowest version
+        "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl",
+        "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl",
+    )
+
+_tests.append(_test_multiple_mvs_match)
+
+def _test_multiple_mvs_match_override_more_specific(env):
+    got = _select_whl(
+        whls = [
+            "pkg-0.0.1-py3-none-musllinux_1_4_x86_64.whl",
+            "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl",
+            "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl",
+        ],
+        whl_platform_tags = [
+            "musllinux_*_x86_64",  # default to something
+            "musllinux_1_3_x86_64",  # override the previous
+        ],
+        whl_abi_tags = ["none"],
+        python_version = "3.12",
+        limit = 2,
+    )
+    _match(
+        env,
+        got,
+        # Should be the same as without the `*` match
+        "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl",
+        "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl",
+    )
+
+_tests.append(_test_multiple_mvs_match_override_more_specific)
+
+def _test_multiple_mvs_match_override_less_specific(env):
+    got = _select_whl(
+        whls = [
+            "pkg-0.0.1-py3-none-musllinux_1_4_x86_64.whl",
+        ],
+        whl_platform_tags = [
+            "musllinux_1_3_x86_64",  # default to 1.3
+            "musllinux_*_x86_64",  # then override to something less specific
+        ],
+        whl_abi_tags = ["none"],
+        python_version = "3.12",
+        limit = 2,
+    )
+    _match(
+        env,
+        got,
+        "pkg-0.0.1-py3-none-musllinux_1_4_x86_64.whl",
+    )
+
+_tests.append(_test_multiple_mvs_match_override_less_specific)
+
 def _test_android(env):
     got = _select_whl(
         whls = [