diff --git a/CHANGELOG.md b/CHANGELOG.md index 55d0d3fa2f..b62898fade 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,13 @@ END_UNRELEASED_TEMPLATE length errors due to too long environment variables. * (bootstrap) {obj}`--bootstrap_impl=script` now supports the `-S` interpreter setting. +* (pypi) We now use the Minimal Version Selection (MVS) algorithm to select + the right wheel when there are multiple wheels for the target platform + (e.g. `musllinux_1_1_x86_64` and `musllinux_1_2_x86_64`). If the user + wants to set the minimum version for the selection algorithm, use the + {attr}`pip.defaults.whl_platform_tags` attribute to configure that. If + `musllinux_*_x86_64` is specified, we will chose the lowest available + wheel version. Fixes [#3250](https://github.com/bazel-contrib/rules_python/issues/3250). {#v0-0-0-added} ### Added diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 4708c8e53a..c14912c2c9 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -492,35 +492,37 @@ preference. Will always include `"any"` even if it is not specified. The items in this list can contain a single `*` character that is equivalent to matching the -latest available version component in the platform_tag. Note, if the wheel platform tag does not -have a version component, e.g. `linux_x86_64` or `win_amd64`, then `*` will act as a regular -character. - -We will always select the highest available `platform_tag` version that is compatible with the -target platform. +lowest available version component in the platform_tag. If the wheel platform tag does not +have a version component, e.g. `linux_x86_64` or `win_amd64`, then `*` will act as a regular character. :::{note} +Normally, the `*` in the matcher means that we will target the lowest platform version that we can +and will give preference to whls built targeting the older versions of the platform. If you +specify the version, then we will use the MVS (Minimal Version Selection) algorithm to select the +compatible wheel. As such, you need to keep in mind how to configure the target platforms to +select a particular wheel of your preference. + We select a single wheel and the last match will take precedence, if the platform_tag that we match has a version component (e.g. `android_x_arch`, then the version `x` will be used in the -matching algorithm). - -If the matcher you provide has `*`, then we will match a wheel with the highest available target platform, i.e. if `musllinux_1_1_arch` and `musllinux_1_2_arch` are both present, then we will select `musllinux_1_2_arch`. -Otherwise we will select the highest available version that is equal or lower to the specifier, i.e. if `manylinux_2_12` and `manylinux_2_17` wheels are present and the matcher is `manylinux_2_15`, then we will match `manylinux_2_12` but not `manylinux_2_17`. -::: - -:::{note} -The following tag prefixes should be used instead of the legacy equivalents: -* `manylinux_2_5` instead of `manylinux1` -* `manylinux_2_12` instead of `manylinux2010` -* `manylinux_2_17` instead of `manylinux2014` - -When parsing the whl filenames `rules_python` will automatically transform wheel filenames to the -latest format. +MVS matching algorithm). + +Common patterns: +* To select any versioned wheel for an ``, ``, use `_*_`, e.g. + `manylinux_2_17_x86_64`. +* To exclude versions up to `X.Y` - **submit a PR supporting this feature**. +* To exclude versions above `X.Y`, provide the full platform tag specifier, e.g. + `musllinux_1_2_x86_64`, which will ensure that no wheels with `musllinux_1_3_x86_64` or higher + are selected. ::: :::{seealso} See official [docs](https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#platform-tag) for more information. ::: +:::{versionchanged} VERSION_NEXT_FEATURE +The matching of versioned platforms have been switched to MVS (Minimal Version Selection) +algorithm for easier evaluation logic and fewer surprises. The legacy platform tags are +supported from this version without extra handling from the user. +::: """, ), } | AUTH_ATTRS diff --git a/python/private/pypi/select_whl.bzl b/python/private/pypi/select_whl.bzl index e9db1886e7..b32fc68f01 100644 --- a/python/private/pypi/select_whl.bzl +++ b/python/private/pypi/select_whl.bzl @@ -10,6 +10,21 @@ _MANYLINUX = "manylinux" _MACOSX = "macosx" _MUSLLINUX = "musllinux" +# Taken from https://peps.python.org/pep-0600/ +_LEGACY_ALIASES = { + "manylinux1_i686": "manylinux_2_5_i686", + "manylinux1_x86_64": "manylinux_2_5_x86_64", + "manylinux2010_i686": "manylinux_2_12_i686", + "manylinux2010_x86_64": "manylinux_2_12_x86_64", + "manylinux2014_aarch64": "manylinux_2_17_aarch64", + "manylinux2014_armv7l": "manylinux_2_17_armv7l", + "manylinux2014_i686": "manylinux_2_17_i686", + "manylinux2014_ppc64": "manylinux_2_17_ppc64", + "manylinux2014_ppc64le": "manylinux_2_17_ppc64le", + "manylinux2014_s390x": "manylinux_2_17_s390x", + "manylinux2014_x86_64": "manylinux_2_17_x86_64", +} + def _value_priority(*, tag, values): keys = [] for priority, wp in enumerate(values): @@ -18,17 +33,61 @@ def _value_priority(*, tag, values): return max(keys) if keys else None -def _platform_tag_priority(*, tag, values): - # Implements matching platform tag - # https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/ - - if not ( +def _is_platform_tag_versioned(tag): + return ( tag.startswith(_ANDROID) or tag.startswith(_IOS) or tag.startswith(_MACOSX) or tag.startswith(_MANYLINUX) or tag.startswith(_MUSLLINUX) - ): + ) + +def _parse_platform_tags(tags): + """A helper function that parses all of the platform tags. + + The main idea is to make this more robust and have better debug messages about which will + is compatible and which is not with the target platform. + """ + ret = [] + replacements = {} + for tag in tags: + tag = _LEGACY_ALIASES.get(tag, tag) + + if not _is_platform_tag_versioned(tag): + ret.append(tag) + continue + + want_os, sep, tail = tag.partition("_") + if not sep: + fail("could not parse the tag: {}".format(tag)) + + want_major, _, tail = tail.partition("_") + if want_major == "*": + # the expected match is any version + want_arch = tail + elif want_os.startswith(_ANDROID): + want_arch = tail + else: + # drop the minor version segment + _, _, want_arch = tail.partition("_") + + placeholder = "{}_*_{}".format(want_os, want_arch) + replacements[placeholder] = tag + if placeholder in ret: + ret.remove(placeholder) + + ret.append(placeholder) + + return [ + replacements.get(p, p) + for p in ret + ] + +def _platform_tag_priority(*, tag, values): + # Implements matching platform tag + # https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/ + + if not _is_platform_tag_versioned(tag): res = _value_priority(tag = tag, values = values) if res == None: return res @@ -39,7 +98,7 @@ def _platform_tag_priority(*, tag, values): os, _, tail = tag.partition("_") major, _, tail = tail.partition("_") - if not os.startswith(_ANDROID): + if not tag.startswith(_ANDROID): minor, _, arch = tail.partition("_") else: minor = "0" @@ -65,7 +124,7 @@ def _platform_tag_priority(*, tag, values): want_major = "" want_minor = "" want_arch = tail - elif os.startswith(_ANDROID): + elif tag.startswith(_ANDROID): # we set it to `0` above, so setting the `want_minor` her to `0` will make things # consistent. want_minor = "0" @@ -81,7 +140,7 @@ def _platform_tag_priority(*, tag, values): # if want_major is defined, then we know that we don't have a `*` in the matcher. want_version = (int(want_major), int(want_minor)) if want_major else None if not want_version or version <= want_version: - keys.append((priority, version)) + keys.append((priority, (-version[0], -version[1]))) return max(keys) if keys else None @@ -222,7 +281,7 @@ def select_whl( implementation_name = implementation_name, python_version = python_version, whl_abi_tags = whl_abi_tags, - whl_platform_tags = whl_platform_tags, + whl_platform_tags = _parse_platform_tags(whl_platform_tags), logger = logger, ) diff --git a/tests/pypi/select_whl/select_whl_tests.bzl b/tests/pypi/select_whl/select_whl_tests.bzl index 28e17ba3b3..1c28fcca5f 100644 --- a/tests/pypi/select_whl/select_whl_tests.bzl +++ b/tests/pypi/select_whl/select_whl_tests.bzl @@ -131,7 +131,6 @@ def _test_not_select_abi3(env): whl_abi_tags = ["none"], python_version = "3.13", limit = 2, - debug = True, ) _match( env, @@ -232,6 +231,34 @@ def _test_select_by_supported_cp_version(env): _tests.append(_test_select_by_supported_cp_version) +def _test_legacy_manylinux(env): + for legacy, replacement in { + "manylinux1": "manylinux_2_5", + "manylinux2010": "manylinux_2_12", + "manylinux2014": "manylinux_2_17", + }.items(): + for plat in [legacy, replacement]: + whls = [ + "pkg-0.0.1-py3-none-{}_x86_64.whl".format(plat), + "pkg-0.0.1-py3-none-any.whl", + ] + + got = _select_whl( + whls = whls, + whl_platform_tags = ["{}_x86_64".format(legacy)], + whl_abi_tags = ["none"], + python_version = "3.10", + ) + want = _select_whl( + whls = whls, + whl_platform_tags = ["{}_x86_64".format(replacement)], + whl_abi_tags = ["none"], + python_version = "3.10", + ) + _match(env, [got], want.filename) + +_tests.append(_test_legacy_manylinux) + def _test_supported_cp_version_manylinux(env): whls = [ "pkg-0.0.1-py2.py3-none-manylinux_1_1_x86_64.whl", @@ -406,9 +433,10 @@ def _test_multiple_musllinux(env): _match( env, got, - # select the one with the highest version that is matching - "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + # select the one with the lowest version that is matching because we want to + # increase the compatibility "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl", + "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl", ) _tests.append(_test_multiple_musllinux) @@ -423,17 +451,85 @@ def _test_multiple_musllinux_exact_params(env): whl_abi_tags = ["none"], python_version = "3.12", limit = 2, + debug = True, ) _match( env, got, - # select the one with the lowest version, because of the input to the function - "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl", + # 1.2 is not within the candidates because it is not compatible "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl", ) _tests.append(_test_multiple_musllinux_exact_params) +def _test_multiple_mvs_match(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-musllinux_1_4_x86_64.whl", + "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl", + "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + ], + whl_platform_tags = ["musllinux_1_3_x86_64"], + whl_abi_tags = ["none"], + python_version = "3.12", + limit = 2, + ) + _match( + env, + got, + # select the one with the lowest version + "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl", + "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + ) + +_tests.append(_test_multiple_mvs_match) + +def _test_multiple_mvs_match_override_more_specific(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-musllinux_1_4_x86_64.whl", + "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl", + "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + ], + whl_platform_tags = [ + "musllinux_*_x86_64", # default to something + "musllinux_1_3_x86_64", # override the previous + ], + whl_abi_tags = ["none"], + python_version = "3.12", + limit = 2, + ) + _match( + env, + got, + # Should be the same as without the `*` match + "pkg-0.0.1-py3-none-musllinux_1_2_x86_64.whl", + "pkg-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + ) + +_tests.append(_test_multiple_mvs_match_override_more_specific) + +def _test_multiple_mvs_match_override_less_specific(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-musllinux_1_4_x86_64.whl", + ], + whl_platform_tags = [ + "musllinux_1_3_x86_64", # default to 1.3 + "musllinux_*_x86_64", # then override to something less specific + ], + whl_abi_tags = ["none"], + python_version = "3.12", + limit = 2, + ) + _match( + env, + got, + "pkg-0.0.1-py3-none-musllinux_1_4_x86_64.whl", + ) + +_tests.append(_test_multiple_mvs_match_override_less_specific) + def _test_android(env): got = _select_whl( whls = [