feat(pypi): support freethreaded in experimental_index_url (#2460)

With this we:
* Fix the previous behaviour where `abi3` wheels would be selected when
  freethreaded builds are selected. Whilst this may work in practise
  sometimes, I am not sure it has been supported by reading PEP703.
* Start selecting `cp313t` wheels when we scan what is available on
PyPI.
* Ensure that the `whl_library` repository rule handles `cp313t` wheel
  extraction.
* Generate `cp313t` config_settings so that we can use them in
  `pkg_aliases`.
* Generate `cp313t` references in `pkg_aliases` macro.
* Add the 3.13 deps to dev_pip for testing.

Also tested by manually running:
```
$ bazel cquery --//python/config_settings:python_version=3.13 --//python/config_settings:py_freethreaded=yes 'kind("py_library rule", deps(@dev_pip//markupsafe))'
INFO: Analyzed target @@_main~pip~dev_pip//markupsafe:markupsafe (3 packages loaded, 4091 targets configured).
INFO: Found 1 target...
@@_main~pip~dev_pip_313_markupsafe_cp313_cp313t_manylinux_2_17_x86_64_c0ef13ea//:pkg (008c5a5)
$bazel build --//python/config_settings:python_version=3.13 --//python/config_settings:py_freethreaded=yes @dev_pip//markupsafe
```

Fixes #2386

Author: aignas

MODULE.bazel

@@ -121,6 +121,13 @@ dev_pip.parse(
     python_version = "3.11",
     requirements_lock = "//docs:requirements.txt",
 )
+dev_pip.parse(
+    download_only = True,
+    experimental_index_url = "https://pypi.org/simple",
+    hub_name = "dev_pip",
+    python_version = "3.13.0",
+    requirements_lock = "//docs:requirements.txt",
+)
 dev_pip.parse(
     download_only = True,
     experimental_index_url = "https://pypi.org/simple",

python/config_settings/BUILD.bazel

@@ -106,6 +106,12 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "is_py_non_freethreaded",
+    flag_values = {":py_freethreaded": FreeThreadedFlag.NO},
+    visibility = ["//visibility:public"],
+)
+
 # pip.parse related flags
 
 string_flag(

python/private/pypi/config_settings.bzl

@@ -20,20 +20,21 @@ that matches the target platform. We can leverage this fact to ensure that the
 most specialized wheels are used by default with the users being able to
 configure string_flag values to select the less specialized ones.
 
-The list of specialization of the dists goes like follows:
+The list of specialization of the dists goes like follows (cpxyt stands for freethreaded
+environments):
 * sdist
 * py*-none-any.whl
 * py*-abi3-any.whl
-* py*-cpxy-any.whl
+* py*-cpxy-any.whl or py*-cpxyt-any.whl
 * cp*-none-any.whl
 * cp*-abi3-any.whl
-* cp*-cpxy-plat.whl
+* cp*-cpxy-any.whl or cp*-cpxyt-any.whl
 * py*-none-plat.whl
 * py*-abi3-plat.whl
-* py*-cpxy-plat.whl
+* py*-cpxy-plat.whl or py*-cpxyt-plat.whl
 * cp*-none-plat.whl
 * cp*-abi3-plat.whl
-* cp*-cpxy-plat.whl
+* cp*-cpxy-plat.whl or cp*-cpxyt-plat.whl
 
 Note, that here the specialization of musl vs manylinux wheels is the same in
 order to ensure that the matching fails if the user requests for `musl` and we don't have it or vice versa.
@@ -46,19 +47,24 @@ FLAGS = struct(
     **{
         f: str(Label("//python/config_settings:" + f))
         for f in [
-            "python_version",
+            "is_pip_whl_auto",
+            "is_pip_whl_no",
+            "is_pip_whl_only",
+            "is_py_freethreaded",
+            "is_py_non_freethreaded",
             "pip_whl_glibc_version",
             "pip_whl_muslc_version",
             "pip_whl_osx_arch",
             "pip_whl_osx_version",
             "py_linux_libc",
-            "is_pip_whl_no",
-            "is_pip_whl_only",
-            "is_pip_whl_auto",
+            "python_version",
         ]
     }
 )
 
+_DEFAULT = "//conditions:default"
+_INCOMPATIBLE = "@platforms//:incompatible"
+
 # Here we create extra string flags that are just to work with the select
 # selecting the most specialized match. We don't allow the user to change
 # them.
@@ -170,52 +176,70 @@ def _dist_config_settings(*, suffix, plat_flag_values, **kwargs):
         **kwargs
     )
 
-    for name, f in [
-        ("py_none", _flags.whl_py2_py3),
-        ("py3_none", _flags.whl_py3),
-        ("py3_abi3", _flags.whl_py3_abi3),
-        ("cp3x_none", _flags.whl_pycp3x),
-        ("cp3x_abi3", _flags.whl_pycp3x_abi3),
-        ("cp3x_cp", _flags.whl_pycp3x_abicp),
+    used_flags = {}
+
+    # NOTE @aignas 2024-12-01: the abi3 is not compatible with freethreaded
+    # builds as per PEP703 (https://peps.python.org/pep-0703/#backwards-compatibility)
+    #
+    # The discussion here also reinforces this notion:
+    # https://discuss.python.org/t/pep-703-making-the-global-interpreter-lock-optional-3-12-updates/26503/99
+
+    for name, f, abi in [
+        ("py_none", _flags.whl_py2_py3, None),
+        ("py3_none", _flags.whl_py3, None),
+        ("py3_abi3", _flags.whl_py3_abi3, (FLAGS.is_py_non_freethreaded,)),
+        ("cp3x_none", _flags.whl_pycp3x, None),
+        ("cp3x_abi3", _flags.whl_pycp3x_abi3, (FLAGS.is_py_non_freethreaded,)),
+        # The below are not specializations of one another, they are variants
+        ("cp3x_cp", _flags.whl_pycp3x_abicp, (FLAGS.is_py_non_freethreaded,)),
+        ("cp3x_cpt", _flags.whl_pycp3x_abicp, (FLAGS.is_py_freethreaded,)),
     ]:
-        if f in flag_values:
+        if (f, abi) in used_flags:
             # This should never happen as all of the different whls should have
-            # unique flag values.
+            # unique flag values
             fail("BUG: the flag {} is attempted to be added twice to the list".format(f))
         else:
             flag_values[f] = ""
+            used_flags[(f, abi)] = True
 
         _dist_config_setting(
             name = "{}_any{}".format(name, suffix),
             flag_values = flag_values,
             is_pip_whl = FLAGS.is_pip_whl_only,
+            abi = abi,
             **kwargs
         )
 
     generic_flag_values = flag_values
+    generic_used_flags = used_flags
 
     for (suffix, flag_values) in plat_flag_values:
+        used_flags = {(f, None): True for f in flag_values} | generic_used_flags
         flag_values = flag_values | generic_flag_values
 
-        for name, f in [
-            ("py_none", _flags.whl_plat),
-            ("py3_none", _flags.whl_plat_py3),
-            ("py3_abi3", _flags.whl_plat_py3_abi3),
-            ("cp3x_none", _flags.whl_plat_pycp3x),
-            ("cp3x_abi3", _flags.whl_plat_pycp3x_abi3),
-            ("cp3x_cp", _flags.whl_plat_pycp3x_abicp),
+        for name, f, abi in [
+            ("py_none", _flags.whl_plat, None),
+            ("py3_none", _flags.whl_plat_py3, None),
+            ("py3_abi3", _flags.whl_plat_py3_abi3, (FLAGS.is_py_non_freethreaded,)),
+            ("cp3x_none", _flags.whl_plat_pycp3x, None),
+            ("cp3x_abi3", _flags.whl_plat_pycp3x_abi3, (FLAGS.is_py_non_freethreaded,)),
+            # The below are not specializations of one another, they are variants
+            ("cp3x_cp", _flags.whl_plat_pycp3x_abicp, (FLAGS.is_py_non_freethreaded,)),
+            ("cp3x_cpt", _flags.whl_plat_pycp3x_abicp, (FLAGS.is_py_freethreaded,)),
         ]:
-            if f in flag_values:
+            if (f, abi) in used_flags:
                 # This should never happen as all of the different whls should have
                 # unique flag values.
                 fail("BUG: the flag {} is attempted to be added twice to the list".format(f))
             else:
                 flag_values[f] = ""
+                used_flags[(f, abi)] = True
 
             _dist_config_setting(
                 name = "{}_{}".format(name, suffix),
                 flag_values = flag_values,
                 is_pip_whl = FLAGS.is_pip_whl_only,
+                abi = abi,
                 **kwargs
             )
 
@@ -285,7 +309,7 @@ def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions):
 
     return ret
 
-def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None, native = native, **kwargs):
+def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None, abi = None, native = native, **kwargs):
     """A macro to create a target that matches is_pip_whl_auto and one more value.
 
     Args:
@@ -294,6 +318,10 @@ def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None,
             `is_pip_whl_auto` when evaluating the config setting.
         is_python: The python version config_setting to match.
         python_version: The python version name.
+        abi: {type}`tuple[Label]` A collection of ABI config settings that are
+            compatible with the given dist config setting. For example, if only
+            non-freethreaded python builds are allowed, add
+            FLAGS.is_py_non_freethreaded here.
         native (struct): The struct containing alias and config_setting rules
             to use for creating the objects. Can be overridden for unit tests
             reasons.
@@ -306,9 +334,9 @@ def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None,
     native.alias(
         name = "is_cp{}_{}".format(python_version, name) if python_version else "is_{}".format(name),
         actual = select({
-            # First match by the python version
-            is_python: _name,
-            "//conditions:default": is_python,
+            # First match by the python version and then by ABI
+            is_python: _name + ("_abi" if abi else ""),
+            _DEFAULT: _INCOMPATIBLE,
         }),
         visibility = visibility,
     )
@@ -325,12 +353,23 @@ def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None,
     config_setting_name = _name + "_setting"
     native.config_setting(name = config_setting_name, **kwargs)
 
+    if abi:
+        native.alias(
+            name = _name + "_abi",
+            actual = select(
+                {k: _name for k in abi} | {
+                    _DEFAULT: _INCOMPATIBLE,
+                },
+            ),
+            visibility = visibility,
+        )
+
     # Next match by the `pip_whl` flag value and then match by the flags that
     # are intrinsic to the distribution.
     native.alias(
         name = _name,
         actual = select({
-            "//conditions:default": FLAGS.is_pip_whl_auto,
+            _DEFAULT: _INCOMPATIBLE,
             FLAGS.is_pip_whl_auto: config_setting_name,
             is_pip_whl: config_setting_name,
         }),

python/private/pypi/pkg_aliases.bzl

@@ -308,7 +308,9 @@ def get_filename_config_settings(
         else:
             py = "py3"
 
-        if parsed.abi_tag.startswith("cp"):
+        if parsed.abi_tag.startswith("cp") and parsed.abi_tag.endswith("t"):
+            abi = "cpt"
+        elif parsed.abi_tag.startswith("cp"):
             abi = "cp"
         else:
             abi = parsed.abi_tag

python/private/pypi/whl_library.bzl

@@ -287,7 +287,7 @@ def _whl_library_impl(rctx):
                 p.target_platform
                 for p in whl_target_platforms(
                     platform_tag = parsed_whl.platform_tag,
-                    abi_tag = parsed_whl.abi_tag,
+                    abi_tag = parsed_whl.abi_tag.strip("tm"),
                 )
             ]
 

python/private/pypi/whl_target_platforms.bzl

@@ -89,6 +89,10 @@ def select_whls(*, whls, want_platforms = [], logger = None):
         want_abis[abi] = None
         want_abis[abi + "m"] = None
 
+        # Also add freethreaded wheels if we find them since we started supporting them
+        _want_platforms["{}t_{}".format(abi, os_cpu)] = None
+        want_abis[abi + "t"] = None
+
     want_platforms = sorted(_want_platforms)
 
     candidates = {}

tests/pypi/config_settings/config_settings_tests.bzl

@@ -39,6 +39,7 @@ _flag = struct(
     pip_whl_osx_arch = lambda x: (str(Label("//python/config_settings:pip_whl_osx_arch")), str(x)),
     py_linux_libc = lambda x: (str(Label("//python/config_settings:py_linux_libc")), str(x)),
     python_version = lambda x: (str(Label("//python/config_settings:python_version")), str(x)),
+    py_freethreaded = lambda x: (str(Label("//python/config_settings:py_freethreaded")), str(x)),
 )
 
 def _analysis_test(*, name, dist, want, config_settings = [_flag.platform("linux_aarch64")]):
@@ -286,6 +287,38 @@ def _test_py_none_any_versioned(name):
 
 _tests.append(_test_py_none_any_versioned)
 
+def _test_cp_whl_is_not_prefered_over_py3_non_freethreaded(name):
+    _analysis_test(
+        name = name,
+        dist = {
+            "is_cp3.7_cp3x_abi3_any": "py3_abi3",
+            "is_cp3.7_cp3x_cpt_any": "cp",
+            "is_cp3.7_cp3x_none_any": "py3",
+        },
+        want = "py3_abi3",
+        config_settings = [
+            _flag.py_freethreaded("no"),
+        ],
+    )
+
+_tests.append(_test_cp_whl_is_not_prefered_over_py3_non_freethreaded)
+
+def _test_cp_whl_is_not_prefered_over_py3_freethreaded(name):
+    _analysis_test(
+        name = name,
+        dist = {
+            "is_cp3.7_cp3x_abi3_any": "py3_abi3",
+            "is_cp3.7_cp3x_cp_any": "cp",
+            "is_cp3.7_cp3x_none_any": "py3",
+        },
+        want = "py3",
+        config_settings = [
+            _flag.py_freethreaded("yes"),
+        ],
+    )
+
+_tests.append(_test_cp_whl_is_not_prefered_over_py3_freethreaded)
+
 def _test_cp_cp_whl(name):
     _analysis_test(
         name = name,
@@ -412,6 +445,7 @@ def _test_windows(name):
         name = name,
         dist = {
             "is_cp3.7_cp3x_cp_windows_x86_64": "whl",
+            "is_cp3.7_cp3x_cpt_windows_x86_64": "whl_freethreaded",
         },
         want = "whl",
         config_settings = [
@@ -421,6 +455,22 @@ def _test_windows(name):
 
 _tests.append(_test_windows)
 
+def _test_windows_freethreaded(name):
+    _analysis_test(
+        name = name,
+        dist = {
+            "is_cp3.7_cp3x_cp_windows_x86_64": "whl",
+            "is_cp3.7_cp3x_cpt_windows_x86_64": "whl_freethreaded",
+        },
+        want = "whl_freethreaded",
+        config_settings = [
+            _flag.platform("windows_x86_64"),
+            _flag.py_freethreaded("yes"),
+        ],
+    )
+
+_tests.append(_test_windows_freethreaded)
+
 def _test_osx(name):
     _analysis_test(
         name = name,

tests/pypi/pkg_aliases/pkg_aliases_test.bzl

@@ -287,6 +287,14 @@ def _test_multiplatform_whl_aliases_filename(env):
             filename = "foo-0.0.1-py3-none-any.whl",
             version = "3.1",
         ): "foo-py3-0.0.1",
+        whl_config_setting(
+            filename = "foo-0.0.1-cp313-cp313-any.whl",
+            version = "3.1",
+        ): "foo-cp-0.0.1",
+        whl_config_setting(
+            filename = "foo-0.0.1-cp313-cp313t-any.whl",
+            version = "3.1",
+        ): "foo-cpt-0.0.1",
         whl_config_setting(
             filename = "foo-0.0.2-py3-none-any.whl",
             version = "3.1",
@@ -303,6 +311,8 @@ def _test_multiplatform_whl_aliases_filename(env):
         osx_versions = [],
     )
     want = {
+        "//_config:is_cp3.1_cp3x_cp_any": "foo-cp-0.0.1",
+        "//_config:is_cp3.1_cp3x_cpt_any": "foo-cpt-0.0.1",
         "//_config:is_cp3.1_py3_none_any": "foo-py3-0.0.1",
         "//_config:is_cp3.1_py3_none_any_linux_aarch64": "foo-0.0.2",
         "//_config:is_cp3.1_py3_none_any_linux_x86_64": "foo-0.0.2",

tests/pypi/whl_target_platforms/select_whl_tests.bzl

@@ -27,6 +27,10 @@ WHL_LIST = [
     "pkg-0.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl",
     "pkg-0.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
     "pkg-0.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl",
+    "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl",
+    "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl",
+    "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl",
+    "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl",
     "pkg-0.0.1-cp311-cp311-musllinux_1_1_aarch64.whl",
     "pkg-0.0.1-cp311-cp311-musllinux_1_1_i686.whl",
     "pkg-0.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl",
@@ -269,6 +273,22 @@ def _test_prefer_manylinux_wheels(env):
 
 _tests.append(_test_prefer_manylinux_wheels)
 
+def _test_freethreaded_wheels(env):
+    # Check we prefer platform specific wheels
+    got = _select_whls(whls = WHL_LIST, want_platforms = ["cp313_linux_x86_64"])
+    _match(
+        env,
+        got,
+        "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl",
+        "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl",
+        "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl",
+        "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl",
+        "pkg-0.0.1-cp39-abi3-any.whl",
+        "pkg-0.0.1-py3-none-any.whl",
+    )
+
+_tests.append(_test_freethreaded_wheels)
+
 def select_whl_test_suite(name):
     """Create the test suite.