feat: freethreaded support for the builder API

aignas · aignas · commit bc8424bd442a · 2025-07-07T00:30:57.000+09:00
Stacked on #3058 This is a continuation of #3058 where we define freethreaded platforms. They need to be used only for particular python versions so I included an extra marker configuration attribute where we are using pipstar marker evaluation before using the platform. I think this in general will be a useful tool to configure only particular platforms for particular python versions Work towards #2548, since this shows how we can define custom platforms Work towards #2747 TODO: - [ ] Fix the remaining expectations in the unit tests. Maybe make the tests less brittle and define platforms for unit testing.
diff --git a/MODULE.bazel b/MODULE.bazel
@@ -70,14 +70,22 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
         config_settings = [
             "@platforms//cpu:{}".format(cpu),
             "@platforms//os:linux",
+            "//python/config_settings:_is_py_freethreaded_{}".format(
+                "yes" if freethreaded else "no",
+            ),
         ],
         env = {"platform_version": "0"},
+        marker = "python_version ~= \"3.13\"" if freethreaded else "",
         os_name = "linux",
-        platform = "linux_{}".format(cpu),
+        platform = "linux_{}{}".format(cpu, freethreaded),
         platform_tags = [
             "linux_*_{}".format(cpu),
             "manylinux_*_{}".format(cpu),
         ],
+        want_abis = [
+            "cp{0}{1}t",
+            "none",
+        ] if freethreaded else [],
     )
     for cpu in [
         "x86_64",
@@ -89,6 +97,10 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
         "ppc",
         "s390x",
     ]
+    for freethreaded in [
+        "",
+        "_freethreaded",
+    ]
 ]
 
 [
@@ -97,16 +109,24 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
         config_settings = [
             "@platforms//cpu:{}".format(cpu),
             "@platforms//os:osx",
+            "//python/config_settings:_is_py_freethreaded_{}".format(
+                "yes" if freethreaded else "no",
+            ),
         ],
         # We choose the oldest non-EOL version at the time when we release `rules_python`.
         # See https://endoflife.date/macos
         env = {"platform_version": "14.0"},
+        marker = "python_version ~= \"3.13\"" if freethreaded else "",
         os_name = "osx",
-        platform = "osx_{}".format(cpu),
+        platform = "osx_{}{}".format(cpu, freethreaded),
         platform_tags = [
             "macosx_*_{}".format(suffix)
             for suffix in platform_tag_cpus
         ],
+        want_abis = [
+            "cp{0}{1}t",
+            "none",
+        ] if freethreaded else [],
     )
     for cpu, platform_tag_cpus in {
         "aarch64": [
@@ -118,19 +138,38 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
             "x86_64",
         ],
     }.items()
+    for freethreaded in [
+        "",
+        "_freethreaded",
+    ]
+]
+
+[
+    pip.default(
+        arch_name = "x86_64",
+        config_settings = [
+            "@platforms//cpu:x86_64",
+            "@platforms//os:windows",
+            "//python/config_settings:_is_py_freethreaded_{}".format(
+                "yes" if freethreaded else "no",
+            ),
+        ],
+        env = {"platform_version": "0"},
+        marker = "python_version ~= \"3.13\"" if freethreaded else "",
+        os_name = "windows",
+        platform = "windows_x86_64{}".format(freethreaded),
+        platform_tags = ["win_amd64"],
+        want_abis = [
+            "cp{0}{1}t",
+            "none",
+        ] if freethreaded else [],
+    )
+    for freethreaded in [
+        "",
+        "_freethreaded",
+    ]
 ]
 
-pip.default(
-    arch_name = "x86_64",
-    config_settings = [
-        "@platforms//cpu:x86_64",
-        "@platforms//os:windows",
-    ],
-    env = {"platform_version": "0"},
-    os_name = "windows",
-    platform = "windows_x86_64",
-    platform_tags = ["win_amd64"],
-)
 pip.parse(
     # NOTE @aignas 2024-10-26: We have an integration test that depends on us
     # being able to build sdists for this hub, so explicitly set this to False.
diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl
@@ -30,6 +30,7 @@ load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json")
 load(":parse_requirements.bzl", "parse_requirements")
 load(":parse_whl_name.bzl", "parse_whl_name")
 load(":pep508_env.bzl", "env")
+load(":pep508_evaluate.bzl", "evaluate")
 load(":pip_repository_attrs.bzl", "ATTRS")
 load(":requirements_files_by_platform.bzl", "requirements_files_by_platform")
 load(":simpleapi_download.bzl", "simpleapi_download")
@@ -83,8 +84,13 @@ def _platforms(*, python_version, minor_mapping, config):
             os = values.os_name,
             arch = values.arch_name,
         )) | values.env
+
+        if values.marker and not evaluate(values.marker, env = env_):
+            continue
+
         platforms[key] = struct(
             env = env_,
+            tripple = "{}_{}_{}".format(abi, values.os_name, values.arch_name),
             want_abis = [
                 v.format(*python_version.split("."))
                 for v in values.want_abis
@@ -190,17 +196,19 @@ def _create_whl_repos(
         whl_group_mapping = {}
         requirement_cycles = {}
 
+    platforms = _platforms(
+        python_version = pip_attr.python_version,
+        minor_mapping = minor_mapping,
+        config = config,
+    )
+
     if evaluate_markers:
         # This is most likely unit tests
         pass
     elif config.enable_pipstar:
         evaluate_markers = lambda _, requirements: evaluate_markers_star(
             requirements = requirements,
-            platforms = _platforms(
-                python_version = pip_attr.python_version,
-                minor_mapping = minor_mapping,
-                config = config,
-            ),
+            platforms = platforms,
         )
     else:
         # NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either
@@ -219,7 +227,14 @@ def _create_whl_repos(
         # spin up a Python interpreter.
         evaluate_markers = lambda module_ctx, requirements: evaluate_markers_py(
             module_ctx,
-            requirements = requirements,
+            requirements = {
+                k: {
+                    # TODO @aignas 2025-07-06: should we leave this as is?
+                    p: platforms[p].tripple
+                    for p in plats
+                }
+                for k, plats in requirements.items()
+            },
             python_interpreter = pip_attr.python_interpreter,
             python_interpreter_target = python_interpreter_target,
             srcs = pip_attr._evaluate_markers_srcs,
@@ -235,18 +250,14 @@ def _create_whl_repos(
             requirements_osx = pip_attr.requirements_darwin,
             requirements_windows = pip_attr.requirements_windows,
             extra_pip_args = pip_attr.extra_pip_args,
-            platforms = sorted(config.platforms),  # here we only need keys
+            platforms = sorted(platforms),  # here we only need keys
             python_version = full_version(
                 version = pip_attr.python_version,
                 minor_mapping = minor_mapping,
             ),
             logger = logger,
         ),
-        platforms = _platforms(
-            python_version = pip_attr.python_version,
-            minor_mapping = minor_mapping,
-            config = config,
-        ),
+        platforms = platforms,
         extra_pip_args = pip_attr.extra_pip_args,
         get_index_urls = get_index_urls,
         evaluate_markers = evaluate_markers,
@@ -320,6 +331,17 @@ def _create_whl_repos(
                 ))
 
             whl_libraries[repo_name] = repo.args
+            if "experimental_target_platforms" in repo.args:
+                whl_libraries[repo_name] |= {
+                    "experimental_target_platforms": sorted({
+                        # TODO @aignas 2025-07-07: this should be solved in a better way
+                        # TODO @aignas 2025-07-07: add a unit test
+                        platforms[candidate].tripple.partition("_")[-1]: None
+                        for p in repo.args["experimental_target_platforms"]
+                        for candidate in platforms
+                        if candidate.endswith(p)
+                    }),
+                }
             whl_map.setdefault(whl.name, {})[repo.config_setting] = repo_name
 
     return struct(
@@ -385,7 +407,7 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net
         ),
     )
 
-def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, want_abis, platform_tags, override = False):
+def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, want_abis, platform_tags, marker, override = False):
     """Set the value in the config if the value is provided"""
     config.setdefault("platforms", {})
     if platform and (os_name or arch_name or config_settings or platform_tags or env):
@@ -406,21 +428,25 @@ def _configure(config, *, platform, os_name, arch_name, config_settings, env = {
             # the lowest priority one needs to be the first one
             platform_tags = ["any"] + platform_tags
 
+        want_abis = want_abis or [
+            "cp{0}{1}",
+            "abi3",
+            "none",
+        ]
+        env = {
+            # default to this
+            "implementation_name": "cpython",
+        } | env
+
         config["platforms"][platform] = struct(
             name = platform.replace("-", "_").lower(),
-            os_name = os_name,
             arch_name = arch_name,
             config_settings = config_settings,
-            want_abis = want_abis or [
-                "cp{0}{1}",
-                "abi3",
-                "none",
-            ],
+            env = env,
+            marker = marker,
+            os_name = os_name,
             platform_tags = platform_tags,
-            env = {
-                # default to this
-                "implementation_name": "cpython",
-            } | env,
+            want_abis = want_abis,
         )
     else:
         config["platforms"].pop(platform)
@@ -491,6 +517,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
                 env = tag.env,
                 os_name = tag.os_name,
                 platform = tag.platform,
+                marker = tag.marker,
                 platform_tags = tag.platform_tags,
                 want_abis = tag.want_abis,
                 override = mod.is_root,
@@ -823,6 +850,12 @@ Supported keys:
 ::::{note}
 This is only used if the {envvar}`RULES_PYTHON_ENABLE_PIPSTAR` is enabled.
 ::::
+""",
+    ),
+    "marker": attr.string(
+        doc = """\
+A marker which will be evaluated to disable the target platform for certain python versions. This
+is especially useful when defining freethreaded platform variants.
 """,
     ),
     # The values for PEP508 env marker evaluation during the lock file parsing
diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl
@@ -94,7 +94,12 @@ def _pip_repository_impl(rctx):
         extra_pip_args = rctx.attr.extra_pip_args,
         evaluate_markers = lambda rctx, requirements: evaluate_markers_py(
             rctx,
-            requirements = requirements,
+            requirements = {
+                # NOTE @aignas 2025-07-07: because we don't distinguish between
+                # freethreaded and non-freethreaded, it is a 1:1 mapping.
+                req: {p: p for p in plats}
+                for req, plats in requirements.items()
+            },
             python_interpreter = rctx.attr.python_interpreter,
             python_interpreter_target = rctx.attr.python_interpreter_target,
             srcs = rctx.attr._evaluate_markers_srcs,
diff --git a/python/private/pypi/requirements_files_by_platform.bzl b/python/private/pypi/requirements_files_by_platform.bzl
@@ -37,7 +37,9 @@ def _default_platforms(*, filter, platforms):
         if not prefix:
             return platforms
 
-        match = [p for p in platforms if p.startswith(prefix)]
+        match = [p for p in platforms if p.startswith(prefix) or (
+            p.startswith("cp") and p.partition("_")[-1].startswith(prefix)
+        )]
     else:
         match = [p for p in platforms if filter in p]
 
@@ -140,7 +142,7 @@ def requirements_files_by_platform(
     if logger:
         logger.debug(lambda: "Platforms from pip args: {}".format(platforms_from_args))
 
-    default_platforms = [_platform(p, python_version) for p in platforms]
+    default_platforms = platforms
 
     if platforms_from_args:
         lock_files = [
@@ -252,6 +254,6 @@ def requirements_files_by_platform(
 
     ret = {}
     for plat, file in requirements.items():
-        ret.setdefault(file, []).append(plat)
+        ret.setdefault(file, []).append(_platform(plat, python_version = python_version))
 
     return ret
diff --git a/python/private/pypi/requirements_parser/resolve_target_platforms.py b/python/private/pypi/requirements_parser/resolve_target_platforms.py
@@ -50,8 +50,8 @@ def main():
         hashes = prefix + hashes
 
         req = Requirement(entry)
-        for p in target_platforms:
-            (platform,) = Platform.from_string(p)
+        for p, tripple in target_platforms.items():
+            (platform,) = Platform.from_string(tripple)
             if not req.marker or req.marker.evaluate(platform.env_markers("")):
                 response.setdefault(requirement_line, []).append(p)
 
diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl
