feat: freethreaded support for the builder API

aignas · aignas · commit ce4dd591a347 · 2025-07-06T13:03:09.000+09:00
Stacked on bazel-contrib#3058 This is a continuation of bazel-contrib#3058 where we define freethreaded platforms. They need to be used only for particular python versions so I included an extra marker configuration attribute where we are using pipstar marker evaluation before using the platform. I think this in general will be a useful tool to configure only particular platforms for particular python versions Work towards bazel-contrib#2548, since this shows how we can define custom platforms Work towards bazel-contrib#2747 TODO: - [ ] Fix the remaining expectations in the unit tests. Maybe make the tests less brittle and define platforms for unit testing.
diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl
@@ -30,6 +30,7 @@ load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json")
 load(":parse_requirements.bzl", "parse_requirements")
 load(":parse_whl_name.bzl", "parse_whl_name")
 load(":pep508_env.bzl", "env")
+load(":pep508_evaluate.bzl", "evaluate")
 load(":pip_repository_attrs.bzl", "ATTRS")
 load(":requirements_files_by_platform.bzl", "requirements_files_by_platform")
 load(":simpleapi_download.bzl", "simpleapi_download")
@@ -83,6 +84,10 @@ def _platforms(*, python_version, minor_mapping, config):
             os = values.os_name,
             arch = values.arch_name,
         )) | values.env
+
+        if values.marker and not evaluate(values.marker, env = env_):
+            continue
+
         platforms[key] = struct(
             env = env_,
             want_abis = [
@@ -190,17 +195,19 @@ def _create_whl_repos(
         whl_group_mapping = {}
         requirement_cycles = {}
 
+    platforms = _platforms(
+        python_version = pip_attr.python_version,
+        minor_mapping = minor_mapping,
+        config = config,
+    )
+
     if evaluate_markers:
         # This is most likely unit tests
         pass
     elif config.enable_pipstar:
         evaluate_markers = lambda _, requirements: evaluate_markers_star(
             requirements = requirements,
-            platforms = _platforms(
-                python_version = pip_attr.python_version,
-                minor_mapping = minor_mapping,
-                config = config,
-            ),
+            platforms = platforms,
         )
     else:
         # NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either
@@ -235,18 +242,14 @@ def _create_whl_repos(
             requirements_osx = pip_attr.requirements_darwin,
             requirements_windows = pip_attr.requirements_windows,
             extra_pip_args = pip_attr.extra_pip_args,
-            platforms = sorted(config.platforms),  # here we only need keys
+            platforms = sorted(platforms),  # here we only need keys
             python_version = full_version(
                 version = pip_attr.python_version,
                 minor_mapping = minor_mapping,
             ),
             logger = logger,
         ),
-        platforms = _platforms(
-            python_version = pip_attr.python_version,
-            minor_mapping = minor_mapping,
-            config = config,
-        ),
+        platforms = platforms,
         extra_pip_args = pip_attr.extra_pip_args,
         get_index_urls = get_index_urls,
         evaluate_markers = evaluate_markers,
@@ -385,7 +388,7 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net
         ),
     )
 
-def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, want_abis, platform_tags, override = False):
+def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, want_abis, platform_tags, marker, override = False):
     """Set the value in the config if the value is provided"""
     config.setdefault("platforms", {})
     if platform and (os_name or arch_name or config_settings or platform_tags or env):
@@ -406,21 +409,25 @@ def _configure(config, *, platform, os_name, arch_name, config_settings, env = {
             # the lowest priority one needs to be the first one
             platform_tags = ["any"] + platform_tags
 
+        want_abis = want_abis or [
+            "cp{0}{1}",
+            "abi3",
+            "none",
+        ]
+        env = {
+            # default to this
+            "implementation_name": "cpython",
+        } | env
+
         config["platforms"][platform] = struct(
             name = platform.replace("-", "_").lower(),
-            os_name = os_name,
             arch_name = arch_name,
             config_settings = config_settings,
-            want_abis = want_abis or [
-                "cp{0}{1}",
-                "abi3",
-                "none",
-            ],
+            env = env,
+            marker = marker,
+            os_name = os_name,
             platform_tags = platform_tags,
-            env = {
-                # default to this
-                "implementation_name": "cpython",
-            } | env,
+            want_abis = want_abis,
         )
     else:
         config["platforms"].pop(platform)
@@ -432,79 +439,106 @@ def _set_defaults(defaults):
     a little problematic.
     """
 
-    # NOTE: We have this so that it is easier to maintain unit tests assuming certain
-    # defaults
-    for cpu in [
-        "x86_64",
-        "aarch64",
-        # TODO @aignas 2025-05-19: only leave tier 0-1 cpus when stabilizing the
-        # `pip.default` extension. i.e. drop the below values - users will have to
-        # define themselves if they need them.
-        "arm",
-        "ppc",
-        "s390x",
-    ]:
-        _configure(
-            defaults,
-            arch_name = cpu,
-            os_name = "linux",
-            platform = "linux_{}".format(cpu),
-            want_abis = [],
-            config_settings = [
-                "@platforms//os:linux",
-                "@platforms//cpu:{}".format(cpu),
+    for suffix, config in {
+        "": struct(
+            want_abis = [
+                "cp{0}{1}",
+                "abi3",
+                "none",
             ],
-            platform_tags = [
-                "linux_*_{}".format(cpu),
-                "manylinux_*_{}".format(cpu),
+            freethreaded = "no",
+            marker = "",
+        ),
+        "_freethreaded": struct(
+            want_abis = [
+                "cp{0}{1}t",
+                "none",
             ],
-            env = {
-                "platform_version": "0",
-            },
-        )
-    for cpu, platform_tag_cpus in {
-        "aarch64": ["universal2", "arm64"],
-        "x86_64": ["universal2", "x86_64"],
+            freethreaded = "yes",
+            marker = "python_version ~= \"3.13\"",
+        ),
     }.items():
-        _configure(
-            defaults,
-            arch_name = cpu,
-            os_name = "osx",
-            platform = "osx_{}".format(cpu),
-            config_settings = [
-                "@platforms//os:osx",
-                "@platforms//cpu:{}".format(cpu),
-            ],
-            want_abis = [],
-            platform_tags = [
-                "macosx_*_{}".format(suffix)
-                for suffix in platform_tag_cpus
-            ],
-            # We choose the oldest non-EOL version at the time when we release `rules_python`.
-            # See https://endoflife.date/macos
-            env = {
-                "platform_version": "14.0",
-            },
-        )
+        freethreaded_flag = Label("//python/config_settings:_is_py_freethreaded_{}".format(config.freethreaded))
+
+        # NOTE: We have this so that it is easier to maintain unit tests assuming certain
+        # defaults
+        for cpu in [
+            "x86_64",
+            "aarch64",
+            # TODO @aignas 2025-05-19: only leave tier 0-1 cpus when stabilizing the
+            # `pip.default` extension. i.e. drop the below values - users will have to
+            # define themselves if they need them.
+            "arm",
+            "ppc",
+            "s390x",
+        ]:
+            _configure(
+                defaults,
+                arch_name = cpu,
+                os_name = "linux",
+                platform = "linux_{}{}".format(cpu, suffix),
+                want_abis = config.want_abis,
+                marker = config.marker,
+                config_settings = [
+                    "@platforms//os:linux",
+                    "@platforms//cpu:{}".format(cpu),
+                    freethreaded_flag,
+                ],
+                platform_tags = [
+                    "linux_*_{}".format(cpu),
+                    "manylinux_*_{}".format(cpu),
+                ],
+                env = {
+                    "platform_version": "0",
+                },
+            )
+        for cpu, platform_tag_cpus in {
+            "aarch64": ["universal2", "arm64"],
+            "x86_64": ["universal2", "x86_64"],
+        }.items():
+            _configure(
+                defaults,
+                arch_name = cpu,
+                os_name = "osx",
+                platform = "osx_{}{}".format(cpu, suffix),
+                config_settings = [
+                    "@platforms//os:osx",
+                    "@platforms//cpu:{}".format(cpu),
+                    freethreaded_flag,
+                ],
+                want_abis = config.want_abis,
+                marker = config.marker,
+                platform_tags = [
+                    "macosx_*_{}".format(suffix)
+                    for suffix in platform_tag_cpus
+                ],
+                # We choose the oldest non-EOL version at the time when we release `rules_python`.
+                # See https://endoflife.date/macos
+                env = {
+                    "platform_version": "14.0",
+                },
+            )
 
-    for cpu, platform_tags in {
-        "x86_64": ["win_amd64"],
-    }.items():
-        _configure(
-            defaults,
-            arch_name = cpu,
-            os_name = "windows",
-            platform = "windows_{}".format(cpu),
-            config_settings = [
-                "@platforms//os:windows",
-                "@platforms//cpu:{}".format(cpu),
-            ],
-            want_abis = [],
-            platform_tags = platform_tags,
-            env = {
-                "platform_version": "0",
-            },
-        )
+        for cpu, platform_tags in {
+            "x86_64": ["win_amd64"],
+        }.items():
+            _configure(
+                defaults,
+                arch_name = cpu,
+                os_name = "windows",
+                platform = "windows_{}{}".format(cpu, suffix),
+                config_settings = [
+                    "@platforms//os:windows",
+                    "@platforms//cpu:{}".format(cpu),
+                    freethreaded_flag,
+                ],
+                want_abis = config.want_abis,
+                marker = config.marker,
+                platform_tags = platform_tags,
+                env = {
+                    "platform_version": "0",
+                },
+            )
 
 def parse_modules(
         module_ctx,
@@ -573,6 +607,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
                 env = tag.env,
                 os_name = tag.os_name,
                 platform = tag.platform,
+                marker = tag.marker,
                 platform_tags = tag.platform_tags,
                 want_abis = tag.want_abis,
                 override = mod.is_root,
@@ -905,6 +940,12 @@ Supported keys:
 ::::{note}
 This is only used if the {envvar}`RULES_PYTHON_ENABLE_PIPSTAR` is enabled.
 ::::
+""",
+    ),
+    "marker": attr.string(
+        doc = """\
+A marker which will be evaluated to disable the target platform for certain python versions. This
+is especially useful when defining freethreaded platform variants.
 """,
     ),
     # The values for PEP508 env marker evaluation during the lock file parsing
diff --git a/python/private/pypi/requirements_files_by_platform.bzl b/python/private/pypi/requirements_files_by_platform.bzl
@@ -37,7 +37,9 @@ def _default_platforms(*, filter, platforms):
         if not prefix:
             return platforms
 
-        match = [p for p in platforms if p.startswith(prefix)]
+        # TODO @aignas 2025-07-06: this assumes that the platforms start with
+        # cp313.3_<suffix>
+        match = [p for p in platforms if p.partition("_")[-1].startswith(prefix)]
     else:
         match = [p for p in platforms if filter in p]
 
@@ -140,7 +142,7 @@ def requirements_files_by_platform(
     if logger:
         logger.debug(lambda: "Platforms from pip args: {}".format(platforms_from_args))
 
-    default_platforms = [_platform(p, python_version) for p in platforms]
+    default_platforms = platforms
 
     if platforms_from_args:
         lock_files = [
diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl
@@ -84,6 +84,7 @@ def _default(
         platform = None,
         platform_tags = None,
         env = None,
+        marker = None,
         want_abis = None):
     return struct(
         arch_name = arch_name,
@@ -92,6 +93,7 @@ def _default(
         platform_tags = platform_tags or [],
         config_settings = config_settings,
         env = env or {},
+        marker = marker or "",
         want_abis = want_abis or [],
     )
 
@@ -1068,7 +1070,7 @@ def _test_pipstar_platforms(env):
                         ("osx", "aarch64"),
                     ]
                 ] + [
-                    _default(platform = name)
+                    _default(platform = name + suffix)
                     for name in [
                         "linux_x86_64",
                         "linux_aarch64",
@@ -1079,6 +1081,7 @@ def _test_pipstar_platforms(env):
                         "osx_aarch64",
                         "windows_x86_64",
                     ]
+                    for suffix in ["", "_freethreaded"]
                 ],
                 parse = [
                     _parse(
