feat(toolchain): support freethreaded toolchains (#2372)

aignas · web-flow · commit 2af0020b38d0 · 2024-11-11T04:19:12.000Z
Before this PR freethreaded toolchains were not possible to be used, this adds the minimum plumbing to get the things working. Coverage support is also added. Whilst at it: - Add plumbing to print checksums only for a particular python version. - Bump the remaining toolchain versions that used to use the 20241008 release - Pass around the loaded platform list so that we are only defining toolchains for the platforms that we have loaded the hermetic toolchain for. Tested: ``` $ bazel run --//python/config_settings:python_version=3.13.0 --//python/config_settings:py_freethreaded="yes" //python/private:current_interpreter_executable ... Python 3.13.0 experimental free-threading build (main, Oct 16 2024, 03:26:14) [Clang 18.1.8 ] on linux Type "help", "copyright", "credits" or "license" for more information. >>> ``` Closes #2129. Work towards #2386.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -16,6 +16,10 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0  # Use the ref you want to point at
+    hooks:
+      - id: check-merge-conflict
   - repo: https://github.com/keith/pre-commit-buildifier
     rev: 6.1.0
     hooks:
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -28,14 +28,24 @@ A brief description of the categories of changes:
 {#v0-0-0-changed}
 ### Changed
 * (deps) bazel_skylib 1.6.1 -> 1.7.1
+* (toolchains) Use the latest indygreg toolchain release [20241016] for Python versions:
+    * 3.9.20
+    * 3.10.15
+    * 3.11.10
+    * 3.12.7
+    * 3.13.0
+
+[20241016]: https://github.com/indygreg/python-build-standalone/releases/tag/20241016
 
 {#v0-0-0-fixed}
 ### Fixed
 * Nothing yet
 
 {#v0-0-0-added}
 ### Added
-* Nothing yet
+* (toolchain) Support for freethreaded Python toolchains is now available. Use
+  the config flag `//python/config_settings:py_freethreaded` to toggle the
+  selection of the free-threaded toolchains.
 
 {#v0-0-0-removed}
 ### Removed
diff --git a/docs/api/rules_python/python/config_settings/index.md b/docs/api/rules_python/python/config_settings/index.md
@@ -149,6 +149,16 @@ Values:
 :::
 ::::
 
+::::{bzl:flag} py_freethreaded
+Set whether to use an interpreter with the experimental freethreaded option set to true.
+
+Values:
+* `no`: Use regular Python toolchains, default.
+* `yes`: Use the experimental Python toolchain with freethreaded compile option enabled.
+:::{versionadded} 0.38.0
+:::
+::::
+
 ::::{bzl:flag} pip_whl
 Set what distributions are used in the `pip` integration.
 
diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel
@@ -5,6 +5,7 @@ load(
     "AddSrcsToRunfilesFlag",
     "BootstrapImplFlag",
     "ExecToolsToolchainFlag",
+    "FreeThreadedFlag",
     "PrecompileFlag",
     "PrecompileSourceRetentionFlag",
 )
@@ -92,6 +93,19 @@ string_flag(
     visibility = ["//visibility:public"],
 )
 
+string_flag(
+    name = "py_freethreaded",
+    build_setting_default = FreeThreadedFlag.NO,
+    values = sorted(FreeThreadedFlag.__members__.values()),
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "is_py_freethreaded",
+    flag_values = {":py_freethreaded": FreeThreadedFlag.YES},
+    visibility = ["//visibility:public"],
+)
+
 # pip.parse related flags
 
 string_flag(
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
@@ -256,6 +256,7 @@ bzl_library(
     deps = [
         ":py_toolchain_suite_bzl",
         ":text_util_bzl",
+        "//python:versions_bzl",
     ],
 )
 
diff --git a/python/private/coverage_deps.bzl b/python/private/coverage_deps.bzl
@@ -80,11 +80,23 @@ _coverage_deps = {
             "https://files.pythonhosted.org/packages/b9/67/e1413d5a8591622a46dd04ff80873b04c849268831ed5c304c16433e7e30/coverage-7.6.1-cp313-cp313-macosx_11_0_arm64.whl",
             "a6d3adcf24b624a7b778533480e32434a39ad8fa30c315208f6d3e5542aeb6e9",
         ),
+        "aarch64-apple-darwin-freethreaded": (
+            "https://files.pythonhosted.org/packages/c4/ae/b5d58dff26cade02ada6ca612a76447acd69dccdbb3a478e9e088eb3d4b9/coverage-7.6.1-cp313-cp313t-macosx_11_0_arm64.whl",
+            "502753043567491d3ff6d08629270127e0c31d4184c4c8d98f92c26f65019962",
+        ),
         "aarch64-unknown-linux-gnu": (
+            "https://files.pythonhosted.org/packages/14/5b/9dec847b305e44a5634d0fb8498d135ab1d88330482b74065fcec0622224/coverage-7.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
+            "d0c212c49b6c10e6951362f7c6df3329f04c2b1c28499563d4035d964ab8e08c",
+        ),
+        "aarch64-unknown-linux-gnu-freethreaded": (
             "https://files.pythonhosted.org/packages/b8/d7/62095e355ec0613b08dfb19206ce3033a0eedb6f4a67af5ed267a8800642/coverage-7.6.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
             "6a89ecca80709d4076b95f89f308544ec8f7b4727e8a547913a35f16717856cb",
         ),
         "x86_64-unknown-linux-gnu": (
+            "https://files.pythonhosted.org/packages/f7/95/d2fd31f1d638df806cae59d7daea5abf2b15b5234016a5ebb502c2f3f7ee/coverage-7.6.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+            "78b260de9790fd81e69401c2dc8b17da47c8038176a79092a89cb2b7d945d060",
+        ),
+        "x86_64-unknown-linux-gnu-freethreaded": (
             "https://files.pythonhosted.org/packages/8b/61/a7a6a55dd266007ed3b1df7a3386a0d760d014542d72f7c2c6938483b7bd/coverage-7.6.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
             "13b0a73a0896988f053e4fbb7de6d93388e6dd292b0d87ee51d106f2c11b465b",
         ),
diff --git a/python/private/flags.bzl b/python/private/flags.bzl
@@ -122,3 +122,13 @@ PrecompileSourceRetentionFlag = enum(
     OMIT_SOURCE = "omit_source",
     get_effective_value = _precompile_source_retention_flag_get_effective_value,
 )
+
+# Used for matching freethreaded toolchains and would have to be used in wheels
+# as well.
+# buildifier: disable=name-conventions
+FreeThreadedFlag = enum(
+    # Use freethreaded python toolchain and wheels.
+    YES = "yes",
+    # Do not use freethreaded python toolchain and wheels.
+    NO = "no",
+)
diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl
@@ -21,6 +21,8 @@ load(":glob_excludes.bzl", "glob_excludes")
 load(":py_exec_tools_toolchain.bzl", "py_exec_tools_toolchain")
 load(":semver.bzl", "semver")
 
+_IS_FREETHREADED = Label("//python/config_settings:is_py_freethreaded")
+
 def define_hermetic_runtime_toolchain_impl(
         *,
         name,
@@ -45,7 +47,7 @@ def define_hermetic_runtime_toolchain_impl(
         python_version: {type}`str` The Python version, in `major.minor.micro`
             format.
         python_bin: {type}`str` The path to the Python binary within the
-            repositoroy.
+            repository.
         coverage_tool: {type}`str` optional target to the coverage tool to
             use.
     """
@@ -67,19 +69,23 @@ def define_hermetic_runtime_toolchain_impl(
             exclude = [
                 # Unused shared libraries. `python` executable and the `:libpython` target
                 # depend on `libpython{python_version}.so.1.0`.
-                "lib/libpython{major}.{minor}.so".format(**version_dict),
+                "lib/libpython{major}.{minor}*.so".format(**version_dict),
                 # static libraries
                 "lib/**/*.a",
                 # tests for the standard libraries.
-                "lib/python{major}.{minor}/**/test/**".format(**version_dict),
-                "lib/python{major}.{minor}/**/tests/**".format(**version_dict),
-                "**/__pycache__/*.pyc.*",  # During pyc creation, temp files named *.pyc.NNN are created
+                "lib/python{major}.{minor}*/**/test/**".format(**version_dict),
+                "lib/python{major}.{minor}*/**/tests/**".format(**version_dict),
+                # During pyc creation, temp files named *.pyc.NNN are created
+                "**/__pycache__/*.pyc.*",
             ] + glob_excludes.version_dependent_exclusions() + extra_files_glob_exclude,
         ),
     )
     cc_import(
         name = "interface",
-        interface_library = "libs/python{major}{minor}.lib".format(**version_dict),
+        interface_library = select({
+            _IS_FREETHREADED: "libs/python{major}{minor}t.lib".format(**version_dict),
+            "//conditions:default": "libs/python{major}{minor}.lib".format(**version_dict),
+        }),
         system_provided = True,
     )
 
@@ -96,14 +102,62 @@ def define_hermetic_runtime_toolchain_impl(
         hdrs = [":includes"],
         includes = [
             "include",
-            "include/python{major}.{minor}".format(**version_dict),
-            "include/python{major}.{minor}m".format(**version_dict),
+        ] + select({
+            _IS_FREETHREADED: [
+                "include/python{major}.{minor}t".format(**version_dict),
+            ],
+            "//conditions:default": [
+                "include/python{major}.{minor}".format(**version_dict),
+                "include/python{major}.{minor}m".format(**version_dict),
+            ],
+        }),
+    )
+    native.config_setting(
+        name = "is_freethreaded_linux",
+        flag_values = {
+            Label("//python/config_settings:py_freethreaded"): "yes",
+        },
+        constraint_values = [
+            "@platforms//os:linux",
         ],
+        visibility = ["//visibility:private"],
     )
+    native.config_setting(
+        name = "is_freethreaded_osx",
+        flag_values = {
+            Label("//python/config_settings:py_freethreaded"): "yes",
+        },
+        constraint_values = [
+            "@platforms//os:osx",
+        ],
+        visibility = ["//visibility:private"],
+    )
+    native.config_setting(
+        name = "is_freethreaded_windows",
+        flag_values = {
+            Label("//python/config_settings:py_freethreaded"): "yes",
+        },
+        constraint_values = [
+            "@platforms//os:windows",
+        ],
+        visibility = ["//visibility:private"],
+    )
+
     cc_library(
         name = "libpython",
         hdrs = [":includes"],
         srcs = select({
+            ":is_freethreaded_linux": [
+                "lib/libpython{major}.{minor}t.so".format(**version_dict),
+                "lib/libpython{major}.{minor}t.so.1.0".format(**version_dict),
+            ],
+            ":is_freethreaded_osx": [
+                "lib/libpython{major}.{minor}t.dylib".format(**version_dict),
+            ],
+            ":is_freethreaded_windows": [
+                "python3.dll",
+                "libs/python{major}{minor}t.lib".format(**version_dict),
+            ],
             "@platforms//os:linux": [
                 "lib/libpython{major}.{minor}.so".format(**version_dict),
                 "lib/libpython{major}.{minor}.so.1.0".format(**version_dict),
@@ -132,12 +186,18 @@ def define_hermetic_runtime_toolchain_impl(
             "micro": str(version_info.patch),
             "minor": str(version_info.minor),
         },
-        # Convert empty string to None
-        coverage_tool = coverage_tool or None,
+        coverage_tool = select({
+            # Convert empty string to None
+            ":coverage_enabled": coverage_tool or None,
+            "//conditions:default": None,
+        }),
         python_version = "PY3",
         implementation_name = "cpython",
         # See https://peps.python.org/pep-3147/ for pyc tag infix format
-        pyc_tag = "cpython-{major}{minor}".format(**version_dict),
+        pyc_tag = select({
+            _IS_FREETHREADED: "cpython-{major}{minor}t".format(**version_dict),
+            "//conditions:default": "cpython-{major}{minor}".format(**version_dict),
+        }),
     )
 
     py_runtime_pair(
diff --git a/python/private/python.bzl b/python/private/python.bzl
@@ -213,6 +213,7 @@ def parse_modules(*, module_ctx, _fail = fail):
 def _python_impl(module_ctx):
     py = parse_modules(module_ctx = module_ctx)
 
+    loaded_platforms = {}
     for toolchain_info in py.toolchains:
         # Ensure that we pass the full version here.
         full_python_version = full_version(
@@ -228,7 +229,7 @@ def _python_impl(module_ctx):
         kwargs.update(py.config.kwargs.get(toolchain_info.python_version, {}))
         kwargs.update(py.config.kwargs.get(full_python_version, {}))
         kwargs.update(py.config.default)
-        python_register_toolchains(
+        loaded_platforms[full_python_version] = python_register_toolchains(
             name = toolchain_info.name,
             _internal_bzlmod_toolchain_call = True,
             **kwargs
@@ -257,6 +258,7 @@ def _python_impl(module_ctx):
             for i in range(len(py.toolchains))
         ],
         toolchain_user_repository_names = [t.name for t in py.toolchains],
+        loaded_platforms = loaded_platforms,
     )
 
     # This is require in order to support multiple version py_test
@@ -464,7 +466,7 @@ def _get_toolchain_config(*, modules, _fail = fail):
             "url": {
                 platform: [item["url"]]
                 for platform in item["sha256"]
-            },
+            } if type(item["url"]) == type("") else item["url"],
         }
         for version, item in TOOL_VERSIONS.items()
     }
diff --git a/python/private/python_register_toolchains.bzl b/python/private/python_register_toolchains.bzl
@@ -73,6 +73,9 @@ def python_register_toolchains(
         minor_mapping: {type}`dict[str, str]` contains a mapping from `X.Y` to `X.Y.Z`
             version.
         **kwargs: passed to each {obj}`python_repository` call.
+
+    Returns:
+        On bzlmod this returns the loaded platform labels. Otherwise None.
     """
     bzlmod_toolchain_call = kwargs.pop("_internal_bzlmod_toolchain_call", False)
     if bzlmod_toolchain_call:
@@ -168,11 +171,13 @@ def python_register_toolchains(
 
     # in bzlmod we write out our own toolchain repos
     if bzlmod_toolchain_call:
-        return
+        return loaded_platforms
 
     toolchains_repo(
         name = toolchain_repo_name,
         python_version = python_version,
         set_python_version_constraint = set_python_version_constraint,
         user_repository_name = name,
+        platforms = loaded_platforms,
     )
+    return None
diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl
@@ -15,7 +15,7 @@
 """This file contains repository rules and macros to support toolchain registration.
 """
 
-load("//python:versions.bzl", "PLATFORMS")
+load("//python:versions.bzl", "FREETHREADED", "PLATFORMS")
 load(":auth.bzl", "get_auth")
 load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
 load(":text_util.bzl", "render")
@@ -63,8 +63,12 @@ def _python_repository_impl(rctx):
     platform = rctx.attr.platform
     python_version = rctx.attr.python_version
     python_version_info = python_version.split(".")
-    python_short_version = "{0}.{1}".format(*python_version_info)
     release_filename = rctx.attr.release_filename
+    version_suffix = "t" if FREETHREADED in release_filename else ""
+    python_short_version = "{0}.{1}{suffix}".format(
+        suffix = version_suffix,
+        *python_version_info
+    )
     urls = rctx.attr.urls or [rctx.attr.url]
     auth = get_auth(rctx, urls)
 
diff --git a/python/private/pythons_hub.bzl b/python/private/pythons_hub.bzl
@@ -14,6 +14,7 @@
 
 "Repo rule used by bzlmod extension to create a repo that has a map of Python interpreters and their labels"
 
+load("//python:versions.bzl", "PLATFORMS")
 load(":text_util.bzl", "render")
 load(":toolchains_repo.bzl", "python_toolchain_build_file_content")
 
@@ -46,7 +47,8 @@ def _hub_build_file_content(
         python_versions,
         set_python_version_constraints,
         user_repository_names,
-        workspace_location):
+        workspace_location,
+        loaded_platforms):
     """This macro iterates over each of the lists and returns the toolchain content.
 
     python_toolchain_build_file_content is called to generate each of the toolchain
@@ -65,6 +67,11 @@ def _hub_build_file_content(
                 python_version = python_versions[i],
                 set_python_version_constraint = set_python_version_constraints[i],
                 user_repository_name = user_repository_names[i],
+                loaded_platforms = {
+                    k: v
+                    for k, v in PLATFORMS.items()
+                    if k in loaded_platforms[python_versions[i]]
+                },
             )
             for i in range(len(python_versions))
         ],
@@ -103,6 +110,7 @@ def _hub_repo_impl(rctx):
             rctx.attr.toolchain_set_python_version_constraints,
             rctx.attr.toolchain_user_repository_names,
             rctx.attr._rules_python_workspace,
+            rctx.attr.loaded_platforms,
         ),
         executable = False,
     )
@@ -149,6 +157,9 @@ This rule also writes out the various toolchains for the different Python versio
             doc = "Default Python version for the build in `X.Y` or `X.Y.Z` format.",
             mandatory = True,
         ),
+        "loaded_platforms": attr.string_list_dict(
+            doc = "The list of loaded platforms keyed by the toolchain full python version",
+        ),
         "minor_mapping": attr.string_dict(
             doc = "The minor mapping of the `X.Y` to `X.Y.Z` format that is used in config settings.",
             mandatory = True,
diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl
diff --git a/python/versions.bzl b/python/versions.bzl
diff --git a/tools/private/update_deps/update_coverage_deps.py b/tools/private/update_deps/update_coverage_deps.py

Original file line number	Diff line number	Diff line change
`@@ -256,6 +256,7 @@ bzl_library(`
`256`	`256`	`deps = [`
`257`	`257`	`":py_toolchain_suite_bzl",`
`258`	`258`	`":text_util_bzl",`
	`259`	`+ "//python:versions_bzl",`
`259`	`260`	`],`
`260`	`261`	`)`
`261`	`262`