Skip to content

Commit 66c133d

Browse files
committed
feat(pypi): freethreaded support for the builder API
DO NOT MERGE: stacked on #3058 This is a continuation of #3058 where we define freethreaded platforms. They need to be used only for particular python versions so I included an extra marker configuration attribute where we are using pipstar marker evaluation before using the platform. I think this in general will be a useful tool to configure only particular platforms for particular python versions Work towards #2548, since this shows how we can define custom platforms Work towards #2747
1 parent 0b8596d commit 66c133d

File tree

6 files changed

+119
-46
lines changed

6 files changed

+119
-46
lines changed

MODULE.bazel

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -70,11 +70,15 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
7070
config_settings = [
7171
"@platforms//cpu:{}".format(cpu),
7272
"@platforms//os:linux",
73+
"//python/config_settings:_is_py_freethreaded_{}".format(
74+
"yes" if freethreaded else "no",
75+
),
7376
],
7477
env = {"platform_version": "0"},
78+
marker = "python_version >= '3.13'" if freethreaded else "",
7579
os_name = "linux",
76-
platform = "linux_{}".format(cpu),
77-
whl_abi_tags = [
80+
platform = "linux_{}{}".format(cpu, freethreaded),
81+
whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [
7882
"abi3",
7983
"cp{major}{minor}",
8084
],
@@ -87,6 +91,10 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
8791
"x86_64",
8892
"aarch64",
8993
]
94+
for freethreaded in [
95+
"",
96+
"_freethreaded",
97+
]
9098
]
9199

92100
[
@@ -95,13 +103,17 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
95103
config_settings = [
96104
"@platforms//cpu:{}".format(cpu),
97105
"@platforms//os:osx",
106+
"//python/config_settings:_is_py_freethreaded_{}".format(
107+
"yes" if freethreaded else "no",
108+
),
98109
],
99110
# We choose the oldest non-EOL version at the time when we release `rules_python`.
100111
# See https://endoflife.date/macos
101112
env = {"platform_version": "14.0"},
113+
marker = "python_version >= '3.13'" if freethreaded else "",
102114
os_name = "osx",
103-
platform = "osx_{}".format(cpu),
104-
whl_abi_tags = [
115+
platform = "osx_{}{}".format(cpu, freethreaded),
116+
whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [
105117
"abi3",
106118
"cp{major}{minor}",
107119
],
@@ -120,6 +132,10 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
120132
"x86_64",
121133
],
122134
}.items()
135+
for freethreaded in [
136+
"",
137+
"_freethreaded",
138+
]
123139
]
124140

125141
[
@@ -128,11 +144,15 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
128144
config_settings = [
129145
"@platforms//cpu:{}".format(cpu),
130146
"@platforms//os:windows",
147+
"//python/config_settings:_is_py_freethreaded_{}".format(
148+
"yes" if freethreaded else "no",
149+
),
131150
],
132151
env = {"platform_version": "0"},
152+
marker = "python_version >= '3.13'" if freethreaded else "",
133153
os_name = "windows",
134-
platform = "windows_{}".format(cpu),
135-
whl_abi_tags = [
154+
platform = "windows_{}{}".format(cpu, freethreaded),
155+
whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [
136156
"abi3",
137157
"cp{major}{minor}",
138158
],
@@ -141,6 +161,10 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
141161
for cpu, whl_platform_tags in {
142162
"x86_64": ["win_amd64"],
143163
}.items()
164+
for freethreaded in [
165+
"",
166+
"_freethreaded",
167+
]
144168
]
145169

146170
pip.parse(

python/private/pypi/extension.bzl

Lines changed: 53 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json")
3030
load(":parse_requirements.bzl", "parse_requirements")
3131
load(":parse_whl_name.bzl", "parse_whl_name")
3232
load(":pep508_env.bzl", "env")
33+
load(":pep508_evaluate.bzl", "evaluate")
3334
load(":pip_repository_attrs.bzl", "ATTRS")
3435
load(":python_tag.bzl", "python_tag")
3536
load(":requirements_files_by_platform.bzl", "requirements_files_by_platform")
@@ -80,21 +81,27 @@ def _platforms(*, python_version, minor_mapping, config):
8081
for platform, values in config.platforms.items():
8182
# TODO @aignas 2025-07-07: this is probably doing the parsing of the version too
8283
# many times.
83-
key = "{}{}{}.{}_{}".format(
84+
abi = "{}{}{}.{}".format(
8485
python_tag(values.env["implementation_name"]),
8586
python_version.release[0],
8687
python_version.release[1],
8788
python_version.release[2],
88-
platform,
8989
)
90+
key = "{}_{}".format(abi, platform)
91+
92+
env_ = env(
93+
env = values.env,
94+
os = values.os_name,
95+
arch = values.arch_name,
96+
python_version = python_version.string,
97+
)
98+
99+
if values.marker and not evaluate(values.marker, env = env_):
100+
continue
90101

91102
platforms[key] = struct(
92-
env = env(
93-
env = values.env,
94-
os = values.os_name,
95-
arch = values.arch_name,
96-
python_version = python_version.string,
97-
),
103+
env = env_,
104+
triple = "{}_{}_{}".format(abi, values.os_name, values.arch_name),
98105
whl_abi_tags = [
99106
v.format(
100107
major = python_version.release[0],
@@ -203,17 +210,19 @@ def _create_whl_repos(
203210
whl_group_mapping = {}
204211
requirement_cycles = {}
205212

213+
platforms = _platforms(
214+
python_version = pip_attr.python_version,
215+
minor_mapping = minor_mapping,
216+
config = config,
217+
)
218+
206219
if evaluate_markers:
207220
# This is most likely unit tests
208221
pass
209222
elif config.enable_pipstar:
210223
evaluate_markers = lambda _, requirements: evaluate_markers_star(
211224
requirements = requirements,
212-
platforms = _platforms(
213-
python_version = pip_attr.python_version,
214-
minor_mapping = minor_mapping,
215-
config = config,
216-
),
225+
platforms = platforms,
217226
)
218227
else:
219228
# NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either
@@ -232,7 +241,13 @@ def _create_whl_repos(
232241
# spin up a Python interpreter.
233242
evaluate_markers = lambda module_ctx, requirements: evaluate_markers_py(
234243
module_ctx,
235-
requirements = requirements,
244+
requirements = {
245+
k: {
246+
p: platforms[p].triple
247+
for p in plats
248+
}
249+
for k, plats in requirements.items()
250+
},
236251
python_interpreter = pip_attr.python_interpreter,
237252
python_interpreter_target = python_interpreter_target,
238253
srcs = pip_attr._evaluate_markers_srcs,
@@ -248,18 +263,14 @@ def _create_whl_repos(
248263
requirements_osx = pip_attr.requirements_darwin,
249264
requirements_windows = pip_attr.requirements_windows,
250265
extra_pip_args = pip_attr.extra_pip_args,
251-
platforms = sorted(config.platforms), # here we only need keys
266+
platforms = sorted(platforms), # here we only need keys
252267
python_version = full_version(
253268
version = pip_attr.python_version,
254269
minor_mapping = minor_mapping,
255270
),
256271
logger = logger,
257272
),
258-
platforms = _platforms(
259-
python_version = pip_attr.python_version,
260-
minor_mapping = minor_mapping,
261-
config = config,
262-
),
273+
platforms = platforms,
263274
extra_pip_args = pip_attr.extra_pip_args,
264275
get_index_urls = get_index_urls,
265276
evaluate_markers = evaluate_markers,
@@ -346,6 +357,16 @@ def _create_whl_repos(
346357
))
347358

348359
whl_libraries[repo_name] = repo.args
360+
if not config.enable_pipstar and "experimental_target_platforms" in repo.args:
361+
whl_libraries[repo_name] |= {
362+
"experimental_target_platforms": sorted({
363+
# TODO @aignas 2025-07-07: this should be solved in a better way
364+
platforms[candidate].triple.partition("_")[-1]: None
365+
for p in repo.args["experimental_target_platforms"]
366+
for candidate in platforms
367+
if candidate.endswith(p)
368+
}),
369+
}
349370
whl_map.setdefault(whl.name, {})[repo.config_setting] = repo_name
350371

351372
return struct(
@@ -434,14 +455,15 @@ def _configure(
434455
arch_name,
435456
config_settings,
436457
env = {},
458+
marker,
437459
whl_abi_tags,
438460
whl_platform_tags,
439461
override = False):
440462
"""Set the value in the config if the value is provided"""
441463
config.setdefault("platforms", {})
442464

443465
if platform and (
444-
os_name or arch_name or config_settings or whl_abi_tags or whl_platform_tags or env
466+
os_name or arch_name or config_settings or whl_abi_tags or whl_platform_tags or env or marker
445467
):
446468
if not override and config["platforms"].get(platform):
447469
return
@@ -455,6 +477,7 @@ def _configure(
455477
"arch_name": arch_name,
456478
"config_settings": config_settings,
457479
"env": env,
480+
"marker": marker,
458481
"name": platform.replace("-", "_").lower(),
459482
"os_name": os_name,
460483
"whl_abi_tags": whl_abi_tags,
@@ -470,7 +493,7 @@ def _configure(
470493
else:
471494
config["platforms"].pop(platform)
472495

473-
def _plat(*, name, arch_name, os_name, config_settings = [], env = {}, whl_abi_tags = [], whl_platform_tags = []):
496+
def _plat(*, name, arch_name, os_name, config_settings = [], env = {}, marker = "", whl_abi_tags = [], whl_platform_tags = []):
474497
# NOTE @aignas 2025-07-08: the least preferred is the first item in the list
475498
if "any" not in whl_platform_tags:
476499
# the lowest priority one needs to be the first one
@@ -490,6 +513,7 @@ def _plat(*, name, arch_name, os_name, config_settings = [], env = {}, whl_abi_t
490513
# defaults for env
491514
"implementation_name": "cpython",
492515
} | env,
516+
marker = marker,
493517
whl_abi_tags = whl_abi_tags,
494518
whl_platform_tags = whl_platform_tags,
495519
)
@@ -524,6 +548,7 @@ def build_config(
524548
config_settings = tag.config_settings,
525549
env = tag.env,
526550
os_name = tag.os_name,
551+
marker = tag.marker,
527552
platform = platform,
528553
override = mod.is_root,
529554
whl_abi_tags = tag.whl_abi_tags,
@@ -533,8 +558,6 @@ def build_config(
533558
# attribute.
534559
# * for index/downloader config. This includes all of those attributes for
535560
# overrides, etc. Index overrides per platform could be also used here.
536-
# * for whl selection - selecting preferences of which `platform_tag`s we should use
537-
# for what. We could also model the `cp313t` freethreaded as separate platforms.
538561
)
539562

540563
return struct(
@@ -928,6 +951,12 @@ Supported keys:
928951
::::{note}
929952
This is only used if the {envvar}`RULES_PYTHON_ENABLE_PIPSTAR` is enabled.
930953
::::
954+
""",
955+
),
956+
"marker": attr.string(
957+
doc = """\
958+
A marker which will be evaluated to disable the target platform for certain python versions. This
959+
is especially useful when defining freethreaded platform variants.
931960
""",
932961
),
933962
# The values for PEP508 env marker evaluation during the lock file parsing

python/private/pypi/pip_repository.bzl

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,12 @@ def _pip_repository_impl(rctx):
9494
extra_pip_args = rctx.attr.extra_pip_args,
9595
evaluate_markers = lambda rctx, requirements: evaluate_markers_py(
9696
rctx,
97-
requirements = requirements,
97+
requirements = {
98+
# NOTE @aignas 2025-07-07: because we don't distinguish between
99+
# freethreaded and non-freethreaded, it is a 1:1 mapping.
100+
req: {p: p for p in plats}
101+
for req, plats in requirements.items()
102+
},
98103
python_interpreter = rctx.attr.python_interpreter,
99104
python_interpreter_target = rctx.attr.python_interpreter_target,
100105
srcs = rctx.attr._evaluate_markers_srcs,

python/private/pypi/requirements_files_by_platform.bzl

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@ def _default_platforms(*, filter, platforms):
3737
if not prefix:
3838
return platforms
3939

40-
match = [p for p in platforms if p.startswith(prefix)]
40+
match = [p for p in platforms if p.startswith(prefix) or (
41+
p.startswith("cp") and p.partition("_")[-1].startswith(prefix)
42+
)]
4143
else:
4244
match = [p for p in platforms if filter in p]
4345

@@ -140,7 +142,7 @@ def requirements_files_by_platform(
140142
if logger:
141143
logger.debug(lambda: "Platforms from pip args: {}".format(platforms_from_args))
142144

143-
default_platforms = [_platform(p, python_version) for p in platforms]
145+
default_platforms = platforms
144146

145147
if platforms_from_args:
146148
lock_files = [
@@ -252,6 +254,6 @@ def requirements_files_by_platform(
252254

253255
ret = {}
254256
for plat, file in requirements.items():
255-
ret.setdefault(file, []).append(plat)
257+
ret.setdefault(file, []).append(_platform(plat, python_version = python_version))
256258

257259
return ret

python/private/pypi/requirements_parser/resolve_target_platforms.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ def main():
5050
hashes = prefix + hashes
5151

5252
req = Requirement(entry)
53-
for p in target_platforms:
54-
(platform,) = Platform.from_string(p)
53+
for p, triple in target_platforms.items():
54+
(platform,) = Platform.from_string(triple)
5555
if not req.marker or req.marker.evaluate(platform.env_markers("")):
5656
response.setdefault(requirement_line, []).append(p)
5757

0 commit comments

Comments
 (0)