Skip to content

feat(pypi): incrementally build platform configuration #3112 #3112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

feat(pypi): incrementally build platform configuration #3112 #3112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6f3dab1
fix(pypi): reuse select dicts for constructing the env
aignas Jul 20, 2025
31967c5
feat(pypi): add a standards compliant python_tag creator
aignas Jul 20, 2025
9792058
feat(pypi): implement a new whl selection algorithm
aignas Jul 21, 2025
7d36af2
feat(pypi): incrementally build platform configuration
aignas Jul 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 21 additions & 5 deletions python/private/pypi/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,9 @@ bzl_library(
bzl_library(
name = "pep508_env_bzl",
srcs = ["pep508_env.bzl"],
deps = [
"//python/private:version_bzl",
],
)

bzl_library(
Expand All @@ -263,11 +266,6 @@ bzl_library(
],
)

bzl_library(
name = "pep508_platform_bzl",
srcs = ["pep508_platform.bzl"],
)

bzl_library(
name = "pep508_requirement_bzl",
srcs = ["pep508_requirement.bzl"],
Expand Down Expand Up @@ -338,6 +336,14 @@ bzl_library(
],
)

bzl_library(
name = "python_tag_bzl",
srcs = ["python_tag.bzl"],
deps = [
"//python/private:version_bzl",
],
)

bzl_library(
name = "render_pkg_aliases_bzl",
srcs = ["render_pkg_aliases.bzl"],
Expand All @@ -359,6 +365,16 @@ bzl_library(
],
)

bzl_library(
name = "select_whl_bzl",
srcs = ["select_whl.bzl"],
deps = [
":parse_whl_name_bzl",
":python_tag_bzl",
"//python/private:version_bzl",
],
)

bzl_library(
name = "simpleapi_download_bzl",
srcs = ["simpleapi_download.bzl"],
Expand Down
120 changes: 75 additions & 45 deletions python/private/pypi/extension.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,12 @@ def _platforms(*, python_version, minor_mapping, config):

for platform, values in config.platforms.items():
key = "{}_{}".format(abi, platform)
platforms[key] = env(struct(
abi = abi,
platforms[key] = env(
env = values.env,
os = values.os_name,
arch = values.arch_name,
)) | values.env
python_version = python_version,
)
return platforms

def _create_whl_repos(
Expand Down Expand Up @@ -376,26 +377,80 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net
),
)

def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, override = False):
"""Set the value in the config if the value is provided"""
config.setdefault("platforms", {})
if platform:
if not override and config.get("platforms", {}).get(platform):
return
def _plat(*, name, arch_name, os_name, config_settings = [], env = {}):
return struct(
name = name,
arch_name = arch_name,
os_name = os_name,
config_settings = config_settings,
env = env,
)

def _configure(config, *, override = False, **kwargs):
"""Set the value in the config if the value is provided"""
env = kwargs.get("env")
if env:
for key in env:
if key not in _SUPPORTED_PEP508_KEYS:
fail("Unsupported key in the PEP508 environment: {}".format(key))

config["platforms"][platform] = struct(
name = platform.replace("-", "_").lower(),
os_name = os_name,
arch_name = arch_name,
config_settings = config_settings,
env = env,
)
else:
config["platforms"].pop(platform)
for key, value in kwargs.items():
if value and (override or key not in config):
config[key] = value

def build_config(
*,
module_ctx,
enable_pipstar):
"""Parse 'configure' and 'default' extension tags
Args:
module_ctx: {type}`module_ctx` module context.
enable_pipstar: {type}`bool` a flag to enable dropping Python dependency for
evaluation of the extension.
Returns:
A struct with the configuration.
"""
defaults = {
"platforms": {},
}
for mod in module_ctx.modules:
if not (mod.is_root or mod.name == "rules_python"):
continue

for tag in mod.tags.default:
platform = tag.platform
if platform:
specific_config = defaults["platforms"].setdefault(platform, {})
_configure(
specific_config,
arch_name = tag.arch_name,
config_settings = tag.config_settings,
env = tag.env,
os_name = tag.os_name,
name = platform.replace("-", "_").lower(),
override = mod.is_root,
)

if platform and not (tag.arch_name or tag.config_settings or tag.env or tag.os_name):
defaults["platforms"].pop(platform)

# TODO @aignas 2025-05-19: add more attr groups:
# * for AUTH - the default `netrc` usage could be configured through a common
# attribute.
# * for index/downloader config. This includes all of those attributes for
# overrides, etc. Index overrides per platform could be also used here.
# * for whl selection - selecting preferences of which `platform_tag`s we should use
# for what. We could also model the `cp313t` freethreaded as separate platforms.

return struct(
platforms = {
name: _plat(**values)
for name, values in defaults["platforms"].items()
},
enable_pipstar = enable_pipstar,
)

def parse_modules(
module_ctx,
Expand Down Expand Up @@ -447,33 +502,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
srcs_exclude_glob = whl_mod.srcs_exclude_glob,
)

defaults = {
"enable_pipstar": enable_pipstar,
"platforms": {},
}
for mod in module_ctx.modules:
if not (mod.is_root or mod.name == "rules_python"):
continue

for tag in mod.tags.default:
_configure(
defaults,
arch_name = tag.arch_name,
config_settings = tag.config_settings,
env = tag.env,
os_name = tag.os_name,
platform = tag.platform,
override = mod.is_root,
# TODO @aignas 2025-05-19: add more attr groups:
# * for AUTH - the default `netrc` usage could be configured through a common
# attribute.
# * for index/downloader config. This includes all of those attributes for
# overrides, etc. Index overrides per platform could be also used here.
# * for whl selection - selecting preferences of which `platform_tag`s we should use
# for what. We could also model the `cp313t` freethreaded as separate platforms.
)

config = struct(**defaults)
config = build_config(module_ctx = module_ctx, enable_pipstar = enable_pipstar)

# TODO @aignas 2025-06-03: Merge override API with the builder?
_overriden_whl_set = {}
Expand Down Expand Up @@ -658,6 +687,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
k: dict(sorted(args.items()))
for k, args in sorted(whl_libraries.items())
},
config = config,
)

def _pip_impl(module_ctx):
Expand Down
86 changes: 53 additions & 33 deletions python/private/pypi/pep508_env.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,20 @@
"""This module is for implementing PEP508 environment definition.
"""

load("//python/private:version.bzl", "version")

_DEFAULT = "//conditions:default"

# Here we store the aliases in the platform so that the users can specify any valid target in
# there.
_cpu_aliases = {
"arm": "aarch32",
"arm64": "aarch64",
}
_os_aliases = {
"macos": "osx",
}

# See https://stackoverflow.com/a/45125525
platform_machine_aliases = {
# These pairs mean the same hardware, but different values may be used
Expand Down Expand Up @@ -59,7 +73,7 @@ platform_machine_select_map = {
"@platforms//cpu:x86_64": "x86_64",
# The value is empty string if it cannot be determined:
# https://docs.python.org/3/library/platform.html#platform.machine
"//conditions:default": "",
_DEFAULT: "",
}

# Platform system returns results from the `uname` call.
Expand All @@ -73,7 +87,7 @@ _platform_system_values = {
"linux": "Linux",
"netbsd": "NetBSD",
"openbsd": "OpenBSD",
"osx": "Darwin",
"osx": "Darwin", # NOTE: macos is an alias to osx, we handle it through _os_aliases
"windows": "Windows",
}

Expand All @@ -83,7 +97,7 @@ platform_system_select_map = {
} | {
# The value is empty string if it cannot be determined:
# https://docs.python.org/3/library/platform.html#platform.machine
"//conditions:default": "",
_DEFAULT: "",
}

# The copy of SO [answer](https://stackoverflow.com/a/13874620) containing
Expand Down Expand Up @@ -123,72 +137,78 @@ _sys_platform_values = {
"ios": "ios",
"linux": "linux",
"openbsd": "openbsd",
"osx": "darwin",
"osx": "darwin", # NOTE: macos is an alias to osx, we handle it through _os_aliases
"wasi": "wasi",
"windows": "win32",
}

sys_platform_select_map = {
# These values are decided by the sys.platform docs.
"@platforms//os:{}".format(bazel_os): py_platform
for bazel_os, py_platform in _sys_platform_values.items()
} | {
# For lack of a better option, use empty string. No standard doc/spec
# about sys_platform value.
"//conditions:default": "",
_DEFAULT: "",
}

# The "java" value is documented, but with Jython defunct,
# shouldn't occur in practice.
# The os.name value is technically a property of the runtime, not the
# targetted runtime OS, but the distinction shouldn't matter if
# things are properly configured.
_os_name_values = {
"linux": "posix",
"osx": "posix",
"windows": "nt",
}

os_name_select_map = {
"@platforms//os:{}".format(bazel_os): py_os
for bazel_os, py_os in _os_name_values.items()
} | {
"//conditions:default": "posix",
"@platforms//os:windows": "nt",
_DEFAULT: "posix",
}

def env(target_platform, *, extra = None):
def _set_default(env, env_key, m, key):
"""Set the default value in the env if it is not already set."""
default = m.get(key, m[_DEFAULT])
env.setdefault(env_key, default)

def env(*, env = None, os, arch, python_version = "", extra = None):
"""Return an env target platform
NOTE: This is for use during the loading phase. For the analysis phase,
`env_marker_setting()` constructs the env dict.
Args:
target_platform: {type}`str` the target platform identifier, e.g.
`cp33_linux_aarch64`
env: {type}`str` the environment.
os: {type}`str` the OS name.
arch: {type}`str` the CPU name.
python_version: {type}`str` the full python version.
extra: {type}`str` the extra value to be added into the env.
Returns:
A dict that can be used as `env` in the marker evaluation.
"""
env = create_env()
env = env or {}
env = env | create_env()
if extra != None:
env["extra"] = extra

if target_platform.abi:
minor_version, _, micro_version = target_platform.abi[3:].partition(".")
micro_version = micro_version or "0"
env = env | {
"implementation_version": "3.{}.{}".format(minor_version, micro_version),
"python_full_version": "3.{}.{}".format(minor_version, micro_version),
"python_version": "3.{}".format(minor_version),
}
if target_platform.os and target_platform.arch:
os = target_platform.os
if python_version:
v = version.parse(python_version)
major = v.release[0]
minor = v.release[1]
micro = v.release[2] if len(v.release) > 2 else 0
env = env | {
"os_name": _os_name_values.get(os, ""),
"platform_machine": target_platform.arch,
"platform_system": _platform_system_values.get(os, ""),
"sys_platform": _sys_platform_values.get(os, ""),
"implementation_version": "{}.{}.{}".format(major, minor, micro),
"python_full_version": "{}.{}.{}".format(major, minor, micro),
"python_version": "{}.{}".format(major, minor),
}

if os:
os = "@platforms//os:{}".format(_os_aliases.get(os, os))
_set_default(env, "os_name", os_name_select_map, os)
_set_default(env, "platform_system", platform_system_select_map, os)
_set_default(env, "sys_platform", sys_platform_select_map, os)

if arch:
arch = "@platforms//cpu:{}".format(_cpu_aliases.get(arch, arch))
_set_default(env, "platform_machine", platform_machine_select_map, arch)

set_missing_env_defaults(env)

return env
Expand Down
Loading