Skip to content

chore(pypi): enable pipstar by default #3121

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 11 commits into
base: main
Choose a base branch
from
Draft
15 changes: 15 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ END_UNRELEASED_TEMPLATE

{#v0-0-0-changed}
### Changed
* (deps) Upgraded to `bazel-skylib` version
[1.8.1](https://github.com/bazelbuild/bazel-skylib/releases/tag/1.8.1)
to remove deprecation warnings.
* (gazelle) For package mode, resolve dependencies when imports are relative
to the package path. This is enabled via the
`# gazelle:experimental_allow_relative_imports` true directive ({gh-issue}`2203`).
Expand All @@ -69,8 +72,16 @@ END_UNRELEASED_TEMPLATE
* (toolchain) Python 3.13 now references 3.13.5
* (gazelle) Switched back to smacker/go-tree-sitter, fixing
[#2630](https://github.com/bazel-contrib/rules_python/issues/2630)
* (pypi) From now on the list of default platforms only includes `linux_x86_64`, `linux_aarch64`,
`osx_x86_64`, `osx_aarch64` and `windows_x86_64`. If you are on other platforms, you need to
use the `pip.default` to configure it yourself. If you are interested in graduating the
platform, consider helping set us up CI for them and update the documentation.
* (ci) We are now testing on Ubuntu 22.04 for RBE and non-RBE configurations.
* (core) #!/usr/bin/env bash is now used as a shebang in the stage1 bootstrap template.
* (pypi) From now on `pipstar` is enabled by default for all builds. To disable it, consider
setting env variable `RULES_PYTHON_ENABLE_PIPSTAR` to `0`. If there are reasons why you have to
keep it disabled, it is a bug and we would like to hear about it, please create a ticket in our
github issue tracker.

{#v0-0-0-fixed}
### Fixed
Expand All @@ -88,6 +99,10 @@ END_UNRELEASED_TEMPLATE
([#3043](https://github.com/bazel-contrib/rules_python/issues/3043)).
* (pypi) The pipstar `defaults` configuration now supports any custom platform
name.
* (pypi) The selection of the whls has been changed and should no longer result
in ambiguous select matches ({gh-issue}`2759`) and should be much more efficient
when running `bazel query` due to fewer repositories being included
({gh-issue}`2849`).
* Multi-line python imports (e.g. with escaped newlines) are now correctly processed by Gazelle.
* (toolchains) `local_runtime_repo` works with multiarch Debian with Python 3.8
([#3099](https://github.com/bazel-contrib/rules_python/issues/3099)).
Expand Down
99 changes: 77 additions & 22 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ module(
)

bazel_dep(name = "bazel_features", version = "1.21.0")
bazel_dep(name = "bazel_skylib", version = "1.7.1")
bazel_dep(name = "bazel_skylib", version = "1.8.1")
bazel_dep(name = "rules_cc", version = "0.0.16")
bazel_dep(name = "platforms", version = "0.0.11")

Expand Down Expand Up @@ -70,20 +70,30 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
config_settings = [
"@platforms//cpu:{}".format(cpu),
"@platforms//os:linux",
"//python/config_settings:_is_py_freethreaded_{}".format(
"yes" if freethreaded else "no",
),
],
env = {"platform_version": "0"},
marker = "python_version >= '3.13'" if freethreaded else "",
os_name = "linux",
platform = "linux_{}".format(cpu),
platform = "linux_{}{}".format(cpu, freethreaded),
whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [
"abi3",
"cp{major}{minor}",
],
whl_platform_tags = [
"linux_{}".format(cpu),
"manylinux_*_{}".format(cpu),
],
)
for cpu in [
"x86_64",
"aarch64",
# TODO @aignas 2025-05-19: only leave tier 0-1 cpus when stabilizing the
# `pip.default` extension. i.e. drop the below values - users will have to
# define themselves if they need them.
"arm",
"ppc",
"s390x",
]
for freethreaded in [
"",
"_freethreaded",
]
]

Expand All @@ -93,32 +103,77 @@ pip = use_extension("//python/extensions:pip.bzl", "pip")
config_settings = [
"@platforms//cpu:{}".format(cpu),
"@platforms//os:osx",
"//python/config_settings:_is_py_freethreaded_{}".format(
"yes" if freethreaded else "no",
),
],
# We choose the oldest non-EOL version at the time when we release `rules_python`.
# See https://endoflife.date/macos
#
# Keep in sync with "pip_whl_osx_version"
env = {"platform_version": "14.0"},
marker = "python_version >= '3.13'" if freethreaded else "",
os_name = "osx",
platform = "osx_{}".format(cpu),
platform = "osx_{}{}".format(cpu, freethreaded),
whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [
"abi3",
"cp{major}{minor}",
],
whl_platform_tags = [
"macosx_*_{}".format(suffix)
for suffix in platform_tag_cpus
],
)
for cpu in [
"aarch64",
"x86_64",
for cpu, platform_tag_cpus in {
"aarch64": [
"universal2",
"arm64",
],
"x86_64": [
"universal2",
"x86_64",
],
}.items()
for freethreaded in [
"",
"_freethreaded",
]
]

[
pip.default(
arch_name = cpu,
config_settings = [
"@platforms//cpu:{}".format(cpu),
"@platforms//os:windows",
"//python/config_settings:_is_py_freethreaded_{}".format(
"yes" if freethreaded else "no",
),
],
env = {"platform_version": "0"},
marker = "python_version >= '3.13'" if freethreaded else "",
os_name = "windows",
platform = "windows_{}{}".format(cpu, freethreaded),
whl_abi_tags = ["cp{major}{minor}t"] if freethreaded else [
"abi3",
"cp{major}{minor}",
],
whl_platform_tags = whl_platform_tags,
)
for cpu, whl_platform_tags in {
"x86_64": ["win_amd64"],
}.items()
for freethreaded in [
"",
"_freethreaded",
]
]

pip.default(
arch_name = "x86_64",
config_settings = [
"@platforms//cpu:x86_64",
"@platforms//os:windows",
],
env = {"platform_version": "0"},
os_name = "windows",
platform = "windows_x86_64",
)
pip.parse(
# NOTE @aignas 2024-10-26: We have an integration test that depends on us
# being able to build sdists for this hub, so explicitly set this to False.
#
# how do we test sdists? Maybe just worth adding a single sdist somewhere?
download_only = False,
experimental_index_url = "https://pypi.org/simple",
hub_name = "rules_python_publish_deps",
Expand Down
17 changes: 17 additions & 0 deletions examples/bzlmod/MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,23 @@ pip.whl_mods(
)
use_repo(pip, "whl_mods_hub")

# Because below we are using `windows_aarch64` platform, we have to define various
# properties for it.
pip.default(
arch_name = "aarch64",
config_settings = [
"@platforms//os:windows",
"@platforms//cpu:aarch64",
],
env = {
"platform_version": "0",
},
os_name = "windows",
platform = "windows_aarch64",
whl_abi_tags = [], # default to all ABIs
whl_platform_tags = ["win_amd64"],
)

# To fetch pip dependencies, use pip.parse. We can pass in various options,
# but typically we pass requirements and the Python version. The Python
# version must have been configured by a corresponding `python.toolchain()`
Expand Down
6 changes: 5 additions & 1 deletion python/config_settings/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,11 @@ string_flag(

string_flag(
name = "pip_whl_osx_version",
build_setting_default = "",
# We choose the oldest non-EOL version at the time when we release `rules_python`.
# See https://endoflife.date/macos
#
# Keep in sync with MODULE.bazel
build_setting_default = "14.0",
# NOTE: Only public because it is used in pip hub repos.
visibility = ["//visibility:public"],
)
Expand Down
3 changes: 2 additions & 1 deletion python/private/internal_config_repo.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,11 @@ such as globals available to Bazel versions, or propagating user environment
settings for rules to later use.
"""

load(":bzlmod_enabled.bzl", "BZLMOD_ENABLED")
load(":repo_utils.bzl", "repo_utils")

_ENABLE_PIPSTAR_ENVVAR_NAME = "RULES_PYTHON_ENABLE_PIPSTAR"
_ENABLE_PIPSTAR_DEFAULT = "0"
_ENABLE_PIPSTAR_DEFAULT = "1" if BZLMOD_ENABLED else "0"
_ENABLE_PYSTAR_ENVVAR_NAME = "RULES_PYTHON_ENABLE_PYSTAR"
_ENABLE_PYSTAR_DEFAULT = "1"
_ENABLE_DEPRECATION_WARNINGS_ENVVAR_NAME = "RULES_PYTHON_DEPRECATION_WARNINGS"
Expand Down
35 changes: 23 additions & 12 deletions python/private/pypi/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -116,11 +116,11 @@ bzl_library(
":parse_whl_name_bzl",
":pep508_env_bzl",
":pip_repository_attrs_bzl",
":python_tag_bzl",
":simpleapi_download_bzl",
":whl_config_setting_bzl",
":whl_library_bzl",
":whl_repo_name_bzl",
":whl_target_platforms_bzl",
"//python/private:full_version_bzl",
"//python/private:normalize_name_bzl",
"//python/private:version_bzl",
Expand Down Expand Up @@ -209,7 +209,7 @@ bzl_library(
":parse_requirements_txt_bzl",
":pypi_repo_utils_bzl",
":requirements_files_by_platform_bzl",
":whl_target_platforms_bzl",
":select_whl_bzl",
"//python/private:normalize_name_bzl",
"//python/private:repo_utils_bzl",
],
Expand Down Expand Up @@ -252,6 +252,9 @@ bzl_library(
bzl_library(
name = "pep508_env_bzl",
srcs = ["pep508_env.bzl"],
deps = [
"//python/private:version_bzl",
],
)

bzl_library(
Expand All @@ -263,11 +266,6 @@ bzl_library(
],
)

bzl_library(
name = "pep508_platform_bzl",
srcs = ["pep508_platform.bzl"],
)

bzl_library(
name = "pep508_requirement_bzl",
srcs = ["pep508_requirement.bzl"],
Expand Down Expand Up @@ -322,8 +320,6 @@ bzl_library(
srcs = ["pkg_aliases.bzl"],
deps = [
":labels_bzl",
":parse_whl_name_bzl",
":whl_target_platforms_bzl",
"//python/private:text_util_bzl",
"@bazel_skylib//lib:selects",
],
Expand All @@ -338,14 +334,20 @@ bzl_library(
],
)

bzl_library(
name = "python_tag_bzl",
srcs = ["python_tag.bzl"],
deps = [
"//python/private:version_bzl",
],
)

bzl_library(
name = "render_pkg_aliases_bzl",
srcs = ["render_pkg_aliases.bzl"],
deps = [
":generate_group_library_build_bazel_bzl",
":parse_whl_name_bzl",
":whl_config_setting_bzl",
":whl_target_platforms_bzl",
"//python/private:normalize_name_bzl",
"//python/private:text_util_bzl",
],
Expand All @@ -359,6 +361,16 @@ bzl_library(
],
)

bzl_library(
name = "select_whl_bzl",
srcs = ["select_whl.bzl"],
deps = [
":parse_whl_name_bzl",
":python_tag_bzl",
"//python/private:version_bzl",
],
)

bzl_library(
name = "simpleapi_download_bzl",
srcs = ["simpleapi_download.bzl"],
Expand Down Expand Up @@ -422,5 +434,4 @@ bzl_library(
bzl_library(
name = "whl_target_platforms_bzl",
srcs = ["whl_target_platforms.bzl"],
deps = [":parse_whl_name_bzl"],
)
Loading