diff --git a/CHANGELOG.md b/CHANGELOG.md index 82aeda8117..3abc10ff33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,9 @@ Unreleased changes template. env var. * (pypi) Downgraded versions of packages: `pip` from `24.3.2` to `24.0.0` and `packaging` from `24.2` to `24.0`. +* (toolchain) `coverage_tool` now accepts a special value of the `//python:none` + that acts as a label that means "no coverage tool". +* (toolchain) The coverage deps are now registered as part of `py_repositories`. {#v0-0-0-fixed} ### Fixed diff --git a/MODULE.bazel b/MODULE.bazel index 7034357f61..0f6252aa16 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,6 +19,33 @@ use_repo( "pypi__build", "pypi__click", "pypi__colorama", + "pypi__coverage", + "pypi__coverage_cp310_aarch64_apple_darwin", + "pypi__coverage_cp310_aarch64_unknown_linux_gnu", + "pypi__coverage_cp310_x86_64_apple_darwin", + "pypi__coverage_cp310_x86_64_unknown_linux_gnu", + "pypi__coverage_cp311_aarch64_apple_darwin", + "pypi__coverage_cp311_aarch64_unknown_linux_gnu", + "pypi__coverage_cp311_x86_64_apple_darwin", + "pypi__coverage_cp311_x86_64_unknown_linux_gnu", + "pypi__coverage_cp312_aarch64_apple_darwin", + "pypi__coverage_cp312_aarch64_unknown_linux_gnu", + "pypi__coverage_cp312_x86_64_apple_darwin", + "pypi__coverage_cp312_x86_64_unknown_linux_gnu", + "pypi__coverage_cp313_aarch64_apple_darwin", + "pypi__coverage_cp313_aarch64_apple_darwin_freethreaded", + "pypi__coverage_cp313_aarch64_unknown_linux_gnu", + "pypi__coverage_cp313_aarch64_unknown_linux_gnu_freethreaded", + "pypi__coverage_cp313_x86_64_unknown_linux_gnu", + "pypi__coverage_cp313_x86_64_unknown_linux_gnu_freethreaded", + "pypi__coverage_cp38_aarch64_apple_darwin", + "pypi__coverage_cp38_aarch64_unknown_linux_gnu", + "pypi__coverage_cp38_x86_64_apple_darwin", + "pypi__coverage_cp38_x86_64_unknown_linux_gnu", + "pypi__coverage_cp39_aarch64_apple_darwin", + "pypi__coverage_cp39_aarch64_unknown_linux_gnu", + "pypi__coverage_cp39_x86_64_apple_darwin", + "pypi__coverage_cp39_x86_64_unknown_linux_gnu", "pypi__importlib_metadata", "pypi__installer", "pypi__more_itertools", diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 14f52c541b..c0543f00b5 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -52,6 +52,12 @@ filegroup( visibility = ["//python:__pkg__"], ) +alias( + name = "coverage", + actual = "@pypi__coverage//coverage", + visibility = ["//visibility:public"], +) + bzl_library( name = "attributes_bzl", srcs = ["attributes.bzl"], @@ -134,7 +140,8 @@ bzl_library( srcs = ["coverage_deps.bzl"], deps = [ ":bazel_tools_bzl", - ":version_label_bzl", + "//python/private/pypi:hub_repository_bzl", + "//python/private/pypi:whl_config_setting_bzl", ], ) diff --git a/python/private/coverage_deps.bzl b/python/private/coverage_deps.bzl index e80e8ee910..d06fd2714c 100644 --- a/python/private/coverage_deps.bzl +++ b/python/private/coverage_deps.bzl @@ -17,7 +17,8 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") -load("//python/private:version_label.bzl", "version_label") +load("//python/private/pypi:hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") +load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # START: maintained by 'bazel run //tools/private/update_deps:update_coverage_deps ' _coverage_deps = { @@ -142,49 +143,63 @@ _coverage_deps = { _coverage_patch = Label("//python/private:coverage.patch") -def coverage_dep(name, python_version, platform, visibility): - """Register a single coverage dependency based on the python version and platform. +def coverage_deps(name): + """Register all coverage dependencies. - Args: - name: The name of the registered repository. - python_version: The full python version. - platform: The platform, which can be found in //python:versions.bzl PLATFORMS dict. - visibility: The visibility of the coverage tool. + This exposes them through a hub repository. - Returns: - The label of the coverage tool if the platform is supported, otherwise - None. + Args: + name: The name of the hub repository. """ - if "windows" in platform: - # NOTE @aignas 2023-01-19: currently we do not support windows as the - # upstream coverage wrapper is written in shell. Do not log any warning - # for now as it is not actionable. - return None - abi = "cp" + version_label(python_version) - url, sha256 = _coverage_deps.get(abi, {}).get(platform, (None, "")) + whl_map = { + str(Label("//python:none")): [ + whl_config_setting(config_setting = "//conditions:default"), + ], + } + for abi, values in _coverage_deps.items(): + for platform, (url, sha256) in values.items(): + spoke_name = "{}_{}_{}".format(name, abi, platform).replace("-", "_") + _, _, filename = url.rpartition("/") - if url == None: - # Some wheels are not present for some builds, so let's silently ignore those. - return None + config_setting = whl_config_setting( + version = "3.{}".format(abi[3:]), + filename = filename, + ) - maybe( - http_archive, - name = name, - build_file_content = """ + whl_map[spoke_name] = [config_setting] + + # NOTE @aignas 2025-02-04: under `bzlmod` the `maybe` function is noop. + # This is only kept for `WORKSPACE` compatibility. + maybe( + http_archive, + name = spoke_name, + build_file_content = """ filegroup( - name = "coverage", + name = "pkg", srcs = ["coverage/__main__.py"], data = glob(["coverage/*.py", "coverage/**/*.py", "coverage/*.so"]), - visibility = {visibility}, + visibility = ["{visibility}"], ) - """.format( - visibility = visibility, - ), - patch_args = ["-p1"], - patches = [_coverage_patch], - sha256 = sha256, - type = "zip", - urls = [url], - ) + """.format( + visibility = "@{}//:__subpackages__".format(name), + ), + patch_args = ["-p1"], + patches = [_coverage_patch], + sha256 = sha256, + type = "zip", + urls = [url], + ) - return "@{name}//:coverage".format(name = name) + maybe( + hub_repository, + name = name, + repo_name = name, + whl_map = { + "coverage": whl_config_settings_to_json(whl_map), + }, + add_requirements_bzl = False, + extra_hub_aliases = {}, + packages = [], + groups = {}, + ) diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index 64d721ecad..c35ae595c3 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -23,6 +23,7 @@ load(":py_exec_tools_toolchain.bzl", "py_exec_tools_toolchain") load(":semver.bzl", "semver") _IS_FREETHREADED = Label("//python/config_settings:is_py_freethreaded") +_NONE = Label("//python:none") def define_hermetic_runtime_toolchain_impl( *, @@ -49,7 +50,7 @@ def define_hermetic_runtime_toolchain_impl( format. python_bin: {type}`str` The path to the Python binary within the repository. - coverage_tool: {type}`str` optional target to the coverage tool to + coverage_tool: {type}`Label` optional target to the coverage tool to use. """ _ = name # @unused @@ -204,8 +205,8 @@ def define_hermetic_runtime_toolchain_impl( }, coverage_tool = select({ # Convert empty string to None - ":coverage_enabled": coverage_tool or None, - "//conditions:default": None, + ":coverage_enabled": coverage_tool, + "//conditions:default": _NONE, }), python_version = "PY3", implementation_name = "cpython", diff --git a/python/private/py_repositories.bzl b/python/private/py_repositories.bzl index 46ca903df4..49d15df96e 100644 --- a/python/private/py_repositories.bzl +++ b/python/private/py_repositories.bzl @@ -18,6 +18,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archi load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load("//python:versions.bzl", "MINOR_MAPPING", "TOOL_VERSIONS") load("//python/private/pypi:deps.bzl", "pypi_deps") +load(":coverage_deps.bzl", "coverage_deps") load(":internal_config_repo.bzl", "internal_config_repo") load(":pythons_hub.bzl", "hub_repo") @@ -34,6 +35,9 @@ def py_repositories(): internal_config_repo, name = "rules_python_internal", ) + coverage_deps( + name = "pypi__coverage", + ) maybe( hub_repo, name = "pythons_hub", diff --git a/python/private/py_runtime_rule.bzl b/python/private/py_runtime_rule.bzl index 5ce8161cf0..2ca7b1c04a 100644 --- a/python/private/py_runtime_rule.bzl +++ b/python/private/py_runtime_rule.bzl @@ -25,6 +25,9 @@ load(":util.bzl", "IS_BAZEL_7_OR_HIGHER") _py_builtins = py_internal +# We need to use the resolved alias value to the sentinel +_NONE = Label("//python/private:sentinel") + def _py_runtime_impl(ctx): interpreter_path = ctx.attr.interpreter_path or None # Convert empty string to None interpreter = ctx.attr.interpreter @@ -61,7 +64,7 @@ def _py_runtime_impl(ctx): else: fail("interpreter must be an executable target or must produce exactly one file.") - if ctx.attr.coverage_tool: + if ctx.attr.coverage_tool and ctx.attr.coverage_tool.label != _NONE: coverage_di = ctx.attr.coverage_tool[DefaultInfo] if _is_singleton_depset(coverage_di.files): @@ -69,7 +72,7 @@ def _py_runtime_impl(ctx): elif coverage_di.files_to_run and coverage_di.files_to_run.executable: coverage_tool = coverage_di.files_to_run.executable else: - fail("coverage_tool must be an executable target or must produce exactly one file.") + fail("coverage_tool must be an executable target or must produce exactly one file, got: {}".format(ctx.attr.coverage_tool.label)) coverage_files = depset(transitive = [ coverage_di.files, @@ -234,6 +237,11 @@ The entry point for the tool must be loadable by a Python interpreter (e.g. a `.py` or `.pyc` file). It must accept the command line arguments of [`coverage.py`](https://coverage.readthedocs.io), at least including the `run` and `lcov` subcommands. + +:::{versionchanged} VERSION_NEXT_PATCH +If set to {obj}`//python:none`, then it will be treated the same as if the attribute +is unset. +::: """, ), "files": attr.label_list( diff --git a/python/private/pypi/deps.bzl b/python/private/pypi/deps.bzl index 31a5201659..b1baaaa0e8 100644 --- a/python/private/pypi/deps.bzl +++ b/python/private/pypi/deps.bzl @@ -16,6 +16,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("//python/private:coverage_deps.bzl", "coverage_deps") _RULE_DEPS = [ # START: maintained by 'bazel run //tools/private/update_deps:update_pip_deps' @@ -145,3 +146,5 @@ def pypi_deps(): type = "zip", build_file_content = _GENERIC_WHEEL, ) + + coverage_deps(name = "pypi__coverage") diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 405c22f60e..84c22d64f9 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -84,7 +84,7 @@ def _create_whl_repos( used during the `repository_rule` and must be always compatible with the host. Returns a {type}`struct` with the following attributes: - whl_map: {type}`dict[str, list[struct]]` the output is keyed by the + whl_map: {type}`dict[str, dict[struct, str]]` the output is keyed by the normalized package name and the values are the instances of the {bzl:obj}`whl_config_setting` return values. exposed_packages: {type}`dict[str, Any]` this is just a way to diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 48245b4106..a836bb5f37 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -26,7 +26,6 @@ exports_files(["requirements.bzl"]) """ def _impl(rctx): - bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() aliases = render_multiplatform_pkg_aliases( aliases = { key: _whl_config_settings_from_json(values) @@ -38,6 +37,12 @@ def _impl(rctx): for path, contents in aliases.items(): rctx.file(path, contents) + if not rctx.attr.add_requirements_bzl: + rctx.file("BUILD.bazel", "") + return + + bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + # NOTE: we are using the canonical name with the double '@' in order to # always uniquely identify a repository, as the labels are being passed as # a string and the resolution of the label happens at the call-site of the @@ -63,6 +68,10 @@ def _impl(rctx): hub_repository = repository_rule( attrs = { + "add_requirements_bzl": attr.bool( + mandatory = False, + default = True, + ), "extra_hub_aliases": attr.string_list_dict( doc = "Extra aliases to make for specific wheels in the hub repo.", mandatory = True, diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index a9eee7be88..09c6c6ed7d 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -123,6 +123,7 @@ def pkg_aliases( actual, group_name = None, extra_aliases = None, + target_names = None, **kwargs): """Create aliases for an actual package. @@ -135,6 +136,9 @@ def pkg_aliases( the aliases to point to mapping to repositories. The keys are passed to bazel skylib's `selects.with_or`, so they can be tuples as well. group_name: {type}`str` The group name that the pkg belongs to. + target_names: {type}`dict[str, str]` the dictionary of aliases that will be + made by default. Defaults to `pkg`, `whl`, `data` and `dist_info` labels. + This will be altered slightly if the {attr}`group_name` is set. extra_aliases: {type}`list[str]` The extra aliases to be created. **kwargs: extra kwargs to pass to {bzl:obj}`get_filename_config_settings`. """ @@ -146,12 +150,14 @@ def pkg_aliases( actual = ":" + PY_LIBRARY_PUBLIC_LABEL, ) - target_names = { - PY_LIBRARY_PUBLIC_LABEL: PY_LIBRARY_IMPL_LABEL if group_name else PY_LIBRARY_PUBLIC_LABEL, - WHEEL_FILE_PUBLIC_LABEL: WHEEL_FILE_IMPL_LABEL if group_name else WHEEL_FILE_PUBLIC_LABEL, - DATA_LABEL: DATA_LABEL, - DIST_INFO_LABEL: DIST_INFO_LABEL, - } | { + if target_names == None: + target_names = { + PY_LIBRARY_PUBLIC_LABEL: PY_LIBRARY_IMPL_LABEL if group_name else PY_LIBRARY_PUBLIC_LABEL, + WHEEL_FILE_PUBLIC_LABEL: WHEEL_FILE_IMPL_LABEL if group_name else WHEEL_FILE_PUBLIC_LABEL, + DATA_LABEL: DATA_LABEL, + DIST_INFO_LABEL: DIST_INFO_LABEL, + } + target_names = target_names | { x: x for x in extra_aliases or [] } @@ -190,7 +196,7 @@ def pkg_aliases( v: "@{repo}//:{target_name}".format( repo = repo, target_name = name, - ) if repo != _INCOMPATIBLE else repo + ) if repo not in [_INCOMPATIBLE, str(_LABEL_NONE)] else repo for v, repo in actual.items() }, ) diff --git a/python/private/python.bzl b/python/private/python.bzl index ec6f73e41f..f93754f11f 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -364,6 +364,10 @@ def _process_single_version_overrides(*, tag, _fail = fail, default): for platform in sha256 } + if tag.coverage_tool: + for platform in sha256: + available_versions[tag.python_version].setdefault("coverage_tool", {})[platform] = tag.coverage_tool + available_versions[tag.python_version] = {k: v for k, v in override.items() if v} if tag.distutils_content: @@ -744,6 +748,19 @@ class. ::: """, attrs = { + "coverage_tool": attr.label( + doc = """\ +The coverage tool to be used for a particular Python interpreter. This can override +`rules_python` defaults. + +This usually is an alias pointing to the right target based on the target +platform. For the restrictions on the underlying targets see +{attr}`py_runtime.coverage_tool` for details. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), # NOTE @aignas 2024-09-01: all of the attributes except for `version` # can be part of the `python.toolchain` call. That would make it more # ergonomic to define new toolchains and to override values for old @@ -818,6 +835,8 @@ configuration, please use {obj}`single_version_override`. doc = """\ The coverage tool to be used for a particular Python interpreter. This can override `rules_python` defaults. + +See {attr}`py_runtime.coverage_tool` for details. """, ), "patch_strip": attr.int( diff --git a/python/private/python_register_toolchains.bzl b/python/private/python_register_toolchains.bzl index cd3e9cbed7..4b96d934f0 100644 --- a/python/private/python_register_toolchains.bzl +++ b/python/private/python_register_toolchains.bzl @@ -23,7 +23,6 @@ load( "TOOL_VERSIONS", "get_release_info", ) -load(":coverage_deps.bzl", "coverage_dep") load(":full_version.bzl", "full_version") load(":python_repository.bzl", "python_repository") load( @@ -33,6 +32,8 @@ load( "toolchains_repo", ) +_NONE = Label("//python:none") + # Wrapper macro around everything above, this is the primary API. def python_register_toolchains( name, @@ -89,21 +90,6 @@ def python_register_toolchains( toolchain_repo_name = "{name}_toolchains".format(name = name) - # When using unreleased Bazel versions, the version is an empty string - if native.bazel_version: - bazel_major = int(native.bazel_version.split(".")[0]) - if bazel_major < 6: - if register_coverage_tool: - # buildifier: disable=print - print(( - "WARNING: ignoring register_coverage_tool=True when " + - "registering @{name}: Bazel 6+ required, got {version}" - ).format( - name = name, - version = native.bazel_version, - )) - register_coverage_tool = False - loaded_platforms = [] for platform in PLATFORMS.keys(): sha256 = tool_versions[python_version]["sha256"].get(platform, None) @@ -117,18 +103,7 @@ def python_register_toolchains( coverage_tool = None coverage_tool = tool_versions[python_version].get("coverage_tool", {}).get(platform, None) if register_coverage_tool and coverage_tool == None: - coverage_tool = coverage_dep( - name = "{name}_{platform}_coverage".format( - name = name, - platform = platform, - ), - python_version = python_version, - platform = platform, - visibility = ["@{name}_{platform}//:__subpackages__".format( - name = name, - platform = platform, - )], - ) + coverage_tool = "@pypi__coverage//coverage" python_repository( name = "{name}_{platform}".format( @@ -143,7 +118,7 @@ def python_register_toolchains( release_filename = release_filename, urls = urls, strip_prefix = strip_prefix, - coverage_tool = coverage_tool, + coverage_tool = coverage_tool or _NONE, **kwargs ) if register_toolchains: diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl index c7407c8f2c..cc1563c5e9 100644 --- a/python/private/python_repository.bzl +++ b/python/private/python_repository.bzl @@ -208,11 +208,6 @@ def _python_repository_impl(rctx): "lib/**", ) - if "windows" in platform: - coverage_tool = None - else: - coverage_tool = rctx.attr.coverage_tool - build_content = """\ # Generated by python/private/python_repositories.bzl @@ -233,7 +228,7 @@ define_hermetic_runtime_toolchain_impl( extra_files_glob_include = render.list(glob_include), python_bin = render.str(python_bin), python_version = render.str(rctx.attr.python_version), - coverage_tool = render.str(coverage_tool), + coverage_tool = render.str(str(rctx.attr.coverage_tool)), ) rctx.delete("python") rctx.symlink(python_bin, "python") @@ -271,7 +266,7 @@ python_repository = repository_rule( "auth_patterns": attr.string_dict( doc = "Override mapping of hostnames to authorization patterns; mirrors the eponymous attribute from http_archive", ), - "coverage_tool": attr.string( + "coverage_tool": attr.label( doc = """ This is a target to use for collecting code coverage information from {rule}`py_binary` and {rule}`py_test` targets. @@ -280,6 +275,11 @@ The target is accepted as a string by the python_repository and evaluated within the context of the toolchain repository. For more information see {attr}`py_runtime.coverage_tool`. + +:::{versionchanged} VERSION_NEXT_FEATURE +This now has a type of {type}`Label` instead of a simple {type}`str` so that users can +more easily provide their own coverage_tool. +::: """, ), "distutils": attr.label(