feat: Expose Python C headers through the toolchain. (#1287)

This allows getting a build's `cc_library` of Python headers through
toolchain resolution instead of having to use the underlying toolchain's
repository `:python_headers` target directly.

Without this feature, it's not possible to reliably and correctly get
the C information about the runtime a build is going to use. Existing
solutions require carefully setting up repo names, external state,
and/or using specific build rules. In comparison, with this feature,
consumers are able to simply ask for the current headers via a helper
target or manually lookup the toolchain and pull the relevant
information; toolchain resolution handles finding the correct headers.

The basic way this works is by registering a second toolchain to carry
C/C++ related information; as such, it is named `py_cc_toolchain`. The
py cc toolchain has the same constraint settings as the regular py
toolchain; an expected invariant is that there is a 1:1 correspondence
between the two. This base functionality allows a consuming rule
implementation to use toolchain resolution to find the Python C
toolchain information.

Usually what downstream consumers need are the headers to feed into
another `cc_library` (or equivalent) target, so, rather than have every
project re-implement the same "lookup and forward cc_library info"
logic,
this is provided by the `//python/cc:current_py_cc_headers` target.
Targets that need the headers can then depend on that target as if it
was a `cc_library` target.

Work towards #824

Author: rickeylev

.bazelci/presubmit.yml

@@ -113,8 +113,19 @@ tasks:
     <<: *reusable_config
     name: Test on RBE using minimum supported Bazel version
     platform: rbe_ubuntu1604
+    build_flags:
+      # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1,
+      # which prevents cc toolchain autodetection from working correctly
+      # on Bazel 5.4 and earlier. To workaround this, manually specify the
+      # build kite cc toolchain.
+      - "--extra_toolchains=@buildkite_config//config:cc-toolchain"
     test_flags:
       - "--test_tag_filters=-integration-test,-acceptance-test"
+      # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1,
+      # which prevents cc toolchain autodetection from working correctly
+      # on Bazel 5.4 and earlier. To workaround this, manually specify the
+      # build kite cc toolchain.
+      - "--extra_toolchains=@buildkite_config//config:cc-toolchain"
   rbe:
     <<: *reusable_config
     name: Test on RBE

.bazelrc

@@ -3,8 +3,8 @@
 # This lets us glob() up all the files inside the examples to make them inputs to tests
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_point,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_point,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
 
 test --test_output=errors
 

docs/BUILD.bazel

@@ -25,6 +25,8 @@ _DOCS = {
     "packaging": "//docs:packaging-docs",
     "pip": "//docs:pip-docs",
     "pip_repository": "//docs:pip-repository",
+    "py_cc_toolchain": "//docs:py_cc_toolchain-docs",
+    "py_cc_toolchain_info": "//docs:py_cc_toolchain_info-docs",
     "python": "//docs:core-docs",
 }
 
@@ -134,6 +136,25 @@ stardoc(
     deps = [":packaging_bzl"],
 )
 
+stardoc(
+    name = "py_cc_toolchain-docs",
+    out = "py_cc_toolchain.md_",
+    # NOTE: The public file isn't used as the input because it would document
+    # the macro, which doesn't have the attribute documentation. The macro
+    # doesn't do anything interesting to users, so bypass it to avoid having to
+    # copy/paste all the rule's doc in the macro.
+    input = "//python/private:py_cc_toolchain_rule.bzl",
+    target_compatible_with = _NOT_WINDOWS,
+    deps = ["//python/private:py_cc_toolchain_bzl"],
+)
+
+stardoc(
+    name = "py_cc_toolchain_info-docs",
+    out = "py_cc_toolchain_info.md_",
+    input = "//python/cc:py_cc_toolchain_info.bzl",
+    deps = ["//python/cc:py_cc_toolchain_info_bzl"],
+)
+
 [
     diff_test(
         name = "check_" + k,

docs/py_cc_toolchain.md

@@ -33,6 +33,7 @@ licenses(["notice"])
 filegroup(
     name = "distribution",
     srcs = glob(["**"]) + [
+        "//python/cc:distribution",
         "//python/config_settings:distribution",
         "//python/constraints:distribution",
         "//python/private:distribution",

docs/py_cc_toolchain_info.md

@@ -0,0 +1,44 @@
+# Package for C/C++ specific functionality of the Python rules.
+
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load("//python/private:current_py_cc_headers.bzl", "current_py_cc_headers")
+load("//python/private:util.bzl", "BZLMOD_ENABLED")
+
+package(
+    default_visibility = ["//:__subpackages__"],
+)
+
+# This target provides the C headers for whatever the current toolchain is
+# for the consuming rule. It basically acts like a cc_library by forwarding
+# on the providers for the underlying cc_library that the toolchain is using.
+current_py_cc_headers(
+    name = "current_py_cc_headers",
+    # Building this directly will fail unless a py cc toolchain is registered,
+    # and it's only under bzlmod that one is registered by default.
+    tags = [] if BZLMOD_ENABLED else ["manual"],
+    visibility = ["//visibility:public"],
+)
+
+toolchain_type(
+    name = "toolchain_type",
+    visibility = ["//visibility:public"],
+)
+
+bzl_library(
+    name = "py_cc_toolchain_bzl",
+    srcs = ["py_cc_toolchain.bzl"],
+    visibility = ["//visibility:public"],
+    deps = ["//python/private:py_cc_toolchain_bzl"],
+)
+
+bzl_library(
+    name = "py_cc_toolchain_info_bzl",
+    srcs = ["py_cc_toolchain_info.bzl"],
+    visibility = ["//visibility:public"],
+    deps = ["//python/private:py_cc_toolchain_info_bzl"],
+)
+
+filegroup(
+    name = "distribution",
+    srcs = glob(["**"]),
+)

python/BUILD.bazel

@@ -0,0 +1,19 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Public entry point for py_cc_toolchain rule."""
+
+load("//python/private:py_cc_toolchain_macro.bzl", _py_cc_toolchain = "py_cc_toolchain")
+
+py_cc_toolchain = _py_cc_toolchain

python/cc/BUILD.bazel

@@ -0,0 +1,23 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provider for C/C++ information about the Python runtime.
+
+NOTE: This is a beta-quality feature. APIs subject to change until
+https://github.com/bazelbuild/rules_python/issues/824 is considered done.
+"""
+
+load("//python/private:py_cc_toolchain_info.bzl", _PyCcToolchainInfo = "PyCcToolchainInfo")
+
+PyCcToolchainInfo = _PyCcToolchainInfo

python/cc/py_cc_toolchain.bzl

@@ -51,6 +51,28 @@ bzl_library(
     deps = ["@bazel_skylib//lib:types"],
 )
 
+bzl_library(
+    name = "py_cc_toolchain_bzl",
+    srcs = [
+        "py_cc_toolchain_macro.bzl",
+        "py_cc_toolchain_rule.bzl",
+    ],
+    visibility = [
+        "//docs:__subpackages__",
+        "//python/cc:__pkg__",
+    ],
+    deps = [
+        ":py_cc_toolchain_info_bzl",
+        ":util_bzl",
+    ],
+)
+
+bzl_library(
+    name = "py_cc_toolchain_info_bzl",
+    srcs = ["py_cc_toolchain_info.bzl"],
+    visibility = ["//python/cc:__pkg__"],
+)
+
 # @bazel_tools can't define bzl_library itself, so we just put a wrapper around it.
 bzl_library(
     name = "bazel_tools_bzl",
@@ -73,6 +95,7 @@ exports_files(
         "reexports.bzl",
         "stamp.bzl",
         "util.bzl",
+        "py_cc_toolchain_rule.bzl",
     ],
     visibility = ["//docs:__pkg__"],
 )