diff --git a/MODULE.bazel b/MODULE.bazel index 2637b11bf920d..be33a5a52b25f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -115,7 +115,27 @@ register_toolchains( dev_dependency = True, ) -# Proto toolchains +# Define toolchains that use pre-built protoc binaries. +protoc = use_extension("//bazel/private:prebuilt_protoc_extension.bzl", "protoc") +use_repo( + protoc, + "prebuilt_protoc.linux_aarch_64", + "prebuilt_protoc.osx_aarch_64", + "prebuilt_protoc.linux_ppcle_64", + "prebuilt_protoc.linux_s390_64", + "prebuilt_protoc.linux_x86_32", + "prebuilt_protoc.linux_x86_64", + "prebuilt_protoc.osx_x86_64", + "prebuilt_protoc.win32", + "prebuilt_protoc.win64", +) + +# However this registration only matters if the config_setting for prefer_prebuilt_protoc is true, +# using --@protobuf//bazel/toolchains:prefer_prebuilt_protoc +register_toolchains("//bazel/private/toolchains/prebuilt:all") + +# From-source protobuf toolchains +# Fallback if nothing is already registered register_toolchains("//bazel/private/toolchains:all") SUPPORTED_PYTHON_VERSIONS = [ diff --git a/bazel/private/prebuilt_protoc_extension.bzl b/bazel/private/prebuilt_protoc_extension.bzl new file mode 100644 index 0000000000000..844caff42b933 --- /dev/null +++ b/bazel/private/prebuilt_protoc_extension.bzl @@ -0,0 +1,18 @@ +"Module extensions for use under bzlmod" + +load("//toolchain:platforms.bzl", "PROTOBUF_PLATFORMS") +load("//bazel/private:prebuilt_protoc_toolchain.bzl", "prebuilt_protoc_repo") + +def create_all_toolchain_repos(name, version): + for platform in PROTOBUF_PLATFORMS.keys(): + prebuilt_protoc_repo( + # We must replace hyphen with underscore to workaround rules_python py_proto_library constraint + name = ".".join([name, platform.replace("-", "_")]), + platform = platform, + version = version, + ) + +protoc = module_extension( + # TODO: replace version number here during release, maybe with git archive .gitattributes config + implementation = lambda module_ctx: create_all_toolchain_repos("prebuilt_protoc", "v33.0") +) diff --git a/bazel/private/prebuilt_protoc_toolchain.bzl b/bazel/private/prebuilt_protoc_toolchain.bzl new file mode 100644 index 0000000000000..7f6abf229e05c --- /dev/null +++ b/bazel/private/prebuilt_protoc_toolchain.bzl @@ -0,0 +1,62 @@ +"Repository rule that downloads a pre-compiled protoc from our official release for a single platform." + +load(":prebuilt_tool_integrity.bzl", "RELEASED_BINARY_INTEGRITY") +load("//toolchain:platforms.bzl", "PROTOBUF_PLATFORMS") + +def release_version_to_artifact_name(release_version, platform): + # versions have a "v" prefix like "v28.0" + stripped_version = release_version.removeprefix("v") + + # release candidate versions like "v29.0-rc3" have artifact names + # like "protoc-29.0-rc-3-osx-x86_64.zip" + artifact_version = stripped_version.replace("rc", "rc-") + + return "{}-{}-{}.zip".format( + "protoc", + artifact_version, + platform, + ) + +def _prebuilt_protoc_repo_impl(rctx): + release_version = rctx.attr.version + filename = release_version_to_artifact_name( + release_version, + rctx.attr.platform, + ) + rctx.download_and_extract( + url = "https://github.com/protocolbuffers/protobuf/releases/download/{}/{}".format( + release_version, + filename, + ), + sha256 = RELEASED_BINARY_INTEGRITY[filename], + ) + + rctx.file("BUILD.bazel", """\ +# Generated by @protobuf//bazel/private:prebuilt_protoc_toolchain.bzl +load("@com_google_protobuf//bazel/toolchains:proto_toolchain.bzl", "proto_toolchain") + +package(default_visibility = ["//visibility:public"]) + +proto_toolchain( + name = "prebuilt_protoc_toolchain", + proto_compiler = "{protoc_label}", +) +""".format( + protoc_label = "bin/protoc.exe" if rctx.attr.platform.startswith("win") else "bin/protoc", + )) + +prebuilt_protoc_repo = repository_rule( + doc = "Download a pre-built protoc and create a concrete toolchains for it", + implementation = _prebuilt_protoc_repo_impl, + attrs = { + "platform": attr.string( + doc = "A platform that protobuf ships a release for", + mandatory = True, + values = PROTOBUF_PLATFORMS.keys(), + ), + "version": attr.string( + doc = "Release tag from protocolbuffers/protobuf repo, e.g. 'v25.3'", + mandatory = True, + ), + }, +) diff --git a/bazel/private/prebuilt_tool_integrity.bzl b/bazel/private/prebuilt_tool_integrity.bzl index 029aa8a033664..c9435079f8a0b 100644 --- a/bazel/private/prebuilt_tool_integrity.bzl +++ b/bazel/private/prebuilt_tool_integrity.bzl @@ -6,15 +6,20 @@ so that the integrity of the prebuilt tools is included in the release artifact. The checked in content is only here to allow load() statements in the sources to resolve. """ -# Create a mapping for every tool name to the hash of /dev/null -NULLSHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +# Fetched from the last release of protobuf, so the example can work RELEASED_BINARY_INTEGRITY = { - "-".join([ - "protoc", - os, - arch, - ]): NULLSHA - for [os, arch] in { - "linux": ["aarch_64", "x86_64"], - } + "MODULE.bazel.intoto.jsonl": "32ee438bf7e3a210a6b7d5e2d272900a2e7f4a1b4f0992fdf0490281771cac3e", + "protobuf-33.0.tar.gz": "7a796fd9a7947d51e098ebb065d8f8b45ea0ac313ac89cc083456b3005329a1a", + "protobuf-33.0.zip": "aaddf29b205ed915100a5fd096e8252842b67da9accfb7ba91ec3680ea307e45", + "protoc-33.0-linux-aarch_64.zip": "4b96bc91f8b54d829b8c3ca2207ff1ceb774843321e4fa5a68502faece584272", + "protoc-33.0-linux-ppcle_64.zip": "4eb7682900d01e4848fe9b30b9beeffaf9ed2a8d7e8d310c50ed521dbb33411c", + "protoc-33.0-linux-s390_64.zip": "96ee21d761e93bbfa7095ed14e769446c8d9790fecfbd7d6962e858350e0da95", + "protoc-33.0-linux-x86_32.zip": "49edaf078e48d4f45b17be31076ac7dbf64474cd7f1ee3b2cac0938bf0f778f3", + "protoc-33.0-linux-x86_64.zip": "d99c011b799e9e412064244f0be417e5d76c9b6ace13a2ac735330fa7d57ad8f", + "protoc-33.0-osx-aarch_64.zip": "3cf55dd47118bd2efda9cd26b74f8bbbfcf5beb1bf606bc56ad4c001b543f6d3", + "protoc-33.0-osx-universal_binary.zip": "88c0a52f048827d6892cd3403e3ae4181208ab261f93428c86d1736f536a60ec", + "protoc-33.0-osx-x86_64.zip": "e4e50a703147a92d1a5a2d3a34c9e41717f67ade67d4be72b9a466eb8f22fe87", + "protoc-33.0-win32.zip": "3941cc8aeb0e8f59f2143b65f594088f726bb857550dabae5d0dee3bf1392dd1", + "protoc-33.0-win64.zip": "3742cd49c8b6bd78b6760540367eb0ff62fa70a1032e15dafe131bfaf296986a", + "source.json.intoto.jsonl": "139f2eabd41a050cfde345589fb565cad4fb4f2d86dc084ad9bca3cf0806b5d6" } diff --git a/bazel/private/toolchains/prebuilt/BUILD.bazel b/bazel/private/toolchains/prebuilt/BUILD.bazel new file mode 100644 index 0000000000000..961d693117130 --- /dev/null +++ b/bazel/private/toolchains/prebuilt/BUILD.bazel @@ -0,0 +1,40 @@ +"""Create lazy definitions to reference the pre-built protoc toolchains. + +Ensures that Bazel only downloads required binaries for selected toolchains. + +This follows guidance here: +https://docs.bazel.build/versions/main/skylark/deploying.html#registering-toolchains +" +Note that in order to resolve toolchains in the analysis phase +Bazel needs to analyze all toolchain targets that are registered. +Bazel will not need to analyze all targets referenced by toolchain.toolchain attribute. +If in order to register toolchains you need to perform complex computation in the repository, +consider splitting the repository with toolchain targets +from the repository with _toolchain targets. +Former will be always fetched, +and the latter will only be fetched when user actually needs to build code. +" + +The "complex computation" in our case is simply downloading our pre-built protoc binaries. +This guidance tells us how to avoid that: we put the toolchain targets in the alias repository +with only the toolchain attribute pointing into the platform-specific repositories. + +These can be registered in the workspace file or passed to --extra_toolchains flag. +By default all these toolchains are registered by the protoc module extension +so users don't normally need to interact with these targets. +""" + +load("//toolchain:platforms.bzl", "PROTOBUF_PLATFORMS") +[ + toolchain( + name = "{}_toolchain".format(platform.replace("-", "_")), + exec_compatible_with = meta["compatible_with"], + # Toolchain resolution will only permit this toolchain if the config_setting for prefer_prebuilt_protoc is true, + target_settings = ["@com_google_protobuf//bazel/toolchains:prefer_prebuilt_protoc.flag_set"], + # Bazel does not follow this attribute during analysis, so the referenced repo + # will only be fetched if this toolchain is selected. + toolchain = "@prebuilt_protoc.{}//:prebuilt_protoc_toolchain".format(platform.replace("-", "_")), + toolchain_type = "@com_google_protobuf//bazel/private:proto_toolchain_type", + ) + for platform, meta in PROTOBUF_PLATFORMS.items() +] diff --git a/bazel/toolchains/BUILD b/bazel/toolchains/BUILD index 2f629a69e5fa2..7291ae7e0cd3f 100644 --- a/bazel/toolchains/BUILD +++ b/bazel/toolchains/BUILD @@ -1,4 +1,5 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") package(default_applicable_licenses = ["//:license"]) @@ -39,3 +40,16 @@ filegroup( "//bazel:__pkg__", ], ) + +# The public API users set +bool_flag( + name = "prefer_prebuilt_protoc", + # TODO(alexeagle): this should be True after the feature is vetted with some adoption + build_setting_default = False, + visibility = ["//visibility:public"], +) + +config_setting( + name = "prefer_prebuilt_protoc.flag_set", + flag_values = {":prefer_prebuilt_protoc": "true"}, +) diff --git a/examples/.gitignore b/examples/.gitignore index 229542d23ce92..be0a4547c08fa 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -1,2 +1,2 @@ # Ignore the bazel symlinks -/bazel-* +bazel-* diff --git a/examples/example_without_cc_toolchain/.bazelrc b/examples/example_without_cc_toolchain/.bazelrc new file mode 100644 index 0000000000000..d03f5ec1efec0 --- /dev/null +++ b/examples/example_without_cc_toolchain/.bazelrc @@ -0,0 +1,6 @@ +# Simulate a non-functional CC toolchain +common --per_file_copt=external/.*protobuf.*@--THIS_CC_TOOLCHAIN_IS_BROKEN +common --host_per_file_copt=external/.*protobuf.*@--THIS_CC_TOOLCHAIN_IS_BROKEN +# But, users should be able to use pre-built protoc toolchains instead. +common --incompatible_enable_proto_toolchain_resolution +common --@protobuf//bazel/toolchains:prefer_prebuilt_protoc \ No newline at end of file diff --git a/examples/example_without_cc_toolchain/BUILD.bazel b/examples/example_without_cc_toolchain/BUILD.bazel new file mode 100644 index 0000000000000..c6b047c8bf60e --- /dev/null +++ b/examples/example_without_cc_toolchain/BUILD.bazel @@ -0,0 +1,6 @@ +load("@protobuf//bazel:proto_library.bzl", "proto_library") + +proto_library( + name = "empty_proto", + srcs = ["empty.proto"], +) diff --git a/examples/example_without_cc_toolchain/MODULE.bazel b/examples/example_without_cc_toolchain/MODULE.bazel new file mode 100644 index 0000000000000..3a283ffb5278a --- /dev/null +++ b/examples/example_without_cc_toolchain/MODULE.bazel @@ -0,0 +1,13 @@ +"""Bazel module dependencies""" + +module( + name = "com_google_protobuf-example-without-cc-toolchain", + version = "0.0.0", + compatibility_level = 1, +) + +bazel_dep(name = "protobuf") +local_path_override( + module_name = "protobuf", + path = "../..", +) diff --git a/examples/example_without_cc_toolchain/README.md b/examples/example_without_cc_toolchain/README.md new file mode 100644 index 0000000000000..810010de387ad --- /dev/null +++ b/examples/example_without_cc_toolchain/README.md @@ -0,0 +1,2 @@ +This example demonstrates what happens when a Bazel user doesn't have a proper CC toolchain installed. +This case commonly happens in projects with no C++ code, so they don't have a hermetic method of building C++ code. diff --git a/examples/example_without_cc_toolchain/empty.proto b/examples/example_without_cc_toolchain/empty.proto new file mode 100644 index 0000000000000..1a4aede5748d5 --- /dev/null +++ b/examples/example_without_cc_toolchain/empty.proto @@ -0,0 +1,5 @@ +edition = "2023"; + +package examples.without.cc.toolchain; + +message EmptyMessage {} diff --git a/toolchain/platforms.bzl b/toolchain/platforms.bzl new file mode 100644 index 0000000000000..dbd386a1044bc --- /dev/null +++ b/toolchain/platforms.bzl @@ -0,0 +1,64 @@ +"List of published platforms on protobuf GitHub releases" + +# Keys are chosen to match the filenames published on protocolbuffers/protobuf releases +# NB: keys in this list are nearly identical to /toolchain/BUILD.bazel#TOOLCHAINS +# Perhaps we should share code. +PROTOBUF_PLATFORMS = { + # "k8", # this is in /toolchain/BUILD.bazel but not a released platform + # "osx-universal_binary", # this is not in /toolchain/BUILD.bazel + # but also Bazel will never request it, as we have a darwin binary for each architecture + "linux-aarch_64": { + "compatible_with": [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], + }, + "linux-ppcle_64": { + "compatible_with": [ + "@platforms//os:linux", + "@platforms//cpu:ppc64le", + ], + }, + "linux-s390_64": { + "compatible_with": [ + "@platforms//os:linux", + "@platforms//cpu:s390x", + ], + }, + "linux-x86_32": { + "compatible_with": [ + "@platforms//os:linux", + "@platforms//cpu:x86_32", + ], + }, + "linux-x86_64": { + "compatible_with": [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + }, + "osx-aarch_64": { + "compatible_with": [ + "@platforms//os:macos", + "@platforms//cpu:aarch64", + ], + }, + "osx-x86_64": { + "compatible_with": [ + "@platforms//os:macos", + "@platforms//cpu:x86_64", + ], + }, + "win32": { + "compatible_with": [ + "@platforms//os:windows", + "@platforms//cpu:x86_32", + ], + }, + "win64": { + "compatible_with": [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], + }, +}