WIP: wire up prebuilt protoc toolchain

MODULE.bazel

@@ -115,7 +115,27 @@ register_toolchains(
     dev_dependency = True,
 )
 
-# Proto toolchains
+# Define toolchains that use pre-built protoc binaries.
+protoc = use_extension("//bazel/private:prebuilt_protoc_extension.bzl", "protoc")
+use_repo(
+    protoc,
+    "prebuilt_protoc.linux_aarch_64",
+    "prebuilt_protoc.osx_aarch_64",
+    "prebuilt_protoc.linux_ppcle_64",
+    "prebuilt_protoc.linux_s390_64",
+    "prebuilt_protoc.linux_x86_32",
+    "prebuilt_protoc.linux_x86_64",
+    "prebuilt_protoc.osx_x86_64",
+    "prebuilt_protoc.win32",
+    "prebuilt_protoc.win64",
+)
+
+# However this registration only matters if the config_setting for prefer_prebuilt_protoc is true,
+# using --@protobuf//bazel/toolchains:prefer_prebuilt_protoc
+register_toolchains("//bazel/private/toolchains/prebuilt:all")
+
+# From-source protobuf toolchains
+# Fallback if nothing is already registered
 register_toolchains("//bazel/private/toolchains:all")
 
 SUPPORTED_PYTHON_VERSIONS = [

bazel/private/prebuilt_protoc_extension.bzl

@@ -0,0 +1,18 @@
+"Module extensions for use under bzlmod"
+
+load("//toolchain:platforms.bzl", "PROTOBUF_PLATFORMS")
+load("//bazel/private:prebuilt_protoc_toolchain.bzl", "prebuilt_protoc_repo")
+
+def create_all_toolchain_repos(name, version):
+    for platform in PROTOBUF_PLATFORMS.keys():
+        prebuilt_protoc_repo(
+            # We must replace hyphen with underscore to workaround rules_python py_proto_library constraint
+            name = ".".join([name, platform.replace("-", "_")]),
+            platform = platform,
+            version = version,
+        )
+
+protoc = module_extension(
+    # TODO: replace version number here during release, maybe with git archive .gitattributes config
+    implementation = lambda module_ctx: create_all_toolchain_repos("prebuilt_protoc", "v33.0")
+)

bazel/private/prebuilt_protoc_toolchain.bzl

@@ -0,0 +1,62 @@
+"Repository rule that downloads a pre-compiled protoc from our official release for a single platform."
+
+load(":prebuilt_tool_integrity.bzl", "RELEASED_BINARY_INTEGRITY")
+load("//toolchain:platforms.bzl", "PROTOBUF_PLATFORMS")
+
+def release_version_to_artifact_name(release_version, platform):
+    # versions have a "v" prefix like "v28.0"
+    stripped_version = release_version.removeprefix("v")
+
+    # release candidate versions like "v29.0-rc3" have artifact names
+    # like "protoc-29.0-rc-3-osx-x86_64.zip"
+    artifact_version = stripped_version.replace("rc", "rc-")
+
+    return "{}-{}-{}.zip".format(
+        "protoc",
+        artifact_version,
+        platform,
+    )
+
+def _prebuilt_protoc_repo_impl(rctx):
+    release_version = rctx.attr.version
+    filename = release_version_to_artifact_name(
+        release_version,
+        rctx.attr.platform,
+    )
+    rctx.download_and_extract(
+        url = "https://github.com/protocolbuffers/protobuf/releases/download/{}/{}".format(
+            release_version,
+            filename,
+        ),
+        sha256 = RELEASED_BINARY_INTEGRITY[filename],
+    )
+
+    rctx.file("BUILD.bazel", """\
+# Generated by @protobuf//bazel/private:prebuilt_protoc_toolchain.bzl
+load("@com_google_protobuf//bazel/toolchains:proto_toolchain.bzl", "proto_toolchain")
+
+package(default_visibility = ["//visibility:public"])
+
+proto_toolchain(
+    name = "prebuilt_protoc_toolchain",
+    proto_compiler = "{protoc_label}",
+)
+""".format(
+        protoc_label = ":bin/protoc.exe" if rctx.attr.platform.startswith("win") else ":bin/protoc",
+    ))
+
+prebuilt_protoc_repo = repository_rule(
+    doc = "Download a pre-built protoc and create a concrete toolchains for it",
+    implementation = _prebuilt_protoc_repo_impl,
+    attrs = {
+        "platform": attr.string(
+            doc = "A platform that protobuf ships a release for",
+            mandatory = True,
+            values = PROTOBUF_PLATFORMS.keys(),
+        ),
+        "version": attr.string(
+            doc = "Release tag from protocolbuffers/protobuf repo, e.g. 'v25.3'",
+            mandatory = True,
+        ),
+    },
+)

bazel/private/prebuilt_tool_integrity.bzl

@@ -6,15 +6,20 @@ so that the integrity of the prebuilt tools is included in the release artifact.
 The checked in content is only here to allow load() statements in the sources to resolve.
 """
 
-# Create a mapping for every tool name to the hash of /dev/null
-NULLSHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+# Fetched from the last release of protobuf, so the example can work
 RELEASED_BINARY_INTEGRITY = {
-    "-".join([
-        "protoc",
-        os,
-        arch,
-    ]): NULLSHA
-    for [os, arch] in {
-        "linux": ["aarch_64", "x86_64"],
-    }
+  "MODULE.bazel.intoto.jsonl": "32ee438bf7e3a210a6b7d5e2d272900a2e7f4a1b4f0992fdf0490281771cac3e",
+  "protobuf-33.0.tar.gz": "7a796fd9a7947d51e098ebb065d8f8b45ea0ac313ac89cc083456b3005329a1a",
+  "protobuf-33.0.zip": "aaddf29b205ed915100a5fd096e8252842b67da9accfb7ba91ec3680ea307e45",
+  "protoc-33.0-linux-aarch_64.zip": "4b96bc91f8b54d829b8c3ca2207ff1ceb774843321e4fa5a68502faece584272",
+  "protoc-33.0-linux-ppcle_64.zip": "4eb7682900d01e4848fe9b30b9beeffaf9ed2a8d7e8d310c50ed521dbb33411c",
+  "protoc-33.0-linux-s390_64.zip": "96ee21d761e93bbfa7095ed14e769446c8d9790fecfbd7d6962e858350e0da95",
+  "protoc-33.0-linux-x86_32.zip": "49edaf078e48d4f45b17be31076ac7dbf64474cd7f1ee3b2cac0938bf0f778f3",
+  "protoc-33.0-linux-x86_64.zip": "d99c011b799e9e412064244f0be417e5d76c9b6ace13a2ac735330fa7d57ad8f",
+  "protoc-33.0-osx-aarch_64.zip": "3cf55dd47118bd2efda9cd26b74f8bbbfcf5beb1bf606bc56ad4c001b543f6d3",
+  "protoc-33.0-osx-universal_binary.zip": "88c0a52f048827d6892cd3403e3ae4181208ab261f93428c86d1736f536a60ec",
+  "protoc-33.0-osx-x86_64.zip": "e4e50a703147a92d1a5a2d3a34c9e41717f67ade67d4be72b9a466eb8f22fe87",
+  "protoc-33.0-win32.zip": "3941cc8aeb0e8f59f2143b65f594088f726bb857550dabae5d0dee3bf1392dd1",
+  "protoc-33.0-win64.zip": "3742cd49c8b6bd78b6760540367eb0ff62fa70a1032e15dafe131bfaf296986a",
+  "source.json.intoto.jsonl": "139f2eabd41a050cfde345589fb565cad4fb4f2d86dc084ad9bca3cf0806b5d6"
 }

bazel/private/toolchains/prebuilt/BUILD.bazel

@@ -0,0 +1,40 @@
+"""Create lazy definitions to reference the pre-built protoc toolchains.
+
+Ensures that Bazel only downloads required binaries for selected toolchains.
+
+This follows guidance here:
+https://docs.bazel.build/versions/main/skylark/deploying.html#registering-toolchains
+"
+Note that in order to resolve toolchains in the analysis phase
+Bazel needs to analyze all toolchain targets that are registered.
+Bazel will not need to analyze all targets referenced by toolchain.toolchain attribute.
+If in order to register toolchains you need to perform complex computation in the repository,
+consider splitting the repository with toolchain targets
+from the repository with <LANG>_toolchain targets.
+Former will be always fetched,
+and the latter will only be fetched when user actually needs to build <LANG> code.
+"
+
+The "complex computation" in our case is simply downloading our pre-built protoc binaries.
+This guidance tells us how to avoid that: we put the toolchain targets in the alias repository
+with only the toolchain attribute pointing into the platform-specific repositories.
+
+These can be registered in the workspace file or passed to --extra_toolchains flag.
+By default all these toolchains are registered by the protoc module extension
+so users don't normally need to interact with these targets.
+"""
+
+load("//toolchain:platforms.bzl", "PROTOBUF_PLATFORMS")
+[
+    toolchain(
+        name = "{}_toolchain".format(platform.replace("-", "_")),
+        exec_compatible_with = meta["compatible_with"],
+        # Toolchain resolution will only permit this toolchain if the config_setting for prefer_prebuilt_protoc is true,
+        target_settings = ["@com_google_protobuf//bazel/toolchains:prefer_prebuilt_protoc.flag_set"],
+        # Bazel does not follow this attribute during analysis, so the referenced repo
+        # will only be fetched if this toolchain is selected.
+        toolchain = "@prebuilt_protoc.{}//:prebuilt_protoc_toolchain".format(platform.replace("-", "_")),
+        toolchain_type = "@com_google_protobuf//bazel/private:proto_toolchain_type",
+    )
+    for platform, meta in PROTOBUF_PLATFORMS.items()
+]

bazel/toolchains/BUILD

@@ -1,4 +1,5 @@
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
 
 package(default_applicable_licenses = ["//:license"])
 
@@ -39,3 +40,15 @@ filegroup(
         "//bazel:__pkg__",
     ],
 )
+
+# The public API users set
+bool_flag(
+    name = "prefer_prebuilt_protoc",
+    build_setting_default = False,
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "prefer_prebuilt_protoc.flag_set",
+    flag_values = {":prefer_prebuilt_protoc": "true"},
+)

examples/.gitignore

@@ -1,2 +1,2 @@
 # Ignore the bazel symlinks
-/bazel-*
+bazel-*

examples/example_without_cc_toolchain/.bazelrc

@@ -0,0 +1,6 @@
+# Simulate a non-functional CC toolchain
+common --per_file_copt=external/.*protobuf.*@--THIS_CC_TOOLCHAIN_IS_BROKEN
+common --host_per_file_copt=external/.*protobuf.*@--THIS_CC_TOOLCHAIN_IS_BROKEN
+# But, users should be able to use pre-built protoc toolchains instead.
+common --incompatible_enable_proto_toolchain_resolution
+common --@protobuf//bazel/toolchains:prefer_prebuilt_protoc

examples/example_without_cc_toolchain/BUILD.bazel

@@ -0,0 +1,6 @@
+load("@protobuf//bazel:proto_library.bzl", "proto_library")
+
+proto_library(
+    name = "empty_proto",
+    srcs = ["empty.proto"],
+)

examples/example_without_cc_toolchain/MODULE.bazel

@@ -0,0 +1,13 @@
+"""Bazel module dependencies"""
+
+module(
+    name = "com_google_protobuf-example-without-cc-toolchain",
+    version = "0.0.0",
+    compatibility_level = 1,
+)
+
+bazel_dep(name = "protobuf")
+local_path_override(
+    module_name = "protobuf",
+    path = "../..",
+)

examples/example_without_cc_toolchain/README.md

@@ -0,0 +1,2 @@
+This example demonstrates what happens when a Bazel user doesn't have a proper CC toolchain installed.
+This case commonly happens in projects with no C++ code, so they don't have a hermetic method of building C++ code.

examples/example_without_cc_toolchain/empty.proto

@@ -0,0 +1,5 @@
+edition = "2023";
+
+package examples.without.cc.toolchain;
+
+message EmptyMessage {}

toolchain/platforms.bzl

@@ -0,0 +1,64 @@
+"List of published platforms on protobuf GitHub releases"
+
+# Keys are chosen to match the filenames published on protocolbuffers/protobuf releases
+# NB: keys in this list are nearly identical to /toolchain/BUILD.bazel#TOOLCHAINS
+# Perhaps we should share code.
+PROTOBUF_PLATFORMS = {
+    # "k8", # this is in /toolchain/BUILD.bazel but not a released platform
+    # "osx-universal_binary", # this is not in /toolchain/BUILD.bazel
+    # but also Bazel will never request it, as we have a darwin binary for each architecture
+    "linux-aarch_64": {
+        "compatible_with": [
+            "@platforms//os:linux",
+            "@platforms//cpu:aarch64",
+        ],
+    },
+    "linux-ppcle_64": {
+        "compatible_with": [
+            "@platforms//os:linux",
+            "@platforms//cpu:ppc64le",
+        ],
+    },
+    "linux-s390_64": {
+        "compatible_with": [
+            "@platforms//os:linux",
+            "@platforms//cpu:s390x",
+        ],
+    },
+    "linux-x86_32": {
+        "compatible_with": [
+            "@platforms//os:linux",
+            "@platforms//cpu:x86_32",
+        ],
+    },
+    "linux-x86_64": {
+        "compatible_with": [
+            "@platforms//os:linux",
+            "@platforms//cpu:x86_64",
+        ],
+    },
+    "osx-aarch_64": {
+        "compatible_with": [
+            "@platforms//os:macos",
+            "@platforms//cpu:aarch64",
+        ],
+    },
+    "osx-x86_64": {
+        "compatible_with": [
+            "@platforms//os:macos",
+            "@platforms//cpu:x86_64",
+        ],
+    },
+    "win32": {
+        "compatible_with": [
+            "@platforms//os:windows",
+            "@platforms//cpu:x86_32",
+        ],
+    },
+    "win64": {
+        "compatible_with": [
+            "@platforms//os:windows",
+            "@platforms//cpu:x86_64",
+        ],
+    },
+}