WIP: wire up prebuilt protoc toolchain

bazel/extensions.bzl

@@ -0,0 +1,61 @@
+"Module extensions for use under bzlmod"
+
+load("@proto_bazel_features//:features.bzl", "bazel_features")
+load("//bazel/toolchains:prebuilt_toolchains.bzl", "prebuilt_toolchains_repo", "PROTOC_PLATFORMS")
+load("//bazel/private:prebuilt_protoc_toolchain.bzl", "prebuilt_protoc_repo")
+
+DEFAULT_REPOSITORY = "prebuilt_protoc_hub"
+
+def create_all_toolchain_repos(name, version):
+    for platform in PROTOC_PLATFORMS.keys():
+        prebuilt_protoc_repo(
+            # We must replace hyphen with underscore to workaround rules_python py_proto_library constraint
+            name = ".".join([name, platform.replace("-", "_")]),
+            platform = platform,
+            version = version,
+        )
+    # name will be mangled by bzlmod into apparent name, so pass an extra copy that is preserved
+    prebuilt_toolchains_repo(name = name, user_repository_name = name)
+
+def _proto_extension_impl(module_ctx):
+    registrations = {}
+    root_name = None
+    for mod in module_ctx.modules:
+        for toolchain in mod.tags.prebuilt_toolchain:
+            if toolchain.name != DEFAULT_REPOSITORY and not mod.is_root:
+                fail("""\
+                Only the root module may override the default name for the toolchain.
+                This prevents conflicting registrations in the global namespace of external repos.
+                """)
+
+            # Ensure the root wins in case of differences
+            if mod.is_root:
+                create_all_toolchain_repos(toolchain.name, toolchain.version)
+                root_name = toolchain.name
+            elif toolchain.name not in registrations.keys():
+                registrations[toolchain.name] = toolchain
+    for name, toolchain in registrations.items():
+        if name != root_name:
+            create_all_toolchain_repos(name, toolchain.version)
+
+    if bazel_features.external_deps.extension_metadata_has_reproducible:
+        return module_ctx.extension_metadata(reproducible = True)
+    else:
+        return None
+
+protoc = module_extension(
+    implementation = _proto_extension_impl,
+    tag_classes = {
+        # buildifier: disable=unsorted-dict-items
+        "prebuilt_toolchain": tag_class(attrs = {
+            "name": attr.string(
+                doc = """\
+                Base name for generated repositories, allowing more than one toolchain to be registered.
+                Overriding the default is only permitted in the root module.
+                """,
+                default = DEFAULT_REPOSITORY,
+            ),
+            "version": attr.string(doc = "A tag of protocolbuffers/protobuf repository."),
+        }),
+    },
+)

bazel/private/prebuilt_protoc_toolchain.bzl

@@ -0,0 +1,62 @@
+"Repository rule that downloads a pre-compiled protoc from our official release for a single platform."
+
+load(":prebuilt_tool_integrity.bzl", "RELEASED_BINARY_INTEGRITY")
+load("//bazel/toolchains:prebuilt_toolchains.bzl", "PROTOC_PLATFORMS")
+
+def release_version_to_artifact_name(release_version, platform):
+    # versions have a "v" prefix like "v28.0"
+    stripped_version = release_version.removeprefix("v")
+
+    # release candidate versions like "v29.0-rc3" have artifact names
+    # like "protoc-29.0-rc-3-osx-x86_64.zip"
+    artifact_version = stripped_version.replace("rc", "rc-")
+
+    return "{}-{}-{}.zip".format(
+        "protoc",
+        artifact_version,
+        platform,
+    )
+
+def _prebuilt_protoc_repo_impl(rctx):
+    release_version = rctx.attr.version
+    filename = release_version_to_artifact_name(
+        release_version,
+        rctx.attr.platform,
+    )
+    rctx.download_and_extract(
+        url = "https://github.com/protocolbuffers/protobuf/releases/download/{}/{}".format(
+            release_version,
+            filename,
+        ),
+        sha256 = RELEASED_BINARY_INTEGRITY[filename],
+    )
+
+    rctx.file("BUILD.bazel", """\
+# Generated by @protobuf//bazel/private:prebuilt_protoc_toolchain.bzl
+load("@com_google_protobuf//bazel/toolchains:proto_toolchain.bzl", "proto_toolchain")
+
+package(default_visibility = ["//visibility:public"])
+
+proto_toolchain(
+    name = "prebuilt_protoc_toolchain",
+    proto_compiler = "{protoc_label}",
+)
+""".format(
+        protoc_label = ":bin/protoc.exe" if rctx.attr.platform.startswith("win") else ":bin/protoc",
+    ))
+
+prebuilt_protoc_repo = repository_rule(
+    doc = "Download a pre-built protoc and create a concrete toolchains for it",
+    implementation = _prebuilt_protoc_repo_impl,
+    attrs = {
+        "platform": attr.string(
+            doc = "A platform that protobuf ships a release for",
+            mandatory = True,
+            values = PROTOC_PLATFORMS.keys(),
+        ),
+        "version": attr.string(
+            doc = "Release tag from protocolbuffers/protobuf repo, e.g. 'v25.3'",
+            mandatory = True,
+        ),
+    },
+)

bazel/private/prebuilt_tool_integrity.bzl

@@ -6,15 +6,20 @@ so that the integrity of the prebuilt tools is included in the release artifact.
 The checked in content is only here to allow load() statements in the sources to resolve.
 """
 
-# Create a mapping for every tool name to the hash of /dev/null
-NULLSHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+# Fetched from the last release of protobuf, so the example can work
 RELEASED_BINARY_INTEGRITY = {
-    "-".join([
-        "protoc",
-        os,
-        arch,
-    ]): NULLSHA
-    for [os, arch] in {
-        "linux": ["aarch_64", "x86_64"],
-    }
+  "MODULE.bazel.intoto.jsonl": "32ee438bf7e3a210a6b7d5e2d272900a2e7f4a1b4f0992fdf0490281771cac3e",
+  "protobuf-33.0.tar.gz": "7a796fd9a7947d51e098ebb065d8f8b45ea0ac313ac89cc083456b3005329a1a",
+  "protobuf-33.0.zip": "aaddf29b205ed915100a5fd096e8252842b67da9accfb7ba91ec3680ea307e45",
+  "protoc-33.0-linux-aarch_64.zip": "4b96bc91f8b54d829b8c3ca2207ff1ceb774843321e4fa5a68502faece584272",
+  "protoc-33.0-linux-ppcle_64.zip": "4eb7682900d01e4848fe9b30b9beeffaf9ed2a8d7e8d310c50ed521dbb33411c",
+  "protoc-33.0-linux-s390_64.zip": "96ee21d761e93bbfa7095ed14e769446c8d9790fecfbd7d6962e858350e0da95",
+  "protoc-33.0-linux-x86_32.zip": "49edaf078e48d4f45b17be31076ac7dbf64474cd7f1ee3b2cac0938bf0f778f3",
+  "protoc-33.0-linux-x86_64.zip": "d99c011b799e9e412064244f0be417e5d76c9b6ace13a2ac735330fa7d57ad8f",
+  "protoc-33.0-osx-aarch_64.zip": "3cf55dd47118bd2efda9cd26b74f8bbbfcf5beb1bf606bc56ad4c001b543f6d3",
+  "protoc-33.0-osx-universal_binary.zip": "88c0a52f048827d6892cd3403e3ae4181208ab261f93428c86d1736f536a60ec",
+  "protoc-33.0-osx-x86_64.zip": "e4e50a703147a92d1a5a2d3a34c9e41717f67ade67d4be72b9a466eb8f22fe87",
+  "protoc-33.0-win32.zip": "3941cc8aeb0e8f59f2143b65f594088f726bb857550dabae5d0dee3bf1392dd1",
+  "protoc-33.0-win64.zip": "3742cd49c8b6bd78b6760540367eb0ff62fa70a1032e15dafe131bfaf296986a",
+  "source.json.intoto.jsonl": "139f2eabd41a050cfde345589fb565cad4fb4f2d86dc084ad9bca3cf0806b5d6"
 }

bazel/toolchains/prebuilt_toolchains.bzl

@@ -0,0 +1,122 @@
+"""Create a "hub" repository to reference the pre-built protoc toolchains.
+
+Ensures that Bazel only downloads required binaries for selected toolchains.
+
+This follows guidance here:
+https://docs.bazel.build/versions/main/skylark/deploying.html#registering-toolchains
+"
+Note that in order to resolve toolchains in the analysis phase
+Bazel needs to analyze all toolchain targets that are registered.
+Bazel will not need to analyze all targets referenced by toolchain.toolchain attribute.
+If in order to register toolchains you need to perform complex computation in the repository,
+consider splitting the repository with toolchain targets
+from the repository with <LANG>_toolchain targets.
+Former will be always fetched,
+and the latter will only be fetched when user actually needs to build <LANG> code.
+"
+
+The "complex computation" in our case is simply downloading our pre-built protoc binaries.
+This guidance tells us how to avoid that: we put the toolchain targets in the alias repository
+with only the toolchain attribute pointing into the platform-specific repositories.
+"""
+
+# Keys are chosen to match the filenames published on protocolbuffers/protobuf releases
+# NB: keys in this list are nearly identical to /toolchain/BUILD.bazel#TOOLCHAINS
+# Perhaps we should share code.
+PROTOC_PLATFORMS = {
+    # "k8", # this is in /toolchain/BUILD.bazel but not a released platform
+    # "osx-universal_binary", # this is not in /toolchain/BUILD.bazel
+    # but also Bazel will never request it, as we have a darwin binary for each architecture
+    "linux-aarch_64": {
+        "compatible_with": [
+            "@platforms//os:linux",
+            "@platforms//cpu:aarch64",
+        ],
+    },
+    "linux-ppcle_64": {
+        "compatible_with": [
+            "@platforms//os:linux",
+            "@platforms//cpu:ppc64le",
+        ],
+    },
+    "linux-s390_64": {
+        "compatible_with": [
+            "@platforms//os:linux",
+            "@platforms//cpu:s390x",
+        ],
+    },
+    "linux-x86_32": {
+        "compatible_with": [
+            "@platforms//os:linux",
+            "@platforms//cpu:x86_32",
+        ],
+    },
+    "linux-x86_64": {
+        "compatible_with": [
+            "@platforms//os:linux",
+            "@platforms//cpu:x86_64",
+        ],
+    },
+    "osx-aarch_64": {
+        "compatible_with": [
+            "@platforms//os:macos",
+            "@platforms//cpu:aarch64",
+        ],
+    },
+    "osx-x86_64": {
+        "compatible_with": [
+            "@platforms//os:macos",
+            "@platforms//cpu:x86_64",
+        ],
+    },
+    "win32": {
+        "compatible_with": [
+            "@platforms//os:windows",
+            "@platforms//cpu:x86_32",
+        ],
+    },
+    "win64": {
+        "compatible_with": [
+            "@platforms//os:windows",
+            "@platforms//cpu:x86_64",
+        ],
+    },
+}
+
+prebuilt_toolchains_repo = repository_rule(
+    doc = """\
+    Creates a single repository with toolchain definitions for all known platforms
+    that can be registered or selected.
+    """,
+    attrs = {
+        "user_repository_name": attr.string(
+            doc = """What the user chose for the base name.
+            Needed since bzlmod apparent name has extra tilde segments.
+            """,
+            mandatory = True,
+        ),
+    },
+    implementation = lambda rctx: rctx.file("BUILD.bazel", """\
+# Generated by @protobuf//bazel/toolchains:prebuilt_toolchains.bzl
+#
+# These can be registered in the workspace file or passed to --extra_toolchains flag.
+# By default all these toolchains are registered by the protoc module extension
+# so users don't normally need to interact with these targets.
+
+""" + "\n".join(["""\
+toolchain(
+    name = "{platform}_toolchain",
+    exec_compatible_with = {compatible_with},
+    # Bazel does not follow this attribute during analysis, so the referenced repo
+    # will only be fetched if this toolchain is selected.
+    toolchain = "@{user_repository_name}.{platform}//:prebuilt_protoc_toolchain",
+    toolchain_type = "@com_google_protobuf//bazel/private:proto_toolchain_type",
+)
+""".format(
+            platform = platform.replace("-", "_"),
+            user_repository_name = rctx.attr.user_repository_name,
+            compatible_with = meta["compatible_with"],
+        )
+    for platform, meta in PROTOC_PLATFORMS.items()]
+)),
+)

examples/.gitignore

@@ -1,2 +1,2 @@
 # Ignore the bazel symlinks
-/bazel-*
+bazel-*

examples/example_without_cc_toolchain/.bazelrc

@@ -0,0 +1,5 @@
+# Simulate a non-functional CC toolchain
+common --per_file_copt=external/.*protobuf.*@--THIS_CC_TOOLCHAIN_IS_BROKEN
+common --host_per_file_copt=external/.*protobuf.*@--THIS_CC_TOOLCHAIN_IS_BROKEN
+# But, users should be able to use pre-built protoc toolchains instead.
+common --incompatible_enable_proto_toolchain_resolution

examples/example_without_cc_toolchain/BUILD.bazel

@@ -0,0 +1,6 @@
+load("@protobuf//bazel:proto_library.bzl", "proto_library")
+
+proto_library(
+    name = "empty_proto",
+    srcs = ["empty.proto"],
+)

examples/example_without_cc_toolchain/MODULE.bazel

@@ -0,0 +1,22 @@
+"""Bazel module dependencies"""
+
+module(
+    name = "com_google_protobuf-example-without-cc-toolchain",
+    version = "0.0.0",
+    compatibility_level = 1,
+)
+
+bazel_dep(name = "protobuf")
+local_path_override(
+    module_name = "protobuf",
+    path = "../..",
+)
+
+protoc = use_extension("@protobuf//bazel:extensions.bzl", "protoc")
+protoc.prebuilt_toolchain(
+    name = "prebuilt_protoc",
+    # FIXME: users shouldn't be able to vary the version here from the protobuf module
+    version = "v33.0",
+)
+use_repo(protoc, "prebuilt_protoc", "prebuilt_protoc.osx_aarch_64")
+register_toolchains("@prebuilt_protoc//:all")

examples/example_without_cc_toolchain/README.md

@@ -0,0 +1,2 @@
+This example demonstrates what happens when a Bazel user doesn't have a proper CC toolchain installed.
+This case commonly happens in projects with no C++ code, so they don't have a hermetic method of building C++ code.

examples/example_without_cc_toolchain/empty.proto

@@ -0,0 +1,5 @@
+edition = "2023";
+
+package examples.without.cc.toolchain;
+
+message EmptyMessage {}