bazel: make build fully hermetic (#31006)

This PR makes our Bazel build fully hermetic, mainly by refactoring how
we built the `librocksdb-sys` crate. Previously we had to rely on some
hacks because `librocksdb-sys` uses bindgen which was tricky to get
working.

Specifically the changes this PR makes are:
* upgrades to [`clang
18.1.8-5`](https://github.com/MaterializeInc/toolchains/releases/tag/clang-18.1.8-5)
* upgrades to newer version of `toolchains_llvm` that includes an
[upstream
fix](bazel-contrib/toolchains_llvm#438)
* adds `rust_bindgen` toolchains with new `libclang` package from our
toolchains repo
* refactors the rocksdb build to be hermetic

### Motivation

Improves the quality of our build, allows us to move to a "minimum" CI
builder image for faster build times

### Tips for reviewer

<!--
Leave some tips for your reviewer, like:

    * The diff is much smaller if viewed with whitespace hidden.
    * [Some function/module/file] deserves extra attention.
* [Some function/module/file] is pure code movement and only needs a
skim.

Delete this section if no tips.
-->

### Checklist

- [ ] This PR has adequate test coverage / QA involvement has been duly
considered. ([trigger-ci for additional test/nightly
runs](https://trigger-ci.dev.materialize.com/))
- [ ] This PR has an associated up-to-date [design
doc](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/design/README.md),
is a design doc
([template](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/design/00000000_template.md)),
or is sufficiently small to not require a design.
  <!-- Reference the design in the description. -->
- [ ] If this PR evolves [an existing `$T ⇔ Proto$T`
mapping](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/command-and-response-binary-encoding.md)
(possibly in a backwards-incompatible way), then it is tagged with a
`T-proto` label.
- [ ] If this PR will require changes to cloud orchestration or tests,
there is a companion cloud PR to account for those changes that is
tagged with the release-blocker label
([example](https://github.com/MaterializeInc/cloud/pull/5021)).
<!-- Ask in #team-cloud on Slack if you need help preparing the cloud
PR. -->
- [ ] If this PR includes major [user-facing behavior
changes](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/guide-changes.md#what-changes-require-a-release-note),
I have pinged the relevant PM to schedule a changelog post.

Author: ParkMyCar

WORKSPACE

@@ -155,11 +155,6 @@ rules_foreign_cc_dependencies(make_version = "4.2")
 # All of the clang related tools are provided under the `@llvm_toolchain_llvm`
 # repo. To see what's available run `bazel query @llvm_toolchain_llvm//...`.
 
-# Version of the "toolchains_llvm" rule set, _not_ the version of clang/llvm.
-TOOLCHAINS_LLVM_VERSION = "1.0.0"
-
-TOOLCHAINS_LLVM_INTEGRITY = "sha256-6RxDYfmQEaVIFOGvvlxDbg0ymHEUajzVjCOitK+1Bzc="
-
 # System roots that we use, this is where clang will search for things like libc.
 
 _SYSROOT_DARWIN_BUILD_FILE = """
@@ -222,15 +217,21 @@ http_archive(
 LLVM_VERSION = "18.1.8"
 
 # We have a few variants of our clang toolchain, either improving how it's built or adding new tools.
-LLVM_VERSION_SUFFIX = "4"
+LLVM_VERSION_SUFFIX = "5"
+
+# Version of the "toolchains_llvm" rule set, _not_ the version of clang/llvm.
+#
+# We depend on a commit that includes <https://github.com/bazel-contrib/toolchains_llvm/pull/438>.
+TOOLCHAINS_LLVM_VERSION = "9f0a7cb0f752ffd430a5c80d749a2e84cb348876"
+
+TOOLCHAINS_LLVM_INTEGRITY = "sha256-9SY8+RwP3KPfaLtjQGzJmknOcxEpTkmu/h1ntaljYdw="
 
 maybe(
     http_archive,
     name = "toolchains_llvm",
-    canonical_id = "{0}".format(TOOLCHAINS_LLVM_VERSION),
     integrity = TOOLCHAINS_LLVM_INTEGRITY,
     strip_prefix = "toolchains_llvm-{0}".format(TOOLCHAINS_LLVM_VERSION),
-    url = "https://github.com/bazel-contrib/toolchains_llvm/releases/download/{0}/toolchains_llvm-{0}.tar.gz".format(TOOLCHAINS_LLVM_VERSION),
+    url = "https://github.com/bazel-contrib/toolchains_llvm/archive/{0}.tar.gz".format(TOOLCHAINS_LLVM_VERSION),
 )
 
 load("@toolchains_llvm//toolchain:deps.bzl", "bazel_toolchain_dependencies")
@@ -243,10 +244,10 @@ llvm_toolchain(
     name = "llvm_toolchain",
     llvm_version = LLVM_VERSION,
     sha256 = {
-        "darwin-aarch64": "41d8dea52d18c4e8b90c4fcd31965f9f297df9f40a38a33d60748dbe7f8330b8",
-        "darwin-x86_64": "291b8dd844aa896b98393c5d3beaee57f294768039eacdf9ef5e96ed9d3f62d7",
-        "linux-aarch64": "fe8f9e283ab43e963daf9ffb18742e134ad239b56078d61ef9a289ff642784ed",
-        "linux-x86_64": "8b725ec14e48bc1cb3698309506e29cd94ff3b823976ebb306e9c3ef84480c16",
+        "darwin-aarch64": "d28437d58615ccae434c289bcf5e0d10f031c732d7b60a5090541dbbbd381a01",
+        "darwin-x86_64": "f0c103bc2a19ffee7b55df2798188f085756e5668adc7c18d05c7f6705871828",
+        "linux-aarch64": "e9695cb9a8d068236b1c902af877be396eb0110d87525db49c68911b571f6976",
+        "linux-x86_64": "4d0bbec4acd23006d1353d0cd255144df8c910981f5474f3030bfde3dc75ccfa",
     },
     sysroot = {
         "darwin-aarch64": "@sysroot_darwin_universal//:sysroot",
@@ -338,7 +339,7 @@ RUST_VERSION = "1.83.0"
 
 RUST_NIGHTLY_VERSION = "nightly/2024-12-02"
 
-load("//misc/bazel/toolchains:rust.bzl", "rust_toolchains")
+load("//misc/bazel/toolchains:rust.bzl", "bindgen_toolchains", "rust_toolchains")
 
 rust_toolchains(
     [
@@ -413,6 +414,25 @@ rust_toolchains(
     },
 )
 
+# Rust `bindgen`
+#
+# Rules and Toolchains for running [`bindgen`](https://github.com/rust-lang/rust-bindgen)
+# a tool for generating Rust FFI bindings to C.
+
+load("@rules_rust//bindgen:repositories.bzl", "rust_bindgen_dependencies")
+
+rust_bindgen_dependencies()
+
+bindgen_toolchains(
+    "{0}-{1}".format(LLVM_VERSION, LLVM_VERSION_SUFFIX),
+    {
+        "darwin_aarch64": "sha256-YHP+DGnW3QmbKi93KYVOEq7oym0cbJga0MOgarWksds=",
+        "darwin_x86_64": "sha256-fA4eCuliHPYf3DBqsSVIQ53TyCfIUJJ3LaM5U3ZxxRc=",
+        "linux_aarch64": "sha256-GIB3n6vX/jpyFn2e7hI/pRzlP0fPeVhkVu2YDQUDS3A=",
+        "linux_x86_64": "sha256-a8BzSw50LoMYtEZretLgqizUFcpGpBGYa18rT9EnFgI=",
+    },
+)
+
 # Load all dependencies for crate_universe.
 load("@rules_rust//crate_universe:repositories.bzl", "crate_universe_dependencies")
 
@@ -438,33 +458,16 @@ crates_repository(
             deps = [":decnumber"],
         )],
         "librocksdb-sys": [crate.annotation(
-            additive_build_file = "@//misc/bazel/c_deps:rust-sys/BUILD.rocksdb.bazel",
             # Note: The below targets are from the additive build file.
-            #
-            # HACK(parkmycar): The `librocksdb-sys` build script runs bindgen for us, and to
-            # support cross compiling we need to provide the sysroot to the build script so
-            # bindgen can find it. Providing the sysroot and relying on the raw paths is quite
-            # fragile, the fix is to use `@rules_rust//bindgen/...` rules with our Clang toolchain.
-            build_script_data = [
-                ":rocksdb_lib",
-                ":rocksdb_include",
-                ":snappy_lib",
-                "@linux_sysroot-aarch64//:sysroot",
-                "@linux_sysroot-x86_64//:sysroot",
-            ],
-            build_script_env = {
-                "ROCKSDB_STATIC": "true",
-                "ROCKSDB_LIB_DIR": "$(execpath :rocksdb_lib)",
-                "ROCKSDB_INCLUDE_DIR": "$(execpath :rocksdb_include)",
-                "SNAPPY_STATIC": "true",
-                "SNAPPY_LIB_DIR": "$(execpath :snappy_lib)",
-                "BINDGEN_EXTRA_CLANG_ARGS_aarch64-unknown-linux-gnu": "--sysroot=external/linux_sysroot-aarch64",
-                "BINDGEN_EXTRA_CLANG_ARGS_x86_64-unknown-linux-gnu": "--sysroot=external/linux_sysroot-x86_64",
+            additive_build_file = "@//misc/bazel/c_deps:rust-sys/BUILD.rocksdb.bazel",
+            compile_data = [":out_dir"],
+            gen_build_script = False,
+            rustc_env = {
+                "OUT_DIR": "$(execpath :out_dir)",
             },
-            compile_data = [
-                ":rocksdb_lib",
-                ":rocksdb_include",
-                ":snappy_lib",
+            deps = [
+                ":bindings",
+                ":rocksdb",
             ],
         )],
         "tikv-jemalloc-sys": [crate.annotation(

misc/bazel/c_deps/rust-sys/BUILD.rocksdb.bazel

@@ -13,105 +13,36 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("@aspect_bazel_lib//lib:copy_file.bzl", "copy_file")
+"""Additive BUILD file for the librocksdb-sys Rust crate."""
+
 load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory")
 load("@bazel_skylib//rules:select_file.bzl", "select_file")
 load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake")
+load("@rules_rust//bindgen:defs.bzl", "rust_bindgen")
 
-"""Additive BUILD file for the librocksdb-sys Rust crate."""
-
-# Copied from https://github.com/tensorflow/tensorflow/blob/bdd8bf316e4ab7d699127d192d30eb614a158462/third_party/snappy.BUILD
+# Derived from <https://github.com/rust-rocksdb/rust-rocksdb/blob/7f9cba4a819e76d8022733b4c82509aec6056938/librocksdb-sys/build.rs>
 
 cc_library(
     name = "snappy",
     srcs = [
         "snappy/snappy.cc",
-        "snappy/snappy.h",
-        "snappy/snappy-internal.h",
+        "snappy/snappy-c.cc",
         "snappy/snappy-sinksource.cc",
-        "snappy/snappy-sinksource.h",
-        "snappy/snappy-stubs-internal.cc",
-        "snappy/snappy-stubs-internal.h",
-        ":config_h",
-        ":snappy_stubs_public_h",
     ],
     hdrs = [
+        "snappy-stubs-public.h",
         "snappy/snappy.h",
+        "snappy/snappy-c.h",
+        "snappy/snappy-internal.h",
+        "snappy/snappy-sinksource.h",
+        "snappy/snappy-stubs-internal.h",
     ],
-    copts = [
-        "-DHAVE_CONFIG_H",
-        "-fno-exceptions",
-        "-Wno-sign-compare",
-        "-Wno-shift-negative-value",
-        "-Wno-implicit-function-declaration",
-    ],
-    defines = ["HAVE_SYS_UIO_H"],
-    includes = ["snappy/."],
-)
-
-genrule(
-    name = "config_h",
-    outs = ["snappy/config.h"],
-    cmd = "\n".join([
-        "cat <<'EOF' >$@",
-        "#define HAVE_STDDEF_H 1",
-        "#define HAVE_STDINT_H 1",
-        "",
-        "#ifdef __has_builtin",
-        "#  if !defined(HAVE_BUILTIN_EXPECT) && __has_builtin(__builtin_expect)",
-        "#    define HAVE_BUILTIN_EXPECT 1",
-        "#  endif",
-        "#  if !defined(HAVE_BUILTIN_CTZ) && __has_builtin(__builtin_ctzll)",
-        "#    define HAVE_BUILTIN_CTZ 1",
-        "#  endif",
-        "#elif defined(__GNUC__) && (__GNUC__ > 3 || __GNUC__ == 3 && __GNUC_MINOR__ >= 4)",
-        "#  ifndef HAVE_BUILTIN_EXPECT",
-        "#    define HAVE_BUILTIN_EXPECT 1",
-        "#  endif",
-        "#  ifndef HAVE_BUILTIN_CTZ",
-        "#    define HAVE_BUILTIN_CTZ 1",
-        "#  endif",
-        "#endif",
-        "",
-        "#ifdef __has_include",
-        "#  if !defined(HAVE_BYTESWAP_H) && __has_include(<byteswap.h>)",
-        "#    define HAVE_BYTESWAP_H 1",
-        "#  endif",
-        "#  if !defined(HAVE_UNISTD_H) && __has_include(<unistd.h>)",
-        "#    define HAVE_UNISTD_H 1",
-        "#  endif",
-        "#  if !defined(HAVE_SYS_ENDIAN_H) && __has_include(<sys/endian.h>)",
-        "#    define HAVE_SYS_ENDIAN_H 1",
-        "#  endif",
-        "#  if !defined(HAVE_SYS_MMAN_H) && __has_include(<sys/mman.h>)",
-        "#    define HAVE_SYS_MMAN_H 1",
-        "#  endif",
-        "#  if !defined(HAVE_SYS_UIO_H) && __has_include(<sys/uio.h>)",
-        "#    define HAVE_SYS_UIO_H 1",
-        "#  endif",
-        "#endif",
+    copts = ["-std=c++11"],
+    includes = [
         "",
-        "#ifndef SNAPPY_IS_BIG_ENDIAN",
-        "#  ifdef __s390x__",
-        "#    define SNAPPY_IS_BIG_ENDIAN 1",
-        "#  elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__",
-        "#    define SNAPPY_IS_BIG_ENDIAN 1",
-        "#  endif",
-        "#endif",
-        "EOF",
-    ]),
-)
-
-genrule(
-    name = "snappy_stubs_public_h",
-    srcs = ["snappy/snappy-stubs-public.h.in"],
-    outs = ["snappy/snappy-stubs-public.h"],
-    cmd = ("sed " +
-           "-e 's/$${\\(.*\\)_01}/\\1/g' " +
-           "-e 's/$${SNAPPY_MAJOR}/1/g' " +
-           "-e 's/$${SNAPPY_MINOR}/1/g' " +
-           "-e 's/$${SNAPPY_PATCHLEVEL}/4/g' " +
-           "$< >$@"),
+        "snappy",
+    ],
+    local_defines = ["NDEBUG=1"],
 )
 
 filegroup(
@@ -174,6 +105,7 @@ cmake(
         },
         no_match_error = "Building rocksdb for the specified CPU is not supported.",
     ),
+    includes = ["include/rocksdb/c.h"],
     lib_source = ":rocksdb_srcs",
     out_static_libs = ["librocksdb.a"],
     targets = ["rocksdb"],
@@ -186,52 +118,44 @@ cmake(
     ],
 )
 
-filegroup(
-    name = "out_dir",
-    srcs = [":rocksdb"],
-    visibility = ["//visibility:public"],
-)
-
 select_file(
-    name = "librocksdb",
-    srcs = ":out_dir",
+    name = "librocksdb_a",
+    srcs = ":rocksdb",
     subpath = "librocksdb.a",
 )
 
-copy_file(
-    name = "librocksdb_copy",
-    src = ":librocksdb",
-    out = "librocksdb.a",
-    allow_symlink = False,
-)
-
-copy_to_directory(
-    name = "rocksdb_lib",
-    srcs = [":librocksdb_copy"],
-    visibility = ["//visibility:public"],
+filegroup(
+    name = "rocksdb_include",
+    srcs = glob(
+        include = ["rocksdb/include/rocksdb/**/*.h"],
+    ),
 )
 
-# Copy the include folder so we can specify `ROCKSDB_INCLUDE_DIR`
-#
-# Note: We used to use `select_file` here but it generated symlinks that
-# overlapped with others and spammed the logs with WARNINGs.
-copy_to_directory(
-    name = "rocksdb_include",
-    srcs = [":out_dir"],
-    out = "include",
-    root_paths = ["rocksdb/include"],
-    visibility = ["//visibility:public"],
+# We need to expose the header files with the rocksdb static lib. The `cmake`
+# rules doesn't give us a way to do that, so we manually piece it together.
+cc_import(
+    name = "librocksdb",
+    hdrs = [":rocksdb_include"],
+    static_library = ":librocksdb_a",
 )
 
-select_file(
-    name = "libsnappy",
-    srcs = ":snappy",
-    subpath = "libsnappy.a",
+rust_bindgen(
+    name = "bindings",
+    bindgen_flags = [
+        "--no-derive-debug",
+        "--blocklist-type=max_align_t",
+        "--ctypes-prefix=libc",
+    ],
+    cc_lib = ":librocksdb",
+    header = "rocksdb/include/rocksdb/c.h",
 )
 
+# Place the generated artifacts into an OUT_DIR.
+#
+# TODO(parkmycar): <https://github.com/bazelbuild/rules_rust/issues/3184>
 copy_to_directory(
-    name = "snappy_lib",
-    srcs = [":libsnappy"],
+    name = "out_dir",
+    srcs = [":bindings"],
     visibility = ["//visibility:public"],
 )