Skip to content

bazel: make build fully hermetic #31006

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 13, 2025
Merged

bazel: make build fully hermetic #31006

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 41 additions & 38 deletions WORKSPACE
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,11 +155,6 @@ rules_foreign_cc_dependencies(make_version = "4.2")
# All of the clang related tools are provided under the `@llvm_toolchain_llvm`
# repo. To see what's available run `bazel query @llvm_toolchain_llvm//...`.

# Version of the "toolchains_llvm" rule set, _not_ the version of clang/llvm.
TOOLCHAINS_LLVM_VERSION = "1.0.0"

TOOLCHAINS_LLVM_INTEGRITY = "sha256-6RxDYfmQEaVIFOGvvlxDbg0ymHEUajzVjCOitK+1Bzc="

# System roots that we use, this is where clang will search for things like libc.

_SYSROOT_DARWIN_BUILD_FILE = """
Expand Down Expand Up @@ -222,15 +217,21 @@ http_archive(
LLVM_VERSION = "18.1.8"

# We have a few variants of our clang toolchain, either improving how it's built or adding new tools.
LLVM_VERSION_SUFFIX = "4"
LLVM_VERSION_SUFFIX = "5"

# Version of the "toolchains_llvm" rule set, _not_ the version of clang/llvm.
#
# We depend on a commit that includes <https://github.com/bazel-contrib/toolchains_llvm/pull/438>.
TOOLCHAINS_LLVM_VERSION = "9f0a7cb0f752ffd430a5c80d749a2e84cb348876"

TOOLCHAINS_LLVM_INTEGRITY = "sha256-9SY8+RwP3KPfaLtjQGzJmknOcxEpTkmu/h1ntaljYdw="

maybe(
http_archive,
name = "toolchains_llvm",
canonical_id = "{0}".format(TOOLCHAINS_LLVM_VERSION),
integrity = TOOLCHAINS_LLVM_INTEGRITY,
strip_prefix = "toolchains_llvm-{0}".format(TOOLCHAINS_LLVM_VERSION),
url = "https://github.com/bazel-contrib/toolchains_llvm/releases/download/{0}/toolchains_llvm-{0}.tar.gz".format(TOOLCHAINS_LLVM_VERSION),
url = "https://github.com/bazel-contrib/toolchains_llvm/archive/{0}.tar.gz".format(TOOLCHAINS_LLVM_VERSION),
)

load("@toolchains_llvm//toolchain:deps.bzl", "bazel_toolchain_dependencies")
Expand All @@ -243,10 +244,10 @@ llvm_toolchain(
name = "llvm_toolchain",
llvm_version = LLVM_VERSION,
sha256 = {
"darwin-aarch64": "41d8dea52d18c4e8b90c4fcd31965f9f297df9f40a38a33d60748dbe7f8330b8",
"darwin-x86_64": "291b8dd844aa896b98393c5d3beaee57f294768039eacdf9ef5e96ed9d3f62d7",
"linux-aarch64": "fe8f9e283ab43e963daf9ffb18742e134ad239b56078d61ef9a289ff642784ed",
"linux-x86_64": "8b725ec14e48bc1cb3698309506e29cd94ff3b823976ebb306e9c3ef84480c16",
"darwin-aarch64": "d28437d58615ccae434c289bcf5e0d10f031c732d7b60a5090541dbbbd381a01",
"darwin-x86_64": "f0c103bc2a19ffee7b55df2798188f085756e5668adc7c18d05c7f6705871828",
"linux-aarch64": "e9695cb9a8d068236b1c902af877be396eb0110d87525db49c68911b571f6976",
"linux-x86_64": "4d0bbec4acd23006d1353d0cd255144df8c910981f5474f3030bfde3dc75ccfa",
},
sysroot = {
"darwin-aarch64": "@sysroot_darwin_universal//:sysroot",
Expand Down Expand Up @@ -338,7 +339,7 @@ RUST_VERSION = "1.83.0"

RUST_NIGHTLY_VERSION = "nightly/2024-12-02"

load("//misc/bazel/toolchains:rust.bzl", "rust_toolchains")
load("//misc/bazel/toolchains:rust.bzl", "bindgen_toolchains", "rust_toolchains")

rust_toolchains(
[
Expand Down Expand Up @@ -413,6 +414,25 @@ rust_toolchains(
},
)

# Rust `bindgen`
#
# Rules and Toolchains for running [`bindgen`](https://github.com/rust-lang/rust-bindgen)
# a tool for generating Rust FFI bindings to C.

load("@rules_rust//bindgen:repositories.bzl", "rust_bindgen_dependencies")

rust_bindgen_dependencies()

bindgen_toolchains(
"{0}-{1}".format(LLVM_VERSION, LLVM_VERSION_SUFFIX),
{
"darwin_aarch64": "sha256-YHP+DGnW3QmbKi93KYVOEq7oym0cbJga0MOgarWksds=",
"darwin_x86_64": "sha256-fA4eCuliHPYf3DBqsSVIQ53TyCfIUJJ3LaM5U3ZxxRc=",
"linux_aarch64": "sha256-GIB3n6vX/jpyFn2e7hI/pRzlP0fPeVhkVu2YDQUDS3A=",
"linux_x86_64": "sha256-a8BzSw50LoMYtEZretLgqizUFcpGpBGYa18rT9EnFgI=",
},
)

# Load all dependencies for crate_universe.
load("@rules_rust//crate_universe:repositories.bzl", "crate_universe_dependencies")

Expand All @@ -438,33 +458,16 @@ crates_repository(
deps = [":decnumber"],
)],
"librocksdb-sys": [crate.annotation(
additive_build_file = "@//misc/bazel/c_deps:rust-sys/BUILD.rocksdb.bazel",
# Note: The below targets are from the additive build file.
#
# HACK(parkmycar): The `librocksdb-sys` build script runs bindgen for us, and to
# support cross compiling we need to provide the sysroot to the build script so
# bindgen can find it. Providing the sysroot and relying on the raw paths is quite
# fragile, the fix is to use `@rules_rust//bindgen/...` rules with our Clang toolchain.
build_script_data = [
":rocksdb_lib",
":rocksdb_include",
":snappy_lib",
"@linux_sysroot-aarch64//:sysroot",
"@linux_sysroot-x86_64//:sysroot",
],
build_script_env = {
"ROCKSDB_STATIC": "true",
"ROCKSDB_LIB_DIR": "$(execpath :rocksdb_lib)",
"ROCKSDB_INCLUDE_DIR": "$(execpath :rocksdb_include)",
"SNAPPY_STATIC": "true",
"SNAPPY_LIB_DIR": "$(execpath :snappy_lib)",
"BINDGEN_EXTRA_CLANG_ARGS_aarch64-unknown-linux-gnu": "--sysroot=external/linux_sysroot-aarch64",
"BINDGEN_EXTRA_CLANG_ARGS_x86_64-unknown-linux-gnu": "--sysroot=external/linux_sysroot-x86_64",
additive_build_file = "@//misc/bazel/c_deps:rust-sys/BUILD.rocksdb.bazel",
compile_data = [":out_dir"],
gen_build_script = False,
rustc_env = {
"OUT_DIR": "$(execpath :out_dir)",
},
compile_data = [
":rocksdb_lib",
":rocksdb_include",
":snappy_lib",
deps = [
":bindings",
":rocksdb",
],
)],
"tikv-jemalloc-sys": [crate.annotation(
Expand Down
162 changes: 43 additions & 119 deletions misc/bazel/c_deps/rust-sys/BUILD.rocksdb.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,105 +13,36 @@
# See the License for the specific language governing permissions and
# limitations under the License.

load("@aspect_bazel_lib//lib:copy_file.bzl", "copy_file")
"""Additive BUILD file for the librocksdb-sys Rust crate."""

load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory")
load("@bazel_skylib//rules:select_file.bzl", "select_file")
load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake")
load("@rules_rust//bindgen:defs.bzl", "rust_bindgen")

"""Additive BUILD file for the librocksdb-sys Rust crate."""

# Copied from https://github.com/tensorflow/tensorflow/blob/bdd8bf316e4ab7d699127d192d30eb614a158462/third_party/snappy.BUILD
# Derived from <https://github.com/rust-rocksdb/rust-rocksdb/blob/7f9cba4a819e76d8022733b4c82509aec6056938/librocksdb-sys/build.rs>

cc_library(
name = "snappy",
srcs = [
"snappy/snappy.cc",
"snappy/snappy.h",
"snappy/snappy-internal.h",
"snappy/snappy-c.cc",
"snappy/snappy-sinksource.cc",
"snappy/snappy-sinksource.h",
"snappy/snappy-stubs-internal.cc",
"snappy/snappy-stubs-internal.h",
":config_h",
":snappy_stubs_public_h",
],
hdrs = [
"snappy-stubs-public.h",
"snappy/snappy.h",
"snappy/snappy-c.h",
"snappy/snappy-internal.h",
"snappy/snappy-sinksource.h",
"snappy/snappy-stubs-internal.h",
],
copts = [
"-DHAVE_CONFIG_H",
"-fno-exceptions",
"-Wno-sign-compare",
"-Wno-shift-negative-value",
"-Wno-implicit-function-declaration",
],
defines = ["HAVE_SYS_UIO_H"],
includes = ["snappy/."],
)

genrule(
name = "config_h",
outs = ["snappy/config.h"],
cmd = "\n".join([
"cat <<'EOF' >$@",
"#define HAVE_STDDEF_H 1",
"#define HAVE_STDINT_H 1",
"",
"#ifdef __has_builtin",
"# if !defined(HAVE_BUILTIN_EXPECT) && __has_builtin(__builtin_expect)",
"# define HAVE_BUILTIN_EXPECT 1",
"# endif",
"# if !defined(HAVE_BUILTIN_CTZ) && __has_builtin(__builtin_ctzll)",
"# define HAVE_BUILTIN_CTZ 1",
"# endif",
"#elif defined(__GNUC__) && (__GNUC__ > 3 || __GNUC__ == 3 && __GNUC_MINOR__ >= 4)",
"# ifndef HAVE_BUILTIN_EXPECT",
"# define HAVE_BUILTIN_EXPECT 1",
"# endif",
"# ifndef HAVE_BUILTIN_CTZ",
"# define HAVE_BUILTIN_CTZ 1",
"# endif",
"#endif",
"",
"#ifdef __has_include",
"# if !defined(HAVE_BYTESWAP_H) && __has_include(<byteswap.h>)",
"# define HAVE_BYTESWAP_H 1",
"# endif",
"# if !defined(HAVE_UNISTD_H) && __has_include(<unistd.h>)",
"# define HAVE_UNISTD_H 1",
"# endif",
"# if !defined(HAVE_SYS_ENDIAN_H) && __has_include(<sys/endian.h>)",
"# define HAVE_SYS_ENDIAN_H 1",
"# endif",
"# if !defined(HAVE_SYS_MMAN_H) && __has_include(<sys/mman.h>)",
"# define HAVE_SYS_MMAN_H 1",
"# endif",
"# if !defined(HAVE_SYS_UIO_H) && __has_include(<sys/uio.h>)",
"# define HAVE_SYS_UIO_H 1",
"# endif",
"#endif",
copts = ["-std=c++11"],
includes = [
"",
"#ifndef SNAPPY_IS_BIG_ENDIAN",
"# ifdef __s390x__",
"# define SNAPPY_IS_BIG_ENDIAN 1",
"# elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__",
"# define SNAPPY_IS_BIG_ENDIAN 1",
"# endif",
"#endif",
"EOF",
]),
)

genrule(
name = "snappy_stubs_public_h",
srcs = ["snappy/snappy-stubs-public.h.in"],
outs = ["snappy/snappy-stubs-public.h"],
cmd = ("sed " +
"-e 's/$${\\(.*\\)_01}/\\1/g' " +
"-e 's/$${SNAPPY_MAJOR}/1/g' " +
"-e 's/$${SNAPPY_MINOR}/1/g' " +
"-e 's/$${SNAPPY_PATCHLEVEL}/4/g' " +
"$< >$@"),
"snappy",
],
local_defines = ["NDEBUG=1"],
)

filegroup(
Expand Down Expand Up @@ -174,6 +105,7 @@ cmake(
},
no_match_error = "Building rocksdb for the specified CPU is not supported.",
),
includes = ["include/rocksdb/c.h"],
lib_source = ":rocksdb_srcs",
out_static_libs = ["librocksdb.a"],
targets = ["rocksdb"],
Expand All @@ -186,52 +118,44 @@ cmake(
],
)

filegroup(
name = "out_dir",
srcs = [":rocksdb"],
visibility = ["//visibility:public"],
)

select_file(
name = "librocksdb",
srcs = ":out_dir",
name = "librocksdb_a",
srcs = ":rocksdb",
subpath = "librocksdb.a",
)

copy_file(
name = "librocksdb_copy",
src = ":librocksdb",
out = "librocksdb.a",
allow_symlink = False,
)

copy_to_directory(
name = "rocksdb_lib",
srcs = [":librocksdb_copy"],
visibility = ["//visibility:public"],
filegroup(
name = "rocksdb_include",
srcs = glob(
include = ["rocksdb/include/rocksdb/**/*.h"],
),
)

# Copy the include folder so we can specify `ROCKSDB_INCLUDE_DIR`
#
# Note: We used to use `select_file` here but it generated symlinks that
# overlapped with others and spammed the logs with WARNINGs.
copy_to_directory(
name = "rocksdb_include",
srcs = [":out_dir"],
out = "include",
root_paths = ["rocksdb/include"],
visibility = ["//visibility:public"],
# We need to expose the header files with the rocksdb static lib. The `cmake`
# rules doesn't give us a way to do that, so we manually piece it together.
cc_import(
name = "librocksdb",
hdrs = [":rocksdb_include"],
static_library = ":librocksdb_a",
)

select_file(
name = "libsnappy",
srcs = ":snappy",
subpath = "libsnappy.a",
rust_bindgen(
name = "bindings",
bindgen_flags = [
"--no-derive-debug",
"--blocklist-type=max_align_t",
"--ctypes-prefix=libc",
],
cc_lib = ":librocksdb",
header = "rocksdb/include/rocksdb/c.h",
)

# Place the generated artifacts into an OUT_DIR.
#
# TODO(parkmycar): <https://github.com/bazelbuild/rules_rust/issues/3184>
copy_to_directory(
name = "snappy_lib",
srcs = [":libsnappy"],
name = "out_dir",
srcs = [":bindings"],
visibility = ["//visibility:public"],
)

Expand Down
Loading
Loading