Skip to content

Commit 8700093

Browse files
committed
start, reword rocksdb build, use rust_bindgen rule
* upgrades to clang 18.1.8-5 * upgrades to newer version of toolchains_llvm * adds rust_bindgen toolchains with new libclang package from our toolchains repo * refactors the rocksdb build to be hermetic
1 parent b1db073 commit 8700093

File tree

5 files changed

+260
-165
lines changed

5 files changed

+260
-165
lines changed

WORKSPACE

Lines changed: 41 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -155,11 +155,6 @@ rules_foreign_cc_dependencies(make_version = "4.2")
155155
# All of the clang related tools are provided under the `@llvm_toolchain_llvm`
156156
# repo. To see what's available run `bazel query @llvm_toolchain_llvm//...`.
157157

158-
# Version of the "toolchains_llvm" rule set, _not_ the version of clang/llvm.
159-
TOOLCHAINS_LLVM_VERSION = "1.0.0"
160-
161-
TOOLCHAINS_LLVM_INTEGRITY = "sha256-6RxDYfmQEaVIFOGvvlxDbg0ymHEUajzVjCOitK+1Bzc="
162-
163158
# System roots that we use, this is where clang will search for things like libc.
164159

165160
_SYSROOT_DARWIN_BUILD_FILE = """
@@ -222,15 +217,21 @@ http_archive(
222217
LLVM_VERSION = "18.1.8"
223218

224219
# We have a few variants of our clang toolchain, either improving how it's built or adding new tools.
225-
LLVM_VERSION_SUFFIX = "4"
220+
LLVM_VERSION_SUFFIX = "5"
221+
222+
# Version of the "toolchains_llvm" rule set, _not_ the version of clang/llvm.
223+
#
224+
# We depend on a commit that includes <https://github.com/bazel-contrib/toolchains_llvm/pull/438>.
225+
TOOLCHAINS_LLVM_VERSION = "9f0a7cb0f752ffd430a5c80d749a2e84cb348876"
226+
227+
TOOLCHAINS_LLVM_INTEGRITY = "sha256-9SY8+RwP3KPfaLtjQGzJmknOcxEpTkmu/h1ntaljYdw="
226228

227229
maybe(
228230
http_archive,
229231
name = "toolchains_llvm",
230-
canonical_id = "{0}".format(TOOLCHAINS_LLVM_VERSION),
231232
integrity = TOOLCHAINS_LLVM_INTEGRITY,
232233
strip_prefix = "toolchains_llvm-{0}".format(TOOLCHAINS_LLVM_VERSION),
233-
url = "https://github.com/bazel-contrib/toolchains_llvm/releases/download/{0}/toolchains_llvm-{0}.tar.gz".format(TOOLCHAINS_LLVM_VERSION),
234+
url = "https://github.com/bazel-contrib/toolchains_llvm/archive/{0}.tar.gz".format(TOOLCHAINS_LLVM_VERSION),
234235
)
235236

236237
load("@toolchains_llvm//toolchain:deps.bzl", "bazel_toolchain_dependencies")
@@ -243,10 +244,10 @@ llvm_toolchain(
243244
name = "llvm_toolchain",
244245
llvm_version = LLVM_VERSION,
245246
sha256 = {
246-
"darwin-aarch64": "41d8dea52d18c4e8b90c4fcd31965f9f297df9f40a38a33d60748dbe7f8330b8",
247-
"darwin-x86_64": "291b8dd844aa896b98393c5d3beaee57f294768039eacdf9ef5e96ed9d3f62d7",
248-
"linux-aarch64": "fe8f9e283ab43e963daf9ffb18742e134ad239b56078d61ef9a289ff642784ed",
249-
"linux-x86_64": "8b725ec14e48bc1cb3698309506e29cd94ff3b823976ebb306e9c3ef84480c16",
247+
"darwin-aarch64": "d28437d58615ccae434c289bcf5e0d10f031c732d7b60a5090541dbbbd381a01",
248+
"darwin-x86_64": "f0c103bc2a19ffee7b55df2798188f085756e5668adc7c18d05c7f6705871828",
249+
"linux-aarch64": "e9695cb9a8d068236b1c902af877be396eb0110d87525db49c68911b571f6976",
250+
"linux-x86_64": "4d0bbec4acd23006d1353d0cd255144df8c910981f5474f3030bfde3dc75ccfa",
250251
},
251252
sysroot = {
252253
"darwin-aarch64": "@sysroot_darwin_universal//:sysroot",
@@ -338,7 +339,7 @@ RUST_VERSION = "1.83.0"
338339

339340
RUST_NIGHTLY_VERSION = "nightly/2024-12-02"
340341

341-
load("//misc/bazel/toolchains:rust.bzl", "rust_toolchains")
342+
load("//misc/bazel/toolchains:rust.bzl", "bindgen_toolchains", "rust_toolchains")
342343

343344
rust_toolchains(
344345
[
@@ -413,6 +414,25 @@ rust_toolchains(
413414
},
414415
)
415416

417+
# Rust `bindgen`
418+
#
419+
# Rules and Toolchains for running [`bindgen`](https://github.com/rust-lang/rust-bindgen)
420+
# a tool for generating Rust FFI bindings to C.
421+
422+
load("@rules_rust//bindgen:repositories.bzl", "rust_bindgen_dependencies")
423+
424+
rust_bindgen_dependencies()
425+
426+
bindgen_toolchains(
427+
"{0}-{1}".format(LLVM_VERSION, LLVM_VERSION_SUFFIX),
428+
{
429+
"darwin_aarch64": "sha256-YHP+DGnW3QmbKi93KYVOEq7oym0cbJga0MOgarWksds=",
430+
"darwin_x86_64": "sha256-fA4eCuliHPYf3DBqsSVIQ53TyCfIUJJ3LaM5U3ZxxRc=",
431+
"linux_aarch64": "sha256-GIB3n6vX/jpyFn2e7hI/pRzlP0fPeVhkVu2YDQUDS3A=",
432+
"linux_x86_64": "sha256-a8BzSw50LoMYtEZretLgqizUFcpGpBGYa18rT9EnFgI=",
433+
},
434+
)
435+
416436
# Load all dependencies for crate_universe.
417437
load("@rules_rust//crate_universe:repositories.bzl", "crate_universe_dependencies")
418438

@@ -438,33 +458,16 @@ crates_repository(
438458
deps = [":decnumber"],
439459
)],
440460
"librocksdb-sys": [crate.annotation(
441-
additive_build_file = "@//misc/bazel/c_deps:rust-sys/BUILD.rocksdb.bazel",
442461
# Note: The below targets are from the additive build file.
443-
#
444-
# HACK(parkmycar): The `librocksdb-sys` build script runs bindgen for us, and to
445-
# support cross compiling we need to provide the sysroot to the build script so
446-
# bindgen can find it. Providing the sysroot and relying on the raw paths is quite
447-
# fragile, the fix is to use `@rules_rust//bindgen/...` rules with our Clang toolchain.
448-
build_script_data = [
449-
":rocksdb_lib",
450-
":rocksdb_include",
451-
":snappy_lib",
452-
"@linux_sysroot-aarch64//:sysroot",
453-
"@linux_sysroot-x86_64//:sysroot",
454-
],
455-
build_script_env = {
456-
"ROCKSDB_STATIC": "true",
457-
"ROCKSDB_LIB_DIR": "$(execpath :rocksdb_lib)",
458-
"ROCKSDB_INCLUDE_DIR": "$(execpath :rocksdb_include)",
459-
"SNAPPY_STATIC": "true",
460-
"SNAPPY_LIB_DIR": "$(execpath :snappy_lib)",
461-
"BINDGEN_EXTRA_CLANG_ARGS_aarch64-unknown-linux-gnu": "--sysroot=external/linux_sysroot-aarch64",
462-
"BINDGEN_EXTRA_CLANG_ARGS_x86_64-unknown-linux-gnu": "--sysroot=external/linux_sysroot-x86_64",
462+
additive_build_file = "@//misc/bazel/c_deps:rust-sys/BUILD.rocksdb.bazel",
463+
compile_data = [":out_dir"],
464+
gen_build_script = False,
465+
rustc_env = {
466+
"OUT_DIR": "$(execpath :out_dir)",
463467
},
464-
compile_data = [
465-
":rocksdb_lib",
466-
":rocksdb_include",
467-
":snappy_lib",
468+
deps = [
469+
":bindings",
470+
":rocksdb",
468471
],
469472
)],
470473
"tikv-jemalloc-sys": [crate.annotation(

misc/bazel/c_deps/rust-sys/BUILD.rocksdb.bazel

Lines changed: 43 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -13,105 +13,36 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16-
load("@aspect_bazel_lib//lib:copy_file.bzl", "copy_file")
16+
"""Additive BUILD file for the librocksdb-sys Rust crate."""
17+
1718
load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory")
1819
load("@bazel_skylib//rules:select_file.bzl", "select_file")
1920
load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake")
21+
load("@rules_rust//bindgen:defs.bzl", "rust_bindgen")
2022

21-
"""Additive BUILD file for the librocksdb-sys Rust crate."""
22-
23-
# Copied from https://github.com/tensorflow/tensorflow/blob/bdd8bf316e4ab7d699127d192d30eb614a158462/third_party/snappy.BUILD
23+
# Derived from <https://github.com/rust-rocksdb/rust-rocksdb/blob/7f9cba4a819e76d8022733b4c82509aec6056938/librocksdb-sys/build.rs>
2424

2525
cc_library(
2626
name = "snappy",
2727
srcs = [
2828
"snappy/snappy.cc",
29-
"snappy/snappy.h",
30-
"snappy/snappy-internal.h",
29+
"snappy/snappy-c.cc",
3130
"snappy/snappy-sinksource.cc",
32-
"snappy/snappy-sinksource.h",
33-
"snappy/snappy-stubs-internal.cc",
34-
"snappy/snappy-stubs-internal.h",
35-
":config_h",
36-
":snappy_stubs_public_h",
3731
],
3832
hdrs = [
33+
"snappy-stubs-public.h",
3934
"snappy/snappy.h",
35+
"snappy/snappy-c.h",
36+
"snappy/snappy-internal.h",
37+
"snappy/snappy-sinksource.h",
38+
"snappy/snappy-stubs-internal.h",
4039
],
41-
copts = [
42-
"-DHAVE_CONFIG_H",
43-
"-fno-exceptions",
44-
"-Wno-sign-compare",
45-
"-Wno-shift-negative-value",
46-
"-Wno-implicit-function-declaration",
47-
],
48-
defines = ["HAVE_SYS_UIO_H"],
49-
includes = ["snappy/."],
50-
)
51-
52-
genrule(
53-
name = "config_h",
54-
outs = ["snappy/config.h"],
55-
cmd = "\n".join([
56-
"cat <<'EOF' >$@",
57-
"#define HAVE_STDDEF_H 1",
58-
"#define HAVE_STDINT_H 1",
59-
"",
60-
"#ifdef __has_builtin",
61-
"# if !defined(HAVE_BUILTIN_EXPECT) && __has_builtin(__builtin_expect)",
62-
"# define HAVE_BUILTIN_EXPECT 1",
63-
"# endif",
64-
"# if !defined(HAVE_BUILTIN_CTZ) && __has_builtin(__builtin_ctzll)",
65-
"# define HAVE_BUILTIN_CTZ 1",
66-
"# endif",
67-
"#elif defined(__GNUC__) && (__GNUC__ > 3 || __GNUC__ == 3 && __GNUC_MINOR__ >= 4)",
68-
"# ifndef HAVE_BUILTIN_EXPECT",
69-
"# define HAVE_BUILTIN_EXPECT 1",
70-
"# endif",
71-
"# ifndef HAVE_BUILTIN_CTZ",
72-
"# define HAVE_BUILTIN_CTZ 1",
73-
"# endif",
74-
"#endif",
75-
"",
76-
"#ifdef __has_include",
77-
"# if !defined(HAVE_BYTESWAP_H) && __has_include(<byteswap.h>)",
78-
"# define HAVE_BYTESWAP_H 1",
79-
"# endif",
80-
"# if !defined(HAVE_UNISTD_H) && __has_include(<unistd.h>)",
81-
"# define HAVE_UNISTD_H 1",
82-
"# endif",
83-
"# if !defined(HAVE_SYS_ENDIAN_H) && __has_include(<sys/endian.h>)",
84-
"# define HAVE_SYS_ENDIAN_H 1",
85-
"# endif",
86-
"# if !defined(HAVE_SYS_MMAN_H) && __has_include(<sys/mman.h>)",
87-
"# define HAVE_SYS_MMAN_H 1",
88-
"# endif",
89-
"# if !defined(HAVE_SYS_UIO_H) && __has_include(<sys/uio.h>)",
90-
"# define HAVE_SYS_UIO_H 1",
91-
"# endif",
92-
"#endif",
40+
copts = ["-std=c++11"],
41+
includes = [
9342
"",
94-
"#ifndef SNAPPY_IS_BIG_ENDIAN",
95-
"# ifdef __s390x__",
96-
"# define SNAPPY_IS_BIG_ENDIAN 1",
97-
"# elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__",
98-
"# define SNAPPY_IS_BIG_ENDIAN 1",
99-
"# endif",
100-
"#endif",
101-
"EOF",
102-
]),
103-
)
104-
105-
genrule(
106-
name = "snappy_stubs_public_h",
107-
srcs = ["snappy/snappy-stubs-public.h.in"],
108-
outs = ["snappy/snappy-stubs-public.h"],
109-
cmd = ("sed " +
110-
"-e 's/$${\\(.*\\)_01}/\\1/g' " +
111-
"-e 's/$${SNAPPY_MAJOR}/1/g' " +
112-
"-e 's/$${SNAPPY_MINOR}/1/g' " +
113-
"-e 's/$${SNAPPY_PATCHLEVEL}/4/g' " +
114-
"$< >$@"),
43+
"snappy",
44+
],
45+
local_defines = ["NDEBUG=1"],
11546
)
11647

11748
filegroup(
@@ -174,6 +105,7 @@ cmake(
174105
},
175106
no_match_error = "Building rocksdb for the specified CPU is not supported.",
176107
),
108+
includes = ["include/rocksdb/c.h"],
177109
lib_source = ":rocksdb_srcs",
178110
out_static_libs = ["librocksdb.a"],
179111
targets = ["rocksdb"],
@@ -186,52 +118,44 @@ cmake(
186118
],
187119
)
188120

189-
filegroup(
190-
name = "out_dir",
191-
srcs = [":rocksdb"],
192-
visibility = ["//visibility:public"],
193-
)
194-
195121
select_file(
196-
name = "librocksdb",
197-
srcs = ":out_dir",
122+
name = "librocksdb_a",
123+
srcs = ":rocksdb",
198124
subpath = "librocksdb.a",
199125
)
200126

201-
copy_file(
202-
name = "librocksdb_copy",
203-
src = ":librocksdb",
204-
out = "librocksdb.a",
205-
allow_symlink = False,
206-
)
207-
208-
copy_to_directory(
209-
name = "rocksdb_lib",
210-
srcs = [":librocksdb_copy"],
211-
visibility = ["//visibility:public"],
127+
filegroup(
128+
name = "rocksdb_include",
129+
srcs = glob(
130+
include = ["rocksdb/include/rocksdb/**/*.h"],
131+
),
212132
)
213133

214-
# Copy the include folder so we can specify `ROCKSDB_INCLUDE_DIR`
215-
#
216-
# Note: We used to use `select_file` here but it generated symlinks that
217-
# overlapped with others and spammed the logs with WARNINGs.
218-
copy_to_directory(
219-
name = "rocksdb_include",
220-
srcs = [":out_dir"],
221-
out = "include",
222-
root_paths = ["rocksdb/include"],
223-
visibility = ["//visibility:public"],
134+
# We need to expose the header files with the rocksdb static lib. The `cmake`
135+
# rules doesn't give us a way to do that, so we manually piece it together.
136+
cc_import(
137+
name = "librocksdb",
138+
hdrs = [":rocksdb_include"],
139+
static_library = ":librocksdb_a",
224140
)
225141

226-
select_file(
227-
name = "libsnappy",
228-
srcs = ":snappy",
229-
subpath = "libsnappy.a",
142+
rust_bindgen(
143+
name = "bindings",
144+
bindgen_flags = [
145+
"--no-derive-debug",
146+
"--blocklist-type=max_align_t",
147+
"--ctypes-prefix=libc",
148+
],
149+
cc_lib = ":librocksdb",
150+
header = "rocksdb/include/rocksdb/c.h",
230151
)
231152

153+
# Place the generated artifacts into an OUT_DIR.
154+
#
155+
# TODO(parkmycar): <https://github.com/bazelbuild/rules_rust/issues/3184>
232156
copy_to_directory(
233-
name = "snappy_lib",
234-
srcs = [":libsnappy"],
157+
name = "out_dir",
158+
srcs = [":bindings"],
235159
visibility = ["//visibility:public"],
236160
)
237161

0 commit comments

Comments
 (0)