diff --git a/Cargo.toml b/Cargo.toml index e3912a8490..984de3a0a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,2 +1,8 @@ [workspace] -members = ["jemallocator", "jemallocator-global", "jemalloc-ctl", "jemalloc-sys"] +members = [ + "jemallocator", + "jemallocator-global", + "jemalloc-ctl", + "jemalloc-sys", + "test-dylib", +] diff --git a/ci/run.sh b/ci/run.sh index 61182f318e..56ef686006 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -32,7 +32,7 @@ cargo test --target "${TARGET}" --features stats cargo test --target "${TARGET}" --features 'debug profiling' cargo test --target "${TARGET}" \ - --features unprefixed_malloc_on_supported_platforms + --features override_allocator_on_supported_platforms cargo test --target "${TARGET}" --no-default-features cargo test --target "${TARGET}" --no-default-features \ --features background_threads_runtime_support @@ -48,7 +48,7 @@ cargo test --target "${TARGET}" --release cargo test --target "${TARGET}" --manifest-path jemalloc-sys/Cargo.toml cargo test --target "${TARGET}" \ --manifest-path jemalloc-sys/Cargo.toml \ - --features unprefixed_malloc_on_supported_platforms + --features override_allocator_on_supported_platforms # FIXME: jemalloc-ctl fails in the following targets case "${TARGET}" in @@ -77,3 +77,14 @@ cargo test --target "${TARGET}" \ # # The Alloc trait is unstable: # ${CARGO_CMD} test --target "${TARGET}" --features alloc_trait # fi + +# Test that overriding works in dylibs. +case "$TARGET" in + "i686-unknown-linux-musl") ;; + "x86_64-unknown-linux-musl") ;; + *) + cargo run --target "${TARGET}" \ + -p test-dylib \ + --features override_allocator_on_supported_platforms + ;; +esac diff --git a/jemalloc-sys/Cargo.toml b/jemalloc-sys/Cargo.toml index c99204cf36..a6c1998186 100644 --- a/jemalloc-sys/Cargo.toml +++ b/jemalloc-sys/Cargo.toml @@ -46,6 +46,7 @@ background_threads_runtime_support = [] background_threads = [ "background_threads_runtime_support" ] stats = [] unprefixed_malloc_on_supported_platforms = [] +override_allocator_on_supported_platforms = [ "unprefixed_malloc_on_supported_platforms" ] disable_initial_exec_tls = [] disable_cache_oblivious = [] diff --git a/jemalloc-sys/README.md b/jemalloc-sys/README.md index 1fab7ae630..db3103bd4f 100644 --- a/jemalloc-sys/README.md +++ b/jemalloc-sys/README.md @@ -48,11 +48,11 @@ This crate provides following cargo feature flags: * `stats` (configure `jemalloc` with `--enable-stats`): Enable statistics gathering functionality. See the `jemalloc`'s "`opt.stats_print`" option documentation for usage details. - + * `debug` (configure `jemalloc` with `--enable-debug`): Enable assertions and validation code. This incurs a substantial performance hit, but is very useful during application development. - + * `background_threads_runtime_support` (enabled by default): enables background-threads run-time support when building `jemalloc-sys` on some POSIX targets supported by `jemalloc`. Background threads are disabled at run-time @@ -72,16 +72,33 @@ This crate provides following cargo feature flags: * `unprefixed_malloc_on_supported_platforms`: when disabled, configure `jemalloc` with `--with-jemalloc-prefix=_rjem_`. Enabling this causes symbols like `malloc` to be emitted without a prefix, overriding the ones defined by - libc. This usually causes C and C++ code linked in the same program to use - `jemalloc` as well. On some platforms prefixes are always used because - unprefixing is known to cause segfaults due to allocator mismatches. - + libc. This usually causes C, Objective-C and C++ code linked in the same + program to use `jemalloc` as well. On some platforms prefixes are always used + because unprefixing is known to cause segfaults due to allocator mismatches. + +* `override_allocator_on_supported_platforms`: override the system allocator, + even outside Rust code. + + This enables the `unprefixed_malloc_on_supported_platforms` feature, with the + addition that it forces overriding the allocator even if `malloc` and `free` + would not usually have been seen by the linker. It also overrides the + allocator on Apple platforms. + + Note that to use this, the `jemalloc-sys` crate must actually be visible to + `rustc` (it is not enough to only declare it in `Cargo.toml`). This can be + done by adding: + ```rust + use jemalloc_sys as _; + ``` + + In your `main.rs`. + * `disable_initial_exec_tls` (disabled by default): when enabled, jemalloc is - built with the `--disable-initial-exec-tls` option. It disables the - initial-exec TLS model for jemalloc's internal thread-local storage (on those - platforms that support explicit settings). This can allow jemalloc to be + built with the `--disable-initial-exec-tls` option. It disables the + initial-exec TLS model for jemalloc's internal thread-local storage (on those + platforms that support explicit settings). This can allow jemalloc to be dynamically loaded after program startup (e.g. using dlopen). If you encounter - the error `yourlib.so: cannot allocate memory in static TLS block`, you'll + the error `yourlib.so: cannot allocate memory in static TLS block`, you'll likely want to enable this. * `disable_cache_oblivious` (disabled by default): when enabled, jemalloc is @@ -104,7 +121,7 @@ hyphens `-` are replaced with underscores `_`(see variable, the `/etc/malloc.conf` symlink, and the `MALLOC_CONF` environment variable (note: this variable might be prefixed as `_RJEM_MALLOC_CONF`). For example, to change the default decay time for dirty pages to 30 seconds: - + ``` JEMALLOC_SYS_WITH_MALLOC_CONF=dirty_decay_ms:30000 ``` @@ -115,17 +132,17 @@ hyphens `-` are replaced with underscores `_`(see allocator page size equal to the system page size, so this option need not be specified unless the system page size may change between configuration and execution, e.g. when cross compiling. - + * `JEMALLOC_SYS_WITH_LG_HUGEPAGE=`: Specify the base 2 log of the system huge page size. This option is useful when cross compiling, or when overriding the default for systems that do not explicitly support huge pages. - - + + * `JEMALLOC_SYS_WITH_LG_QUANTUM=`: Specify the base 2 log of the minimum allocation alignment. jemalloc needs to know the minimum alignment that meets the following C standard requirement (quoted from the April 12, 2011 draft of the C11 standard): - + > The pointer returned if the allocation succeeds is suitably aligned so that > it may be assigned to a pointer to any type of object with a fundamental > alignment requirement and then used to access such an object or an array of diff --git a/jemalloc-sys/build.rs b/jemalloc-sys/build.rs index 3fc5b9fe4a..9ec19241d6 100644 --- a/jemalloc-sys/build.rs +++ b/jemalloc-sys/build.rs @@ -111,10 +111,16 @@ fn main() { .iter() .any(|i| target.contains(i)) { - warning!( - "Unprefixed `malloc` requested on unsupported platform `{}` => using prefixed `malloc`", - target - ); + // Apple targets don't support unprefixed, but they do support + // overriding (if you do the `zone_register` trick), so no need to + // warn there. + let override_ = env::var("CARGO_FEATURE_OVERRIDE_ALLOCATOR_ON_SUPPORTED_PLATFORMS").is_ok(); + if !target.contains("apple") || !override_ { + warning!( + "Unprefixed `malloc` requested on unsupported platform `{}` => using prefixed `malloc`", + target + ); + } use_prefix = true; } diff --git a/jemalloc-sys/src/env.rs b/jemalloc-sys/src/env.rs index 2053ad3068..88703d3598 100644 --- a/jemalloc-sys/src/env.rs +++ b/jemalloc-sys/src/env.rs @@ -21,4 +21,4 @@ pub static NO_BG_THREAD_TARGETS: &[&str] = &["musl"]; // https://github.com/rust-lang/rust/commit/e3b414d8612314e74e2b0ebde1ed5c6997d28e8d // https://github.com/rust-lang/rust/commit/9f3de647326fbe50e0e283b9018ab7c41abccde3 // https://github.com/rust-lang/rust/commit/ed015456a114ae907a36af80c06f81ea93182a24 -pub static NO_UNPREFIXED_MALLOC_TARGETS: &[&str] = &["android", "dragonfly", "darwin"]; +pub static NO_UNPREFIXED_MALLOC_TARGETS: &[&str] = &["android", "dragonfly", "apple"]; diff --git a/jemalloc-sys/src/lib.rs b/jemalloc-sys/src/lib.rs index 078096dba4..4e5573c6c4 100644 --- a/jemalloc-sys/src/lib.rs +++ b/jemalloc-sys/src/lib.rs @@ -890,3 +890,86 @@ pub type extent_merge_t = unsafe extern "C" fn( mod env; pub use env::*; + +// When using the `"override_allocator_on_supported_platforms"` feature flag, +// the user wants us to globally override the system allocator. +// +// However, since we build `jemalloc` as a static library (an archive), the +// linker may decide to not care about our overrides if it can't directly see +// references to the symbols, see the following link for details: +// +// +// This is problematic if `jemalloc_sys` is used from a library that by itself +// doesn't allocate, while invoking other shared libraries that do. +// +// Another especially problematic case would be something like the following: +// +// ``` +// // Call `malloc` whose symbol is looked up statically. +// let ptr = libc::malloc(42); +// +// // But use a dynamically looked up `free`. +// let free = libc::dlsym(null_mut(), c"free".as_ptr()); +// let free = transmute::<*mut c_void, unsafe extern "C" fn(*mut c_void)>(free); +// free(ptr); +// ``` +// +// Since if the `malloc` and `free` provided by `jemalloc` end up in different +// object files in the archive (NOTE: In practice, this is unlikely to be an +// issue, since `jemalloc.c` contains all the implementations and is compiled +// as a single object file), the linker would think that only `malloc` was +// used, and would never load the `free` that we also want (and hence we'd end +// up executing jemalloc's `malloc` and the system's `free`, which is UB). +// +// To avoid this problem, we make sure that all the allocator functions are +// visible to the linker, such that it will always override all of them. +// +// We do this by referencing these symbols in `#[used]` statics, which makes +// them known to `rustc`, which will reference them in a `symbols.o` stub file +// that is later passed to the linker. See the following link for details on +// how this works: +// + +#[cfg(all( + feature = "override_allocator_on_supported_platforms", + not(target_vendor = "apple") +))] +mod set_up_statics { + use super::*; + + #[used] + static USED_MALLOC: unsafe extern "C" fn(usize) -> *mut c_void = malloc; + #[used] + static USED_CALLOC: unsafe extern "C" fn(usize, usize) -> *mut c_void = calloc; + #[used] + static USED_POSIX_MEMALIGN: unsafe extern "C" fn(*mut *mut c_void, usize, usize) -> c_int = + posix_memalign; + #[used] + static USED_ALIGNED_ALLOC: unsafe extern "C" fn(usize, usize) -> *mut c_void = aligned_alloc; + #[used] + static USED_REALLOC: unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void = realloc; + #[used] + static USED_FREE: unsafe extern "C" fn(*mut c_void) = free; +} + +// On macOS, jemalloc doesn't directly override malloc/free, but instead +// registers itself with the allocator's zone APIs in a ctor (`zone_register` +// is marked with `__attribute__((constructor))`). +// +// Similarly to above though, for the Mach-O linker to actually consider ctors +// as "used" when defined in an archive member in a static library, so we need +// to explicitly reference the function via. Rust's `#[used]`. + +#[cfg(all( + feature = "override_allocator_on_supported_platforms", + target_vendor = "apple" +))] +#[used] +static USED_ZONE_REGISTER: unsafe extern "C" fn() = { + extern "C" { + #[cfg_attr(prefixed, link_name = "_rjem_je_zone_register")] + #[cfg_attr(not(prefixed), link_name = "je_zone_register")] + fn zone_register(); + } + zone_register +}; diff --git a/jemalloc-sys/tests/unprefixed_malloc.rs b/jemalloc-sys/tests/unprefixed_malloc.rs index 7b9228d624..17fb75d569 100644 --- a/jemalloc-sys/tests/unprefixed_malloc.rs +++ b/jemalloc-sys/tests/unprefixed_malloc.rs @@ -9,3 +9,16 @@ fn malloc_is_prefixed() { fn malloc_is_overridden() { assert_eq!(tikv_jemalloc_sys::malloc as usize, libc::malloc as usize) } + +#[cfg(any( + not(prefixed), + all( + feature = "override_allocator_on_supported_platforms", + target_vendor = "apple" + ), +))] +#[test] +fn malloc_and_libc_are_interoperable_when_overridden() { + let ptr = unsafe { tikv_jemalloc_sys::malloc(42) }; + unsafe { libc::free(ptr) }; +} diff --git a/jemallocator/Cargo.toml b/jemallocator/Cargo.toml index bee3663596..4a3844a447 100644 --- a/jemallocator/Cargo.toml +++ b/jemallocator/Cargo.toml @@ -53,6 +53,10 @@ stats = ["tikv-jemalloc-sys/stats"] background_threads_runtime_support = ["tikv-jemalloc-sys/background_threads_runtime_support"] background_threads = ["tikv-jemalloc-sys/background_threads"] unprefixed_malloc_on_supported_platforms = ["tikv-jemalloc-sys/unprefixed_malloc_on_supported_platforms"] +override_allocator_on_supported_platforms = [ + "unprefixed_malloc_on_supported_platforms", + "tikv-jemalloc-sys/override_allocator_on_supported_platforms", +] disable_initial_exec_tls = ["tikv-jemalloc-sys/disable_initial_exec_tls"] disable_cache_oblivious = ["tikv-jemalloc-sys/disable_cache_oblivious"] diff --git a/test-dylib/Cargo.toml b/test-dylib/Cargo.toml new file mode 100644 index 0000000000..e67c74eaa2 --- /dev/null +++ b/test-dylib/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "test-dylib" +version = "0.0.0" +license = "MIT OR Apache-2.0" +description = "A test helper for jemalloc-sys" +edition = "2018" +publish = false + +[dependencies] +libc = { version = "^0.2.8", default-features = false } +tikv-jemalloc-sys = { path = "../jemalloc-sys" } + +[build-dependencies] +cc = "^1.0.13" + +[features] +override_allocator_on_supported_platforms = [ + "tikv-jemalloc-sys/override_allocator_on_supported_platforms", +] + +[[bin]] +name = "test-dylib" +test = false diff --git a/test-dylib/build.rs b/test-dylib/build.rs new file mode 100644 index 0000000000..eefbc32eaa --- /dev/null +++ b/test-dylib/build.rs @@ -0,0 +1,29 @@ +//! Build shared library `dep.c`. +use std::{env, path::PathBuf}; + +fn main() { + println!("cargo:rerun-if-changed=src/dep.c"); + + let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap()); + + // NOTE: Only for testing, extension is wrong when cross-compiling. + let dylib = out_dir.join(format!( + "{}dep{}", + env::consts::DLL_PREFIX, + env::consts::DLL_SUFFIX + )); + + let status = cc::Build::new() + .get_compiler() + .to_command() + .arg("src/dep.c") + .arg("-shared") + .arg("-o") + .arg(&dylib) + .status() + .unwrap(); + assert!(status.success()); + + println!("cargo:rustc-link-lib=dylib=dep"); + println!("cargo:rustc-link-search=native={}", out_dir.display()); +} diff --git a/test-dylib/src/dep.c b/test-dylib/src/dep.c new file mode 100644 index 0000000000..eefbdc5b71 --- /dev/null +++ b/test-dylib/src/dep.c @@ -0,0 +1,21 @@ +#define _GNU_SOURCE +#include +#include +#include + +const char* dep_lookup_malloc_address(void) { + Dl_info info; + if (!dladdr((void *)malloc, &info)) { + printf("failed finding `malloc`\n"); + abort(); + } + return info.dli_fname; +} + +void* dep_malloc(size_t size) { + return malloc(size); +} + +void dep_free(void* ptr) { + free(ptr); +} diff --git a/test-dylib/src/main.rs b/test-dylib/src/main.rs new file mode 100644 index 0000000000..590a16ca11 --- /dev/null +++ b/test-dylib/src/main.rs @@ -0,0 +1,51 @@ +//! Test that the symbol address of `malloc` in shared libraries come from +//! the place we'd expect it to. +use std::ffi::{c_char, c_void, CStr}; + +use tikv_jemalloc_sys as _; + +extern "C-unwind" { + fn dep_lookup_malloc_address() -> *const c_char; + fn dep_malloc(size: libc::size_t) -> *mut c_void; + fn dep_free(ptr: *mut c_void); +} + +fn lookup_malloc_address() -> *const c_char { + unsafe { + let mut info: libc::Dl_info = core::mem::zeroed(); + let fnptr: unsafe extern "C" fn(libc::size_t) -> *mut c_void = libc::malloc; + let fnptr = fnptr as *const c_void; + if libc::dladdr(fnptr, &mut info) == 0 { + libc::printf(b"failed finding `malloc`\n\0".as_ptr().cast()); + libc::abort(); + } + info.dli_fname + } +} +fn main() { + // Check that pointers created with `malloc` in a dylib dependency can be + // free'd with `free` here, or vice-versa. + let ptr = unsafe { libc::malloc(10) }; + unsafe { dep_free(ptr) }; + let ptr = unsafe { dep_malloc(10) }; + unsafe { libc::free(ptr) }; + + // If overidden, test that the same is true for `tikv_jemalloc_sys`' + // symbols being interoperable with `free`. + if cfg!(feature = "override_allocator_on_supported_platforms") { + let ptr = unsafe { tikv_jemalloc_sys::malloc(10) }; + unsafe { dep_free(ptr) }; + let ptr = unsafe { tikv_jemalloc_sys::malloc(10) }; + unsafe { libc::free(ptr) }; + + let ptr = unsafe { libc::malloc(10) }; + unsafe { tikv_jemalloc_sys::free(ptr) }; + let ptr = unsafe { dep_malloc(10) }; + unsafe { tikv_jemalloc_sys::free(ptr) }; + } + + // Extra check that the `malloc` symbol was actually from the same place. + let dep = unsafe { CStr::from_ptr(dep_lookup_malloc_address()) }; + let here = unsafe { CStr::from_ptr(lookup_malloc_address()) }; + assert_eq!(dep, here); +}