feat: add jemalloc as optional custom allocator (#1679)

* Add jemalloc, snmalloc, and tcmalloc.

* Remove unnecessary check. rustc yelled at me if I tried to enable two.

* Adjust snmalloc following instructions.

* Remove tcmalloc since I can't get it to compile.

* Fix jemalloc loading with dlopen.

* Remove snmalloc.

Author: mbutrovich

native/Cargo.lock

@@ -39,6 +39,7 @@ arrow = { workspace = true }
 parquet = { workspace = true, default-features = false, features = ["experimental"] }
 futures = { workspace = true }
 mimalloc = { version = "*", default-features = false, optional = true }
+tikv-jemallocator = { version = "0.6.0", optional = true, features = ["disable_initial_exec_tls"] }
 tokio = { version = "1", features = ["rt-multi-thread"] }
 async-trait = { workspace = true }
 log = "0.4"
@@ -51,13 +52,13 @@ snap = "1.1"
 # we disable default features in lz4_flex to force the use of the faster unsafe encoding and decoding implementation
 lz4_flex = { version = "0.11.3", default-features = false }
 zstd = "0.13.3"
-rand = { workspace = true}
+rand = { workspace = true }
 num = { workspace = true }
 bytes = { workspace = true }
 tempfile = "3.8.0"
 itertools = "0.14.0"
 paste = "1.0.14"
-datafusion = {  workspace = true }
+datafusion = { workspace = true }
 once_cell = "1.18.0"
 regex = { workspace = true }
 crc32fast = "1.3.2"
@@ -80,7 +81,8 @@ datafusion-functions-nested = { version = "47.0.0" }
 
 [features]
 default = []
-hdfs=["datafusion-comet-objectstore-hdfs"]
+hdfs = ["datafusion-comet-objectstore-hdfs"]
+jemalloc = ["tikv-jemallocator"]
 
 # exclude optional packages from cargo machete verifications
 [package.metadata.cargo-machete]

native/core/Cargo.toml

@@ -36,9 +36,13 @@ use log4rs::{
     encode::pattern::PatternEncoder,
     Config,
 };
+use once_cell::sync::OnceCell;
+
+#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
+use tikv_jemallocator::Jemalloc;
+
 #[cfg(feature = "mimalloc")]
 use mimalloc::MiMalloc;
-use once_cell::sync::OnceCell;
 
 use errors::{try_unwrap_or_throw, CometError, CometResult};
 
@@ -50,6 +54,10 @@ pub mod execution;
 mod jvm_bridge;
 pub mod parquet;
 
+#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
+#[global_allocator]
+static GLOBAL: Jemalloc = Jemalloc;
+
 #[cfg(feature = "mimalloc")]
 #[global_allocator]
 static GLOBAL: MiMalloc = MiMalloc;