Merge pull request #3325 from Pana/chore/bumpJemalloc2

feat: add memory&cpu pprof

Author: Pana

Cargo.lock

@@ -1,5 +1,5 @@
-# Use workspace section to allow test all cases under root folder (cargo test --all).
 [workspace]
+resolver = "2"
 exclude = ["tools"]
 members = [
     # bins
@@ -111,11 +111,10 @@ members = [
     "crates/config",
     "crates/cfxcore/types",
     "crates/cfxcore/pow",
-    "crates/cfxcore/errors",
+    "crates/cfxcore/errors", 
+    "crates/util/mallocator",
 ]
 
-resolver = "2"
-
 [workspace.package]
 version = "3.0.0"
 authors = ["Conflux Network"]
@@ -252,6 +251,7 @@ diem-network-address-encryption = { path = "./crates/pos/config/management/netwo
 cfx-tasks = { path = "./crates/tasks" }
 # eest_types = { path = "./crates/eest_types" }
 cfx-config = { path = "./crates/config" }
+cfx-mallocator-utils = { path = "./crates/util/mallocator" }
 
 # basics
 bytes = "1.9"
@@ -328,9 +328,18 @@ mio = { version = "1.0.4", features = ["os-ext", "net", "os-poll"] }
 mio-timer = { git = "https://github.com/Conflux-Chain/mio-timer", rev = "91a02cc" }
 mio-misc = { git = "https://github.com/Conflux-Chain/mio-misc", rev = "27ad80f" }
 reqwest = "0.12"
+axum = "0.8"
+mockito = "1.5.0"
 
-# jemallocator
-tikv-jemallocator = "0.6"
+# mallocator
+tikv-jemallocator = { version = "0.6", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] }
+tikv-jemalloc-ctl = { version = "0.6", features = ["use_std", "stats"] }
+tikv-jemalloc-sys = { version = "0.6", features = ["profiling"] }
+jemalloc_pprof = { version = "0.8", features = ["symbolize", "flamegraph"] }
+# CPU profiling
+pprof = { version = "0.15", features = ["flamegraph", "protobuf-codec"] }
+tracy-client = "0.18.0"
+snmalloc-rs = { version = "0.3.7", features = ["build_cc"] }
 
 # crypto & hash
 #fixed-hash = "0.5"

Cargo.toml

@@ -41,21 +41,30 @@ tokio = { workspace = true, features = ["rt"] }
 bls-signatures = { workspace = true }
 cfx-executor = { workspace = true }
 cfx-execute-helper = { workspace = true }
+cfx-mallocator-utils = { workspace = true }
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 tikv-jemallocator = { workspace = true }
 
 [features]
 default = ["jemalloc-global"]
 deadlock-detection = ["parking_lot/deadlock_detection"]
-jemalloc-global = ["malloc_size_of/jemalloc-global"]
 u64-mpt-db-key = ["client/u64_mpt_db_key"]
 # According to Rust's feature unification, when a feature is enabled for a dependency in the root package,
 # it will be enabled across all paths depending on that package. 
 # (https://doc.rust-lang.org/cargo/reference/features.html#feature-unification)
 blst-portable = ["bls-signatures/blst-portable"]
 align_evm = ["cfx-executor/align_evm", "cfx-execute-helper/align_evm"]
+# mallocator related features
+jemalloc-global = ["malloc_size_of/jemalloc", "cfx-mallocator-utils/jemalloc"]
+jemalloc-prof = ["cfx-mallocator-utils/jemalloc", "cfx-mallocator-utils/jemalloc-prof"]
+tracy-allocator = ["cfx-mallocator-utils/tracy-allocator"]
+# Because jemalloc is default and preferred over snmalloc when both features are
+# enabled, `--no-default-features` should be used when enabling snmalloc or
+# snmalloc-native.
+snmalloc = ["cfx-mallocator-utils/snmalloc"]
+snmalloc-native = ["cfx-mallocator-utils/snmalloc-native"]
 
 [dev-dependencies]
 tokio = { workspace = true, features = ["full"] }
-mockito = "1.5.0"
+mockito = { workspace = true }

bins/conflux/Cargo.toml

@@ -2,6 +2,17 @@
 // Conflux is free software and distributed under GNU General Public License.
 // See http://www.gnu.org/licenses/
 
+#[cfg(all(not(target_env = "msvc"), feature = "jemalloc-global"))]
+#[global_allocator]
+static ALLOC: cfx_mallocator_utils::allocator::Allocator =
+    cfx_mallocator_utils::allocator::new_allocator();
+// jemalloc profiling config
+#[allow(non_upper_case_globals)]
+#[export_name = "malloc_conf"]
+#[cfg(all(not(target_env = "msvc"), feature = "jemalloc-prof"))]
+pub static malloc_conf: &[u8] =
+    b"prof:true,prof_active:true,lg_prof_sample:19\0"; // 512kb
+
 #[cfg(test)]
 mod test;
 

bins/conflux/src/main.rs

@@ -79,6 +79,7 @@ cfx-parity-trace-types = { workspace = true }
 cfx-tasks = { workspace = true }
 cfx-config = { workspace = true }
 cfxcore-types = { workspace = true }
+cfx-mallocator-utils = { workspace = true }
 
 [dev-dependencies]
 criterion = { workspace = true }

crates/client/Cargo.toml

@@ -21,7 +21,6 @@ use parking_lot::{Condvar, Mutex};
 use rand_08::{prelude::StdRng, rngs::OsRng, SeedableRng};
 use threadpool::ThreadPool;
 
-use crate::keylib::KeyPair;
 use blockgen::BlockGenerator;
 use cfx_executor::machine::{Machine, VmFactory};
 use cfx_parameters::genesis::{
@@ -65,6 +64,7 @@ use cfx_config::{parse_config_address_string, Configuration};
 
 use crate::{
     accounts::{account_provider, keys_path},
+    keylib::KeyPair,
     rpc::{
         extractor::RpcExtractor,
         impls::{
@@ -74,6 +74,7 @@ use crate::{
         launch_async_rpc_servers, setup_debug_rpc_apis, setup_public_rpc_apis,
     },
 };
+use cfx_mallocator_utils::start_pprf_server;
 use cfxcore::consensus::pos_handler::read_initial_nodes_from_file;
 
 pub mod delegate_convert;
@@ -656,9 +657,6 @@ pub fn initialize_not_light_node_modules(
         accounts,
     ));
 
-    let task_manager = TaskManager::new(tokio_runtime.handle().clone());
-    let task_executor = task_manager.executor();
-
     let debug_rpc_http_server = super::rpc::start_http(
         conf.local_http_config(),
         setup_debug_rpc_apis(
@@ -718,7 +716,8 @@ pub fn initialize_not_light_node_modules(
         setup_public_rpc_apis(common_impl, rpc_impl, pubsub, &conf),
     )?;
 
-    network.start();
+    let task_manager = TaskManager::new(tokio_runtime.handle().clone());
+    let task_executor = task_manager.executor();
 
     let eth_rpc_server_handle =
         tokio_runtime.block_on(launch_async_rpc_servers(
@@ -730,8 +729,21 @@ pub fn initialize_not_light_node_modules(
             conf,
         ))?;
 
+    // start pprf server, which is used to serve the pprof data for heap
+    // profiling
+    if let Some(pprf_addr) = conf.raw_conf.profiling_listen_addr.as_ref() {
+        let pprf_addr = pprf_addr.clone();
+        let _pprf_server_handle = tokio_runtime.spawn(async move {
+            if let Err(e) = start_pprf_server(&pprf_addr).await {
+                eprintln!("Error starting pprof server: {}", e);
+            }
+        });
+    }
+
     metrics::initialize(conf.metrics_config(), task_executor.clone());
 
+    network.start();
+
     Ok((
         data_man,
         pow,

crates/client/src/common/mod.rs

@@ -134,6 +134,7 @@ build_config! {
         (metrics_output_file, (Option<String>), None)
         (metrics_report_interval_ms, (u64), 3_000)
         (metrics_prometheus_listen_addr, (Option<String>), None)
+        (profiling_listen_addr, (Option<String>), None)
         (rocksdb_disable_wal, (bool), false)
         (txgen_account_count, (usize), 10)
 

crates/config/src/configuration.rs

@@ -14,7 +14,7 @@ parking_lot = { workspace = true }
 smallvec = { workspace = true }
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
-tikv-jemallocator = { workspace = true }
+tikv-jemallocator = { workspace = true, features = ["profiling", "unprefixed_malloc_on_supported_platforms"] }
 
 [features]
-jemalloc-global = []
+jemalloc = []

crates/util/malloc_size_of/Cargo.toml

@@ -10,13 +10,6 @@
 
 //! A reduced fork of Firefox's malloc_size_of crate, for bundling with
 //! WebRender.
-
-#[cfg(all(not(target_env = "msvc"), feature = "jemalloc-global"))]
-use tikv_jemallocator::Jemalloc;
-#[cfg(all(not(target_env = "msvc"), feature = "jemalloc-global"))]
-#[global_allocator]
-static GLOBAL: Jemalloc = Jemalloc;
-
 use cfg_if::cfg_if;
 use cfx_types::{
     AddressWithSpace, AllChainID, Space, SpaceMap, H160, H256, H512, U256, U512,
@@ -555,7 +548,7 @@ mod usable_size {
                 HeapSize(heap, 0, ptr) as usize
             }
 
-        } else if #[cfg(feature = "jemalloc-global")] {
+        } else if #[cfg(feature = "jemalloc")] {
 
             /// Use of jemalloc usable size C function through jemallocator crate call.
             pub unsafe extern "C" fn malloc_usable_size(ptr: *const c_void) -> usize {

crates/util/malloc_size_of/src/lib.rs

@@ -0,0 +1,42 @@
+[package]
+name = "cfx-mallocator-utils"
+version.workspace = true
+authors.workspace = true
+description.workspace = true
+documentation.workspace = true
+homepage.workspace = true
+keywords.workspace = true
+repository.workspace = true
+license-file.workspace = true
+edition.workspace = true
+
+[dependencies]
+chrono = { workspace = true }
+jemalloc_pprof = { workspace = true }
+pprof = { workspace = true }
+log = { workspace = true }
+tokio = { workspace = true }
+axum = { workspace = true }
+cfg-if = { workspace = true }
+tracy-client = { workspace = true, optional = true, features = ["demangle"] }
+
+[target.'cfg(unix)'.dependencies]
+tikv-jemallocator = { workspace = true, optional = true }
+snmalloc-rs = { workspace = true, optional = true }
+libc = "0.2"
+
+[features]
+jemalloc = ["dep:tikv-jemallocator"]
+
+# Enables jemalloc profiling features
+jemalloc-prof = ["jemalloc", "tikv-jemallocator?/profiling"]
+
+# Wraps the selected allocator in the tracy profiling allocator
+tracy-allocator = ["dep:tracy-client"]
+
+snmalloc = ["dep:snmalloc-rs"]
+
+# Enables the snmalloc-rs `native-cpu` feature, which optimizes snmalloc for the
+# native CPU of the host machine. Not sure why this feature is not derived from
+# RUSTFLAGS or enabled when `target-cpu=native`.
+snmalloc-native = ["snmalloc", "snmalloc-rs/native-cpu"]