diff --git a/Cargo.lock b/Cargo.lock index 151c41100c2..ac58baba0f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1980,6 +1980,12 @@ version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" +[[package]] +name = "bytemuck" +version = "1.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677" + [[package]] name = "byteorder" version = "1.5.0" @@ -3332,6 +3338,15 @@ dependencies = [ "syn 2.0.105", ] +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", +] + [[package]] name = "env_logger" version = "0.9.3" @@ -3345,6 +3360,16 @@ dependencies = [ "termcolor", ] +[[package]] +name = "env_logger" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +dependencies = [ + "env_filter", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -3531,7 +3556,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b439cac1603a75866d038ec54f17264f06ca0c1b155b266ffe23b8195d3ad3d" dependencies = [ "chrono", - "env_logger", + "env_logger 0.9.3", "futures", "log", "thiserror 1.0.69", @@ -4610,6 +4635,28 @@ dependencies = [ "serde", ] +[[package]] +name = "inferno" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96d2465363ed2d81857759fc864cf6bb7997f79327aec028d65bd7989393685" +dependencies = [ + "ahash 0.8.12", + "clap", + "crossbeam-channel", + "crossbeam-utils", + "dashmap 6.1.0", + "env_logger 0.11.8", + "indexmap 2.10.0", + "itoa", + "log", + "num-format", + "once_cell", + "quick-xml", + "rgb", + "str_stack", +] + [[package]] name = "insta" version = "1.43.1" @@ -4708,6 +4755,23 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jemalloc_pprof" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74ff642505c7ce8d31c0d43ec0e235c6fd4585d9b8172d8f9dd04d36590200b5" +dependencies = [ + "anyhow", + "libc", + "mappings", + "once_cell", + "pprof_util", + "tempfile", + "tikv-jemalloc-ctl", + "tokio", + "tracing", +] + [[package]] name = "jobserver" version = "0.1.33" @@ -5412,7 +5476,7 @@ dependencies = [ "linera-service-graphql-client", "linera-version", "linera-views", - "prost", + "prost 0.14.1", "reqwest 0.11.27", "sqlx", "thiserror 1.0.69", @@ -5491,7 +5555,9 @@ version = "0.16.0" dependencies = [ "anyhow", "axum", + "jemalloc_pprof", "prometheus", + "thiserror 1.0.69", "tokio", "tokio-util", "tracing", @@ -5550,7 +5616,7 @@ dependencies = [ "papaya", "prometheus", "proptest", - "prost", + "prost 0.14.1", "rand 0.8.5", "rcgen", "serde", @@ -5680,7 +5746,7 @@ dependencies = [ "port-selector", "prometheus", "proptest", - "prost", + "prost 0.14.1", "quick_cache", "rand 0.8.5", "reqwest 0.11.27", @@ -5694,6 +5760,7 @@ dependencies = [ "test-log", "test-strategy", "thiserror 1.0.69", + "tikv-jemallocator", "tokio", "tokio-stream", "tokio-util", @@ -5774,7 +5841,7 @@ dependencies = [ "linera-version", "linera-views", "proptest", - "prost", + "prost 0.14.1", "serde", "serde-reflection", "similar-asserts", @@ -6232,6 +6299,19 @@ dependencies = [ "syn 2.0.105", ] +[[package]] +name = "mappings" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4d277bb50d4508057e7bddd7fcd19ef4a4cc38051b6a5a36868d75ae2cbeb9" +dependencies = [ + "anyhow", + "libc", + "once_cell", + "pprof_util", + "tracing", +] + [[package]] name = "matchers" version = "0.1.0" @@ -7234,6 +7314,21 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "pprof_util" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9aba4251d95ac86f14c33e688d57a9344bfcff29e9b0c5a063fc66b5facc8a1" +dependencies = [ + "anyhow", + "backtrace", + "flate2", + "inferno", + "num", + "paste", + "prost 0.13.5", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -7394,6 +7489,16 @@ dependencies = [ "unarray", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive 0.13.5", +] + [[package]] name = "prost" version = "0.14.1" @@ -7401,7 +7506,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.14.1", ] [[package]] @@ -7417,13 +7522,26 @@ dependencies = [ "once_cell", "petgraph", "prettyplease", - "prost", + "prost 0.14.1", "prost-types", "regex", "syn 2.0.105", "tempfile", ] +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 2.0.105", +] + [[package]] name = "prost-derive" version = "0.14.1" @@ -7443,7 +7561,7 @@ version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" dependencies = [ - "prost", + "prost 0.14.1", ] [[package]] @@ -7517,6 +7635,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", +] + [[package]] name = "quick_cache" version = "0.6.16" @@ -8081,6 +8208,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "rgb" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce" +dependencies = [ + "bytemuck", +] + [[package]] name = "ring" version = "0.17.14" @@ -9414,6 +9550,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4af28eeb7c18ac2dbdb255d40bee63f203120e1db6b0024b177746ebec7049c1" +[[package]] +name = "str_stack" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" + [[package]] name = "stringprep" version = "0.1.5" @@ -9774,6 +9916,37 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "tikv-jemalloc-ctl" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f21f216790c8df74ce3ab25b534e0718da5a1916719771d3fec23315c99e468b" +dependencies = [ + "libc", + "paste", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3c60906412afa9c2b5b5a48ca6a5abe5736aec9eb48ad05037a677e52e4e2d" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cec5ff18518d81584f477e9bfdf957f5bb0979b0bac3af4ca30b5b3ae2d2865" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "time" version = "0.3.41" @@ -10063,7 +10236,7 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a82868bf299e0a1d2e8dce0dc33a46c02d6f045b2c1f1d6cc8dc3d0bf1812ef" dependencies = [ - "prost", + "prost 0.14.1", "tokio", "tokio-stream", "tonic", @@ -10077,7 +10250,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" dependencies = [ "bytes", - "prost", + "prost 0.14.1", "tonic", ] @@ -10103,7 +10276,7 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34da53e8387581d66db16ff01f98a70b426b091fdf76856e289d5c1bd386ed7b" dependencies = [ - "prost", + "prost 0.14.1", "prost-types", "tokio", "tokio-stream", diff --git a/Cargo.toml b/Cargo.toml index bcf5e4e180d..7f05653505f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,6 +136,7 @@ indexed_db_futures = "0.4.1" insta = "1.36.1" is-terminal = "0.4.12" itertools = "0.14.0" +jemalloc_pprof = "0.8.1" js-sys = "0.3.70" k256 = { version = "0.13.4", default-features = false, features = [ "ecdsa", @@ -233,6 +234,7 @@ test-log = { version = "0.2.15", default-features = false, features = [ test-strategy = "0.3.1" thiserror = "1.0.65" thiserror-context = "0.1.1" +tikv-jemallocator = "0.6.0" tokio = "1.36.0" tokio-stream = "0.1.14" tokio-test = "0.4.3" diff --git a/docker/Dockerfile b/docker/Dockerfile index a44185eccbe..b233b01346c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -25,7 +25,7 @@ ARG binaries= ARG copy=${binaries:+_copy} ARG build_flag=--release ARG build_folder=release -ARG build_features=scylladb,metrics +ARG build_features=scylladb,metrics,memory-profiling ARG rustflags="-C force-frame-pointers=yes" FROM rust:1.74-slim-bookworm AS builder @@ -39,7 +39,8 @@ ARG rustflags RUN apt-get update && apt-get install -y \ pkg-config \ protobuf-compiler \ - clang + clang \ + make COPY examples examples COPY linera-base linera-base diff --git a/kubernetes/linera-validator/grafana-dashboards/profiling/jemalloc-memory.json b/kubernetes/linera-validator/grafana-dashboards/profiling/jemalloc-memory.json new file mode 100644 index 00000000000..c67f579228d --- /dev/null +++ b/kubernetes/linera-validator/grafana-dashboards/profiling/jemalloc-memory.json @@ -0,0 +1,146 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "P02E4190217B50628" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "displayMode": "flamegraph" + }, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "P02E4190217B50628" + }, + "groupBy": [], + "labelSelector": "{service_name=\"memory/default/shards/\"}", + "profileTypeId": "memory:inuse_space:bytes:space:bytes", + "queryType": "profile", + "refId": "A" + } + ], + "title": "Memory Allocation Flamegraph - Server", + "type": "flamegraph" + }, + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "${datasource}" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 2, + "options": { + "displayMode": "flamegraph" + }, + "targets": [ + { + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "${datasource}" + }, + "groupBy": [], + "labelSelector": "{service_name=\"memory/default/proxy/\"}", + "profileTypeId": "memory:inuse_space:bytes:space:bytes", + "queryType": "profile", + "refId": "A" + } + ], + "title": "Memory Allocation Flamegraph - Proxy", + "type": "flamegraph" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "linera", + "memory", + "profiling" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Pyroscope", + "value": "P02E4190217B50628" + }, + "hide": 0, + "includeAll": false, + "label": "Pyroscope Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "grafana-pyroscope-datasource", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus Datasource", + "multi": false, + "name": "prometheus_datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Memory", + "uid": "linera-memory-profiling", + "version": 1, + "weekStart": "" +} diff --git a/kubernetes/linera-validator/templates/grafana-pyroscope-dashboard-config.yaml b/kubernetes/linera-validator/templates/grafana-pyroscope-dashboard-config.yaml index 254a6d080c9..b553a4d713e 100644 --- a/kubernetes/linera-validator/templates/grafana-pyroscope-dashboard-config.yaml +++ b/kubernetes/linera-validator/templates/grafana-pyroscope-dashboard-config.yaml @@ -7,4 +7,5 @@ metadata: annotations: grafana_folder: "Profiling" data: - cpu_profiling.json: {{ .Files.Get "grafana-dashboards/profiling/cpu.json" | quote }} \ No newline at end of file + cpu_profiling.json: {{ .Files.Get "grafana-dashboards/profiling/cpu.json" | quote }} + memory_profiling.json: {{ .Files.Get "grafana-dashboards/profiling/jemalloc-memory.json" | quote }} \ No newline at end of file diff --git a/linera-faucet/server/src/lib.rs b/linera-faucet/server/src/lib.rs index 8ebe294c596..871bd2475d3 100644 --- a/linera-faucet/server/src/lib.rs +++ b/linera-faucet/server/src/lib.rs @@ -42,7 +42,7 @@ use linera_execution::{ Committee, ExecutionError, Operation, }; #[cfg(feature = "metrics")] -use linera_metrics::prometheus_server; +use linera_metrics::monitoring_server; use linera_storage::{Clock as _, Storage}; use serde::{Deserialize, Serialize}; use tokio::sync::{oneshot, Notify}; @@ -662,7 +662,7 @@ where let index_handler = axum::routing::get(graphiql).post(Self::index_handler); #[cfg(feature = "metrics")] - prometheus_server::start_metrics(self.metrics_address(), cancellation_token.clone()); + monitoring_server::start_metrics(self.metrics_address(), cancellation_token.clone()); let app = Router::new() .route("/", index_handler) diff --git a/linera-metrics/Cargo.toml b/linera-metrics/Cargo.toml index 4e062d5880a..24071d627e5 100644 --- a/linera-metrics/Cargo.toml +++ b/linera-metrics/Cargo.toml @@ -14,10 +14,18 @@ version.workspace = true [lints] workspace = true +[features] +memory-profiling = ["jemalloc_pprof"] + [dependencies] anyhow.workspace = true axum.workspace = true +jemalloc_pprof = { workspace = true, features = [ + "symbolize", + "flamegraph", +], optional = true } prometheus.workspace = true +thiserror.workspace = true tokio = { workspace = true, features = ["full"] } tokio-util.workspace = true tracing.workspace = true diff --git a/linera-metrics/src/lib.rs b/linera-metrics/src/lib.rs index 88f8e0dd1d2..f7569f510b8 100644 --- a/linera-metrics/src/lib.rs +++ b/linera-metrics/src/lib.rs @@ -3,4 +3,7 @@ //! A library for Linera server metrics. -pub mod prometheus_server; +pub mod monitoring_server; + +#[cfg(feature = "memory-profiling")] +pub mod memory_profiler; diff --git a/linera-metrics/src/memory_profiler.rs b/linera-metrics/src/memory_profiler.rs new file mode 100644 index 00000000000..271236712e2 --- /dev/null +++ b/linera-metrics/src/memory_profiler.rs @@ -0,0 +1,139 @@ +// Copyright (c) Zefchain Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//! Safe jemalloc memory profiling with jemalloc_pprof integration +//! +//! This module provides HTTP endpoints for pprof format profiles using pull model. +//! Profiles are generated on-demand when endpoints are requested by Grafana Alloy. + +use axum::{ + http::{header, StatusCode}, + response::IntoResponse, +}; +use jemalloc_pprof::PROF_CTL; +use thiserror::Error; +use tracing::{error, info}; + +#[derive(Debug, Error)] +pub enum MemoryProfilerError { + #[error("jemalloc profiling not activated - check malloc_conf configuration")] + JemallocProfilingNotActivated, + #[error("PROF_CTL not available - ensure jemalloc is built with profiling support")] + ProfCtlNotAvailable, + #[error("another profiler is already running")] + AnotherProfilerAlreadyRunning, +} + +/// Memory profiler using safe jemalloc_pprof wrapper (pull model only) +pub struct MemoryProfiler; + +impl MemoryProfiler { + pub fn check_prof_ctl() -> Result<(), MemoryProfilerError> { + // Check if jemalloc profiling is available + if let Some(prof_ctl) = PROF_CTL.as_ref() { + let prof_ctl = prof_ctl + .try_lock() + .map_err(|_| MemoryProfilerError::AnotherProfilerAlreadyRunning)?; + + if !prof_ctl.activated() { + error!("jemalloc profiling not activated"); + return Err(MemoryProfilerError::JemallocProfilingNotActivated); + } + + info!("✓ jemalloc memory profiling is ready"); + } else { + error!("PROF_CTL not available"); + return Err(MemoryProfilerError::ProfCtlNotAvailable); + } + + Ok(()) + } + + /// HTTP endpoint for heap profile - returns fresh pprof data + pub async fn heap_profile() -> Result { + info!("Serving heap profile via /debug/pprof"); + + match Self::collect_heap_profile().await { + Ok(profile_data) => { + info!("✓ Serving heap profile ({} bytes)", profile_data.len()); + Ok(( + StatusCode::OK, + [(header::CONTENT_TYPE, "application/octet-stream")], + profile_data, + )) + } + Err(e) => { + error!("Failed to collect heap profile: {}", e); + Err(StatusCode::INTERNAL_SERVER_ERROR) + } + } + } + + /// HTTP endpoint for flamegraph - returns SVG flamegraph + pub async fn heap_flamegraph() -> Result { + info!("Serving heap flamegraph via /debug/flamegraph"); + + match Self::collect_heap_flamegraph().await { + Ok(flamegraph_svg) => { + info!("✓ Serving heap flamegraph ({} bytes)", flamegraph_svg.len()); + Ok(( + StatusCode::OK, + [(header::CONTENT_TYPE, "image/svg+xml")], + flamegraph_svg, + )) + } + Err(e) => { + error!("Failed to collect heap flamegraph: {}", e); + Err(StatusCode::INTERNAL_SERVER_ERROR) + } + } + } + + /// Collect heap profile on-demand using safe jemalloc_pprof wrapper + async fn collect_heap_profile() -> anyhow::Result> { + if let Some(prof_ctl) = PROF_CTL.as_ref() { + let mut prof_ctl = prof_ctl.lock().await; + + if !prof_ctl.activated() { + return Err(anyhow::anyhow!("jemalloc profiling not activated")); + } + + match prof_ctl.dump_pprof() { + Ok(profile) => { + info!("✓ Collected heap profile ({} bytes)", profile.len()); + Ok(profile) + } + Err(e) => { + error!("Failed to dump pprof profile: {}", e); + Err(anyhow::anyhow!("Failed to dump pprof profile: {}", e)) + } + } + } else { + Err(anyhow::anyhow!("PROF_CTL not available")) + } + } + + /// Collect heap flamegraph using prof_ctl.dump_flamegraph() + async fn collect_heap_flamegraph() -> anyhow::Result> { + if let Some(prof_ctl) = PROF_CTL.as_ref() { + let mut prof_ctl = prof_ctl.lock().await; + + if !prof_ctl.activated() { + return Err(anyhow::anyhow!("jemalloc profiling not activated")); + } + + match prof_ctl.dump_flamegraph() { + Ok(flamegraph_bytes) => { + info!("✓ Generated flamegraph ({} bytes)", flamegraph_bytes.len()); + Ok(flamegraph_bytes) + } + Err(e) => { + error!("Failed to dump flamegraph: {}", e); + Err(anyhow::anyhow!("Failed to dump flamegraph: {}", e)) + } + } + } else { + Err(anyhow::anyhow!("PROF_CTL not available")) + } + } +} diff --git a/linera-metrics/src/monitoring_server.rs b/linera-metrics/src/monitoring_server.rs new file mode 100644 index 00000000000..3e43591c793 --- /dev/null +++ b/linera-metrics/src/monitoring_server.rs @@ -0,0 +1,80 @@ +// Copyright (c) Zefchain Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use std::fmt::Debug; + +use axum::{http::StatusCode, response::IntoResponse, routing::get, Router}; +use tokio::net::ToSocketAddrs; +use tokio_util::sync::CancellationToken; +use tracing::info; + +#[cfg(feature = "memory-profiling")] +use crate::memory_profiler::MemoryProfiler; + +pub fn start_metrics( + address: impl ToSocketAddrs + Debug + Send + 'static, + shutdown_signal: CancellationToken, +) { + info!("Starting to serve metrics on {:?}", address); + + #[cfg(feature = "memory-profiling")] + let app = { + // Try to add memory profiling endpoint + match MemoryProfiler::check_prof_ctl() { + Ok(()) => { + info!("Memory profiling available, enabling /debug/pprof and /debug/flamegraph endpoints"); + Router::new() + .route("/metrics", get(serve_metrics)) + .route("/debug/pprof", get(MemoryProfiler::heap_profile)) + .route("/debug/flamegraph", get(MemoryProfiler::heap_flamegraph)) + } + Err(e) => { + tracing::warn!( + "Memory profiling not available: {}, serving metrics-only", + e + ); + Router::new().route("/metrics", get(serve_metrics)) + } + } + }; + + #[cfg(not(feature = "memory-profiling"))] + let app = Router::new().route("/metrics", get(serve_metrics)); + + tokio::spawn(async move { + if let Err(e) = axum::serve(tokio::net::TcpListener::bind(address).await.unwrap(), app) + .with_graceful_shutdown(shutdown_signal.cancelled_owned()) + .await + { + panic!("Error serving metrics: {}", e); + } + }); +} + +async fn serve_metrics() -> Result { + let metric_families = prometheus::gather(); + Ok(prometheus::TextEncoder::new() + .encode_to_string(&metric_families) + .map_err(anyhow::Error::from)?) +} + +struct AxumError(anyhow::Error); + +impl IntoResponse for AxumError { + fn into_response(self) -> axum::response::Response { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Something went wrong: {}", self.0), + ) + .into_response() + } +} + +impl From for AxumError +where + E: Into, +{ + fn from(err: E) -> Self { + Self(err.into()) + } +} diff --git a/linera-metrics/src/prometheus_server.rs b/linera-metrics/src/prometheus_server.rs deleted file mode 100644 index 0bccea0436c..00000000000 --- a/linera-metrics/src/prometheus_server.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) Zefchain Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::fmt::Debug; - -use axum::{http::StatusCode, response::IntoResponse, routing::get, Router}; -use tokio::net::ToSocketAddrs; -use tokio_util::sync::CancellationToken; -use tracing::info; - -pub fn start_metrics( - address: impl ToSocketAddrs + Debug + Send + 'static, - shutdown_signal: CancellationToken, -) { - info!("Starting to serve metrics on {:?}", address); - let prometheus_router = Router::new().route("/metrics", get(serve_metrics)); - - tokio::spawn(async move { - if let Err(e) = axum::serve( - tokio::net::TcpListener::bind(address).await.unwrap(), - prometheus_router, - ) - .with_graceful_shutdown(shutdown_signal.cancelled_owned()) - .await - { - panic!("Error serving metrics: {}", e); - } - }); -} - -async fn serve_metrics() -> Result { - let metric_families = prometheus::gather(); - Ok(prometheus::TextEncoder::new() - .encode_to_string(&metric_families) - .map_err(anyhow::Error::from)?) -} - -struct AxumError(anyhow::Error); - -impl IntoResponse for AxumError { - fn into_response(self) -> axum::response::Response { - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Something went wrong: {}", self.0), - ) - .into_response() - } -} - -impl From for AxumError -where - E: Into, -{ - fn from(err: E) -> Self { - Self(err.into()) - } -} diff --git a/linera-service/Cargo.toml b/linera-service/Cargo.toml index 52ff73bb28d..fee66af236d 100644 --- a/linera-service/Cargo.toml +++ b/linera-service/Cargo.toml @@ -52,6 +52,13 @@ metrics = [ "linera-faucet-server/metrics", "linera-metrics", ] +jemalloc = ["tikv-jemallocator"] +memory-profiling = [ + "metrics", + "jemalloc", + "tikv-jemallocator/profiling", + "linera-metrics/memory-profiling", +] storage-service = ["linera-storage-service"] [dependencies] @@ -119,6 +126,10 @@ serde_json.workspace = true stdext = { workspace = true, optional = true } tempfile.workspace = true thiserror.workspace = true +tikv-jemallocator = { workspace = true, features = [ + "profiling", + "unprefixed_malloc_on_supported_platforms", +], optional = true } tokio = { workspace = true, features = ["full"] } tokio-stream.workspace = true tokio-util.workspace = true diff --git a/linera-service/src/cli/main.rs b/linera-service/src/cli/main.rs index e553eb2dbc0..1722c6875d7 100644 --- a/linera-service/src/cli/main.rs +++ b/linera-service/src/cli/main.rs @@ -4,6 +4,18 @@ #![recursion_limit = "256"] +#[cfg(feature = "jemalloc")] +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +// jemalloc configuration for memory profiling with jemalloc_pprof +// prof:true,prof_active:true - Enable profiling from start +// lg_prof_sample:19 - Sample every 512KB for good detail/overhead balance +#[cfg(feature = "memory-profiling")] +#[allow(non_upper_case_globals)] +#[export_name = "malloc_conf"] +pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0"; + use std::{ collections::{BTreeMap, BTreeSet}, env, @@ -42,6 +54,8 @@ use linera_execution::{ WasmRuntime, WithWasmDefault as _, }; use linera_faucet_server::{FaucetConfig, FaucetService}; +#[cfg(with_metrics)] +use linera_metrics::monitoring_server; use linera_persistent::{self as persistent, Persist, PersistExt as _}; use linera_service::{ cli::{ @@ -877,6 +891,16 @@ impl Runnable for Job { let shutdown_notifier = CancellationToken::new(); tokio::spawn(listen_for_shutdown_signals(shutdown_notifier.clone())); + // Start metrics server for benchmark monitoring + #[cfg(with_metrics)] + { + let metrics_address = std::net::SocketAddr::from(([127, 0, 0, 1], 0)); + monitoring_server::start_metrics( + metrics_address, + shutdown_notifier.clone(), + ); + } + let shared_context = std::sync::Arc::new(futures::lock::Mutex::new(context)); let chain_listener = ChainListener::new( listener_config, @@ -1129,6 +1153,16 @@ impl Runnable for Job { let shutdown_notifier = CancellationToken::new(); tokio::spawn(listen_for_shutdown_signals(shutdown_notifier.clone())); + // Start metrics server for multi-process benchmark monitoring + #[cfg(with_metrics)] + { + let metrics_address = std::net::SocketAddr::from(([127, 0, 0, 1], 0)); + monitoring_server::start_metrics( + metrics_address, + shutdown_notifier.clone(), + ); + } + let mut join_set = JoinSet::new(); let children_pids: Vec = children.iter().filter_map(|c| c.id()).collect(); diff --git a/linera-service/src/exporter/main.rs b/linera-service/src/exporter/main.rs index 86ead954dd9..e0ff4006d96 100644 --- a/linera-service/src/exporter/main.rs +++ b/linera-service/src/exporter/main.rs @@ -10,7 +10,7 @@ use exporter_service::ExporterService; use futures::FutureExt; use linera_base::listen_for_shutdown_signals; #[cfg(with_metrics)] -use linera_metrics::prometheus_server; +use linera_metrics::monitoring_server; use linera_rpc::NodeOptions; use linera_service::{ config::BlockExporterConfig, @@ -104,7 +104,7 @@ impl Runnable for ExporterContext { tokio::spawn(listen_for_shutdown_signals(shutdown_notifier.clone())); #[cfg(with_metrics)] - prometheus_server::start_metrics(self.config.metrics_address(), shutdown_notifier.clone()); + monitoring_server::start_metrics(self.config.metrics_address(), shutdown_notifier.clone()); let (sender, handle) = start_block_processor_task( storage, diff --git a/linera-service/src/proxy/grpc.rs b/linera-service/src/proxy/grpc.rs index 6977c340743..b113b3d3d83 100644 --- a/linera-service/src/proxy/grpc.rs +++ b/linera-service/src/proxy/grpc.rs @@ -23,7 +23,7 @@ use linera_core::{ JoinSetExt as _, }; #[cfg(with_metrics)] -use linera_metrics::prometheus_server; +use linera_metrics::monitoring_server; use linera_rpc::{ config::{ProxyConfig, ShardConfig, TlsConfig, ValidatorInternalNetworkConfig}, grpc::{ @@ -248,7 +248,7 @@ where let mut join_set = JoinSet::new(); #[cfg(with_metrics)] - prometheus_server::start_metrics(self.metrics_address(), shutdown_signal.clone()); + monitoring_server::start_metrics(self.metrics_address(), shutdown_signal.clone()); let (health_reporter, health_service) = tonic_health::server::health_reporter(); health_reporter diff --git a/linera-service/src/proxy/main.rs b/linera-service/src/proxy/main.rs index 331603f7c0f..8fda32fff90 100644 --- a/linera-service/src/proxy/main.rs +++ b/linera-service/src/proxy/main.rs @@ -1,6 +1,18 @@ // Copyright (c) Zefchain Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +#[cfg(feature = "jemalloc")] +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +// jemalloc configuration for memory profiling with jemalloc_pprof +// prof:true,prof_active:true - Enable profiling from start +// lg_prof_sample:19 - Sample every 512KB for good detail/overhead balance +#[cfg(feature = "memory-profiling")] +#[allow(non_upper_case_globals)] +#[export_name = "malloc_conf"] +pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0"; + use std::{net::SocketAddr, path::PathBuf, time::Duration}; use anyhow::{anyhow, bail, ensure, Result}; @@ -10,7 +22,7 @@ use linera_base::listen_for_shutdown_signals; use linera_client::config::ValidatorServerConfig; use linera_core::{node::NodeError, JoinSetExt as _}; #[cfg(with_metrics)] -use linera_metrics::prometheus_server; +use linera_metrics::monitoring_server; use linera_rpc::{ config::{ NetworkProtocol, ShardConfig, ValidatorInternalNetworkPreConfig, @@ -245,7 +257,7 @@ where let address = self.get_listen_address(); #[cfg(with_metrics)] - Self::start_metrics(address, shutdown_signal.clone()); + monitoring_server::start_metrics(address, shutdown_signal.clone()); self.public_config .protocol @@ -274,11 +286,6 @@ where .metrics_port } - #[cfg(with_metrics)] - pub fn start_metrics(address: SocketAddr, shutdown_signal: CancellationToken) { - prometheus_server::start_metrics(address, shutdown_signal) - } - fn get_listen_address(&self) -> SocketAddr { SocketAddr::from(([0, 0, 0, 0], self.port())) } diff --git a/linera-service/src/server.rs b/linera-service/src/server.rs index 639614cb5a8..3b34a91f45f 100644 --- a/linera-service/src/server.rs +++ b/linera-service/src/server.rs @@ -2,6 +2,18 @@ // Copyright (c) Zefchain Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +#[cfg(feature = "jemalloc")] +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +// jemalloc configuration for memory profiling with jemalloc_pprof +// prof:true,prof_active:true - Enable profiling from start +// lg_prof_sample:19 - Sample every 512KB for good detail/overhead balance +#[cfg(feature = "memory-profiling")] +#[allow(non_upper_case_globals)] +#[export_name = "malloc_conf"] +pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0"; + use std::{ borrow::Cow, num::NonZeroU16, @@ -20,7 +32,7 @@ use linera_client::config::{CommitteeConfig, ValidatorConfig, ValidatorServerCon use linera_core::{worker::WorkerState, JoinSetExt as _}; use linera_execution::{WasmRuntime, WithWasmDefault}; #[cfg(with_metrics)] -use linera_metrics::prometheus_server; +use linera_metrics::monitoring_server; use linera_persistent::{self as persistent, Persist}; use linera_rpc::{ config::{ @@ -103,7 +115,10 @@ impl ServerContext { #[cfg(with_metrics)] if let Some(port) = shard.metrics_port { - Self::start_metrics(&listen_address, port, shutdown_signal.clone()); + monitoring_server::start_metrics( + (listen_address.clone(), port), + shutdown_signal.clone(), + ); } let server_handle = simple::Server::new( @@ -146,7 +161,10 @@ impl ServerContext { for (state, shard_id, shard) in states { #[cfg(with_metrics)] if let Some(port) = shard.metrics_port { - Self::start_metrics(listen_address, port, shutdown_signal.clone()); + monitoring_server::start_metrics( + (listen_address.to_string(), port), + shutdown_signal.clone(), + ); } let server_handle = grpc::GrpcServer::spawn( @@ -176,11 +194,6 @@ impl ServerContext { join_set } - #[cfg(with_metrics)] - fn start_metrics(host: &str, port: u16, shutdown_signal: CancellationToken) { - prometheus_server::start_metrics((host.to_owned(), port), shutdown_signal); - } - fn get_listen_address() -> String { // Allow local IP address to be different from the public one. "0.0.0.0".to_string()