From 47045b3205a6e004cfe7633874d0a8f4d5173251 Mon Sep 17 00:00:00 2001 From: Martin Tomka Date: Mon, 10 Feb 2025 10:10:37 +0100 Subject: [PATCH 1/2] Add an example demonstrating the lock contention --- examples/metrics-basic/src/main.rs | 1 + examples/metrics-lock-contention/Cargo.toml | 10 +++ examples/metrics-lock-contention/README.md | 19 ++++ examples/metrics-lock-contention/src/main.rs | 91 ++++++++++++++++++++ 4 files changed, 121 insertions(+) create mode 100644 examples/metrics-lock-contention/Cargo.toml create mode 100644 examples/metrics-lock-contention/README.md create mode 100644 examples/metrics-lock-contention/src/main.rs diff --git a/examples/metrics-basic/src/main.rs b/examples/metrics-basic/src/main.rs index bbb6a24614..0924c45496 100644 --- a/examples/metrics-basic/src/main.rs +++ b/examples/metrics-basic/src/main.rs @@ -10,6 +10,7 @@ fn init_meter_provider() -> opentelemetry_sdk::metrics::SdkMeterProvider { // Build exporter using Delta Temporality (Defaults to Temporality::Cumulative) // .with_temporality(opentelemetry_sdk::metrics::Temporality::Delta) .build(); + let provider = SdkMeterProvider::builder() .with_periodic_exporter(exporter) .with_resource( diff --git a/examples/metrics-lock-contention/Cargo.toml b/examples/metrics-lock-contention/Cargo.toml new file mode 100644 index 0000000000..a7955ea7fa --- /dev/null +++ b/examples/metrics-lock-contention/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "metrics-lock-contention" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[dependencies] +opentelemetry = { path = "../../opentelemetry", features = ["metrics"] } +opentelemetry_sdk = { path = "../../opentelemetry-sdk", features = ["metrics"] } diff --git a/examples/metrics-lock-contention/README.md b/examples/metrics-lock-contention/README.md new file mode 100644 index 0000000000..e97c5c8353 --- /dev/null +++ b/examples/metrics-lock-contention/README.md @@ -0,0 +1,19 @@ +# OpenTelemetry Lock Contention for Metrics - Example + +This example demonstrates the performance difference of using a shared instance of `MeterProvider` in multiple threads vs +having a dedicated instance of `MeterProvider` for each thread. + +## Usage + +To run the example using a shared `MeterProvider`: + +```shell +cargo run --release -- shared +``` + +To run the example using a per-thread `MeterProvider`: + +```shell +cargo run --release -- per-thread +``` + diff --git a/examples/metrics-lock-contention/src/main.rs b/examples/metrics-lock-contention/src/main.rs new file mode 100644 index 0000000000..cfb960f10a --- /dev/null +++ b/examples/metrics-lock-contention/src/main.rs @@ -0,0 +1,91 @@ +use opentelemetry::metrics::MeterProvider; +use opentelemetry::KeyValue; +use opentelemetry_sdk::metrics::{ManualReader, SdkMeterProvider}; +use std::env::args; +use std::fmt::Debug; +use std::str::FromStr; +use std::thread::{spawn, JoinHandle}; +use std::time::{Duration, Instant}; + +const NUM_THREADS: usize = 10; + +const RUN_TIME: Duration = Duration::from_secs(120); + +fn main() { + let mode: Mode = args().nth(1).unwrap_or("shared".to_string()).parse().unwrap(); + + println!("Running with mode: {:?}, Duration: {}s, Threads: {}", mode, RUN_TIME.as_secs(), NUM_THREADS); + + let handles: Vec> = match mode { + Mode::Shared => { + let provider = create_meter_provider(); + (0..NUM_THREADS).map(move |_|start_work(provider.clone())).collect() + } + Mode::PerThread => { + (0..NUM_THREADS).map(move |_|start_work(create_meter_provider())).collect() + } + }; + + let sum = handles + .into_iter() + .map(|h| h.join().unwrap()) + .sum::(); + + println!("Reported Metrics: {} millions", (sum / 1_000_000)); +} + +fn start_work(meter_provider: SdkMeterProvider) -> JoinHandle { + let histogram = meter_provider.meter("dummy").f64_histogram("histogram").build(); + spawn(move || { + let mut count = 0_usize; + let now = Instant::now(); + + loop { + histogram.record( + 10.5, + &[ + KeyValue::new("mykey1", "myvalue1"), + KeyValue::new("mykey2", "myvalue2"), + KeyValue::new("mykey3", "myvalue3"), + KeyValue::new("mykey4", "myvalue4"), + KeyValue::new("mykey5", "myvalue5"), + KeyValue::new("mykey6", "myvalue6"), + KeyValue::new("mykey7", "myvalue7"), + ], + ); + + count = count.checked_add(1).unwrap(); + + if now.elapsed() > RUN_TIME { + break; + } + } + + count + }) +} + +#[derive(Debug, Clone, Copy)] +enum Mode { + Shared, + PerThread +} + +impl FromStr for Mode { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + match s { + "shared" => Ok(Mode::Shared), + "per-thread" => Ok(Mode::PerThread), + _ => Err("invalid mode"), + } + } +} + + +fn create_meter_provider() -> SdkMeterProvider { + SdkMeterProvider::builder() + .with_reader(ManualReader::default()) + .build() +} From 3e1c1bcc1e3ae1815185131c4fd60d34d595b512 Mon Sep 17 00:00:00 2001 From: Martin Tomka Date: Wed, 12 Feb 2025 13:10:54 +0100 Subject: [PATCH 2/2] refine the example, drop some overhead --- examples/metrics-lock-contention/src/main.rs | 32 +++++++++----------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/examples/metrics-lock-contention/src/main.rs b/examples/metrics-lock-contention/src/main.rs index cfb960f10a..7a52301bc7 100644 --- a/examples/metrics-lock-contention/src/main.rs +++ b/examples/metrics-lock-contention/src/main.rs @@ -4,6 +4,8 @@ use opentelemetry_sdk::metrics::{ManualReader, SdkMeterProvider}; use std::env::args; use std::fmt::Debug; use std::str::FromStr; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::thread::{spawn, JoinHandle}; use std::time::{Duration, Instant}; @@ -13,19 +15,27 @@ const RUN_TIME: Duration = Duration::from_secs(120); fn main() { let mode: Mode = args().nth(1).unwrap_or("shared".to_string()).parse().unwrap(); + let exit_signal = Arc::new(AtomicBool::new(false)); println!("Running with mode: {:?}, Duration: {}s, Threads: {}", mode, RUN_TIME.as_secs(), NUM_THREADS); let handles: Vec> = match mode { Mode::Shared => { let provider = create_meter_provider(); - (0..NUM_THREADS).map(move |_|start_work(provider.clone())).collect() + let signal = Arc::clone(&exit_signal); + (0..NUM_THREADS).map(move |_|start_work(provider.clone(), signal.clone())).collect() } Mode::PerThread => { - (0..NUM_THREADS).map(move |_|start_work(create_meter_provider())).collect() + let signal = Arc::clone(&exit_signal); + (0..NUM_THREADS).map(move |_|start_work(create_meter_provider(), signal.clone())).collect() } }; + _ = spawn(move || { + std::thread::sleep(RUN_TIME); + exit_signal.store(true, Ordering::Relaxed); + }); + let sum = handles .into_iter() .map(|h| h.join().unwrap()) @@ -34,29 +44,17 @@ fn main() { println!("Reported Metrics: {} millions", (sum / 1_000_000)); } -fn start_work(meter_provider: SdkMeterProvider) -> JoinHandle { +fn start_work(meter_provider: SdkMeterProvider, exit_signal: Arc) -> JoinHandle { let histogram = meter_provider.meter("dummy").f64_histogram("histogram").build(); spawn(move || { let mut count = 0_usize; let now = Instant::now(); loop { - histogram.record( - 10.5, - &[ - KeyValue::new("mykey1", "myvalue1"), - KeyValue::new("mykey2", "myvalue2"), - KeyValue::new("mykey3", "myvalue3"), - KeyValue::new("mykey4", "myvalue4"), - KeyValue::new("mykey5", "myvalue5"), - KeyValue::new("mykey6", "myvalue6"), - KeyValue::new("mykey7", "myvalue7"), - ], - ); - + histogram.record(10.5, &[]); count = count.checked_add(1).unwrap(); - if now.elapsed() > RUN_TIME { + if exit_signal.load(Ordering::Relaxed) { break; } }