diff --git a/crates/cargo-codspeed/src/walltime_results.rs b/crates/cargo-codspeed/src/walltime_results.rs index 2ce30087..731981a0 100644 --- a/crates/cargo-codspeed/src/walltime_results.rs +++ b/crates/cargo-codspeed/src/walltime_results.rs @@ -53,10 +53,17 @@ impl WalltimeBenchmark { impl From for WalltimeBenchmark { fn from(value: RawWallTimeData) -> Self { - let times_ns: Vec = value.times_ns.iter().map(|&t| t as f64).collect(); - let mut data = Data::new(times_ns.clone()); + let total_time = value.times_per_round_ns.iter().sum::() as f64 / 1_000_000_000.0; + let time_per_iteration_per_round_ns: Vec<_> = value + .times_per_round_ns + .into_iter() + .zip(&value.iters_per_round) + .map(|(time_per_round, iter_per_round)| time_per_round / iter_per_round) + .map(|t| t as f64) + .collect::>(); + + let mut data = Data::new(time_per_iteration_per_round_ns); let rounds = data.len() as u64; - let total_time = times_ns.iter().sum::() / 1_000_000_000.0; let mean_ns = data.mean().unwrap(); @@ -91,7 +98,9 @@ impl From for WalltimeBenchmark { let min_ns = data.min(); let max_ns = data.max(); - let iter_per_round = value.iter_per_round as u64; + // TODO(COD-1056): We currently only support single iteration count per round + let iter_per_round = (value.iters_per_round.iter().sum::() + / value.iters_per_round.len() as u128) as u64; let warmup_iters = 0; // FIXME: add warmup detection let stats = BenchmarkStats { @@ -172,9 +181,9 @@ mod tests { }; let raw_bench = RawWallTimeData { metadata, - iter_per_round: 1, + iters_per_round: vec![1], max_time_ns: None, - times_ns: vec![42], + times_per_round_ns: vec![42], }; let benchmark: WalltimeBenchmark = raw_bench.into(); @@ -183,4 +192,31 @@ mod tests { assert_eq!(benchmark.stats.max_ns, 42.); assert_eq!(benchmark.stats.mean_ns, 42.); } + + #[test] + fn test_parse_bench_with_variable_iterations() { + let metadata = BenchmarkMetadata { + name: "benchmark".to_string(), + uri: "test::benchmark".to_string(), + }; + + let raw_bench = RawWallTimeData { + metadata, + iters_per_round: vec![1, 2, 3, 4, 5, 6], + max_time_ns: None, + times_per_round_ns: vec![42, 42 * 2, 42 * 3, 42 * 4, 42 * 5, 42 * 6], + }; + + let total_rounds = raw_bench.iters_per_round.iter().sum::() as f64; + + let benchmark: WalltimeBenchmark = raw_bench.into(); + assert_eq!(benchmark.stats.stdev_ns, 0.); + assert_eq!(benchmark.stats.min_ns, 42.); + assert_eq!(benchmark.stats.max_ns, 42.); + assert_eq!(benchmark.stats.mean_ns, 42.); + assert_eq!( + benchmark.stats.total_time, + 42. * total_rounds / 1_000_000_000.0 + ); + } } diff --git a/crates/codspeed/src/walltime.rs b/crates/codspeed/src/walltime.rs index fea658ac..e51804b9 100644 --- a/crates/codspeed/src/walltime.rs +++ b/crates/codspeed/src/walltime.rs @@ -15,24 +15,24 @@ pub struct BenchmarkMetadata { pub struct RawWallTimeData { #[serde(flatten)] pub metadata: BenchmarkMetadata, - pub iter_per_round: u32, + pub iters_per_round: Vec, + pub times_per_round_ns: Vec, pub max_time_ns: Option, - pub times_ns: Vec, } impl RawWallTimeData { fn from_runtime_data( name: String, uri: String, - iter_per_round: u32, + iters_per_round: Vec, + times_per_round_ns: Vec, max_time_ns: Option, - times_ns: Vec, ) -> Self { RawWallTimeData { metadata: BenchmarkMetadata { name, uri }, - iter_per_round, + iters_per_round, max_time_ns, - times_ns, + times_per_round_ns, } } @@ -50,13 +50,37 @@ impl RawWallTimeData { /// Entry point called in patched integration to harvest raw walltime data /// /// `CODSPEED_CARGO_WORKSPACE_ROOT` is expected to be set for this to work +/// +/// # Arguments +/// +/// - `scope`: The used integration, e.g. "divan" or "criterion" +/// - `name`: The name of the benchmark +/// - `uri`: The URI of the benchmark +/// - `iters_per_round`: The number of iterations for each round (=sample_size), e.g. `[1, 2, 3]` (variable) or `[2, 2, 2, 2]` (constant). +/// - `times_per_round_ns`: The measured time for each round in nanoseconds, e.g. `[1000, 2000, 3000]` +/// - `max_time_ns`: The time limit for the benchmark in nanoseconds (if defined) +/// +/// # Pseudo-code +/// +/// ```text +/// let sample_count = /* The number of executions for the same benchmark. */ +/// let sample_size = iters_per_round = vec![/* The number of iterations within each sample. */]; +/// for round in 0..sample_count { +/// let times_per_round_ns = 0; +/// for iteration in 0..sample_size[round] { +/// run_benchmark(); +/// times_per_round_ns += /* measured execution time */; +/// } +/// } +/// ``` +/// pub fn collect_raw_walltime_results( scope: &str, name: String, uri: String, - iter_per_round: u32, + iters_per_round: Vec, + times_per_round_ns: Vec, max_time_ns: Option, - times_ns: Vec, ) { if !crate::utils::running_with_codspeed_runner() { return; @@ -66,7 +90,13 @@ pub fn collect_raw_walltime_results( eprintln!("codspeed failed to get workspace root. skipping"); return; }; - let data = RawWallTimeData::from_runtime_data(name, uri, iter_per_round, max_time_ns, times_ns); + let data = RawWallTimeData::from_runtime_data( + name, + uri, + iters_per_round, + times_per_round_ns, + max_time_ns, + ); data.dump_to_results(&workspace_root, scope); } diff --git a/crates/criterion_compat/benches/criterion_integration/mod.rs b/crates/criterion_compat/benches/criterion_integration/mod.rs index 2a0f0ddc..655739ef 100644 --- a/crates/criterion_compat/benches/criterion_integration/mod.rs +++ b/crates/criterion_compat/benches/criterion_integration/mod.rs @@ -6,6 +6,7 @@ pub mod iter_with_large_setup; pub mod iter_with_setup; pub mod measurement_overhead; pub mod sampling_mode; +pub mod sleep; pub mod special_characters; pub mod with_inputs; diff --git a/crates/criterion_compat/benches/criterion_integration/sleep.rs b/crates/criterion_compat/benches/criterion_integration/sleep.rs new file mode 100644 index 00000000..5cfdd09d --- /dev/null +++ b/crates/criterion_compat/benches/criterion_integration/sleep.rs @@ -0,0 +1,22 @@ +use codspeed_criterion_compat::{criterion_group, Criterion}; +use std::time::Duration; + +fn sleep_benchmarks(c: &mut Criterion) { + c.bench_function("sleep_1ms", |b| { + b.iter(|| std::thread::sleep(Duration::from_millis(1))) + }); + + c.bench_function("sleep_10ms", |b| { + b.iter(|| std::thread::sleep(Duration::from_millis(10))) + }); + + c.bench_function("sleep_50ms", |b| { + b.iter(|| std::thread::sleep(Duration::from_millis(50))) + }); + + c.bench_function("sleep_100ms", |b| { + b.iter(|| std::thread::sleep(Duration::from_millis(100))) + }); +} + +criterion_group!(benches, sleep_benchmarks); diff --git a/crates/criterion_compat/benches/criterion_integration_main.rs b/crates/criterion_compat/benches/criterion_integration_main.rs index ef3e4ed1..40b1f31b 100644 --- a/crates/criterion_compat/benches/criterion_integration_main.rs +++ b/crates/criterion_compat/benches/criterion_integration_main.rs @@ -13,5 +13,6 @@ criterion_main! { criterion_integration::measurement_overhead::benches, criterion_integration::custom_measurement::benches, criterion_integration::sampling_mode::benches, + criterion_integration::sleep::benches, criterion_integration::async_measurement_overhead::benches, } diff --git a/crates/criterion_compat/criterion_fork/src/analysis/mod.rs b/crates/criterion_compat/criterion_fork/src/analysis/mod.rs index a0120eb9..c8eab967 100644 --- a/crates/criterion_compat/criterion_fork/src/analysis/mod.rs +++ b/crates/criterion_compat/criterion_fork/src/analysis/mod.rs @@ -258,7 +258,7 @@ pub(crate) fn common( } if criterion.should_save_baseline() && ::codspeed::utils::running_with_codspeed_runner() { - codspeed::collect_walltime_results(id, criterion, &iters, avg_times); + codspeed::collect_walltime_results(id, criterion, &iters, ×); } } @@ -293,7 +293,7 @@ mod codspeed { id: &BenchmarkId, c: &Criterion, iters: &[f64], - avg_times: &[f64], + times: &[f64], ) { let (uri, bench_name) = create_uri_and_name(id, c); @@ -306,17 +306,17 @@ mod codspeed { } } - let avg_iter_per_round = iters.iter().sum::() / iters.len() as f64; + let iters_per_round = iters.iter().map(|t| *t as u128).collect(); + let times_per_round_ns = times.iter().map(|t| *t as u128).collect(); let max_time_ns = Some(c.config.measurement_time.as_nanos()); - let times_ns = avg_times.iter().map(|t| *t as u128).collect(); ::codspeed::walltime::collect_raw_walltime_results( "criterion", bench_name, uri, - avg_iter_per_round as u32, + iters_per_round, + times_per_round_ns, max_time_ns, - times_ns, ); } } diff --git a/crates/divan_compat/Cargo.toml b/crates/divan_compat/Cargo.toml index 5f8bbac3..60399fa9 100644 --- a/crates/divan_compat/Cargo.toml +++ b/crates/divan_compat/Cargo.toml @@ -25,3 +25,7 @@ codspeed-divan-compat-macros = { version = "=2.10.1", path = './macros' } [[bench]] name = "basic_example" harness = false + +[[bench]] +name = "sleep_benches" +harness = false diff --git a/crates/divan_compat/benches/sleep_benches.rs b/crates/divan_compat/benches/sleep_benches.rs new file mode 100644 index 00000000..7a8931ec --- /dev/null +++ b/crates/divan_compat/benches/sleep_benches.rs @@ -0,0 +1,29 @@ +#[divan::bench] +fn sleep_1ms() { + std::thread::sleep(std::time::Duration::from_millis(1)); +} + +#[divan::bench] +fn sleep_10ms() { + std::thread::sleep(std::time::Duration::from_millis(10)); +} + +#[divan::bench] +fn sleep_50ms() { + std::thread::sleep(std::time::Duration::from_millis(50)); +} + +#[divan::bench] +fn sleep_100ms() { + std::thread::sleep(std::time::Duration::from_millis(100)); +} + +// Tests COD-1044, do not modify the sample size or count! +#[divan::bench(sample_size = 3, sample_count = 6)] +fn sleep_100ms_with_custom_sample() { + std::thread::sleep(std::time::Duration::from_millis(100)); +} + +fn main() { + codspeed_divan_compat::main(); +} diff --git a/crates/divan_compat/divan_fork/src/divan.rs b/crates/divan_compat/divan_fork/src/divan.rs index 374eb7ed..b6f22871 100644 --- a/crates/divan_compat/divan_fork/src/divan.rs +++ b/crates/divan_compat/divan_fork/src/divan.rs @@ -424,7 +424,9 @@ mod codspeed { }; let iter_per_round = bench_context.samples.sample_size; - let times_ns: Vec<_> = + let iters_per_round = + vec![iter_per_round as u128; bench_context.samples.time_samples.len()]; + let times_per_round_ns: Vec<_> = bench_context.samples.time_samples.iter().map(|s| s.duration.picos / 1_000).collect(); let max_time_ns = bench_context.options.max_time.map(|t| t.as_nanos()); @@ -441,9 +443,9 @@ mod codspeed { "divan", bench_name, uri, - iter_per_round, + iters_per_round, + times_per_round_ns, max_time_ns, - times_ns, ); } }