diff --git a/Cargo.lock b/Cargo.lock index ae5e776d..cfd917b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -469,12 +469,10 @@ dependencies = [ "clap", "codspeed", "fs_extra", - "glob", "itertools 0.14.0", "predicates", "serde", "serde_json", - "statrs", "termcolor", "uuid", ] @@ -601,10 +599,12 @@ dependencies = [ "anyhow", "bincode", "colored", + "glob", "libc", "nix", "serde", "serde_json", + "statrs", "tempfile", "uuid", ] diff --git a/crates/cargo-codspeed/Cargo.toml b/crates/cargo-codspeed/Cargo.toml index e752e8a8..1925e063 100644 --- a/crates/cargo-codspeed/Cargo.toml +++ b/crates/cargo-codspeed/Cargo.toml @@ -27,8 +27,6 @@ anstyle = "1.0.8" serde = { workspace = true } serde_json = { workspace = true } codspeed = { path = "../codspeed", version = "=2.10.1" } -glob = "0.3.2" -statrs = "0.18.0" [dev-dependencies] assert_cmd = "2.0.15" diff --git a/crates/cargo-codspeed/src/main.rs b/crates/cargo-codspeed/src/main.rs index 82f1e573..d9e6e251 100644 --- a/crates/cargo-codspeed/src/main.rs +++ b/crates/cargo-codspeed/src/main.rs @@ -4,7 +4,6 @@ mod helpers; mod measurement_mode; mod prelude; mod run; -mod walltime_results; use crate::prelude::*; use std::{env::args_os, process::exit}; diff --git a/crates/cargo-codspeed/src/run.rs b/crates/cargo-codspeed/src/run.rs index 8beedfbc..866f14e8 100644 --- a/crates/cargo-codspeed/src/run.rs +++ b/crates/cargo-codspeed/src/run.rs @@ -3,12 +3,10 @@ use crate::{ helpers::get_codspeed_target_dir, measurement_mode::MeasurementMode, prelude::*, - walltime_results::{WalltimeBenchmark, WalltimeResults}, }; use anyhow::Context; use cargo_metadata::{Metadata, Package}; -use codspeed::walltime::get_raw_result_dir_from_workspace_root; -use glob::glob; +use codspeed::walltime_results::WalltimeResults; use std::{ io::{self, Write}, os::unix::process::ExitStatusExt, @@ -103,7 +101,7 @@ pub fn run_benches( let codspeed_target_dir = get_codspeed_target_dir(metadata, measurement_mode); let workspace_root = metadata.workspace_root.as_std_path(); if measurement_mode == MeasurementMode::Walltime { - clear_raw_walltime_data(workspace_root)?; + WalltimeResults::clear(workspace_root)?; } let benches = filters.benches_to_run(codspeed_target_dir, metadata)?; if benches.is_empty() { @@ -181,35 +179,14 @@ pub fn run_benches( Ok(()) } -fn clear_raw_walltime_data(workspace_root: &Path) -> Result<()> { - let raw_results_dir = get_raw_result_dir_from_workspace_root(workspace_root); - std::fs::remove_dir_all(&raw_results_dir).ok(); // ignore errors when the directory does not exist - std::fs::create_dir_all(&raw_results_dir).context("Failed to create raw_results directory")?; - Ok(()) -} - fn aggregate_raw_walltime_data(workspace_root: &Path) -> Result<()> { - // retrieve data from `{workspace_root}/target/codspeed/raw_results/{scope}/*.json - let walltime_benchmarks = glob(&format!( - "{}/**/*.json", - get_raw_result_dir_from_workspace_root(workspace_root) - .to_str() - .unwrap(), - ))? - .map(|sample| { - let sample = sample?; - let raw_walltime_data: codspeed::walltime::RawWallTimeData = - serde_json::from_reader(std::fs::File::open(&sample)?)?; - Ok(WalltimeBenchmark::from(raw_walltime_data)) - }) - .collect::>>()?; - - if walltime_benchmarks.is_empty() { + let results = WalltimeResults::collect_walltime_results(workspace_root)?; + if results.benchmarks().is_empty() { eprintln!("No walltime benchmarks found"); return Ok(()); } - for bench in &walltime_benchmarks { + for bench in results.benchmarks() { if bench.is_invalid() { eprintln!( "Warning: Benchmark {} was possibly optimized away", @@ -224,7 +201,6 @@ fn aggregate_raw_walltime_data(workspace_root: &Path) -> Result<()> { .join("results"); std::fs::create_dir_all(&results_folder).context("Failed to create results folder")?; - let results = WalltimeResults::from_benchmarks(walltime_benchmarks); let results_path = results_folder.join(format!("{}.json", std::process::id())); let mut results_file = std::fs::File::create(&results_path).context("Failed to create results file")?; diff --git a/crates/cargo-codspeed/src/walltime_results.rs b/crates/cargo-codspeed/src/walltime_results.rs deleted file mode 100644 index 731981a0..00000000 --- a/crates/cargo-codspeed/src/walltime_results.rs +++ /dev/null @@ -1,222 +0,0 @@ -use codspeed::walltime::{BenchmarkMetadata, RawWallTimeData}; -use serde::{Deserialize, Serialize}; -use statrs::statistics::{Data, Distribution, Max, Min, OrderStatistics}; - -const IQR_OUTLIER_FACTOR: f64 = 1.5; -const STDEV_OUTLIER_FACTOR: f64 = 3.0; - -#[derive(Debug, Serialize, Deserialize)] -struct BenchmarkStats { - min_ns: f64, - max_ns: f64, - mean_ns: f64, - stdev_ns: f64, - - q1_ns: f64, - median_ns: f64, - q3_ns: f64, - - rounds: u64, - total_time: f64, - iqr_outlier_rounds: u64, - stdev_outlier_rounds: u64, - iter_per_round: u64, - warmup_iters: u64, -} - -#[derive(Debug, Serialize, Deserialize, Default)] -struct BenchmarkConfig { - warmup_time_ns: Option, - min_round_time_ns: Option, - max_time_ns: Option, - max_rounds: Option, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct WalltimeBenchmark { - #[serde(flatten)] - metadata: BenchmarkMetadata, - - config: BenchmarkConfig, - stats: BenchmarkStats, -} - -impl WalltimeBenchmark { - pub(crate) fn is_invalid(&self) -> bool { - self.stats.min_ns < f64::EPSILON - } - - pub(crate) fn name(&self) -> &str { - &self.metadata.name - } -} - -impl From for WalltimeBenchmark { - fn from(value: RawWallTimeData) -> Self { - let total_time = value.times_per_round_ns.iter().sum::() as f64 / 1_000_000_000.0; - let time_per_iteration_per_round_ns: Vec<_> = value - .times_per_round_ns - .into_iter() - .zip(&value.iters_per_round) - .map(|(time_per_round, iter_per_round)| time_per_round / iter_per_round) - .map(|t| t as f64) - .collect::>(); - - let mut data = Data::new(time_per_iteration_per_round_ns); - let rounds = data.len() as u64; - - let mean_ns = data.mean().unwrap(); - - let stdev_ns = if data.len() < 2 { - // std_dev() returns f64::NAN if data has less than two entries, so we have to - // manually handle this case. - 0.0 - } else { - data.std_dev().unwrap() - }; - - let q1_ns = data.quantile(0.25); - let median_ns = data.median(); - let q3_ns = data.quantile(0.75); - - let iqr_ns = q3_ns - q1_ns; - let iqr_outlier_rounds = data - .iter() - .filter(|&&t| { - t < q1_ns - IQR_OUTLIER_FACTOR * iqr_ns || t > q3_ns + IQR_OUTLIER_FACTOR * iqr_ns - }) - .count() as u64; - - let stdev_outlier_rounds = data - .iter() - .filter(|&&t| { - t < mean_ns - STDEV_OUTLIER_FACTOR * stdev_ns - || t > mean_ns + STDEV_OUTLIER_FACTOR * stdev_ns - }) - .count() as u64; - - let min_ns = data.min(); - let max_ns = data.max(); - - // TODO(COD-1056): We currently only support single iteration count per round - let iter_per_round = (value.iters_per_round.iter().sum::() - / value.iters_per_round.len() as u128) as u64; - let warmup_iters = 0; // FIXME: add warmup detection - - let stats = BenchmarkStats { - min_ns, - max_ns, - mean_ns, - stdev_ns, - q1_ns, - median_ns, - q3_ns, - rounds, - total_time, - iqr_outlier_rounds, - stdev_outlier_rounds, - iter_per_round, - warmup_iters, - }; - - WalltimeBenchmark { - metadata: BenchmarkMetadata { - name: value.metadata.name, - uri: value.metadata.uri, - }, - config: BenchmarkConfig { - max_time_ns: value.max_time_ns.map(|t| t as f64), - ..Default::default() - }, - stats, - } - } -} - -#[derive(Debug, Serialize, Deserialize)] -struct Instrument { - #[serde(rename = "type")] - type_: String, -} - -#[derive(Debug, Serialize, Deserialize)] -struct Creator { - name: String, - version: String, - pid: u32, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct WalltimeResults { - creator: Creator, - instrument: Instrument, - benchmarks: Vec, -} - -impl WalltimeResults { - pub fn from_benchmarks(benchmarks: Vec) -> Self { - WalltimeResults { - instrument: Instrument { - type_: "walltime".to_string(), - }, - creator: Creator { - name: "codspeed-rust".to_string(), - version: env!("CARGO_PKG_VERSION").to_string(), - pid: std::process::id(), - }, - benchmarks, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_single_benchmark() { - let metadata = BenchmarkMetadata { - name: "benchmark".to_string(), - uri: "test::benchmark".to_string(), - }; - let raw_bench = RawWallTimeData { - metadata, - iters_per_round: vec![1], - max_time_ns: None, - times_per_round_ns: vec![42], - }; - - let benchmark: WalltimeBenchmark = raw_bench.into(); - assert_eq!(benchmark.stats.stdev_ns, 0.); - assert_eq!(benchmark.stats.min_ns, 42.); - assert_eq!(benchmark.stats.max_ns, 42.); - assert_eq!(benchmark.stats.mean_ns, 42.); - } - - #[test] - fn test_parse_bench_with_variable_iterations() { - let metadata = BenchmarkMetadata { - name: "benchmark".to_string(), - uri: "test::benchmark".to_string(), - }; - - let raw_bench = RawWallTimeData { - metadata, - iters_per_round: vec![1, 2, 3, 4, 5, 6], - max_time_ns: None, - times_per_round_ns: vec![42, 42 * 2, 42 * 3, 42 * 4, 42 * 5, 42 * 6], - }; - - let total_rounds = raw_bench.iters_per_round.iter().sum::() as f64; - - let benchmark: WalltimeBenchmark = raw_bench.into(); - assert_eq!(benchmark.stats.stdev_ns, 0.); - assert_eq!(benchmark.stats.min_ns, 42.); - assert_eq!(benchmark.stats.max_ns, 42.); - assert_eq!(benchmark.stats.mean_ns, 42.); - assert_eq!( - benchmark.stats.total_time, - 42. * total_rounds / 1_000_000_000.0 - ); - } -} diff --git a/crates/codspeed/Cargo.toml b/crates/codspeed/Cargo.toml index 350270a6..e2d66754 100644 --- a/crates/codspeed/Cargo.toml +++ b/crates/codspeed/Cargo.toml @@ -21,10 +21,12 @@ keywords = ["codspeed", "benchmark"] anyhow = { workspace = true } bincode = "1.3.3" colored = "2.0.0" +glob = "0.3.2" libc = "^0.2" nix = { version = "0.29.0", features = ["fs"] } serde = { workspace = true } serde_json = { workspace = true } +statrs = "0.18.0" uuid = { version = "1.12.1", features = ["v4"] } [[bench]] diff --git a/crates/codspeed/src/lib.rs b/crates/codspeed/src/lib.rs index 87ad3453..ec32f26d 100644 --- a/crates/codspeed/src/lib.rs +++ b/crates/codspeed/src/lib.rs @@ -5,4 +5,4 @@ mod measurement; mod request; mod shared; pub mod utils; -pub mod walltime; +pub mod walltime_results; diff --git a/crates/codspeed/src/walltime.rs b/crates/codspeed/src/walltime.rs deleted file mode 100644 index e51804b9..00000000 --- a/crates/codspeed/src/walltime.rs +++ /dev/null @@ -1,111 +0,0 @@ -use std::{ - io::Write, - path::{Path, PathBuf}, -}; - -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Serialize, Deserialize)] -pub struct BenchmarkMetadata { - pub name: String, - pub uri: String, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct RawWallTimeData { - #[serde(flatten)] - pub metadata: BenchmarkMetadata, - pub iters_per_round: Vec, - pub times_per_round_ns: Vec, - pub max_time_ns: Option, -} - -impl RawWallTimeData { - fn from_runtime_data( - name: String, - uri: String, - iters_per_round: Vec, - times_per_round_ns: Vec, - max_time_ns: Option, - ) -> Self { - RawWallTimeData { - metadata: BenchmarkMetadata { name, uri }, - iters_per_round, - max_time_ns, - times_per_round_ns, - } - } - - fn dump_to_results(&self, workspace_root: &Path, scope: &str) { - let output_dir = get_raw_result_dir_from_workspace_root(workspace_root).join(scope); - std::fs::create_dir_all(&output_dir).unwrap(); - let bench_id = uuid::Uuid::new_v4().to_string(); - let output_path = output_dir.join(format!("{}.json", bench_id)); - let mut writer = std::fs::File::create(&output_path).expect("Failed to create the file"); - serde_json::to_writer_pretty(&mut writer, self).expect("Failed to write the data"); - writer.flush().expect("Failed to flush the writer"); - } -} - -/// Entry point called in patched integration to harvest raw walltime data -/// -/// `CODSPEED_CARGO_WORKSPACE_ROOT` is expected to be set for this to work -/// -/// # Arguments -/// -/// - `scope`: The used integration, e.g. "divan" or "criterion" -/// - `name`: The name of the benchmark -/// - `uri`: The URI of the benchmark -/// - `iters_per_round`: The number of iterations for each round (=sample_size), e.g. `[1, 2, 3]` (variable) or `[2, 2, 2, 2]` (constant). -/// - `times_per_round_ns`: The measured time for each round in nanoseconds, e.g. `[1000, 2000, 3000]` -/// - `max_time_ns`: The time limit for the benchmark in nanoseconds (if defined) -/// -/// # Pseudo-code -/// -/// ```text -/// let sample_count = /* The number of executions for the same benchmark. */ -/// let sample_size = iters_per_round = vec![/* The number of iterations within each sample. */]; -/// for round in 0..sample_count { -/// let times_per_round_ns = 0; -/// for iteration in 0..sample_size[round] { -/// run_benchmark(); -/// times_per_round_ns += /* measured execution time */; -/// } -/// } -/// ``` -/// -pub fn collect_raw_walltime_results( - scope: &str, - name: String, - uri: String, - iters_per_round: Vec, - times_per_round_ns: Vec, - max_time_ns: Option, -) { - if !crate::utils::running_with_codspeed_runner() { - return; - } - let workspace_root = std::env::var("CODSPEED_CARGO_WORKSPACE_ROOT").map(PathBuf::from); - let Ok(workspace_root) = workspace_root else { - eprintln!("codspeed failed to get workspace root. skipping"); - return; - }; - let data = RawWallTimeData::from_runtime_data( - name, - uri, - iters_per_round, - times_per_round_ns, - max_time_ns, - ); - data.dump_to_results(&workspace_root, scope); -} - -// FIXME: This assumes that the cargo target dir is `target`, and duplicates information with -// `cargo-codspeed::helpers::get_codspeed_target_dir` -pub fn get_raw_result_dir_from_workspace_root(workspace_root: &Path) -> PathBuf { - workspace_root - .join("target") - .join("codspeed") - .join("walltime") - .join("raw_results") -} diff --git a/crates/codspeed/src/walltime_results.rs b/crates/codspeed/src/walltime_results.rs new file mode 100644 index 00000000..55fa41fa --- /dev/null +++ b/crates/codspeed/src/walltime_results.rs @@ -0,0 +1,327 @@ +use anyhow::{Context, Result}; +use std::{ + io::Write, + path::{Path, PathBuf}, +}; + +use serde::{Deserialize, Serialize}; +use statrs::statistics::{Data, Distribution, Max, Min, OrderStatistics}; + +const IQR_OUTLIER_FACTOR: f64 = 1.5; +const STDEV_OUTLIER_FACTOR: f64 = 3.0; + +#[derive(Debug, Serialize, Deserialize)] +pub struct BenchmarkMetadata { + pub name: String, + pub uri: String, +} + +#[derive(Debug, Serialize, Deserialize)] +struct BenchmarkStats { + min_ns: f64, + max_ns: f64, + mean_ns: f64, + stdev_ns: f64, + + q1_ns: f64, + median_ns: f64, + q3_ns: f64, + + rounds: u64, + total_time: f64, + iqr_outlier_rounds: u64, + stdev_outlier_rounds: u64, + iter_per_round: u64, + warmup_iters: u64, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +struct BenchmarkConfig { + warmup_time_ns: Option, + min_round_time_ns: Option, + max_time_ns: Option, + max_rounds: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct WalltimeBenchmark { + #[serde(flatten)] + metadata: BenchmarkMetadata, + + config: BenchmarkConfig, + stats: BenchmarkStats, +} + +impl WalltimeBenchmark { + /// Entry point called in patched integration to harvest raw walltime data + /// + /// `CODSPEED_CARGO_WORKSPACE_ROOT` is expected to be set for this to work + /// + /// # Arguments + /// + /// - `scope`: The used integration, e.g. "divan" or "criterion" + /// - `name`: The name of the benchmark + /// - `uri`: The URI of the benchmark + /// - `iters_per_round`: The number of iterations for each round (=sample_size), e.g. `[1, 2, 3]` (variable) or `[2, 2, 2, 2]` (constant). + /// - `times_per_round_ns`: The measured time for each round in nanoseconds, e.g. `[1000, 2000, 3000]` + /// - `max_time_ns`: The time limit for the benchmark in nanoseconds (if defined) + /// + /// # Pseudo-code + /// + /// ```text + /// let sample_count = /* The number of executions for the same benchmark. */ + /// let sample_size = iters_per_round = vec![/* The number of iterations within each sample. */]; + /// for round in 0..sample_count { + /// let times_per_round_ns = 0; + /// for iteration in 0..sample_size[round] { + /// run_benchmark(); + /// times_per_round_ns += /* measured execution time */; + /// } + /// } + /// ``` + /// + pub fn collect_raw_walltime_results( + scope: &str, + name: String, + uri: String, + iters_per_round: Vec, + times_per_round_ns: Vec, + max_time_ns: Option, + ) { + if !crate::utils::running_with_codspeed_runner() { + return; + } + let workspace_root = std::env::var("CODSPEED_CARGO_WORKSPACE_ROOT").map(PathBuf::from); + let Ok(workspace_root) = workspace_root else { + eprintln!("codspeed failed to get workspace root. skipping"); + return; + }; + let data = WalltimeBenchmark::from_runtime_data( + name, + uri, + iters_per_round, + times_per_round_ns, + max_time_ns, + ); + data.dump_to_results(&workspace_root, scope); + } + + pub fn from_runtime_data( + name: String, + uri: String, + iters_per_round: Vec, + times_per_round_ns: Vec, + max_time_ns: Option, + ) -> Self { + let total_time = times_per_round_ns.iter().sum::() as f64 / 1_000_000_000.0; + let time_per_iteration_per_round_ns: Vec<_> = times_per_round_ns + .into_iter() + .zip(&iters_per_round) + .map(|(time_per_round, iter_per_round)| time_per_round / iter_per_round) + .map(|t| t as f64) + .collect::>(); + + let mut data = Data::new(time_per_iteration_per_round_ns); + let rounds = data.len() as u64; + + let mean_ns = data.mean().unwrap(); + + let stdev_ns = if data.len() < 2 { + // std_dev() returns f64::NAN if data has less than two entries, so we have to + // manually handle this case. + 0.0 + } else { + data.std_dev().unwrap() + }; + + let q1_ns = data.quantile(0.25); + let median_ns = data.median(); + let q3_ns = data.quantile(0.75); + + let iqr_ns = q3_ns - q1_ns; + let iqr_outlier_rounds = data + .iter() + .filter(|&&t| { + t < q1_ns - IQR_OUTLIER_FACTOR * iqr_ns || t > q3_ns + IQR_OUTLIER_FACTOR * iqr_ns + }) + .count() as u64; + + let stdev_outlier_rounds = data + .iter() + .filter(|&&t| { + t < mean_ns - STDEV_OUTLIER_FACTOR * stdev_ns + || t > mean_ns + STDEV_OUTLIER_FACTOR * stdev_ns + }) + .count() as u64; + + let min_ns = data.min(); + let max_ns = data.max(); + + // TODO(COD-1056): We currently only support single iteration count per round + let iter_per_round = + (iters_per_round.iter().sum::() / iters_per_round.len() as u128) as u64; + let warmup_iters = 0; // FIXME: add warmup detection + + let stats = BenchmarkStats { + min_ns, + max_ns, + mean_ns, + stdev_ns, + q1_ns, + median_ns, + q3_ns, + rounds, + total_time, + iqr_outlier_rounds, + stdev_outlier_rounds, + iter_per_round, + warmup_iters, + }; + + WalltimeBenchmark { + metadata: BenchmarkMetadata { name, uri }, + config: BenchmarkConfig { + max_time_ns: max_time_ns.map(|t| t as f64), + ..Default::default() + }, + stats, + } + } + + fn dump_to_results(&self, workspace_root: &Path, scope: &str) { + let output_dir = + WalltimeResults::result_dir_from_workspace_root(workspace_root).join(scope); + std::fs::create_dir_all(&output_dir).unwrap(); + let bench_id = uuid::Uuid::new_v4().to_string(); + let output_path = output_dir.join(format!("{}.json", bench_id)); + let mut writer = std::fs::File::create(&output_path).expect("Failed to create the file"); + serde_json::to_writer_pretty(&mut writer, self).expect("Failed to write the data"); + writer.flush().expect("Failed to flush the writer"); + } + + pub fn is_invalid(&self) -> bool { + self.stats.min_ns < f64::EPSILON + } + + pub fn name(&self) -> &str { + &self.metadata.name + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct Instrument { + #[serde(rename = "type")] + type_: String, +} + +#[derive(Debug, Serialize, Deserialize)] +struct Creator { + name: String, + version: String, + pid: u32, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct WalltimeResults { + creator: Creator, + instrument: Instrument, + benchmarks: Vec, +} + +impl WalltimeResults { + pub fn collect_walltime_results(workspace_root: &Path) -> Result { + // retrieve data from `{workspace_root}/target/codspeed/raw_results/{scope}/*.json + let benchmarks = glob::glob(&format!( + "{}/**/*.json", + WalltimeResults::result_dir_from_workspace_root(workspace_root) + .to_str() + .unwrap(), + ))? + .map(|sample| -> Result<_> { + let sample = sample?; + serde_json::from_reader::<_, WalltimeBenchmark>(std::fs::File::open(&sample)?) + .context("Failed to read benchmark data") + }) + .collect::>>()?; + + Ok(WalltimeResults { + instrument: Instrument { + type_: "walltime".to_string(), + }, + creator: Creator { + name: "codspeed-rust".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + pid: std::process::id(), + }, + benchmarks, + }) + } + + pub fn clear(workspace_root: &Path) -> Result<()> { + let raw_results_dir = WalltimeResults::result_dir_from_workspace_root(workspace_root); + std::fs::remove_dir_all(&raw_results_dir).ok(); // ignore errors when the directory does not exist + std::fs::create_dir_all(&raw_results_dir) + .context("Failed to create raw_results directory")?; + Ok(()) + } + + // FIXME: This assumes that the cargo target dir is `target`, and duplicates information with + // `cargo-codspeed::helpers::get_codspeed_target_dir` + fn result_dir_from_workspace_root(workspace_root: &Path) -> PathBuf { + workspace_root + .join("target") + .join("codspeed") + .join("walltime") + .join("raw_results") + } + + pub fn benchmarks(&self) -> &[WalltimeBenchmark] { + &self.benchmarks + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const NAME: &str = "benchmark"; + const URI: &str = "test::benchmark"; + + #[test] + fn test_parse_single_benchmark() { + let benchmark = WalltimeBenchmark::from_runtime_data( + NAME.to_string(), + URI.to_string(), + vec![1], + vec![42], + None, + ); + assert_eq!(benchmark.stats.stdev_ns, 0.); + assert_eq!(benchmark.stats.min_ns, 42.); + assert_eq!(benchmark.stats.max_ns, 42.); + assert_eq!(benchmark.stats.mean_ns, 42.); + } + + #[test] + fn test_parse_bench_with_variable_iterations() { + let iters_per_round = vec![1, 2, 3, 4, 5, 6]; + let total_rounds = iters_per_round.iter().sum::() as f64; + + let benchmark = WalltimeBenchmark::from_runtime_data( + NAME.to_string(), + URI.to_string(), + iters_per_round, + vec![42, 42 * 2, 42 * 3, 42 * 4, 42 * 5, 42 * 6], + None, + ); + + assert_eq!(benchmark.stats.stdev_ns, 0.); + assert_eq!(benchmark.stats.min_ns, 42.); + assert_eq!(benchmark.stats.max_ns, 42.); + assert_eq!(benchmark.stats.mean_ns, 42.); + assert_eq!( + benchmark.stats.total_time, + 42. * total_rounds / 1_000_000_000.0 + ); + } +} diff --git a/crates/criterion_compat/criterion_fork/src/analysis/mod.rs b/crates/criterion_compat/criterion_fork/src/analysis/mod.rs index c8eab967..815dcf27 100644 --- a/crates/criterion_compat/criterion_fork/src/analysis/mod.rs +++ b/crates/criterion_compat/criterion_fork/src/analysis/mod.rs @@ -310,7 +310,7 @@ mod codspeed { let times_per_round_ns = times.iter().map(|t| *t as u128).collect(); let max_time_ns = Some(c.config.measurement_time.as_nanos()); - ::codspeed::walltime::collect_raw_walltime_results( + ::codspeed::walltime_results::WalltimeBenchmark::collect_raw_walltime_results( "criterion", bench_name, uri, diff --git a/crates/divan_compat/divan_fork/src/divan.rs b/crates/divan_compat/divan_fork/src/divan.rs index b6f22871..3953aff5 100644 --- a/crates/divan_compat/divan_fork/src/divan.rs +++ b/crates/divan_compat/divan_fork/src/divan.rs @@ -439,7 +439,7 @@ mod codspeed { } } - ::codspeed::walltime::collect_raw_walltime_results( + ::codspeed::walltime_results::WalltimeBenchmark::collect_raw_walltime_results( "divan", bench_name, uri,