feat: parse and convert raw results into walltime results

Author: not-matthias

go-runner/src/results/mod.rs

@@ -0,0 +1,2 @@
+pub mod raw_result;
+pub mod walltime_results;

go-runner/src/results/raw_result.rs

@@ -0,0 +1,95 @@
+use std::path::Path;
+
+use serde::{Deserialize, Serialize};
+
+use crate::results::walltime_results::WalltimeBenchmark;
+
+// WARN: Keep in sync with Golang "testing" fork (benchmark.go)
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RawResult {
+    pub benchmark_name: String,
+    pub pid: u32,
+    pub codspeed_time_per_round_ns: Vec<u64>,
+
+    #[serde(default)]
+    pub codspeed_iters_per_round: Vec<u64>,
+}
+
+impl RawResult {
+    pub fn parse(content: &str) -> anyhow::Result<Self> {
+        serde_json::from_str(content)
+            .map_err(|e| anyhow::anyhow!("Failed to parse raw result: {}", e))
+    }
+
+    pub fn parse_folder<P: AsRef<Path>>(folder: P) -> anyhow::Result<Vec<Self>> {
+        let glob_pattern = folder.as_ref().join("raw_results").join("*.json");
+        Ok(glob::glob(&glob_pattern.to_string_lossy())?
+            .filter_map(Result::ok)
+            .filter_map(|path| {
+                let content = std::fs::read_to_string(&path).ok()?;
+                Self::parse(&content).ok()
+            })
+            .collect())
+    }
+
+    pub fn into_walltime_benchmark(self, file_path: Option<String>) -> WalltimeBenchmark {
+        let name = self.benchmark_name;
+
+        let file = file_path.as_deref().unwrap_or("unknown");
+        let uri = format!("{file}::{name}");
+
+        let times_per_round_ns = self
+            .codspeed_time_per_round_ns
+            .iter()
+            .map(|t| *t as u128)
+            .collect::<Vec<_>>();
+        let iters_per_round = if self.codspeed_iters_per_round.is_empty() {
+            vec![1; times_per_round_ns.len()]
+        } else {
+            self.codspeed_iters_per_round
+                .iter()
+                .map(|i| *i as u128)
+                .collect()
+        };
+
+        WalltimeBenchmark::from_runtime_data(name, uri, iters_per_round, times_per_round_ns, None)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_raw_result_deserialization() {
+        let json_data = r#"{
+    "benchmark_name": "BenchmarkFibonacci20-16",
+    "pid": 777767,
+    "codspeed_time_per_round_ns": [1000, 2000, 3000]
+}"#;
+        let result: RawResult = serde_json::from_str(json_data).unwrap();
+
+        assert_eq!(result.benchmark_name, "BenchmarkFibonacci20-16");
+        assert_eq!(result.pid, 777767);
+        assert_eq!(result.codspeed_time_per_round_ns.len(), 3);
+        assert_eq!(result.codspeed_iters_per_round.len(), 0); // Default: 1 per round
+    }
+
+    #[test]
+    fn test_into_walltime_benchmark_with_file_path() {
+        let raw_result = RawResult {
+            benchmark_name: "BenchmarkFibonacci20-16".to_string(),
+            pid: 777767,
+            codspeed_time_per_round_ns: vec![1000, 2000, 3000],
+            codspeed_iters_per_round: vec![],
+        };
+
+        // Test with file path - should not panic and create successfully
+        let _walltime_bench = raw_result
+            .clone()
+            .into_walltime_benchmark(Some("pkg/foo/fib_test.go".to_string()));
+
+        // Test without file path (should default to TODO) - should not panic and create successfully
+        let _walltime_bench_no_path = raw_result.into_walltime_benchmark(None);
+    }
+}

go-runner/src/results/walltime_results.rs

@@ -0,0 +1,213 @@
+// NOTE: This file was taken from `codspeed-rust` and modified a bit to fit this project.
+
+use anyhow::Result;
+
+use serde::{Deserialize, Serialize};
+use statrs::statistics::{Data, Distribution, Max, Min, OrderStatistics};
+
+const IQR_OUTLIER_FACTOR: f64 = 1.5;
+const STDEV_OUTLIER_FACTOR: f64 = 3.0;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct BenchmarkMetadata {
+    pub name: String,
+    pub uri: String,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct BenchmarkStats {
+    min_ns: f64,
+    max_ns: f64,
+    mean_ns: f64,
+    stdev_ns: f64,
+
+    q1_ns: f64,
+    median_ns: f64,
+    q3_ns: f64,
+
+    rounds: u64,
+    total_time: f64,
+    iqr_outlier_rounds: u64,
+    stdev_outlier_rounds: u64,
+    iter_per_round: u64,
+    warmup_iters: u64,
+}
+
+#[derive(Debug, Serialize, Deserialize, Default)]
+struct BenchmarkConfig {
+    warmup_time_ns: Option<f64>,
+    min_round_time_ns: Option<f64>,
+    max_time_ns: Option<f64>,
+    max_rounds: Option<u64>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct WalltimeBenchmark {
+    #[serde(flatten)]
+    pub metadata: BenchmarkMetadata,
+
+    config: BenchmarkConfig,
+    stats: BenchmarkStats,
+}
+
+impl WalltimeBenchmark {
+    pub fn from_runtime_data(
+        name: String,
+        uri: String,
+        iters_per_round: Vec<u128>,
+        times_per_round_ns: Vec<u128>,
+        max_time_ns: Option<u128>,
+    ) -> Self {
+        let total_time = times_per_round_ns.iter().sum::<u128>() as f64 / 1_000_000_000.0;
+        let time_per_iteration_per_round_ns: Vec<_> = times_per_round_ns
+            .into_iter()
+            .zip(&iters_per_round)
+            .map(|(time_per_round, iter_per_round)| time_per_round / iter_per_round)
+            .map(|t| t as f64)
+            .collect::<Vec<f64>>();
+
+        let mut data = Data::new(time_per_iteration_per_round_ns);
+        let rounds = data.len() as u64;
+
+        let mean_ns = data.mean().unwrap();
+
+        let stdev_ns = if data.len() < 2 {
+            // std_dev() returns f64::NAN if data has less than two entries, so we have to
+            // manually handle this case.
+            0.0
+        } else {
+            data.std_dev().unwrap()
+        };
+
+        let q1_ns = data.quantile(0.25);
+        let median_ns = data.median();
+        let q3_ns = data.quantile(0.75);
+
+        let iqr_ns = q3_ns - q1_ns;
+        let iqr_outlier_rounds = data
+            .iter()
+            .filter(|&&t| {
+                t < q1_ns - IQR_OUTLIER_FACTOR * iqr_ns || t > q3_ns + IQR_OUTLIER_FACTOR * iqr_ns
+            })
+            .count() as u64;
+
+        let stdev_outlier_rounds = data
+            .iter()
+            .filter(|&&t| {
+                t < mean_ns - STDEV_OUTLIER_FACTOR * stdev_ns
+                    || t > mean_ns + STDEV_OUTLIER_FACTOR * stdev_ns
+            })
+            .count() as u64;
+
+        let min_ns = data.min();
+        let max_ns = data.max();
+
+        // TODO(COD-1056): We currently only support single iteration count per round
+        let iter_per_round =
+            (iters_per_round.iter().sum::<u128>() / iters_per_round.len() as u128) as u64;
+        let warmup_iters = 0; // FIXME: add warmup detection
+
+        let stats = BenchmarkStats {
+            min_ns,
+            max_ns,
+            mean_ns,
+            stdev_ns,
+            q1_ns,
+            median_ns,
+            q3_ns,
+            rounds,
+            total_time,
+            iqr_outlier_rounds,
+            stdev_outlier_rounds,
+            iter_per_round,
+            warmup_iters,
+        };
+
+        WalltimeBenchmark {
+            metadata: BenchmarkMetadata { name, uri },
+            config: BenchmarkConfig {
+                max_time_ns: max_time_ns.map(|t| t as f64),
+                ..Default::default()
+            },
+            stats,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Instrument {
+    #[serde(rename = "type")]
+    type_: String,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Creator {
+    pub name: String,
+    pub version: String,
+    pub pid: u32,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct WalltimeResults {
+    creator: Creator,
+    instrument: Instrument,
+    pub benchmarks: Vec<WalltimeBenchmark>,
+}
+
+impl WalltimeResults {
+    pub fn new(benchmarks: Vec<WalltimeBenchmark>, creator: Creator) -> Result<Self> {
+        Ok(WalltimeResults {
+            instrument: Instrument {
+                type_: "walltime".to_string(),
+            },
+            creator,
+            benchmarks,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const NAME: &str = "benchmark";
+    const URI: &str = "test::benchmark";
+
+    #[test]
+    fn test_parse_single_benchmark() {
+        let benchmark = WalltimeBenchmark::from_runtime_data(
+            NAME.to_string(),
+            URI.to_string(),
+            vec![1],
+            vec![42],
+            None,
+        );
+        assert_eq!(benchmark.stats.stdev_ns, 0.);
+        assert_eq!(benchmark.stats.min_ns, 42.);
+        assert_eq!(benchmark.stats.max_ns, 42.);
+        assert_eq!(benchmark.stats.mean_ns, 42.);
+    }
+
+    #[test]
+    fn test_parse_bench_with_variable_iterations() {
+        let iters_per_round = vec![1, 2, 3, 4, 5, 6];
+        let total_rounds = iters_per_round.iter().sum::<u128>() as f64;
+
+        let benchmark = WalltimeBenchmark::from_runtime_data(
+            NAME.to_string(),
+            URI.to_string(),
+            iters_per_round,
+            vec![42, 42 * 2, 42 * 3, 42 * 4, 42 * 5, 42 * 6],
+            None,
+        );
+
+        assert_eq!(benchmark.stats.stdev_ns, 0.);
+        assert_eq!(benchmark.stats.min_ns, 42.);
+        assert_eq!(benchmark.stats.max_ns, 42.);
+        assert_eq!(benchmark.stats.mean_ns, 42.);
+        assert_eq!(
+            benchmark.stats.total_time,
+            42. * total_rounds / 1_000_000_000.0
+        );
+    }
+}