fix(cargo-codspeed): properly forward stdout compiler messages

Author: art049

crates/cargo-codspeed/src/build.rs

@@ -43,10 +43,16 @@ impl BuildOptions<'_> {
 
         let mut built_benches = Vec::new();
         for message in Message::parse_stream(reader) {
-            if let Message::CompilerArtifact(artifact) =
-                message.expect("Failed to parse compiler message")
-            {
-                if artifact.target.is_kind(TargetKind::Bench) {
+            match message.expect("Failed to parse message") {
+                Message::CompilerMessage(msg) => {
+                    println!("{}", &msg.message);
+                }
+                Message::TextLine(line) => {
+                    println!("{}", line);
+                }
+                Message::CompilerArtifact(artifact)
+                    if artifact.target.is_kind(TargetKind::Bench) =>
+                {
                     let package = workspace_packages
                         .iter()
                         .find(|p| p.id == artifact.package_id)
@@ -71,6 +77,7 @@ impl BuildOptions<'_> {
                         });
                     }
                 }
+                _ => {}
             }
         }
 

crates/cargo-codspeed/src/walltime_results.rs

@@ -0,0 +1,144 @@
+use codspeed::walltime::{BenchmarkMetadata, RawWallTimeData};
+use serde::{Deserialize, Serialize};
+use statrs::statistics::{Data, Distribution, Max, Min, OrderStatistics};
+
+const IQR_OUTLIER_FACTOR: f64 = 1.5;
+const STDEV_OUTLIER_FACTOR: f64 = 3.0;
+
+#[derive(Debug, Serialize, Deserialize)]
+struct BenchmarkStats {
+    min_ns: f64,
+    max_ns: f64,
+    mean_ns: f64,
+    stdev_ns: f64,
+
+    q1_ns: f64,
+    median_ns: f64,
+    q3_ns: f64,
+
+    rounds: u64,
+    total_time: f64,
+    iqr_outlier_rounds: u64,
+    stdev_outlier_rounds: u64,
+    iter_per_round: u64,
+    warmup_iters: u64,
+}
+
+#[derive(Debug, Serialize, Deserialize, Default)]
+struct BenchmarkConfig {
+    warmup_time_ns: Option<f64>,
+    min_round_time_ns: Option<f64>,
+    max_time_ns: Option<f64>,
+    max_rounds: Option<u64>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct WalltimeBenchmark {
+    #[serde(flatten)]
+    metadata: BenchmarkMetadata,
+
+    config: BenchmarkConfig,
+    stats: BenchmarkStats,
+}
+
+impl From<RawWallTimeData> for WalltimeBenchmark {
+    fn from(value: RawWallTimeData) -> Self {
+        let times_ns: Vec<f64> = value.times_ns.iter().map(|&t| t as f64).collect();
+        let mut data = Data::new(times_ns.clone());
+        let rounds = data.len() as u64;
+        let total_time = times_ns.iter().sum::<f64>() / 1_000_000_000.0;
+
+        let mean_ns = data.mean().unwrap();
+        let stdev_ns = data.std_dev().unwrap();
+
+        let q1_ns = data.quantile(0.25);
+        let median_ns = data.median();
+        let q3_ns = data.quantile(0.75);
+
+        let iqr_ns = q3_ns - q1_ns;
+        let iqr_outlier_rounds = data
+            .iter()
+            .filter(|&&t| {
+                t < q1_ns - IQR_OUTLIER_FACTOR * iqr_ns || t > q3_ns + IQR_OUTLIER_FACTOR * iqr_ns
+            })
+            .count() as u64;
+
+        let stdev_outlier_rounds = data
+            .iter()
+            .filter(|&&t| {
+                t < mean_ns - STDEV_OUTLIER_FACTOR * stdev_ns
+                    || t > mean_ns + STDEV_OUTLIER_FACTOR * stdev_ns
+            })
+            .count() as u64;
+
+        let min_ns = data.min();
+        let max_ns = data.max();
+
+        let iter_per_round = value.iter_per_round as u64;
+        let warmup_iters = 0; // FIXME: add warmup detection
+
+        let stats = BenchmarkStats {
+            min_ns,
+            max_ns,
+            mean_ns,
+            stdev_ns,
+            q1_ns,
+            median_ns,
+            q3_ns,
+            rounds,
+            total_time,
+            iqr_outlier_rounds,
+            stdev_outlier_rounds,
+            iter_per_round,
+            warmup_iters,
+        };
+
+        WalltimeBenchmark {
+            metadata: BenchmarkMetadata {
+                name: value.metadata.name,
+                uri: value.metadata.uri,
+            },
+            config: BenchmarkConfig {
+                max_time_ns: value.max_time_ns.map(|t| t as f64),
+                ..Default::default()
+            },
+            stats,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Instrument {
+    #[serde(rename = "type")]
+    type_: String,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Creator {
+    name: String,
+    version: String,
+    pid: u32,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct WalltimeResults {
+    creator: Creator,
+    instrument: Instrument,
+    benchmarks: Vec<WalltimeBenchmark>,
+}
+
+impl WalltimeResults {
+    pub fn from_benchmarks(benchmarks: Vec<WalltimeBenchmark>) -> Self {
+        WalltimeResults {
+            instrument: Instrument {
+                type_: "walltime".to_string(),
+            },
+            creator: Creator {
+                name: "codspeed-rust".to_string(),
+                version: env!("CARGO_PKG_VERSION").to_string(),
+                pid: std::process::id(),
+            },
+            benchmarks,
+        }
+    }
+}