feat(cargo-codspeed): support walltime runs from with cargo-codspeed

Co-authored-by: Adrien Cacciaguerra <[email protected]>

Author: art049

Cargo.lock

@@ -19,11 +19,16 @@ keywords = ["codspeed", "benchmark", "cargo"]
 
 [dependencies]
 cargo_metadata = "0.19.1"
-clap = { version = "=4.5.17", features = ["derive"] }
+clap = { version = "=4.5.17", features = ["derive", "env"] }
 termcolor = "1.4"
 anyhow = "1.0.86"
 itertools = "0.13.0"
 anstyle = "1.0.8"
+serde = { workspace = true }
+serde_json = { workspace = true }
+codspeed = { path = "../codspeed", version = "=2.7.2" }
+glob = "0.3.2"
+statrs = "0.18.0"
 
 [dev-dependencies]
 assert_cmd = "2.0.15"

crates/cargo-codspeed/Cargo.toml

@@ -1,4 +1,4 @@
-use crate::{prelude::*, run::run_benches};
+use crate::{measurement_mode::MeasurementMode, prelude::*, run::run_benches};
 use cargo_metadata::MetadataCommand;
 use clap::{Args, Parser, Subcommand};
 use std::{ffi::OsString, process::exit};
@@ -12,6 +12,13 @@ struct Cli {
     #[arg(short, long, global = true)]
     quiet: bool,
 
+    /// The measurement tool to use for measuring performance.
+    /// Automatically set to `walltime` on macro runners
+    // This is an Option even if MeasurementMode has a default because
+    // the default is dynamic and this would mislead the user
+    #[arg(short, long, global = true, env = "CODSPEED_RUNNER_MODE")]
+    measurement_mode: Option<MeasurementMode>,
+
     #[command(subcommand)]
     command: Commands,
 }
@@ -63,6 +70,9 @@ pub fn run(args: impl Iterator<Item = OsString>) -> Result<()> {
     let metadata = MetadataCommand::new().exec()?;
     let cli = Cli::try_parse_from(args)?;
 
+    let measurement_mode = cli.measurement_mode.unwrap_or_default();
+    eprintln!("[cargo-codspeed] Measurement mode: {measurement_mode:?}\n");
+
     let res = match cli.command {
         Commands::Build {
             filters,
@@ -71,9 +81,16 @@ pub fn run(args: impl Iterator<Item = OsString>) -> Result<()> {
         } => {
             let features =
                 features.map(|f| f.split([' ', ',']).map(|s| s.to_string()).collect_vec());
-            build_benches(&metadata, filters, features, profile, cli.quiet)
+            build_benches(
+                &metadata,
+                filters,
+                features,
+                profile,
+                cli.quiet,
+                measurement_mode,
+            )
         }
-        Commands::Run { filters } => run_benches(&metadata, filters),
+        Commands::Run { filters } => run_benches(&metadata, filters, measurement_mode),
     };
 
     if let Err(e) = res {

crates/cargo-codspeed/src/app.rs

@@ -1,6 +1,7 @@
 use crate::{
     app::{Filters, PackageFilters},
     helpers::{clear_dir, get_codspeed_target_dir},
+    measurement_mode::MeasurementMode,
     prelude::*,
 };
 use cargo_metadata::{camino::Utf8PathBuf, Message, Metadata, TargetKind};
@@ -21,10 +22,15 @@ struct BuiltBench {
 impl BuildOptions<'_> {
     /// Builds the benchmarks by invoking cargo
     /// Returns a list of built benchmarks, with path to associated executables
-    fn build(&self, metadata: &Metadata, quiet: bool) -> Result<Vec<BuiltBench>> {
+    fn build(
+        &self,
+        metadata: &Metadata,
+        quiet: bool,
+        measurement_mode: MeasurementMode,
+    ) -> Result<Vec<BuiltBench>> {
         let workspace_packages = metadata.workspace_packages();
 
-        let mut cargo = self.build_command();
+        let mut cargo = self.build_command(measurement_mode);
         if quiet {
             cargo.arg("--quiet");
         }
@@ -99,18 +105,20 @@ impl BuildOptions<'_> {
     }
 
     /// Generates a subcommand to build the benchmarks by invoking cargo and forwarding the filters
-    /// This command explicitely ignores the `self.benches`: all benches are built
-    fn build_command(&self) -> Command {
+    /// This command explicitly ignores the `self.benches`: all benches are built
+    fn build_command(&self, measurement_mode: MeasurementMode) -> Command {
         let mut cargo = Command::new("cargo");
         cargo.args(["build", "--benches"]);
 
-        cargo.env(
-            "RUSTFLAGS",
-            format!(
-                "{} -g --cfg codspeed",
-                std::env::var("RUSTFLAGS").unwrap_or_else(|_| "".into())
-            ),
-        );
+        let mut rust_flags = std::env::var("RUSTFLAGS").unwrap_or_else(|_| "".into());
+        // Add debug info (equivalent to -g)
+        rust_flags.push_str(" -C debuginfo=2");
+
+        // Add the codspeed cfg flag if instrumentation mode is enabled
+        if measurement_mode == MeasurementMode::Instrumentation {
+            rust_flags.push_str(" --cfg codspeed");
+        }
+        cargo.env("RUSTFLAGS", rust_flags);
 
         if let Some(features) = self.features {
             cargo.arg("--features").arg(features.join(","));
@@ -150,13 +158,14 @@ pub fn build_benches(
     features: Option<Vec<String>>,
     profile: String,
     quiet: bool,
+    measurement_mode: MeasurementMode,
 ) -> Result<()> {
     let built_benches = BuildOptions {
         filters,
         features: &features,
         profile: &profile,
     }
-    .build(metadata, quiet)?;
+    .build(metadata, quiet, measurement_mode)?;
 
     if built_benches.is_empty() {
         bail!(

crates/cargo-codspeed/src/build.rs

@@ -1,8 +1,10 @@
 mod app;
 mod build;
 mod helpers;
+mod measurement_mode;
 mod prelude;
 mod run;
+mod walltime_results;
 
 use crate::prelude::*;
 use std::{env::args_os, process::exit};

crates/cargo-codspeed/src/main.rs

@@ -0,0 +1,20 @@
+use clap::ValueEnum;
+use serde::Serialize;
+use std::env;
+
+#[derive(Debug, Clone, ValueEnum, Serialize, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum MeasurementMode {
+    Walltime,
+    Instrumentation,
+}
+
+impl Default for MeasurementMode {
+    fn default() -> Self {
+        if env::var("CODSPEED_ENV").is_ok() {
+            MeasurementMode::Instrumentation
+        } else {
+            MeasurementMode::Walltime
+        }
+    }
+}

crates/cargo-codspeed/src/measurement_mode.rs

@@ -1,10 +1,18 @@
 use crate::{
     app::{Filters, PackageFilters},
     helpers::get_codspeed_target_dir,
+    measurement_mode::MeasurementMode,
     prelude::*,
+    walltime_results::{WalltimeBenchmark, WalltimeResults},
 };
+use anyhow::Context;
 use cargo_metadata::{Metadata, Package};
-use std::{io, path::PathBuf};
+use codspeed::walltime::get_raw_result_dir_from_workspace_root;
+use glob::glob;
+use std::{
+    io::{self, Write},
+    path::{Path, PathBuf},
+};
 
 struct BenchToRun {
     bench_path: PathBuf,
@@ -86,8 +94,16 @@ impl PackageFilters {
     }
 }
 
-pub fn run_benches(metadata: &Metadata, filters: Filters) -> Result<()> {
+pub fn run_benches(
+    metadata: &Metadata,
+    filters: Filters,
+    measurement_mode: MeasurementMode,
+) -> Result<()> {
     let codspeed_target_dir = get_codspeed_target_dir(metadata);
+    let workspace_root = metadata.workspace_root.as_std_path();
+    if measurement_mode == MeasurementMode::Walltime {
+        clear_raw_walltime_data(workspace_root)?;
+    }
     let benches = filters.benches_to_run(codspeed_target_dir, metadata)?;
     if benches.is_empty() {
         bail!("No benchmarks found. Run `cargo codspeed build` first.");
@@ -125,9 +141,16 @@ pub fn run_benches(metadata: &Metadata, filters: Filters) -> Result<()> {
         // while CARGO_MANIFEST_DIR returns the path to the sub package
         let workspace_root = metadata.workspace_root.clone();
         eprintln!("Running {} {}", &bench.package_name, bench_name);
-        std::process::Command::new(&bench.bench_path)
+        let mut command = std::process::Command::new(&bench.bench_path);
+        command
             .env("CODSPEED_CARGO_WORKSPACE_ROOT", workspace_root)
-            .current_dir(&bench.working_directory)
+            .current_dir(&bench.working_directory);
+
+        if measurement_mode == MeasurementMode::Walltime {
+            command.arg("--bench"); // Walltime targets need this additional argument (inherited from running them with `cargo bench`)
+        }
+
+        command
             .status()
             .map_err(|e| anyhow!("failed to execute the benchmark process: {}", e))
             .and_then(|status| {
@@ -143,5 +166,55 @@ pub fn run_benches(metadata: &Metadata, filters: Filters) -> Result<()> {
         eprintln!("Done running {}", bench_name);
     }
     eprintln!("Finished running {} benchmark suite(s)", to_run.len());
+
+    if measurement_mode == MeasurementMode::Walltime {
+        aggregate_raw_walltime_data(workspace_root)?;
+    }
+
+    Ok(())
+}
+
+fn clear_raw_walltime_data(workspace_root: &Path) -> Result<()> {
+    let raw_results_dir = get_raw_result_dir_from_workspace_root(workspace_root);
+    std::fs::remove_dir_all(&raw_results_dir).ok(); // ignore errors when the directory does not exist
+    std::fs::create_dir_all(&raw_results_dir).context("Failed to create raw_results directory")?;
+    Ok(())
+}
+
+fn aggregate_raw_walltime_data(workspace_root: &Path) -> Result<()> {
+    // retrieve data from `{workspace_root}/target/codspeed/raw_results/{scope}/*.json
+    let walltime_benchmarks = glob(&format!(
+        "{}/**/*.json",
+        get_raw_result_dir_from_workspace_root(workspace_root)
+            .to_str()
+            .unwrap(),
+    ))?
+    .map(|sample| {
+        let sample = sample?;
+        let raw_walltime_data: codspeed::walltime::RawWallTimeData =
+            serde_json::from_reader(std::fs::File::open(&sample)?)?;
+        Ok(WalltimeBenchmark::from(raw_walltime_data))
+    })
+    .collect::<Result<Vec<_>>>()?;
+
+    if walltime_benchmarks.is_empty() {
+        eprintln!("No walltime benchmarks found");
+        return Ok(());
+    }
+
+    let results_folder = std::env::var("CODSPEED_PROFILE_FOLDER")
+        .map(PathBuf::from)
+        .unwrap_or_else(|_| workspace_root.join("target/codspeed/profiles"))
+        .join("results");
+    std::fs::create_dir_all(&results_folder).context("Failed to create results folder")?;
+
+    let results = WalltimeResults::from_benchmarks(walltime_benchmarks);
+    let results_path = results_folder.join(format!("{}.json", std::process::id()));
+    let mut results_file =
+        std::fs::File::create(&results_path).context("Failed to create results file")?;
+    serde_json::to_writer_pretty(&results_file, &results)?;
+    results_file
+        .flush()
+        .context("Failed to flush results file")?;
     Ok(())
 }