feat!(cargo-codspeed): allow benchmark filtering by name in cargo codspeed run

GuillaumeLagrange · GuillaumeLagrange · commit e0fe0305b82d · 2025-09-25T16:52:36.000+02:00
There previously was a confusion between benchmark name, aka the name
the user gives to individual benchmarks, and benchmark targets.

Now, benchmark target filtering is only done through `--workspace`,
`--package` and `--exclude` cli flags for both `cargo codspeed build`
and `cargo codspeed run`.

In addition, `cargo codspeed run` now properly forwards the `benchname`
positional argument to the benchmark target, in order to filter
benchmarks ran. This is a breaking change, but it makes `cargo-codspeed`
more in line with `cargo bench`
diff --git a/crates/cargo-codspeed/src/app.rs b/crates/cargo-codspeed/src/app.rs
@@ -21,58 +21,55 @@ struct Cli {
     command: Commands,
 }
 
+const PACKAGE_HELP: &str = "Package Selection";
 #[derive(Args)]
 pub(crate) struct PackageFilters {
     /// Select all packages in the workspace
-    #[arg(long)]
+    #[arg(long, help_heading = PACKAGE_HELP)]
     pub(crate) workspace: bool,
-    /// Exclude packages
-    #[arg(long)]
+    /// Package to exclude
+    #[arg(long, value_name = "SPEC", help_heading = PACKAGE_HELP)]
     pub(crate) exclude: Vec<String>,
-    /// Package to select (builds all workspace package by default)
-    #[arg(short, long)]
+    /// Package to select
+    #[arg(short, long, value_name= "SPEC", help_heading = PACKAGE_HELP)]
     pub(crate) package: Vec<String>,
 }
 
-#[derive(Args)]
-pub(crate) struct Filters {
-    /// Optional list of benchmarks to build (builds all benchmarks by default)
-    pub(crate) bench: Option<Vec<String>>,
-    #[command(flatten)]
-    pub(crate) package: PackageFilters,
-}
-
+const FEATURE_HELP: &str = "Feature Selection";
+const COMPILATION_HELP: &str = "Compilation Options";
 #[derive(Subcommand)]
 enum Commands {
     /// Build the benchmarks
     Build {
         #[command(flatten)]
-        filters: Filters,
+        package_filters: PackageFilters,
 
         /// Space or comma separated list of features to activate
-        #[arg(short = 'F', long)]
+        #[arg(short = 'F', long, help_heading = FEATURE_HELP)]
         features: Option<String>,
 
         /// Activate all available features of all selected packages.
-        #[arg(long)]
+        #[arg(long, help_heading = FEATURE_HELP)]
         all_features: bool,
 
         /// Do not activate the `default` feature of the selected packages.
-        #[arg(long)]
+        #[arg(long, help_heading = FEATURE_HELP)]
         no_default_features: bool,
 
         /// Number of parallel jobs, defaults to # of CPUs.
-        #[arg(short, long)]
+        #[arg(short, long, help_heading = COMPILATION_HELP)]
         jobs: Option<u32>,
 
         /// Build the benchmarks with the specified profile
-        #[arg(long, default_value = "bench")]
+        #[arg(long, default_value = "bench", help_heading = COMPILATION_HELP)]
         profile: String,
     },
     /// Run the previously built benchmarks
     Run {
+        /// If specified, only run benches containing this string in their names
+        bench_name: Option<String>,
         #[command(flatten)]
-        filters: Filters,
+        package_filters: PackageFilters,
     },
 }
 
@@ -85,7 +82,7 @@ pub fn run(args: impl Iterator<Item = OsString>) -> Result<()> {
 
     let res = match cli.command {
         Commands::Build {
-            filters,
+            package_filters,
             features,
             all_features,
             jobs,
@@ -113,15 +110,18 @@ pub fn run(args: impl Iterator<Item = OsString>) -> Result<()> {
                 features.map(|f| f.split([' ', ',']).map(|s| s.to_string()).collect_vec());
             build_benches(
                 &metadata,
-                filters,
+                package_filters,
                 features,
                 profile,
                 cli.quiet,
                 measurement_mode,
                 passthrough_flags,
             )
         }
-        Commands::Run { filters } => run_benches(&metadata, filters, measurement_mode),
+        Commands::Run {
+            bench_name,
+            package_filters,
+        } => run_benches(&metadata, bench_name, package_filters, measurement_mode),
     };
 
     if let Err(e) = res {
diff --git a/crates/cargo-codspeed/src/build.rs b/crates/cargo-codspeed/src/build.rs
@@ -1,5 +1,5 @@
 use crate::{
-    app::{Filters, PackageFilters},
+    app::PackageFilters,
     helpers::{clear_dir, get_codspeed_target_dir},
     measurement_mode::MeasurementMode,
     prelude::*,
@@ -8,7 +8,7 @@ use cargo_metadata::{camino::Utf8PathBuf, Message, Metadata, TargetKind};
 use std::process::{exit, Command, Stdio};
 
 struct BuildOptions<'a> {
-    filters: Filters,
+    package_filters: PackageFilters,
     features: &'a Option<Vec<String>>,
     profile: &'a str,
     passthrough_flags: &'a Vec<String>,
@@ -49,6 +49,19 @@ impl BuildOptions<'_> {
         );
 
         let mut built_benches = Vec::new();
+
+        let package_names = self
+            .package_filters
+            .packages_from_flags(metadata)
+            .map_err(|e| {
+                // Avoid leaving an orphan cargo process, even if something went wrong
+                cargo.wait().expect("Could not get cargo's exist status");
+                e
+            })?
+            .into_iter()
+            .map(|t| t.name.clone())
+            .collect::<Vec<_>>();
+
         for message in Message::parse_stream(reader) {
             match message.expect("Failed to parse message") {
                 // Those messages will include build errors and warnings even if stderr also contain some of them
@@ -66,19 +79,14 @@ impl BuildOptions<'_> {
                         .find(|p| p.id == artifact.package_id)
                         .expect("Could not find package");
 
-                    let bench_name = artifact.target.name;
+                    let bench_target_name = artifact.target.name;
 
-                    let add_bench_to_codspeed_dir = match &self.filters.bench {
-                        Some(allowed_bench_names) => allowed_bench_names
-                            .iter()
-                            .any(|allowed_bench_name| bench_name.contains(allowed_bench_name)),
-                        None => true,
-                    };
+                    let add_bench_to_codspeed_dir = package_names.iter().contains(&package.name);
 
                     if add_bench_to_codspeed_dir {
                         built_benches.push(BuiltBench {
                             package: package.name.clone(),
-                            bench: bench_name,
+                            bench: bench_target_name,
                             executable_path: artifact
                                 .executable
                                 .expect("Unexpected missing executable path"),
@@ -171,7 +179,7 @@ impl BuildOptions<'_> {
 
         cargo.arg("--profile").arg(self.profile);
 
-        self.filters.package.add_cargo_args(&mut cargo);
+        self.package_filters.add_cargo_args(&mut cargo);
 
         cargo
     }
@@ -199,15 +207,15 @@ impl PackageFilters {
 
 pub fn build_benches(
     metadata: &Metadata,
-    filters: Filters,
+    package_filters: PackageFilters,
     features: Option<Vec<String>>,
     profile: String,
     quiet: bool,
     measurement_mode: MeasurementMode,
     passthrough_flags: Vec<String>,
 ) -> Result<()> {
     let built_benches = BuildOptions {
-        filters,
+        package_filters,
         features: &features,
         profile: &profile,
         passthrough_flags: &passthrough_flags,
diff --git a/crates/cargo-codspeed/src/run.rs b/crates/cargo-codspeed/src/run.rs
@@ -1,7 +1,5 @@
 use crate::{
-    app::{Filters, PackageFilters},
-    helpers::get_codspeed_target_dir,
-    measurement_mode::MeasurementMode,
+    app::PackageFilters, helpers::get_codspeed_target_dir, measurement_mode::MeasurementMode,
     prelude::*,
 };
 use anyhow::Context;
@@ -22,13 +20,37 @@ struct BenchToRun {
     package_name: String,
 }
 
-impl Filters {
+impl PackageFilters {
+    /// Logic taken from [cargo::ops::Packages](https://docs.rs/cargo/0.85.0/src/cargo/ops/cargo_compile/packages.rs.html#34-42)
+    pub(crate) fn packages_from_flags<'a>(
+        &self,
+        metadata: &'a Metadata,
+    ) -> Result<Vec<&'a Package>> {
+        Ok(
+            match (self.workspace, self.exclude.len(), self.package.len()) {
+                (false, 0, 0) => metadata.workspace_default_packages(),
+                (false, 0, _) => metadata
+                    .workspace_packages()
+                    .into_iter()
+                    .filter(|p| self.package.contains(&p.name))
+                    .collect(),
+                (false, _, _) => bail!("--exclude can only be used with --workspace"),
+                (true, 0, _) => metadata.workspace_packages(),
+                (true, _, _) => metadata
+                    .workspace_packages()
+                    .into_iter()
+                    .filter(|p| !self.exclude.contains(&p.name))
+                    .collect(),
+            },
+        )
+    }
+
     fn benches_to_run(
         &self,
         codspeed_target_dir: PathBuf,
         metadata: &Metadata,
     ) -> Result<Vec<BenchToRun>> {
-        let packages = self.package.packages(metadata)?;
+        let packages = self.packages_from_flags(metadata)?;
 
         let mut benches = vec![];
         for package in packages {
@@ -63,80 +85,25 @@ impl Filters {
     }
 }
 
-impl PackageFilters {
-    /// Logic taken from [cargo::ops::Packages](https://docs.rs/cargo/0.85.0/src/cargo/ops/cargo_compile/packages.rs.html#34-42)
-    fn packages<'a>(&self, metadata: &'a Metadata) -> Result<Vec<&'a Package>> {
-        Ok(
-            match (self.workspace, self.exclude.len(), self.package.len()) {
-                (false, 0, 0) => metadata.workspace_default_packages(),
-                (false, 0, _) => metadata
-                    .workspace_packages()
-                    .into_iter()
-                    .filter(|p| {
-                        self.package
-                            .iter()
-                            .any(|allowed_package| p.name.contains(allowed_package))
-                    })
-                    .collect(),
-                (false, _, _) => bail!("--exclude can only be used with --workspace"),
-                (true, 0, _) => metadata.workspace_packages(),
-                (true, _, _) => metadata
-                    .workspace_packages()
-                    .into_iter()
-                    .filter(|p| {
-                        !self
-                            .exclude
-                            .iter()
-                            .any(|denied_package| p.name.contains(denied_package))
-                    })
-                    .collect(),
-            },
-        )
-    }
-}
-
 pub fn run_benches(
     metadata: &Metadata,
-    filters: Filters,
+    bench_name_filter: Option<String>,
+    package_filters: PackageFilters,
     measurement_mode: MeasurementMode,
 ) -> Result<()> {
     let codspeed_target_dir = get_codspeed_target_dir(metadata, measurement_mode);
     let workspace_root = metadata.workspace_root.as_std_path();
     if measurement_mode == MeasurementMode::Walltime {
         WalltimeResults::clear(workspace_root)?;
     }
-    let benches = filters.benches_to_run(codspeed_target_dir, metadata)?;
+    let benches = package_filters.benches_to_run(codspeed_target_dir, metadata)?;
     if benches.is_empty() {
         bail!("No benchmarks found. Run `cargo codspeed build` first.");
     }
 
-    let mut to_run = vec![];
-    if let Some(allowed_bench_names) = filters.bench {
-        // Make sure all benchmarks are found
-        let mut not_found = vec![];
-        for allowed_bench_name in allowed_bench_names.iter() {
-            let bench = benches
-                .iter()
-                .find(|b| b.bench_name.contains(allowed_bench_name));
-
-            if let Some(bench) = bench {
-                to_run.push(bench);
-            } else {
-                not_found.push(allowed_bench_name);
-            }
-        }
+    eprintln!("Collected {} benchmark suite(s) to run", benches.len());
 
-        if !not_found.is_empty() {
-            bail!(
-                "The following benchmarks to run were not found: {}",
-                not_found.iter().join(", ")
-            );
-        }
-    } else {
-        to_run = benches.iter().collect();
-    }
-    eprintln!("Collected {} benchmark suite(s) to run", to_run.len());
-    for bench in to_run.iter() {
+    for bench in benches.iter() {
         let bench_name = &bench.bench_name;
         // workspace_root is needed since file! returns the path relatively to the workspace root
         // while CARGO_MANIFEST_DIR returns the path to the sub package
@@ -151,6 +118,10 @@ pub fn run_benches(
             command.arg("--bench"); // Walltime targets need this additional argument (inherited from running them with `cargo bench`)
         }
 
+        if let Some(bench_name_filter) = bench_name_filter.as_ref() {
+            command.arg(bench_name_filter);
+        }
+
         command
             .status()
             .map_err(|e| anyhow!("failed to execute the benchmark process: {}", e))
@@ -177,7 +148,7 @@ pub fn run_benches(
             })?;
         eprintln!("Done running {bench_name}");
     }
-    eprintln!("Finished running {} benchmark suite(s)", to_run.len());
+    eprintln!("Finished running {} benchmark suite(s)", benches.len());
 
     if measurement_mode == MeasurementMode::Walltime {
         aggregate_raw_walltime_data(workspace_root)?;
