Add --exact-match CLI argument for exact matching benchmark names

To avoid problems with prefixes/suffixes in programmatic usage of the collector.

Author: Kobzol

collector/README.md

@@ -135,6 +135,9 @@ The following options alter the behaviour of the `bench_local` subcommand.
   dedicated to artifact sizes (ending with `-tiny`).
 - `--id <ID>` the identifier that will be used to identify the results in the
   database.
+- `--exact-match <BENCHMARKS>`: comma-separated list of benchmark names that should be
+  executed. The names have to match exactly. Cannot be combined with
+  `--include`/`--exclude`/`--exclude-suffix`.
 - `--include <INCLUDE>`: the inverse of `--exclude`. The argument is a
   comma-separated list of benchmark prefixes. When this option is specified, a
   benchmark is included in the run only if its name matches one of the given

collector/src/bin/collector.rs

@@ -40,6 +40,7 @@ use collector::compile::benchmark::scenario::Scenario;
 use collector::compile::benchmark::target::Target;
 use collector::compile::benchmark::{
     compile_benchmark_dir, get_compile_benchmarks, ArtifactType, Benchmark, BenchmarkName,
+    CompileBenchmarkFilter,
 };
 use collector::compile::execute::bencher::BenchProcessor;
 use collector::compile::execute::profiler::{ProfileProcessor, Profiler};
@@ -337,6 +338,16 @@ struct LocalOptions {
     #[arg(long, value_delimiter = ',')]
     include: Vec<String>,
 
+    /// Include only benchmarks in this comma-separated list
+    #[arg(
+        long,
+        value_delimiter = ',',
+        conflicts_with("include"),
+        conflicts_with("exclude"),
+        conflicts_with("exclude_suffix")
+    )]
+    exact_match: Vec<String>,
+
     /// Include only benchmarks belonging to the given categories.
     #[arg(long, value_parser = EnumArgParser::<Category>::default(), default_value = "Primary,Secondary")]
     category: MultiEnumValue<Category>,
@@ -688,6 +699,25 @@ enum DownloadSubcommand {
     },
 }
 
+impl<'a> From<&'a LocalOptions> for CompileBenchmarkFilter<'a> {
+    fn from(value: &'a LocalOptions) -> Self {
+        if !value.exact_match.is_empty() {
+            Self::Exact(&value.exact_match)
+        } else if !value.include.is_empty()
+            || !value.exclude.is_empty()
+            || !value.exclude_suffix.is_empty()
+        {
+            Self::Fuzzy {
+                include: &value.include,
+                exclude: &value.exclude,
+                exclude_suffix: &value.exclude_suffix,
+            }
+        } else {
+            Self::All
+        }
+    }
+}
+
 fn main_result() -> anyhow::Result<i32> {
     env_logger::init();
 
@@ -884,12 +914,7 @@ fn main_result() -> anyhow::Result<i32> {
                 target_triple,
             )?;
 
-            let mut benchmarks = get_compile_benchmarks(
-                &compile_benchmark_dir,
-                &local.include,
-                &local.exclude,
-                &local.exclude_suffix,
-            )?;
+            let mut benchmarks = get_compile_benchmarks(&compile_benchmark_dir, (&local).into())?;
             benchmarks.retain(|b| local.category.0.contains(&b.category()));
 
             let artifact_id = ArtifactId::Commit(Commit {
@@ -1006,9 +1031,11 @@ fn main_result() -> anyhow::Result<i32> {
 
                         let mut benchmarks = get_compile_benchmarks(
                             &compile_benchmark_dir,
-                            &split_args(include),
-                            &split_args(exclude),
-                            &[],
+                            CompileBenchmarkFilter::Fuzzy {
+                                include: &split_args(include),
+                                exclude: &split_args(exclude),
+                                exclude_suffix: &[],
+                            },
                         )?;
                         benchmarks.retain(|b| b.category().is_primary_or_secondary());
 
@@ -1102,12 +1129,7 @@ fn main_result() -> anyhow::Result<i32> {
             let scenarios = &opts.scenarios.0;
             let backends = &opts.codegen_backends.0;
 
-            let mut benchmarks = get_compile_benchmarks(
-                &compile_benchmark_dir,
-                &local.include,
-                &local.exclude,
-                &local.exclude_suffix,
-            )?;
+            let mut benchmarks = get_compile_benchmarks(&compile_benchmark_dir, (&local).into())?;
             benchmarks.retain(|b| local.category.0.contains(&b.category()));
 
             let mut errors = BenchmarkErrors::new();
@@ -1315,12 +1337,7 @@ fn binary_stats_compile(
         Profile::Opt => CargoProfile::Release,
         _ => return Err(anyhow::anyhow!("Only Debug and Opt profiles are supported")),
     };
-    let benchmarks = get_compile_benchmarks(
-        &compile_benchmark_dir(),
-        &local.include,
-        &local.exclude,
-        &local.exclude_suffix,
-    )?;
+    let benchmarks = get_compile_benchmarks(&compile_benchmark_dir(), (&local).into())?;
     for benchmark in benchmarks {
         println!("Stats for benchmark `{}`", benchmark.name);
         println!("{}", "-".repeat(20));
@@ -1713,7 +1730,7 @@ fn bench_published_artifact(
     };
 
     // Exclude benchmarks that don't work with a stable compiler.
-    let mut compile_benchmarks = get_compile_benchmarks(dirs.compile, &[], &[], &[])?;
+    let mut compile_benchmarks = get_compile_benchmarks(dirs.compile, CompileBenchmarkFilter::All)?;
     compile_benchmarks.retain(|b| b.category().is_stable());
 
     let runtime_suite = rt.block_on(load_runtime_benchmarks(

collector/src/compile/benchmark/mod.rs

@@ -466,14 +466,22 @@ pub fn compile_benchmark_dir() -> PathBuf {
     PathBuf::from("collector/compile-benchmarks")
 }
 
+pub enum CompileBenchmarkFilter<'a> {
+    All,
+    /// Select benchmarks exactly matching the given benchmark names.
+    Exact(&'a [String]),
+    /// Select benchmarks matching the given prefixes/suffixes.
+    Fuzzy {
+        include: &'a [String],
+        exclude: &'a [String],
+        exclude_suffix: &'a [String],
+    },
+}
+
 pub fn get_compile_benchmarks(
     benchmark_dir: &Path,
-    include: &[String],
-    exclude: &[String],
-    exclude_suffix: &[String],
+    filter: CompileBenchmarkFilter<'_>,
 ) -> anyhow::Result<Vec<Benchmark>> {
-    let mut benchmarks = Vec::new();
-
     let mut paths = Vec::new();
     for entry in std::fs::read_dir(benchmark_dir)
         .with_context(|| format!("failed to list benchmark dir '{}'", benchmark_dir.display()))?
@@ -493,6 +501,40 @@ pub fn get_compile_benchmarks(
         paths.push((path, name));
     }
 
+    let mut benchmarks = match filter {
+        CompileBenchmarkFilter::All => paths
+            .into_iter()
+            .map(|(path, name)| Benchmark::new(name, path))
+            .collect::<anyhow::Result<Vec<Benchmark>>>()?,
+        CompileBenchmarkFilter::Exact(names) => paths
+            .into_iter()
+            .filter(|(_, name)| names.contains(name))
+            .map(|(path, name)| Benchmark::new(name, path))
+            .collect::<anyhow::Result<Vec<Benchmark>>>()?,
+        CompileBenchmarkFilter::Fuzzy {
+            include,
+            exclude,
+            exclude_suffix,
+        } => select_benchmarks_fuzzy(paths, include, exclude, exclude_suffix)?,
+    };
+
+    benchmarks.sort_by_key(|benchmark| benchmark.name.clone());
+
+    if benchmarks.is_empty() {
+        eprintln!("Warning: no benchmarks selected! Try less strict filters.");
+    }
+
+    Ok(benchmarks)
+}
+
+fn select_benchmarks_fuzzy(
+    paths: Vec<(PathBuf, String)>,
+    include: &[String],
+    exclude: &[String],
+    exclude_suffix: &[String],
+) -> anyhow::Result<Vec<Benchmark>> {
+    let mut benchmarks = Vec::new();
+
     // For each --include/--exclude entry, we count how many times it's used,
     // to enable `check_for_unused` below.
     fn to_hashmap(xyz: &[String]) -> HashMap<&str, usize> {
@@ -551,12 +593,6 @@ Expected zero or more entries or substrings from list: {:?}."#,
     check_for_unused("exclude", excludes)?;
     check_for_unused("exclude-suffix", exclude_suffixes)?;
 
-    benchmarks.sort_by_key(|benchmark| benchmark.name.clone());
-
-    if benchmarks.is_empty() {
-        eprintln!("Warning: no benchmarks selected! Try less strict filters.");
-    }
-
     Ok(benchmarks)
 }
 
@@ -578,17 +614,19 @@ fn substring_matches(
 
 #[cfg(test)]
 mod tests {
-    use crate::compile::benchmark::get_compile_benchmarks;
+    use crate::compile::benchmark::{get_compile_benchmarks, CompileBenchmarkFilter};
     use std::path::Path;
 
     #[test]
     fn check_compile_benchmarks() {
         // Check that we can deserialize all perf-config.json files in the compile benchmark
         // directory.
         let root = env!("CARGO_MANIFEST_DIR");
-        let benchmarks =
-            get_compile_benchmarks(&Path::new(root).join("compile-benchmarks"), &[], &[], &[])
-                .unwrap();
+        let benchmarks = get_compile_benchmarks(
+            &Path::new(root).join("compile-benchmarks"),
+            CompileBenchmarkFilter::All,
+        )
+        .unwrap();
         assert!(!benchmarks.is_empty());
     }
 }