tests: test that cargo-bench works for all compat layers

Author: art049

crates/cargo-codspeed/tests/.gitignore

@@ -0,0 +1,2 @@
+*/Cargo.lock
+*/target/

crates/cargo-codspeed/tests/simple.in/Cargo.toml renamed to crates/cargo-codspeed/tests/simple-bencher.in/Cargo.toml

@@ -5,9 +5,8 @@ edition = "2021"
 publish = false
 
 [dependencies]
-bencher = "0.1.5"
 codspeed = { path = "../../../codspeed" }
-codspeed-bencher-compat = { path = "../../../bencher_compat" }
+bencher = { package = "codspeed-bencher-compat", path = "../../../bencher_compat" }
 
 [workspace]
 

crates/cargo-codspeed/tests/simple.in/benches/another_bencher_example.rs renamed to crates/cargo-codspeed/tests/simple-bencher.in/benches/another_bencher_example.rs

@@ -1,5 +1,5 @@
+use bencher::{benchmark_group, benchmark_main, Bencher};
 use codspeed::codspeed::black_box;
-use codspeed_bencher_compat::{benchmark_group, benchmark_main, Bencher};
 
 pub fn a(bench: &mut Bencher) {
     bench.iter(|| (0..100).fold(0, |x, y| black_box(x + y)))

crates/cargo-codspeed/tests/simple.in/benches/bencher_example.rs renamed to crates/cargo-codspeed/tests/simple-bencher.in/benches/bencher_example.rs

@@ -1,5 +1,5 @@
+use bencher::{benchmark_group, benchmark_main, Bencher};
 use codspeed::codspeed::black_box;
-use codspeed_bencher_compat::{benchmark_group, benchmark_main, Bencher};
 
 pub fn a(bench: &mut Bencher) {
     bench.iter(|| (0..100).fold(0, |x, y| black_box(x + y)))

crates/cargo-codspeed/tests/simple.rs renamed to crates/cargo-codspeed/tests/simple-bencher.rs

@@ -1,9 +1,10 @@
+use assert_cmd::assert::OutputAssertExt;
 use predicates::str::contains;
 
 mod helpers;
 use helpers::*;
 
-const DIR: &str = "tests/simple.in";
+const DIR: &str = "tests/simple-bencher.in";
 
 #[test]
 fn test_simple_run_without_build() {
@@ -69,3 +70,15 @@ fn test_simple_build_and_run_single() {
         .stderr(contains("another_bencher_example"));
     teardown(dir);
 }
+
+#[test]
+fn test_simple_cargo_bench_no_run() {
+    let dir = setup(DIR, Project::Simple);
+    std::process::Command::new("cargo")
+        .arg("bench")
+        .arg("--no-run")
+        .current_dir(&dir)
+        .assert()
+        .success();
+    teardown(dir);
+}

crates/cargo-codspeed/tests/simple-criterion.in/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "simple-criterion"
+version = "0.0.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+criterion = { package = "codspeed-criterion-compat", path = "../../../criterion_compat" }
+
+[workspace]
+
+
+[[bench]]
+name = "criterion_example"
+harness = false
+
+[[bench]]
+name = "another_criterion_example"
+harness = false

crates/cargo-codspeed/tests/simple-criterion.in/benches/another_criterion_example.rs

@@ -0,0 +1,24 @@
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+
+fn bubble_sort(arr: &mut Vec<i32>) {
+    let n = arr.len();
+    for i in 0..n {
+        for j in 0..(n - i - 1) {
+            if arr[j] > arr[j + 1] {
+                arr.swap(j, j + 1);
+            }
+        }
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    c.bench_function("bubble sort", |b| {
+        b.iter(|| {
+            let mut data = vec![64, 34, 25, 12, 22, 11, 90];
+            bubble_sort(black_box(&mut data));
+        })
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

crates/cargo-codspeed/tests/simple-criterion.in/benches/criterion_example.rs

@@ -0,0 +1,16 @@
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+
+fn fibonacci(n: u64) -> u64 {
+    match n {
+        0 => 1,
+        1 => 1,
+        n => fibonacci(n - 1) + fibonacci(n - 2),
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    c.bench_function("fib 20", |b| b.iter(|| fibonacci(black_box(20))));
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

crates/cargo-codspeed/tests/simple-criterion.rs

@@ -0,0 +1,84 @@
+use assert_cmd::assert::OutputAssertExt;
+use predicates::str::contains;
+
+mod helpers;
+use helpers::*;
+
+const DIR: &str = "tests/simple-criterion.in";
+
+#[test]
+fn test_criterion_run_without_build() {
+    let dir = setup(DIR, Project::Simple);
+    cargo_codspeed(&dir)
+        .arg("run")
+        .assert()
+        .failure()
+        .stderr(contains("Error: No benchmarks found."));
+    teardown(dir);
+}
+
+#[test]
+fn test_criterion_build() {
+    let dir = setup(DIR, Project::Simple);
+    cargo_codspeed(&dir)
+        .arg("build")
+        .assert()
+        .success()
+        .stderr(contains("Built 2 benchmark suite(s)"));
+    teardown(dir);
+}
+
+#[test]
+fn test_criterion_build_and_run() {
+    let dir = setup(DIR, Project::Simple);
+    cargo_codspeed(&dir).arg("build").assert().success();
+    cargo_codspeed(&dir)
+        .arg("run")
+        .assert()
+        .success()
+        .stderr(contains("Finished running 2 benchmark suite(s)"));
+    teardown(dir);
+}
+
+#[test]
+fn test_criterion_build_single() {
+    let dir = setup(DIR, Project::Simple);
+    cargo_codspeed(&dir)
+        .arg("build")
+        .arg("another_criterion_example")
+        .assert()
+        .success()
+        .stderr(contains("Built 1 benchmark suite(s)"))
+        .stderr(contains("another_criterion_example"));
+    teardown(dir);
+}
+
+#[test]
+fn test_criterion_build_and_run_single() {
+    let dir = setup(DIR, Project::Simple);
+    cargo_codspeed(&dir)
+        .arg("build")
+        .arg("another_criterion_example")
+        .assert()
+        .success();
+    cargo_codspeed(&dir)
+        .arg("run")
+        .arg("another_criterion_example")
+        .assert()
+        .success()
+        .stderr(contains("Finished running 1 benchmark suite(s)"))
+        .stderr(contains("another_criterion_example"));
+    teardown(dir);
+}
+
+#[test]
+fn test_criterion_cargo_bench_no_run() {
+    let dir = setup(DIR, Project::Simple);
+    std::process::Command::new("cargo")
+        .arg("bench")
+        .arg("--no-run")
+        .current_dir(&dir)
+        .assert()
+        .success();
+    teardown(dir);
+}

crates/cargo-codspeed/tests/simple-divan.in/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "simple-divan"
+version = "0.0.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+divan = { package = "codspeed-divan-compat", path = "../../../divan_compat" }
+
+[workspace]
+
+
+[[bench]]
+name = "divan_example"
+harness = false
+
+[[bench]]
+name = "another_divan_example"
+harness = false