feat: add more test projects; add custom binary dir to run tests in parallel

Author: not-matthias

.gitmodules

@@ -1,3 +1,12 @@
 [submodule "go-runner/testdata/projects/caddy"]
 	path = go-runner/testdata/projects/caddy
 	url = https://github.com/caddyserver/caddy.git
+[submodule "go-runner/testdata/projects/fzf"]
+	path = go-runner/testdata/projects/fzf
+	url = https://github.com/junegunn/fzf.git
+[submodule "go-runner/testdata/projects/opentelemetry-go"]
+	path = go-runner/testdata/projects/opentelemetry-go
+	url = https://github.com/open-telemetry/opentelemetry-go.git
+[submodule "go-runner/testdata/projects/golang-benchmarks"]
+	path = go-runner/testdata/projects/golang-benchmarks
+	url = https://github.com/SimonWaldherr/golang-benchmarks.git

go-runner/src/integration_tests.rs

@@ -1,51 +1,67 @@
 use crate::prelude::*;
 use std::path::{Path, PathBuf};
+use std::sync::Mutex;
 use tempfile::TempDir;
 
-fn setup_test_project(project_name: &str) -> (TempDir, PathBuf) {
-    let project_path = Path::new("testdata/projects").join(project_name);
-    let temp_dir = TempDir::new().unwrap();
-    let test_dir = temp_dir.path();
+fn setup_test_project(project_name: &str) -> anyhow::Result<TempDir> {
+    let project_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("testdata/projects")
+        .join(project_name);
+    println!("Project path: {}", project_path.display());
 
-    crate::utils::copy_dir_all(&project_path, test_dir.join(project_name)).unwrap();
-    let project_test_path = test_dir.join(project_name);
+    let temp_dir = TempDir::new()?;
+    crate::utils::copy_dir_all(&project_path, &temp_dir)?;
 
-    std::env::set_current_dir(&project_test_path).unwrap();
+    Ok(temp_dir)
+}
+
+fn assert_benchmarks_created(binary_dir: &Path, n: usize) {
+    assert!(binary_dir.exists());
 
-    (temp_dir, project_test_path)
+    let entries: Vec<_> = std::fs::read_dir(binary_dir)
+        .unwrap()
+        .collect::<Result<Vec<_>, _>>()
+        .unwrap();
+    assert_eq!(entries.len(), n);
 }
 
-/// Integration tests with real Go projects to ensure end-to-end functionality
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    fn assert_benchmarks_created(n: usize) {
-        let codspeed_dir = std::env::current_dir()
-            .unwrap()
-            .join(".codspeed")
-            .join("walltime");
-        assert!(codspeed_dir.exists());
-
-        let entries: Vec<_> = std::fs::read_dir(&codspeed_dir)
-            .unwrap()
-            .collect::<Result<Vec<_>, _>>()
-            .unwrap();
-        assert_eq!(entries.len(), n);
-    }
+fn run_benchmark_for_project(project_name: &str) {
+    let temp_dir = setup_test_project(project_name).unwrap();
 
-    #[test]
-    fn test_caddy_benchmarks() {
-        let (_temp_dir, caddy_test_path) = setup_test_project("caddy");
+    let binary_dir = temp_dir.path().join(".codspeed").join("walltime");
+    let binaries = crate::build_benchmarks(temp_dir.path(), &binary_dir).unwrap();
+    assert!(!binaries.is_empty(), "No benchmark binaries were created");
+    assert_benchmarks_created(&binary_dir, binaries.len());
 
-        let binaries = crate::build_benchmarks(&caddy_test_path).unwrap();
-        assert!(!binaries.is_empty(), "No benchmark binaries were created");
-        assert_benchmarks_created(binaries.len());
+    // Mutex to prevent concurrent tests from interfering with CODSPEED_PROFILE_FOLDER env var
+    static ENV_MUTEX: Mutex<()> = Mutex::new(());
+    let _guard = ENV_MUTEX.lock().unwrap();
 
-        // FIXME: Find a better solution for this
-        unsafe {
-            std::env::set_var("CODSPEED_PROFILE_FOLDER", caddy_test_path.join("profile"));
-        }
-        crate::run_benchmarks(".").unwrap();
+    unsafe { std::env::set_var("CODSPEED_PROFILE_FOLDER", temp_dir.path().join("profile")) };
+    if let Err(error) = crate::run_benchmarks(".", &binary_dir) {
+        panic!("Benchmarks couldn't run: {error}");
     }
+
+    // TODO:  Assert that we have a results.json?
+}
+
+#[ignore = "doesn't work atm"]
+#[test]
+fn test_caddy_benchmarks() {
+    run_benchmark_for_project("caddy");
+}
+
+#[test]
+fn test_fzf_benchmarks() {
+    run_benchmark_for_project("fzf");
+}
+
+#[test]
+fn test_opentelemetry_go_benchmarks() {
+    run_benchmark_for_project("opentelemetry-go");
+}
+
+#[test]
+fn test_golang_benchmarks() {
+    run_benchmark_for_project("golang-benchmarks");
 }

go-runner/src/lib.rs

@@ -13,7 +13,7 @@ pub(crate) mod utils;
 mod integration_tests;
 
 /// Builds the specified go project and writes the benchmarks to the .codspeed folder.
-pub fn build_benchmarks(project_dir: &Path) -> anyhow::Result<Vec<PathBuf>> {
+pub fn build_benchmarks(project_dir: &Path, binary_dir: &Path) -> anyhow::Result<Vec<PathBuf>> {
     // 1. Benchmark and package discovery
     let benchmark_packages = builder::discovery::run(project_dir)?;
     info!("Discovered {} packages", benchmark_packages.len());
@@ -37,18 +37,16 @@ pub fn build_benchmarks(project_dir: &Path) -> anyhow::Result<Vec<PathBuf>> {
         binaries.push(builder::runner::build(&runner_path)?);
     }
 
-    // 4. Copy them to the .codspeed folder
-    let codspeed_dir = std::env::current_dir()?.join(".codspeed").join("walltime");
-
-    debug!("Creating codspeed directory: {}", codspeed_dir.display());
-    std::fs::create_dir_all(&codspeed_dir)?;
+    // 4. Copy them to the binary folder
+    debug!("Creating binary directory: {}", binary_dir.display());
+    std::fs::create_dir_all(binary_dir)?;
 
     let mut copied_binaries = Vec::new();
     for binary_path in &binaries {
         // Create a unique filename to avoid accidentally overwriting existing benchmarks.
         let filename = binary_path.file_name().unwrap().to_string_lossy();
         let unique_filename = format!("{}_{:08x}", filename, rand::random::<u32>());
-        let target_path = codspeed_dir.join(unique_filename);
+        let target_path = binary_dir.join(unique_filename);
 
         debug!(
             "Copying {} to {}",
@@ -62,10 +60,9 @@ pub fn build_benchmarks(project_dir: &Path) -> anyhow::Result<Vec<PathBuf>> {
     Ok(copied_binaries)
 }
 
-pub fn run_benchmarks(bench: &str) -> anyhow::Result<()> {
+pub fn run_benchmarks(bench: &str, binary_dir: &Path) -> anyhow::Result<()> {
     // 1. Discover the benchmarks
-    let codspeed_dir = std::env::current_dir()?.join(".codspeed").join("walltime");
-    let benchmark_packages = std::fs::read_dir(&codspeed_dir)?
+    let benchmark_packages = std::fs::read_dir(binary_dir)?
         .filter_map(Result::ok)
         .filter(|entry| entry.path().is_file())
         .map(|entry| entry.path())
@@ -77,7 +74,8 @@ pub fn run_benchmarks(bench: &str) -> anyhow::Result<()> {
 
         let cmd = std::process::Command::new(bench_path)
             .arg(format!("-test.bench={bench}"))
-            .output()?;
+            .output()
+            .context("Failed to execute benchmark command")?;
 
         // Check if the command was successful
         if !cmd.status.success() {

go-runner/src/main.rs

@@ -37,12 +37,16 @@ fn main() -> anyhow::Result<()> {
 
     let cli = Cli::parse();
 
+    let binary_dir = std::env::current_dir()
+        .unwrap()
+        .join(".codspeed")
+        .join("walltime");
     match &cli.command {
         Commands::Build { path } => {
-            go_runner::build_benchmarks(path)?;
+            go_runner::build_benchmarks(path, &binary_dir)?;
         }
         Commands::Run { bench } => {
-            go_runner::run_benchmarks(bench)?;
+            go_runner::run_benchmarks(bench, &binary_dir)?;
         }
     }
 

go-runner/testdata/projects/.gitignore

@@ -0,0 +1,3 @@
+# Ignore .git directories in test projects
+*/.git
+*/.git/*