feat: add caddy integration test; fix bug where benchmark was ovewritten

Author: not-matthias

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "go-runner/testdata/projects/caddy"]
+	path = go-runner/testdata/projects/caddy
+	url = https://github.com/caddyserver/caddy.git

go-runner/Cargo.lock

@@ -22,3 +22,4 @@ thiserror = "2.0.12"
 [dev-dependencies]
 insta = { version = "1.43.1", features = ["json"] }
 rstest = "0.25.0"
+tempfile = "3.14.0"

go-runner/Cargo.toml

@@ -1,26 +1,13 @@
+use std::fs;
 use std::path::{Path, PathBuf};
-use std::{fs, io};
 
 use handlebars::Handlebars;
 use serde::{Deserialize, Serialize};
 
 use crate::builder::{BenchmarkData, GoPackage};
+use crate::utils;
 use crate::{builder::patcher, prelude::*};
 
-fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> io::Result<()> {
-    fs::create_dir_all(&dst)?;
-    for entry in fs::read_dir(src)? {
-        let entry = entry?;
-        let ty = entry.file_type()?;
-        if ty.is_dir() {
-            copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?;
-        } else {
-            fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?;
-        }
-    }
-    Ok(())
-}
-
 // FIXME: Move this to the discovery -> Create a new GoPackage which has a list of benchmarks
 fn find_benchmark_files(
     target_dir: &Path,
@@ -86,7 +73,7 @@ pub fn run(package: &GoPackage) -> anyhow::Result<(PathBuf, PathBuf)> {
         fs::remove_dir_all(&target_dir).context("Failed to remove existing target directory")?;
     }
     std::fs::create_dir_all(&target_dir).context("Failed to create target directory")?;
-    copy_dir_all(&package.module.dir, &target_dir)?;
+    utils::copy_dir_all(&package.module.dir, &target_dir)?;
 
     // Get files that need to be renamed first
     let files = package
@@ -96,12 +83,10 @@ pub fn run(package: &GoPackage) -> anyhow::Result<(PathBuf, PathBuf)> {
     // Calculate the relative path from module root to package directory
     let package_dir = Path::new(&package.dir);
     let module_dir = Path::new(&package.module.dir);
-    let relative_package_path = package_dir.strip_prefix(module_dir).context(
-        format!(
-            "Package dir {} is not within module dir {}",
-            package.dir, package.module.dir
-        )
-    )?;
+    let relative_package_path = package_dir.strip_prefix(module_dir).context(format!(
+        "Package dir {} is not within module dir {}",
+        package.dir, package.module.dir
+    ))?;
     info!("Relative package path: {}", relative_package_path.display());
 
     // 2. Find benchmark files and patch only their imports
@@ -120,13 +105,11 @@ pub fn run(package: &GoPackage) -> anyhow::Result<(PathBuf, PathBuf)> {
                 .replace("_test", "_codspeed"),
         );
 
-        fs::rename(&old_path, &new_path).context(
-            format!(
-                "Failed to rename {} to {}",
-                old_path.display(),
-                new_path.display()
-            )
-        )?;
+        fs::rename(&old_path, &new_path).context(format!(
+            "Failed to rename {} to {}",
+            old_path.display(),
+            new_path.display()
+        ))?;
     }
 
     // 4. Generate the codspeed/runner.go file using the template

go-runner/src/builder/templater.rs

@@ -0,0 +1,51 @@
+use crate::prelude::*;
+use std::path::{Path, PathBuf};
+use tempfile::TempDir;
+
+fn setup_test_project(project_name: &str) -> (TempDir, PathBuf) {
+    let project_path = Path::new("testdata/projects").join(project_name);
+    let temp_dir = TempDir::new().unwrap();
+    let test_dir = temp_dir.path();
+
+    crate::utils::copy_dir_all(&project_path, test_dir.join(project_name)).unwrap();
+    let project_test_path = test_dir.join(project_name);
+
+    std::env::set_current_dir(&project_test_path).unwrap();
+
+    (temp_dir, project_test_path)
+}
+
+/// Integration tests with real Go projects to ensure end-to-end functionality
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn assert_benchmarks_created(n: usize) {
+        let codspeed_dir = std::env::current_dir()
+            .unwrap()
+            .join(".codspeed")
+            .join("walltime");
+        assert!(codspeed_dir.exists());
+
+        let entries: Vec<_> = std::fs::read_dir(&codspeed_dir)
+            .unwrap()
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap();
+        assert_eq!(entries.len(), n);
+    }
+
+    #[test]
+    fn test_caddy_benchmarks() {
+        let (_temp_dir, caddy_test_path) = setup_test_project("caddy");
+
+        let binaries = crate::build_benchmarks(&caddy_test_path).unwrap();
+        assert!(!binaries.is_empty(), "No benchmark binaries were created");
+        assert_benchmarks_created(binaries.len());
+
+        // FIXME: Find a better solution for this
+        unsafe {
+            std::env::set_var("CODSPEED_PROFILE_FOLDER", caddy_test_path.join("profile"));
+        }
+        crate::run_benchmarks(".").unwrap();
+    }
+}

go-runner/src/integration_tests.rs

@@ -7,6 +7,10 @@ use std::{
 mod builder;
 pub mod prelude;
 mod results;
+pub(crate) mod utils;
+
+#[cfg(test)]
+mod integration_tests;
 
 /// Builds the specified go project and writes the benchmarks to the .codspeed folder.
 pub fn build_benchmarks(project_dir: &Path) -> anyhow::Result<Vec<PathBuf>> {
@@ -33,19 +37,29 @@ pub fn build_benchmarks(project_dir: &Path) -> anyhow::Result<Vec<PathBuf>> {
         binaries.push(builder::runner::build(&runner_path)?);
     }
 
-    // 3. Copy them to the .codspeed folder
+    // 4. Copy them to the .codspeed folder
     let codspeed_dir = std::env::current_dir()?.join(".codspeed").join("walltime");
 
     debug!("Creating codspeed directory: {}", codspeed_dir.display());
     std::fs::create_dir_all(&codspeed_dir)?;
 
-    for binary in &binaries {
-        let target_path = codspeed_dir.join(binary.file_name().unwrap());
-        debug!("Copying {} to {}", binary.display(), target_path.display());
-        std::fs::copy(binary, &target_path)?;
+    let mut copied_binaries = Vec::new();
+    for binary_path in &binaries {
+        // Create a unique filename to avoid accidentally overwriting existing benchmarks.
+        let filename = binary_path.file_name().unwrap().to_string_lossy();
+        let unique_filename = format!("{}_{:08x}", filename, rand::random::<u32>());
+        let target_path = codspeed_dir.join(unique_filename);
+
+        debug!(
+            "Copying {} to {}",
+            binary_path.display(),
+            target_path.display()
+        );
+        std::fs::copy(binary_path, &target_path)?;
+        copied_binaries.push(target_path);
     }
 
-    Ok(binaries)
+    Ok(copied_binaries)
 }
 
 pub fn run_benchmarks(bench: &str) -> anyhow::Result<()> {
@@ -70,8 +84,8 @@ pub fn run_benchmarks(bench: &str) -> anyhow::Result<()> {
             let stdout = String::from_utf8_lossy(&cmd.stdout);
             let stderr = String::from_utf8_lossy(&cmd.stderr);
 
-            info!("Command output: {stdout}");
-            info!("Command error output: {stderr}");
+            warn!("Command output: {stdout}");
+            warn!("Command error output: {stderr}");
 
             bail!(
                 "Failed to run benchmark: {}. Exit status: {}",

go-runner/src/lib.rs

@@ -0,0 +1,20 @@
+use std::path::Path;
+use std::{fs, io};
+
+pub fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> io::Result<()> {
+    fs::create_dir_all(&dst)?;
+    for entry in fs::read_dir(src)? {
+        let entry = entry?;
+        let ty = entry.file_type()?;
+        if ty.is_dir() {
+            if entry.file_name() == ".git" {
+                continue;
+            }
+
+            copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?;
+        } else {
+            fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?;
+        }
+    }
+    Ok(())
+}