feat: add exec-harness binary

This binary is in charge of providing a minimal instrument-hooks wrapper
around the command it is given.

Author: GuillaumeLagrange

Cargo.lock

@@ -12,7 +12,7 @@ path = "src/main.rs"
 
 [dependencies]
 anyhow = "1.0.75"
-clap = { version = "4.4.8", features = ["derive", "env", "color"] }
+clap = { workspace = true }
 itertools = "0.11.0"
 lazy_static = "1.4.0"
 log = "0.4.20"
@@ -76,7 +76,10 @@ rstest_reuse = "0.7.0"
 shell-quote = "0.7.2"
 
 [workspace]
-members = ["crates/runner-shared"]
+members = ["crates/exec-harness", "crates/runner-shared"]
+
+[workspace.dependencies]
+clap = { version = "4.4.8", features = ["derive", "env", "color"] }
 
 [workspace.metadata.release]
 sign-tag = true

Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "exec-harness"
+version = "4.4.1"
+edition = "2024"
+
+[dependencies]
+anyhow = { workspace = true }
+codspeed = "4.1.0"
+clap = { workspace = true }
+serde_json = { workspace = true }
+serde = { workspace = true }
+nix = { workspace = true, features = ["signal"] }

crates/exec-harness/Cargo.toml

@@ -0,0 +1,116 @@
+use crate::walltime::WalltimeResults;
+use clap::Parser;
+use codspeed::instrument_hooks::InstrumentHooks;
+use codspeed::walltime_results::WalltimeBenchmark;
+use std::path::PathBuf;
+use std::process;
+
+mod walltime;
+
+#[derive(Parser, Debug)]
+#[command(name = "exec-harness")]
+#[command(about = "CodSpeed exec harness - wraps commands with performance instrumentation")]
+struct Args {
+    /// Optional benchmark name (defaults to command filename)
+    #[arg(long)]
+    name: Option<String>,
+
+    /// The command and arguments to execute
+    command: Vec<String>,
+}
+
+fn main() {
+    let args = Args::parse();
+
+    if args.command.is_empty() {
+        eprintln!("Error: No command provided");
+        process::exit(1);
+    }
+
+    // Derive benchmark name from command if not provided
+    let bench_name = args.name.unwrap_or_else(|| {
+        // Extract filename from command path
+        let cmd = &args.command[0];
+        std::path::Path::new(cmd)
+            .file_name()
+            .and_then(|n| n.to_str())
+            .map(|s| s.to_string())
+            .unwrap_or_else(|| "exec_benchmark".to_string())
+    });
+    // TODO: Better URI generation
+    let bench_uri = format!("standalone_run::{bench_name}");
+
+    let hooks = InstrumentHooks::instance();
+
+    // TODO: Change this to avoid impersonating `codspeed-rust`
+    hooks
+        .set_integration("codspeed-rust", env!("CARGO_PKG_VERSION"))
+        .unwrap();
+
+    const NUM_ITERATIONS: usize = 1;
+    let mut times_per_round_ns = Vec::with_capacity(NUM_ITERATIONS);
+
+    hooks.start_benchmark().unwrap();
+    for _ in 0..NUM_ITERATIONS {
+        // Spawn the command
+        let mut child = match process::Command::new(&args.command[0])
+            .args(&args.command[1..])
+            .spawn()
+        {
+            Ok(child) => child,
+            Err(e) => {
+                eprintln!("Failed to spawn command: {e}");
+                process::exit(1);
+            }
+        };
+        // Start monotonic timer for this iteration
+        let bench_start = InstrumentHooks::current_timestamp();
+
+        // Wait for the process to complete
+        let status = match child.wait() {
+            Ok(status) => status,
+            Err(e) => {
+                eprintln!("Failed to wait for command: {e}");
+                process::exit(1);
+            }
+        };
+
+        // Measure elapsed time
+        let bench_end = InstrumentHooks::current_timestamp();
+        hooks.add_benchmark_timestamps(bench_start, bench_end);
+
+        // Exit immediately if any iteration fails
+        if !status.success() {
+            eprintln!("Command failed with exit code: {:?}", status.code());
+            process::exit(status.code().unwrap_or(1));
+        }
+
+        // Calculate and store the elapsed time in nanoseconds
+        let elapsed_ns = (bench_end - bench_start) as u128;
+        times_per_round_ns.push(elapsed_ns);
+    }
+
+    hooks.stop_benchmark().unwrap();
+    hooks.set_executed_benchmark(&bench_uri).unwrap();
+
+    // Collect walltime results
+    let max_time_ns = times_per_round_ns.iter().copied().max();
+    let walltime_benchmark = WalltimeBenchmark::from_runtime_data(
+        bench_name.clone(),
+        bench_uri.clone(),
+        vec![1; NUM_ITERATIONS],
+        times_per_round_ns,
+        max_time_ns,
+    );
+
+    let walltime_results = WalltimeResults::from_benchmarks(vec![walltime_benchmark])
+        .expect("Failed to create walltime results");
+
+    walltime_results
+        .save_to_file(
+            std::env::var("CODSPEED_PROFILE_FOLDER")
+                .map(PathBuf::from)
+                .unwrap_or_else(|_| std::env::current_dir().unwrap().join(".codspeed")),
+        )
+        .expect("Failed to save walltime results");
+}

crates/exec-harness/src/main.rs

@@ -0,0 +1,62 @@
+use anyhow::Context;
+use anyhow::Result;
+use codspeed::walltime_results::WalltimeBenchmark;
+use serde::Deserialize;
+use serde::Serialize;
+use std::path::Path;
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Instrument {
+    #[serde(rename = "type")]
+    type_: String,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Creator {
+    name: String,
+    version: String,
+    pid: u32,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct WalltimeResults {
+    creator: Creator,
+    instrument: Instrument,
+    benchmarks: Vec<WalltimeBenchmark>,
+}
+
+impl WalltimeResults {
+    pub fn from_benchmarks(benchmarks: Vec<WalltimeBenchmark>) -> Result<Self> {
+        Ok(WalltimeResults {
+            instrument: Instrument {
+                type_: "walltime".to_string(),
+            },
+            creator: Creator {
+                name: "codspeed-rust".to_string(),
+                version: env!("CARGO_PKG_VERSION").to_string(),
+                pid: std::process::id(),
+            },
+            benchmarks,
+        })
+    }
+
+    pub fn save_to_file<P: AsRef<Path>>(&self, profile_folder: P) -> Result<()> {
+        let results_path = {
+            let results_dir = profile_folder.as_ref().join("results");
+            std::fs::create_dir_all(&results_dir).with_context(|| {
+                format!(
+                    "Failed to create results directory: {}",
+                    results_dir.display()
+                )
+            })?;
+
+            results_dir.join(format!("{}.json", self.creator.pid))
+        };
+
+        let file = std::fs::File::create(&results_path)
+            .with_context(|| format!("Failed to create file: {}", results_path.display()))?;
+        serde_json::to_writer_pretty(file, &self)
+            .with_context(|| format!("Failed to write JSON to file: {}", results_path.display()))?;
+        Ok(())
+    }
+}