--wip-- [skip ci]

Author: GuillaumeLagrange

Cargo.lock

@@ -11,7 +11,7 @@ path = "src/main.rs"
 
 
 [dependencies]
-anyhow = "1.0.75"
+anyhow = { workspace = true }
 clap = { workspace = true }
 itertools = "0.11.0"
 lazy_static = "1.4.0"
@@ -26,8 +26,8 @@ reqwest = { version = "0.11.22", features = [
 ] }
 reqwest-middleware = "0.2.4"
 reqwest-retry = "0.3.0"
-serde = { version = "1.0.192", features = ["derive"] }
-serde_json = { version = "1.0.108", features = ["preserve_order"] }
+serde = { workspace = true }
+serde_json = { workspace = true }
 url = "2.4.1"
 sha256 = "1.4.0"
 tokio = { version = "1", features = ["macros", "rt"] }
@@ -51,10 +51,10 @@ async-trait = "0.1.82"
 libc = "0.2.171"
 bincode = "1.3.3"
 object = "0.36.7"
-linux-perf-data = "0.11.0"
+linux-perf-data = { git = "ssh://[email protected]/CodSpeedHQ/linux-perf-data.git", branch = "feat/support-perf-pipe-data-parsing" }
 debugid = "0.8.0"
 memmap2 = "0.9.5"
-nix = { version = "0.29.0", features = ["fs", "time", "user"] }
+nix = { workspace = true, features = ["fs", "time", "user"] }
 futures = "0.3.31"
 runner-shared = { path = "crates/runner-shared" }
 shellexpand = { version = "3.1.1", features = ["tilde"] }
@@ -79,7 +79,11 @@ shell-quote = "0.7.2"
 members = ["crates/exec-harness", "crates/runner-shared"]
 
 [workspace.dependencies]
+anyhow = "1.0.75"
 clap = { version = "4.4.8", features = ["derive", "env", "color"] }
+nix = "0.29.0"
+serde = { version = "1.0.192", features = ["derive"] }
+serde_json = { version = "1.0.108", features = ["preserve_order"] }
 
 [workspace.metadata.release]
 sign-tag = true

Cargo.toml

@@ -4,5 +4,9 @@ version = "4.4.1"
 edition = "2024"
 
 [dependencies]
+anyhow = { workspace = true }
 codspeed = "4.1.0"
 clap = { workspace = true }
+serde_json = { workspace = true }
+serde = { workspace = true }
+nix = { workspace = true, features = ["signal"] }

crates/exec-harness/Cargo.toml

@@ -1,5 +1,11 @@
+use crate::walltime::WalltimeResults;
 use clap::Parser;
 use codspeed::instrument_hooks::InstrumentHooks;
+use codspeed::walltime_results::WalltimeBenchmark;
+use std::path::PathBuf;
+use std::process;
+
+mod walltime;
 
 #[derive(Parser, Debug)]
 #[command(name = "exec-harness")]
@@ -18,7 +24,7 @@ fn main() {
 
     if args.command.is_empty() {
         eprintln!("Error: No command provided");
-        std::process::exit(1);
+        process::exit(1);
     }
 
     // Derive benchmark name from command if not provided
@@ -34,27 +40,75 @@ fn main() {
 
     let hooks = InstrumentHooks::instance();
 
+    // TODO: Change this to avoid impersonating `codspeed-rust`
     hooks
-        .set_integration("codspeed-exec", env!("CARGO_PKG_VERSION"))
+        .set_integration("codspeed-rust", env!("CARGO_PKG_VERSION"))
         .unwrap();
+
+    const NUM_ITERATIONS: usize = 10;
+    let mut times_per_round_ns = Vec::with_capacity(NUM_ITERATIONS);
+
     hooks.start_benchmark().unwrap();
+    for _ in 0..NUM_ITERATIONS {
+        // Start monotonic timer for this iteration
+        let bench_start = InstrumentHooks::current_timestamp();
 
-    // Execute the command
-    let status = std::process::Command::new(&args.command[0])
-        .args(&args.command[1..])
-        .status();
+        // Spawn the command
+        let mut child = match process::Command::new(&args.command[0])
+            .args(&args.command[1..])
+            .spawn()
+        {
+            Ok(child) => child,
+            Err(e) => {
+                eprintln!("Failed to spawn command: {e}");
+                process::exit(1);
+            }
+        };
+        // Wait for the process to complete
+        let status = match child.wait() {
+            Ok(status) => status,
+            Err(e) => {
+                eprintln!("Failed to wait for command: {e}");
+                process::exit(1);
+            }
+        };
 
-    hooks.stop_benchmark().unwrap();
-    hooks.set_executed_benchmark(&bench_name).unwrap();
+        // Measure elapsed time
+        let bench_end = InstrumentHooks::current_timestamp();
+        hooks.add_benchmark_timestamps(bench_start, bench_end);
 
-    // Propagate exit code
-    match status {
-        Ok(exit_status) => {
-            std::process::exit(exit_status.code().unwrap_or(1));
-        }
-        Err(e) => {
-            eprintln!("Failed to execute command: {e}");
-            std::process::exit(1);
+        // Exit immediately if any iteration fails
+        if !status.success() {
+            eprintln!("Command failed with exit code: {:?}", status.code());
+            process::exit(status.code().unwrap_or(1));
         }
+
+        // Calculate and store the elapsed time in nanoseconds
+        let elapsed_ns = (bench_end - bench_start) as u128;
+        times_per_round_ns.push(elapsed_ns);
     }
+
+    hooks.stop_benchmark().unwrap();
+    hooks.set_executed_benchmark(&bench_name).unwrap();
+
+    // Collect walltime results
+    let max_time_ns = times_per_round_ns.iter().copied().max();
+    let walltime_benchmark = WalltimeBenchmark::from_runtime_data(
+        bench_name.clone(),
+        format!("standalone_run::{bench_name}"),
+        vec![1; NUM_ITERATIONS],
+        times_per_round_ns,
+        max_time_ns,
+    );
+
+    let walltime_results = WalltimeResults::from_benchmarks(vec![walltime_benchmark])
+        .expect("Failed to create walltime results");
+
+    walltime_results
+        .save_to_file(
+            std::env::var("CODSPEED_PROFILE_FOLDER")
+                .map(PathBuf::from)
+                .unwrap_or_else(|_| std::env::current_dir().unwrap().join(".codspeed")),
+        )
+        .expect("Failed to save walltime results");
 }

crates/exec-harness/src/main.rs

@@ -0,0 +1,62 @@
+use anyhow::Context;
+use anyhow::Result;
+use codspeed::walltime_results::WalltimeBenchmark;
+use serde::Deserialize;
+use serde::Serialize;
+use std::path::Path;
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Instrument {
+    #[serde(rename = "type")]
+    type_: String,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Creator {
+    name: String,
+    version: String,
+    pid: u32,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct WalltimeResults {
+    creator: Creator,
+    instrument: Instrument,
+    benchmarks: Vec<WalltimeBenchmark>,
+}
+
+impl WalltimeResults {
+    pub fn from_benchmarks(benchmarks: Vec<WalltimeBenchmark>) -> Result<Self> {
+        Ok(WalltimeResults {
+            instrument: Instrument {
+                type_: "walltime".to_string(),
+            },
+            creator: Creator {
+                name: "codspeed-rust".to_string(),
+                version: env!("CARGO_PKG_VERSION").to_string(),
+                pid: std::process::id(),
+            },
+            benchmarks,
+        })
+    }
+
+    pub fn save_to_file<P: AsRef<Path>>(&self, profile_folder: P) -> Result<()> {
+        let results_path = {
+            let results_dir = profile_folder.as_ref().join("results");
+            std::fs::create_dir_all(&results_dir).with_context(|| {
+                format!(
+                    "Failed to create results directory: {}",
+                    results_dir.display()
+                )
+            })?;
+
+            results_dir.join(format!("{}.json", self.creator.pid))
+        };
+
+        let file = std::fs::File::create(&results_path)
+            .with_context(|| format!("Failed to create file: {}", results_path.display()))?;
+        serde_json::to_writer_pretty(file, &self)
+            .with_context(|| format!("Failed to write JSON to file: {}", results_path.display()))?;
+        Ok(())
+    }
+}

crates/exec-harness/src/walltime.rs

@@ -212,7 +212,7 @@ impl CodSpeedAPIClient {
             Err(err) if err.contains_error_code("UNAUTHENTICATED") => {
                 bail!("Your session has expired, please login again using `codspeed auth login`")
             }
-            Err(err) => bail!("Failed to fetch local run report: {err}"),
+            Err(err) => bail!("Failed to fetch local run report: {err:?}"),
         }
     }
 }

src/api_client.rs

@@ -4,9 +4,6 @@ use crate::prelude::*;
 use clap::Args;
 use std::path::Path;
 
-mod run_with_harness;
-use run_with_harness::wrap_command_with_harness;
-
 #[derive(Args, Debug)]
 pub struct ExecArgs {
     #[command(flatten)]
@@ -26,10 +23,14 @@ pub async fn run(
     codspeed_config: &CodSpeedConfig,
     setup_cache_dir: Option<&Path>,
 ) -> Result<()> {
-    // Wrap the user's command with exec-harness BEFORE creating config
-    let wrapped_command = wrap_command_with_harness(&args.command, args.name.as_deref())?;
+    // Assume exec-harness is in the PATH for now
+    let wrapped_command = std::iter::once("exec-harness".to_string())
+        .chain(args.command)
+        .collect::<Vec<String>>();
+
+    let warped_command_string = wrapped_command.join(" ");
 
-    info!("Executing: {}", wrapped_command.join(" "));
+    info!("Executing: {warped_command_string}");
 
     // Convert ExecArgs to executor::Config using shared args
     let config = crate::executor::Config {
@@ -54,7 +55,7 @@ pub async fn run(
             })
             .transpose()?,
         working_directory: args.shared.working_directory,
-        command: wrapped_command.join(" "), // Use wrapped command
+        command: warped_command_string,
         mode: args.shared.mode,
         instruments: crate::instruments::Instruments { mongodb: None }, // exec doesn't support MongoDB
         enable_perf: args.shared.perf_run_args.enable_perf,