From e5883dbb3c9fc1993aa62f8d592fbdb602075fb0 Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Sat, 8 Mar 2025 17:30:28 +0100 Subject: [PATCH 1/2] feat: explicitely disable ASLR with the walltime executor --- src/run/runner/wall_time/executor.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/run/runner/wall_time/executor.rs b/src/run/runner/wall_time/executor.rs index 6bb03520..e04bd94d 100644 --- a/src/run/runner/wall_time/executor.rs +++ b/src/run/runner/wall_time/executor.rs @@ -8,6 +8,7 @@ use crate::run::runner::helpers::run_command_with_log_pipe::run_command_with_log use crate::run::runner::{ExecutorName, RunData, RunnerMode}; use crate::run::{check_system::SystemInfo, config::Config}; use async_trait::async_trait; +use std::env::consts::ARCH; use std::fs::canonicalize; use std::process::Command; @@ -26,7 +27,8 @@ impl Executor for WallTimeExecutor { run_data: &RunData, _mongo_tracer: &Option, ) -> Result<()> { - let mut cmd = Command::new("sh"); + let mut cmd = Command::new("setarch"); + cmd.arg(ARCH).arg("-R"); cmd.envs(get_base_injected_env( RunnerMode::WallTime, @@ -38,7 +40,8 @@ impl Executor for WallTimeExecutor { cmd.current_dir(abs_cwd); } - cmd.args(["-c", get_bench_command(config)?.as_str()]); + // Configure perf + cmd.args(["sh", "-c", get_bench_command(config)?.as_str()]); debug!("cmd: {:?}", cmd); let status = run_command_with_log_pipe(cmd) From 47961c4e104a5dbe16cccb97ba9277a15c0a199a Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Sat, 8 Mar 2025 18:15:26 +0100 Subject: [PATCH 2/2] --wip-- [skip ci] --- README.md | 13 +++++++++++-- codspeed.yml | 4 ++++ src/run/config.rs | 7 +++++++ src/run/mod.rs | 5 +++++ 4 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 codspeed.yml diff --git a/README.md b/README.md index ab996214..401461a0 100644 --- a/README.md +++ b/README.md @@ -38,13 +38,17 @@ Refer to the [releases page](https://github.com/CodSpeedHQ/runner/releases) to s > [!NOTE] > For now, the CLI only supports Ubuntu 20.04, 22.04, 24.04 and Debian 11, 12. +### Authentication + First, authenticate with your CodSpeed account: ```bash codspeed auth login ``` -Then, run benchmarks with the following command: +### Running benchmarks + +Run benchmarks with the following command (with a benchmark harness of your choice): ```bash codspeed run @@ -59,7 +63,12 @@ codspeed run pytest ./tests --codspeed codspeed run pnpm vitest bench ``` +It's also possible to run standalone benchmark processes: + +```bash +codspeed run --standalone curl https://example.com ``` + Usage: codspeed run [OPTIONS] [COMMAND]... Arguments: @@ -82,4 +91,4 @@ Use the `CODSPEED_LOG` environment variable to set the logging level: ```bash CODSPEED_LOG=debug codspeed run ... -``` +```` diff --git a/codspeed.yml b/codspeed.yml new file mode 100644 index 00000000..01031f1d --- /dev/null +++ b/codspeed.yml @@ -0,0 +1,4 @@ +benchmarks: + - name: random bench + command: echo "hello world" + harnessed: false diff --git a/src/run/config.rs b/src/run/config.rs index 6ab959a6..124c4e14 100644 --- a/src/run/config.rs +++ b/src/run/config.rs @@ -9,6 +9,8 @@ pub struct Config { pub upload_url: Url, pub token: Option, pub working_directory: Option, + + pub standalone: bool, pub command: String, pub instruments: Instruments, @@ -35,6 +37,7 @@ impl Config { instruments: Instruments::test(), skip_upload: false, skip_setup: false, + standalone: false, } } } @@ -50,6 +53,7 @@ impl TryFrom for Config { .map_err(|e| anyhow!("Invalid upload URL: {}, {}", raw_upload_url, e))?; Ok(Self { upload_url, + standalone: args.standalone, token: args.token, working_directory: args.working_directory, instruments, @@ -77,6 +81,7 @@ mod tests { skip_upload: false, skip_setup: false, command: vec!["cargo".into(), "codspeed".into(), "bench".into()], + standalone: false, }) .unwrap(); assert_eq!(config.upload_url, Url::parse(DEFAULT_UPLOAD_URL).unwrap()); @@ -86,6 +91,7 @@ mod tests { assert!(!config.skip_upload); assert!(!config.skip_setup); assert_eq!(config.command, "cargo codspeed bench"); + assert!(!config.standalone); } #[test] @@ -99,6 +105,7 @@ mod tests { skip_upload: true, skip_setup: true, command: vec!["cargo".into(), "codspeed".into(), "bench".into()], + standalone: true, }) .unwrap(); diff --git a/src/run/mod.rs b/src/run/mod.rs index 960524c0..97f37323 100644 --- a/src/run/mod.rs +++ b/src/run/mod.rs @@ -38,6 +38,10 @@ fn show_banner() { #[derive(Args, Debug)] pub struct RunArgs { + /// Is the benchmark command a standalone command (i.e. not using a benchmarking framework) + #[arg(long, default_value = "false")] + pub standalone: bool, + /// The upload URL to use for uploading the results, useful for on-premises installations #[arg(long)] pub upload_url: Option, @@ -83,6 +87,7 @@ impl RunArgs { /// Constructs a new `RunArgs` with default values for testing purposes pub fn test() -> Self { Self { + standalone: false, upload_url: None, token: None, working_directory: None,