From 0a6883050d18a14cbd1f55a8476309e26c26c45d Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 5 Oct 2025 11:05:38 -0600 Subject: [PATCH 01/24] Prototype macOS support --- Cargo.toml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 69e338bf..ff3acb7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,7 +94,13 @@ ci = "github" # The installers to generate for each app installers = ["shell"] # Target platforms to build apps for (Rust target-triple syntax) -targets = ["aarch64-unknown-linux-musl", "x86_64-unknown-linux-musl"] +targets = [ + "aarch64-unknown-linux-musl", + "x86_64-unknown-linux-musl", + # macOS targets (Apple Silicon and Intel) + "aarch64-apple-darwin", + "x86_64-apple-darwin", +] # The archive format to use for non-windows builds (defaults .tar.xz) unix-archive = ".tar.gz" # Which actions to run on pull requests @@ -108,3 +114,6 @@ install-updater = false [workspace.metadata.dist.github-custom-runners] aarch64-unknown-linux-musl = "buildjet-2vcpu-ubuntu-2204-arm" +# Use GitHub-hosted macOS runners for apple-darwin targets by default +aarch64-apple-darwin = "macos-latest" +x86_64-apple-darwin = "macos-latest" From 74c3c84551af12d8c8fa512bef178c099c39d4f4 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 5 Oct 2025 11:38:39 -0600 Subject: [PATCH 02/24] Prototype macOS support --- src/run/runner/mod.rs | 15 ++++++- src/run/runner/valgrind/executor.rs | 24 ++++++++++- src/run/runner/valgrind/measure.rs | 63 +++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 3 deletions(-) diff --git a/src/run/runner/mod.rs b/src/run/runner/mod.rs index 978493b6..45686d44 100644 --- a/src/run/runner/mod.rs +++ b/src/run/runner/mod.rs @@ -31,7 +31,20 @@ pub const EXECUTOR_TARGET: &str = "executor"; pub fn get_executor_from_mode(mode: &RunnerMode) -> Box { match mode { - RunnerMode::Instrumentation => Box::new(ValgrindExecutor), + RunnerMode::Instrumentation => { + // Valgrind/Callgrind is not available on macOS (notably arm64 macOS). + // If the user requested Instrumentation mode on macOS, fall back to + // the WallTime executor so the produced archive and upload metadata + // accurately reflect what was collected (no Callgrind profiles). + #[cfg(target_os = "macos")] + { + Box::new(WallTimeExecutor::new()) + } + #[cfg(not(target_os = "macos"))] + { + Box::new(ValgrindExecutor) + } + } RunnerMode::Walltime => Box::new(WallTimeExecutor::new()), } } diff --git a/src/run/runner/valgrind/executor.rs b/src/run/runner/valgrind/executor.rs index 13dbbd67..a5f7cd96 100644 --- a/src/run/runner/valgrind/executor.rs +++ b/src/run/runner/valgrind/executor.rs @@ -18,7 +18,18 @@ impl Executor for ValgrindExecutor { } async fn setup(&self, system_info: &SystemInfo) -> Result<()> { - install_valgrind(system_info).await?; + // Valgrind / Callgrind is not supported on macOS (notably arm64 macOS). + // Instead of failing fast, allow the executor to run but skip installing + // Valgrind. The measure implementation contains a macOS fallback that + // runs the benchmark without instrumentation so users can still run + // benchmarks locally on macOS. + if cfg!(target_os = "macos") { + warn!( + "Valgrind/Callgrind is not supported on macOS: skipping Valgrind installation. Benchmarks will run without instrumentation." + ); + } else { + install_valgrind(system_info).await?; + } if let Err(error) = venv_compat::symlink_libpython(None) { warn!("Failed to symlink libpython"); @@ -35,7 +46,16 @@ impl Executor for ValgrindExecutor { run_data: &RunData, mongo_tracer: &Option, ) -> Result<()> { - //TODO: add valgrind version check + // On macOS, callgrind is not available. Let the measure function handle + // the macOS fallback (it will run the benchmark without instrumentation) + // so users can still run benchmarks locally. On non-macOS platforms we + // proceed with the regular Valgrind-based instrumentation. + // TODO: add valgrind version check for non-macOS platforms + if cfg!(target_os = "macos") { + info!( + "Running Valgrind executor on macOS: benchmarks will run without Callgrind instrumentation." + ); + } measure::measure(config, &run_data.profile_folder, mongo_tracer).await?; Ok(()) diff --git a/src/run/runner/valgrind/measure.rs b/src/run/runner/valgrind/measure.rs index 460010a2..0f70a9c5 100644 --- a/src/run/runner/valgrind/measure.rs +++ b/src/run/runner/valgrind/measure.rs @@ -79,6 +79,69 @@ pub async fn measure( profile_folder: &Path, mongo_tracer: &Option, ) -> Result<()> { + // valgrind (callgrind) is a Linux-only tool and is not available on macOS + // (notably arm64 macOS). On macOS we fall back to running the benchmark + // without instrumentation so users can still run benchmarks locally. + if cfg!(target_os = "macos") { + warn!( + "Valgrind/Callgrind is not available on macOS: running the benchmark without instrumentation. Results will not include callgrind profiles." + ); + + // Create the wrapper script and status file + let script_path = create_run_script()?; + let cmd_status_path = tempfile::NamedTempFile::new()?.into_temp_path(); + + // Prepare the command that will execute the benchmark wrapper + let bench_cmd = get_bench_command(config)?; + let mut cmd = Command::new(script_path.to_str().unwrap()); + cmd.args([bench_cmd.as_str(), cmd_status_path.to_str().unwrap()]); + + // Configure the environment similar to other runners, but use Walltime + // mode since we don't have instrumentation available. + cmd.envs(get_base_injected_env(RunnerMode::Walltime, profile_folder)) + .env("PYTHONMALLOC", "malloc") + .env( + "PATH", + format!( + "{}:{}:{}", + introspected_nodejs::setup() + .map_err(|e| anyhow!("failed to setup NodeJS introspection. {e}"))? + .to_string_lossy(), + introspected_golang::setup() + .map_err(|e| anyhow!("failed to setup Go introspection. {e}"))? + .to_string_lossy(), + env::var("PATH").unwrap_or_default(), + ), + ); + + if let Some(cwd) = &config.working_directory { + let abs_cwd = canonicalize(cwd)?; + cmd.current_dir(abs_cwd); + } + + if let Some(mongo_tracer) = mongo_tracer { + mongo_tracer.apply_run_command_transformations(&mut cmd)?; + } + + debug!("cmd: {cmd:?}"); + let _status = run_command_with_log_pipe(cmd) + .await + .map_err(|e| anyhow!("failed to execute the benchmark process. {e}"))?; + + // Check the exit code which was written to the file by the wrapper script. + let cmd_status = { + let content = std::fs::read_to_string(&cmd_status_path)?; + content + .parse::() + .map_err(|e| anyhow!("unable to retrieve the program exit code. {e}"))? + }; + debug!("Program exit code = {cmd_status}"); + if cmd_status != 0 { + bail!("failed to execute the benchmark process, exit code: {cmd_status}"); + } + + return Ok(()); + } // Create the command let mut cmd = Command::new("setarch"); cmd.arg(ARCH).arg("-R"); From 8dc3ec75929ffffbb440bf302db7534d4309e753 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 5 Oct 2025 12:09:47 -0600 Subject: [PATCH 03/24] Add simple installer script --- scripts/simple-installer.sh | 145 ++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 scripts/simple-installer.sh diff --git a/scripts/simple-installer.sh b/scripts/simple-installer.sh new file mode 100644 index 00000000..e9a25466 --- /dev/null +++ b/scripts/simple-installer.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# Simple installer for codspeed-runner +# This script clones the repository (or uses the current directory), builds the +# `codspeed` binary with cargo in release mode, and installs it to the target +# directory (default: /usr/local/bin). It intentionally avoids cargo-dist and +# the GitHub release flow so you can build and install locally or from CI. + +set -euo pipefail + +REPO_URL="https://github.com/jzombie/codspeed-runner.git" +REF="main" +INSTALL_DIR="/usr/local/bin" +TMP_DIR="" +NO_RUSTUP="false" +QUIET="false" + +usage() { + cat <] [--ref ] [--install-dir ] [--no-rustup] [--quiet] + +Options: + --repo Git repository URL (default: ${REPO_URL}) + --ref Git ref to checkout (branch, tag, or commit). Default: ${REF} + --install-dir Where to install the built binary. Default: ${INSTALL_DIR} + --no-rustup Do not attempt to install rustup if cargo is missing + --quiet Minimize output + -h, --help Show this help message + +Example: + curl -fsSL https://example.com/codspeed-runner-installer.sh | bash -s -- --ref feature/my-branch + +This script will clone the repository to a temporary directory, build the +`codspeed` binary with `cargo build --release`, and copy it to +${INSTALL_DIR}. Sudo may be used to write to the install directory. +EOF +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --repo) + REPO_URL="$2"; shift 2;; + --ref) + REF="$2"; shift 2;; + --install-dir) + INSTALL_DIR="$2"; shift 2;; + --no-rustup) + NO_RUSTUP="true"; shift 1;; + --quiet) + QUIET="true"; shift 1;; + -h|--help) + usage; exit 0;; + --) + shift; break;; + *) + echo "Unknown argument: $1" >&2; usage; exit 1;; + esac +done + +log() { + if [ "$QUIET" != "true" ]; then + echo "$@" + fi +} + +fail() { + echo "Error: $@" >&2 + exit 1 +} + +cleanup() { + if [ -n "$TMP_DIR" ] && [ -d "$TMP_DIR" ]; then + rm -rf "$TMP_DIR" + fi +} +trap cleanup EXIT + +check_command() { + command -v "$1" >/dev/null 2>&1 +} + +ensure_rust() { + if check_command cargo; then + log "Found cargo" + return 0 + fi + + if [ "$NO_RUSTUP" = "true" ]; then + fail "cargo is not installed and --no-rustup was passed. Install Rust toolchain first."; + fi + + log "Rust toolchain not found. Installing rustup (non-interactive)..." + # Install rustup non-interactively + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y || fail "failed to install rustup" + export PATH="$HOME/.cargo/bin:$PATH" + check_command cargo || fail "cargo still not available after rustup install" +} + +main() { + ensure_rust + + # Create temp dir + TMP_DIR=$(mktemp -d -t codspeed-installer-XXXX) + log "Using temporary directory: $TMP_DIR" + + # Clone the requested ref + log "Cloning ${REPO_URL} (ref: ${REF})..." + git clone --depth 1 --branch "$REF" "$REPO_URL" "$TMP_DIR" || { + # Try cloning default branch and then checking out ref (for commit-ish refs) + log "Shallow clone failed for ref $REF, attempting full clone and checkout" + rm -rf "$TMP_DIR" + TMP_DIR=$(mktemp -d -t codspeed-installer-XXXX) + git clone "$REPO_URL" "$TMP_DIR" || fail "failed to clone repo" + (cd "$TMP_DIR" && git fetch --all --tags && git checkout "$REF") || fail "failed to checkout ref $REF" + } + + # Build + log "Building codspeed (release)..." + (cd "$TMP_DIR" && cargo build --release) || fail "cargo build failed" + + # Locate built binary + BIN_PATH="$TMP_DIR/target/release/codspeed" + if [ ! -x "$BIN_PATH" ]; then + fail "Built binary not found at $BIN_PATH" + fi + + # Ensure install dir exists + if [ ! -d "$INSTALL_DIR" ]; then + log "Creating install directory $INSTALL_DIR" + mkdir -p "$INSTALL_DIR" || fail "failed to create install dir" + fi + + # Copy binary (use sudo if required) + DEST="$INSTALL_DIR/codspeed" + if [ -w "$INSTALL_DIR" ]; then + cp "$BIN_PATH" "$DEST" || fail "failed to copy binary to $DEST" + else + log "Installing to $DEST with sudo" + sudo cp "$BIN_PATH" "$DEST" || fail "sudo copy failed" + fi + + log "Installed codspeed to $DEST" + log "Run 'codspeed --help' to verify" +} + +main "$@" From 6aeb03c139a6e1e87ed4910e2f53118cfacf8c7b Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 5 Oct 2025 12:23:10 -0600 Subject: [PATCH 04/24] Patch macOS support --- src/run/check_system.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/run/check_system.rs b/src/run/check_system.rs index efe7a29e..edbd5a99 100644 --- a/src/run/check_system.rs +++ b/src/run/check_system.rs @@ -135,7 +135,14 @@ pub fn check_system(system_info: &SystemInfo) -> Result<()> { return Ok(()); } - match system_info.arch.as_str() { + // Normalize common architecture strings (macOS reports `arm64` on Apple + // Silicon; treat it as `aarch64` which the rest of the codebase expects). + let arch = match system_info.arch.as_str() { + "arm64" => "aarch64", + other => other, + }; + + match arch { "x86_64" | "aarch64" => { warn!( "Unofficially supported system: {} {}. Continuing with best effort support.", From a24692541e598110afd06f20c86d61f999765f93 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 5 Oct 2025 12:26:46 -0600 Subject: [PATCH 05/24] Remove `x86_64-apple-darwin` --- Cargo.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ff3acb7f..91371645 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -99,7 +99,6 @@ targets = [ "x86_64-unknown-linux-musl", # macOS targets (Apple Silicon and Intel) "aarch64-apple-darwin", - "x86_64-apple-darwin", ] # The archive format to use for non-windows builds (defaults .tar.xz) unix-archive = ".tar.gz" @@ -116,4 +115,4 @@ install-updater = false aarch64-unknown-linux-musl = "buildjet-2vcpu-ubuntu-2204-arm" # Use GitHub-hosted macOS runners for apple-darwin targets by default aarch64-apple-darwin = "macos-latest" -x86_64-apple-darwin = "macos-latest" + From 2729c14d6230b9db59c7b57ce75718a51db33edb Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 5 Oct 2025 12:27:11 -0600 Subject: [PATCH 06/24] Remove extra trailing line break --- Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 91371645..b2ce254f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -115,4 +115,3 @@ install-updater = false aarch64-unknown-linux-musl = "buildjet-2vcpu-ubuntu-2204-arm" # Use GitHub-hosted macOS runners for apple-darwin targets by default aarch64-apple-darwin = "macos-latest" - From 84a0430328dffb2e220909c777fc002679969579 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 5 Oct 2025 12:48:17 -0600 Subject: [PATCH 07/24] Add workaround for asking for sudo password on macOS --- src/run/runner/wall_time/executor.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/run/runner/wall_time/executor.rs b/src/run/runner/wall_time/executor.rs index 8c6d9854..d4eda779 100644 --- a/src/run/runner/wall_time/executor.rs +++ b/src/run/runner/wall_time/executor.rs @@ -177,7 +177,16 @@ impl Executor for WallTimeExecutor { run_data: &RunData, _mongo_tracer: &Option, ) -> Result<()> { - let mut cmd = Command::new("sudo"); + // Note: (jzombie) Workaround for asking for `sudo password` on macOS + // Only use sudo when perf is enabled and available; otherwise run the + // benchmark directly to avoid prompting for passwords on CI (macOS + // runners, etc.). + let use_sudo = self.perf.is_some() && config.enable_perf; + let mut cmd = if use_sudo { + Command::new("sudo") + } else { + Command::new("sh") + }; if let Some(cwd) = &config.working_directory { let abs_cwd = canonicalize(cwd)?; From 75901d13e2e0014f4682b202c4816308841e1df8 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 5 Oct 2025 12:53:33 -0600 Subject: [PATCH 08/24] Add TODO --- src/run/runner/wall_time/executor.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/run/runner/wall_time/executor.rs b/src/run/runner/wall_time/executor.rs index d4eda779..7999c931 100644 --- a/src/run/runner/wall_time/executor.rs +++ b/src/run/runner/wall_time/executor.rs @@ -177,10 +177,12 @@ impl Executor for WallTimeExecutor { run_data: &RunData, _mongo_tracer: &Option, ) -> Result<()> { - // Note: (jzombie) Workaround for asking for `sudo password` on macOS + // Note: (jzombie) Workaround for asking for `sudo password` on macOS. // Only use sudo when perf is enabled and available; otherwise run the // benchmark directly to avoid prompting for passwords on CI (macOS // runners, etc.). + // TODO: There is also a `run_with_sudo` in `setup.rs` that might be a + // better way to approach this let use_sudo = self.perf.is_some() && config.enable_perf; let mut cmd = if use_sudo { Command::new("sudo") From 0fddfe3735c7127f95072f785dd03b1af400b7ec Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 5 Oct 2025 13:02:41 -0600 Subject: [PATCH 09/24] Debug: ` /bin/sh: /bin/sh: cannot execute binary file` --- src/run/runner/wall_time/executor.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/run/runner/wall_time/executor.rs b/src/run/runner/wall_time/executor.rs index 7999c931..5b4872d7 100644 --- a/src/run/runner/wall_time/executor.rs +++ b/src/run/runner/wall_time/executor.rs @@ -184,6 +184,8 @@ impl Executor for WallTimeExecutor { // TODO: There is also a `run_with_sudo` in `setup.rs` that might be a // better way to approach this let use_sudo = self.perf.is_some() && config.enable_perf; + // If we need sudo, the command will be `sudo sh -c ''`. + // Otherwise we invoke the shell directly as `sh -c ''`. let mut cmd = if use_sudo { Command::new("sudo") } else { @@ -202,9 +204,21 @@ impl Executor for WallTimeExecutor { if let Some(perf) = &self.perf && config.enable_perf { + // Perf runner expects the `cmd` to be either `sudo` (so that + // it becomes `sudo sh -c ...`) or `sh` (in which case the + // perf runner will arrange execution itself). Pass through. perf.run(cmd, &bench_cmd, config).await } else { - cmd.args(["sh", "-c", &bench_cmd]); + // Add the appropriate arguments depending on whether we're + // invoking via sudo or directly. When using sudo, the args + // must be: `sh -c ''`. When not using sudo, the + // args are just `-c ''` because the program is + // already `sh`. + if use_sudo { + cmd.args(["sh", "-c", &bench_cmd]); + } else { + cmd.args(["-c", &bench_cmd]); + } debug!("cmd: {cmd:?}"); run_command_with_log_pipe(cmd).await From 3e125ba2828954c1364012ea1a556b42023c5ce0 Mon Sep 17 00:00:00 2001 From: Jeremy Harris Date: Sun, 5 Oct 2025 13:09:23 -0600 Subject: [PATCH 10/24] Skip `systemd-run` if not on Linux --- src/run/runner/wall_time/executor.rs | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/run/runner/wall_time/executor.rs b/src/run/runner/wall_time/executor.rs index 5b4872d7..15591e58 100644 --- a/src/run/runner/wall_time/executor.rs +++ b/src/run/runner/wall_time/executor.rs @@ -148,10 +148,27 @@ impl WallTimeExecutor { // - We have to pass the environment variables because `--scope` only inherits the system and not the user environment variables. let uid = nix::unistd::Uid::current().as_raw(); let gid = nix::unistd::Gid::current().as_raw(); - let cmd = format!( - "systemd-run {quiet_flag} --scope --slice=codspeed.slice --same-dir --uid={uid} --gid={gid} -- bash {}", - script_file.path().display() - ); + // Prefer using systemd-run on Linux hosts when available since it + // provides the `--scope` isolation we need. On macOS (or when + // systemd isn't installed) fall back to invoking `bash