diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1362cb12..bec70ebd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,10 @@ jobs: extra_args: --all-files tests: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 with: diff --git a/Cargo.toml b/Cargo.toml index 69e338bf..c765ab66 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,7 +94,12 @@ ci = "github" # The installers to generate for each app installers = ["shell"] # Target platforms to build apps for (Rust target-triple syntax) -targets = ["aarch64-unknown-linux-musl", "x86_64-unknown-linux-musl"] +targets = [ + "aarch64-unknown-linux-musl", + "x86_64-unknown-linux-musl", + # macOS targets (Apple Silicon) + "aarch64-apple-darwin", +] # The archive format to use for non-windows builds (defaults .tar.xz) unix-archive = ".tar.gz" # Which actions to run on pull requests @@ -108,3 +113,5 @@ install-updater = false [workspace.metadata.dist.github-custom-runners] aarch64-unknown-linux-musl = "buildjet-2vcpu-ubuntu-2204-arm" +# Use GitHub-hosted macOS runners for apple-darwin targets by default +aarch64-apple-darwin = "macos-latest" diff --git a/scripts/simple-installer.sh b/scripts/simple-installer.sh new file mode 100644 index 00000000..e9a25466 --- /dev/null +++ b/scripts/simple-installer.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# Simple installer for codspeed-runner +# This script clones the repository (or uses the current directory), builds the +# `codspeed` binary with cargo in release mode, and installs it to the target +# directory (default: /usr/local/bin). It intentionally avoids cargo-dist and +# the GitHub release flow so you can build and install locally or from CI. + +set -euo pipefail + +REPO_URL="https://github.com/jzombie/codspeed-runner.git" +REF="main" +INSTALL_DIR="/usr/local/bin" +TMP_DIR="" +NO_RUSTUP="false" +QUIET="false" + +usage() { + cat <] [--ref ] [--install-dir ] [--no-rustup] [--quiet] + +Options: + --repo Git repository URL (default: ${REPO_URL}) + --ref Git ref to checkout (branch, tag, or commit). Default: ${REF} + --install-dir Where to install the built binary. Default: ${INSTALL_DIR} + --no-rustup Do not attempt to install rustup if cargo is missing + --quiet Minimize output + -h, --help Show this help message + +Example: + curl -fsSL https://example.com/codspeed-runner-installer.sh | bash -s -- --ref feature/my-branch + +This script will clone the repository to a temporary directory, build the +`codspeed` binary with `cargo build --release`, and copy it to +${INSTALL_DIR}. Sudo may be used to write to the install directory. +EOF +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --repo) + REPO_URL="$2"; shift 2;; + --ref) + REF="$2"; shift 2;; + --install-dir) + INSTALL_DIR="$2"; shift 2;; + --no-rustup) + NO_RUSTUP="true"; shift 1;; + --quiet) + QUIET="true"; shift 1;; + -h|--help) + usage; exit 0;; + --) + shift; break;; + *) + echo "Unknown argument: $1" >&2; usage; exit 1;; + esac +done + +log() { + if [ "$QUIET" != "true" ]; then + echo "$@" + fi +} + +fail() { + echo "Error: $@" >&2 + exit 1 +} + +cleanup() { + if [ -n "$TMP_DIR" ] && [ -d "$TMP_DIR" ]; then + rm -rf "$TMP_DIR" + fi +} +trap cleanup EXIT + +check_command() { + command -v "$1" >/dev/null 2>&1 +} + +ensure_rust() { + if check_command cargo; then + log "Found cargo" + return 0 + fi + + if [ "$NO_RUSTUP" = "true" ]; then + fail "cargo is not installed and --no-rustup was passed. Install Rust toolchain first."; + fi + + log "Rust toolchain not found. Installing rustup (non-interactive)..." + # Install rustup non-interactively + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y || fail "failed to install rustup" + export PATH="$HOME/.cargo/bin:$PATH" + check_command cargo || fail "cargo still not available after rustup install" +} + +main() { + ensure_rust + + # Create temp dir + TMP_DIR=$(mktemp -d -t codspeed-installer-XXXX) + log "Using temporary directory: $TMP_DIR" + + # Clone the requested ref + log "Cloning ${REPO_URL} (ref: ${REF})..." + git clone --depth 1 --branch "$REF" "$REPO_URL" "$TMP_DIR" || { + # Try cloning default branch and then checking out ref (for commit-ish refs) + log "Shallow clone failed for ref $REF, attempting full clone and checkout" + rm -rf "$TMP_DIR" + TMP_DIR=$(mktemp -d -t codspeed-installer-XXXX) + git clone "$REPO_URL" "$TMP_DIR" || fail "failed to clone repo" + (cd "$TMP_DIR" && git fetch --all --tags && git checkout "$REF") || fail "failed to checkout ref $REF" + } + + # Build + log "Building codspeed (release)..." + (cd "$TMP_DIR" && cargo build --release) || fail "cargo build failed" + + # Locate built binary + BIN_PATH="$TMP_DIR/target/release/codspeed" + if [ ! -x "$BIN_PATH" ]; then + fail "Built binary not found at $BIN_PATH" + fi + + # Ensure install dir exists + if [ ! -d "$INSTALL_DIR" ]; then + log "Creating install directory $INSTALL_DIR" + mkdir -p "$INSTALL_DIR" || fail "failed to create install dir" + fi + + # Copy binary (use sudo if required) + DEST="$INSTALL_DIR/codspeed" + if [ -w "$INSTALL_DIR" ]; then + cp "$BIN_PATH" "$DEST" || fail "failed to copy binary to $DEST" + else + log "Installing to $DEST with sudo" + sudo cp "$BIN_PATH" "$DEST" || fail "sudo copy failed" + fi + + log "Installed codspeed to $DEST" + log "Run 'codspeed --help' to verify" +} + +main "$@" diff --git a/src/run/check_system.rs b/src/run/check_system.rs index efe7a29e..edbd5a99 100644 --- a/src/run/check_system.rs +++ b/src/run/check_system.rs @@ -135,7 +135,14 @@ pub fn check_system(system_info: &SystemInfo) -> Result<()> { return Ok(()); } - match system_info.arch.as_str() { + // Normalize common architecture strings (macOS reports `arm64` on Apple + // Silicon; treat it as `aarch64` which the rest of the codebase expects). + let arch = match system_info.arch.as_str() { + "arm64" => "aarch64", + other => other, + }; + + match arch { "x86_64" | "aarch64" => { warn!( "Unofficially supported system: {} {}. Continuing with best effort support.", diff --git a/src/run/runner/mod.rs b/src/run/runner/mod.rs index 978493b6..45686d44 100644 --- a/src/run/runner/mod.rs +++ b/src/run/runner/mod.rs @@ -31,7 +31,20 @@ pub const EXECUTOR_TARGET: &str = "executor"; pub fn get_executor_from_mode(mode: &RunnerMode) -> Box { match mode { - RunnerMode::Instrumentation => Box::new(ValgrindExecutor), + RunnerMode::Instrumentation => { + // Valgrind/Callgrind is not available on macOS (notably arm64 macOS). + // If the user requested Instrumentation mode on macOS, fall back to + // the WallTime executor so the produced archive and upload metadata + // accurately reflect what was collected (no Callgrind profiles). + #[cfg(target_os = "macos")] + { + Box::new(WallTimeExecutor::new()) + } + #[cfg(not(target_os = "macos"))] + { + Box::new(ValgrindExecutor) + } + } RunnerMode::Walltime => Box::new(WallTimeExecutor::new()), } } diff --git a/src/run/runner/tests.rs b/src/run/runner/tests.rs index 36316216..9ae55c9f 100644 --- a/src/run/runner/tests.rs +++ b/src/run/runner/tests.rs @@ -6,6 +6,7 @@ use crate::run::runner::valgrind::executor::ValgrindExecutor; use crate::run::{RunnerMode, runner::wall_time::executor::WallTimeExecutor}; use rstest_reuse::{self, *}; use shell_quote::{Bash, QuoteRefExt}; +use std::fs; use tempfile::TempDir; use tokio::sync::{OnceCell, Semaphore, SemaphorePermit}; @@ -123,6 +124,13 @@ async fn create_test_setup() -> (SystemInfo, RunData, TempDir) { let system_info = SystemInfo::new().unwrap(); let temp_dir = TempDir::new().unwrap(); + let walltime_dir = temp_dir + .path() + .join("target") + .join("codspeed") + .join("walltime"); + fs::create_dir_all(&walltime_dir).unwrap(); + fs::write(walltime_dir.join(".placeholder"), b"codspeed").unwrap(); let run_data = RunData { profile_folder: temp_dir.path().to_path_buf(), }; @@ -223,10 +231,11 @@ mod walltime { #[rstest::rstest] #[tokio::test] async fn test_walltime_executor(#[case] cmd: &str, #[values(false, true)] enable_perf: bool) { - let (system_info, run_data, _temp_dir) = create_test_setup().await; + let (system_info, run_data, temp_dir) = create_test_setup().await; let (_permit, executor) = get_walltime_executor().await; - let config = walltime_config(cmd, enable_perf); + let mut config = walltime_config(cmd, enable_perf); + config.working_directory = Some(temp_dir.path().to_string_lossy().into_owned()); executor .run(&config, &system_info, &run_data, &None) .await @@ -240,17 +249,21 @@ mod walltime { #[case] env_case: (&str, &str), #[values(false, true)] enable_perf: bool, ) { - let (system_info, run_data, _temp_dir) = create_test_setup().await; + let (system_info, run_data, temp_dir) = create_test_setup().await; let (_permit, executor) = get_walltime_executor().await; let (env_var, env_value) = env_case; - temp_env::async_with_vars(&[(env_var, Some(env_value))], async { - let cmd = env_var_validation_script(env_var, env_value); - let config = walltime_config(&cmd, enable_perf); - executor - .run(&config, &system_info, &run_data, &None) - .await - .unwrap(); + temp_env::async_with_vars(&[(env_var, Some(env_value))], { + let workspace = temp_dir.path().to_path_buf(); + async move { + let cmd = env_var_validation_script(env_var, env_value); + let mut config = walltime_config(&cmd, enable_perf); + config.working_directory = Some(workspace.to_string_lossy().into_owned()); + executor + .run(&config, &system_info, &run_data, &None) + .await + .unwrap(); + } }) .await; } diff --git a/src/run/runner/valgrind/executor.rs b/src/run/runner/valgrind/executor.rs index 13dbbd67..a5f7cd96 100644 --- a/src/run/runner/valgrind/executor.rs +++ b/src/run/runner/valgrind/executor.rs @@ -18,7 +18,18 @@ impl Executor for ValgrindExecutor { } async fn setup(&self, system_info: &SystemInfo) -> Result<()> { - install_valgrind(system_info).await?; + // Valgrind / Callgrind is not supported on macOS (notably arm64 macOS). + // Instead of failing fast, allow the executor to run but skip installing + // Valgrind. The measure implementation contains a macOS fallback that + // runs the benchmark without instrumentation so users can still run + // benchmarks locally on macOS. + if cfg!(target_os = "macos") { + warn!( + "Valgrind/Callgrind is not supported on macOS: skipping Valgrind installation. Benchmarks will run without instrumentation." + ); + } else { + install_valgrind(system_info).await?; + } if let Err(error) = venv_compat::symlink_libpython(None) { warn!("Failed to symlink libpython"); @@ -35,7 +46,16 @@ impl Executor for ValgrindExecutor { run_data: &RunData, mongo_tracer: &Option, ) -> Result<()> { - //TODO: add valgrind version check + // On macOS, callgrind is not available. Let the measure function handle + // the macOS fallback (it will run the benchmark without instrumentation) + // so users can still run benchmarks locally. On non-macOS platforms we + // proceed with the regular Valgrind-based instrumentation. + // TODO: add valgrind version check for non-macOS platforms + if cfg!(target_os = "macos") { + info!( + "Running Valgrind executor on macOS: benchmarks will run without Callgrind instrumentation." + ); + } measure::measure(config, &run_data.profile_folder, mongo_tracer).await?; Ok(()) diff --git a/src/run/runner/valgrind/measure.rs b/src/run/runner/valgrind/measure.rs index 460010a2..0f70a9c5 100644 --- a/src/run/runner/valgrind/measure.rs +++ b/src/run/runner/valgrind/measure.rs @@ -79,6 +79,69 @@ pub async fn measure( profile_folder: &Path, mongo_tracer: &Option, ) -> Result<()> { + // valgrind (callgrind) is a Linux-only tool and is not available on macOS + // (notably arm64 macOS). On macOS we fall back to running the benchmark + // without instrumentation so users can still run benchmarks locally. + if cfg!(target_os = "macos") { + warn!( + "Valgrind/Callgrind is not available on macOS: running the benchmark without instrumentation. Results will not include callgrind profiles." + ); + + // Create the wrapper script and status file + let script_path = create_run_script()?; + let cmd_status_path = tempfile::NamedTempFile::new()?.into_temp_path(); + + // Prepare the command that will execute the benchmark wrapper + let bench_cmd = get_bench_command(config)?; + let mut cmd = Command::new(script_path.to_str().unwrap()); + cmd.args([bench_cmd.as_str(), cmd_status_path.to_str().unwrap()]); + + // Configure the environment similar to other runners, but use Walltime + // mode since we don't have instrumentation available. + cmd.envs(get_base_injected_env(RunnerMode::Walltime, profile_folder)) + .env("PYTHONMALLOC", "malloc") + .env( + "PATH", + format!( + "{}:{}:{}", + introspected_nodejs::setup() + .map_err(|e| anyhow!("failed to setup NodeJS introspection. {e}"))? + .to_string_lossy(), + introspected_golang::setup() + .map_err(|e| anyhow!("failed to setup Go introspection. {e}"))? + .to_string_lossy(), + env::var("PATH").unwrap_or_default(), + ), + ); + + if let Some(cwd) = &config.working_directory { + let abs_cwd = canonicalize(cwd)?; + cmd.current_dir(abs_cwd); + } + + if let Some(mongo_tracer) = mongo_tracer { + mongo_tracer.apply_run_command_transformations(&mut cmd)?; + } + + debug!("cmd: {cmd:?}"); + let _status = run_command_with_log_pipe(cmd) + .await + .map_err(|e| anyhow!("failed to execute the benchmark process. {e}"))?; + + // Check the exit code which was written to the file by the wrapper script. + let cmd_status = { + let content = std::fs::read_to_string(&cmd_status_path)?; + content + .parse::() + .map_err(|e| anyhow!("unable to retrieve the program exit code. {e}"))? + }; + debug!("Program exit code = {cmd_status}"); + if cmd_status != 0 { + bail!("failed to execute the benchmark process, exit code: {cmd_status}"); + } + + return Ok(()); + } // Create the command let mut cmd = Command::new("setarch"); cmd.arg(ARCH).arg("-R"); diff --git a/src/run/runner/wall_time/executor.rs b/src/run/runner/wall_time/executor.rs index 8c6d9854..42acdd6c 100644 --- a/src/run/runner/wall_time/executor.rs +++ b/src/run/runner/wall_time/executor.rs @@ -17,6 +17,25 @@ use std::path::{Path, PathBuf}; use std::process::Command; use tempfile::NamedTempFile; +fn shell_quote(value: &str) -> String { + if value.is_empty() { + "''".to_string() + } else if !value.contains('\'') { + format!("'{value}'") + } else { + let mut quoted = String::with_capacity(value.len() + 2); + quoted.push('\''); + for (idx, part) in value.split('\'').enumerate() { + if idx > 0 { + quoted.push_str("'\"'\"'"); + } + quoted.push_str(part); + } + quoted.push('\''); + quoted + } +} + struct HookScriptsGuard { post_bench_script: PathBuf, } @@ -101,17 +120,23 @@ impl WallTimeExecutor { ) -> Result<(NamedTempFile, NamedTempFile, String)> { let bench_cmd = get_bench_command(config)?; - let system_env = get_exported_system_env()?; + let use_systemd = cfg!(target_os = "linux") + && std::process::Command::new("which") + .arg("systemd-run") + .output() + .map(|o| o.status.success()) + .unwrap_or(false); + let base_injected_env = get_base_injected_env(RunnerMode::Walltime, &run_data.profile_folder) .into_iter() - .map(|(k, v)| format!("export {k}={v}",)) + .map(|(k, v)| format!("export {k}={}", shell_quote(&v))) .collect::>() .join("\n"); let path_env = std::env::var("PATH").unwrap_or_default(); - let path_env = format!( - "export PATH={}:{}:{}", + let path_value = format!( + "{}:{}:{}", introspected_nodejs::setup() .map_err(|e| anyhow!("failed to setup NodeJS introspection. {e}"))? .to_string_lossy(), @@ -120,8 +145,14 @@ impl WallTimeExecutor { .to_string_lossy(), path_env ); + let path_env = format!("export PATH={}", shell_quote(&path_value)); - let combined_env = format!("{system_env}\n{base_injected_env}\n{path_env}"); + let combined_env = if use_systemd { + let system_env = get_exported_system_env()?; + format!("{system_env}\n{base_injected_env}\n{path_env}") + } else { + format!("{base_injected_env}\n{path_env}") + }; let mut env_file = NamedTempFile::new()?; env_file.write_all(combined_env.as_bytes())?; @@ -148,10 +179,20 @@ impl WallTimeExecutor { // - We have to pass the environment variables because `--scope` only inherits the system and not the user environment variables. let uid = nix::unistd::Uid::current().as_raw(); let gid = nix::unistd::Gid::current().as_raw(); - let cmd = format!( - "systemd-run {quiet_flag} --scope --slice=codspeed.slice --same-dir --uid={uid} --gid={gid} -- bash {}", - script_file.path().display() - ); + // Prefer using systemd-run on Linux hosts when available since it + // provides the `--scope` isolation we need. On macOS (or when + // systemd isn't installed) fall back to invoking `bash