feat(test): add execution benchmarks (#1556)

Closes INT-3706

- Moved existing benchmarks to `benchmarks/prove` crate
- Added a new `benchmarks/execute` binary crate that runs execution for
a list of guest programs
- Added programs for benchmarking in `benchmarks/guest`
- Added a ci workflow to run execution benchmarks
- Note: Timing metrics for summary are currently being parsed from logs.
I'll change it to use proper metrics in a follow up

Author: shuklaayush

.github/workflows/benchmarks-execute.yml

@@ -0,0 +1,112 @@
+name: "benchmarks-execute"
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+    branches: ["**"]
+    paths:
+      - "benchmarks/execute/**"
+      - "crates/circuits/**"
+      - "crates/toolchain/**"
+      - "crates/prof/**"
+      - "crates/sdk/**"
+      - "crates/vm/**"
+      - "extensions/**"
+      - "Cargo.toml"
+      - ".github/workflows/benchmarks-execute.yml"
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  execute-benchmarks:
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - runner=8cpu-linux-x64
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Run execution benchmarks
+        working-directory: benchmarks/execute
+        run: cargo run | tee benchmark_output.log
+
+      - name: Parse benchmark results
+        working-directory: benchmarks/execute
+        run: |
+          # Determine if running in GitHub Actions environment
+          if [ -n "$GITHUB_STEP_SUMMARY" ]; then
+            SUMMARY_FILE="$GITHUB_STEP_SUMMARY"
+            echo "### Benchmark Results Summary" >> "$SUMMARY_FILE"
+          else
+            SUMMARY_FILE="benchmark_summary.md"
+            echo "### Benchmark Results Summary" > "$SUMMARY_FILE"
+            echo "Saving summary to $SUMMARY_FILE"
+          fi
+
+          # Set up summary table header
+          echo "| Program | Total Time (ms) |" >> "$SUMMARY_FILE"
+          echo "| ------- | --------------- |" >> "$SUMMARY_FILE"
+
+          # Variables to track current program and total time
+          current_program=""
+          total_time=0
+
+          # Process the output file line by line
+          while IFS= read -r line; do
+            # Check if line contains "Running program" message
+            if [[ $line =~ ｉ\ \[info\]:\ Running\ program:\ ([a-zA-Z0-9_-]+) ]]; then
+              # If we were processing a program, output its results
+              if [[ -n "$current_program" ]]; then
+                echo "| $current_program | $total_time |" >> "$SUMMARY_FILE"
+              fi
+
+              # Start tracking new program
+              current_program="${BASH_REMATCH[1]}"
+              total_time=0
+            fi
+
+            # Check for program completion to catch programs that might have no execution segments
+            if [[ $line =~ ｉ\ \[info\]:\ Completed\ program:\ ([a-zA-Z0-9_-]+) ]]; then
+              completed_program="${BASH_REMATCH[1]}"
+              # If no segments were found for this program, ensure it's still in the output
+              if [[ "$current_program" == "$completed_program" && $total_time == 0 ]]; then
+                echo "| $current_program | 0 |" >> "$SUMMARY_FILE"
+                current_program=""
+              fi
+            fi
+
+            # Check if line contains execution time (looking for the format with ms or s)
+            if [[ $line =~ execute_segment\ \[\ ([0-9.]+)(ms|s)\ \|\ [0-9.]+%\ \]\ segment ]]; then
+              segment_time="${BASH_REMATCH[1]}"
+              unit="${BASH_REMATCH[2]}"
+
+              # Convert to milliseconds if in seconds
+              if [[ "$unit" == "s" ]]; then
+                segment_time=$(echo "scale=6; $segment_time * 1000" | bc)
+              fi
+
+              # Add segment time to total
+              total_time=$(echo "scale=6; $total_time + $segment_time" | bc)
+            fi
+          done < benchmark_output.log
+
+          # Output the last program result if there was one
+          if [[ -n "$current_program" ]]; then
+            echo "| $current_program | $total_time |" >> "$SUMMARY_FILE"
+          fi
+
+          # If not in GitHub Actions, print the summary to the terminal
+          if [ -z "$GITHUB_STEP_SUMMARY" ]; then
+            echo -e "\nBenchmark Summary:"
+            cat "$SUMMARY_FILE"
+          fi

.github/workflows/benchmarks.yml

@@ -7,7 +7,7 @@ on:
     types: [opened, synchronize, reopened, labeled]
     branches: ["**"]
     paths:
-      - "benchmarks/**"
+      - "benchmarks/prove/**"
       - "crates/circuits/**"
       - "crates/toolchain/**"
       - "crates/prof/**"

.gitignore

@@ -33,3 +33,6 @@ metrics.json
 # Profiling
 **/flamegraph.svg
 **/profile.json
+
+# openvm generated files
+crates/cli/openvm/

Cargo.lock

@@ -9,7 +9,9 @@ license = "MIT"
 
 [workspace]
 members = [
-    "benchmarks",
+    "benchmarks/utils",
+    "benchmarks/execute",
+    "benchmarks/prove",
     "crates/prof",
     "crates/sdk",
     "crates/cli",
@@ -127,6 +129,7 @@ openvm-custom-insn = { path = "crates/toolchain/custom_insn", default-features =
 openvm-circuit = { path = "crates/vm", default-features = false }
 openvm-circuit-derive = { path = "crates/vm/derive", default-features = false }
 openvm-continuations = { path = "crates/continuations", default-features = false }
+cargo-openvm = { path = "crates/cli", default-features = false }
 
 # Extensions
 openvm-rv32im-circuit = { path = "extensions/rv32im/circuit", default-features = false }
@@ -159,6 +162,9 @@ openvm-pairing-circuit = { path = "extensions/pairing/circuit", default-features
 openvm-pairing-transpiler = { path = "extensions/pairing/transpiler", default-features = false }
 openvm-pairing-guest = { path = "extensions/pairing/guest", default-features = false }
 
+# Benchmarking
+openvm-benchmarks-utils = { path = "benchmarks/utils", default-features = false }
+
 # Plonky3
 p3-field = { git = "https://github.com/Plonky3/Plonky3.git", rev = "1ba4e5c" }
 p3-baby-bear = { git = "https://github.com/Plonky3/Plonky3.git", features = [
@@ -180,6 +186,7 @@ snark-verifier-sdk = { version = "0.2.0", default-features = false, features = [
 ] }
 halo2curves-axiom = "0.7.0"
 
+cargo_metadata = "0.18"
 tracing = "0.1.40"
 bon = "3.2.0"
 serde_json = "1.0.117"

Cargo.toml

@@ -0,0 +1,50 @@
+[package]
+name = "openvm-benchmarks-execute"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+
+[dependencies]
+openvm-benchmarks-utils.workspace = true
+cargo-openvm.workspace = true
+openvm-circuit.workspace = true
+openvm-sdk.workspace = true
+openvm-stark-sdk.workspace = true
+openvm-transpiler.workspace = true
+openvm-rv32im-circuit.workspace = true
+openvm-rv32im-transpiler.workspace = true
+openvm-keccak256-circuit.workspace = true
+openvm-keccak256-transpiler.workspace = true
+
+clap = { version = "4.5.9", features = ["derive", "env"] }
+eyre.workspace = true
+tracing.workspace = true
+derive_more = { workspace = true, features = ["from"] }
+
+tracing-subscriber = { version = "0.3.17", features = ["std", "env-filter"] }
+
+[dev-dependencies]
+criterion = { version = "0.5", features = ["html_reports"] }
+
+[features]
+default = ["mimalloc"]
+profiling = ["openvm-sdk/profiling"]
+aggregation = []
+mimalloc = ["openvm-circuit/mimalloc"]
+jemalloc = ["openvm-circuit/jemalloc"]
+jemalloc-prof = ["openvm-circuit/jemalloc-prof"]
+nightly-features = ["openvm-circuit/nightly-features"]
+
+[[bench]]
+name = "fibonacci_execute"
+harness = false
+
+[[bench]]
+name = "regex_execute"
+harness = false
+
+[package.metadata.cargo-shear]
+ignored = ["derive_more"]

benchmarks/execute/Cargo.toml

@@ -1,5 +1,5 @@
 use criterion::{criterion_group, criterion_main, Criterion};
-use openvm_benchmarks::utils::{build_bench, get_programs_dir};
+use openvm_benchmarks_utils::{build_elf, get_programs_dir};
 use openvm_circuit::arch::{instructions::exe::VmExe, VmExecutor};
 use openvm_rv32im_circuit::Rv32ImConfig;
 use openvm_rv32im_transpiler::{
@@ -10,7 +10,9 @@ use openvm_stark_sdk::p3_baby_bear::BabyBear;
 use openvm_transpiler::{transpiler::Transpiler, FromElf};
 
 fn benchmark_function(c: &mut Criterion) {
-    let elf = build_bench(get_programs_dir().join("fibonacci"), "release").unwrap();
+    let program_dir = get_programs_dir().join("fibonacci");
+    let elf = build_elf(&program_dir, "release").unwrap();
+
     let exe = VmExe::from_elf(
         elf,
         Transpiler::<BabyBear>::default()

benchmarks/benches/fibonacci_execute.rs renamed to benchmarks/execute/benches/fibonacci_execute.rs

@@ -1,5 +1,5 @@
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use openvm_benchmarks::utils::{build_bench, get_programs_dir};
+use openvm_benchmarks_utils::{build_elf, get_programs_dir};
 use openvm_circuit::arch::{instructions::exe::VmExe, VmExecutor};
 use openvm_keccak256_circuit::Keccak256Rv32Config;
 use openvm_keccak256_transpiler::Keccak256TranspilerExtension;
@@ -11,7 +11,9 @@ use openvm_stark_sdk::p3_baby_bear::BabyBear;
 use openvm_transpiler::{transpiler::Transpiler, FromElf};
 
 fn benchmark_function(c: &mut Criterion) {
-    let elf = build_bench(get_programs_dir().join("regex"), "release").unwrap();
+    let program_dir = get_programs_dir().join("regex");
+    let elf = build_elf(&program_dir, "release").unwrap();
+
     let exe = VmExe::from_elf(
         elf,
         Transpiler::<BabyBear>::default()
@@ -27,7 +29,7 @@ fn benchmark_function(c: &mut Criterion) {
     let config = Keccak256Rv32Config::default();
     let executor = VmExecutor::<BabyBear, Keccak256Rv32Config>::new(config);
 
-    let data = include_str!("../programs/regex/regex_email.txt");
+    let data = include_str!("../../guest/regex/regex_email.txt");
 
     let fe_bytes = data.to_owned().into_bytes();
     group.bench_function("execute", |b| {

benchmarks/benches/regex_execute.rs renamed to benchmarks/execute/benches/regex_execute.rs

@@ -25,7 +25,7 @@ fn main() {
     let config = Keccak256Rv32Config::default();
     let executor = VmExecutor::<BabyBear, Keccak256Rv32Config>::new(config);
 
-    let data = include_str!("../programs/regex/regex_email.txt");
+    let data = include_str!("../../guest/regex/regex_email.txt");
 
     let timer = std::time::Instant::now();
     executor