perf: execution and tracegen rewrite

.config/nextest.toml

@@ -0,0 +1,13 @@
+# Nextest configuration for OpenVM project
+
+# Define test groups with different weights
+[[profile.default.overrides]]
+# Match all tests with "persistent" in their name
+filter = 'test(~persistent)'
+# Give these tests 5x the default weight because they use more memory
+threads-required = 16
+
+# custom profile for heavy tests
+[profile.heavy]
+# Run fewer tests in parallel for heavy workloads
+test-threads = 2

.github/workflows/benchmark-call.yml

@@ -49,7 +49,7 @@ on:
       features:
         type: string
         required: false
-        description: Host features, comma separated (aggregation,profiling)
+        description: Host features, comma separated (aggregation,perf-metrics)
   workflow_call:
     inputs:
       benchmark_name:
@@ -102,12 +102,12 @@ on:
       features:
         type: string
         required: false
-        description: Host features, comma separated (aggregation,profiling)
+        description: Host features, comma separated (aggregation,perf-metrics)
 
 env:
   S3_METRICS_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/github/metrics
   S3_FLAMEGRAPHS_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/github/flamegraphs
-  FEATURE_FLAGS: "bench-metrics,parallel,nightly-features"
+  FEATURE_FLAGS: "metrics,parallel,nightly-features"
   INPUT_ARGS: ""
   CARGO_NET_GIT_FETCH_WITH_CLI: "true"
 
@@ -170,7 +170,7 @@ jobs:
             ROOT_ARG="--root_log_blowup ${{ inputs.root_log_blowup }}"
             INTERNAL_ARG="--internal_log_blowup ${{ inputs.internal_log_blowup }}"
             bash ./extensions/native/recursion/trusted_setup_s3.sh
-            PARAMS_DIR=$(pwd)/params
+            PARAMS_DIR=$HOME/.openvm/params
             PARAMS_ARG="--kzg-params-dir $PARAMS_DIR"
             echo "INPUT_ARGS=${ROOT_ARG} ${INTERNAL_ARG} ${PARAMS_ARG} ${INPUT_ARGS}" >> $GITHUB_ENV
           fi
@@ -230,11 +230,11 @@ jobs:
           s5cmd cp $METRIC_PATH ${{ env.S3_METRICS_PATH }}/${METRIC_NAME}-${current_sha}.json
 
       - name: Install inferno-flamegraph
-        if: ${{ contains(env.FEATURE_FLAGS, 'profiling') }}
+        if: ${{ contains(env.FEATURE_FLAGS, 'perf-metrics') }}
         run: cargo install inferno
 
       - name: Generate flamegraphs
-        if: ${{ contains(env.FEATURE_FLAGS, 'profiling') }}
+        if: ${{ contains(env.FEATURE_FLAGS, 'perf-metrics') }}
         run: |
           if [[ -f $METRIC_PATH ]]; then
             GUEST_SYMBOLS_PATH="${METRIC_PATH%.json}.syms"
@@ -250,9 +250,15 @@ jobs:
           fi
 
       ##########################################################################
-      # Update s3 for latest main metrics upon a push event                    #
+      # Update s3 for latest branch metrics upon a push event                  #
       ##########################################################################
-      - name: Update latest main result in s3
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+      - name: Update latest branch result in s3
+        if: github.event_name == 'push'
         run: |
-          s5cmd cp $METRIC_PATH "${{ env.S3_METRICS_PATH }}/main-${METRIC_NAME}.json"
+          if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
+            # for backwards compatibility
+            REF_HASH="main"
+          else
+            REF_HASH=$(echo "${{ github.ref }}" | sha256sum | cut -d' ' -f1)
+          fi
+          s5cmd cp $METRIC_PATH "${{ env.S3_METRICS_PATH }}/${REF_HASH}-${METRIC_NAME}.json"

.github/workflows/benchmarks-execute.yml

@@ -1,8 +1,9 @@
-name: "benchmarks-execute"
+name: "Execution benchmarks"
 
 on:
   push:
-    branches: ["main"]
+    # TODO(ayush): remove after feat/new-execution is merged
+    branches: ["main", "feat/new-execution"]
   pull_request:
     types: [opened, synchronize, reopened, labeled]
     branches: ["**"]
@@ -18,95 +19,101 @@ on:
       - ".github/workflows/benchmarks-execute.yml"
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
 env:
   CARGO_TERM_COLOR: always
+  S3_FIXTURES_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/fixtures
+  JEMALLOC_SYS_WITH_MALLOC_CONF: "retain:true,background_thread:true,metadata_thp:always,thp:always,dirty_decay_ms:10000,muzzy_decay_ms:10000,abort_conf:true"
 
 jobs:
-  execute-benchmarks:
+  codspeed-walltime-benchmarks:
+    name: Run codspeed walltime benchmarks
     runs-on:
       - runs-on=${{ github.run_id }}
-      - runner=8cpu-linux-x64
+      - family=m5a.xlarge # 2.5Ghz clock speed
+      - image=ubuntu24-full-x64
+      - extras=s3-cache
+
+    env:
+      CODSPEED_RUNNER_MODE: walltime
+
     steps:
+      - uses: runs-on/action@v1
       - uses: actions/checkout@v4
-
-      - name: Set up Rust
-        uses: actions-rs/toolchain@v1
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
         with:
-          profile: minimal
-          toolchain: stable
-          override: true
+          cache-on-failure: true
 
-      - name: Run execution benchmarks
-        working-directory: benchmarks/execute
-        run: cargo run | tee benchmark_output.log
+      - name: Install architecture specific tools
+        run: |
+          source ci/scripts/utils.sh
+          install_s5cmd
 
-      - name: Parse benchmark results
+      - name: Pull fixtures from S3
+        run: |
+          mkdir -p benchmarks/fixtures
+          s5cmd cp "${{ env.S3_FIXTURES_PATH }}/*" benchmarks/fixtures/ || echo "No fixtures found in S3"
+
+      - name: Install cargo-binstall
+        uses: cargo-bins/cargo-binstall@main
+      - name: Install codspeed
+        run: cargo binstall --no-confirm --force cargo-codspeed
+
+      - name: Build benchmarks
         working-directory: benchmarks/execute
+        run: cargo codspeed build --profile maxperf
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v3
+        with:
+          working-directory: benchmarks/execute
+          run: cargo codspeed run
+          token: ${{ secrets.CODSPEED_TOKEN }}
+
+  codspeed-instrumentation-benchmarks:
+    name: Run codspeed instrumentation benchmarks
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - family=m5a.xlarge
+      - image=ubuntu24-full-x64
+      - extras=s3-cache
+    if: github.event_name != 'pull_request'
+
+    env:
+      CODSPEED_RUNNER_MODE: instrumentation
+
+    steps:
+      - uses: runs-on/action@v1
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+
+      - name: Install architecture specific tools
         run: |
-          # Determine if running in GitHub Actions environment
-          if [ -n "$GITHUB_STEP_SUMMARY" ]; then
-            SUMMARY_FILE="$GITHUB_STEP_SUMMARY"
-            echo "### Benchmark Results Summary" >> "$SUMMARY_FILE"
-          else
-            SUMMARY_FILE="benchmark_summary.md"
-            echo "### Benchmark Results Summary" > "$SUMMARY_FILE"
-            echo "Saving summary to $SUMMARY_FILE"
-          fi
-
-          # Set up summary table header
-          echo "| Program | Total Time (ms) |" >> "$SUMMARY_FILE"
-          echo "| ------- | --------------- |" >> "$SUMMARY_FILE"
-
-          # Variables to track current program and total time
-          current_program=""
-          total_time=0
-
-          # Process the output file line by line
-          while IFS= read -r line; do
-            # Check if line contains "Running program" message
-            if [[ $line =~ ｉ\ \[info\]:\ Running\ program:\ ([a-zA-Z0-9_-]+) ]]; then
-              # If we were processing a program, output its results
-              if [[ -n "$current_program" ]]; then
-                echo "| $current_program | $total_time |" >> "$SUMMARY_FILE"
-              fi
-
-              # Start tracking new program
-              current_program="${BASH_REMATCH[1]}"
-              total_time=0
-            fi
-
-            # Check for program completion to catch programs that might have no execution segments
-            if [[ $line =~ ｉ\ \[info\]:\ Completed\ program:\ ([a-zA-Z0-9_-]+) ]]; then
-              completed_program="${BASH_REMATCH[1]}"
-              # If no segments were found for this program, ensure it's still in the output
-              if [[ "$current_program" == "$completed_program" && $total_time == 0 ]]; then
-                echo "| $current_program | 0 |" >> "$SUMMARY_FILE"
-                current_program=""
-              fi
-            fi
-
-            # Check if line contains execution time (looking for the format with ms or s)
-            if [[ $line =~ execute_segment\ \[\ ([0-9.]+)(ms|s)\ \|\ [0-9.]+%\ \]\ segment ]]; then
-              segment_time="${BASH_REMATCH[1]}"
-              unit="${BASH_REMATCH[2]}"
-
-              # Convert to milliseconds if in seconds
-              if [[ "$unit" == "s" ]]; then
-                segment_time=$(echo "scale=6; $segment_time * 1000" | bc)
-              fi
-
-              # Add segment time to total
-              total_time=$(echo "scale=6; $total_time + $segment_time" | bc)
-            fi
-          done < benchmark_output.log
-
-          # Output the last program result if there was one
-          if [[ -n "$current_program" ]]; then
-            echo "| $current_program | $total_time |" >> "$SUMMARY_FILE"
-          fi
-
-          # If not in GitHub Actions, print the summary to the terminal
-          if [ -z "$GITHUB_STEP_SUMMARY" ]; then
-            echo -e "\nBenchmark Summary:"
-            cat "$SUMMARY_FILE"
-          fi
+          source ci/scripts/utils.sh
+          install_s5cmd
+
+      - name: Pull fixtures from S3
+        run: |
+          mkdir -p benchmarks/fixtures
+          s5cmd cp "${{ env.S3_FIXTURES_PATH }}/*" benchmarks/fixtures/ || echo "No fixtures found in S3"
+
+      - name: Install cargo-binstall
+        uses: cargo-bins/cargo-binstall@main
+      - name: Install codspeed
+        run: cargo binstall --no-confirm --force cargo-codspeed
+
+      - name: Build benchmarks
+        working-directory: benchmarks/execute
+        run: cargo codspeed build
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v3
+        with:
+          working-directory: benchmarks/execute
+          run: cargo codspeed run
+          token: ${{ secrets.CODSPEED_TOKEN }}

.github/workflows/benchmarks-upload-fixtures.yml

@@ -0,0 +1,42 @@
+name: "Upload benchmark fixtures"
+
+on:
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+  S3_FIXTURES_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/fixtures
+
+jobs:
+  generate-fixtures:
+    name: Generate and upload benchmark fixtures
+    runs-on:
+      - runs-on=${{ github.run_id }}
+      - runner=64cpu-linux-arm64
+      - family=m7
+      - extras=s3-cache
+
+    steps:
+      - uses: runs-on/action@v1
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+
+      - name: Install architecture specific tools
+        run: |
+          source ci/scripts/utils.sh
+          install_s5cmd
+
+      - name: Generate fixtures
+        run: cargo r -r --bin generate-fixtures --features generate-fixtures
+
+      - name: Upload fixtures to S3
+        run: |
+          if [ -d "benchmarks/fixtures" ]; then
+            s5cmd cp benchmarks/fixtures/ ${{ env.S3_FIXTURES_PATH }}/
+          else
+            echo "No fixtures directory found"
+            exit 1
+          fi

.github/workflows/benchmarks.yml

@@ -2,7 +2,7 @@ name: "OpenVM Benchmarks: Coordinate Runner & Reporting"
 
 on:
   push:
-    branches: ["main"]
+    branches: ["main", "feat/new-execution"]
   pull_request:
     types: [opened, synchronize, reopened, labeled]
     branches: ["**"]
@@ -89,7 +89,7 @@ jobs:
             FEATURE_FLAGS="aggregation,${FEATURE_FLAGS}"
           fi
           if [[ "${{ github.event.inputs.flamegraphs }}" == "true" ]]; then
-            FEATURE_FLAGS="profiling,${FEATURE_FLAGS}"
+            FEATURE_FLAGS="perf-metrics,${FEATURE_FLAGS}"
           fi
 
           matrix=$(jq -c --argjson run_e2e $RUN_E2E --arg features "$FEATURE_FLAGS" '
@@ -211,9 +211,21 @@ jobs:
           json_file_list=$(echo -n "$json_files" | paste -sd "," -)
           echo $json_file_list
 
-          prev_json_files=$(echo $matrix | jq -r '
+          # For PRs, get the latest commit from the target branch
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            if [[ "${{ github.base_ref }}" == "main" ]]; then
+              REF_HASH="main"
+            else
+              REF_HASH=$(echo "refs/heads/${{ github.base_ref }}" | sha256sum | cut -d' ' -f1)
+            fi
+            echo "Target branch REF_HASH: $REF_HASH"
+          else
+            REF_HASH="main"
+          fi
+
+          prev_json_files=$(echo $matrix | jq -r --arg target "$REF_HASH" '
             .[] |
-            "main-\(.id).json"')
+            "\($target)-\(.id).json"')
           prev_json_file_list=$(echo -n "$prev_json_files" | paste -sd "," -)
           echo $prev_json_file_list
 

.github/workflows/build.yml

@@ -16,6 +16,7 @@ jobs:
     runs-on:
       - runs-on=${{ github.run_id }}
       - runner=64cpu-linux-arm64
+      - image=ubuntu24-full-arm64
       - extras=s3-cache
     steps:
       - uses: runs-on/action@v1