fix: Micro benchmark flamegraph (#1598)

Fix micro benchmarks after circuit flamegraph changes

Author: manh9203

.github/workflows/benchmark-call.yml

@@ -106,6 +106,7 @@ on:
 
 env:
   S3_METRICS_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/github/metrics
+  S3_FLAMEGRAPHS_PATH: s3://openvm-public-data-sandbox-us-east-1/benchmark/github/flamegraphs
   FEATURE_FLAGS: "bench-metrics,parallel,nightly-features"
   INPUT_ARGS: ""
   CARGO_NET_GIT_FETCH_WITH_CLI: "true"
@@ -228,16 +229,25 @@ jobs:
 
           s5cmd cp $METRIC_PATH ${{ env.S3_METRICS_PATH }}/${METRIC_NAME}-${current_sha}.json
 
-      # - name: Install inferno-flamegraph
-      #   run: cargo install inferno
+      - name: Install inferno-flamegraph
+        if: ${{ contains(env.FEATURE_FLAGS, 'profiling') }}
+        run: cargo install inferno
 
-      # - name: Generate flamegraphs
-      #   run: |
-      #     if [[ -f $METRIC_PATH ]]; then
-      #       python3 ci/scripts/metric_unify/flamegraph.py $METRIC_PATH
-      #       s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.PUBLIC_S3_PATH }}/${current_sha}/"
-      #       echo "UPLOAD_FLAMEGRAPHS=1" >> $GITHUB_ENV
-      #     fi
+      - name: Generate flamegraphs
+        if: ${{ contains(env.FEATURE_FLAGS, 'profiling') }}
+        run: |
+          if [[ -f $METRIC_PATH ]]; then
+            GUEST_SYMBOLS_PATH="${METRIC_PATH%.json}.syms"
+            if [[ -f $GUEST_SYMBOLS_PATH ]]; then
+              echo "Generating flamegraphs with guest symbols"
+              python3 ci/scripts/metric_unify/flamegraph.py $METRIC_PATH --guest-symbols $GUEST_SYMBOLS_PATH
+            else
+              echo "No guest symbols found, generating flamegraphs without symbols"
+              python3 ci/scripts/metric_unify/flamegraph.py $METRIC_PATH
+            fi
+            s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.S3_FLAMEGRAPHS_PATH }}/${METRIC_NAME}-${current_sha}/"
+            echo "UPLOAD_FLAMEGRAPHS=1" >> $GITHUB_ENV
+          fi
 
       ##########################################################################
       # Update s3 for latest main metrics upon a push event                    #

ci/scripts/bench.py

@@ -50,6 +50,8 @@ def run_cargo_command(
     # Prepare the environment variables
     env = os.environ.copy()  # Copy current environment variables
     env["OUTPUT_PATH"] = output_path
+    if "profiling" in feature_flags:
+        env["GUEST_SYMBOLS_PATH"] = os.path.splitext(output_path)[0] + ".syms"
     env["RUSTFLAGS"] = "-Ctarget-cpu=native"
 
     # Run the subprocess with the updated environment

ci/scripts/utils.sh

@@ -18,12 +18,6 @@ add_metadata_and_flamegraphs() {
       }')
     echo "inputs: $inputs"
 
-    if [[ "$FLAMEGRAPHS" == 'true' ]]; then
-      repo_root=$(git rev-parse --show-toplevel)
-      python3 ${repo_root}/ci/scripts/metric_unify/flamegraph.py $metric_path
-      s5cmd cp "${repo_root}/.bench_metrics/flamegraphs/*.svg" "${S3_FLAMEGRAPHS_PATH}/${CURRENT_SHA}/"
-    fi
-
     if [ ! -z "$inputs" ]; then
       max_segment_length=$(echo "$inputs" | jq -r '.max_segment_length')
       instance_type=$(echo "$inputs" | jq -r '.instance_type')
@@ -54,13 +48,12 @@ add_metadata() {
         echo "<details>" >> $result_path
         echo "<summary>Flamegraphs</summary>" >> $result_path
         echo "" >> $result_path
-        repo_root=$(git rev-parse --show-toplevel)
-        for file in $repo_root/.bench_metrics/flamegraphs/*.svg; do
-        filename=$(basename "$file")
-            flamegraph_url=https://openvm-public-data-sandbox-us-east-1.s3.us-east-1.amazonaws.com/benchmark/github/flamegraphs/${CURRENT_SHA}/${filename}
+        benchmark_name=$(basename "$result_path" | cut -d'-' -f1)
+        flamegraph_files=$(s5cmd ls ${S3_FLAMEGRAPHS_PATH}/${benchmark_name}-${CURRENT_SHA}/*.svg | awk '{print $4}' | xargs -n1 basename)
+        for file in $flamegraph_files; do
+            flamegraph_url=https://openvm-public-data-sandbox-us-east-1.s3.us-east-1.amazonaws.com/benchmark/github/flamegraphs/${benchmark_name}-${CURRENT_SHA}/${file}
             echo "[![]($flamegraph_url)]($flamegraph_url)" >> $result_path
         done
-        rm -f ${repo_root}/.bench_metrics/flamegraphs/*.svg
         echo "" >> $result_path
         echo "</details>" >> $result_path
         echo "" >> $result_path

crates/toolchain/transpiler/src/elf.rs

@@ -132,7 +132,8 @@ impl Elf {
                     }
                 }
 
-                let guest_symbols_path = std::env::var("GUEST_SYMBOLS_PATH")?;
+                let guest_symbols_path = std::env::var("GUEST_SYMBOLS_PATH")
+                    .map_err(|e| eyre::eyre!("{e}: GUEST_SYMBOLS_PATH"))?;
                 let mut guest_symbols_file =
                     std::fs::File::create(&guest_symbols_path).map_err(|e| {
                         eyre::eyre!(

docs/crates/benchmarks.md

@@ -120,10 +120,10 @@ for more detailed profiling we generate special flamegraphs that visualize VM-sp
 The benchmark must be run with special configuration so that additional metrics are collected for profiling. Note that the additional metric collection will slow down the benchmark. To run a benchmark with the additional profiling, run the following command:
 
 ```bash
-OUTPUT_PATH="metrics.json" cargo run --release --bin <benchmark_name> --features profiling -- --profiling
+OUTPUT_PATH="metrics.json" GUEST_SYMBOLS_PATH="guest.syms" cargo run --release --bin <benchmark_name> --features profiling -- --profiling
 ```
 
-Add `--features aggregation,profiling` to run with leaf aggregation. The `profiling` feature tells the VM to run with additional metric collection. The `--profiling` CLI argument tells the script to build the guest program with `profile=profiling` so that the guest program is compiled without stripping debug symbols.
+Add `--features aggregation,profiling` to run with leaf aggregation. The `profiling` feature tells the VM to run with additional metric collection. The `--profiling` CLI argument tells the script to build the guest program with `profile=profiling` so that the guest program is compiled without stripping debug symbols. When the `profiling` feature is enabled, the `GUEST_SYMBOLS_PATH` environment variable must be set to the file path where function symbols of the guest program will be exported. Those symbols are then used to annotate the flamegraph with function names.
 
 After the collected metrics are written to `$OUTPUT_PATH`, these flamegraphs can be generated if you have [inferno-flamegraph](https://crates.io/crates/inferno) installed. Install via
 
@@ -134,7 +134,7 @@ cargo install inferno
 Then run
 
 ```bash
-python <repo_root>/ci/scripts/metric_unify/flamegraph.py $OUTPUT_PATH
+python <repo_root>/ci/scripts/metric_unify/flamegraph.py $OUTPUT_PATH --guest-symbols $GUEST_SYMBOLS_PATH
 ```
 
 The flamegraphs will be written to `*.svg` files in `.bench_metrics/flamegraphs` with respect to the repo root.