feat(bb): --print_bench (#16673)

While the pieces were there before, I would say the culmination of them amounts to a new feature. Now if you do --print_bench or BB_BENCH=1 on a bb or a test (or even a bench to get clearer numbers) you will get a listing like below

Author: ludamad

.vscode/launch.json

@@ -36,8 +36,8 @@
       "name": "Debug CIVC transaction",
       "type": "lldb",
       "request": "launch",
-      "program": "${workspaceFolder}/barretenberg/cpp/build-debug-no-avm/bin/bb",
-      "args": ["prove", "--scheme", "client_ivc", "--output_path", ".", "--ivc_inputs_path", "ivc-inputs.msgpack"],
+      "program": "${workspaceFolder}/barretenberg/cpp/build-debug-fast-no-avm/bin/bb",
+      "args": ["prove", "--scheme", "client_ivc", "--output_path", ".", "--ivc_inputs_path", "ivc-inputs.msgpack", "--print_bench"],
       "cwd": "${workspaceFolder}/yarn-project/end-to-end/example-app-ivc-inputs-out/ecdsar1+transfer_0_recursions+sponsored_fpc",
       "initCommands": [
         "command script import ${workspaceFolder}/barretenberg/cpp/scripts/lldb_format.py"

barretenberg/README.md

@@ -450,7 +450,7 @@ In terms of general usage, you should be able to use scrolling or the WASD keys
 
 ##### Adding Zones
 
-Zones are how you can keep track of where you are relative in the code and how you can bucket allocations together. All of the colored blocks in the Main Thread row and other threads' rows refer to zones. You can nest zones in deeper and deeper scopes, which leads to stacks of these zones. To add a named zone, all you have to do is add PROFILE_THIS() or PROFILE_THIS_NAME(<name>) to a scope and it will create a zone. Note that you can't create multiple zones in the same scope.
+Zones are how you can keep track of where you are relative in the code and how you can bucket allocations together. All of the colored blocks in the Main Thread row and other threads' rows refer to zones. You can nest zones in deeper and deeper scopes, which leads to stacks of these zones. To add a named zone, all you have to do is add BB_BENCH_TRACY() or BB_BENCH_TRACY_NAME(<name>) to a scope and it will create a zone. Note that you can't create multiple zones in the same scope.
 
 ##### Analyzing Fragmentation
 

barretenberg/cpp/bootstrap.sh

@@ -308,12 +308,13 @@ case "$cmd" in
     export AZTEC_CACHE_COMMIT=$commit_hash
     # TODO currently does nothing! to reinstate in cache_download
     export FORCE_CACHE_DOWNLOAD=${FORCE_CACHE_DOWNLOAD:-1}
-    BOOTSTRAP_AFTER=barretenberg BOOSTRAP_TO=yarn-project ../../bootstrap.sh
+    # make sure that disabling the aztec VM does not interfere with cache results from CI.
+    DISABLE_AZTEC_VM="" BOOTSTRAP_AFTER=barretenberg BOOSTRAP_TO=yarn-project ../../bootstrap.sh
 
     rm -rf bench-out
 
     # Recreation of logic from bench.
-    ../../yarn-project/end-to-end/bootstrap.sh build_bench
+    DISABLE_AZTEC_VM="" ../../yarn-project/end-to-end/bootstrap.sh build_bench
 
     # Extract and filter benchmark commands by flow name and wasm/no-wasm
     function ivc_bench_cmds {

barretenberg/cpp/cmake/module.cmake

@@ -26,7 +26,7 @@ target_sources(
 function(barretenberg_module MODULE_NAME)
     file(GLOB_RECURSE SOURCE_FILES *.cpp)
     file(GLOB_RECURSE HEADER_FILES *.hpp *.tcc)
-    list(FILTER SOURCE_FILES EXCLUDE REGEX ".*\.(fuzzer|test|bench).cpp$")
+    list(FILTER SOURCE_FILES EXCLUDE REGEX ".*\\.(fuzzer|test|bench)\\.cpp$")
 
     target_sources(
         barretenberg_headers
@@ -202,8 +202,8 @@ function(barretenberg_module MODULE_NAME)
             )
         endforeach()
     endif()
-
     file(GLOB_RECURSE BENCH_SOURCE_FILES *.bench.cpp)
+
     if(BENCH_SOURCE_FILES AND NOT FUZZING)
         foreach(BENCHMARK_SOURCE ${BENCH_SOURCE_FILES})
             get_filename_component(BENCHMARK_NAME ${BENCHMARK_SOURCE} NAME_WE) # extract name without extension

barretenberg/cpp/scripts/analyze_client_ivc_bench.py

@@ -11,7 +11,7 @@ cd $(dirname $0)/..
 
 # Measure the benchmarks with ops time counting
 ./scripts/benchmark_remote.sh "$TARGET"\
-                              "BB_USE_OP_COUNT_TIME=1 ./$TARGET --benchmark_filter=$FILTER\
+                              "BB_BENCH=1 ./$TARGET --benchmark_filter=$FILTER\
                                          --benchmark_out=$TARGET.json\
                                          --benchmark_out_format=json"\
                               clang20\

barretenberg/cpp/scripts/analyze_protogalaxy_bench.py

@@ -15,16 +15,9 @@ cd $(dirname $0)/..
 scp $BB_SSH_KEY ../../yarn-project/end-to-end/example-app-ivc-inputs-out/$FLOW/ivc-inputs.msgpack $BB_SSH_INSTANCE:$BB_SSH_CPP_PATH/build/
 
 # Measure the benchmarks with ops time counting
+
 ./scripts/benchmark_remote.sh "$TARGET"\
                               "./$TARGET prove -o output --ivc_inputs_path ivc-inputs.msgpack --scheme client_ivc\
-                              --op_counts_out=$TARGET.json"\
+                              --print_bench"\
                               clang20\
                               "$BUILD_DIR"
-
-# Retrieve output from benching instance
-cd $BUILD_DIR
-scp $BB_SSH_KEY $BB_SSH_INSTANCE:$BB_SSH_CPP_PATH/build/$TARGET.json .
-
-# Analyze the results
-cd ../
-python3 ./scripts/analyze_client_ivc_bench.py --json "$TARGET.json" --benchmark "" --prefix "$BUILD_DIR"

barretenberg/cpp/scripts/benchmark_client_ivc.sh

@@ -10,7 +10,7 @@ cd $(dirname $0)/..
 
 # Measure the benchmarks with ops time counting
 ./scripts/benchmark_remote.sh protogalaxy_bench\
-                              "BB_USE_OP_COUNT_TIME=1 ./protogalaxy_bench --benchmark_filter=$FILTER\
+                              "BB_BENCH=1 ./protogalaxy_bench --benchmark_filter=$FILTER\
                                                   --benchmark_out=$TARGET.json\
                                                   --benchmark_out_format=json"\
                               clang20\

barretenberg/cpp/scripts/benchmark_example_ivc_flow_remote.sh

@@ -44,20 +44,18 @@ function verify_ivc_flow {
 function run_bb_cli_bench {
   local runtime="$1"
   local output="$2"
-  local args="$3"
+  shift 2
 
   if [[ "$runtime" == "native" ]]; then
-    memusage "./$native_build_dir/bin/bb" $args \
-      --op_counts_out=$output/op-counts.json || {
-      echo "bb native failed with args: $args"
+    memusage "./$native_build_dir/bin/bb" "$@" || {
+      echo "bb native failed with args: $@"
       exit 1
     }
   else # wasm
     export WASMTIME_ALLOWED_DIRS="--dir=$flow_folder --dir=$output"
     # TODO support wasm op count time preset
-    memusage scripts/wasmtime.sh $WASMTIME_ALLOWED_DIRS ./build-wasm-threads/bin/bb $args \
-      --op_counts_out=$output/op-counts.json || {
-      echo "bb wasm failed with args: $args"
+    memusage scripts/wasmtime.sh $WASMTIME_ALLOWED_DIRS ./build-wasm-threads/bin/bb "$@" || {
+      echo "bb wasm failed with args: $@"
       exit 1
     }
   fi
@@ -76,11 +74,7 @@ function client_ivc_flow {
   mkdir -p "$output"
   export MEMUSAGE_OUT="$output/peak-memory-mb.txt"
 
-  run_bb_cli_bench "$runtime" "$output" "prove -o $output --ivc_inputs_path $flow_folder/ivc-inputs.msgpack --scheme client_ivc -v"
-
-  if [[ "$runtime" != wasm ]]; then
-    python3 scripts/analyze_client_ivc_bench.py --prefix . --json $output/op-counts.json --benchmark ""
-  fi
+  run_bb_cli_bench "$runtime" "$output" prove -o $output --ivc_inputs_path $flow_folder/ivc-inputs.msgpack --scheme client_ivc -v --print_bench
 
   local end=$(date +%s%N)
   local elapsed_ns=$(( end - start ))