test: print benchmark numbers at end of benchmark test

This will allows us to keep track of the numbers

Signed-off-by: Patrick Roy <[email protected]>

Author: roypat

.buildkite/pipeline_pr_no_block.py

@@ -26,7 +26,7 @@
 
 optional_grp = group(
     "❓ Optional",
-    "./tools/devtool -y test -c 1-10 -m 0 -- ../tests/integration_tests/ -m no_block_pr",
+    "./tools/devtool -y test -c 1-10 -m 0 -- ../tests/integration_tests/ -m no_block_pr --log-cli-level=INFO",
     **defaults,
 )
 

src/vmm/benches/cpu_templates.rs

@@ -45,7 +45,7 @@ pub fn cpu_template_benchmark(c: &mut Criterion) {
 
 criterion_group! {
     name = cpu_template_benches;
-    config = Criterion::default().sample_size(200).output_directory(Path::new("../../build/vmm_benchmark/cpu_templates"));
+    config = Criterion::default().sample_size(200);
     targets = cpu_template_benchmark
 }
 

tests/integration_tests/performance/test_benchmarks.py

@@ -2,6 +2,8 @@
 # SPDX-License-Identifier: Apache-2.0
 """Optional benchmarks-do-not-regress test"""
 
+import json
+import logging
 import os
 import platform
 
@@ -11,6 +13,7 @@
 from host_tools.cargo_build import cargo
 
 TARGET_BRANCH = os.environ.get("BUILDKITE_PULL_REQUEST_BASE_BRANCH") or "main"
+LOGGER = logging.getLogger(__name__)
 
 
 @pytest.mark.no_block_pr
@@ -41,6 +44,16 @@ def test_no_regression_relative_to_target_branch():
         if "Performance has regressed." in result
     )
 
+    for benchmark in os.listdir("../build/cargo_target/criterion"):
+        with open(
+            f"../build/cargo_target/criterion/{benchmark}/new/estimates.json",
+            encoding="utf-8",
+        ) as file:
+            data = json.load(file)
+        average_ns = data["mean"]["point_estimate"]
+
+        LOGGER.info("%s mean: %iµs", benchmark, average_ns / 1000)
+
     # If this string is anywhere in stdout, then at least one of our benchmarks
     # is now performing worse with the PR changes.
     assert not regressions_only, "\n" + regressions_only