feat(google_benchmark): only run benchmark once in instrumentation mode

Author: GuillaumeLagrange

google_benchmark/CMakeLists.txt

@@ -97,6 +97,8 @@ option(BENCHMARK_ENABLE_ASSEMBLY_TESTS "Enable building and running the assembly
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules")
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
+# Codspeed specific configuration
+include(Codspeed)
 
 # Read the git tags to determine the project version
 include(GetGitVersion)

google_benchmark/cmake/Codspeed.cmake

@@ -0,0 +1,27 @@
+# Step 1: Check if CODSPEED_MODE is set via the command line
+
+# CMake cache kind of breaks this mechanism, keeping it for first time
+# defaulting
+if(NOT DEFINED CODSPEED_MODE)
+    # Step 2: Check the environment variable CODSPEED_MODE
+    if(DEFINED $ENV{CODSPEED_RUNNER_MODE})
+        set(CODSPEED_MODE $ENV{CODSPEED_RUNNER_MODE} FORCE)
+    else()
+        # Step 3: Default to "instrumentation" if no value is provided
+        set(CODSPEED_MODE "instrumentation")
+    endif()
+endif()
+
+# Define a preprocessor macro based on the build mode
+if(CODSPEED_MODE STREQUAL "instrumentation")
+    target_compile_definitions(codspeed INTERFACE -DCODSPEED_INSTRUMENTATION)
+elseif(CODSPEED_MODE STREQUAL "walltime")
+    target_compile_definitions(codspeed INTERFACE -DCODSPEED_WALLTIME)
+else()
+    message(
+        FATAL_ERROR
+        "Invalid build mode: ${CODSPEED_MODE}. Use 'instrumentation' or 'walltime'."
+    )
+endif()
+
+message(STATUS "Build mode set to: ${CODSPEED_MODE}")

google_benchmark/src/CMakeLists.txt

@@ -82,9 +82,8 @@ set_target_properties(benchmark_main PROPERTIES
 )
 target_link_libraries(benchmark_main PUBLIC benchmark::benchmark)
 
-add_subdirectory(${PROJECT_SOURCE_DIR}/../core codspeed)
-
-target_link_libraries(benchmark PRIVATE codspeed)
+# TODO: Find a way to not expose codspeed headers to downstream users
+target_link_libraries(benchmark PUBLIC codspeed)
 
 set(generated_dir "${PROJECT_BINARY_DIR}")
 

google_benchmark/src/benchmark.cc

@@ -382,9 +382,13 @@ void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks,
   // Note the file_reporter can be null.
   BM_CHECK(display_reporter != nullptr);
 
-  hello_codspeed();
-
   // Determine the width of the name field using a minimum width of 10.
+#ifdef CODSPEED_INSTRUMENTATION
+  std::cout << "Codspeed mode: instrumentation" << "\n";
+#elif defined(CODSPEED_WALLTIME)
+  std::cout << "Codspeed mode walltime" << "\n";
+#endif
+
   bool might_have_aggregates = FLAGS_benchmark_repetitions > 1;
   size_t name_field_width = 10;
   size_t stat_field_width = 0;

google_benchmark/src/benchmark_api_internal.cc

@@ -2,6 +2,7 @@
 
 #include <cinttypes>
 
+#include "codspeed.h"
 #include "string_util.h"
 
 namespace benchmark {
@@ -89,6 +90,21 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx,
   teardown_ = benchmark_.teardown_;
 }
 
+#ifdef CODSPEED_INSTRUMENTATION
+State BenchmarkInstance::RunInstrumented(
+    CodSpeed* codspeed, internal::ThreadTimer* timer,
+    internal::ThreadManager* manager,
+    internal::PerfCountersMeasurement* perf_counters_measurement,
+    ProfilerManager* profiler_manager) const {
+  State st(name_.function_name, 1, args_, 0, 1, timer, manager,
+           perf_counters_measurement, profiler_manager);
+  codspeed->start_benchmark(name().str());
+  benchmark_.Run(st);
+  codspeed->end_benchmark();
+  return st;
+}
+#endif
+
 State BenchmarkInstance::Run(
     IterationCount iters, int thread_id, internal::ThreadTimer* timer,
     internal::ThreadManager* manager,

google_benchmark/src/benchmark_api_internal.h

@@ -9,6 +9,9 @@
 #include <vector>
 
 #include "benchmark/benchmark.h"
+#ifdef CODSPEED_INSTRUMENTATION
+#include "codspeed.h"
+#endif
 #include "commandlineflags.h"
 
 namespace benchmark {
@@ -34,7 +37,13 @@ class BenchmarkInstance {
   BigO complexity() const { return complexity_; }
   BigOFunc* complexity_lambda() const { return complexity_lambda_; }
   const std::vector<Statistics>& statistics() const { return statistics_; }
-  int repetitions() const { return repetitions_; }
+  int repetitions() const {
+#ifdef CODSPEED_INSTRUMENTATION
+    return 1;
+#else
+    return repetitions_;
+#endif
+  }
   double min_time() const { return min_time_; }
   double min_warmup_time() const { return min_warmup_time_; }
   IterationCount iterations() const { return iterations_; }
@@ -47,6 +56,14 @@ class BenchmarkInstance {
             internal::PerfCountersMeasurement* perf_counters_measurement,
             ProfilerManager* profiler_manager) const;
 
+#ifdef CODSPEED_INSTRUMENTATION
+  State RunInstrumented(
+      CodSpeed* codspeed, internal::ThreadTimer* timer,
+      internal::ThreadManager* manager,
+      internal::PerfCountersMeasurement* perf_counters_measurement,
+      ProfilerManager* profiler_manager) const;
+#endif
+
  private:
   BenchmarkName name_;
   Benchmark& benchmark_;

google_benchmark/src/benchmark_runner.cc

@@ -42,6 +42,7 @@
 #include <utility>
 
 #include "check.h"
+#include "codspeed.h"
 #include "colorprint.h"
 #include "commandlineflags.h"
 #include "complexity.h"
@@ -464,6 +465,16 @@ void BenchmarkRunner::DoOneRepetition() {
 
   const bool is_the_first_repetition = num_repetitions_done == 0;
 
+#ifdef CODSPEED_INSTRUMENTATION
+  std::unique_ptr<internal::ThreadManager> manager;
+  manager.reset(new internal::ThreadManager(b.threads()));
+  internal::ThreadTimer timer = internal::ThreadTimer::Create();
+  State st = b.RunInstrumented(CodSpeed::getInstance(), &timer, manager.get(),
+                               nullptr, nullptr);
+
+  return;
+#endif
+
   // In case a warmup phase is requested by the benchmark, run it now.
   // After running the warmup phase the BenchmarkRunner should be in a state as
   // this warmup never happened except the fact that warmup_done is set. Every