feat(google_benchmark): only run benchmark once in instrumentation mode

Author: GuillaumeLagrange

google_benchmark/CMakeLists.txt

@@ -97,6 +97,8 @@ option(BENCHMARK_ENABLE_ASSEMBLY_TESTS "Enable building and running the assembly
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules")
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
+# Codspeed specific configuration
+include(Codspeed)
 
 # Read the git tags to determine the project version
 include(GetGitVersion)

google_benchmark/cmake/Codspeed.cmake

@@ -0,0 +1,20 @@
+# Step 1: Check if CODSPEED_MODE is set via the command line
+
+if(DEFINED CODSPEED_MODE)
+    # Define a preprocessor macro based on the build mode
+    if(CODSPEED_MODE STREQUAL "instrumentation")
+        target_compile_definitions(
+            codspeed
+            INTERFACE -DCODSPEED_INSTRUMENTATION
+        )
+    elseif(CODSPEED_MODE STREQUAL "walltime")
+        target_compile_definitions(codspeed INTERFACE -DCODSPEED_WALLTIME)
+    else()
+        message(
+            FATAL_ERROR
+            "Invalid build mode: ${CODSPEED_MODE}. Use 'instrumentation' or 'walltime'."
+        )
+    endif()
+endif()
+
+message(STATUS "Build mode set to: ${CODSPEED_MODE}")

google_benchmark/src/CMakeLists.txt

@@ -82,9 +82,8 @@ set_target_properties(benchmark_main PROPERTIES
 )
 target_link_libraries(benchmark_main PUBLIC benchmark::benchmark)
 
-add_subdirectory(${PROJECT_SOURCE_DIR}/../core codspeed)
-
-target_link_libraries(benchmark PRIVATE codspeed)
+# TODO: Find a way to not expose codspeed headers to downstream users
+target_link_libraries(benchmark PUBLIC codspeed)
 
 set(generated_dir "${PROJECT_BINARY_DIR}")
 

google_benchmark/src/benchmark.cc

@@ -382,9 +382,16 @@ void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks,
   // Note the file_reporter can be null.
   BM_CHECK(display_reporter != nullptr);
 
-  hello_codspeed();
-
+#ifdef CODSPEED_ENABLED
+  auto& Err = display_reporter->GetErrorStream();
   // Determine the width of the name field using a minimum width of 10.
+#ifdef CODSPEED_INSTRUMENTATION
+  Err << "Codspeed mode: instrumentation" << "\n";
+#elif defined(CODSPEED_WALLTIME)
+  Err << "Codspeed mode: walltime" << "\n";
+#endif
+#endif  // CODSPEED_ENABLED
+
   bool might_have_aggregates = FLAGS_benchmark_repetitions > 1;
   size_t name_field_width = 10;
   size_t stat_field_width = 0;

google_benchmark/src/benchmark_api_internal.cc

@@ -2,7 +2,9 @@
 
 #include <cinttypes>
 
+#include "codspeed.h"
 #include "string_util.h"
+#include "thread_timer.h"
 
 namespace benchmark {
 namespace internal {
@@ -89,6 +91,28 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx,
   teardown_ = benchmark_.teardown_;
 }
 
+#ifdef CODSPEED_INSTRUMENTATION
+State BenchmarkInstance::RunInstrumented(
+    CodSpeed* codspeed, internal::ThreadTimer* timer,
+    internal::ThreadManager* manager,
+    internal::PerfCountersMeasurement* perf_counters_measurement,
+    ProfilerManager* profiler_manager) const {
+  State st(name_.function_name, 1, args_, 0, 1, timer, manager,
+           perf_counters_measurement, profiler_manager);
+  // Do one repetition to avoid flakiness due to inconcistencies in CPU cache
+  // from execution order
+
+  internal::ThreadTimer warmup_timer = internal::ThreadTimer::Create();
+  State warmup_state(name_.function_name, 1, args_, 0, 1, &warmup_timer,
+                     manager, perf_counters_measurement, profiler_manager);
+  benchmark_.Run(warmup_state);
+  codspeed->start_benchmark(name().str());
+  benchmark_.Run(st);
+  codspeed->end_benchmark();
+  return st;
+}
+#endif
+
 State BenchmarkInstance::Run(
     IterationCount iters, int thread_id, internal::ThreadTimer* timer,
     internal::ThreadManager* manager,

google_benchmark/src/benchmark_api_internal.h

@@ -9,6 +9,9 @@
 #include <vector>
 
 #include "benchmark/benchmark.h"
+#ifdef CODSPEED_INSTRUMENTATION
+#include "codspeed.h"
+#endif
 #include "commandlineflags.h"
 
 namespace benchmark {
@@ -34,7 +37,13 @@ class BenchmarkInstance {
   BigO complexity() const { return complexity_; }
   BigOFunc* complexity_lambda() const { return complexity_lambda_; }
   const std::vector<Statistics>& statistics() const { return statistics_; }
-  int repetitions() const { return repetitions_; }
+  int repetitions() const {
+#ifdef CODSPEED_INSTRUMENTATION
+    return 1;
+#else
+    return repetitions_;
+#endif
+  }
   double min_time() const { return min_time_; }
   double min_warmup_time() const { return min_warmup_time_; }
   IterationCount iterations() const { return iterations_; }
@@ -47,6 +56,14 @@ class BenchmarkInstance {
             internal::PerfCountersMeasurement* perf_counters_measurement,
             ProfilerManager* profiler_manager) const;
 
+#ifdef CODSPEED_INSTRUMENTATION
+  State RunInstrumented(
+      CodSpeed* codspeed, internal::ThreadTimer* timer,
+      internal::ThreadManager* manager,
+      internal::PerfCountersMeasurement* perf_counters_measurement,
+      ProfilerManager* profiler_manager) const;
+#endif
+
  private:
   BenchmarkName name_;
   Benchmark& benchmark_;

google_benchmark/src/benchmark_runner.cc

@@ -42,6 +42,7 @@
 #include <utility>
 
 #include "check.h"
+#include "codspeed.h"
 #include "colorprint.h"
 #include "commandlineflags.h"
 #include "complexity.h"
@@ -460,6 +461,18 @@ void BenchmarkRunner::RunProfilerManager(IterationCount profile_iterations) {
 }
 
 void BenchmarkRunner::DoOneRepetition() {
+#ifdef CODSPEED_INSTRUMENTATION
+  std::unique_ptr<internal::ThreadManager> manager;
+  manager.reset(new internal::ThreadManager(b.threads()));
+  internal::ThreadTimer timer = internal::ThreadTimer::Create();
+  b.Setup();
+  State st = b.RunInstrumented(CodSpeed::getInstance(), &timer, manager.get(),
+                               nullptr, nullptr);
+  b.Teardown();
+
+  return;
+#endif
+
   assert(HasRepeatsRemaining() && "Already done all repetitions?");
 
   const bool is_the_first_repetition = num_repetitions_done == 0;