feat: only run benchmark once in instrumentation mode

GuillaumeLagrange · GuillaumeLagrange · commit 3e2d001bfefc · 2025-02-21T10:51:00.000+01:00
This also adds support for codspeed mode definition at build time
diff --git a/google_benchmark/CMakeLists.txt b/google_benchmark/CMakeLists.txt
@@ -97,6 +97,8 @@ option(BENCHMARK_ENABLE_ASSEMBLY_TESTS "Enable building and running the assembly
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules")
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
+# Codspeed specific configuration
+include(Codspeed)
 
 # Read the git tags to determine the project version
 include(GetGitVersion)
diff --git a/google_benchmark/cmake/Codspeed.cmake b/google_benchmark/cmake/Codspeed.cmake
@@ -0,0 +1,27 @@
+# Step 1: Check if CODSPEED_MODE is set via the command line
+
+# CMake cache kind of breaks this mechanism, keeping it for first time
+# defaulting
+if(NOT DEFINED CODSPEED_MODE)
+  # Step 2: Check the environment variable CODSPEED_MODE
+  if(DEFINED $ENV{CODSPEED_RUNNER_MODE})
+    set(CODSPEED_MODE $ENV{CODSPEED_RUNNER_MODE} FORCE)
+  else()
+    # Step 3: Default to "instrumentation" if no value is provided
+    set(CODSPEED_MODE "instrumentation")
+  endif()
+endif()
+
+# Define a preprocessor macro based on the build mode
+if(CODSPEED_MODE STREQUAL "instrumentation")
+  target_compile_definitions(codspeed INTERFACE -DCODSPEED_INSTRUMENTATION)
+elseif(CODSPEED_MODE STREQUAL "walltime")
+  target_compile_definitions(codspeed INTERFACE -DCODSPEED_WALLTIME)
+else()
+  message(
+    FATAL_ERROR
+      "Invalid build mode: ${CODSPEED_MODE}. Use 'instrumentation' or 'walltime'."
+  )
+endif()
+
+message(STATUS "Build mode set to: ${CODSPEED_MODE}")
diff --git a/google_benchmark/src/CMakeLists.txt b/google_benchmark/src/CMakeLists.txt
@@ -1,42 +1,36 @@
-#Allow the source files to find headers in src /
+# Allow the source files to find headers in src /
 include(GNUInstallDirs)
 include_directories(${PROJECT_SOURCE_DIR}/src)
 
-if (DEFINED BENCHMARK_CXX_LINKER_FLAGS)
+if(DEFINED BENCHMARK_CXX_LINKER_FLAGS)
   list(APPEND CMAKE_SHARED_LINKER_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS})
   list(APPEND CMAKE_MODULE_LINKER_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS})
 endif()
 
-file(GLOB
-  SOURCE_FILES
-    *.cc
-    ${PROJECT_SOURCE_DIR}/include/benchmark/*.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/*.h)
+file(GLOB SOURCE_FILES *.cc ${PROJECT_SOURCE_DIR}/include/benchmark/*.h
+     ${CMAKE_CURRENT_SOURCE_DIR}/*.h)
 file(GLOB BENCHMARK_MAIN "benchmark_main.cc")
 foreach(item ${BENCHMARK_MAIN})
   list(REMOVE_ITEM SOURCE_FILES "${item}")
 endforeach()
 
 add_library(benchmark ${SOURCE_FILES})
 add_library(benchmark::benchmark ALIAS benchmark)
-set_target_properties(benchmark PROPERTIES
-  OUTPUT_NAME "benchmark"
-  VERSION ${GENERIC_LIB_VERSION}
-  SOVERSION ${GENERIC_LIB_SOVERSION}
-)
-target_include_directories(benchmark PUBLIC
-  $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
-)
+set_target_properties(
+  benchmark
+  PROPERTIES OUTPUT_NAME "benchmark"
+             VERSION ${GENERIC_LIB_VERSION}
+             SOVERSION ${GENERIC_LIB_SOVERSION})
+target_include_directories(
+  benchmark PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>)
 
 set_property(
   SOURCE benchmark.cc
   APPEND
-  PROPERTY COMPILE_DEFINITIONS
-  BENCHMARK_VERSION="${VERSION}"
-)
+  PROPERTY COMPILE_DEFINITIONS BENCHMARK_VERSION="${VERSION}")
 
 # libpfm, if available
-if (PFM_FOUND)
+if(PFM_FOUND)
   target_link_libraries(benchmark PRIVATE PFM::libpfm)
   target_compile_definitions(benchmark PRIVATE -DHAVE_LIBPFM)
 endif()
@@ -55,7 +49,6 @@ if(HAVE_LIB_RT)
   target_link_libraries(benchmark PRIVATE rt)
 endif(HAVE_LIB_RT)
 
-
 # We need extra libraries on Windows
 if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
   target_link_libraries(benchmark PRIVATE shlwapi)
@@ -67,24 +60,23 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
   set(BENCHMARK_PRIVATE_LINK_LIBRARIES -lkstat)
 endif()
 
-if (NOT BUILD_SHARED_LIBS)
+if(NOT BUILD_SHARED_LIBS)
   target_compile_definitions(benchmark PUBLIC -DBENCHMARK_STATIC_DEFINE)
 endif()
 
 # Benchmark main library
 add_library(benchmark_main "benchmark_main.cc")
 add_library(benchmark::benchmark_main ALIAS benchmark_main)
-set_target_properties(benchmark_main PROPERTIES
-  OUTPUT_NAME "benchmark_main"
-  VERSION ${GENERIC_LIB_VERSION}
-  SOVERSION ${GENERIC_LIB_SOVERSION}
-  DEFINE_SYMBOL benchmark_EXPORTS
-)
+set_target_properties(
+  benchmark_main
+  PROPERTIES OUTPUT_NAME "benchmark_main"
+             VERSION ${GENERIC_LIB_VERSION}
+             SOVERSION ${GENERIC_LIB_SOVERSION}
+             DEFINE_SYMBOL benchmark_EXPORTS)
 target_link_libraries(benchmark_main PUBLIC benchmark::benchmark)
 
-add_subdirectory(${PROJECT_SOURCE_DIR}/../core codspeed)
-
-target_link_libraries(benchmark PRIVATE codspeed)
+# TODO: Find a way to not expose codspeed headers to downstream users
+target_link_libraries(benchmark PUBLIC codspeed)
 
 set(generated_dir "${PROJECT_BINARY_DIR}")
 
@@ -100,57 +92,57 @@ set(namespace "${PROJECT_NAME}::")
 
 include(CMakePackageConfigHelpers)
 
-configure_package_config_file (
-  ${PROJECT_SOURCE_DIR}/cmake/Config.cmake.in
-  ${project_config}
+configure_package_config_file(
+  ${PROJECT_SOURCE_DIR}/cmake/Config.cmake.in ${project_config}
   INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
-  NO_SET_AND_CHECK_MACRO
-  NO_CHECK_REQUIRED_COMPONENTS_MACRO
-)
+  NO_SET_AND_CHECK_MACRO NO_CHECK_REQUIRED_COMPONENTS_MACRO)
 write_basic_package_version_file(
-  "${version_config}" VERSION ${GENERIC_LIB_VERSION} COMPATIBILITY SameMajorVersion
-)
+  "${version_config}"
+  VERSION ${GENERIC_LIB_VERSION}
+  COMPATIBILITY SameMajorVersion)
 
-configure_file("${PROJECT_SOURCE_DIR}/cmake/benchmark.pc.in" "${pkg_config}" @ONLY)
-configure_file("${PROJECT_SOURCE_DIR}/cmake/benchmark_main.pc.in" "${pkg_config_main}" @ONLY)
+configure_file("${PROJECT_SOURCE_DIR}/cmake/benchmark.pc.in" "${pkg_config}"
+               @ONLY)
+configure_file("${PROJECT_SOURCE_DIR}/cmake/benchmark_main.pc.in"
+               "${pkg_config_main}" @ONLY)
 
-export (
+export(
   TARGETS ${targets_to_export}
   NAMESPACE "${namespace}"
-  FILE ${generated_dir}/${targets_export_name}.cmake
-)
+  FILE ${generated_dir}/${targets_export_name}.cmake)
 
-if (BENCHMARK_ENABLE_INSTALL)
-  # Install target (will install the library to specified CMAKE_INSTALL_PREFIX variable)
+if(BENCHMARK_ENABLE_INSTALL)
+  # Install target (will install the library to specified CMAKE_INSTALL_PREFIX
+  # variable)
   install(
     TARGETS ${targets_to_export}
     EXPORT ${targets_export_name}
     ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
     LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
     RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-    INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+    INCLUDES
+    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 
   install(
     DIRECTORY "${PROJECT_SOURCE_DIR}/include/benchmark"
               "${PROJECT_BINARY_DIR}/include/benchmark"
     DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-    FILES_MATCHING PATTERN "*.*h")
+    FILES_MATCHING
+    PATTERN "*.*h")
 
-  install(
-      FILES "${project_config}" "${version_config}"
-      DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
+  install(FILES "${project_config}" "${version_config}"
+          DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
 
-  install(
-      FILES "${pkg_config}" "${pkg_config_main}"
-      DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
+  install(FILES "${pkg_config}" "${pkg_config_main}"
+          DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 
   install(
-      EXPORT "${targets_export_name}"
-      NAMESPACE "${namespace}"
-      DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
+    EXPORT "${targets_export_name}"
+    NAMESPACE "${namespace}"
+    DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
 endif()
 
-if (BENCHMARK_ENABLE_DOXYGEN)
+if(BENCHMARK_ENABLE_DOXYGEN)
   find_package(Doxygen REQUIRED)
   set(DOXYGEN_QUIET YES)
   set(DOXYGEN_RECURSIVE YES)
@@ -164,22 +156,17 @@ if (BENCHMARK_ENABLE_DOXYGEN)
   set(DOXYGEN_BINARY_TOC YES)
   set(DOXYGEN_TOC_EXPAND YES)
   set(DOXYGEN_USE_MDFILE_AS_MAINPAGE "index.md")
-  doxygen_add_docs(benchmark_doxygen
-    docs
-    include
-    src
-    ALL
+  doxygen_add_docs(
+    benchmark_doxygen docs include src ALL
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     COMMENT "Building documentation with Doxygen.")
-  if (BENCHMARK_ENABLE_INSTALL AND BENCHMARK_INSTALL_DOCS)
-    install(
-      DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/html/"
-      DESTINATION ${CMAKE_INSTALL_DOCDIR})
+  if(BENCHMARK_ENABLE_INSTALL AND BENCHMARK_INSTALL_DOCS)
+    install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/html/"
+            DESTINATION ${CMAKE_INSTALL_DOCDIR})
   endif()
 else()
-  if (BENCHMARK_ENABLE_INSTALL AND BENCHMARK_INSTALL_DOCS)
-    install(
-      DIRECTORY "${PROJECT_SOURCE_DIR}/docs/"
-      DESTINATION ${CMAKE_INSTALL_DOCDIR})
+  if(BENCHMARK_ENABLE_INSTALL AND BENCHMARK_INSTALL_DOCS)
+    install(DIRECTORY "${PROJECT_SOURCE_DIR}/docs/"
+            DESTINATION ${CMAKE_INSTALL_DOCDIR})
   endif()
 endif()
diff --git a/google_benchmark/src/benchmark.cc b/google_benchmark/src/benchmark.cc
@@ -382,9 +382,13 @@ void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks,
   // Note the file_reporter can be null.
   BM_CHECK(display_reporter != nullptr);
 
-  hello_codspeed();
-
   // Determine the width of the name field using a minimum width of 10.
+#ifdef CODSPEED_INSTRUMENTATION
+  std::cout << "Codspeed mode: instrumentation" << "\n";
+#elif defined(CODSPEED_WALLTIME)
+  std::cout << "Codspeed mode walltime" << "\n";
+#endif
+
   bool might_have_aggregates = FLAGS_benchmark_repetitions > 1;
   size_t name_field_width = 10;
   size_t stat_field_width = 0;
diff --git a/google_benchmark/src/benchmark_api_internal.cc b/google_benchmark/src/benchmark_api_internal.cc
@@ -2,6 +2,7 @@
 
 #include <cinttypes>
 
+#include "codspeed.h"
 #include "string_util.h"
 
 namespace benchmark {
@@ -89,6 +90,21 @@ BenchmarkInstance::BenchmarkInstance(Benchmark* benchmark, int family_idx,
   teardown_ = benchmark_.teardown_;
 }
 
+#ifdef CODSPEED_INSTRUMENTATION
+State BenchmarkInstance::RunInstrumented(
+    CodSpeed* codspeed, internal::ThreadTimer* timer,
+    internal::ThreadManager* manager,
+    internal::PerfCountersMeasurement* perf_counters_measurement,
+    ProfilerManager* profiler_manager) const {
+  State st(name_.function_name, 1, args_, 0, 1, timer, manager,
+           perf_counters_measurement, profiler_manager);
+  codspeed->start_benchmark(name().str());
+  benchmark_.Run(st);
+  codspeed->end_benchmark();
+  return st;
+}
+#endif
+
 State BenchmarkInstance::Run(
     IterationCount iters, int thread_id, internal::ThreadTimer* timer,
     internal::ThreadManager* manager,
diff --git a/google_benchmark/src/benchmark_api_internal.h b/google_benchmark/src/benchmark_api_internal.h
@@ -9,6 +9,9 @@
 #include <vector>
 
 #include "benchmark/benchmark.h"
+#ifdef CODSPEED_INSTRUMENTATION
+#include "codspeed.h"
+#endif
 #include "commandlineflags.h"
 
 namespace benchmark {
@@ -34,7 +37,13 @@ class BenchmarkInstance {
   BigO complexity() const { return complexity_; }
   BigOFunc* complexity_lambda() const { return complexity_lambda_; }
   const std::vector<Statistics>& statistics() const { return statistics_; }
-  int repetitions() const { return repetitions_; }
+  int repetitions() const {
+#ifdef CODSPEED_INSTRUMENTATION
+    return 1;
+#else
+    return repetitions_;
+#endif
+  }
   double min_time() const { return min_time_; }
   double min_warmup_time() const { return min_warmup_time_; }
   IterationCount iterations() const { return iterations_; }
@@ -47,6 +56,14 @@ class BenchmarkInstance {
             internal::PerfCountersMeasurement* perf_counters_measurement,
             ProfilerManager* profiler_manager) const;
 
+#ifdef CODSPEED_INSTRUMENTATION
+  State RunInstrumented(
+      CodSpeed* codspeed, internal::ThreadTimer* timer,
+      internal::ThreadManager* manager,
+      internal::PerfCountersMeasurement* perf_counters_measurement,
+      ProfilerManager* profiler_manager) const;
+#endif
+
  private:
   BenchmarkName name_;
   Benchmark& benchmark_;
diff --git a/google_benchmark/src/benchmark_runner.cc b/google_benchmark/src/benchmark_runner.cc
@@ -42,6 +42,7 @@
 #include <utility>
 
 #include "check.h"
+#include "codspeed.h"
 #include "colorprint.h"
 #include "commandlineflags.h"
 #include "complexity.h"
@@ -464,6 +465,16 @@ void BenchmarkRunner::DoOneRepetition() {
 
   const bool is_the_first_repetition = num_repetitions_done == 0;
 
+#ifdef CODSPEED_INSTRUMENTATION
+  std::unique_ptr<internal::ThreadManager> manager;
+  manager.reset(new internal::ThreadManager(b.threads()));
+  internal::ThreadTimer timer = internal::ThreadTimer::Create();
+  State st = b.RunInstrumented(CodSpeed::getInstance(), &timer, manager.get(),
+                               nullptr, nullptr);
+
+  return;
+#endif
+
   // In case a warmup phase is requested by the benchmark, run it now.
   // After running the warmup phase the BenchmarkRunner should be in a state as
   // this warmup never happened except the fact that warmup_done is set. Every
