--wip-- [skip ci]

Author: GuillaumeLagrange

core/CMakeLists.txt

@@ -1,11 +1,8 @@
 cmake_minimum_required(VERSION 3.10)
 
-set(CODSPEED_VERSION 1.0)
+set(CODSPEED_VERSION 1.0.0)
 
-project(
-  codspeed
-  VERSION ${CODSPEED_VERSION}
-  LANGUAGES CXX)
+project(codspeed VERSION ${CODSPEED_VERSION} LANGUAGES CXX)
 
 # Specify the C++ standard
 set(CMAKE_CXX_STANDARD 17)
@@ -22,10 +19,12 @@ add_compile_definitions(CODSPEED_VERSION="${CODSPEED_VERSION}")
 
 # Disable valgrind compilation errors
 if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
-  # Disable the old-style-cast warning for the specific target
-  target_compile_options(codspeed PRIVATE -Wno-old-style-cast)
+    # Disable the old-style-cast warning for the specific target
+    target_compile_options(codspeed PRIVATE -Wno-old-style-cast)
 endif()
 
 # Specify the include directories for users of the library
 target_include_directories(
-  codspeed PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>)
+    codspeed
+    PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
+)

core/src/codspeed.cpp

@@ -18,7 +18,7 @@ std::string join(const std::vector<std::string> &elements,
 
 CodSpeed::CodSpeed() : is_instrumented(measurement_is_instrumented()) {
   if (!is_instrumented) {
-    std::cout
+    std::cerr
         << "NOTICE: codspeed is enabled, but no performance measurement will "
            "be made since it's running in an unknown environment."
         << std::endl;
@@ -48,6 +48,6 @@ void CodSpeed::end_benchmark() {
   std::string action_str = is_instrumented ? "Measured" : "Checked";
   std::string group_str =
       group_stack.empty() ? "" : " (group: " + join(group_stack, "/") + ")";
-  std::cout << action_str << ": " << current_benchmark << group_str
+  std::cerr << action_str << ": " << current_benchmark << group_str
             << std::endl;
 }

examples/google_benchmark/CMakeLists.txt

@@ -1,20 +1,15 @@
 cmake_minimum_required(VERSION 3.12)
 include(FetchContent)
 
-project(
-  codspeed_picobench_compat
-  VERSION 0.0.0
-  LANGUAGES CXX)
+project(codspeed_picobench_compat VERSION 0.0.0 LANGUAGES CXX)
 
 set(BENCHMARK_DOWNLOAD_DEPENDENCIES ON)
 
-# FetchContent_Declare( google_benchmark GIT_REPOSITORY
-# https://github.com/CodSpeedHQ/codspeed-cpp GIT_TAG
-# origin/cod-291-implement-the-core-instrumentation-library-in-c SOURCE_SUBDIR
-# google_benchmark)
-
-FetchContent_Declare(google_benchmark
-                     SOURCE_DIR ${CMAKE_SOURCE_DIR}/../../google_benchmark)
+FetchContent_Declare(
+    google_benchmark
+    SOURCE_DIR
+    ${CMAKE_SOURCE_DIR}/../../google_benchmark
+)
 
 FetchContent_MakeAvailable(google_benchmark)
 

flake.nix

@@ -38,7 +38,7 @@
             buildInputs =
               with pkgs;
               [
-                cmake-language-server
+                gersemi
                 clang-tools
               ]
               ++ commonBuildInputs;

google_benchmark/cmake/Codspeed.cmake

@@ -1,37 +1,43 @@
 add_subdirectory(${PROJECT_SOURCE_DIR}/../core codspeed)
 
 execute_process(
-  COMMAND git rev-parse --show-toplevel
-  OUTPUT_VARIABLE GIT_ROOT_DIR
-  OUTPUT_STRIP_TRAILING_WHITESPACE)
+    COMMAND git rev-parse --show-toplevel
+    OUTPUT_VARIABLE GIT_ROOT_DIR
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    RESULT_VARIABLE GIT_COMMAND_RESULT
+)
 
-target_compile_definitions(codspeed
-                           INTERFACE -DCODSPEED_GIT_ROOT_DIR="${GIT_ROOT_DIR}")
+if(NOT GIT_COMMAND_RESULT EQUAL 0)
+    message(
+        WARNING
+        "Failed to determine the git root directory.\
+        Check that git is in your PATH, and that you are in a git repository.\
+        Continuing, but codspeed features will not be useable"
+    )
+    # Default to user's cmake source directory
+    set(GIT_ROOT_DIR ${CMAKE_SOURCE_DIR})
+endif()
 
-# Step 1: Check if CODSPEED_MODE is set via the command line
+target_compile_definitions(
+    codspeed
+    INTERFACE -DCODSPEED_GIT_ROOT_DIR="${GIT_ROOT_DIR}"
+)
 
-# CMake cache kind of breaks this mechanism, keeping it for first time
-# defaulting
-if(NOT DEFINED CODSPEED_MODE)
-    # Step 2: Check the environment variable CODSPEED_MODE
-    if(DEFINED $ENV{CODSPEED_RUNNER_MODE})
-        set(CODSPEED_MODE $ENV{CODSPEED_RUNNER_MODE} FORCE)
+if(DEFINED CODSPEED_MODE)
+    # Define a preprocessor macro based on the build mode
+    if(CODSPEED_MODE STREQUAL "instrumentation")
+        target_compile_definitions(
+            codspeed
+            INTERFACE -DCODSPEED_INSTRUMENTATION
+        )
+    elseif(CODSPEED_MODE STREQUAL "walltime")
+        target_compile_definitions(codspeed INTERFACE -DCODSPEED_WALLTIME)
     else()
-        # Step 3: Default to "instrumentation" if no value is provided
-        set(CODSPEED_MODE "instrumentation")
+        message(
+            FATAL_ERROR
+            "Invalid build mode: ${CODSPEED_MODE}. Use 'instrumentation' or 'walltime'."
+        )
     endif()
 endif()
 
-# Define a preprocessor macro based on the build mode
-if(CODSPEED_MODE STREQUAL "instrumentation")
-    target_compile_definitions(codspeed INTERFACE -DCODSPEED_INSTRUMENTATION)
-elseif(CODSPEED_MODE STREQUAL "walltime")
-    target_compile_definitions(codspeed INTERFACE -DCODSPEED_WALLTIME)
-else()
-    message(
-        FATAL_ERROR
-        "Invalid build mode: ${CODSPEED_MODE}. Use 'instrumentation' or 'walltime'."
-    )
-endif()
-
 message(STATUS "Build mode set to: ${CODSPEED_MODE}")

google_benchmark/src/benchmark.cc

@@ -379,14 +379,16 @@ void Report(BenchmarkReporter* display_reporter,
 void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks,
                    BenchmarkReporter* display_reporter,
                    BenchmarkReporter* file_reporter) {
+  auto& Err = display_reporter->GetErrorStream();
+
   // Note the file_reporter can be null.
   BM_CHECK(display_reporter != nullptr);
 
   // Determine the width of the name field using a minimum width of 10.
 #ifdef CODSPEED_INSTRUMENTATION
-  std::cout << "Codspeed mode: instrumentation" << "\n";
+  Err << "Codspeed mode: instrumentation" << "\n";
 #elif defined(CODSPEED_WALLTIME)
-  std::cout << "Codspeed mode walltime" << "\n";
+  Err << "Codspeed mode: walltime" << "\n";
 #endif
 
   bool might_have_aggregates = FLAGS_benchmark_repetitions > 1;

google_benchmark/src/benchmark_api_internal.cc

@@ -98,6 +98,9 @@ State BenchmarkInstance::RunInstrumented(
     ProfilerManager* profiler_manager) const {
   State st(name_.function_name, 1, args_, 0, 1, timer, manager,
            perf_counters_measurement, profiler_manager);
+  // Do one repetition to avoid flakiness due to inconcistencies in CPU cache
+  // from execution order
+  benchmark_.Run(st);
   codspeed->start_benchmark(name().str());
   benchmark_.Run(st);
   codspeed->end_benchmark();

google_benchmark/src/benchmark_runner.cc

@@ -461,20 +461,22 @@ void BenchmarkRunner::RunProfilerManager(IterationCount profile_iterations) {
 }
 
 void BenchmarkRunner::DoOneRepetition() {
-  assert(HasRepeatsRemaining() && "Already done all repetitions?");
-
-  const bool is_the_first_repetition = num_repetitions_done == 0;
-
 #ifdef CODSPEED_INSTRUMENTATION
   std::unique_ptr<internal::ThreadManager> manager;
   manager.reset(new internal::ThreadManager(b.threads()));
   internal::ThreadTimer timer = internal::ThreadTimer::Create();
+  b.Setup();
   State st = b.RunInstrumented(CodSpeed::getInstance(), &timer, manager.get(),
                                nullptr, nullptr);
+  b.Teardown();
 
   return;
 #endif
 
+  assert(HasRepeatsRemaining() && "Already done all repetitions?");
+
+  const bool is_the_first_repetition = num_repetitions_done == 0;
+
   // In case a warmup phase is requested by the benchmark, run it now.
   // After running the warmup phase the BenchmarkRunner should be in a state as
   // this warmup never happened except the fact that warmup_done is set. Every