feat(google_benchmark): add walltime support

GuillaumeLagrange · GuillaumeLagrange · commit e7a1994acf22 · 2025-03-10T12:33:07.000+01:00
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
@@ -12,7 +12,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED True)
 include_directories(include)
 
 # Add the library
-add_library(codspeed src/codspeed.cpp)
+add_library(codspeed src/codspeed.cpp src/walltime.cpp)
 
 # Version
 add_compile_definitions(CODSPEED_VERSION="${CODSPEED_VERSION}")
diff --git a/core/include/codspeed.h b/core/include/codspeed.h
@@ -27,4 +27,18 @@ class CodSpeed {
   bool is_instrumented;
 };
 
+// Times are per iteration
+struct RawWalltimeBenchmark {
+  std::string name;
+  std::string uri;
+  long iter_per_round;
+  double mean_ns;
+  double median_ns;
+  double stdev_ns;
+  std::vector<double> round_times_ns;
+};
+
+void generate_codspeed_walltime_report(
+    const std::vector<RawWalltimeBenchmark> &walltime_data_list);
+
 #endif // CODSPEED_H
diff --git a/core/src/codspeed.cpp b/core/src/codspeed.cpp
@@ -38,7 +38,16 @@ void CodSpeed::pop_group() {
 }
 
 void CodSpeed::start_benchmark(const std::string &name) {
-  current_benchmark = name;
+  std::string uri = name;
+
+  // Sanity check URI and add a placeholder if format is wrong
+  if (name.find("::") == std::string::npos) {
+    std::string uri = "unknown_file::" + name;
+    std::cout << "WARNING: Benchmark name does not contain '::'. Using URI: "
+              << uri << std::endl;
+  }
+
+  current_benchmark = uri;
   measurement_start();
 }
 
diff --git a/core/src/walltime.cpp b/core/src/walltime.cpp
@@ -0,0 +1,225 @@
+#include "codspeed.h"
+#include <algorithm>
+#include <cmath>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <numeric>
+#include <sstream>
+#include <string>
+#include <thread>
+#include <vector>
+
+const double IQR_OUTLIER_FACTOR = 1.5;
+const double STDEV_OUTLIER_FACTOR = 3.0;
+
+// Times are per iteration
+struct BenchmarkStats {
+  double min_ns;
+  double max_ns;
+  double mean_ns;
+  double stdev_ns;
+  double q1_ns;
+  double median_ns;
+  double q3_ns;
+  uint64_t rounds;
+  double total_time;
+  uint64_t iqr_outlier_rounds;
+  uint64_t stdev_outlier_rounds;
+  long iter_per_round;
+  uint64_t warmup_iters;
+};
+
+struct BenchmarkMetadata {
+  std::string name;
+  std::string uri;
+};
+
+struct CodspeedWalltimeBenchmark {
+  BenchmarkMetadata metadata;
+  BenchmarkStats stats;
+};
+
+double compute_quantile(const std::vector<double> &data, double quantile) {
+  size_t n = data.size();
+  if (n == 0)
+    return 0.0;
+
+  double pos = quantile * (n - 1);
+  size_t k = static_cast<size_t>(pos);
+  double d = pos - k;
+
+  if (k + 1 < n) {
+    return data[k] + d * (data[k + 1] - data[k]);
+  }
+  return data[k];
+}
+
+void compute_iqr_and_outliers(const std::vector<double> &times_ns, double mean,
+                              double median, double stdev, double &q1,
+                              double &q3, double &iqr,
+                              size_t &iqr_outlier_rounds,
+                              size_t &stdev_outlier_rounds) {
+  std::vector<double> sorted_times = times_ns;
+  std::sort(sorted_times.begin(), sorted_times.end());
+
+  q1 = compute_quantile(sorted_times, 0.25);
+  q3 = compute_quantile(sorted_times, 0.75);
+
+  iqr = q3 - q1;
+
+  const double IQR_OUTLIER_FACTOR = 1.5;
+  const double STDEV_OUTLIER_FACTOR = 3.0;
+
+  iqr_outlier_rounds =
+      std::count_if(sorted_times.begin(), sorted_times.end(),
+                    [q1, q3, iqr, IQR_OUTLIER_FACTOR](double x) {
+                      return x < q1 - IQR_OUTLIER_FACTOR * iqr ||
+                             x > q3 + IQR_OUTLIER_FACTOR * iqr;
+                    });
+
+  stdev_outlier_rounds =
+      std::count_if(sorted_times.begin(), sorted_times.end(),
+                    [mean, stdev, STDEV_OUTLIER_FACTOR](double x) {
+                      return x < mean - STDEV_OUTLIER_FACTOR * stdev ||
+                             x > mean + STDEV_OUTLIER_FACTOR * stdev;
+                    });
+}
+
+void write_codspeed_benchmarks_to_json(
+    const std::vector<CodspeedWalltimeBenchmark> &benchmarks) {
+  std::ostringstream oss;
+
+  std::string creator_name = "codspeed-cpp";
+  std::string creator_version = CODSPEED_VERSION;
+  std::thread::id creator_pid = std::this_thread::get_id();
+  std::string instrument_type = "walltime";
+
+  oss << "{\n";
+  oss << "  \"creator\": {\n";
+  oss << "    \"name\": \"" << creator_name << "\",\n";
+  oss << "    \"version\": \"" << creator_version << "\",\n";
+  oss << "    \"pid\": " << creator_pid << "\n";
+  oss << "  },\n";
+  oss << "  \"instrument\": {\n";
+  oss << "    \"type\": \"" << instrument_type << "\"\n";
+  oss << "  },\n";
+  oss << "  \"benchmarks\": [\n";
+
+  for (size_t i = 0; i < benchmarks.size(); ++i) {
+    const auto &benchmark = benchmarks[i];
+    const auto &stats = benchmark.stats;
+    const auto &metadata = benchmark.metadata;
+
+    oss << "    {\n";
+    oss << "      \"name\": \"" << metadata.name << "\",\n";
+    oss << "      \"uri\": \"" << metadata.uri << "\",\n";
+    // TODO: Manage config fields from actual configuration
+    oss << "      \"config\": {\n";
+    oss << "        \"warmup_time_ns\": null,\n";
+    oss << "        \"min_round_time_ns\": null,\n";
+    oss << "        \"max_time_ns\": null,\n";
+    oss << "        \"max_rounds\": null\n";
+    oss << "      },\n";
+    oss << "      \"stats\": {\n";
+    oss << "        \"min_ns\": " << stats.min_ns << ",\n";
+    oss << "        \"max_ns\": " << stats.max_ns << ",\n";
+    oss << "        \"mean_ns\": " << stats.mean_ns << ",\n";
+    oss << "        \"stdev_ns\": " << stats.stdev_ns << ",\n";
+    oss << "        \"q1_ns\": " << stats.q1_ns << ",\n";
+    oss << "        \"median_ns\": " << stats.median_ns << ",\n";
+    oss << "        \"q3_ns\": " << stats.q3_ns << ",\n";
+    oss << "        \"rounds\": " << stats.rounds << ",\n";
+    oss << "        \"total_time\": " << stats.total_time << ",\n";
+    oss << "        \"iqr_outlier_rounds\": " << stats.iqr_outlier_rounds
+        << ",\n";
+    oss << "        \"stdev_outlier_rounds\": " << stats.stdev_outlier_rounds
+        << ",\n";
+    oss << "        \"iter_per_round\": " << stats.iter_per_round << ",\n";
+    oss << "        \"warmup_iters\": " << stats.warmup_iters << "\n";
+    oss << "      }\n";
+    oss << "    }";
+
+    if (i < benchmarks.size() - 1) {
+      oss << ",";
+    }
+    oss << "\n";
+  }
+
+  oss << "  ]\n";
+  oss << "}";
+
+  // Determine the directory path
+  const char *profile_folder = std::getenv("CODSPEED_PROFILE_FOLDER");
+  std::string directory = profile_folder ? profile_folder : ".";
+
+  // Create the results directory if it does not exist
+  std::filesystem::path results_path = directory + "/results";
+  if (!std::filesystem::exists(results_path)) {
+    if (!std::filesystem::create_directories(results_path)) {
+      std::cerr << "Failed to create directory: " << results_path << std::endl;
+      return;
+    }
+  }
+
+  // Create the file path
+  std::ostringstream file_path;
+  file_path << results_path.string() << "/" << creator_pid << ".json";
+
+  // Write to file
+  std::ofstream out_file(file_path.str());
+  if (out_file.is_open()) {
+    out_file << oss.str();
+    out_file.close();
+    std::cout << "JSON written to " << file_path.str() << std::endl;
+  } else {
+    std::cerr << "Unable to open file " << file_path.str() << std::endl;
+  }
+}
+
+void generate_codspeed_walltime_report(
+    const std::vector<RawWalltimeBenchmark> &raw_walltime_benchmarks) {
+  std::vector<CodspeedWalltimeBenchmark> codspeed_walltime_benchmarks;
+
+  for (const auto &raw_benchmark : raw_walltime_benchmarks) {
+    CodspeedWalltimeBenchmark codspeed_benchmark;
+    codspeed_benchmark.metadata = {raw_benchmark.name, raw_benchmark.uri};
+
+    double total_time =
+        std::accumulate(raw_benchmark.round_times_ns.begin(),
+                        raw_benchmark.round_times_ns.end(), 0.0);
+
+    double mean = raw_benchmark.mean_ns;
+    double median = raw_benchmark.median_ns;
+    double stdev = raw_benchmark.stdev_ns;
+    double q1, q3, iqr;
+    size_t iqr_outlier_rounds, stdev_outlier_rounds;
+    compute_iqr_and_outliers(raw_benchmark.round_times_ns, mean, median, stdev,
+                             q1, q3, iqr, iqr_outlier_rounds,
+                             stdev_outlier_rounds);
+
+    // Populate stats
+    codspeed_benchmark.stats = {
+        *std::min_element(raw_benchmark.round_times_ns.begin(),
+                          raw_benchmark.round_times_ns.end()),
+        *std::max_element(raw_benchmark.round_times_ns.begin(),
+                          raw_benchmark.round_times_ns.end()),
+        mean,
+        stdev,
+        q1,
+        median,
+        q3,
+        raw_benchmark.round_times_ns.size(),
+        total_time,
+        iqr_outlier_rounds,
+        stdev_outlier_rounds,
+        raw_benchmark.iter_per_round,
+        0 // TODO: warmup_iters
+    };
+
+    codspeed_walltime_benchmarks.push_back(codspeed_benchmark);
+  }
+
+  write_codspeed_benchmarks_to_json(codspeed_walltime_benchmarks);
+}
diff --git a/google_benchmark/cmake/Codspeed.cmake b/google_benchmark/cmake/Codspeed.cmake
@@ -24,6 +24,7 @@ target_compile_definitions(
 )
 
 if(DEFINED CODSPEED_MODE)
+    target_compile_definitions(codspeed INTERFACE -DCODSPEED_ENABLED)
     # Define a preprocessor macro based on the build mode
     if(CODSPEED_MODE STREQUAL "instrumentation")
         target_compile_definitions(
diff --git a/google_benchmark/include/benchmark/benchmark.h b/google_benchmark/include/benchmark/benchmark.h
@@ -1438,7 +1438,7 @@ class Fixture : public internal::Benchmark {
   static ::benchmark::internal::Benchmark const* const BENCHMARK_PRIVATE_NAME( \
       n) [[maybe_unused]]
 
-#ifdef CODSPEED_INSTRUMENTATION
+#ifdef CODSPEED_ENABLED
 #include <filesystem>
 #define BENCHMARK(...)                                                   \
   BENCHMARK_PRIVATE_DECLARE(_benchmark_) =                               \
diff --git a/google_benchmark/src/benchmark.cc b/google_benchmark/src/benchmark.cc
@@ -349,6 +349,51 @@ void FlushStreams(BenchmarkReporter* reporter) {
   std::flush(reporter->GetErrorStream());
 }
 
+#ifdef CODSPEED_WALLTIME
+// We use real time by default, but we could offer CPU time usage as a build
+// option, open an issue if you need it.
+RawWalltimeBenchmark generate_raw_walltime_data(const RunResults& run_results) {
+  RawWalltimeBenchmark walltime_data;
+
+  for (const auto& run : run_results.non_aggregates) {
+    walltime_data.uri = run.benchmark_name();
+    size_t pos = walltime_data.uri.rfind("::");
+
+    if (pos != std::string::npos) {
+      walltime_data.name = walltime_data.uri.substr(pos + 2);
+    } else {
+      // Fallback to a placeholder uri, but something is wrong
+      walltime_data.name = walltime_data.uri;
+      walltime_data.uri = "unknown_file::" + walltime_data.uri;
+    }
+
+    walltime_data.iter_per_round = run.iterations;
+    walltime_data.round_times_ns.push_back(run.GetAdjustedRealTime());
+  }
+
+  if (run_results.aggregates_only.empty()) {
+    // If run has no aggreagates, it means that only one round was performed.
+    // Use this time as a mean, median, and set stdev to 0.
+    double only_round_time_ns = walltime_data.round_times_ns[0];
+    walltime_data.mean_ns = only_round_time_ns;
+    walltime_data.median_ns = only_round_time_ns;
+    walltime_data.stdev_ns = 0;
+  } else {
+    for (const auto& aggregate_run : run_results.aggregates_only) {
+      if (aggregate_run.aggregate_name == "mean") {
+        walltime_data.mean_ns = aggregate_run.GetAdjustedRealTime();
+      } else if (aggregate_run.aggregate_name == "median") {
+        walltime_data.median_ns = aggregate_run.GetAdjustedRealTime();
+      } else if (aggregate_run.aggregate_name == "stddev") {
+        walltime_data.stdev_ns = aggregate_run.GetAdjustedRealTime();
+      }
+    }
+  }
+
+  return walltime_data;
+}
+#endif
+
 // Reports in both display and file reporters.
 void Report(BenchmarkReporter* display_reporter,
             BenchmarkReporter* file_reporter, const RunResults& run_results) {
@@ -481,6 +526,9 @@ void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks,
       std::shuffle(repetition_indices.begin(), repetition_indices.end(), g);
     }
 
+#ifdef CODSPEED_WALLTIME
+    std::vector<RawWalltimeBenchmark> codspeed_walltime_data;
+#endif
     for (size_t repetition_index : repetition_indices) {
       internal::BenchmarkRunner& runner = runners[repetition_index];
       runner.DoOneRepetition();
@@ -511,8 +559,15 @@ void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks,
         }
       }
 
+#ifdef CODSPEED_WALLTIME
+      codspeed_walltime_data.push_back(generate_raw_walltime_data(run_results));
+#endif
+
       Report(display_reporter, file_reporter, run_results);
     }
+#ifdef CODSPEED_WALLTIME
+    generate_codspeed_walltime_report(codspeed_walltime_data);
+#endif
   }
   display_reporter->Finalize();
   if (file_reporter != nullptr) {
diff --git a/google_benchmark/src/benchmark_name.cc b/google_benchmark/src/benchmark_name.cc
@@ -16,7 +16,7 @@
 
 namespace benchmark {
 
-#ifdef CODSPEED_INSTRUMENTATION
+#ifdef CODSPEED_ENABLED
 BENCHMARK_EXPORT std::string BenchmarkName::str() const {
   if (args.empty()) {
     return function_name;

Original file line number	Diff line number	Diff line change
`@@ -24,6 +24,7 @@ target_compile_definitions(`
`24`	`24`	`)`
`25`	`25`
`26`	`26`	`if(DEFINED CODSPEED_MODE)`
	`27`	`+ target_compile_definitions(codspeed INTERFACE -DCODSPEED_ENABLED)`
`27`	`28`	`# Define a preprocessor macro based on the build mode`
`28`	`29`	`if(CODSPEED_MODE STREQUAL "instrumentation")`
`29`	`30`	`target_compile_definitions(`