From e7a1994acf224da7b484fece12c50de9c63c81d3 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 26 Feb 2025 12:46:22 +0100 Subject: [PATCH 1/3] feat(google_benchmark): add walltime support --- core/CMakeLists.txt | 2 +- core/include/codspeed.h | 14 ++ core/src/codspeed.cpp | 11 +- core/src/walltime.cpp | 225 ++++++++++++++++++ google_benchmark/cmake/Codspeed.cmake | 1 + .../include/benchmark/benchmark.h | 2 +- google_benchmark/src/benchmark.cc | 55 +++++ google_benchmark/src/benchmark_name.cc | 2 +- 8 files changed, 308 insertions(+), 4 deletions(-) create mode 100644 core/src/walltime.cpp diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index fa94d6b..36e5de6 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -12,7 +12,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED True) include_directories(include) # Add the library -add_library(codspeed src/codspeed.cpp) +add_library(codspeed src/codspeed.cpp src/walltime.cpp) # Version add_compile_definitions(CODSPEED_VERSION="${CODSPEED_VERSION}") diff --git a/core/include/codspeed.h b/core/include/codspeed.h index 78ad431..e1b7ea4 100644 --- a/core/include/codspeed.h +++ b/core/include/codspeed.h @@ -27,4 +27,18 @@ class CodSpeed { bool is_instrumented; }; +// Times are per iteration +struct RawWalltimeBenchmark { + std::string name; + std::string uri; + long iter_per_round; + double mean_ns; + double median_ns; + double stdev_ns; + std::vector round_times_ns; +}; + +void generate_codspeed_walltime_report( + const std::vector &walltime_data_list); + #endif // CODSPEED_H diff --git a/core/src/codspeed.cpp b/core/src/codspeed.cpp index 327fc0c..1e43c06 100644 --- a/core/src/codspeed.cpp +++ b/core/src/codspeed.cpp @@ -38,7 +38,16 @@ void CodSpeed::pop_group() { } void CodSpeed::start_benchmark(const std::string &name) { - current_benchmark = name; + std::string uri = name; + + // Sanity check URI and add a placeholder if format is wrong + if (name.find("::") == std::string::npos) { + std::string uri = "unknown_file::" + name; + std::cout << "WARNING: Benchmark name does not contain '::'. Using URI: " + << uri << std::endl; + } + + current_benchmark = uri; measurement_start(); } diff --git a/core/src/walltime.cpp b/core/src/walltime.cpp new file mode 100644 index 0000000..bf4f4aa --- /dev/null +++ b/core/src/walltime.cpp @@ -0,0 +1,225 @@ +#include "codspeed.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +const double IQR_OUTLIER_FACTOR = 1.5; +const double STDEV_OUTLIER_FACTOR = 3.0; + +// Times are per iteration +struct BenchmarkStats { + double min_ns; + double max_ns; + double mean_ns; + double stdev_ns; + double q1_ns; + double median_ns; + double q3_ns; + uint64_t rounds; + double total_time; + uint64_t iqr_outlier_rounds; + uint64_t stdev_outlier_rounds; + long iter_per_round; + uint64_t warmup_iters; +}; + +struct BenchmarkMetadata { + std::string name; + std::string uri; +}; + +struct CodspeedWalltimeBenchmark { + BenchmarkMetadata metadata; + BenchmarkStats stats; +}; + +double compute_quantile(const std::vector &data, double quantile) { + size_t n = data.size(); + if (n == 0) + return 0.0; + + double pos = quantile * (n - 1); + size_t k = static_cast(pos); + double d = pos - k; + + if (k + 1 < n) { + return data[k] + d * (data[k + 1] - data[k]); + } + return data[k]; +} + +void compute_iqr_and_outliers(const std::vector ×_ns, double mean, + double median, double stdev, double &q1, + double &q3, double &iqr, + size_t &iqr_outlier_rounds, + size_t &stdev_outlier_rounds) { + std::vector sorted_times = times_ns; + std::sort(sorted_times.begin(), sorted_times.end()); + + q1 = compute_quantile(sorted_times, 0.25); + q3 = compute_quantile(sorted_times, 0.75); + + iqr = q3 - q1; + + const double IQR_OUTLIER_FACTOR = 1.5; + const double STDEV_OUTLIER_FACTOR = 3.0; + + iqr_outlier_rounds = + std::count_if(sorted_times.begin(), sorted_times.end(), + [q1, q3, iqr, IQR_OUTLIER_FACTOR](double x) { + return x < q1 - IQR_OUTLIER_FACTOR * iqr || + x > q3 + IQR_OUTLIER_FACTOR * iqr; + }); + + stdev_outlier_rounds = + std::count_if(sorted_times.begin(), sorted_times.end(), + [mean, stdev, STDEV_OUTLIER_FACTOR](double x) { + return x < mean - STDEV_OUTLIER_FACTOR * stdev || + x > mean + STDEV_OUTLIER_FACTOR * stdev; + }); +} + +void write_codspeed_benchmarks_to_json( + const std::vector &benchmarks) { + std::ostringstream oss; + + std::string creator_name = "codspeed-cpp"; + std::string creator_version = CODSPEED_VERSION; + std::thread::id creator_pid = std::this_thread::get_id(); + std::string instrument_type = "walltime"; + + oss << "{\n"; + oss << " \"creator\": {\n"; + oss << " \"name\": \"" << creator_name << "\",\n"; + oss << " \"version\": \"" << creator_version << "\",\n"; + oss << " \"pid\": " << creator_pid << "\n"; + oss << " },\n"; + oss << " \"instrument\": {\n"; + oss << " \"type\": \"" << instrument_type << "\"\n"; + oss << " },\n"; + oss << " \"benchmarks\": [\n"; + + for (size_t i = 0; i < benchmarks.size(); ++i) { + const auto &benchmark = benchmarks[i]; + const auto &stats = benchmark.stats; + const auto &metadata = benchmark.metadata; + + oss << " {\n"; + oss << " \"name\": \"" << metadata.name << "\",\n"; + oss << " \"uri\": \"" << metadata.uri << "\",\n"; + // TODO: Manage config fields from actual configuration + oss << " \"config\": {\n"; + oss << " \"warmup_time_ns\": null,\n"; + oss << " \"min_round_time_ns\": null,\n"; + oss << " \"max_time_ns\": null,\n"; + oss << " \"max_rounds\": null\n"; + oss << " },\n"; + oss << " \"stats\": {\n"; + oss << " \"min_ns\": " << stats.min_ns << ",\n"; + oss << " \"max_ns\": " << stats.max_ns << ",\n"; + oss << " \"mean_ns\": " << stats.mean_ns << ",\n"; + oss << " \"stdev_ns\": " << stats.stdev_ns << ",\n"; + oss << " \"q1_ns\": " << stats.q1_ns << ",\n"; + oss << " \"median_ns\": " << stats.median_ns << ",\n"; + oss << " \"q3_ns\": " << stats.q3_ns << ",\n"; + oss << " \"rounds\": " << stats.rounds << ",\n"; + oss << " \"total_time\": " << stats.total_time << ",\n"; + oss << " \"iqr_outlier_rounds\": " << stats.iqr_outlier_rounds + << ",\n"; + oss << " \"stdev_outlier_rounds\": " << stats.stdev_outlier_rounds + << ",\n"; + oss << " \"iter_per_round\": " << stats.iter_per_round << ",\n"; + oss << " \"warmup_iters\": " << stats.warmup_iters << "\n"; + oss << " }\n"; + oss << " }"; + + if (i < benchmarks.size() - 1) { + oss << ","; + } + oss << "\n"; + } + + oss << " ]\n"; + oss << "}"; + + // Determine the directory path + const char *profile_folder = std::getenv("CODSPEED_PROFILE_FOLDER"); + std::string directory = profile_folder ? profile_folder : "."; + + // Create the results directory if it does not exist + std::filesystem::path results_path = directory + "/results"; + if (!std::filesystem::exists(results_path)) { + if (!std::filesystem::create_directories(results_path)) { + std::cerr << "Failed to create directory: " << results_path << std::endl; + return; + } + } + + // Create the file path + std::ostringstream file_path; + file_path << results_path.string() << "/" << creator_pid << ".json"; + + // Write to file + std::ofstream out_file(file_path.str()); + if (out_file.is_open()) { + out_file << oss.str(); + out_file.close(); + std::cout << "JSON written to " << file_path.str() << std::endl; + } else { + std::cerr << "Unable to open file " << file_path.str() << std::endl; + } +} + +void generate_codspeed_walltime_report( + const std::vector &raw_walltime_benchmarks) { + std::vector codspeed_walltime_benchmarks; + + for (const auto &raw_benchmark : raw_walltime_benchmarks) { + CodspeedWalltimeBenchmark codspeed_benchmark; + codspeed_benchmark.metadata = {raw_benchmark.name, raw_benchmark.uri}; + + double total_time = + std::accumulate(raw_benchmark.round_times_ns.begin(), + raw_benchmark.round_times_ns.end(), 0.0); + + double mean = raw_benchmark.mean_ns; + double median = raw_benchmark.median_ns; + double stdev = raw_benchmark.stdev_ns; + double q1, q3, iqr; + size_t iqr_outlier_rounds, stdev_outlier_rounds; + compute_iqr_and_outliers(raw_benchmark.round_times_ns, mean, median, stdev, + q1, q3, iqr, iqr_outlier_rounds, + stdev_outlier_rounds); + + // Populate stats + codspeed_benchmark.stats = { + *std::min_element(raw_benchmark.round_times_ns.begin(), + raw_benchmark.round_times_ns.end()), + *std::max_element(raw_benchmark.round_times_ns.begin(), + raw_benchmark.round_times_ns.end()), + mean, + stdev, + q1, + median, + q3, + raw_benchmark.round_times_ns.size(), + total_time, + iqr_outlier_rounds, + stdev_outlier_rounds, + raw_benchmark.iter_per_round, + 0 // TODO: warmup_iters + }; + + codspeed_walltime_benchmarks.push_back(codspeed_benchmark); + } + + write_codspeed_benchmarks_to_json(codspeed_walltime_benchmarks); +} diff --git a/google_benchmark/cmake/Codspeed.cmake b/google_benchmark/cmake/Codspeed.cmake index 3fb86da..958c09a 100644 --- a/google_benchmark/cmake/Codspeed.cmake +++ b/google_benchmark/cmake/Codspeed.cmake @@ -24,6 +24,7 @@ target_compile_definitions( ) if(DEFINED CODSPEED_MODE) + target_compile_definitions(codspeed INTERFACE -DCODSPEED_ENABLED) # Define a preprocessor macro based on the build mode if(CODSPEED_MODE STREQUAL "instrumentation") target_compile_definitions( diff --git a/google_benchmark/include/benchmark/benchmark.h b/google_benchmark/include/benchmark/benchmark.h index b88c83a..2ae1f4a 100644 --- a/google_benchmark/include/benchmark/benchmark.h +++ b/google_benchmark/include/benchmark/benchmark.h @@ -1438,7 +1438,7 @@ class Fixture : public internal::Benchmark { static ::benchmark::internal::Benchmark const* const BENCHMARK_PRIVATE_NAME( \ n) [[maybe_unused]] -#ifdef CODSPEED_INSTRUMENTATION +#ifdef CODSPEED_ENABLED #include #define BENCHMARK(...) \ BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ diff --git a/google_benchmark/src/benchmark.cc b/google_benchmark/src/benchmark.cc index ab4160c..e9574c8 100644 --- a/google_benchmark/src/benchmark.cc +++ b/google_benchmark/src/benchmark.cc @@ -349,6 +349,51 @@ void FlushStreams(BenchmarkReporter* reporter) { std::flush(reporter->GetErrorStream()); } +#ifdef CODSPEED_WALLTIME +// We use real time by default, but we could offer CPU time usage as a build +// option, open an issue if you need it. +RawWalltimeBenchmark generate_raw_walltime_data(const RunResults& run_results) { + RawWalltimeBenchmark walltime_data; + + for (const auto& run : run_results.non_aggregates) { + walltime_data.uri = run.benchmark_name(); + size_t pos = walltime_data.uri.rfind("::"); + + if (pos != std::string::npos) { + walltime_data.name = walltime_data.uri.substr(pos + 2); + } else { + // Fallback to a placeholder uri, but something is wrong + walltime_data.name = walltime_data.uri; + walltime_data.uri = "unknown_file::" + walltime_data.uri; + } + + walltime_data.iter_per_round = run.iterations; + walltime_data.round_times_ns.push_back(run.GetAdjustedRealTime()); + } + + if (run_results.aggregates_only.empty()) { + // If run has no aggreagates, it means that only one round was performed. + // Use this time as a mean, median, and set stdev to 0. + double only_round_time_ns = walltime_data.round_times_ns[0]; + walltime_data.mean_ns = only_round_time_ns; + walltime_data.median_ns = only_round_time_ns; + walltime_data.stdev_ns = 0; + } else { + for (const auto& aggregate_run : run_results.aggregates_only) { + if (aggregate_run.aggregate_name == "mean") { + walltime_data.mean_ns = aggregate_run.GetAdjustedRealTime(); + } else if (aggregate_run.aggregate_name == "median") { + walltime_data.median_ns = aggregate_run.GetAdjustedRealTime(); + } else if (aggregate_run.aggregate_name == "stddev") { + walltime_data.stdev_ns = aggregate_run.GetAdjustedRealTime(); + } + } + } + + return walltime_data; +} +#endif + // Reports in both display and file reporters. void Report(BenchmarkReporter* display_reporter, BenchmarkReporter* file_reporter, const RunResults& run_results) { @@ -481,6 +526,9 @@ void RunBenchmarks(const std::vector& benchmarks, std::shuffle(repetition_indices.begin(), repetition_indices.end(), g); } +#ifdef CODSPEED_WALLTIME + std::vector codspeed_walltime_data; +#endif for (size_t repetition_index : repetition_indices) { internal::BenchmarkRunner& runner = runners[repetition_index]; runner.DoOneRepetition(); @@ -511,8 +559,15 @@ void RunBenchmarks(const std::vector& benchmarks, } } +#ifdef CODSPEED_WALLTIME + codspeed_walltime_data.push_back(generate_raw_walltime_data(run_results)); +#endif + Report(display_reporter, file_reporter, run_results); } +#ifdef CODSPEED_WALLTIME + generate_codspeed_walltime_report(codspeed_walltime_data); +#endif } display_reporter->Finalize(); if (file_reporter != nullptr) { diff --git a/google_benchmark/src/benchmark_name.cc b/google_benchmark/src/benchmark_name.cc index 0407c2a..8a1b78c 100644 --- a/google_benchmark/src/benchmark_name.cc +++ b/google_benchmark/src/benchmark_name.cc @@ -16,7 +16,7 @@ namespace benchmark { -#ifdef CODSPEED_INSTRUMENTATION +#ifdef CODSPEED_ENABLED BENCHMARK_EXPORT std::string BenchmarkName::str() const { if (args.empty()) { return function_name; From 5e7a73610bd9ba3661832ce55c062a147e6f85af Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 26 Feb 2025 12:49:05 +0100 Subject: [PATCH 2/3] ci: build without codpseed in CI --- .github/workflows/ci.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90c499a..5767d78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,3 +56,25 @@ jobs: cd examples/google_benchmark/build-walltime cmake -DCODSPEED_MODE=walltime .. make -j + + build-no-codspeed: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Cache build + uses: actions/cache@v3 + with: + path: examples/google_benchmark/build-no-codspeed + key: ${{ runner.os }}-build-no-codspeed-${{ hashFiles('**/CMakeLists.txt', '**/examples/google_benchmark/**') }} + + - name: Create build directory + run: mkdir -p examples/google_benchmark/build-no-codspeed + + - name: Build benchmark example without codspeed + run: | + cd examples/google_benchmark/build-no-codspeed + cmake .. + make -j From 2a6438ad2ba28c3345f8e5c4d0c238d3093615cf Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 26 Feb 2025 12:51:29 +0100 Subject: [PATCH 3/3] ci: run walltime benchmarks in CI --- .github/workflows/ci.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5767d78..8600e1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,8 +35,8 @@ jobs: run: examples/google_benchmark/build-instrumentation/benchmark_example token: ${{ secrets.CODSPEED_TOKEN }} - build-walltime: - runs-on: ubuntu-latest + walltime: + runs-on: codspeed-macro steps: - name: Checkout code @@ -57,6 +57,13 @@ jobs: cmake -DCODSPEED_MODE=walltime .. make -j + - name: Run the benchmarks + uses: CodSpeedHQ/action@main + with: + run: examples/google_benchmark/build-walltime/benchmark_example + token: ${{ secrets.CODSPEED_TOKEN }} + + build-no-codspeed: runs-on: ubuntu-latest