Skip to content

Commit 27e0505

Browse files
GuillaumeLagrangeGuillaumeLagrange
committed
feat(google_benchmark): add walltime support
1 parent 4601025 commit 27e0505

File tree

8 files changed

+312
-28
lines changed

8 files changed

+312
-28
lines changed

‎core/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED True)
1515
include_directories(include)
1616

1717
# Add the library
18-
add_library(codspeed src/codspeed.cpp src/measurement.cpp)
18+
add_library(codspeed src/codspeed.cpp src/measurement.cpp src/walltime.cpp)
1919

2020
# Version
2121
add_compile_definitions(CODSPEED_VERSION="${CODSPEED_VERSION}")

‎core/include/codspeed.h‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,18 @@ class CodSpeed {
2727
bool is_instrumented;
2828
};
2929

30+
// Times are per iteration
31+
struct RawWalltimeBenchmark {
32+
std::string name;
33+
std::string uri;
34+
long iter_per_round;
35+
double mean_ns;
36+
double median_ns;
37+
double stdev_ns;
38+
std::vector<double> round_times_ns;
39+
};
40+
41+
void generate_codspeed_walltime_report(
42+
const std::vector<RawWalltimeBenchmark> &walltime_data_list);
43+
3044
#endif // CODSPEED_H

‎core/src/codspeed.cpp‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,16 @@ void CodSpeed::pop_group() {
3838
}
3939

4040
void CodSpeed::start_benchmark(const std::string &name) {
41-
current_benchmark = name;
41+
std::string uri = name;
42+
43+
// Sanity check URI and add a placeholder if format is wrong
44+
if (name.find("::") == std::string::npos) {
45+
std::string uri = "unknown_file::" + name;
46+
std::cout << "WARNING: Benchmark name does not contain '::'. Using URI: "
47+
<< uri << std::endl;
48+
}
49+
50+
current_benchmark = uri;
4251
measurement_start();
4352
}
4453

‎core/src/walltime.cpp‎

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
#include "codspeed.h"
2+
#include <algorithm>
3+
#include <cmath>
4+
#include <cstdlib>
5+
#include <fstream>
6+
#include <iostream>
7+
#include <numeric>
8+
#include <sstream>
9+
#include <string>
10+
#include <thread>
11+
#include <vector>
12+
13+
const double IQR_OUTLIER_FACTOR = 1.5;
14+
const double STDEV_OUTLIER_FACTOR = 3.0;
15+
16+
// Times are per iteration
17+
struct BenchmarkStats {
18+
double min_ns;
19+
double max_ns;
20+
double mean_ns;
21+
double stdev_ns;
22+
double q1_ns;
23+
double median_ns;
24+
double q3_ns;
25+
uint64_t rounds;
26+
double total_time;
27+
uint64_t iqr_outlier_rounds;
28+
uint64_t stdev_outlier_rounds;
29+
long iter_per_round;
30+
uint64_t warmup_iters;
31+
};
32+
33+
struct BenchmarkMetadata {
34+
std::string name;
35+
std::string uri;
36+
};
37+
38+
struct CodspeedWalltimeBenchmark {
39+
BenchmarkMetadata metadata;
40+
BenchmarkStats stats;
41+
};
42+
43+
double compute_quantile(const std::vector<double> &data, double quantile) {
44+
size_t n = data.size();
45+
if (n == 0)
46+
return 0.0;
47+
48+
double pos = quantile * (n - 1);
49+
size_t k = static_cast<size_t>(pos);
50+
double d = pos - k;
51+
52+
if (k + 1 < n) {
53+
return data[k] + d * (data[k + 1] - data[k]);
54+
}
55+
return data[k];
56+
}
57+
58+
void compute_iqr_and_outliers(const std::vector<double> &times_ns, double mean,
59+
double median, double stdev, double &q1,
60+
double &q3, double &iqr,
61+
size_t &iqr_outlier_rounds,
62+
size_t &stdev_outlier_rounds) {
63+
std::vector<double> sorted_times = times_ns;
64+
std::sort(sorted_times.begin(), sorted_times.end());
65+
66+
q1 = compute_quantile(sorted_times, 0.25);
67+
q3 = compute_quantile(sorted_times, 0.75);
68+
69+
iqr = q3 - q1;
70+
71+
const double IQR_OUTLIER_FACTOR = 1.5;
72+
const double STDEV_OUTLIER_FACTOR = 3.0;
73+
74+
iqr_outlier_rounds =
75+
std::count_if(sorted_times.begin(), sorted_times.end(),
76+
[q1, q3, iqr, IQR_OUTLIER_FACTOR](double x) {
77+
return x < q1 - IQR_OUTLIER_FACTOR * iqr ||
78+
x > q3 + IQR_OUTLIER_FACTOR * iqr;
79+
});
80+
81+
stdev_outlier_rounds =
82+
std::count_if(sorted_times.begin(), sorted_times.end(),
83+
[mean, stdev, STDEV_OUTLIER_FACTOR](double x) {
84+
return x < mean - STDEV_OUTLIER_FACTOR * stdev ||
85+
x > mean + STDEV_OUTLIER_FACTOR * stdev;
86+
});
87+
}
88+
89+
void write_codspeed_benchmarks_to_json(
90+
const std::vector<CodspeedWalltimeBenchmark> &benchmarks) {
91+
std::ostringstream oss;
92+
93+
std::string creator_name = "codspeed-cpp";
94+
std::string creator_version = CODSPEED_VERSION;
95+
std::thread::id creator_pid = std::this_thread::get_id();
96+
std::string instrument_type = "walltime";
97+
98+
oss << "{\n";
99+
oss << " \"creator\": {\n";
100+
oss << " \"name\": \"" << creator_name << "\",\n";
101+
oss << " \"version\": \"" << creator_version << "\",\n";
102+
oss << " \"pid\": " << creator_pid << "\n";
103+
oss << " },\n";
104+
oss << " \"instrument\": {\n";
105+
oss << " \"type\": \"" << instrument_type << "\"\n";
106+
oss << " },\n";
107+
oss << " \"benchmarks\": [\n";
108+
109+
for (size_t i = 0; i < benchmarks.size(); ++i) {
110+
const auto &benchmark = benchmarks[i];
111+
const auto &stats = benchmark.stats;
112+
const auto &metadata = benchmark.metadata;
113+
114+
oss << " {\n";
115+
oss << " \"name\": \"" << metadata.name << "\",\n";
116+
oss << " \"uri\": \"" << metadata.uri << "\",\n";
117+
// TODO: Manage config fields from actual configuration
118+
oss << " \"config\": {\n";
119+
oss << " \"warmup_time_ns\": null,\n";
120+
oss << " \"min_round_time_ns\": null,\n";
121+
oss << " \"max_time_ns\": null,\n";
122+
oss << " \"max_rounds\": null\n";
123+
oss << " },\n";
124+
oss << " \"stats\": {\n";
125+
oss << " \"min_ns\": " << stats.min_ns << ",\n";
126+
oss << " \"max_ns\": " << stats.max_ns << ",\n";
127+
oss << " \"mean_ns\": " << stats.mean_ns << ",\n";
128+
oss << " \"stdev_ns\": " << stats.stdev_ns << ",\n";
129+
oss << " \"q1_ns\": " << stats.q1_ns << ",\n";
130+
oss << " \"median_ns\": " << stats.median_ns << ",\n";
131+
oss << " \"q3_ns\": " << stats.q3_ns << ",\n";
132+
oss << " \"rounds\": " << stats.rounds << ",\n";
133+
oss << " \"total_time\": " << stats.total_time << ",\n";
134+
oss << " \"iqr_outlier_rounds\": " << stats.iqr_outlier_rounds
135+
<< ",\n";
136+
oss << " \"stdev_outlier_rounds\": " << stats.stdev_outlier_rounds
137+
<< ",\n";
138+
oss << " \"iter_per_round\": " << stats.iter_per_round << ",\n";
139+
oss << " \"warmup_iters\": " << stats.warmup_iters << "\n";
140+
oss << " }\n";
141+
oss << " }";
142+
143+
if (i < benchmarks.size() - 1) {
144+
oss << ",";
145+
}
146+
oss << "\n";
147+
}
148+
149+
oss << " ]\n";
150+
oss << "}";
151+
152+
// Determine the directory path
153+
const char *profile_folder = std::getenv("CODSPEED_PROFILE_FOLDER");
154+
std::string directory = profile_folder ? profile_folder : ".";
155+
156+
// Create the file path
157+
std::ostringstream file_path;
158+
file_path << directory << "/" << creator_pid << ".json";
159+
160+
// Write to file
161+
std::ofstream out_file(file_path.str());
162+
if (out_file.is_open()) {
163+
out_file << oss.str();
164+
out_file.close();
165+
std::cout << "JSON written to " << file_path.str() << std::endl;
166+
} else {
167+
std::cerr << "Unable to open file " << file_path.str() << std::endl;
168+
}
169+
}
170+
171+
void generate_codspeed_walltime_report(
172+
const std::vector<RawWalltimeBenchmark> &raw_walltime_benchmarks) {
173+
std::vector<CodspeedWalltimeBenchmark> codspeed_walltime_benchmarks;
174+
175+
for (const auto &raw_benchmark : raw_walltime_benchmarks) {
176+
CodspeedWalltimeBenchmark codspeed_benchmark;
177+
codspeed_benchmark.metadata = {raw_benchmark.name, raw_benchmark.uri};
178+
179+
double total_time =
180+
std::accumulate(raw_benchmark.round_times_ns.begin(),
181+
raw_benchmark.round_times_ns.end(), 0.0);
182+
183+
double mean = raw_benchmark.mean_ns;
184+
double median = raw_benchmark.median_ns;
185+
double stdev = raw_benchmark.stdev_ns;
186+
double q1, q3, iqr;
187+
size_t iqr_outlier_rounds, stdev_outlier_rounds;
188+
compute_iqr_and_outliers(raw_benchmark.round_times_ns, mean, median, stdev,
189+
q1, q3, iqr, iqr_outlier_rounds,
190+
stdev_outlier_rounds);
191+
192+
// Populate stats
193+
codspeed_benchmark.stats = {
194+
*std::min_element(raw_benchmark.round_times_ns.begin(),
195+
raw_benchmark.round_times_ns.end()),
196+
*std::max_element(raw_benchmark.round_times_ns.begin(),
197+
raw_benchmark.round_times_ns.end()),
198+
mean,
199+
stdev,
200+
q1,
201+
median,
202+
q3,
203+
raw_benchmark.round_times_ns.size(),
204+
total_time,
205+
iqr_outlier_rounds,
206+
stdev_outlier_rounds,
207+
raw_benchmark.iter_per_round,
208+
0 // TODO: warmup_iters
209+
};
210+
211+
codspeed_walltime_benchmarks.push_back(codspeed_benchmark);
212+
}
213+
214+
write_codspeed_benchmarks_to_json(codspeed_walltime_benchmarks);
215+
}

‎google_benchmark/cmake/Codspeed.cmake‎

Lines changed: 11 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,30 +8,19 @@ execute_process(
88
target_compile_definitions(codspeed
99
INTERFACE -DCODSPEED_GIT_ROOT_DIR="${GIT_ROOT_DIR}")
1010

11-
# Step 1: Check if CODSPEED_MODE is set via the command line
12-
13-
# CMake cache kind of breaks this mechanism, keeping it for first time
14-
# defaulting
15-
if(NOT DEFINED CODSPEED_MODE)
16-
# Step 2: Check the environment variable CODSPEED_MODE
17-
if(DEFINED $ENV{CODSPEED_RUNNER_MODE})
18-
set(CODSPEED_MODE $ENV{CODSPEED_RUNNER_MODE} FORCE)
11+
if(DEFINED CODSPEED_MODE)
12+
target_compile_definitions(codspeed INTERFACE -DCODSPEED_ENABLED)
13+
# Define a preprocessor macro based on the build mode
14+
if(CODSPEED_MODE STREQUAL "instrumentation")
15+
target_compile_definitions(codspeed INTERFACE -DCODSPEED_INSTRUMENTATION)
16+
elseif(CODSPEED_MODE STREQUAL "walltime")
17+
target_compile_definitions(codspeed INTERFACE -DCODSPEED_WALLTIME)
1918
else()
20-
# Step 3: Default to "instrumentation" if no value is provided
21-
set(CODSPEED_MODE "instrumentation")
19+
message(
20+
FATAL_ERROR
21+
"Invalid build mode: ${CODSPEED_MODE}. Use 'instrumentation' or 'walltime'."
22+
)
2223
endif()
2324
endif()
2425

25-
# Define a preprocessor macro based on the build mode
26-
if(CODSPEED_MODE STREQUAL "instrumentation")
27-
target_compile_definitions(codspeed INTERFACE -DCODSPEED_INSTRUMENTATION)
28-
elseif(CODSPEED_MODE STREQUAL "walltime")
29-
target_compile_definitions(codspeed INTERFACE -DCODSPEED_WALLTIME)
30-
else()
31-
message(
32-
FATAL_ERROR
33-
"Invalid build mode: ${CODSPEED_MODE}. Use 'instrumentation' or 'walltime'."
34-
)
35-
endif()
36-
3726
message(STATUS "Build mode set to: ${CODSPEED_MODE}")

‎google_benchmark/include/benchmark/benchmark.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1438,7 +1438,7 @@ class Fixture : public internal::Benchmark {
14381438
static ::benchmark::internal::Benchmark const* const BENCHMARK_PRIVATE_NAME( \
14391439
n) [[maybe_unused]]
14401440

1441-
#ifdef CODSPEED_INSTRUMENTATION
1441+
#ifdef CODSPEED_ENABLED
14421442
#include <filesystem>
14431443
#define BENCHMARK(...) \
14441444
BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \

0 commit comments

Comments
 (0)