From 10fe19b873f2de87a6390ced4bb0bc3316f14eae Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 11 Mar 2025 18:10:25 +0100 Subject: [PATCH 01/10] fix(google_benchmark): generate proper uri --- examples/google_benchmark/main.cpp | 54 +++++++++++++++++++ .../include/benchmark/benchmark.h | 41 ++++++++------ 2 files changed, 79 insertions(+), 16 deletions(-) diff --git a/examples/google_benchmark/main.cpp b/examples/google_benchmark/main.cpp index 7886fee..bc6cb07 100644 --- a/examples/google_benchmark/main.cpp +++ b/examples/google_benchmark/main.cpp @@ -1,6 +1,60 @@ #include #include +// Passing Arbitrary Arguments to a Benchmark +// See: +// https://github.com/google/benchmark/blob/main/docs/user_guide.md#passing-arbitrary-arguments-to-a-benchmark +template +void BM_takes_args(benchmark::State &state, Args &&...args) { + auto args_tuple = std::make_tuple(std::move(args)...); + for (auto _ : state) { + } +} +BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); +BENCHMARK_CAPTURE(BM_takes_args, int_test, 42, 43); + +// Benchmark template +// +template void BM_VectorPushBack(benchmark::State &state) { + std::vector v; + for (auto _ : state) { + v.push_back(T()); + } +} +BENCHMARK_TEMPLATE(BM_VectorPushBack, int); +BENCHMARK_TEMPLATE(BM_VectorPushBack, std::string); + +template void BM_Template1(benchmark::State &state) { + T val = T(); + for (auto _ : state) { + benchmark::DoNotOptimize(val++); + } +} +BENCHMARK_TEMPLATE1(BM_Template1, int); + +template void BM_Template2(benchmark::State &state) { + T t = T(); + U u = U(); + for (auto _ : state) { + benchmark::DoNotOptimize(t + u); + } +} +BENCHMARK_TEMPLATE2(BM_Template2, int, double); + +// Benchmark template capture +// +template +void BM_takes_args(benchmark::State &state, ExtraArgs &&...extra_args) { + auto args_tuple = std::make_tuple(std::move(extra_args)...); + for (auto _ : state) { + } +} +BENCHMARK_TEMPLATE1_CAPTURE(BM_takes_args, void, int_string_test, 42, + std::string("abc")); + +BENCHMARK_TEMPLATE2_CAPTURE(BM_takes_args, int, double, two_type_test, 42, + std::string("abc")); + // Function to benchmark static void BM_rand_vector(benchmark::State &state) { std::vector v; diff --git a/google_benchmark/include/benchmark/benchmark.h b/google_benchmark/include/benchmark/benchmark.h index 2ae1f4a..31ec5cb 100644 --- a/google_benchmark/include/benchmark/benchmark.h +++ b/google_benchmark/include/benchmark/benchmark.h @@ -1440,21 +1440,23 @@ class Fixture : public internal::Benchmark { #ifdef CODSPEED_ENABLED #include -#define BENCHMARK(...) \ - BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - std::filesystem::relative(__FILE__, CODSPEED_GIT_ROOT_DIR) \ - .string() + \ - "::" + #__VA_ARGS__, \ - __VA_ARGS__))) +#define CUR_FILE \ + std::filesystem::relative(__FILE__, CODSPEED_GIT_ROOT_DIR).string() + "::" +#define TYPE_START "[" +#define TYPE_END "]" +#define FUNC_TEST_SEP "::" #else +#define CUR_FILE std::string() +#define TYPE_START "<" +#define TYPE_END ">" +#define FUNC_TEST_SEP "/" +#endif + #define BENCHMARK(...) \ BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - #__VA_ARGS__, __VA_ARGS__))) -#endif + CUR_FILE + #__VA_ARGS__, __VA_ARGS__))) // Old-style macros #define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a)) @@ -1479,7 +1481,7 @@ class Fixture : public internal::Benchmark { BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - #func "/" #test_case_name, \ + CUR_FILE + #func FUNC_TEST_SEP #test_case_name, \ [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) // This will register a benchmark for a templatized function. For example: @@ -1494,19 +1496,20 @@ class Fixture : public internal::Benchmark { BENCHMARK_PRIVATE_DECLARE(n) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - #n "<" #a ">", n))) + CUR_FILE + #n TYPE_START #a TYPE_END, n))) #define BENCHMARK_TEMPLATE2(n, a, b) \ BENCHMARK_PRIVATE_DECLARE(n) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - #n "<" #a "," #b ">", n))) + CUR_FILE + #n TYPE_START #a "," #b TYPE_END, n))) #define BENCHMARK_TEMPLATE(n, ...) \ BENCHMARK_PRIVATE_DECLARE(n) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - #n "<" #__VA_ARGS__ ">", n<__VA_ARGS__>))) + CUR_FILE + #n TYPE_START #__VA_ARGS__ TYPE_END, \ + n<__VA_ARGS__>))) // This will register a benchmark for a templatized function, // with the additional arguments specified by `...`. @@ -1520,15 +1523,20 @@ class Fixture : public internal::Benchmark { // /* Registers a benchmark named "BM_takes_args/int_string_test` */ // BENCHMARK_TEMPLATE1_CAPTURE(BM_takes_args, void, int_string_test, 42, // std::string("abc")); +#ifdef CODSPEED_ENABLED +#define BENCHMARK_TEMPLATE1_CAPTURE(func, a, test_case_name, ...) \ + BENCHMARK_CAPTURE(func, test_case_name, __VA_ARGS__) +#else #define BENCHMARK_TEMPLATE1_CAPTURE(func, a, test_case_name, ...) \ BENCHMARK_CAPTURE(func, test_case_name, __VA_ARGS__) +#endif #define BENCHMARK_TEMPLATE2_CAPTURE(func, a, b, test_case_name, ...) \ BENCHMARK_PRIVATE_DECLARE(func) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - #func "<" #a "," #b ">" \ - "/" #test_case_name, \ + CUR_FILE + #func TYPE_START #a \ + "," #b TYPE_END FUNC_TEST_SEP #test_case_name, \ [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) #define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ @@ -1680,6 +1688,7 @@ struct BENCHMARK_EXPORT SystemInfo { // which allows individual fields to be modified or cleared before // building the final name using 'str()'. struct BENCHMARK_EXPORT BenchmarkName { + std::string path; std::string function_name; std::string args; std::string min_time; From 151e9e1c54090cedb0eecf82051f2a6a4a56e86b Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 12 Mar 2025 09:54:31 +0100 Subject: [PATCH 02/10] fix(google_benchmark): remarks from PR --- .../include/benchmark/benchmark.h | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/google_benchmark/include/benchmark/benchmark.h b/google_benchmark/include/benchmark/benchmark.h index 31ec5cb..5279e37 100644 --- a/google_benchmark/include/benchmark/benchmark.h +++ b/google_benchmark/include/benchmark/benchmark.h @@ -1442,14 +1442,27 @@ class Fixture : public internal::Benchmark { #include #define CUR_FILE \ std::filesystem::relative(__FILE__, CODSPEED_GIT_ROOT_DIR).string() + "::" + +// Transforms `BM_Foo` into `BM_Foo[int, double]`. #define TYPE_START "[" #define TYPE_END "]" -#define FUNC_TEST_SEP "::" + +// Transforms `BM_Foo/int_arg` into `BM_Foo[int_arg]`. +#define NAME_START "[" +#define NAME_END "]" + +// Extra space after the comma for readability +#define COMMA ", " #else #define CUR_FILE std::string() + #define TYPE_START "<" #define TYPE_END ">" -#define FUNC_TEST_SEP "/" + +#define NAME_START "/" +#define NAME_END "" + +#define COMMA "," #endif #define BENCHMARK(...) \ @@ -1481,7 +1494,7 @@ class Fixture : public internal::Benchmark { BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - CUR_FILE + #func FUNC_TEST_SEP #test_case_name, \ + CUR_FILE + #func NAME_START #test_case_name NAME_END, \ [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) // This will register a benchmark for a templatized function. For example: @@ -1502,7 +1515,7 @@ class Fixture : public internal::Benchmark { BENCHMARK_PRIVATE_DECLARE(n) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - CUR_FILE + #n TYPE_START #a "," #b TYPE_END, n))) + CUR_FILE + #n TYPE_START #a COMMA #b TYPE_END, n))) #define BENCHMARK_TEMPLATE(n, ...) \ BENCHMARK_PRIVATE_DECLARE(n) = \ @@ -1531,13 +1544,22 @@ class Fixture : public internal::Benchmark { BENCHMARK_CAPTURE(func, test_case_name, __VA_ARGS__) #endif +#ifdef CODSPEED_ENABLED +#define BENCHMARK_TEMPLATE2_CAPTURE(func, a, b, test_case_name, ...) \ + BENCHMARK_PRIVATE_DECLARE(func) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + std::make_unique<::benchmark::internal::FunctionBenchmark>( \ + CUR_FILE + #func "[" #test_case_name COMMA #a COMMA #b "]", \ + [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) +#else #define BENCHMARK_TEMPLATE2_CAPTURE(func, a, b, test_case_name, ...) \ BENCHMARK_PRIVATE_DECLARE(func) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - CUR_FILE + #func TYPE_START #a \ - "," #b TYPE_END FUNC_TEST_SEP #test_case_name, \ + #func "<" #a "," #b ">" \ + "/" #test_case_name, \ [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) +#endif #define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ class BaseClass##_##Method##_Benchmark : public BaseClass { \ @@ -1688,7 +1710,6 @@ struct BENCHMARK_EXPORT SystemInfo { // which allows individual fields to be modified or cleared before // building the final name using 'str()'. struct BENCHMARK_EXPORT BenchmarkName { - std::string path; std::string function_name; std::string args; std::string min_time; From bc287a44e369faf76eba2835f7f04f117a24c0c4 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 12 Mar 2025 10:31:34 +0100 Subject: [PATCH 03/10] chore(google_benchmark): rework benches --- examples/google_benchmark/main.cpp | 52 ++----------------- examples/google_benchmark/template_bench.hpp | 51 ++++++++++++++++++ .../include/benchmark/benchmark.h | 12 ++++- 3 files changed, 66 insertions(+), 49 deletions(-) create mode 100644 examples/google_benchmark/template_bench.hpp diff --git a/examples/google_benchmark/main.cpp b/examples/google_benchmark/main.cpp index bc6cb07..143259c 100644 --- a/examples/google_benchmark/main.cpp +++ b/examples/google_benchmark/main.cpp @@ -1,59 +1,15 @@ +#include "template_bench.hpp" #include #include -// Passing Arbitrary Arguments to a Benchmark -// See: -// https://github.com/google/benchmark/blob/main/docs/user_guide.md#passing-arbitrary-arguments-to-a-benchmark template -void BM_takes_args(benchmark::State &state, Args &&...args) { +void BM_Capture(benchmark::State &state, Args &&...args) { auto args_tuple = std::make_tuple(std::move(args)...); for (auto _ : state) { } } -BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); -BENCHMARK_CAPTURE(BM_takes_args, int_test, 42, 43); - -// Benchmark template -// -template void BM_VectorPushBack(benchmark::State &state) { - std::vector v; - for (auto _ : state) { - v.push_back(T()); - } -} -BENCHMARK_TEMPLATE(BM_VectorPushBack, int); -BENCHMARK_TEMPLATE(BM_VectorPushBack, std::string); - -template void BM_Template1(benchmark::State &state) { - T val = T(); - for (auto _ : state) { - benchmark::DoNotOptimize(val++); - } -} -BENCHMARK_TEMPLATE1(BM_Template1, int); - -template void BM_Template2(benchmark::State &state) { - T t = T(); - U u = U(); - for (auto _ : state) { - benchmark::DoNotOptimize(t + u); - } -} -BENCHMARK_TEMPLATE2(BM_Template2, int, double); - -// Benchmark template capture -// -template -void BM_takes_args(benchmark::State &state, ExtraArgs &&...extra_args) { - auto args_tuple = std::make_tuple(std::move(extra_args)...); - for (auto _ : state) { - } -} -BENCHMARK_TEMPLATE1_CAPTURE(BM_takes_args, void, int_string_test, 42, - std::string("abc")); - -BENCHMARK_TEMPLATE2_CAPTURE(BM_takes_args, int, double, two_type_test, 42, - std::string("abc")); +BENCHMARK_CAPTURE(BM_Capture, int_string_test, 42, std::string("abc")); +BENCHMARK_CAPTURE(BM_Capture, int_test, 42, 43); // Function to benchmark static void BM_rand_vector(benchmark::State &state) { diff --git a/examples/google_benchmark/template_bench.hpp b/examples/google_benchmark/template_bench.hpp new file mode 100644 index 0000000..2e0f141 --- /dev/null +++ b/examples/google_benchmark/template_bench.hpp @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include + +template void BM_Template(benchmark::State &state) { + std::vector v; + for (auto _ : state) { + v.push_back(T()); + } +} +BENCHMARK_TEMPLATE(BM_Template, int); +BENCHMARK_TEMPLATE(BM_Template, std::string); + +// +// + +template void BM_Template1(benchmark::State &state) { + T val = T(); + for (auto _ : state) { + benchmark::DoNotOptimize(val++); + } +} +BENCHMARK_TEMPLATE1(BM_Template1, int); + +// +// + +template void BM_Template2(benchmark::State &state) { + T t = T(); + U u = U(); + for (auto _ : state) { + benchmark::DoNotOptimize(t + u); + } +} +BENCHMARK_TEMPLATE2(BM_Template2, int, double); + +// +// + +template +void BM_Template1_Capture(benchmark::State &state, ExtraArgs &&...extra_args) { + auto args_tuple = std::make_tuple(std::move(extra_args)...); + for (auto _ : state) { + } +} +BENCHMARK_TEMPLATE1_CAPTURE(BM_Template1_Capture, void, int_string_test, 42, + std::string("abc")); +BENCHMARK_TEMPLATE2_CAPTURE(BM_Template1_Capture, int, double, two_type_test, + 42, std::string("abc")); diff --git a/google_benchmark/include/benchmark/benchmark.h b/google_benchmark/include/benchmark/benchmark.h index 5279e37..a90f2eb 100644 --- a/google_benchmark/include/benchmark/benchmark.h +++ b/google_benchmark/include/benchmark/benchmark.h @@ -1537,8 +1537,18 @@ class Fixture : public internal::Benchmark { // BENCHMARK_TEMPLATE1_CAPTURE(BM_takes_args, void, int_string_test, 42, // std::string("abc")); #ifdef CODSPEED_ENABLED + +// BM_Template1_Capture[int_string_test] will be turned into +// BM_Template1_Capture[int_string_test] +#define BENCHMARK_CAPTURE_WITH_NAME(func, func_name, test_case_name, ...) \ + BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + std::make_unique<::benchmark::internal::FunctionBenchmark>( \ + CUR_FILE + #func_name NAME_START #test_case_name NAME_END, \ + [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) + #define BENCHMARK_TEMPLATE1_CAPTURE(func, a, test_case_name, ...) \ - BENCHMARK_CAPTURE(func, test_case_name, __VA_ARGS__) + BENCHMARK_CAPTURE_WITH_NAME(func, func, test_case_name, __VA_ARGS__) #else #define BENCHMARK_TEMPLATE1_CAPTURE(func, a, test_case_name, ...) \ BENCHMARK_CAPTURE(func, test_case_name, __VA_ARGS__) From fcf66d42040fe5031d7bc5dfc50262c4bd925567 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 12 Mar 2025 18:01:21 +0100 Subject: [PATCH 04/10] feat(google_benchmark): add namespace to URI --- core/CMakeLists.txt | 2 +- core/include/codspeed.h | 2 + core/src/uri.cpp | 34 ++++++++++ examples/google_benchmark/template_bench.hpp | 2 + .../include/benchmark/benchmark.h | 62 +++++++++++-------- 5 files changed, 75 insertions(+), 27 deletions(-) create mode 100644 core/src/uri.cpp diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 36e5de6..d10de43 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -12,7 +12,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED True) include_directories(include) # Add the library -add_library(codspeed src/codspeed.cpp src/walltime.cpp) +add_library(codspeed src/codspeed.cpp src/walltime.cpp src/uri.cpp) # Version add_compile_definitions(CODSPEED_VERSION="${CODSPEED_VERSION}") diff --git a/core/include/codspeed.h b/core/include/codspeed.h index e1b7ea4..c0d5077 100644 --- a/core/include/codspeed.h +++ b/core/include/codspeed.h @@ -41,4 +41,6 @@ struct RawWalltimeBenchmark { void generate_codspeed_walltime_report( const std::vector &walltime_data_list); +std::string extract_lambda_namespace(const std::string &pretty_func); + #endif // CODSPEED_H diff --git a/core/src/uri.cpp b/core/src/uri.cpp new file mode 100644 index 0000000..86293cf --- /dev/null +++ b/core/src/uri.cpp @@ -0,0 +1,34 @@ +#include "codspeed.h" +#include +#include + +std::string extract_lambda_namespace(const std::string& pretty_func) { + if (pretty_func.find("(anonymous namespace)") != std::string::npos) { + std::cerr << "[ERROR] Anonymous namespace not supported in " << pretty_func << std::endl; + return {}; + } + +#ifdef __clang__ + // Example: auto outer::test12::(anonymous class)::operator()() const + + std::size_t anon_class_pos = pretty_func.find("(anonymous class)"); + std::size_t space_pos = pretty_func.find(' '); + + if (space_pos == std::string::npos || anon_class_pos == std::string::npos) { + return {}; + } + + return pretty_func.substr(space_pos, anon_class_pos - space_pos) + "::"; +#elif __GNUC__ + // Example: outer::test12:: + + auto lambda_pos = pretty_func.find("::"); + if (lambda_pos == std::string::npos) { + return {}; + } + + return pretty_func.substr(0, lambda_pos) + "::"; +#else +#error "Unsupported compiler" +#endif +} diff --git a/examples/google_benchmark/template_bench.hpp b/examples/google_benchmark/template_bench.hpp index 2e0f141..7f56550 100644 --- a/examples/google_benchmark/template_bench.hpp +++ b/examples/google_benchmark/template_bench.hpp @@ -4,6 +4,7 @@ #include #include +namespace test { template void BM_Template(benchmark::State &state) { std::vector v; for (auto _ : state) { @@ -12,6 +13,7 @@ template void BM_Template(benchmark::State &state) { } BENCHMARK_TEMPLATE(BM_Template, int); BENCHMARK_TEMPLATE(BM_Template, std::string); +} // // diff --git a/google_benchmark/include/benchmark/benchmark.h b/google_benchmark/include/benchmark/benchmark.h index a90f2eb..e4684b1 100644 --- a/google_benchmark/include/benchmark/benchmark.h +++ b/google_benchmark/include/benchmark/benchmark.h @@ -1439,9 +1439,16 @@ class Fixture : public internal::Benchmark { n) [[maybe_unused]] #ifdef CODSPEED_ENABLED +#include + #include + #define CUR_FILE \ std::filesystem::relative(__FILE__, CODSPEED_GIT_ROOT_DIR).string() + "::" +#define NAMESPACE \ + (([]() { return extract_lambda_namespace(__PRETTY_FUNCTION__); })()) + +#define FILE_AND_NAMESPACE CUR_FILE + NAMESPACE // Transforms `BM_Foo` into `BM_Foo[int, double]`. #define TYPE_START "[" @@ -1454,7 +1461,7 @@ class Fixture : public internal::Benchmark { // Extra space after the comma for readability #define COMMA ", " #else -#define CUR_FILE std::string() +#define FILE_AND_NAMESPACE std::string() #define TYPE_START "<" #define TYPE_END ">" @@ -1469,7 +1476,7 @@ class Fixture : public internal::Benchmark { BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - CUR_FILE + #__VA_ARGS__, __VA_ARGS__))) + FILE_AND_NAMESPACE + #__VA_ARGS__, __VA_ARGS__))) // Old-style macros #define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a)) @@ -1490,11 +1497,11 @@ class Fixture : public internal::Benchmark { //} // /* Registers a benchmark named "BM_takes_args/int_string_test` */ // BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); -#define BENCHMARK_CAPTURE(func, test_case_name, ...) \ - BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - CUR_FILE + #func NAME_START #test_case_name NAME_END, \ +#define BENCHMARK_CAPTURE(func, test_case_name, ...) \ + BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + std::make_unique<::benchmark::internal::FunctionBenchmark>( \ + FILE_AND_NAMESPACE + #func NAME_START #test_case_name NAME_END, \ [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) // This will register a benchmark for a templatized function. For example: @@ -1509,19 +1516,20 @@ class Fixture : public internal::Benchmark { BENCHMARK_PRIVATE_DECLARE(n) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - CUR_FILE + #n TYPE_START #a TYPE_END, n))) - -#define BENCHMARK_TEMPLATE2(n, a, b) \ - BENCHMARK_PRIVATE_DECLARE(n) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - CUR_FILE + #n TYPE_START #a COMMA #b TYPE_END, n))) - -#define BENCHMARK_TEMPLATE(n, ...) \ - BENCHMARK_PRIVATE_DECLARE(n) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - CUR_FILE + #n TYPE_START #__VA_ARGS__ TYPE_END, \ + FILE_AND_NAMESPACE + #n TYPE_START #a TYPE_END, n))) + +#define BENCHMARK_TEMPLATE2(n, a, b) \ + BENCHMARK_PRIVATE_DECLARE(n) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + std::make_unique<::benchmark::internal::FunctionBenchmark>( \ + FILE_AND_NAMESPACE + #n TYPE_START #a COMMA #b TYPE_END, \ + n))) + +#define BENCHMARK_TEMPLATE(n, ...) \ + BENCHMARK_PRIVATE_DECLARE(n) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + std::make_unique<::benchmark::internal::FunctionBenchmark>( \ + FILE_AND_NAMESPACE + #n TYPE_START #__VA_ARGS__ TYPE_END, \ n<__VA_ARGS__>))) // This will register a benchmark for a templatized function, @@ -1544,7 +1552,8 @@ class Fixture : public internal::Benchmark { BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - CUR_FILE + #func_name NAME_START #test_case_name NAME_END, \ + FILE_AND_NAMESPACE + \ + #func_name NAME_START #test_case_name NAME_END, \ [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) #define BENCHMARK_TEMPLATE1_CAPTURE(func, a, test_case_name, ...) \ @@ -1555,11 +1564,12 @@ class Fixture : public internal::Benchmark { #endif #ifdef CODSPEED_ENABLED -#define BENCHMARK_TEMPLATE2_CAPTURE(func, a, b, test_case_name, ...) \ - BENCHMARK_PRIVATE_DECLARE(func) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - CUR_FILE + #func "[" #test_case_name COMMA #a COMMA #b "]", \ +#define BENCHMARK_TEMPLATE2_CAPTURE(func, a, b, test_case_name, ...) \ + BENCHMARK_PRIVATE_DECLARE(func) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + std::make_unique<::benchmark::internal::FunctionBenchmark>( \ + FILE_AND_NAMESPACE + #func "[" #test_case_name COMMA #a COMMA #b \ + "]", \ [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) #else #define BENCHMARK_TEMPLATE2_CAPTURE(func, a, b, test_case_name, ...) \ From 5b1a4dbc6fe221a374e7a348705ed49afab54374 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 13 Mar 2025 11:08:35 +0100 Subject: [PATCH 05/10] docs(google_benchmarks): document anonymous namespaces as unsupported --- google_benchmark/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/google_benchmark/README.md b/google_benchmark/README.md index 74c3e17..8da7cf6 100644 --- a/google_benchmark/README.md +++ b/google_benchmark/README.md @@ -99,4 +99,8 @@ $ ./my-bench Checked: main.cpp::BM_memcpy[8192] ``` +### Not supported + +- Declaring benches within anonymous namespaces + For more information, please checkout the [codspeed documentation](https://docs.codspeed.io/benchmarks/cpp) From 740fa0b592ff7e87eba6b72d578fded5937c7198 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 13 Mar 2025 11:10:12 +0100 Subject: [PATCH 06/10] examples(google_benchmark): disable google benchmark lib tests --- examples/google_benchmark/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/google_benchmark/CMakeLists.txt b/examples/google_benchmark/CMakeLists.txt index da8662b..ed72751 100644 --- a/examples/google_benchmark/CMakeLists.txt +++ b/examples/google_benchmark/CMakeLists.txt @@ -3,7 +3,7 @@ include(FetchContent) project(codspeed_picobench_compat VERSION 0.0.0 LANGUAGES CXX) -set(BENCHMARK_DOWNLOAD_DEPENDENCIES ON) +option(BENCHMARK_ENABLE_GTEST_TESTS OFF) FetchContent_Declare( google_benchmark From 96909a25ffb8986dcefdf61df97a73cc297d53cc Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 13 Mar 2025 11:10:56 +0100 Subject: [PATCH 07/10] feat(core): escape double colons in type URI bench arguments --- core/include/codspeed.h | 1 + core/src/codspeed.cpp | 31 +++++++++++++++++++++++++++++++ core/src/walltime.cpp | 16 ++++++++++++++-- google_benchmark/src/benchmark.cc | 2 ++ 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/core/include/codspeed.h b/core/include/codspeed.h index c0d5077..e3b6fd2 100644 --- a/core/include/codspeed.h +++ b/core/include/codspeed.h @@ -42,5 +42,6 @@ void generate_codspeed_walltime_report( const std::vector &walltime_data_list); std::string extract_lambda_namespace(const std::string &pretty_func); +std::string sanitize_bench_args(std::string &text); #endif // CODSPEED_H diff --git a/core/src/codspeed.cpp b/core/src/codspeed.cpp index 1e43c06..1b90d15 100644 --- a/core/src/codspeed.cpp +++ b/core/src/codspeed.cpp @@ -4,6 +4,35 @@ #include #include +// Remove any `::` between brackets at the end to not mess with the URI +// parsing +// FIXME: Remove this bandaid when we migrate to structured benchmark metadata +std::string sanitize_bench_args(std::string &text) { + std::string search = "::"; + std::string replace = "\\:\\:"; + + if (text.back() == ']') { + size_t pos_open = text.rfind('['); + if (pos_open != std::string::npos) { + // Extract the substring between '[' and ']' + size_t pos_close = text.size() - 1; + std::string substring = + text.substr(pos_open + 1, pos_close - pos_open - 1); + + // Perform the search and replace within the substring + size_t pos = substring.find(search); + while (pos != std::string::npos) { + substring.replace(pos, search.length(), replace); + pos = substring.find(search, pos + replace.length()); + } + + // Replace the original substring with the modified one + text.replace(pos_open + 1, pos_close - pos_open - 1, substring); + } + } + return text; +} + std::string join(const std::vector &elements, const std::string &delimiter) { std::string result; @@ -40,6 +69,8 @@ void CodSpeed::pop_group() { void CodSpeed::start_benchmark(const std::string &name) { std::string uri = name; + uri = sanitize_bench_args(uri); + // Sanity check URI and add a placeholder if format is wrong if (name.find("::") == std::string::npos) { std::string uri = "unknown_file::" + name; diff --git a/core/src/walltime.cpp b/core/src/walltime.cpp index bf4f4aa..6e6aa50 100644 --- a/core/src/walltime.cpp +++ b/core/src/walltime.cpp @@ -87,6 +87,18 @@ void compute_iqr_and_outliers(const std::vector ×_ns, double mean, }); } +std::string escapeBackslashes(const std::string &input) { + std::string output; + for (char c : input) { + if (c == '\\') { + output += "\\\\"; + } else { + output += c; + } + } + return output; +} + void write_codspeed_benchmarks_to_json( const std::vector &benchmarks) { std::ostringstream oss; @@ -113,8 +125,8 @@ void write_codspeed_benchmarks_to_json( const auto &metadata = benchmark.metadata; oss << " {\n"; - oss << " \"name\": \"" << metadata.name << "\",\n"; - oss << " \"uri\": \"" << metadata.uri << "\",\n"; + oss << " \"name\": \"" << escapeBackslashes(metadata.name) << "\",\n"; + oss << " \"uri\": \"" << escapeBackslashes(metadata.uri) << "\",\n"; // TODO: Manage config fields from actual configuration oss << " \"config\": {\n"; oss << " \"warmup_time_ns\": null,\n"; diff --git a/google_benchmark/src/benchmark.cc b/google_benchmark/src/benchmark.cc index e9574c8..2d27c42 100644 --- a/google_benchmark/src/benchmark.cc +++ b/google_benchmark/src/benchmark.cc @@ -357,6 +357,8 @@ RawWalltimeBenchmark generate_raw_walltime_data(const RunResults& run_results) { for (const auto& run : run_results.non_aggregates) { walltime_data.uri = run.benchmark_name(); + walltime_data.uri = sanitize_bench_args(walltime_data.uri); + size_t pos = walltime_data.uri.rfind("::"); if (pos != std::string::npos) { From b020955719575e2bee81aa63e6419253ffc50ccb Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 13 Mar 2025 11:15:04 +0100 Subject: [PATCH 08/10] example(google_benchmark): remove use of #pragma once --- examples/google_benchmark/template_bench.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/google_benchmark/template_bench.hpp b/examples/google_benchmark/template_bench.hpp index 7f56550..0625222 100644 --- a/examples/google_benchmark/template_bench.hpp +++ b/examples/google_benchmark/template_bench.hpp @@ -1,4 +1,5 @@ -#pragma once +#ifndef TEMPLATE_BENCH_HPP +#define TEMPLATE_BENCH_HPP #include #include @@ -13,7 +14,7 @@ template void BM_Template(benchmark::State &state) { } BENCHMARK_TEMPLATE(BM_Template, int); BENCHMARK_TEMPLATE(BM_Template, std::string); -} +} // namespace test // // @@ -51,3 +52,5 @@ BENCHMARK_TEMPLATE1_CAPTURE(BM_Template1_Capture, void, int_string_test, 42, std::string("abc")); BENCHMARK_TEMPLATE2_CAPTURE(BM_Template1_Capture, int, double, two_type_test, 42, std::string("abc")); + +#endif From c2668c354233aa67e827487688873d237d82d20b Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 13 Mar 2025 12:14:46 +0100 Subject: [PATCH 09/10] feat(google_benchmark): add tests --- .github/workflows/ci.yml | 33 +++++++++++++++++++++++++++++++++ core/CMakeLists.txt | 6 ++++++ core/src/uri.cpp | 38 ++++++++++++++++++++++++++------------ core/test/CMakeLists.txt | 22 ++++++++++++++++++++++ core/test/uri.cpp | 30 ++++++++++++++++++++++++++++++ 5 files changed, 117 insertions(+), 12 deletions(-) create mode 100644 core/test/CMakeLists.txt create mode 100644 core/test/uri.cpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8600e1a..828d3aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,39 @@ on: workflow_dispatch: jobs: + tests: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Cache build + uses: actions/cache@v3 + with: + path: core/build-tests + key: ${{ runner.os }}-build-tests-${{ hashFiles('**/CMakeLists.txt', '**/examples/google_benchmark/**') }} + + - name: Create build directory + run: mkdir -p core/build-tests + + - name: Build tests + run: | + cd core/build-tests + cmake .. -DENABLE_TESTS=ON + make -j + + - name: Run tests + run: | + cd core/build-tests + GTEST_OUTPUT=json:test-results/ ctest + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: failure() + with: + name: test_results + path: ${{runner.workspace}}/core/build-tests/test/test-results/**/*.json + instrumentation: runs-on: ubuntu-latest diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index d10de43..b6f1657 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -28,3 +28,9 @@ target_include_directories( codspeed PUBLIC $ ) + +option(ENABLE_TESTS "Enable building the unit tests which depend on gtest" OFF) +if(ENABLE_TESTS) + enable_testing() + add_subdirectory(test) +endif() diff --git a/core/src/uri.cpp b/core/src/uri.cpp index 86293cf..2bed0ff 100644 --- a/core/src/uri.cpp +++ b/core/src/uri.cpp @@ -2,32 +2,46 @@ #include #include -std::string extract_lambda_namespace(const std::string& pretty_func) { - if (pretty_func.find("(anonymous namespace)") != std::string::npos) { - std::cerr << "[ERROR] Anonymous namespace not supported in " << pretty_func << std::endl; - return {}; - } - -#ifdef __clang__ - // Example: auto outer::test12::(anonymous class)::operator()() const - - std::size_t anon_class_pos = pretty_func.find("(anonymous class)"); +// Example: auto outer::test12::(anonymous class)::operator()() const +// Returns: outer::test12:: +std::string extract_namespace_clang(const std::string& pretty_func) { + std::size_t anon_class_pos = pretty_func.find("::(anonymous class)"); std::size_t space_pos = pretty_func.find(' '); if (space_pos == std::string::npos || anon_class_pos == std::string::npos) { return {}; } + space_pos += 1; // Skip the space return pretty_func.substr(space_pos, anon_class_pos - space_pos) + "::"; -#elif __GNUC__ - // Example: outer::test12:: +} +// Example: outer::test12:: +// Returns: outer::test12:: +std::string extract_namespace_gcc(const std::string& pretty_func) { auto lambda_pos = pretty_func.find("::"); if (lambda_pos == std::string::npos) { return {}; } return pretty_func.substr(0, lambda_pos) + "::"; +} + +// Has to pass the pretty function from a lambda: +// (([]() { return __PRETTY_FUNCTION__; })()) +// +// Returns: An empty string if the namespace could not be extracted, +// otherwise the namespace with a trailing "::" +std::string extract_lambda_namespace(const std::string& pretty_func) { + if (pretty_func.find("(anonymous namespace)") != std::string::npos) { + std::cerr << "[ERROR] Anonymous namespace not supported in " << pretty_func << std::endl; + return {}; + } + +#ifdef __clang__ + return extract_namespace_clang(pretty_func); +#elif __GNUC__ + return extract_namespace_gcc(pretty_func); #else #error "Unsupported compiler" #endif diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt new file mode 100644 index 0000000..68a9ce6 --- /dev/null +++ b/core/test/CMakeLists.txt @@ -0,0 +1,22 @@ +include(FetchContent) +FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG v1.16.0 +) +FetchContent_MakeAvailable(googletest) + +add_executable(unit_tests + uri.cpp +) + +target_link_libraries(unit_tests + PRIVATE + codspeed + GTest::gtest + GTest::gtest_main +) + +include(GoogleTest) +gtest_discover_tests(unit_tests) + diff --git a/core/test/uri.cpp b/core/test/uri.cpp new file mode 100644 index 0000000..61ca5ec --- /dev/null +++ b/core/test/uri.cpp @@ -0,0 +1,30 @@ + +#include +#include "codspeed.h" + +// Manual definition (to avoid including it in the public header): +std::string extract_namespace_clang(const std::string& func_str); +std::string extract_namespace_gcc(const std::string& func_str); + +TEST(UriTest, TestExtractNamespaceClang) { + EXPECT_EQ(extract_namespace_clang("auto outer::test12::(anonymous class)::operator()() const"), "outer::test12::"); + EXPECT_EQ(extract_namespace_clang("auto outer::(anonymous namespace)::test12::(anonymous class)::operator()() const"), "outer::(anonymous namespace)::test12::"); +} + +TEST(UriTest, TestExtractNamespaceGcc) { + EXPECT_EQ(extract_namespace_gcc("outer::test12::"), "outer::test12::"); + EXPECT_EQ(extract_namespace_gcc("outer::(anonymous namespace)::test12::"), "outer::(anonymous namespace)::test12::"); +} + + +namespace a { +namespace b { +namespace c { +static std::string pretty_func = ([]() { return __PRETTY_FUNCTION__; })(); + +TEST(UriTest, TestExtractNamespace) { + EXPECT_EQ(extract_lambda_namespace(pretty_func), "a::b::c::"); +} +} +} +} From 4686610f6ec4323fdbc89fa19307261d07a55a66 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 13 Mar 2025 15:13:16 +0100 Subject: [PATCH 10/10] feat(core): add tests for type double colon escape --- core/test/CMakeLists.txt | 2 +- core/test/codspeed.cpp | 51 ++++++++++++++++++++++++++++++++++++++++ core/test/uri.cpp | 1 - 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 core/test/codspeed.cpp diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt index 68a9ce6..32524bf 100644 --- a/core/test/CMakeLists.txt +++ b/core/test/CMakeLists.txt @@ -8,6 +8,7 @@ FetchContent_MakeAvailable(googletest) add_executable(unit_tests uri.cpp + codspeed.cpp ) target_link_libraries(unit_tests @@ -19,4 +20,3 @@ target_link_libraries(unit_tests include(GoogleTest) gtest_discover_tests(unit_tests) - diff --git a/core/test/codspeed.cpp b/core/test/codspeed.cpp new file mode 100644 index 0000000..c74a1c3 --- /dev/null +++ b/core/test/codspeed.cpp @@ -0,0 +1,51 @@ +#include "codspeed.h" +#include + +// Manual definition (to avoid including it in the public header): + +TEST(CodSpeedTest, TestSearchAndReplaceBetweenBracketsNamespace) { + std::string no_brackets_input = + "examples/google_benchmark/main.cpp::BM_rand_vector"; + std::string no_brackets_output = + "examples/google_benchmark/main.cpp::BM_rand_vector"; + EXPECT_EQ(sanitize_bench_args(no_brackets_input), no_brackets_output); + + std::string brackets_and_no_escaped_type_input = + "examples/google_benchmark/" + "template_bench.hpp::BM_Template1_Capture[two_type_test, int, double]"; + std::string brackets_and_no_escaped_type_output = + "examples/google_benchmark/" + "template_bench.hpp::BM_Template1_Capture[two_type_test, int, double]"; + EXPECT_EQ(sanitize_bench_args(brackets_and_no_escaped_type_input), + brackets_and_no_escaped_type_output); + + std::string brackets_and_escaped_type_input = + "examples/google_benchmark/" + "template_bench.hpp::test::BM_Template[std::string]"; + std::string brackets_and_escaped_type_output = + "examples/google_benchmark/" + "template_bench.hpp::test::BM_Template[std\\:\\:string]"; + + EXPECT_EQ(sanitize_bench_args(brackets_and_escaped_type_input), + brackets_and_escaped_type_output); + + std::string brackets_and_escaped_types_input = + "examples/google_benchmark/" + "template_bench.hpp::test::BM_Template[std::string, std::string]"; + std::string brackets_and_escaped_types_output = + "examples/google_benchmark/" + "template_bench.hpp::test::BM_Template[std\\:\\:string, std\\:\\:string]"; + + EXPECT_EQ(sanitize_bench_args(brackets_and_escaped_types_input), + brackets_and_escaped_types_output); + + std::string brackets_and_multiple_types_input = + "examples/google_benchmark/" + "template_bench.hpp::test::BM_Template[std::string, int, double]"; + std::string brackets_and_multiple_types_output = + "examples/google_benchmark/" + "template_bench.hpp::test::BM_Template[std\\:\\:string, int, double]"; + + EXPECT_EQ(sanitize_bench_args(brackets_and_multiple_types_input), + brackets_and_multiple_types_output); +} diff --git a/core/test/uri.cpp b/core/test/uri.cpp index 61ca5ec..47a7ef2 100644 --- a/core/test/uri.cpp +++ b/core/test/uri.cpp @@ -1,4 +1,3 @@ - #include #include "codspeed.h"