diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8600e1a..828d3aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,39 @@ on: workflow_dispatch: jobs: + tests: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Cache build + uses: actions/cache@v3 + with: + path: core/build-tests + key: ${{ runner.os }}-build-tests-${{ hashFiles('**/CMakeLists.txt', '**/examples/google_benchmark/**') }} + + - name: Create build directory + run: mkdir -p core/build-tests + + - name: Build tests + run: | + cd core/build-tests + cmake .. -DENABLE_TESTS=ON + make -j + + - name: Run tests + run: | + cd core/build-tests + GTEST_OUTPUT=json:test-results/ ctest + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: failure() + with: + name: test_results + path: ${{runner.workspace}}/core/build-tests/test/test-results/**/*.json + instrumentation: runs-on: ubuntu-latest diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 36e5de6..b6f1657 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -12,7 +12,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED True) include_directories(include) # Add the library -add_library(codspeed src/codspeed.cpp src/walltime.cpp) +add_library(codspeed src/codspeed.cpp src/walltime.cpp src/uri.cpp) # Version add_compile_definitions(CODSPEED_VERSION="${CODSPEED_VERSION}") @@ -28,3 +28,9 @@ target_include_directories( codspeed PUBLIC $ ) + +option(ENABLE_TESTS "Enable building the unit tests which depend on gtest" OFF) +if(ENABLE_TESTS) + enable_testing() + add_subdirectory(test) +endif() diff --git a/core/include/codspeed.h b/core/include/codspeed.h index e1b7ea4..e3b6fd2 100644 --- a/core/include/codspeed.h +++ b/core/include/codspeed.h @@ -41,4 +41,7 @@ struct RawWalltimeBenchmark { void generate_codspeed_walltime_report( const std::vector &walltime_data_list); +std::string extract_lambda_namespace(const std::string &pretty_func); +std::string sanitize_bench_args(std::string &text); + #endif // CODSPEED_H diff --git a/core/src/codspeed.cpp b/core/src/codspeed.cpp index 1e43c06..1b90d15 100644 --- a/core/src/codspeed.cpp +++ b/core/src/codspeed.cpp @@ -4,6 +4,35 @@ #include #include +// Remove any `::` between brackets at the end to not mess with the URI +// parsing +// FIXME: Remove this bandaid when we migrate to structured benchmark metadata +std::string sanitize_bench_args(std::string &text) { + std::string search = "::"; + std::string replace = "\\:\\:"; + + if (text.back() == ']') { + size_t pos_open = text.rfind('['); + if (pos_open != std::string::npos) { + // Extract the substring between '[' and ']' + size_t pos_close = text.size() - 1; + std::string substring = + text.substr(pos_open + 1, pos_close - pos_open - 1); + + // Perform the search and replace within the substring + size_t pos = substring.find(search); + while (pos != std::string::npos) { + substring.replace(pos, search.length(), replace); + pos = substring.find(search, pos + replace.length()); + } + + // Replace the original substring with the modified one + text.replace(pos_open + 1, pos_close - pos_open - 1, substring); + } + } + return text; +} + std::string join(const std::vector &elements, const std::string &delimiter) { std::string result; @@ -40,6 +69,8 @@ void CodSpeed::pop_group() { void CodSpeed::start_benchmark(const std::string &name) { std::string uri = name; + uri = sanitize_bench_args(uri); + // Sanity check URI and add a placeholder if format is wrong if (name.find("::") == std::string::npos) { std::string uri = "unknown_file::" + name; diff --git a/core/src/uri.cpp b/core/src/uri.cpp new file mode 100644 index 0000000..2bed0ff --- /dev/null +++ b/core/src/uri.cpp @@ -0,0 +1,48 @@ +#include "codspeed.h" +#include +#include + +// Example: auto outer::test12::(anonymous class)::operator()() const +// Returns: outer::test12:: +std::string extract_namespace_clang(const std::string& pretty_func) { + std::size_t anon_class_pos = pretty_func.find("::(anonymous class)"); + std::size_t space_pos = pretty_func.find(' '); + + if (space_pos == std::string::npos || anon_class_pos == std::string::npos) { + return {}; + } + space_pos += 1; // Skip the space + + return pretty_func.substr(space_pos, anon_class_pos - space_pos) + "::"; +} + +// Example: outer::test12:: +// Returns: outer::test12:: +std::string extract_namespace_gcc(const std::string& pretty_func) { + auto lambda_pos = pretty_func.find("::"); + if (lambda_pos == std::string::npos) { + return {}; + } + + return pretty_func.substr(0, lambda_pos) + "::"; +} + +// Has to pass the pretty function from a lambda: +// (([]() { return __PRETTY_FUNCTION__; })()) +// +// Returns: An empty string if the namespace could not be extracted, +// otherwise the namespace with a trailing "::" +std::string extract_lambda_namespace(const std::string& pretty_func) { + if (pretty_func.find("(anonymous namespace)") != std::string::npos) { + std::cerr << "[ERROR] Anonymous namespace not supported in " << pretty_func << std::endl; + return {}; + } + +#ifdef __clang__ + return extract_namespace_clang(pretty_func); +#elif __GNUC__ + return extract_namespace_gcc(pretty_func); +#else +#error "Unsupported compiler" +#endif +} diff --git a/core/src/walltime.cpp b/core/src/walltime.cpp index bf4f4aa..6e6aa50 100644 --- a/core/src/walltime.cpp +++ b/core/src/walltime.cpp @@ -87,6 +87,18 @@ void compute_iqr_and_outliers(const std::vector ×_ns, double mean, }); } +std::string escapeBackslashes(const std::string &input) { + std::string output; + for (char c : input) { + if (c == '\\') { + output += "\\\\"; + } else { + output += c; + } + } + return output; +} + void write_codspeed_benchmarks_to_json( const std::vector &benchmarks) { std::ostringstream oss; @@ -113,8 +125,8 @@ void write_codspeed_benchmarks_to_json( const auto &metadata = benchmark.metadata; oss << " {\n"; - oss << " \"name\": \"" << metadata.name << "\",\n"; - oss << " \"uri\": \"" << metadata.uri << "\",\n"; + oss << " \"name\": \"" << escapeBackslashes(metadata.name) << "\",\n"; + oss << " \"uri\": \"" << escapeBackslashes(metadata.uri) << "\",\n"; // TODO: Manage config fields from actual configuration oss << " \"config\": {\n"; oss << " \"warmup_time_ns\": null,\n"; diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt new file mode 100644 index 0000000..32524bf --- /dev/null +++ b/core/test/CMakeLists.txt @@ -0,0 +1,22 @@ +include(FetchContent) +FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG v1.16.0 +) +FetchContent_MakeAvailable(googletest) + +add_executable(unit_tests + uri.cpp + codspeed.cpp +) + +target_link_libraries(unit_tests + PRIVATE + codspeed + GTest::gtest + GTest::gtest_main +) + +include(GoogleTest) +gtest_discover_tests(unit_tests) diff --git a/core/test/codspeed.cpp b/core/test/codspeed.cpp new file mode 100644 index 0000000..c74a1c3 --- /dev/null +++ b/core/test/codspeed.cpp @@ -0,0 +1,51 @@ +#include "codspeed.h" +#include + +// Manual definition (to avoid including it in the public header): + +TEST(CodSpeedTest, TestSearchAndReplaceBetweenBracketsNamespace) { + std::string no_brackets_input = + "examples/google_benchmark/main.cpp::BM_rand_vector"; + std::string no_brackets_output = + "examples/google_benchmark/main.cpp::BM_rand_vector"; + EXPECT_EQ(sanitize_bench_args(no_brackets_input), no_brackets_output); + + std::string brackets_and_no_escaped_type_input = + "examples/google_benchmark/" + "template_bench.hpp::BM_Template1_Capture[two_type_test, int, double]"; + std::string brackets_and_no_escaped_type_output = + "examples/google_benchmark/" + "template_bench.hpp::BM_Template1_Capture[two_type_test, int, double]"; + EXPECT_EQ(sanitize_bench_args(brackets_and_no_escaped_type_input), + brackets_and_no_escaped_type_output); + + std::string brackets_and_escaped_type_input = + "examples/google_benchmark/" + "template_bench.hpp::test::BM_Template[std::string]"; + std::string brackets_and_escaped_type_output = + "examples/google_benchmark/" + "template_bench.hpp::test::BM_Template[std\\:\\:string]"; + + EXPECT_EQ(sanitize_bench_args(brackets_and_escaped_type_input), + brackets_and_escaped_type_output); + + std::string brackets_and_escaped_types_input = + "examples/google_benchmark/" + "template_bench.hpp::test::BM_Template[std::string, std::string]"; + std::string brackets_and_escaped_types_output = + "examples/google_benchmark/" + "template_bench.hpp::test::BM_Template[std\\:\\:string, std\\:\\:string]"; + + EXPECT_EQ(sanitize_bench_args(brackets_and_escaped_types_input), + brackets_and_escaped_types_output); + + std::string brackets_and_multiple_types_input = + "examples/google_benchmark/" + "template_bench.hpp::test::BM_Template[std::string, int, double]"; + std::string brackets_and_multiple_types_output = + "examples/google_benchmark/" + "template_bench.hpp::test::BM_Template[std\\:\\:string, int, double]"; + + EXPECT_EQ(sanitize_bench_args(brackets_and_multiple_types_input), + brackets_and_multiple_types_output); +} diff --git a/core/test/uri.cpp b/core/test/uri.cpp new file mode 100644 index 0000000..47a7ef2 --- /dev/null +++ b/core/test/uri.cpp @@ -0,0 +1,29 @@ +#include +#include "codspeed.h" + +// Manual definition (to avoid including it in the public header): +std::string extract_namespace_clang(const std::string& func_str); +std::string extract_namespace_gcc(const std::string& func_str); + +TEST(UriTest, TestExtractNamespaceClang) { + EXPECT_EQ(extract_namespace_clang("auto outer::test12::(anonymous class)::operator()() const"), "outer::test12::"); + EXPECT_EQ(extract_namespace_clang("auto outer::(anonymous namespace)::test12::(anonymous class)::operator()() const"), "outer::(anonymous namespace)::test12::"); +} + +TEST(UriTest, TestExtractNamespaceGcc) { + EXPECT_EQ(extract_namespace_gcc("outer::test12::"), "outer::test12::"); + EXPECT_EQ(extract_namespace_gcc("outer::(anonymous namespace)::test12::"), "outer::(anonymous namespace)::test12::"); +} + + +namespace a { +namespace b { +namespace c { +static std::string pretty_func = ([]() { return __PRETTY_FUNCTION__; })(); + +TEST(UriTest, TestExtractNamespace) { + EXPECT_EQ(extract_lambda_namespace(pretty_func), "a::b::c::"); +} +} +} +} diff --git a/examples/google_benchmark/CMakeLists.txt b/examples/google_benchmark/CMakeLists.txt index da8662b..ed72751 100644 --- a/examples/google_benchmark/CMakeLists.txt +++ b/examples/google_benchmark/CMakeLists.txt @@ -3,7 +3,7 @@ include(FetchContent) project(codspeed_picobench_compat VERSION 0.0.0 LANGUAGES CXX) -set(BENCHMARK_DOWNLOAD_DEPENDENCIES ON) +option(BENCHMARK_ENABLE_GTEST_TESTS OFF) FetchContent_Declare( google_benchmark diff --git a/examples/google_benchmark/main.cpp b/examples/google_benchmark/main.cpp index 7886fee..143259c 100644 --- a/examples/google_benchmark/main.cpp +++ b/examples/google_benchmark/main.cpp @@ -1,6 +1,16 @@ +#include "template_bench.hpp" #include #include +template +void BM_Capture(benchmark::State &state, Args &&...args) { + auto args_tuple = std::make_tuple(std::move(args)...); + for (auto _ : state) { + } +} +BENCHMARK_CAPTURE(BM_Capture, int_string_test, 42, std::string("abc")); +BENCHMARK_CAPTURE(BM_Capture, int_test, 42, 43); + // Function to benchmark static void BM_rand_vector(benchmark::State &state) { std::vector v; diff --git a/examples/google_benchmark/template_bench.hpp b/examples/google_benchmark/template_bench.hpp new file mode 100644 index 0000000..0625222 --- /dev/null +++ b/examples/google_benchmark/template_bench.hpp @@ -0,0 +1,56 @@ +#ifndef TEMPLATE_BENCH_HPP +#define TEMPLATE_BENCH_HPP + +#include +#include +#include + +namespace test { +template void BM_Template(benchmark::State &state) { + std::vector v; + for (auto _ : state) { + v.push_back(T()); + } +} +BENCHMARK_TEMPLATE(BM_Template, int); +BENCHMARK_TEMPLATE(BM_Template, std::string); +} // namespace test + +// +// + +template void BM_Template1(benchmark::State &state) { + T val = T(); + for (auto _ : state) { + benchmark::DoNotOptimize(val++); + } +} +BENCHMARK_TEMPLATE1(BM_Template1, int); + +// +// + +template void BM_Template2(benchmark::State &state) { + T t = T(); + U u = U(); + for (auto _ : state) { + benchmark::DoNotOptimize(t + u); + } +} +BENCHMARK_TEMPLATE2(BM_Template2, int, double); + +// +// + +template +void BM_Template1_Capture(benchmark::State &state, ExtraArgs &&...extra_args) { + auto args_tuple = std::make_tuple(std::move(extra_args)...); + for (auto _ : state) { + } +} +BENCHMARK_TEMPLATE1_CAPTURE(BM_Template1_Capture, void, int_string_test, 42, + std::string("abc")); +BENCHMARK_TEMPLATE2_CAPTURE(BM_Template1_Capture, int, double, two_type_test, + 42, std::string("abc")); + +#endif diff --git a/google_benchmark/README.md b/google_benchmark/README.md index 74c3e17..8da7cf6 100644 --- a/google_benchmark/README.md +++ b/google_benchmark/README.md @@ -99,4 +99,8 @@ $ ./my-bench Checked: main.cpp::BM_memcpy[8192] ``` +### Not supported + +- Declaring benches within anonymous namespaces + For more information, please checkout the [codspeed documentation](https://docs.codspeed.io/benchmarks/cpp) diff --git a/google_benchmark/include/benchmark/benchmark.h b/google_benchmark/include/benchmark/benchmark.h index 2ae1f4a..e4684b1 100644 --- a/google_benchmark/include/benchmark/benchmark.h +++ b/google_benchmark/include/benchmark/benchmark.h @@ -1439,22 +1439,44 @@ class Fixture : public internal::Benchmark { n) [[maybe_unused]] #ifdef CODSPEED_ENABLED +#include + #include -#define BENCHMARK(...) \ - BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - std::filesystem::relative(__FILE__, CODSPEED_GIT_ROOT_DIR) \ - .string() + \ - "::" + #__VA_ARGS__, \ - __VA_ARGS__))) + +#define CUR_FILE \ + std::filesystem::relative(__FILE__, CODSPEED_GIT_ROOT_DIR).string() + "::" +#define NAMESPACE \ + (([]() { return extract_lambda_namespace(__PRETTY_FUNCTION__); })()) + +#define FILE_AND_NAMESPACE CUR_FILE + NAMESPACE + +// Transforms `BM_Foo` into `BM_Foo[int, double]`. +#define TYPE_START "[" +#define TYPE_END "]" + +// Transforms `BM_Foo/int_arg` into `BM_Foo[int_arg]`. +#define NAME_START "[" +#define NAME_END "]" + +// Extra space after the comma for readability +#define COMMA ", " #else +#define FILE_AND_NAMESPACE std::string() + +#define TYPE_START "<" +#define TYPE_END ">" + +#define NAME_START "/" +#define NAME_END "" + +#define COMMA "," +#endif + #define BENCHMARK(...) \ BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - #__VA_ARGS__, __VA_ARGS__))) -#endif + FILE_AND_NAMESPACE + #__VA_ARGS__, __VA_ARGS__))) // Old-style macros #define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a)) @@ -1475,11 +1497,11 @@ class Fixture : public internal::Benchmark { //} // /* Registers a benchmark named "BM_takes_args/int_string_test` */ // BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); -#define BENCHMARK_CAPTURE(func, test_case_name, ...) \ - BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - #func "/" #test_case_name, \ +#define BENCHMARK_CAPTURE(func, test_case_name, ...) \ + BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + std::make_unique<::benchmark::internal::FunctionBenchmark>( \ + FILE_AND_NAMESPACE + #func NAME_START #test_case_name NAME_END, \ [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) // This will register a benchmark for a templatized function. For example: @@ -1494,19 +1516,21 @@ class Fixture : public internal::Benchmark { BENCHMARK_PRIVATE_DECLARE(n) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - #n "<" #a ">", n))) - -#define BENCHMARK_TEMPLATE2(n, a, b) \ - BENCHMARK_PRIVATE_DECLARE(n) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - #n "<" #a "," #b ">", n))) - -#define BENCHMARK_TEMPLATE(n, ...) \ - BENCHMARK_PRIVATE_DECLARE(n) = \ - (::benchmark::internal::RegisterBenchmarkInternal( \ - std::make_unique<::benchmark::internal::FunctionBenchmark>( \ - #n "<" #__VA_ARGS__ ">", n<__VA_ARGS__>))) + FILE_AND_NAMESPACE + #n TYPE_START #a TYPE_END, n))) + +#define BENCHMARK_TEMPLATE2(n, a, b) \ + BENCHMARK_PRIVATE_DECLARE(n) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + std::make_unique<::benchmark::internal::FunctionBenchmark>( \ + FILE_AND_NAMESPACE + #n TYPE_START #a COMMA #b TYPE_END, \ + n))) + +#define BENCHMARK_TEMPLATE(n, ...) \ + BENCHMARK_PRIVATE_DECLARE(n) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + std::make_unique<::benchmark::internal::FunctionBenchmark>( \ + FILE_AND_NAMESPACE + #n TYPE_START #__VA_ARGS__ TYPE_END, \ + n<__VA_ARGS__>))) // This will register a benchmark for a templatized function, // with the additional arguments specified by `...`. @@ -1520,9 +1544,34 @@ class Fixture : public internal::Benchmark { // /* Registers a benchmark named "BM_takes_args/int_string_test` */ // BENCHMARK_TEMPLATE1_CAPTURE(BM_takes_args, void, int_string_test, 42, // std::string("abc")); +#ifdef CODSPEED_ENABLED + +// BM_Template1_Capture[int_string_test] will be turned into +// BM_Template1_Capture[int_string_test] +#define BENCHMARK_CAPTURE_WITH_NAME(func, func_name, test_case_name, ...) \ + BENCHMARK_PRIVATE_DECLARE(_benchmark_) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + std::make_unique<::benchmark::internal::FunctionBenchmark>( \ + FILE_AND_NAMESPACE + \ + #func_name NAME_START #test_case_name NAME_END, \ + [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) + +#define BENCHMARK_TEMPLATE1_CAPTURE(func, a, test_case_name, ...) \ + BENCHMARK_CAPTURE_WITH_NAME(func, func, test_case_name, __VA_ARGS__) +#else #define BENCHMARK_TEMPLATE1_CAPTURE(func, a, test_case_name, ...) \ BENCHMARK_CAPTURE(func, test_case_name, __VA_ARGS__) +#endif +#ifdef CODSPEED_ENABLED +#define BENCHMARK_TEMPLATE2_CAPTURE(func, a, b, test_case_name, ...) \ + BENCHMARK_PRIVATE_DECLARE(func) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + std::make_unique<::benchmark::internal::FunctionBenchmark>( \ + FILE_AND_NAMESPACE + #func "[" #test_case_name COMMA #a COMMA #b \ + "]", \ + [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) +#else #define BENCHMARK_TEMPLATE2_CAPTURE(func, a, b, test_case_name, ...) \ BENCHMARK_PRIVATE_DECLARE(func) = \ (::benchmark::internal::RegisterBenchmarkInternal( \ @@ -1530,6 +1579,7 @@ class Fixture : public internal::Benchmark { #func "<" #a "," #b ">" \ "/" #test_case_name, \ [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) +#endif #define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ class BaseClass##_##Method##_Benchmark : public BaseClass { \ diff --git a/google_benchmark/src/benchmark.cc b/google_benchmark/src/benchmark.cc index e9574c8..2d27c42 100644 --- a/google_benchmark/src/benchmark.cc +++ b/google_benchmark/src/benchmark.cc @@ -357,6 +357,8 @@ RawWalltimeBenchmark generate_raw_walltime_data(const RunResults& run_results) { for (const auto& run : run_results.non_aggregates) { walltime_data.uri = run.benchmark_name(); + walltime_data.uri = sanitize_bench_args(walltime_data.uri); + size_t pos = walltime_data.uri.rfind("::"); if (pos != std::string::npos) {