From 7862f04f904ffffab49ce6e09e0e6c94486c6614 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 14:33:42 -0700 Subject: [PATCH 01/16] [TEST] Add stress test for histogram metric for multiple threads validation --- sdk/test/metrics/CMakeLists.txt | 3 +- sdk/test/metrics/stress_test.cc | 142 ++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 sdk/test/metrics/stress_test.cc diff --git a/sdk/test/metrics/CMakeLists.txt b/sdk/test/metrics/CMakeLists.txt index 79b8c28a2f..2c4f733665 100644 --- a/sdk/test/metrics/CMakeLists.txt +++ b/sdk/test/metrics/CMakeLists.txt @@ -33,7 +33,8 @@ foreach( metric_reader_test observable_registry_test periodic_exporting_metric_reader_test - instrument_metadata_validator_test) + instrument_metadata_validator_test + stress_test) add_executable(${testname} "${testname}.cc") target_link_libraries( ${testname} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} diff --git a/sdk/test/metrics/stress_test.cc b/sdk/test/metrics/stress_test.cc new file mode 100644 index 0000000000..bc3e643185 --- /dev/null +++ b/sdk/test/metrics/stress_test.cc @@ -0,0 +1,142 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include +#include +#include +#include + +#include "common.h" +#include "opentelemetry/sdk/metrics/meter_provider.h" + +using namespace opentelemetry; +using namespace opentelemetry::sdk::instrumentationscope; +using namespace opentelemetry::sdk::metrics; + +class MockMetricExporterForStress : public opentelemetry::sdk::metrics::PushMetricExporter +{ +public: + MockMetricExporterForStress() = default; + + opentelemetry::sdk::metrics::AggregationTemporality GetAggregationTemporality( + opentelemetry::sdk::metrics::InstrumentType) const noexcept override + { + return AggregationTemporality::kDelta; + } + + opentelemetry::sdk::common::ExportResult Export( + const opentelemetry::sdk::metrics::ResourceMetrics &) noexcept override { return opentelemetry::sdk::common::ExportResult::kSuccess; } + + bool ForceFlush(std::chrono::microseconds) noexcept override { return true; } + + bool Shutdown(std::chrono::microseconds) noexcept override { return true; } +}; + +TEST(HistogramStress, UnsignedInt64) +{ + MeterProvider mp; + auto m = mp.GetMeter("meter1", "version1", "schema1"); + + std::unique_ptr exporter(new MockMetricExporterForStress()); + std::shared_ptr reader{new MockMetricReader(std::move(exporter))}; + mp.AddMetricReader(reader); + + auto h = m->CreateUInt64Histogram("histogram1", "histogram1_description", "histogram1_unit"); + + std::vector actuals; + auto stop_collecting = std::make_shared(false); + auto collect_thread = std::thread([&reader, &actuals, stop_collecting]() { + while (!*stop_collecting) + { + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + reader->Collect([&](ResourceMetrics &rm) { + for (const ScopeMetrics &smd : rm.scope_metric_data_) + { + for (const MetricData &md : smd.metric_data_) + { + for (const PointDataAttributes &dp : md.point_data_attr_) + { + actuals.push_back(opentelemetry::nostd::get(dp.point_data)); + } + } + } + return true; + }); + } + }); + + // + // Start logging threads + // + int hardware_concurrency = std::thread::hardware_concurrency() - 1; + std::vector threads(hardware_concurrency); + + if (hardware_concurrency <= 0) + { + hardware_concurrency = 1; + } + + constexpr int iterations_per_core = 2000000; + auto expected_sum = std::make_shared>(0); + + for (int i = 0; i < hardware_concurrency; ++i) + { + threads[i] = std::thread([&h, expected_sum]() { + std::random_device rd; + std::mt19937 random_engine(rd()); + std::uniform_int_distribution<> gen_random(1, 20000); + + for (int j = 0; j < iterations_per_core; ++j) + { + int64_t val = gen_random(random_engine); + expected_sum->fetch_add(val, std::memory_order_relaxed); + h->Record(val, {}); + } + }); + } + + for (int i = 0; i < hardware_concurrency; ++i) + { + threads[i].join(); + } + + // + // Stop the dedicated collection thread + // + *stop_collecting = true; + collect_thread.join(); + + // + // run the the final collection + // + reader->Collect([&](ResourceMetrics &rm) { + for (const ScopeMetrics &smd : rm.scope_metric_data_) + { + for (const MetricData &md : smd.metric_data_) + { + for (const PointDataAttributes &dp : md.point_data_attr_) + { + actuals.push_back(opentelemetry::nostd::get(dp.point_data)); + } + } + } + return true; + }); + + // + // Aggregate the results + // + int64_t expected_count = hardware_concurrency * iterations_per_core; + int64_t collected_count = 0; + int64_t collected_sum = 0; + for (const auto &actual : actuals) + { + collected_count += actual.count_; + collected_sum += opentelemetry::nostd::get(actual.sum_); + } + + ASSERT_EQ(expected_count, collected_count); + ASSERT_EQ(*expected_sum, collected_sum); +} \ No newline at end of file From 9e224537d8855c289f18861022d5ab40443a2e37 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 14:36:41 -0700 Subject: [PATCH 02/16] format --- sdk/test/metrics/stress_test.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sdk/test/metrics/stress_test.cc b/sdk/test/metrics/stress_test.cc index bc3e643185..3597b25c6c 100644 --- a/sdk/test/metrics/stress_test.cc +++ b/sdk/test/metrics/stress_test.cc @@ -27,7 +27,10 @@ class MockMetricExporterForStress : public opentelemetry::sdk::metrics::PushMetr } opentelemetry::sdk::common::ExportResult Export( - const opentelemetry::sdk::metrics::ResourceMetrics &) noexcept override { return opentelemetry::sdk::common::ExportResult::kSuccess; } + const opentelemetry::sdk::metrics::ResourceMetrics &) noexcept override + { + return opentelemetry::sdk::common::ExportResult::kSuccess; + } bool ForceFlush(std::chrono::microseconds) noexcept override { return true; } @@ -47,7 +50,7 @@ TEST(HistogramStress, UnsignedInt64) std::vector actuals; auto stop_collecting = std::make_shared(false); - auto collect_thread = std::thread([&reader, &actuals, stop_collecting]() { + auto collect_thread = std::thread([&reader, &actuals, stop_collecting]() { while (!*stop_collecting) { std::this_thread::sleep_for(std::chrono::milliseconds(1000)); @@ -79,7 +82,7 @@ TEST(HistogramStress, UnsignedInt64) } constexpr int iterations_per_core = 2000000; - auto expected_sum = std::make_shared>(0); + auto expected_sum = std::make_shared>(0); for (int i = 0; i < hardware_concurrency; ++i) { @@ -128,9 +131,9 @@ TEST(HistogramStress, UnsignedInt64) // // Aggregate the results // - int64_t expected_count = hardware_concurrency * iterations_per_core; + int64_t expected_count = hardware_concurrency * iterations_per_core; int64_t collected_count = 0; - int64_t collected_sum = 0; + int64_t collected_sum = 0; for (const auto &actual : actuals) { collected_count += actual.count_; From b716dc2c6de3cba627450de3581bf3ca7bd8f4b1 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 15:01:27 -0700 Subject: [PATCH 03/16] Add new line to the end of stress test fle --- sdk/test/metrics/stress_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/test/metrics/stress_test.cc b/sdk/test/metrics/stress_test.cc index 3597b25c6c..e807cb1b23 100644 --- a/sdk/test/metrics/stress_test.cc +++ b/sdk/test/metrics/stress_test.cc @@ -142,4 +142,4 @@ TEST(HistogramStress, UnsignedInt64) ASSERT_EQ(expected_count, collected_count); ASSERT_EQ(*expected_sum, collected_sum); -} \ No newline at end of file +} From b673bc0d3b5d3cf90d245bd8b0957d75c9952ef3 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 16:02:20 -0700 Subject: [PATCH 04/16] Fix iywu --- sdk/test/metrics/stress_test.cc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/sdk/test/metrics/stress_test.cc b/sdk/test/metrics/stress_test.cc index e807cb1b23..66d4fe475c 100644 --- a/sdk/test/metrics/stress_test.cc +++ b/sdk/test/metrics/stress_test.cc @@ -3,13 +3,31 @@ #include +#include #include +#include #include #include +#include #include #include "common.h" +#include "opentelemetry/context/context.h" +#include "opentelemetry/metrics/meter.h" +#include "opentelemetry/metrics/sync_instruments.h" +#include "opentelemetry/nostd/function_ref.h" +#include "opentelemetry/nostd/shared_ptr.h" +#include "opentelemetry/nostd/unique_ptr.h" +#include "opentelemetry/nostd/variant.h" +#include "opentelemetry/sdk/common/exporter_utils.h" +#include "opentelemetry/sdk/instrumentationscope/instrumentation_scope.h" +#include "opentelemetry/sdk/metrics/data/metric_data.h" +#include "opentelemetry/sdk/metrics/data/point_data.h" +#include "opentelemetry/sdk/metrics/export/metric_producer.h" +#include "opentelemetry/sdk/metrics/instruments.h" #include "opentelemetry/sdk/metrics/meter_provider.h" +#include "opentelemetry/sdk/metrics/metric_reader.h" +#include "opentelemetry/sdk/metrics/push_metric_exporter.h" using namespace opentelemetry; using namespace opentelemetry::sdk::instrumentationscope; From 58de6d23a6b89185716b8a561abb173a9f34c9b9 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 17:58:53 -0700 Subject: [PATCH 05/16] Fix lambda capture --- sdk/test/metrics/stress_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/test/metrics/stress_test.cc b/sdk/test/metrics/stress_test.cc index 66d4fe475c..9aabc82f2e 100644 --- a/sdk/test/metrics/stress_test.cc +++ b/sdk/test/metrics/stress_test.cc @@ -104,7 +104,7 @@ TEST(HistogramStress, UnsignedInt64) for (int i = 0; i < hardware_concurrency; ++i) { - threads[i] = std::thread([&h, expected_sum]() { + threads[i] = std::thread([&h, expected_sum, iterations_per_core] { std::random_device rd; std::mt19937 random_engine(rd()); std::uniform_int_distribution<> gen_random(1, 20000); From a3f5a9635293c21cd8d9ba8bd1dee2f9445cd84f Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 18:01:43 -0700 Subject: [PATCH 06/16] Make stop_collecting an atomic variable --- sdk/test/metrics/stress_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/test/metrics/stress_test.cc b/sdk/test/metrics/stress_test.cc index 9aabc82f2e..cd81d574f2 100644 --- a/sdk/test/metrics/stress_test.cc +++ b/sdk/test/metrics/stress_test.cc @@ -67,7 +67,7 @@ TEST(HistogramStress, UnsignedInt64) auto h = m->CreateUInt64Histogram("histogram1", "histogram1_description", "histogram1_unit"); std::vector actuals; - auto stop_collecting = std::make_shared(false); + auto stop_collecting = std::make_shared>(false); auto collect_thread = std::thread([&reader, &actuals, stop_collecting]() { while (!*stop_collecting) { From 1d51a7ce783c9a55c403e73a533997617c181163 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 18:08:57 -0700 Subject: [PATCH 07/16] Add comment --- sdk/test/metrics/stress_test.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/test/metrics/stress_test.cc b/sdk/test/metrics/stress_test.cc index cd81d574f2..d94bb7eaaa 100644 --- a/sdk/test/metrics/stress_test.cc +++ b/sdk/test/metrics/stress_test.cc @@ -66,6 +66,9 @@ TEST(HistogramStress, UnsignedInt64) auto h = m->CreateUInt64Histogram("histogram1", "histogram1_description", "histogram1_unit"); + // + // Start a dedicated thread to collect the metrics + // std::vector actuals; auto stop_collecting = std::make_shared>(false); auto collect_thread = std::thread([&reader, &actuals, stop_collecting]() { From cbcd3d5f1b624d7acd18f0be44459dd3a4ceee30 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 18:25:29 -0700 Subject: [PATCH 08/16] Increase timeout for stress run --- sdk/test/metrics/BUILD | 17 +++++++++++++++++ sdk/test/metrics/CMakeLists.txt | 2 +- .../{stress_test.cc => metric_test_stress.cc} | 0 3 files changed, 18 insertions(+), 1 deletion(-) rename sdk/test/metrics/{stress_test.cc => metric_test_stress.cc} (100%) diff --git a/sdk/test/metrics/BUILD b/sdk/test/metrics/BUILD index 519cb1fc1f..4fc8336ce2 100644 --- a/sdk/test/metrics/BUILD +++ b/sdk/test/metrics/BUILD @@ -49,6 +49,23 @@ cc_test( ], ) +cc_test( + name = "stress_tests", + srcs = glob(["*_test_stress.cc"]), + copts = [ + "-DUNIT_TESTING", + ], + tags = [ + "metrics", + "test", + ], + deps = [ + "metrics_common_test_utils", + "@com_google_googletest//:gtest_main", + ], + timeout = "long", +) + otel_cc_benchmark( name = "attributes_processor_benchmark", srcs = [ diff --git a/sdk/test/metrics/CMakeLists.txt b/sdk/test/metrics/CMakeLists.txt index 2c4f733665..dfc3e9f772 100644 --- a/sdk/test/metrics/CMakeLists.txt +++ b/sdk/test/metrics/CMakeLists.txt @@ -34,7 +34,7 @@ foreach( observable_registry_test periodic_exporting_metric_reader_test instrument_metadata_validator_test - stress_test) + metric_test_stress.cc) add_executable(${testname} "${testname}.cc") target_link_libraries( ${testname} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} diff --git a/sdk/test/metrics/stress_test.cc b/sdk/test/metrics/metric_test_stress.cc similarity index 100% rename from sdk/test/metrics/stress_test.cc rename to sdk/test/metrics/metric_test_stress.cc From 2d47d30ef177eca41e9b0c7cff0c7a502bc24bb7 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 18:38:58 -0700 Subject: [PATCH 09/16] Remove .cc from test name --- sdk/test/metrics/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/test/metrics/CMakeLists.txt b/sdk/test/metrics/CMakeLists.txt index dfc3e9f772..7269330708 100644 --- a/sdk/test/metrics/CMakeLists.txt +++ b/sdk/test/metrics/CMakeLists.txt @@ -34,7 +34,7 @@ foreach( observable_registry_test periodic_exporting_metric_reader_test instrument_metadata_validator_test - metric_test_stress.cc) + metric_test_stress) add_executable(${testname} "${testname}.cc") target_link_libraries( ${testname} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} From 23a7cffc1642ff7161a422defaac3c2ca87324ad Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 18:42:39 -0700 Subject: [PATCH 10/16] Fix bazel format --- sdk/test/metrics/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/test/metrics/BUILD b/sdk/test/metrics/BUILD index 4fc8336ce2..dcd7a323f1 100644 --- a/sdk/test/metrics/BUILD +++ b/sdk/test/metrics/BUILD @@ -51,6 +51,7 @@ cc_test( cc_test( name = "stress_tests", + timeout = "long", srcs = glob(["*_test_stress.cc"]), copts = [ "-DUNIT_TESTING", @@ -63,7 +64,6 @@ cc_test( "metrics_common_test_utils", "@com_google_googletest//:gtest_main", ], - timeout = "long", ) otel_cc_benchmark( From ff6c805c912256c4a87fd8a3531cc5671b95204b Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 19:06:31 -0700 Subject: [PATCH 11/16] Remove uncessary capture --- sdk/test/metrics/metric_test_stress.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/test/metrics/metric_test_stress.cc b/sdk/test/metrics/metric_test_stress.cc index d94bb7eaaa..f24d2922fb 100644 --- a/sdk/test/metrics/metric_test_stress.cc +++ b/sdk/test/metrics/metric_test_stress.cc @@ -107,7 +107,7 @@ TEST(HistogramStress, UnsignedInt64) for (int i = 0; i < hardware_concurrency; ++i) { - threads[i] = std::thread([&h, expected_sum, iterations_per_core] { + threads[i] = std::thread([&h, expected_sum] { std::random_device rd; std::mt19937 random_engine(rd()); std::uniform_int_distribution<> gen_random(1, 20000); From a332699eea0fa8d2398c69330b816a66fed69070 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 19:22:53 -0700 Subject: [PATCH 12/16] Set default lambda capture mode to reference --- sdk/test/metrics/metric_test_stress.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/test/metrics/metric_test_stress.cc b/sdk/test/metrics/metric_test_stress.cc index f24d2922fb..183e60c0b5 100644 --- a/sdk/test/metrics/metric_test_stress.cc +++ b/sdk/test/metrics/metric_test_stress.cc @@ -107,7 +107,7 @@ TEST(HistogramStress, UnsignedInt64) for (int i = 0; i < hardware_concurrency; ++i) { - threads[i] = std::thread([&h, expected_sum] { + threads[i] = std::thread([&] { std::random_device rd; std::mt19937 random_engine(rd()); std::uniform_int_distribution<> gen_random(1, 20000); From 2f3082001ab637d630a7e40882faf08e663e2120 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 20:49:05 -0700 Subject: [PATCH 13/16] Rename variable for better clarify --- sdk/test/metrics/metric_test_stress.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sdk/test/metrics/metric_test_stress.cc b/sdk/test/metrics/metric_test_stress.cc index 183e60c0b5..8291fb7cd5 100644 --- a/sdk/test/metrics/metric_test_stress.cc +++ b/sdk/test/metrics/metric_test_stress.cc @@ -94,25 +94,25 @@ TEST(HistogramStress, UnsignedInt64) // // Start logging threads // - int hardware_concurrency = std::thread::hardware_concurrency() - 1; - std::vector threads(hardware_concurrency); + int record_thread_count = std::thread::hardware_concurrency() - 1; + std::vector threads(record_thread_count); - if (hardware_concurrency <= 0) + if (record_thread_count <= 0) { - hardware_concurrency = 1; + record_thread_count = 1; } - constexpr int iterations_per_core = 2000000; + constexpr int iterations_per_thread = 2000000; auto expected_sum = std::make_shared>(0); - for (int i = 0; i < hardware_concurrency; ++i) + for (int i = 0; i < record_thread_count; ++i) { threads[i] = std::thread([&] { std::random_device rd; std::mt19937 random_engine(rd()); std::uniform_int_distribution<> gen_random(1, 20000); - for (int j = 0; j < iterations_per_core; ++j) + for (int j = 0; j < iterations_per_thread; ++j) { int64_t val = gen_random(random_engine); expected_sum->fetch_add(val, std::memory_order_relaxed); @@ -121,7 +121,7 @@ TEST(HistogramStress, UnsignedInt64) }); } - for (int i = 0; i < hardware_concurrency; ++i) + for (int i = 0; i < record_thread_count; ++i) { threads[i].join(); } @@ -152,7 +152,7 @@ TEST(HistogramStress, UnsignedInt64) // // Aggregate the results // - int64_t expected_count = hardware_concurrency * iterations_per_core; + int64_t expected_count = record_thread_count * iterations_per_thread; int64_t collected_count = 0; int64_t collected_sum = 0; for (const auto &actual : actuals) From 2ab73112da99b7d259d2133857f125983bf06105 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 20:53:09 -0700 Subject: [PATCH 14/16] Correct record_thread_count before using it --- sdk/test/metrics/metric_test_stress.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/test/metrics/metric_test_stress.cc b/sdk/test/metrics/metric_test_stress.cc index 8291fb7cd5..ec3d81016b 100644 --- a/sdk/test/metrics/metric_test_stress.cc +++ b/sdk/test/metrics/metric_test_stress.cc @@ -95,13 +95,12 @@ TEST(HistogramStress, UnsignedInt64) // Start logging threads // int record_thread_count = std::thread::hardware_concurrency() - 1; - std::vector threads(record_thread_count); - if (record_thread_count <= 0) { record_thread_count = 1; } + std::vector threads(record_thread_count); constexpr int iterations_per_thread = 2000000; auto expected_sum = std::make_shared>(0); From 293620e4d006098b114611c7f5f61b5fb67aea47 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 20:59:42 -0700 Subject: [PATCH 15/16] Reformat --- sdk/test/metrics/metric_test_stress.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/test/metrics/metric_test_stress.cc b/sdk/test/metrics/metric_test_stress.cc index ec3d81016b..dd486a5b23 100644 --- a/sdk/test/metrics/metric_test_stress.cc +++ b/sdk/test/metrics/metric_test_stress.cc @@ -102,7 +102,7 @@ TEST(HistogramStress, UnsignedInt64) std::vector threads(record_thread_count); constexpr int iterations_per_thread = 2000000; - auto expected_sum = std::make_shared>(0); + auto expected_sum = std::make_shared>(0); for (int i = 0; i < record_thread_count; ++i) { From 6932d544fcc256f902e064b898c9f783728661c8 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 2 May 2025 21:56:14 -0700 Subject: [PATCH 16/16] Test bucket count --- sdk/test/metrics/metric_test_stress.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sdk/test/metrics/metric_test_stress.cc b/sdk/test/metrics/metric_test_stress.cc index dd486a5b23..8337852649 100644 --- a/sdk/test/metrics/metric_test_stress.cc +++ b/sdk/test/metrics/metric_test_stress.cc @@ -156,8 +156,15 @@ TEST(HistogramStress, UnsignedInt64) int64_t collected_sum = 0; for (const auto &actual : actuals) { - collected_count += actual.count_; + int64_t collected_bucket_sum = 0; + for (const auto &count : actual.counts_) + { + collected_bucket_sum += count; + } + ASSERT_EQ(collected_bucket_sum, actual.count_); + collected_sum += opentelemetry::nostd::get(actual.sum_); + collected_count += actual.count_; } ASSERT_EQ(expected_count, collected_count);