Skip to content

Commit c685203

Browse files
Alan4506Alan4506
committed
implement the monitoring interface for performance tests
1 parent a6bfd35 commit c685203

File tree

2 files changed

+344
-0
lines changed

2 files changed

+344
-0
lines changed

‎tests/performance-tests/include/performance_tests/reporting/JsonReportingMetrics.h‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
/**
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0.
4+
*/
5+
6+
#pragma once
7+
8+
#include <aws/core/client/AWSClient.h>
9+
#include <aws/core/http/HttpRequest.h>
10+
#include <aws/core/monitoring/CoreMetrics.h>
11+
#include <aws/core/monitoring/MonitoringFactory.h>
12+
#include <aws/core/monitoring/MonitoringInterface.h>
13+
#include <aws/core/utils/memory/AWSMemory.h>
14+
#include <aws/core/utils/memory/stl/AWSSet.h>
15+
#include <aws/core/utils/memory/stl/AWSString.h>
16+
#include <aws/core/utils/memory/stl/AWSVector.h>
17+
18+
#include <cstdint>
19+
#include <memory>
20+
#include <utility>
21+
22+
namespace PerformanceTest {
23+
namespace Reporting {
24+
/**
25+
* Container for a single performance metric record that stores measurement data and associated metadata.
26+
*/
27+
struct PerformanceMetricRecord {
28+
Aws::String name;
29+
Aws::String description;
30+
Aws::String unit;
31+
int64_t date;
32+
Aws::Vector<int64_t> measurements;
33+
Aws::Vector<std::pair<Aws::String, Aws::String>> dimensions;
34+
};
35+
36+
/**
37+
* An implementation of the MonitoringInterface that collects performance metrics
38+
* and reports them in a JSON format.
39+
*/
40+
class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface {
41+
public:
42+
~JsonReportingMetrics() override;
43+
44+
/**
45+
* Called when an AWS request is started. Returns context for tracking.
46+
* @param serviceName Name of the AWS service
47+
* @param requestName Name of the operation
48+
* @param request HTTP request object
49+
* @return Context pointer (always returns nullptr)
50+
*/
51+
void* OnRequestStarted(const Aws::String& serviceName, const Aws::String& requestName,
52+
const std::shared_ptr<const Aws::Http::HttpRequest>& request) const override;
53+
54+
/**
55+
* Called when an AWS request succeeds. Records performance metrics.
56+
* @param serviceName Name of the AWS service
57+
* @param requestName Name of the operation
58+
* @param request HTTP request object
59+
* @param outcome HTTP response outcome
60+
* @param metrics Core metrics collection containing latency data
61+
* @param context Request context (unused)
62+
*/
63+
void OnRequestSucceeded(const Aws::String& serviceName, const Aws::String& requestName,
64+
const std::shared_ptr<const Aws::Http::HttpRequest>& request, const Aws::Client::HttpResponseOutcome& outcome,
65+
const Aws::Monitoring::CoreMetricsCollection& metrics, void* context) const override;
66+
67+
/**
68+
* Called when an AWS request fails. Records performance metrics.
69+
* @param serviceName Name of the AWS service
70+
* @param requestName Name of the operation
71+
* @param request HTTP request object
72+
* @param outcome HTTP response outcome
73+
* @param metrics Core metrics collection containing latency data
74+
* @param context Request context (unused)
75+
*/
76+
void OnRequestFailed(const Aws::String& serviceName, const Aws::String& requestName,
77+
const std::shared_ptr<const Aws::Http::HttpRequest>& request, const Aws::Client::HttpResponseOutcome& outcome,
78+
const Aws::Monitoring::CoreMetricsCollection& metrics, void* context) const override;
79+
80+
/**
81+
* Called when an AWS request is retried. No action taken.
82+
* @param serviceName Name of the AWS service
83+
* @param requestName Name of the operation
84+
* @param request HTTP request object
85+
* @param context Request context (unused)
86+
*/
87+
void OnRequestRetry(const Aws::String& serviceName, const Aws::String& requestName,
88+
const std::shared_ptr<const Aws::Http::HttpRequest>& request, void* context) const override;
89+
90+
/**
91+
* Called when an AWS request finishes. No action taken.
92+
* @param serviceName Name of the AWS service
93+
* @param requestName Name of the operation
94+
* @param request HTTP request object
95+
* @param context Request context (unused)
96+
*/
97+
void OnFinish(const Aws::String& serviceName, const Aws::String& requestName,
98+
const std::shared_ptr<const Aws::Http::HttpRequest>& request, void* context) const override;
99+
100+
/**
101+
* Sets test dimensions that will be included with all performance records.
102+
* @param dimensions Vector of key-value pairs representing test dimensions (e.g., size, bucket type)
103+
*/
104+
static void SetTestContext(const Aws::Vector<std::pair<Aws::String, Aws::String>>& dimensions);
105+
106+
/**
107+
* Registers specific operations to monitor. If empty, all operations are monitored.
108+
* @param operations Vector of operation names to track (e.g., "PutObject", "GetItem")
109+
*/
110+
static void RegisterOperationsToMonitor(const Aws::Vector<Aws::String>& operations);
111+
112+
/**
113+
* Sets product information to include in the JSON output.
114+
* @param productId Product identifier (e.g., "cpp1")
115+
* @param sdkVersion SDK version string
116+
* @param commitId Git commit identifier
117+
*/
118+
static void SetProductInfo(const Aws::String& productId, const Aws::String& sdkVersion, const Aws::String& commitId);
119+
120+
/**
121+
* Sets the output filename for the JSON performance report.
122+
* @param filename Path to output file (e.g., "s3-perf-results.json")
123+
*/
124+
static void SetOutputFilename(const Aws::String& filename);
125+
126+
private:
127+
/**
128+
* Adds a performance record for a completed AWS service operation.
129+
* @param serviceName Name of the AWS service (e.g., "S3", "DynamoDB")
130+
* @param requestName Name of the operation (e.g., "PutObject", "GetItem")
131+
* @param metricsFromCore Core metrics collection containing latency data
132+
*/
133+
void AddPerformanceRecord(const Aws::String& serviceName, const Aws::String& requestName,
134+
const Aws::Monitoring::CoreMetricsCollection& metricsFromCore) const;
135+
136+
/**
137+
* Outputs aggregated performance metrics to JSON file.
138+
* Groups records by name and dimensions, then writes to configured output file.
139+
*/
140+
void DumpJson() const;
141+
142+
mutable Aws::Vector<PerformanceMetricRecord> m_performanceRecords;
143+
static Aws::Vector<std::pair<Aws::String, Aws::String>> TestDimensions;
144+
static Aws::Set<Aws::String> MonitoredOperations;
145+
static Aws::String ProductId;
146+
static Aws::String SdkVersion;
147+
static Aws::String CommitId;
148+
static Aws::String OutputFilename;
149+
};
150+
151+
/**
152+
* A factory for creating instances of JsonReportingMetrics.
153+
* Used by the AWS SDK monitoring system to instantiate performance metrics collectors.
154+
*/
155+
class JsonReportingMetricsFactory : public Aws::Monitoring::MonitoringFactory {
156+
public:
157+
~JsonReportingMetricsFactory() override = default;
158+
/**
159+
* Creates a new JsonReportingMetrics instance for performance monitoring.
160+
* @return Unique pointer to monitoring interface implementation
161+
*/
162+
Aws::UniquePtr<Aws::Monitoring::MonitoringInterface> CreateMonitoringInstance() const override;
163+
};
164+
} // namespace Reporting
165+
} // namespace PerformanceTest

‎tests/performance-tests/src/reporting/JsonReportingMetrics.cpp‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
/**
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0.
4+
*/
5+
6+
#include <aws/core/client/AWSClient.h>
7+
#include <aws/core/monitoring/CoreMetrics.h>
8+
#include <aws/core/monitoring/HttpClientMetrics.h>
9+
#include <aws/core/monitoring/MonitoringInterface.h>
10+
#include <aws/core/utils/Array.h>
11+
#include <aws/core/utils/DateTime.h>
12+
#include <aws/core/utils/StringUtils.h>
13+
#include <aws/core/utils/json/JsonSerializer.h>
14+
#include <aws/core/utils/memory/AWSMemory.h>
15+
#include <aws/core/utils/memory/stl/AWSMap.h>
16+
#include <aws/core/utils/memory/stl/AWSSet.h>
17+
#include <aws/core/utils/memory/stl/AWSString.h>
18+
#include <aws/core/utils/memory/stl/AWSVector.h>
19+
#include <performance_tests/reporting/JsonReportingMetrics.h>
20+
21+
#include <cstddef>
22+
#include <cstdint>
23+
#include <fstream>
24+
#include <memory>
25+
#include <utility>
26+
27+
using namespace PerformanceTest::Reporting;
28+
29+
Aws::Vector<std::pair<Aws::String, Aws::String>> JsonReportingMetrics::TestDimensions;
30+
Aws::Set<Aws::String> JsonReportingMetrics::MonitoredOperations;
31+
Aws::String JsonReportingMetrics::ProductId = "unknown";
32+
Aws::String JsonReportingMetrics::SdkVersion = "unknown";
33+
Aws::String JsonReportingMetrics::CommitId = "unknown";
34+
Aws::String JsonReportingMetrics::OutputFilename = "performance-test-results.json";
35+
36+
void JsonReportingMetrics::SetTestContext(const Aws::Vector<std::pair<Aws::String, Aws::String>>& dimensions) {
37+
TestDimensions = dimensions;
38+
}
39+
40+
void JsonReportingMetrics::RegisterOperationsToMonitor(const Aws::Vector<Aws::String>& operations) {
41+
MonitoredOperations.clear();
42+
for (const auto& operation : operations) {
43+
MonitoredOperations.insert(operation);
44+
}
45+
}
46+
47+
void JsonReportingMetrics::SetProductInfo(const Aws::String& productId, const Aws::String& sdkVersion, const Aws::String& commitId) {
48+
ProductId = productId;
49+
SdkVersion = sdkVersion;
50+
CommitId = commitId;
51+
}
52+
53+
void JsonReportingMetrics::SetOutputFilename(const Aws::String& filename) { OutputFilename = filename; }
54+
55+
JsonReportingMetrics::~JsonReportingMetrics() { DumpJson(); }
56+
57+
Aws::UniquePtr<Aws::Monitoring::MonitoringInterface> JsonReportingMetricsFactory::CreateMonitoringInstance() const {
58+
return Aws::MakeUnique<JsonReportingMetrics>("JsonReportingMetrics");
59+
}
60+
61+
void JsonReportingMetrics::AddPerformanceRecord(const Aws::String& serviceName, const Aws::String& requestName,
62+
const Aws::Monitoring::CoreMetricsCollection& metricsFromCore) const {
63+
// If no operations are registered, monitor all operations. Otherwise, only monitor registered operations
64+
if (!MonitoredOperations.empty() && MonitoredOperations.find(requestName) == MonitoredOperations.end()) {
65+
return;
66+
}
67+
68+
int64_t durationMs = 0;
69+
Aws::String const latencyKey = Aws::Monitoring::GetHttpClientMetricNameByType(Aws::Monitoring::HttpClientMetricsType::RequestLatency);
70+
71+
auto iterator = metricsFromCore.httpClientMetrics.find(latencyKey);
72+
if (iterator != metricsFromCore.httpClientMetrics.end()) {
73+
durationMs = iterator->second;
74+
}
75+
76+
PerformanceMetricRecord record;
77+
record.name =
78+
Aws::Utils::StringUtils::ToLower(serviceName.c_str()) + "." + Aws::Utils::StringUtils::ToLower(requestName.c_str()) + ".latency";
79+
record.description = "Time to complete " + requestName + " operation";
80+
record.unit = "Milliseconds";
81+
record.date = Aws::Utils::DateTime::Now().Seconds();
82+
record.measurements.push_back(durationMs);
83+
record.dimensions = TestDimensions;
84+
85+
m_performanceRecords.push_back(record);
86+
}
87+
88+
void* JsonReportingMetrics::OnRequestStarted(const Aws::String&, const Aws::String&,
89+
const std::shared_ptr<const Aws::Http::HttpRequest>&) const {
90+
return nullptr;
91+
}
92+
93+
void JsonReportingMetrics::OnRequestSucceeded(const Aws::String& serviceName, const Aws::String& requestName,
94+
const std::shared_ptr<const Aws::Http::HttpRequest>&, const Aws::Client::HttpResponseOutcome&,
95+
const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void*) const {
96+
AddPerformanceRecord(serviceName, requestName, metricsFromCore);
97+
}
98+
99+
void JsonReportingMetrics::OnRequestFailed(const Aws::String& serviceName, const Aws::String& requestName,
100+
const std::shared_ptr<const Aws::Http::HttpRequest>&, const Aws::Client::HttpResponseOutcome&,
101+
const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void*) const {
102+
AddPerformanceRecord(serviceName, requestName, metricsFromCore);
103+
}
104+
105+
void JsonReportingMetrics::OnRequestRetry(const Aws::String&, const Aws::String&, const std::shared_ptr<const Aws::Http::HttpRequest>&,
106+
void*) const {}
107+
108+
void JsonReportingMetrics::OnFinish(const Aws::String&, const Aws::String&, const std::shared_ptr<const Aws::Http::HttpRequest>&,
109+
void*) const {}
110+
111+
void JsonReportingMetrics::DumpJson() const {
112+
if (m_performanceRecords.empty()) {
113+
return;
114+
}
115+
116+
// Group performance records by name and dimensions
117+
Aws::Map<Aws::String, PerformanceMetricRecord> aggregatedRecords;
118+
119+
for (const auto& record : m_performanceRecords) {
120+
Aws::String key = record.name;
121+
for (const auto& dim : record.dimensions) {
122+
key += ":" + Aws::String(dim.first) + "=" + Aws::String(dim.second);
123+
}
124+
125+
if (aggregatedRecords.find(key) == aggregatedRecords.end()) {
126+
aggregatedRecords[key] = record;
127+
} else {
128+
for (const auto& measurement : record.measurements) {
129+
aggregatedRecords[key].measurements.push_back(measurement);
130+
}
131+
}
132+
}
133+
134+
// Create the JSON output
135+
Aws::Utils::Json::JsonValue root;
136+
root.WithString("productId", ProductId);
137+
root.WithString("sdkVersion", SdkVersion);
138+
root.WithString("commitId", CommitId);
139+
140+
Aws::Utils::Array<Aws::Utils::Json::JsonValue> results(aggregatedRecords.size());
141+
size_t index = 0;
142+
143+
for (const auto& pair : aggregatedRecords) {
144+
const auto& record = pair.second;
145+
Aws::Utils::Json::JsonValue jsonMetric;
146+
jsonMetric.WithString("name", record.name);
147+
jsonMetric.WithString("description", record.description);
148+
jsonMetric.WithString("unit", record.unit);
149+
jsonMetric.WithInt64("date", record.date);
150+
151+
if (!record.dimensions.empty()) {
152+
Aws::Utils::Array<Aws::Utils::Json::JsonValue> dimensionsArray(record.dimensions.size());
153+
for (size_t j = 0; j < record.dimensions.size(); ++j) {
154+
Aws::Utils::Json::JsonValue dimension;
155+
dimension.WithString("name", record.dimensions[j].first);
156+
dimension.WithString("value", record.dimensions[j].second);
157+
dimensionsArray[j] = std::move(dimension);
158+
}
159+
jsonMetric.WithArray("dimensions", std::move(dimensionsArray));
160+
}
161+
162+
Aws::Utils::Array<Aws::Utils::Json::JsonValue> measurementsArray(record.measurements.size());
163+
for (size_t j = 0; j < record.measurements.size(); ++j) {
164+
Aws::Utils::Json::JsonValue measurementValue;
165+
measurementValue.AsInt64(record.measurements[j]);
166+
measurementsArray[j] = std::move(measurementValue);
167+
}
168+
jsonMetric.WithArray("measurements", std::move(measurementsArray));
169+
170+
results[index++] = std::move(jsonMetric);
171+
}
172+
173+
root.WithArray("results", std::move(results));
174+
175+
std::ofstream outFile(OutputFilename.c_str());
176+
if (outFile.is_open()) {
177+
outFile << root.View().WriteReadable();
178+
}
179+
}

0 commit comments

Comments
 (0)