Skip to content

Commit 5559d12

Browse files
Alan4506Alan4506
committed
add build-mode as a new test dimension into performance tests
1 parent bd9e3be commit 5559d12

File tree

6 files changed

+57
-25
lines changed

6 files changed

+57
-25
lines changed

‎tests/performance-tests/include/performance-tests/reporting/JsonReportingMetrics.h‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -55,25 +55,26 @@ class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface {
5555
* @param sdkVersion SDK version string
5656
* @param commitId Git commit identifier
5757
* @param outputFilename Path to output file (e.g., "s3-perf-results.json")
58+
* @param buildMode Build mode (e.g., "debug" or "release")
5859
*/
5960
JsonReportingMetrics(const Aws::Set<Aws::String>& monitoredOperations = Aws::Set<Aws::String>(), const Aws::String& productId = "unknown",
6061
const Aws::String& sdkVersion = "unknown", const Aws::String& commitId = "unknown",
61-
const Aws::String& outputFilename = "performance-test-results.json");
62+
const Aws::String& outputFilename = "performance-test-results.json", const Aws::String& buildMode = "unknown");
6263

6364
~JsonReportingMetrics() override;
6465

6566
/**
66-
* Called when an AWS request is started. Returns context for tracking.
67+
* Called when an AWS request is started. Creates and returns context for tracking.
6768
* @param serviceName Name of the AWS service
6869
* @param requestName Name of the operation
6970
* @param request HTTP request object
70-
* @return Context pointer (always returns nullptr)
71+
* @return Context pointer to newly created RequestContext
7172
*/
7273
void* OnRequestStarted(const Aws::String& serviceName, const Aws::String& requestName,
7374
const std::shared_ptr<const Aws::Http::HttpRequest>& request) const override;
7475

7576
/**
76-
* Called when an AWS request succeeds. Records performance metrics.
77+
* Called when an AWS request succeeds. Stores latency metrics in context.
7778
* @param serviceName Name of the AWS service
7879
* @param requestName Name of the operation
7980
* @param request HTTP request object
@@ -86,7 +87,7 @@ class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface {
8687
const Aws::Monitoring::CoreMetricsCollection& metrics, void* context) const override;
8788

8889
/**
89-
* Called when an AWS request fails. Records performance metrics.
90+
* Called when an AWS request fails. Stores latency metrics in context.
9091
* @param serviceName Name of the AWS service
9192
* @param requestName Name of the operation
9293
* @param request HTTP request object
@@ -109,7 +110,7 @@ class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface {
109110
const std::shared_ptr<const Aws::Http::HttpRequest>& request, void* context) const override;
110111

111112
/**
112-
* Called when an AWS request finishes. No action taken.
113+
* Called when an AWS request finishes. Processes stored metrics and cleans up context.
113114
* @param serviceName Name of the AWS service
114115
* @param requestName Name of the operation
115116
* @param request HTTP request object
@@ -161,6 +162,7 @@ class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface {
161162
Aws::String m_sdkVersion;
162163
Aws::String m_commitId;
163164
Aws::String m_outputFilename;
165+
Aws::String m_buildMode;
164166
};
165167

166168
/**
@@ -176,10 +178,12 @@ class JsonReportingMetricsFactory : public Aws::Monitoring::MonitoringFactory {
176178
* @param sdkVersion SDK version string
177179
* @param commitId Git commit identifier
178180
* @param outputFilename Path to output file (e.g., "s3-perf-results.json")
181+
* @param buildMode Build mode (e.g., "debug" or "release")
179182
*/
180183
JsonReportingMetricsFactory(const Aws::Set<Aws::String>& monitoredOperations = Aws::Set<Aws::String>(),
181184
const Aws::String& productId = "unknown", const Aws::String& sdkVersion = "unknown",
182-
const Aws::String& commitId = "unknown", const Aws::String& outputFilename = "performance-test-results.json");
185+
const Aws::String& commitId = "unknown", const Aws::String& outputFilename = "performance-test-results.json",
186+
const Aws::String& buildMode = "unknown");
183187

184188
~JsonReportingMetricsFactory() override = default;
185189

@@ -195,6 +199,7 @@ class JsonReportingMetricsFactory : public Aws::Monitoring::MonitoringFactory {
195199
Aws::String m_sdkVersion;
196200
Aws::String m_commitId;
197201
Aws::String m_outputFilename;
202+
Aws::String m_buildMode;
198203
};
199204
} // namespace Reporting
200205
} // namespace PerformanceTest

‎tests/performance-tests/src/reporting/JsonReportingMetrics.cpp‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include <aws/core/utils/Array.h>
1111
#include <aws/core/utils/DateTime.h>
1212
#include <aws/core/utils/StringUtils.h>
13+
#include <aws/core/utils/UUID.h>
1314
#include <aws/core/utils/json/JsonSerializer.h>
1415
#include <aws/core/utils/memory/AWSMemory.h>
1516
#include <aws/core/utils/memory/stl/AWSMap.h>
@@ -37,27 +38,30 @@ struct PerformanceTest::Reporting::RequestContext {
3738
};
3839

3940
JsonReportingMetrics::JsonReportingMetrics(const Aws::Set<Aws::String>& monitoredOperations, const Aws::String& productId,
40-
const Aws::String& sdkVersion, const Aws::String& commitId, const Aws::String& outputFilename)
41+
const Aws::String& sdkVersion, const Aws::String& commitId, const Aws::String& outputFilename,
42+
const Aws::String& buildMode)
4143
: m_monitoredOperations(monitoredOperations),
4244
m_productId(productId),
4345
m_sdkVersion(sdkVersion),
4446
m_commitId(commitId),
45-
m_outputFilename(outputFilename) {}
47+
m_outputFilename(outputFilename),
48+
m_buildMode(buildMode) {}
4649

4750
JsonReportingMetrics::~JsonReportingMetrics() { DumpJson(); }
4851

4952
JsonReportingMetricsFactory::JsonReportingMetricsFactory(const Aws::Set<Aws::String>& monitoredOperations, const Aws::String& productId,
5053
const Aws::String& sdkVersion, const Aws::String& commitId,
51-
const Aws::String& outputFilename)
54+
const Aws::String& outputFilename, const Aws::String& buildMode)
5255
: m_monitoredOperations(monitoredOperations),
5356
m_productId(productId),
5457
m_sdkVersion(sdkVersion),
5558
m_commitId(commitId),
56-
m_outputFilename(outputFilename) {}
59+
m_outputFilename(outputFilename),
60+
m_buildMode(buildMode) {}
5761

5862
Aws::UniquePtr<Aws::Monitoring::MonitoringInterface> JsonReportingMetricsFactory::CreateMonitoringInstance() const {
5963
return Aws::MakeUnique<JsonReportingMetrics>("JsonReportingMetrics", m_monitoredOperations, m_productId, m_sdkVersion, m_commitId,
60-
m_outputFilename);
64+
m_outputFilename, m_buildMode);
6165
}
6266

6367
void JsonReportingMetrics::StoreLatencyInContext(const Aws::String& serviceName, const Aws::String& requestName,
@@ -184,8 +188,16 @@ void JsonReportingMetrics::DumpJson() const {
184188
jsonMetric.WithInt64("date", record.date.Seconds());
185189

186190
if (!record.dimensions.empty()) {
187-
Aws::Utils::Array<Aws::Utils::Json::JsonValue> dimensionsArray(record.dimensions.size());
191+
Aws::Utils::Array<Aws::Utils::Json::JsonValue> dimensionsArray(record.dimensions.size() + (m_buildMode == "unknown" ? 0 : 1));
188192
size_t dimensionIndex = 0;
193+
194+
if (m_buildMode != "unknown") {
195+
Aws::Utils::Json::JsonValue buildModeDimension;
196+
buildModeDimension.WithString("name", "build-mode");
197+
buildModeDimension.WithString("value", m_buildMode);
198+
dimensionsArray[dimensionIndex++] = std::move(buildModeDimension);
199+
}
200+
189201
for (const auto& dim : record.dimensions) {
190202
Aws::Utils::Json::JsonValue dimension;
191203
dimension.WithString("name", dim.first);

‎tests/performance-tests/src/services/dynamodb/main.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,14 @@ int main(int argc, char** argv) {
2323
cxxopts::Options options("dynamodb-perf-test", "DynamoDB Performance Test");
2424
options.add_options()("r,region", "AWS region", cxxopts::value<std::string>()->default_value("us-east-1"))(
2525
"i,iterations", "Number of iterations", cxxopts::value<int>()->default_value("10"))(
26-
"c,commit-id", "Commit ID", cxxopts::value<std::string>()->default_value("unknown"));
26+
"c,commit-id", "Commit ID", cxxopts::value<std::string>()->default_value("unknown"))(
27+
"b,build-mode", "Build mode", cxxopts::value<std::string>()->default_value("unknown"));
2728

2829
auto const result = options.parse(argc, argv);
2930

3031
Aws::String const region = Aws::Utils::StringUtils::to_string(result["region"].as<std::string>());
3132
Aws::String const commitId = Aws::Utils::StringUtils::to_string(result["commit-id"].as<std::string>());
33+
Aws::String const buildMode = Aws::Utils::StringUtils::to_string(result["build-mode"].as<std::string>());
3234
int const iterations = result["iterations"].as<int>();
3335

3436
Aws::SDKOptions sdkOptions;
@@ -41,7 +43,7 @@ int main(int argc, char** argv) {
4143
}
4244
return Aws::MakeUnique<PerformanceTest::Reporting::JsonReportingMetricsFactory>(
4345
"JsonReportingMetricsFactory", operations, "cpp1", versionStr, commitId,
44-
PerformanceTest::Services::DynamoDB::TestConfig::OutputFilename);
46+
PerformanceTest::Services::DynamoDB::TestConfig::OutputFilename, buildMode);
4547
}};
4648

4749
Aws::InitAPI(sdkOptions);

‎tests/performance-tests/src/services/s3/main.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,15 @@ int main(int argc, char** argv) {
2424
options.add_options()("r,region", "AWS region", cxxopts::value<std::string>()->default_value("us-east-1"))(
2525
"a,az-id", "Availability zone ID", cxxopts::value<std::string>()->default_value("use1-az4"))(
2626
"i,iterations", "Number of iterations", cxxopts::value<int>()->default_value("10"))(
27-
"c,commit-id", "Commit ID", cxxopts::value<std::string>()->default_value("unknown"));
27+
"c,commit-id", "Commit ID", cxxopts::value<std::string>()->default_value("unknown"))(
28+
"b,build-mode", "Build mode", cxxopts::value<std::string>()->default_value("unknown"));
2829

2930
auto const result = options.parse(argc, argv);
3031

3132
Aws::String const region = Aws::Utils::StringUtils::to_string(result["region"].as<std::string>());
3233
Aws::String const availabilityZoneId = Aws::Utils::StringUtils::to_string(result["az-id"].as<std::string>());
3334
Aws::String const commitId = Aws::Utils::StringUtils::to_string(result["commit-id"].as<std::string>());
35+
Aws::String const buildMode = Aws::Utils::StringUtils::to_string(result["build-mode"].as<std::string>());
3436
int const iterations = result["iterations"].as<int>();
3537

3638
Aws::SDKOptions sdkOptions;
@@ -42,7 +44,8 @@ int main(int argc, char** argv) {
4244
operations.insert(operation);
4345
}
4446
return Aws::MakeUnique<PerformanceTest::Reporting::JsonReportingMetricsFactory>(
45-
"JsonReportingMetricsFactory", operations, "cpp1", versionStr, commitId, PerformanceTest::Services::S3::TestConfig::OutputFilename);
47+
"JsonReportingMetricsFactory", operations, "cpp1", versionStr, commitId, PerformanceTest::Services::S3::TestConfig::OutputFilename,
48+
buildMode);
4649
}};
4750

4851
Aws::InitAPI(sdkOptions);

‎tools/scripts/build-tests/run-al2-dynamodb-performance-tests.sh‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,22 +11,25 @@ set -e
1111

1212
DEFAULT_REGION="us-east-1"
1313
DEFAULT_ITERATIONS=10
14+
DEFAULT_BUILD_MODE="unknown"
1415

1516
if [ "$#" -lt 1 ]; then
16-
echo "Error: Missing required argument. Usage: ${0} PREFIX_DIR [-r|--region REGION] [-i|--iterations NUM]"
17+
echo "Error: Missing required argument. Usage: ${0} PREFIX_DIR [-r|--region REGION] [-i|--iterations NUM] [-b|--build-mode BUILD_MODE]"
1718
exit 1
1819
fi
1920

2021
PREFIX_DIR="$1"
21-
shift
22+
shift
2223

2324
REGION="$DEFAULT_REGION"
2425
ITERATIONS="$DEFAULT_ITERATIONS"
26+
BUILD_MODE="$DEFAULT_BUILD_MODE"
2527

2628
while [[ "$#" -gt 0 ]]; do
2729
case "$1" in
2830
-r|--region) REGION="$2"; shift 2 ;;
2931
-i|--iterations) ITERATIONS="$2"; shift 2 ;;
32+
-b|--build-mode) BUILD_MODE="$2"; shift 2 ;;
3033
*) echo "Unknown parameter: $1"; exit 1 ;;
3134
esac
3235
done
@@ -42,6 +45,8 @@ fi
4245
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${PREFIX_DIR}/al2-install/lib64/"
4346

4447
cd "${PREFIX_DIR}/al2-build"
45-
if [ -f "${PREFIX_DIR}/aws-sdk-cpp/tools/scripts/suppressions.txt" ]; then export LSAN_OPTIONS=suppressions="${PREFIX_DIR}/aws-sdk-cpp/tools/scripts/suppressions.txt"; fi
46-
./tests/performance-tests/dynamodb-performance-test --region "$REGION" --iterations "$ITERATIONS" --commit-id "$COMMIT_ID"
48+
if [ "$BUILD_MODE" != "release" ] && [ -f "${PREFIX_DIR}/aws-sdk-cpp/tools/scripts/suppressions.txt" ]; then
49+
export LSAN_OPTIONS=suppressions="${PREFIX_DIR}/aws-sdk-cpp/tools/scripts/suppressions.txt"
50+
fi
51+
./tests/performance-tests/dynamodb-performance-test --region "$REGION" --iterations "$ITERATIONS" --commit-id "$COMMIT_ID" --build-mode "$BUILD_MODE"
4752
cat dynamodb-performance-test-results.json

‎tools/scripts/build-tests/run-al2-s3-performance-tests.sh‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,24 +12,27 @@ set -e
1212
DEFAULT_REGION="us-east-1"
1313
DEFAULT_AZ_ID="use1-az4"
1414
DEFAULT_ITERATIONS=10
15+
DEFAULT_BUILD_MODE="unknown"
1516

1617
if [ "$#" -lt 1 ]; then
17-
echo "Error: Missing required argument. Usage: ${0} PREFIX_DIR [-r|--region REGION] [-a|--az-id AZ_ID] [-i|--iterations NUM]"
18+
echo "Error: Missing required argument. Usage: ${0} PREFIX_DIR [-r|--region REGION] [-a|--az-id AZ_ID] [-i|--iterations NUM] [-b|--build-mode BUILD_MODE]"
1819
exit 1
1920
fi
2021

2122
PREFIX_DIR="$1"
22-
shift
23+
shift
2324

2425
REGION="$DEFAULT_REGION"
2526
AZ_ID="$DEFAULT_AZ_ID"
2627
ITERATIONS="$DEFAULT_ITERATIONS"
28+
BUILD_MODE="$DEFAULT_BUILD_MODE"
2729

2830
while [[ "$#" -gt 0 ]]; do
2931
case "$1" in
3032
-r|--region) REGION="$2"; shift 2 ;;
3133
-a|--az-id) AZ_ID="$2"; shift 2 ;;
3234
-i|--iterations) ITERATIONS="$2"; shift 2 ;;
35+
-b|--build-mode) BUILD_MODE="$2"; shift 2 ;;
3336
*) echo "Unknown parameter: $1"; exit 1 ;;
3437
esac
3538
done
@@ -45,6 +48,8 @@ fi
4548
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${PREFIX_DIR}/al2-install/lib64/"
4649

4750
cd "${PREFIX_DIR}/al2-build"
48-
if [ -f "${PREFIX_DIR}/aws-sdk-cpp/tools/scripts/suppressions.txt" ]; then export LSAN_OPTIONS=suppressions="${PREFIX_DIR}/aws-sdk-cpp/tools/scripts/suppressions.txt"; fi
49-
./tests/performance-tests/s3-performance-test --region "$REGION" --az-id "$AZ_ID" --iterations "$ITERATIONS" --commit-id "$COMMIT_ID"
51+
if [ "$BUILD_MODE" != "release" ] && [ -f "${PREFIX_DIR}/aws-sdk-cpp/tools/scripts/suppressions.txt" ]; then
52+
export LSAN_OPTIONS=suppressions="${PREFIX_DIR}/aws-sdk-cpp/tools/scripts/suppressions.txt"
53+
fi
54+
./tests/performance-tests/s3-performance-test --region "$REGION" --az-id "$AZ_ID" --iterations "$ITERATIONS" --commit-id "$COMMIT_ID" --build-mode "$BUILD_MODE"
5055
cat s3-performance-test-results.json

0 commit comments

Comments
 (0)