From 6a2c33f8ffd801accd406855c8906db3baca5e95 Mon Sep 17 00:00:00 2001 From: sbaluja Date: Tue, 29 Jul 2025 10:34:09 -0400 Subject: [PATCH 01/10] Reuse checksum on streaming request retries --- .../aws/core/AmazonWebServiceRequest.h | 8 ++++++- .../client/features/ChecksumInterceptor.h | 21 ++++++++++++------- .../source/client/AWSClient.cpp | 13 ++++++++++++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h b/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h index 2b2a9e13c7d7..097e152c3ea4 100644 --- a/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h +++ b/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h @@ -35,6 +35,10 @@ namespace Aws typedef std::function RequestRetryHandler; typedef std::function RequestSignedHandler; + struct RetryContext { + std::pair> m_requestHash; + }; + /** * Base level abstraction for all modeled AWS requests */ @@ -222,7 +226,9 @@ namespace Aws */ Aws::Set GetUserAgentFeatures() const { return m_userAgentFeatures; } - inline virtual bool RequestChecksumRequired() const { return false; } + inline virtual bool RequestChecksumRequired() const { return false; } + + mutable Aws::RetryContext m_retryContext; protected: /** * Default does nothing. Override this to convert what would otherwise be the payload of the diff --git a/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h b/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h index a860642bf573..9e61b0f09861 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h @@ -84,42 +84,49 @@ class ChecksumInterceptor : public smithy::interceptor::Interceptor { // AwsAuthSigner decide it. if (request.IsStreaming() && checksumValueAndAlgorithmProvided) { addChecksumFeatureForChecksumName(checksumAlgorithmName, request); - const auto hash = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM, checksumHeader->second); - httpRequest->SetRequestHash(checksumAlgorithmName, hash); + if (httpRequest->GetRequestHash().second == nullptr) { + const auto hash = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM, checksumHeader->second); + httpRequest->SetRequestHash(checksumAlgorithmName, hash); + } } else if (checksumValueAndAlgorithmProvided) { httpRequest->SetHeaderValue(checksumType, checksumHeader->second); } else if (checksumAlgorithmName == "crc64nvme") { request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC64); if (request.IsStreaming()) { - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); + if (httpRequest->GetRequestHash().second == nullptr) + httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); } else { httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateCRC64(*(GetBodyStream(request))))); } } else if (checksumAlgorithmName == "crc32") { request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32); if (request.IsStreaming()) { - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); + if (httpRequest->GetRequestHash().second == nullptr) + httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); } else { httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateCRC32(*(GetBodyStream(request))))); } } else if (checksumAlgorithmName == "crc32c") { request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32C); if (request.IsStreaming()) { - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); + if (httpRequest->GetRequestHash().second == nullptr) + httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); } else { httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateCRC32C(*(GetBodyStream(request))))); } } else if (checksumAlgorithmName == "sha256") { request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA256); if (request.IsStreaming()) { - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); + if (httpRequest->GetRequestHash().second == nullptr) + httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); } else { httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateSHA256(*(GetBodyStream(request))))); } } else if (checksumAlgorithmName == "sha1") { request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA1); if (request.IsStreaming()) { - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); + if (httpRequest->GetRequestHash().second == nullptr) + httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); } else { httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateSHA1(*(GetBodyStream(request))))); } diff --git a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp index 80ece5c90515..14c6687c3ae3 100644 --- a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp +++ b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp @@ -389,6 +389,13 @@ HttpResponseOutcome AWSClient::AttemptExhaustively(const Aws::Http::URI& uri, { newUri.SetAuthority(newEndpoint); } + + // Save checksum information from the original request if we haven't already - safe to assume that the checksum has been finalized, since we have sent and received a response + if (request.m_retryContext.m_requestHash.second == nullptr) { + auto originalRequestHash = httpRequest->GetRequestHash(); + request.m_retryContext.m_requestHash = originalRequestHash; + } + httpRequest = CreateHttpRequest(newUri, method, request.GetResponseStreamFactory()); httpRequest->SetHeaderValue(Http::SDK_INVOCATION_ID_HEADER, invocationId); @@ -920,6 +927,12 @@ void AWSClient::BuildHttpRequest(const Aws::AmazonWebServiceRequest& request, co httpRequest->SetContinueRequestHandle(request.GetContinueRequestHandler()); httpRequest->SetServiceSpecificParameters(request.GetServiceSpecificParameters()); request.AddQueryStringParameters(httpRequest->GetUri()); + + // check for retry context, if present use it + if (request.m_retryContext.m_requestHash.second != nullptr) { + const auto hash = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM, HashingUtils::Base64Encode(request.m_retryContext.m_requestHash.second->GetHash().GetResult())); + httpRequest->SetRequestHash(request.m_retryContext.m_requestHash.first, hash); + } } Aws::String AWSClient::GeneratePresignedUrl(const Aws::Http::URI& uri, Aws::Http::HttpMethod method, long long expirationInSeconds, const std::shared_ptr serviceSpecificParameter) From 17879fdd4a5cedcde6f317878f4800eb66736784 Mon Sep 17 00:00:00 2001 From: sbaluja Date: Tue, 29 Jul 2025 15:16:55 -0400 Subject: [PATCH 02/10] Fix precalculated hash allocationtag namespace --- src/aws-cpp-sdk-core/source/client/AWSClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp index 14c6687c3ae3..efedac0054a5 100644 --- a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp +++ b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp @@ -930,7 +930,7 @@ void AWSClient::BuildHttpRequest(const Aws::AmazonWebServiceRequest& request, co // check for retry context, if present use it if (request.m_retryContext.m_requestHash.second != nullptr) { - const auto hash = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM, HashingUtils::Base64Encode(request.m_retryContext.m_requestHash.second->GetHash().GetResult())); + const auto hash = Aws::MakeShared(smithy::client::AWS_SMITHY_CLIENT_CHECKSUM, HashingUtils::Base64Encode(request.m_retryContext.m_requestHash.second->GetHash().GetResult())); httpRequest->SetRequestHash(request.m_retryContext.m_requestHash.first, hash); } } From ddeb5ed979c4e8c46519d800f7b5c044b085159d Mon Sep 17 00:00:00 2001 From: sbaluja Date: Wed, 30 Jul 2025 14:07:11 -0400 Subject: [PATCH 03/10] ChecksumInterceptor.h - Refactor ModifyBeforeSigning logic for readability/scalability --- .../client/features/ChecksumInterceptor.h | 165 ++++++++++-------- 1 file changed, 92 insertions(+), 73 deletions(-) diff --git a/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h b/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h index 9e61b0f09861..eeb333c66ce9 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h @@ -82,56 +82,10 @@ class ChecksumInterceptor : public smithy::interceptor::Interceptor { // For non-streaming payload, the resolved checksum location is always header. // For streaming payload, the resolved checksum location depends on whether it is an unsigned payload, we let // AwsAuthSigner decide it. - if (request.IsStreaming() && checksumValueAndAlgorithmProvided) { - addChecksumFeatureForChecksumName(checksumAlgorithmName, request); - if (httpRequest->GetRequestHash().second == nullptr) { - const auto hash = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM, checksumHeader->second); - httpRequest->SetRequestHash(checksumAlgorithmName, hash); - } - } else if (checksumValueAndAlgorithmProvided) { - httpRequest->SetHeaderValue(checksumType, checksumHeader->second); - } else if (checksumAlgorithmName == "crc64nvme") { - request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC64); - if (request.IsStreaming()) { - if (httpRequest->GetRequestHash().second == nullptr) - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); - } else { - httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateCRC64(*(GetBodyStream(request))))); - } - } else if (checksumAlgorithmName == "crc32") { - request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32); - if (request.IsStreaming()) { - if (httpRequest->GetRequestHash().second == nullptr) - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); - } else { - httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateCRC32(*(GetBodyStream(request))))); - } - } else if (checksumAlgorithmName == "crc32c") { - request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32C); - if (request.IsStreaming()) { - if (httpRequest->GetRequestHash().second == nullptr) - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); - } else { - httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateCRC32C(*(GetBodyStream(request))))); - } - } else if (checksumAlgorithmName == "sha256") { - request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA256); - if (request.IsStreaming()) { - if (httpRequest->GetRequestHash().second == nullptr) - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); - } else { - httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateSHA256(*(GetBodyStream(request))))); - } - } else if (checksumAlgorithmName == "sha1") { - request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA1); - if (request.IsStreaming()) { - if (httpRequest->GetRequestHash().second == nullptr) - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); - } else { - httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateSHA1(*(GetBodyStream(request))))); - } + if (checksumValueAndAlgorithmProvided) { + handleProvidedChecksum(request, httpRequest, checksumAlgorithmName, checksumType, checksumHeader->second); } else { - AWS_LOGSTREAM_WARN(AWS_SMITHY_CLIENT_CHECKSUM, "Checksum algorithm: " << checksumAlgorithmName << "is not supported by SDK."); + calculateAndSetChecksum(request, httpRequest, checksumAlgorithmName, checksumType); } } } @@ -140,30 +94,7 @@ class ChecksumInterceptor : public smithy::interceptor::Interceptor { if ((!request.GetResponseChecksumAlgorithmNames().empty() && m_responseChecksumValidation == ResponseChecksumValidation::WHEN_SUPPORTED) || request.ShouldValidateResponseChecksum()) { - for (const Aws::String& responseChecksumAlgorithmName : request.GetResponseChecksumAlgorithmNames()) { - const auto responseChecksum = Aws::Utils::StringUtils::ToLower(responseChecksumAlgorithmName.c_str()); - if (responseChecksum == "crc32c") { - std::shared_ptr crc32c = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); - httpRequest->AddResponseValidationHash("crc32c", crc32c); - } else if (responseChecksum == "crc32") { - std::shared_ptr crc32 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); - httpRequest->AddResponseValidationHash("crc32", crc32); - } else if (responseChecksum == "sha1") { - std::shared_ptr sha1 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); - httpRequest->AddResponseValidationHash("sha1", sha1); - } else if (responseChecksum == "sha256") { - std::shared_ptr sha256 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); - httpRequest->AddResponseValidationHash("sha256", sha256); - } else if (responseChecksum == "crc64nvme") { - std::shared_ptr crc64 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); - httpRequest->AddResponseValidationHash("crc64nvme", crc64); - } else { - AWS_LOGSTREAM_WARN(AWS_SMITHY_CLIENT_CHECKSUM, - "Checksum algorithm: " << responseChecksum << " is not supported in validating response body yet."); - } - } - // we have to set the checksum mode to enabled if it was not previously - httpRequest->SetHeaderValue("x-amz-checksum-mode", "enabled"); + SetResponseChecksum(request, httpRequest); } return httpRequest; @@ -240,6 +171,94 @@ class ChecksumInterceptor : public smithy::interceptor::Interceptor { } } + void handleProvidedChecksum(const Aws::AmazonWebServiceRequest& request, std::shared_ptr httpRequest, + const Aws::String& algorithm, const Aws::String& checksumType, const Aws::String& checksumValue) { + if (request.IsStreaming()) { + addChecksumFeatureForChecksumName(algorithm, request); + if (httpRequest->GetRequestHash().second == nullptr) { + auto hash = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM, checksumValue); + httpRequest->SetRequestHash(algorithm, hash); + } + } else { + httpRequest->SetHeaderValue(checksumType, checksumValue); + } + } + + void calculateAndSetChecksum(const Aws::AmazonWebServiceRequest& request, std::shared_ptr httpRequest, + const Aws::String& algorithm, const Aws::String& checksumType) { + static const Aws::UnorderedMap algorithmMap = { + {"crc64nvme", + {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateCRC64(stream)); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC64}}, + {"crc32", + {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateCRC32(stream)); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32}}, + {"crc32c", + {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateCRC32C(stream)); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32C}}, + {"sha256", + {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateSHA256(stream)); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA256}}, + {"sha1", + {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateSHA1(stream)); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA1}} + }; + + auto it = algorithmMap.find(algorithm); + if (it == algorithmMap.end()) { + AWS_LOGSTREAM_WARN(AWS_SMITHY_CLIENT_CHECKSUM, "Checksum algorithm: " << algorithm << " is not supported by SDK."); + return; + } + + request.AddUserAgentFeature(it->second.userAgentFeature); + + if (request.IsStreaming()) { + if (httpRequest->GetRequestHash().second == nullptr) { + httpRequest->SetRequestHash(algorithm, it->second.createHash()); + } + } else { + httpRequest->SetHeaderValue(checksumType, it->second.calculateHash(*GetBodyStream(request))); + } + } + + void SetResponseChecksum(const Aws::AmazonWebServiceRequest& request, std::shared_ptr httpRequest) { + for (const Aws::String& responseChecksumAlgorithmName : request.GetResponseChecksumAlgorithmNames()) { + const auto responseChecksum = Aws::Utils::StringUtils::ToLower(responseChecksumAlgorithmName.c_str()); + if (responseChecksum == "crc32c") { + std::shared_ptr crc32c = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); + httpRequest->AddResponseValidationHash("crc32c", crc32c); + } else if (responseChecksum == "crc32") { + std::shared_ptr crc32 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); + httpRequest->AddResponseValidationHash("crc32", crc32); + } else if (responseChecksum == "sha1") { + std::shared_ptr sha1 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); + httpRequest->AddResponseValidationHash("sha1", sha1); + } else if (responseChecksum == "sha256") { + std::shared_ptr sha256 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); + httpRequest->AddResponseValidationHash("sha256", sha256); + } else if (responseChecksum == "crc64nvme") { + std::shared_ptr crc64 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); + httpRequest->AddResponseValidationHash("crc64nvme", crc64); + } else { + AWS_LOGSTREAM_WARN(AWS_SMITHY_CLIENT_CHECKSUM, + "Checksum algorithm: " << responseChecksum << " is not supported in validating response body yet."); + } + } + // we have to set the checksum mode to enabled if it was not previously + httpRequest->SetHeaderValue("x-amz-checksum-mode", "enabled"); + } + + struct ChecksumHandler { + std::function()> createHash; + std::function calculateHash; + Aws::Client::UserAgentFeature userAgentFeature; + }; + RequestChecksumCalculation m_requestChecksumCalculation{RequestChecksumCalculation::WHEN_SUPPORTED}; ResponseChecksumValidation m_responseChecksumValidation{ResponseChecksumValidation::WHEN_SUPPORTED}; }; From 291e457c438f7c718e70850266792baff4ca2846 Mon Sep 17 00:00:00 2001 From: sbaluja Date: Thu, 31 Jul 2025 10:31:09 -0400 Subject: [PATCH 04/10] ChecksumInterceptor.h - use memory safe Aws::Array instead of Aws::UnorderedMap --- .../client/features/ChecksumInterceptor.h | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h b/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h index eeb333c66ce9..627a16757b2a 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -186,30 +187,30 @@ class ChecksumInterceptor : public smithy::interceptor::Interceptor { void calculateAndSetChecksum(const Aws::AmazonWebServiceRequest& request, std::shared_ptr httpRequest, const Aws::String& algorithm, const Aws::String& checksumType) { - static const Aws::UnorderedMap algorithmMap = { - {"crc64nvme", - {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + static const Aws::Array, 5> algorithmMap = {{ + std::make_pair("crc64nvme", ChecksumHandler{ + []() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateCRC64(stream)); }, - Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC64}}, - {"crc32", - {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC64}), + std::make_pair("crc32", ChecksumHandler{ + []() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateCRC32(stream)); }, - Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32}}, - {"crc32c", - {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32}), + std::make_pair("crc32c", ChecksumHandler{ + []() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateCRC32C(stream)); }, - Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32C}}, - {"sha256", - {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32C}), + std::make_pair("sha256", ChecksumHandler{ + []() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateSHA256(stream)); }, - Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA256}}, - {"sha1", - {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA256}), + std::make_pair("sha1", ChecksumHandler{ + []() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateSHA1(stream)); }, - Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA1}} - }; + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA1}) + }}; - auto it = algorithmMap.find(algorithm); + const auto it = find_if(algorithmMap.begin(), algorithmMap.end(), [&](const std::pair &pair) { return algorithm == pair.first; }); if (it == algorithmMap.end()) { AWS_LOGSTREAM_WARN(AWS_SMITHY_CLIENT_CHECKSUM, "Checksum algorithm: " << algorithm << " is not supported by SDK."); return; From 1d69ae41e37c01cef22d3c23d3263803774fcd8e Mon Sep 17 00:00:00 2001 From: sbaluja Date: Tue, 29 Jul 2025 10:34:09 -0400 Subject: [PATCH 05/10] Reuse checksum on streaming request retries --- .../aws/core/AmazonWebServiceRequest.h | 8 ++++++- .../client/features/ChecksumInterceptor.h | 21 ++++++++++++------- .../source/client/AWSClient.cpp | 13 ++++++++++++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h b/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h index 2b2a9e13c7d7..097e152c3ea4 100644 --- a/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h +++ b/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h @@ -35,6 +35,10 @@ namespace Aws typedef std::function RequestRetryHandler; typedef std::function RequestSignedHandler; + struct RetryContext { + std::pair> m_requestHash; + }; + /** * Base level abstraction for all modeled AWS requests */ @@ -222,7 +226,9 @@ namespace Aws */ Aws::Set GetUserAgentFeatures() const { return m_userAgentFeatures; } - inline virtual bool RequestChecksumRequired() const { return false; } + inline virtual bool RequestChecksumRequired() const { return false; } + + mutable Aws::RetryContext m_retryContext; protected: /** * Default does nothing. Override this to convert what would otherwise be the payload of the diff --git a/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h b/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h index a860642bf573..9e61b0f09861 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h @@ -84,42 +84,49 @@ class ChecksumInterceptor : public smithy::interceptor::Interceptor { // AwsAuthSigner decide it. if (request.IsStreaming() && checksumValueAndAlgorithmProvided) { addChecksumFeatureForChecksumName(checksumAlgorithmName, request); - const auto hash = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM, checksumHeader->second); - httpRequest->SetRequestHash(checksumAlgorithmName, hash); + if (httpRequest->GetRequestHash().second == nullptr) { + const auto hash = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM, checksumHeader->second); + httpRequest->SetRequestHash(checksumAlgorithmName, hash); + } } else if (checksumValueAndAlgorithmProvided) { httpRequest->SetHeaderValue(checksumType, checksumHeader->second); } else if (checksumAlgorithmName == "crc64nvme") { request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC64); if (request.IsStreaming()) { - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); + if (httpRequest->GetRequestHash().second == nullptr) + httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); } else { httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateCRC64(*(GetBodyStream(request))))); } } else if (checksumAlgorithmName == "crc32") { request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32); if (request.IsStreaming()) { - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); + if (httpRequest->GetRequestHash().second == nullptr) + httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); } else { httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateCRC32(*(GetBodyStream(request))))); } } else if (checksumAlgorithmName == "crc32c") { request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32C); if (request.IsStreaming()) { - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); + if (httpRequest->GetRequestHash().second == nullptr) + httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); } else { httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateCRC32C(*(GetBodyStream(request))))); } } else if (checksumAlgorithmName == "sha256") { request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA256); if (request.IsStreaming()) { - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); + if (httpRequest->GetRequestHash().second == nullptr) + httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); } else { httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateSHA256(*(GetBodyStream(request))))); } } else if (checksumAlgorithmName == "sha1") { request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA1); if (request.IsStreaming()) { - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); + if (httpRequest->GetRequestHash().second == nullptr) + httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); } else { httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateSHA1(*(GetBodyStream(request))))); } diff --git a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp index 80ece5c90515..14c6687c3ae3 100644 --- a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp +++ b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp @@ -389,6 +389,13 @@ HttpResponseOutcome AWSClient::AttemptExhaustively(const Aws::Http::URI& uri, { newUri.SetAuthority(newEndpoint); } + + // Save checksum information from the original request if we haven't already - safe to assume that the checksum has been finalized, since we have sent and received a response + if (request.m_retryContext.m_requestHash.second == nullptr) { + auto originalRequestHash = httpRequest->GetRequestHash(); + request.m_retryContext.m_requestHash = originalRequestHash; + } + httpRequest = CreateHttpRequest(newUri, method, request.GetResponseStreamFactory()); httpRequest->SetHeaderValue(Http::SDK_INVOCATION_ID_HEADER, invocationId); @@ -920,6 +927,12 @@ void AWSClient::BuildHttpRequest(const Aws::AmazonWebServiceRequest& request, co httpRequest->SetContinueRequestHandle(request.GetContinueRequestHandler()); httpRequest->SetServiceSpecificParameters(request.GetServiceSpecificParameters()); request.AddQueryStringParameters(httpRequest->GetUri()); + + // check for retry context, if present use it + if (request.m_retryContext.m_requestHash.second != nullptr) { + const auto hash = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM, HashingUtils::Base64Encode(request.m_retryContext.m_requestHash.second->GetHash().GetResult())); + httpRequest->SetRequestHash(request.m_retryContext.m_requestHash.first, hash); + } } Aws::String AWSClient::GeneratePresignedUrl(const Aws::Http::URI& uri, Aws::Http::HttpMethod method, long long expirationInSeconds, const std::shared_ptr serviceSpecificParameter) From 0313bed3cae8c2364dcb8ddc795f16ed743e180e Mon Sep 17 00:00:00 2001 From: sbaluja Date: Tue, 29 Jul 2025 15:16:55 -0400 Subject: [PATCH 06/10] Fix precalculated hash allocationtag namespace --- src/aws-cpp-sdk-core/source/client/AWSClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp index 14c6687c3ae3..efedac0054a5 100644 --- a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp +++ b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp @@ -930,7 +930,7 @@ void AWSClient::BuildHttpRequest(const Aws::AmazonWebServiceRequest& request, co // check for retry context, if present use it if (request.m_retryContext.m_requestHash.second != nullptr) { - const auto hash = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM, HashingUtils::Base64Encode(request.m_retryContext.m_requestHash.second->GetHash().GetResult())); + const auto hash = Aws::MakeShared(smithy::client::AWS_SMITHY_CLIENT_CHECKSUM, HashingUtils::Base64Encode(request.m_retryContext.m_requestHash.second->GetHash().GetResult())); httpRequest->SetRequestHash(request.m_retryContext.m_requestHash.first, hash); } } From ba83b78cbced24cdb7323bb947833dc4ae06499f Mon Sep 17 00:00:00 2001 From: sbaluja Date: Wed, 30 Jul 2025 14:07:11 -0400 Subject: [PATCH 07/10] ChecksumInterceptor.h - Refactor ModifyBeforeSigning logic for readability/scalability --- .../client/features/ChecksumInterceptor.h | 165 ++++++++++-------- 1 file changed, 92 insertions(+), 73 deletions(-) diff --git a/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h b/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h index 9e61b0f09861..eeb333c66ce9 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h @@ -82,56 +82,10 @@ class ChecksumInterceptor : public smithy::interceptor::Interceptor { // For non-streaming payload, the resolved checksum location is always header. // For streaming payload, the resolved checksum location depends on whether it is an unsigned payload, we let // AwsAuthSigner decide it. - if (request.IsStreaming() && checksumValueAndAlgorithmProvided) { - addChecksumFeatureForChecksumName(checksumAlgorithmName, request); - if (httpRequest->GetRequestHash().second == nullptr) { - const auto hash = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM, checksumHeader->second); - httpRequest->SetRequestHash(checksumAlgorithmName, hash); - } - } else if (checksumValueAndAlgorithmProvided) { - httpRequest->SetHeaderValue(checksumType, checksumHeader->second); - } else if (checksumAlgorithmName == "crc64nvme") { - request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC64); - if (request.IsStreaming()) { - if (httpRequest->GetRequestHash().second == nullptr) - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); - } else { - httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateCRC64(*(GetBodyStream(request))))); - } - } else if (checksumAlgorithmName == "crc32") { - request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32); - if (request.IsStreaming()) { - if (httpRequest->GetRequestHash().second == nullptr) - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); - } else { - httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateCRC32(*(GetBodyStream(request))))); - } - } else if (checksumAlgorithmName == "crc32c") { - request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32C); - if (request.IsStreaming()) { - if (httpRequest->GetRequestHash().second == nullptr) - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); - } else { - httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateCRC32C(*(GetBodyStream(request))))); - } - } else if (checksumAlgorithmName == "sha256") { - request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA256); - if (request.IsStreaming()) { - if (httpRequest->GetRequestHash().second == nullptr) - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); - } else { - httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateSHA256(*(GetBodyStream(request))))); - } - } else if (checksumAlgorithmName == "sha1") { - request.AddUserAgentFeature(Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA1); - if (request.IsStreaming()) { - if (httpRequest->GetRequestHash().second == nullptr) - httpRequest->SetRequestHash(checksumAlgorithmName, Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM)); - } else { - httpRequest->SetHeaderValue(checksumType, HashingUtils::Base64Encode(HashingUtils::CalculateSHA1(*(GetBodyStream(request))))); - } + if (checksumValueAndAlgorithmProvided) { + handleProvidedChecksum(request, httpRequest, checksumAlgorithmName, checksumType, checksumHeader->second); } else { - AWS_LOGSTREAM_WARN(AWS_SMITHY_CLIENT_CHECKSUM, "Checksum algorithm: " << checksumAlgorithmName << "is not supported by SDK."); + calculateAndSetChecksum(request, httpRequest, checksumAlgorithmName, checksumType); } } } @@ -140,30 +94,7 @@ class ChecksumInterceptor : public smithy::interceptor::Interceptor { if ((!request.GetResponseChecksumAlgorithmNames().empty() && m_responseChecksumValidation == ResponseChecksumValidation::WHEN_SUPPORTED) || request.ShouldValidateResponseChecksum()) { - for (const Aws::String& responseChecksumAlgorithmName : request.GetResponseChecksumAlgorithmNames()) { - const auto responseChecksum = Aws::Utils::StringUtils::ToLower(responseChecksumAlgorithmName.c_str()); - if (responseChecksum == "crc32c") { - std::shared_ptr crc32c = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); - httpRequest->AddResponseValidationHash("crc32c", crc32c); - } else if (responseChecksum == "crc32") { - std::shared_ptr crc32 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); - httpRequest->AddResponseValidationHash("crc32", crc32); - } else if (responseChecksum == "sha1") { - std::shared_ptr sha1 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); - httpRequest->AddResponseValidationHash("sha1", sha1); - } else if (responseChecksum == "sha256") { - std::shared_ptr sha256 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); - httpRequest->AddResponseValidationHash("sha256", sha256); - } else if (responseChecksum == "crc64nvme") { - std::shared_ptr crc64 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); - httpRequest->AddResponseValidationHash("crc64nvme", crc64); - } else { - AWS_LOGSTREAM_WARN(AWS_SMITHY_CLIENT_CHECKSUM, - "Checksum algorithm: " << responseChecksum << " is not supported in validating response body yet."); - } - } - // we have to set the checksum mode to enabled if it was not previously - httpRequest->SetHeaderValue("x-amz-checksum-mode", "enabled"); + SetResponseChecksum(request, httpRequest); } return httpRequest; @@ -240,6 +171,94 @@ class ChecksumInterceptor : public smithy::interceptor::Interceptor { } } + void handleProvidedChecksum(const Aws::AmazonWebServiceRequest& request, std::shared_ptr httpRequest, + const Aws::String& algorithm, const Aws::String& checksumType, const Aws::String& checksumValue) { + if (request.IsStreaming()) { + addChecksumFeatureForChecksumName(algorithm, request); + if (httpRequest->GetRequestHash().second == nullptr) { + auto hash = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM, checksumValue); + httpRequest->SetRequestHash(algorithm, hash); + } + } else { + httpRequest->SetHeaderValue(checksumType, checksumValue); + } + } + + void calculateAndSetChecksum(const Aws::AmazonWebServiceRequest& request, std::shared_ptr httpRequest, + const Aws::String& algorithm, const Aws::String& checksumType) { + static const Aws::UnorderedMap algorithmMap = { + {"crc64nvme", + {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateCRC64(stream)); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC64}}, + {"crc32", + {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateCRC32(stream)); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32}}, + {"crc32c", + {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateCRC32C(stream)); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32C}}, + {"sha256", + {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateSHA256(stream)); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA256}}, + {"sha1", + {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateSHA1(stream)); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA1}} + }; + + auto it = algorithmMap.find(algorithm); + if (it == algorithmMap.end()) { + AWS_LOGSTREAM_WARN(AWS_SMITHY_CLIENT_CHECKSUM, "Checksum algorithm: " << algorithm << " is not supported by SDK."); + return; + } + + request.AddUserAgentFeature(it->second.userAgentFeature); + + if (request.IsStreaming()) { + if (httpRequest->GetRequestHash().second == nullptr) { + httpRequest->SetRequestHash(algorithm, it->second.createHash()); + } + } else { + httpRequest->SetHeaderValue(checksumType, it->second.calculateHash(*GetBodyStream(request))); + } + } + + void SetResponseChecksum(const Aws::AmazonWebServiceRequest& request, std::shared_ptr httpRequest) { + for (const Aws::String& responseChecksumAlgorithmName : request.GetResponseChecksumAlgorithmNames()) { + const auto responseChecksum = Aws::Utils::StringUtils::ToLower(responseChecksumAlgorithmName.c_str()); + if (responseChecksum == "crc32c") { + std::shared_ptr crc32c = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); + httpRequest->AddResponseValidationHash("crc32c", crc32c); + } else if (responseChecksum == "crc32") { + std::shared_ptr crc32 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); + httpRequest->AddResponseValidationHash("crc32", crc32); + } else if (responseChecksum == "sha1") { + std::shared_ptr sha1 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); + httpRequest->AddResponseValidationHash("sha1", sha1); + } else if (responseChecksum == "sha256") { + std::shared_ptr sha256 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); + httpRequest->AddResponseValidationHash("sha256", sha256); + } else if (responseChecksum == "crc64nvme") { + std::shared_ptr crc64 = Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); + httpRequest->AddResponseValidationHash("crc64nvme", crc64); + } else { + AWS_LOGSTREAM_WARN(AWS_SMITHY_CLIENT_CHECKSUM, + "Checksum algorithm: " << responseChecksum << " is not supported in validating response body yet."); + } + } + // we have to set the checksum mode to enabled if it was not previously + httpRequest->SetHeaderValue("x-amz-checksum-mode", "enabled"); + } + + struct ChecksumHandler { + std::function()> createHash; + std::function calculateHash; + Aws::Client::UserAgentFeature userAgentFeature; + }; + RequestChecksumCalculation m_requestChecksumCalculation{RequestChecksumCalculation::WHEN_SUPPORTED}; ResponseChecksumValidation m_responseChecksumValidation{ResponseChecksumValidation::WHEN_SUPPORTED}; }; From f2dbd60494d63d3d024aaf485ab7178c4ea9dd68 Mon Sep 17 00:00:00 2001 From: sbaluja Date: Thu, 31 Jul 2025 10:31:09 -0400 Subject: [PATCH 08/10] ChecksumInterceptor.h - use memory safe Aws::Array instead of Aws::UnorderedMap --- .../client/features/ChecksumInterceptor.h | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h b/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h index eeb333c66ce9..627a16757b2a 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -186,30 +187,30 @@ class ChecksumInterceptor : public smithy::interceptor::Interceptor { void calculateAndSetChecksum(const Aws::AmazonWebServiceRequest& request, std::shared_ptr httpRequest, const Aws::String& algorithm, const Aws::String& checksumType) { - static const Aws::UnorderedMap algorithmMap = { - {"crc64nvme", - {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + static const Aws::Array, 5> algorithmMap = {{ + std::make_pair("crc64nvme", ChecksumHandler{ + []() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateCRC64(stream)); }, - Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC64}}, - {"crc32", - {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC64}), + std::make_pair("crc32", ChecksumHandler{ + []() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateCRC32(stream)); }, - Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32}}, - {"crc32c", - {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32}), + std::make_pair("crc32c", ChecksumHandler{ + []() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateCRC32C(stream)); }, - Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32C}}, - {"sha256", - {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_CRC32C}), + std::make_pair("sha256", ChecksumHandler{ + []() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateSHA256(stream)); }, - Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA256}}, - {"sha1", - {[]() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA256}), + std::make_pair("sha1", ChecksumHandler{ + []() { return Aws::MakeShared(AWS_SMITHY_CLIENT_CHECKSUM); }, [](Aws::IOStream& stream) { return HashingUtils::Base64Encode(HashingUtils::CalculateSHA1(stream)); }, - Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA1}} - }; + Aws::Client::UserAgentFeature::FLEXIBLE_CHECKSUMS_REQ_SHA1}) + }}; - auto it = algorithmMap.find(algorithm); + const auto it = find_if(algorithmMap.begin(), algorithmMap.end(), [&](const std::pair &pair) { return algorithm == pair.first; }); if (it == algorithmMap.end()) { AWS_LOGSTREAM_WARN(AWS_SMITHY_CLIENT_CHECKSUM, "Checksum algorithm: " << algorithm << " is not supported by SDK."); return; From 28a4eb94ec6086a7b7f52c48dcea8a323fde93ec Mon Sep 17 00:00:00 2001 From: sbaluja Date: Fri, 1 Aug 2025 10:35:16 -0400 Subject: [PATCH 09/10] AmazonWebServiceRequest - RetryContext to be private, and contain a pointer to the request hash pair --- .../include/aws/core/AmazonWebServiceRequest.h | 7 +++++-- src/aws-cpp-sdk-core/source/client/AWSClient.cpp | 15 ++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h b/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h index 097e152c3ea4..591b3df3ffbf 100644 --- a/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h +++ b/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h @@ -36,7 +36,7 @@ namespace Aws typedef std::function RequestSignedHandler; struct RetryContext { - std::pair> m_requestHash; + std::shared_ptr>> m_requestHash = nullptr; }; /** @@ -228,7 +228,9 @@ namespace Aws inline virtual bool RequestChecksumRequired() const { return false; } - mutable Aws::RetryContext m_retryContext; + RetryContext GetRetryContext() const { return m_retryContext; } + + void SetRetryContext(const RetryContext& context) const { m_retryContext = context; } protected: /** * Default does nothing. Override this to convert what would otherwise be the payload of the @@ -248,6 +250,7 @@ namespace Aws RequestRetryHandler m_requestRetryHandler; mutable std::shared_ptr m_serviceSpecificParameters; mutable Aws::Set m_userAgentFeatures; + mutable Aws::RetryContext m_retryContext; }; } // namespace Aws diff --git a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp index efedac0054a5..18bdfd60e474 100644 --- a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp +++ b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp @@ -391,9 +391,13 @@ HttpResponseOutcome AWSClient::AttemptExhaustively(const Aws::Http::URI& uri, } // Save checksum information from the original request if we haven't already - safe to assume that the checksum has been finalized, since we have sent and received a response - if (request.m_retryContext.m_requestHash.second == nullptr) { + RetryContext context = request.GetRetryContext(); + if (context.m_requestHash == nullptr) { auto originalRequestHash = httpRequest->GetRequestHash(); - request.m_retryContext.m_requestHash = originalRequestHash; + if (originalRequestHash.second != nullptr) { + context.m_requestHash = Aws::MakeShared>>(AWS_CLIENT_LOG_TAG, originalRequestHash); + request.SetRetryContext(context); + } } httpRequest = CreateHttpRequest(newUri, method, request.GetResponseStreamFactory()); @@ -929,9 +933,10 @@ void AWSClient::BuildHttpRequest(const Aws::AmazonWebServiceRequest& request, co request.AddQueryStringParameters(httpRequest->GetUri()); // check for retry context, if present use it - if (request.m_retryContext.m_requestHash.second != nullptr) { - const auto hash = Aws::MakeShared(smithy::client::AWS_SMITHY_CLIENT_CHECKSUM, HashingUtils::Base64Encode(request.m_retryContext.m_requestHash.second->GetHash().GetResult())); - httpRequest->SetRequestHash(request.m_retryContext.m_requestHash.first, hash); + RetryContext context = request.GetRetryContext(); + if (context.m_requestHash != nullptr) { + const auto hash = Aws::MakeShared(smithy::client::AWS_SMITHY_CLIENT_CHECKSUM, HashingUtils::Base64Encode(context.m_requestHash->second->GetHash().GetResult())); + httpRequest->SetRequestHash(context.m_requestHash->first, hash); } } From 1e8009da8d12fab7c5ad922b334e2eb6129786f3 Mon Sep 17 00:00:00 2001 From: sbaluja Date: Fri, 1 Aug 2025 10:43:48 -0400 Subject: [PATCH 10/10] Remove explicit nullptr initialization --- src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h b/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h index 591b3df3ffbf..a16c939a31cb 100644 --- a/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h +++ b/src/aws-cpp-sdk-core/include/aws/core/AmazonWebServiceRequest.h @@ -36,7 +36,7 @@ namespace Aws typedef std::function RequestSignedHandler; struct RetryContext { - std::shared_ptr>> m_requestHash = nullptr; + std::shared_ptr>> m_requestHash; }; /**