Skip to content

Commit 6289bf9

Browse files
sbisciglsbiscigl
committed
skip response validation when a composite checksum is used
1 parent 4635ead commit 6289bf9

File tree

6 files changed

+370
-2
lines changed

6 files changed

+370
-2
lines changed

‎src/aws-cpp-sdk-core/include/smithy/client/features/ChecksumInterceptor.h‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,9 +171,17 @@ class ChecksumInterceptor : public smithy::interceptor::Interceptor {
171171
}
172172
for (const auto& hashIterator : httpRequest->GetResponseValidationHashes()) {
173173
Aws::String checksumHeaderKey = Aws::String("x-amz-checksum-") + hashIterator.first;
174-
// TODO: If checksum ends with -#, then skip
175174
if (httpResponse->HasHeader(checksumHeaderKey.c_str())) {
176175
const Aws::String& checksumHeaderValue = httpResponse->GetHeader(checksumHeaderKey);
176+
// Handle composite checksum case where checksum has a trailing value indicating part size
177+
// i.e. c8Sk6w==-2 which denotes a 2 part composite checksum.
178+
const auto compositeChecksumIter = checksumHeaderValue.rfind('-');
179+
if (compositeChecksumIter != Aws::String::npos &&
180+
compositeChecksumIter + 1 < checksumHeaderValue.length() &&
181+
Aws::Utils::StringUtils::ConvertToInt32(checksumHeaderValue.substr(compositeChecksumIter + 1).c_str()) != 0) {
182+
AWS_LOGSTREAM_DEBUG(AWS_SMITHY_CLIENT_CHECKSUM, "Skipping checksum validation for composite checksum: " << checksumHeaderValue);
183+
break;
184+
}
177185
if (HashingUtils::Base64Encode(hashIterator.second->GetHash().GetResult()) != checksumHeaderValue) {
178186
auto error = Aws::Client::AWSError<Aws::Client::CoreErrors>{Aws::Client::CoreErrors::VALIDATION, "",
179187
"Response checksums mismatch", false /*retryable*/};

‎tests/aws-cpp-sdk-core-tests/smithy/client/feature/ChecksumInterceptorTest.cpp‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,3 +179,47 @@ TEST_F(ChecksumInterceptorTest, ChecksumInterceptorShouldValidateBadResponseChec
179179
EXPECT_EQ(Client::CoreErrors::VALIDATION, responseOutcome.GetError().GetErrorType());
180180
EXPECT_EQ("Response checksums mismatch", responseOutcome.GetError().GetMessage());
181181
}
182+
183+
TEST_F(ChecksumInterceptorTest, ChecksumInterceptorShouldSkipCompositeChecksum) {
184+
Aws::Vector<Aws::String> responseValidationChecksumsToValidate{"crc32"};
185+
MockChecksumRequest modeledRequest{"beached things", "crc32", true, true, responseValidationChecksumsToValidate};
186+
InterceptorContext context{modeledRequest};
187+
URI uri{"https://www.timefall.com/bts"};
188+
std::shared_ptr<HttpRequest> request(CreateHttpRequest(uri, HttpMethod::HTTP_GET, Utils::Stream::DefaultResponseStreamFactoryMethod));
189+
context.SetTransmitRequest(request);
190+
const auto requestOutcome = m_interceptor.ModifyBeforeSigning(context);
191+
auto responseHashes = request->GetResponseValidationHashes();
192+
EXPECT_EQ(1ul, responseHashes.size());
193+
EXPECT_EQ("crc32", responseHashes[0].first);
194+
EXPECT_NE(nullptr, responseHashes[0].second);
195+
auto bodyStr = Crt::ByteBufFromCString("beached things");
196+
responseHashes[0].second->Update(bodyStr.buffer, bodyStr.len);
197+
EXPECT_TRUE(requestOutcome.IsSuccess());
198+
std::shared_ptr<HttpResponse> response = Aws::MakeShared<StandardHttpResponse>(ALLOC_TAG, request);
199+
response->AddHeader("x-amz-checksum-crc32", "bb28==-1");
200+
context.SetTransmitResponse(response);
201+
const auto responseOutcome = m_interceptor.ModifyBeforeDeserialization(context);
202+
EXPECT_TRUE(responseOutcome.IsSuccess());
203+
}
204+
205+
TEST_F(ChecksumInterceptorTest, ChecksumInterceptorShouldFailForNonNumericCompositeTrailer) {
206+
Aws::Vector<Aws::String> responseValidationChecksumsToValidate{"crc32"};
207+
MockChecksumRequest modeledRequest{"beached things", "crc32", true, true, responseValidationChecksumsToValidate};
208+
InterceptorContext context{modeledRequest};
209+
URI uri{"https://www.timefall.com/bts"};
210+
std::shared_ptr<HttpRequest> request(CreateHttpRequest(uri, HttpMethod::HTTP_GET, Utils::Stream::DefaultResponseStreamFactoryMethod));
211+
context.SetTransmitRequest(request);
212+
const auto requestOutcome = m_interceptor.ModifyBeforeSigning(context);
213+
auto responseHashes = request->GetResponseValidationHashes();
214+
EXPECT_EQ(1ul, responseHashes.size());
215+
EXPECT_EQ("crc32", responseHashes[0].first);
216+
EXPECT_NE(nullptr, responseHashes[0].second);
217+
auto bodyStr = Crt::ByteBufFromCString("beached things");
218+
responseHashes[0].second->Update(bodyStr.buffer, bodyStr.len);
219+
EXPECT_TRUE(requestOutcome.IsSuccess());
220+
std::shared_ptr<HttpResponse> response = Aws::MakeShared<StandardHttpResponse>(ALLOC_TAG, request);
221+
response->AddHeader("x-amz-checksum-crc32", "bb28==-ABC");
222+
context.SetTransmitResponse(response);
223+
const auto responseOutcome = m_interceptor.ModifyBeforeDeserialization(context);
224+
EXPECT_FALSE(responseOutcome.IsSuccess());
225+
}

‎tests/aws-cpp-sdk-s3-crt-integration-tests/BucketAndObjectOperationTest.cpp‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ namespace
7878
static std::string BASE_PUT_WEIRD_CHARSETS_OBJECTS_BUCKET_NAME = "charsetstest";
7979
static std::string BASE_PUT_OBJECTS_PRESIGNED_URLS_BUCKET_NAME = "presignedtest";
8080
static std::string BASE_PUT_MULTIPART_BUCKET_NAME = "multiparttest";
81+
static std::string BASE_PUT_MULTIPART_COMPOSITE_CHECKSUM_BUCKET_NAME = "multiparttest";
8182
static std::string BASE_ERRORS_TESTING_BUCKET = "errorstest";
8283
static std::string BASE_EVENT_STREAM_TEST_BUCKET_NAME = "eventstream";
8384
static std::string BASE_EVENT_STREAM_LARGE_FILE_TEST_BUCKET_NAME = "largeeventstream";
@@ -117,6 +118,7 @@ namespace
117118
std::ref(BASE_PUT_WEIRD_CHARSETS_OBJECTS_BUCKET_NAME),
118119
std::ref(BASE_PUT_OBJECTS_PRESIGNED_URLS_BUCKET_NAME),
119120
std::ref(BASE_PUT_MULTIPART_BUCKET_NAME),
121+
std::ref(BASE_PUT_MULTIPART_COMPOSITE_CHECKSUM_BUCKET_NAME),
120122
std::ref(BASE_ERRORS_TESTING_BUCKET),
121123
std::ref(BASE_EVENT_STREAM_TEST_BUCKET_NAME),
122124
std::ref(BASE_EVENT_STREAM_LARGE_FILE_TEST_BUCKET_NAME),
@@ -162,6 +164,7 @@ namespace
162164
DeleteBucket(CalculateBucketName(BASE_PUT_OBJECTS_BUCKET_NAME.c_str()));
163165
DeleteBucket(CalculateBucketName(BASE_PUT_OBJECTS_PRESIGNED_URLS_BUCKET_NAME.c_str()));
164166
DeleteBucket(CalculateBucketName(BASE_PUT_MULTIPART_BUCKET_NAME.c_str()));
167+
DeleteBucket(CalculateBucketName(BASE_PUT_MULTIPART_COMPOSITE_CHECKSUM_BUCKET_NAME.c_str()));
165168
DeleteBucket(CalculateBucketName(BASE_ERRORS_TESTING_BUCKET.c_str()));
166169
DeleteBucket(CalculateBucketName(BASE_PUT_WEIRD_CHARSETS_OBJECTS_BUCKET_NAME.c_str()));
167170
DeleteBucket(CalculateBucketName(BASE_EVENT_STREAM_TEST_BUCKET_NAME.c_str()));
@@ -1693,4 +1696,74 @@ namespace
16931696

16941697
Aws::Monitoring::CleanupMonitoring();
16951698
}
1699+
1700+
TEST_F(BucketAndObjectOperationTest, ShouldSkipResponseValidationOnCompositeChecksums) {
1701+
const auto fullBucketName = CalculateBucketName(BASE_PUT_MULTIPART_COMPOSITE_CHECKSUM_BUCKET_NAME.c_str());
1702+
SCOPED_TRACE(Aws::String("FullBucketName ") + fullBucketName);
1703+
CreateBucketRequest createBucketRequest;
1704+
createBucketRequest.SetBucket(fullBucketName);
1705+
createBucketRequest.SetACL(BucketCannedACL::private_);
1706+
1707+
CreateBucketOutcome createBucketOutcome = Client->CreateBucket(createBucketRequest);
1708+
AWS_ASSERT_SUCCESS(createBucketOutcome);
1709+
const CreateBucketResult& createBucketResult = createBucketOutcome.GetResult();
1710+
ASSERT_TRUE(!createBucketResult.GetLocation().empty());
1711+
ASSERT_TRUE(WaitForBucketToPropagate(fullBucketName, Client));
1712+
TagTestBucket(fullBucketName, Client);
1713+
1714+
const Aws::String objectKey{"test-composite-checksum"};
1715+
1716+
const auto createMPUResponse = Client->CreateMultipartUpload(CreateMultipartUploadRequest{}
1717+
.WithBucket(fullBucketName)
1718+
.WithKey(objectKey)
1719+
.WithChecksumType(ChecksumType::COMPOSITE)
1720+
.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32));
1721+
AWS_EXPECT_SUCCESS(createMPUResponse);
1722+
1723+
auto uploadPartOneRequest = UploadPartRequest{}
1724+
.WithBucket(fullBucketName)
1725+
.WithKey(objectKey)
1726+
.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32)
1727+
.WithUploadId(createMPUResponse.GetResult().GetUploadId())
1728+
.WithPartNumber(1);
1729+
1730+
uploadPartOneRequest.SetBody(CreateStreamForUploadPart(5, "Hello from part 1"));
1731+
1732+
const auto partOneUploadResponse = Client->UploadPart(uploadPartOneRequest);
1733+
AWS_EXPECT_SUCCESS(partOneUploadResponse);
1734+
1735+
auto uploadPartTwoRequest = UploadPartRequest{}
1736+
.WithBucket(fullBucketName)
1737+
.WithKey(objectKey)
1738+
.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32)
1739+
.WithUploadId(createMPUResponse.GetResult().GetUploadId())
1740+
.WithPartNumber(2);
1741+
1742+
uploadPartTwoRequest.SetBody(CreateStreamForUploadPart(5, "Hello from part 2"));
1743+
1744+
const auto partTwoUploadResponse = Client->UploadPart(uploadPartTwoRequest);
1745+
AWS_EXPECT_SUCCESS(partTwoUploadResponse);
1746+
1747+
1748+
const auto completeMpuRequest = Client->CompleteMultipartUpload(CompleteMultipartUploadRequest{}
1749+
.WithBucket(fullBucketName)
1750+
.WithKey(objectKey)
1751+
.WithUploadId(createMPUResponse.GetResult().GetUploadId())
1752+
.WithChecksumType(ChecksumType::COMPOSITE)
1753+
.WithMultipartUpload(CompletedMultipartUpload{}.WithParts({
1754+
CompletedPart{}.WithPartNumber(1)
1755+
.WithETag(partOneUploadResponse.GetResult().GetETag())
1756+
.WithChecksumCRC32(partOneUploadResponse.GetResult().GetChecksumCRC32()),
1757+
CompletedPart{}
1758+
.WithPartNumber(2)
1759+
.WithETag(partTwoUploadResponse.GetResult().GetETag())
1760+
.WithChecksumCRC32(partTwoUploadResponse.GetResult().GetChecksumCRC32())
1761+
})));
1762+
AWS_EXPECT_SUCCESS(completeMpuRequest);
1763+
1764+
const auto getObjectResponse = Client->GetObject(GetObjectRequest{}
1765+
.WithBucket(fullBucketName)
1766+
.WithKey(objectKey));
1767+
AWS_EXPECT_SUCCESS(getObjectResponse);
1768+
}
16961769
}

‎tests/aws-cpp-sdk-s3-crt-integration-tests/S3ExpressTest.cpp‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,30 @@ namespace {
297297
}
298298
}
299299

300+
static std::shared_ptr<Aws::StringStream> Create5MbStreamForUploadPart(const char *partTag) {
301+
uint32_t fiveMbSize = 5 * 1024 * 1024;
302+
303+
Aws::StringStream patternStream;
304+
patternStream << "Multi-Part upload Test Part " << partTag << ":" << std::endl;
305+
Aws::String pattern = patternStream.str();
306+
307+
Aws::String scratchString;
308+
scratchString.reserve(fiveMbSize);
309+
310+
// 5MB is a hard minimum for multi part uploads; make sure the final string is at least that long
311+
uint32_t patternCopyCount = static_cast<uint32_t>(fiveMbSize / pattern.size() + 1);
312+
for (uint32_t i = 0; i < patternCopyCount; ++i) {
313+
scratchString.append(pattern);
314+
}
315+
316+
std::shared_ptr<Aws::StringStream> streamPtr = Aws::MakeShared<Aws::StringStream>(ALLOCATION_TAG, scratchString);
317+
318+
streamPtr->seekg(0);
319+
streamPtr->seekp(0, std::ios_base::end);
320+
321+
return streamPtr;
322+
}
323+
300324
protected:
301325
void SetUp() override {
302326
S3Crt::ClientConfiguration configuration;
@@ -517,4 +541,65 @@ namespace {
517541
const auto response = client->PutObject(request);
518542
AWS_EXPECT_SUCCESS(response);
519543
}
544+
545+
TEST_F(S3ExpressTest, ShouldSkipResponseValidationOnCompositeChecksums) {
546+
const auto bucketName = Testing::GetAwsResourcePrefix() + randomString() + S3_EXPRESS_SUFFIX;
547+
const auto createOutcome = CreateBucket(bucketName);
548+
AWS_EXPECT_SUCCESS(createOutcome);
549+
550+
const Aws::String objectKey{"test-composite-checksum"};
551+
552+
const auto createMPUResponse = client->CreateMultipartUpload(CreateMultipartUploadRequest{}
553+
.WithBucket(bucketName)
554+
.WithKey(objectKey)
555+
.WithChecksumType(ChecksumType::COMPOSITE)
556+
.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32));
557+
AWS_EXPECT_SUCCESS(createMPUResponse);
558+
559+
auto uploadPartOneRequest = UploadPartRequest{}
560+
.WithBucket(bucketName)
561+
.WithKey(objectKey)
562+
.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32)
563+
.WithUploadId(createMPUResponse.GetResult().GetUploadId())
564+
.WithPartNumber(1);
565+
566+
uploadPartOneRequest.SetBody(Create5MbStreamForUploadPart("Hello from part 1"));
567+
568+
const auto partOneUploadResponse = client->UploadPart(uploadPartOneRequest);
569+
AWS_EXPECT_SUCCESS(partOneUploadResponse);
570+
571+
auto uploadPartTwoRequest = UploadPartRequest{}
572+
.WithBucket(bucketName)
573+
.WithKey(objectKey)
574+
.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32)
575+
.WithUploadId(createMPUResponse.GetResult().GetUploadId())
576+
.WithPartNumber(2);
577+
578+
uploadPartTwoRequest.SetBody(Create5MbStreamForUploadPart("Hello from part 2"));
579+
580+
const auto partTwoUploadResponse = client->UploadPart(uploadPartTwoRequest);
581+
AWS_EXPECT_SUCCESS(partTwoUploadResponse);
582+
583+
584+
const auto completeMpuRequest = client->CompleteMultipartUpload(CompleteMultipartUploadRequest{}
585+
.WithBucket(bucketName)
586+
.WithKey(objectKey)
587+
.WithUploadId(createMPUResponse.GetResult().GetUploadId())
588+
.WithChecksumType(ChecksumType::COMPOSITE)
589+
.WithMultipartUpload(CompletedMultipartUpload{}.WithParts({
590+
CompletedPart{}.WithPartNumber(1)
591+
.WithETag(partOneUploadResponse.GetResult().GetETag())
592+
.WithChecksumCRC32(partOneUploadResponse.GetResult().GetChecksumCRC32()),
593+
CompletedPart{}
594+
.WithPartNumber(2)
595+
.WithETag(partTwoUploadResponse.GetResult().GetETag())
596+
.WithChecksumCRC32(partTwoUploadResponse.GetResult().GetChecksumCRC32())
597+
})));
598+
AWS_EXPECT_SUCCESS(completeMpuRequest);
599+
600+
const auto getObjectResponse = client->GetObject(GetObjectRequest{}
601+
.WithBucket(bucketName)
602+
.WithKey(objectKey));
603+
AWS_EXPECT_SUCCESS(getObjectResponse);
604+
}
520605
}

‎tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp‎

Lines changed: 74 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ namespace
8585
static std::string BASE_PUT_WEIRD_CHARSETS_OBJECTS_BUCKET_NAME = "charsetstest";
8686
static std::string BASE_PUT_OBJECTS_PRESIGNED_URLS_BUCKET_NAME = "presignedtest";
8787
static std::string BASE_PUT_MULTIPART_BUCKET_NAME = "multiparttest";
88+
static std::string BASE_PUT_MULTIPART_COMPOSITE_CHECKSUM_BUCKET_NAME = "multiparttest";
8889
static std::string BASE_OBJECT_LOCK_BUCKET_NAME = "objectlock";
8990
static std::string BASE_ERRORS_TESTING_BUCKET = "errorstest";
9091
static std::string BASE_INTERRUPT_TESTING_BUCKET = "interrupttest";
@@ -131,6 +132,7 @@ namespace
131132
std::ref(BASE_PUT_WEIRD_CHARSETS_OBJECTS_BUCKET_NAME),
132133
std::ref(BASE_PUT_OBJECTS_PRESIGNED_URLS_BUCKET_NAME),
133134
std::ref(BASE_PUT_MULTIPART_BUCKET_NAME),
135+
std::ref(BASE_PUT_MULTIPART_COMPOSITE_CHECKSUM_BUCKET_NAME),
134136
std::ref(BASE_OBJECT_LOCK_BUCKET_NAME),
135137
std::ref(BASE_ERRORS_TESTING_BUCKET),
136138
std::ref(BASE_INTERRUPT_TESTING_BUCKET),
@@ -2642,4 +2644,75 @@ namespace
26422644
client->GetObject(request);
26432645
EXPECT_FALSE(outcome.IsSuccess());
26442646
}
2645-
}
2647+
2648+
TEST_F(BucketAndObjectOperationTest, ShouldSkipResponseValidationOnCompositeChecksums) {
2649+
const auto fullBucketName = CalculateBucketName(BASE_PUT_MULTIPART_COMPOSITE_CHECKSUM_BUCKET_NAME.c_str());
2650+
m_bucketsToDelete.insert(fullBucketName);
2651+
SCOPED_TRACE(Aws::String("FullBucketName ") + fullBucketName);
2652+
CreateBucketRequest createBucketRequest;
2653+
createBucketRequest.SetBucket(fullBucketName);
2654+
createBucketRequest.SetACL(BucketCannedACL::private_);
2655+
2656+
CreateBucketOutcome createBucketOutcome = CreateBucket(createBucketRequest);
2657+
AWS_ASSERT_SUCCESS(createBucketOutcome);
2658+
const CreateBucketResult& createBucketResult = createBucketOutcome.GetResult();
2659+
ASSERT_TRUE(!createBucketResult.GetLocation().empty());
2660+
ASSERT_TRUE(WaitForBucketToPropagate(fullBucketName, Client));
2661+
TagTestBucket(fullBucketName, Client);
2662+
2663+
const Aws::String objectKey{"test-composite-checksum"};
2664+
2665+
const auto createMPUResponse = Client->CreateMultipartUpload(CreateMultipartUploadRequest{}
2666+
.WithBucket(fullBucketName)
2667+
.WithKey(objectKey)
2668+
.WithChecksumType(ChecksumType::COMPOSITE)
2669+
.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32));
2670+
AWS_EXPECT_SUCCESS(createMPUResponse);
2671+
2672+
auto uploadPartOneRequest = UploadPartRequest{}
2673+
.WithBucket(fullBucketName)
2674+
.WithKey(objectKey)
2675+
.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32)
2676+
.WithUploadId(createMPUResponse.GetResult().GetUploadId())
2677+
.WithPartNumber(1);
2678+
2679+
uploadPartOneRequest.SetBody(Create5MbStreamForUploadPart("Hello from part 1"));
2680+
2681+
const auto partOneUploadResponse = Client->UploadPart(uploadPartOneRequest);
2682+
AWS_EXPECT_SUCCESS(partOneUploadResponse);
2683+
2684+
auto uploadPartTwoRequest = UploadPartRequest{}
2685+
.WithBucket(fullBucketName)
2686+
.WithKey(objectKey)
2687+
.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32)
2688+
.WithUploadId(createMPUResponse.GetResult().GetUploadId())
2689+
.WithPartNumber(2);
2690+
2691+
uploadPartTwoRequest.SetBody(Create5MbStreamForUploadPart("Hello from part 2"));
2692+
2693+
const auto partTwoUploadResponse = Client->UploadPart(uploadPartTwoRequest);
2694+
AWS_EXPECT_SUCCESS(partTwoUploadResponse);
2695+
2696+
2697+
const auto completeMpuRequest = Client->CompleteMultipartUpload(CompleteMultipartUploadRequest{}
2698+
.WithBucket(fullBucketName)
2699+
.WithKey(objectKey)
2700+
.WithUploadId(createMPUResponse.GetResult().GetUploadId())
2701+
.WithChecksumType(ChecksumType::COMPOSITE)
2702+
.WithMultipartUpload(CompletedMultipartUpload{}.WithParts({
2703+
CompletedPart{}.WithPartNumber(1)
2704+
.WithETag(partOneUploadResponse.GetResult().GetETag())
2705+
.WithChecksumCRC32(partOneUploadResponse.GetResult().GetChecksumCRC32()),
2706+
CompletedPart{}
2707+
.WithPartNumber(2)
2708+
.WithETag(partTwoUploadResponse.GetResult().GetETag())
2709+
.WithChecksumCRC32(partTwoUploadResponse.GetResult().GetChecksumCRC32())
2710+
})));
2711+
AWS_EXPECT_SUCCESS(completeMpuRequest);
2712+
2713+
const auto getObjectResponse = Client->GetObject(GetObjectRequest{}
2714+
.WithBucket(fullBucketName)
2715+
.WithKey(objectKey));
2716+
AWS_EXPECT_SUCCESS(getObjectResponse);
2717+
}
2718+
}

0 commit comments

Comments
 (0)