Skip to content

Commit 7c83ec3

Browse files
kai-ionkai-ion
authored
Merge branch 'main' into PR3663
2 parents 5c8d290 + 0d96e78 commit 7c83ec3

File tree

6 files changed

+257
-9
lines changed

6 files changed

+257
-9
lines changed

‎.cspell.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,7 @@
261261
"ossl",
262262
"ccrng",
263263
"KEYWRAP",
264+
"HKDF",
264265
"NVME",
265266
// EC2
266267
"IMDS",

‎src/aws-cpp-sdk-transfer/include/aws/transfer/TransferHandle.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include <aws/core/client/AWSError.h>
1414
#include <aws/core/client/AsyncCallerContext.h>
1515
#include <aws/s3/S3Errors.h>
16+
#include <aws/s3/model/ChecksumAlgorithm.h>
1617
#include <iostream>
1718
#include <atomic>
1819
#include <mutex>
@@ -372,6 +373,7 @@ namespace Aws
372373
* Return empty string on success, string with error message on error.
373374
*/
374375
Aws::String WritePartToDownloadStream(Aws::IOStream* partStream, uint64_t writeOffset);
376+
void AddChecksumForPart(Aws:: IOStream* partStream, const PartPointer& shared);
375377

376378
void ApplyDownloadConfiguration(const DownloadConfiguration& downloadConfig);
377379

@@ -389,6 +391,9 @@ namespace Aws
389391
Aws::String GetChecksum() const { return m_checksum; }
390392
void SetChecksum(const Aws::String& checksum) { this->m_checksum = checksum; }
391393

394+
Aws::S3::Model::ChecksumAlgorithm GetChecksumAlgorithm() const { std::lock_guard<std::mutex> locker(m_getterSetterLock); return m_checksumAlgorithm; }
395+
void SetChecksumAlgorithm (const Aws::S3::Model::ChecksumAlgorithm& checksumAlgorithm) { std::lock_guard<std::mutex> locker(m_getterSetterLock); m_checksumAlgorithm = checksumAlgorithm; }
396+
392397
private:
393398
void CleanupDownloadStream();
394399

@@ -430,6 +435,7 @@ namespace Aws
430435
mutable std::condition_variable m_waitUntilFinishedSignal;
431436
mutable std::mutex m_getterSetterLock;
432437
Aws::String m_checksum;
438+
Aws::S3::Model::ChecksumAlgorithm m_checksumAlgorithm;
433439
};
434440

435441
AWS_TRANSFER_API Aws::OStream& operator << (Aws::OStream& s, TransferStatus status);

‎src/aws-cpp-sdk-transfer/include/aws/transfer/TransferManager.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,13 @@ namespace Aws
144144
* upload. Defaults to CRC64-NVME.
145145
*/
146146
Aws::S3::Model::ChecksumAlgorithm checksumAlgorithm = S3::Model::ChecksumAlgorithm::CRC64NVME;
147+
148+
/**
149+
* Enable checksum validation for downloads. When enabled, checksums will be
150+
* calculated during download and validated against S3 response headers.
151+
* Defaults to true.
152+
*/
153+
bool validateChecksums = true;
147154
};
148155

149156
/**

‎src/aws-cpp-sdk-transfer/source/transfer/TransferHandle.cpp‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@
66
#include <aws/transfer/TransferHandle.h>
77
#include <aws/core/utils/logging/LogMacros.h>
88
#include <aws/core/utils/memory/stl/SimpleStringStream.h>
9+
#include <aws/core/utils/crypto/CRC32.h>
10+
#include <aws/core/utils/crypto/CRC64.h>
11+
#include <aws/core/utils/crypto/Sha1.h>
12+
#include <aws/core/utils/crypto/Sha256.h>
13+
#include <aws/core/utils/crypto/Sha256HMAC.h>
14+
#include "aws/core/utils/HashingUtils.h"
915

1016
#include <cassert>
1117

@@ -370,6 +376,11 @@ namespace Aws
370376
AWS_LOGSTREAM_TRACE(CLASS_TAG, "Transfer handle ID [" << GetId() << "] Restarting transfer.");
371377
m_cancel.store(false);
372378
m_lastPart.store(false);
379+
380+
// Clear checksum state for retry
381+
std::lock_guard<std::mutex> locker(m_getterSetterLock);
382+
m_checksum.clear();
383+
m_checksumAlgorithm = Aws::S3::Model::ChecksumAlgorithm::NOT_SET;
373384
}
374385

375386
bool TransferHandle::ShouldContinue() const
@@ -423,6 +434,25 @@ namespace Aws
423434
return "";
424435
}
425436

437+
void TransferHandle::AddChecksumForPart(Aws::IOStream *partStream, const PartPointer& partState) {
438+
partStream->seekg(0);
439+
Aws::String checksum = "";
440+
if (GetChecksumAlgorithm()==S3::Model::ChecksumAlgorithm::CRC32) {
441+
checksum = Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::Crypto::CRC32().Calculate(*partStream).GetResult());
442+
} else if (GetChecksumAlgorithm()==S3::Model::ChecksumAlgorithm::CRC32C) {
443+
checksum = Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::Crypto::CRC32C().Calculate(*partStream).GetResult());
444+
} else if (GetChecksumAlgorithm()==S3::Model::ChecksumAlgorithm::CRC64NVME) {
445+
checksum = Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::Crypto::CRC64().Calculate(*partStream).GetResult());
446+
} else if (GetChecksumAlgorithm()==S3::Model::ChecksumAlgorithm::SHA1) {
447+
checksum = Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::Crypto::Sha1().Calculate(*partStream).GetResult());
448+
} else if (GetChecksumAlgorithm()==S3::Model::ChecksumAlgorithm::SHA256) {
449+
checksum = Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::Crypto::Sha256().Calculate(*partStream).GetResult());
450+
}
451+
partState->SetChecksum(checksum);
452+
partStream->clear();
453+
partStream->seekg(0);
454+
}
455+
426456
void TransferHandle::ApplyDownloadConfiguration(const DownloadConfiguration& downloadConfig)
427457
{
428458
SetVersionId(downloadConfig.versionId);

‎src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp‎

Lines changed: 88 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
#include <aws/core/utils/memory/stl/AWSStreamFwd.h>
1616
#include <aws/core/utils/memory/stl/AWSStringStream.h>
1717
#include <aws/core/utils/stream/PreallocatedStreamBuf.h>
18+
#include <aws/common/byte_order.h>
19+
#include <cstring>
20+
#include <aws/crt/checksum/CRC.h>
1821
#include <aws/s3/S3Client.h>
1922
#include <aws/s3/model/AbortMultipartUploadRequest.h>
2023
#include <aws/s3/model/CompleteMultipartUploadRequest.h>
@@ -406,12 +409,9 @@ namespace Aws
406409

407410
const auto fullObjectHashCalculator = [](const std::shared_ptr<TransferHandle>& handle, bool isRetry, S3::Model::ChecksumAlgorithm algorithm) -> std::shared_ptr<Aws::Utils::Crypto::Hash> {
408411
if (handle->GetChecksum().empty() && !isRetry) {
409-
if (algorithm == S3::Model::ChecksumAlgorithm::CRC32) {
412+
if (algorithm == S3::Model::ChecksumAlgorithm::CRC32 || algorithm == S3::Model::ChecksumAlgorithm::CRC32C) {
410413
return Aws::MakeShared<Aws::Utils::Crypto::CRC32>("TransferManager");
411414
}
412-
if (algorithm == S3::Model::ChecksumAlgorithm::CRC32C) {
413-
return Aws::MakeShared<Aws::Utils::Crypto::CRC32C>("TransferManager");
414-
}
415415
if (algorithm == S3::Model::ChecksumAlgorithm::SHA1) {
416416
return Aws::MakeShared<Aws::Utils::Crypto::Sha1>("TransferManager");
417417
}
@@ -673,6 +673,10 @@ namespace Aws
673673
{
674674
return outcome.GetResult().GetChecksumCRC32C();
675675
}
676+
else if (m_transferConfig.checksumAlgorithm == S3::Model::ChecksumAlgorithm::CRC64NVME)
677+
{
678+
return outcome.GetResult().GetChecksumCRC64NVME();
679+
}
676680
else if (m_transferConfig.checksumAlgorithm == S3::Model::ChecksumAlgorithm::SHA1)
677681
{
678682
return outcome.GetResult().GetChecksumSHA1();
@@ -965,6 +969,7 @@ namespace Aws
965969
headObjectRequest.SetCustomizedAccessLogTag(m_transferConfig.customizedAccessLogTag);
966970
headObjectRequest.WithBucket(handle->GetBucketName())
967971
.WithKey(handle->GetKey());
972+
headObjectRequest.SetChecksumMode(Aws::S3::Model::ChecksumMode::ENABLED);
968973

969974
if(!handle->GetVersionId().empty())
970975
{
@@ -1000,6 +1005,18 @@ namespace Aws
10001005
handle->SetContentType(headObjectOutcome.GetResult().GetContentType());
10011006
handle->SetMetadata(headObjectOutcome.GetResult().GetMetadata());
10021007
handle->SetEtag(headObjectOutcome.GetResult().GetETag());
1008+
if (headObjectOutcome.GetResult().GetChecksumType() == Aws::S3::Model::ChecksumType::FULL_OBJECT) {
1009+
if (!headObjectOutcome.GetResult().GetChecksumCRC32C().empty()) {
1010+
handle->SetChecksum(headObjectOutcome.GetResult().GetChecksumCRC32C());
1011+
handle->SetChecksumAlgorithm(S3::Model::ChecksumAlgorithm::CRC32C);
1012+
} else if (!headObjectOutcome.GetResult().GetChecksumCRC32().empty()) {
1013+
handle->SetChecksum(headObjectOutcome.GetResult().GetChecksumCRC32());
1014+
handle->SetChecksumAlgorithm(S3::Model::ChecksumAlgorithm::CRC32);
1015+
} else if (!headObjectOutcome.GetResult().GetChecksumCRC64NVME().empty()) {
1016+
handle->SetChecksum(headObjectOutcome.GetResult().GetChecksumCRC64NVME());
1017+
handle->SetChecksumAlgorithm(S3::Model::ChecksumAlgorithm::CRC64NVME);
1018+
}
1019+
}
10031020
/* When bucket versioning is suspended, head object will return "null" for unversioned object.
10041021
* Send following GetObject with "null" as versionId will result in 403 access denied error if your IAM role or policy
10051022
* doesn't have GetObjectVersion permission.
@@ -1204,6 +1221,7 @@ namespace Aws
12041221

12051222
Aws::String errMsg{handle->WritePartToDownloadStream(bufferStream, partState->GetRangeBegin())};
12061223
if (errMsg.empty()) {
1224+
if (m_transferConfig.validateChecksums) { handle->AddChecksumForPart(bufferStream, partState); }
12071225
handle->ChangePartToCompleted(partState, outcome.GetResult().GetETag());
12081226
} else {
12091227
Aws::Client::AWSError<Aws::S3::S3Errors> error(Aws::S3::S3Errors::INTERNAL_FAILURE,
@@ -1239,6 +1257,72 @@ namespace Aws
12391257
{
12401258
if (failedParts.size() == 0 && handle->GetBytesTransferred() == handle->GetBytesTotalSize())
12411259
{
1260+
if (m_transferConfig.validateChecksums && !handle->GetChecksum().empty() &&
1261+
(handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC32 ||
1262+
handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC32C ||
1263+
handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC64NVME)) {
1264+
uint64_t combinedChecksum = 0;
1265+
bool first = true;
1266+
for (const auto& part: handle->GetCompletedParts()) {
1267+
Aws::String checksumStr = part.second->GetChecksum();
1268+
uint64_t partSize = part.second->GetSizeInBytes();
1269+
if (checksumStr.empty()) { continue; }
1270+
auto decoded = Aws::Utils::HashingUtils::Base64Decode(checksumStr);
1271+
const auto* raw = decoded.GetUnderlyingData();
1272+
if (first) {
1273+
if (handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC64NVME) {
1274+
uint64_t partCrcBE = 0;
1275+
std::memcpy(&partCrcBE, raw, sizeof(uint64_t));
1276+
const uint64_t partCrc = aws_ntoh64(partCrcBE);
1277+
combinedChecksum = partCrc;
1278+
} else {
1279+
uint32_t partCrcBE = 0;
1280+
std::memcpy(&partCrcBE, raw, sizeof(uint32_t));
1281+
const uint32_t partCrc = aws_ntoh32(partCrcBE);
1282+
combinedChecksum = partCrc;
1283+
}
1284+
first = false;
1285+
} else {
1286+
if (handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC64NVME) {
1287+
uint64_t partCrcBE = 0;
1288+
std::memcpy(&partCrcBE, raw, sizeof(uint64_t));
1289+
const uint64_t partCrc = aws_ntoh64(partCrcBE);
1290+
combinedChecksum = Aws::Crt::Checksum::CombineCRC64NVME(combinedChecksum, partCrc, partSize);
1291+
}
1292+
else {
1293+
uint32_t partCrcBE = 0;
1294+
std::memcpy(&partCrcBE, raw, sizeof(uint32_t));
1295+
const uint32_t partCrc = aws_ntoh32(partCrcBE);
1296+
if (handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC32) {
1297+
combinedChecksum = Aws::Crt::Checksum::CombineCRC32(static_cast<uint32_t>(combinedChecksum), partCrc, partSize);
1298+
} else {
1299+
combinedChecksum = Aws::Crt::Checksum::CombineCRC32C(static_cast<uint32_t>(combinedChecksum), partCrc, partSize);
1300+
}
1301+
}
1302+
}
1303+
}
1304+
Aws::Utils::ByteBuffer checksumBuffer(handle->GetChecksumAlgorithm()== S3::Model::ChecksumAlgorithm::CRC64NVME ? 8 : 4);
1305+
if (handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC64NVME) {
1306+
const uint64_t be = aws_hton64(combinedChecksum);
1307+
std::memcpy(checksumBuffer.GetUnderlyingData(), &be, sizeof(uint64_t));
1308+
} else {
1309+
const uint32_t be = aws_hton32(static_cast<uint32_t>(combinedChecksum));
1310+
std::memcpy(checksumBuffer.GetUnderlyingData(), &be, sizeof(uint32_t));
1311+
}
1312+
Aws::String combinedChecksumStr = Aws::Utils::HashingUtils::Base64Encode(checksumBuffer);
1313+
if (combinedChecksumStr != handle->GetChecksum()) {
1314+
AWS_LOGSTREAM_ERROR(CLASS_TAG, "Transfer handle [" << handle->GetId()
1315+
<< "] Full-object checksum mismatch. Expected: " << handle->GetChecksum()
1316+
<< ", Calculated: " << combinedChecksumStr);
1317+
Aws::Client::AWSError<Aws::S3::S3Errors> error(Aws::S3::S3Errors::INTERNAL_FAILURE,
1318+
"ChecksumMismatch",
1319+
"Full-object checksum validation failed",
1320+
false);
1321+
handle->SetError(error);
1322+
handle->UpdateStatus(TransferStatus::FAILED);
1323+
TriggerErrorCallback(handle, error);
1324+
}
1325+
}
12421326
outcome.GetResult().GetBody().flush();
12431327
handle->UpdateStatus(TransferStatus::COMPLETED);
12441328
}

0 commit comments

Comments
 (0)