Skip to content

Commit 22e831e

Browse files
sbisciglsbiscigl
committed
add option to ignore content length in S3Crt
1 parent 36382c7 commit 22e831e

File tree

10 files changed

+132
-35
lines changed

10 files changed

+132
-35
lines changed

‎generated/src/aws-cpp-sdk-s3-crt/include/aws/s3-crt/S3CrtClient.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7191,6 +7191,10 @@ namespace Aws
71917191
Aws::Client::XmlOutcome GenerateXmlOutcome(const std::shared_ptr<Http::HttpResponse>& response) const;
71927192
Aws::Client::StreamOutcome GenerateStreamOutcome(const std::shared_ptr<Http::HttpResponse>& response) const;
71937193

7194+
protected:
7195+
void AddContentLengthToRequest(const std::shared_ptr<Aws::Http::HttpRequest>& httpRequest,
7196+
const std::shared_ptr<Aws::IOStream>& body,
7197+
bool isChunked) const override;
71947198
private:
71957199
friend class Aws::Client::ClientWithAsyncTemplateMethods<S3CrtClient>;
71967200
void init(const S3Crt::ClientConfiguration& clientConfiguration,

‎generated/src/aws-cpp-sdk-s3-crt/include/aws/s3-crt/S3CrtClientConfiguration.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,16 @@ namespace Aws
160160
* This setting maps to CRT's network_interface_names_array config.
161161
*/
162162
Aws::Vector<Aws::String> networkInterfaceNames;
163+
164+
/**
165+
* Configuration on how to handle missing content length. By default the SDK attempts to seek the stream
166+
* to find the total length of the stream before streaming. CRT will fallback to multipart upload if there
167+
* is no content length, creating multipart uploads of partSize before knowing the stream has ended.
168+
*/
169+
enum class CONTENT_LENGTH_CONFIGURATION {
170+
SEEK_STREAM,
171+
SKIP_CONTENT_LENGTH,
172+
} contentLenghtConfiguration {CONTENT_LENGTH_CONFIGURATION::SEEK_STREAM};
163173
/* End of S3 CRT specifics */
164174
private:
165175
void LoadS3CrtSpecificConfig(const Aws::String& profileName);

‎generated/src/aws-cpp-sdk-s3-crt/source/S3CrtClient.cpp‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -239,10 +239,12 @@ S3CrtClient::S3CrtClient(const S3Crt::ClientConfiguration& clientConfiguration,
239239
signPayloads,
240240
false),
241241
Aws::MakeShared<S3CrtErrorMarshaller>(ALLOCATION_TAG)),
242-
m_clientConfiguration(clientConfiguration, signPayloads, useVirtualAddressing, USEast1RegionalEndPointOption),
242+
m_clientConfiguration(clientConfiguration),
243243
m_credProvider(Aws::MakeShared<DefaultAWSCredentialsProviderChain>(ALLOCATION_TAG, credentialsProvider)),
244244
m_identityProvider(Aws::MakeShared<DefaultS3ExpressIdentityProvider>(ALLOCATION_TAG, *this))
245245
{
246+
AWS_UNREFERENCED_PARAM(useVirtualAddressing);
247+
AWS_UNREFERENCED_PARAM(USEast1RegionalEndPointOption);
246248
init(clientConfiguration, m_credProvider);
247249
}
248250

@@ -256,10 +258,12 @@ S3CrtClient::S3CrtClient(const AWSCredentials& credentials, const S3Crt::ClientC
256258
signPayloads,
257259
false),
258260
Aws::MakeShared<S3CrtErrorMarshaller>(ALLOCATION_TAG)),
259-
m_clientConfiguration(clientConfiguration, signPayloads, useVirtualAddressing, USEast1RegionalEndPointOption),
261+
m_clientConfiguration(clientConfiguration),
260262
m_credProvider(Aws::MakeShared<SimpleAWSCredentialsProvider>(ALLOCATION_TAG, credentials)),
261263
m_identityProvider(Aws::MakeShared<DefaultS3ExpressIdentityProvider>(ALLOCATION_TAG, *this))
262264
{
265+
AWS_UNREFERENCED_PARAM(useVirtualAddressing);
266+
AWS_UNREFERENCED_PARAM(USEast1RegionalEndPointOption);
263267
init(clientConfiguration, m_credProvider);
264268
}
265269

@@ -274,10 +278,12 @@ S3CrtClient::S3CrtClient(const std::shared_ptr<AWSCredentialsProvider>& credenti
274278
signPayloads,
275279
false),
276280
Aws::MakeShared<S3CrtErrorMarshaller>(ALLOCATION_TAG)),
277-
m_clientConfiguration(clientConfiguration, signPayloads, useVirtualAddressing, USEast1RegionalEndPointOption),
281+
m_clientConfiguration(clientConfiguration, signPayloads),
278282
m_credProvider(credentialsProvider),
279283
m_identityProvider(Aws::MakeShared<DefaultS3ExpressIdentityProvider>(ALLOCATION_TAG, *this))
280284
{
285+
AWS_UNREFERENCED_PARAM(useVirtualAddressing);
286+
AWS_UNREFERENCED_PARAM(USEast1RegionalEndPointOption);
281287
init(clientConfiguration, m_credProvider);
282288
}
283289

@@ -5580,3 +5586,12 @@ bool S3CrtClient::MultipartUploadSupported() const
55805586
{
55815587
return true;
55825588
}
5589+
5590+
void S3CrtClient::AddContentLengthToRequest(const std::shared_ptr<Aws::Http::HttpRequest>& httpRequest,
5591+
const std::shared_ptr<Aws::IOStream>& body,
5592+
bool isChunked) const
5593+
{
5594+
if (m_clientConfiguration.contentLenghtConfiguration == S3CrtClientConfiguration::CONTENT_LENGTH_CONFIGURATION::SEEK_STREAM) {
5595+
BASECLASS::AddContentLengthToRequest(httpRequest, body, isChunked);
5596+
}
5597+
}

‎src/aws-cpp-sdk-core/include/aws/core/client/AWSClient.h‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -285,6 +285,14 @@ namespace Aws
285285
virtual void BuildHttpRequest(const Aws::AmazonWebServiceRequest& request,
286286
const std::shared_ptr<Aws::Http::HttpRequest>& httpRequest) const;
287287

288+
/**
289+
* Adds content-length to the request if the request has a body by attempting to seek the end of
290+
* the body.
291+
*/
292+
virtual void AddContentLengthToRequest(const std::shared_ptr<Aws::Http::HttpRequest>& httpRequest,
293+
const std::shared_ptr<Aws::IOStream>& body,
294+
bool isChunked) const;
295+
288296
/**
289297
* Gets the underlying ErrorMarshaller for subclasses to use.
290298
*/
@@ -303,7 +311,6 @@ namespace Aws
303311
std::shared_ptr<Auth::AWSCredentialsProvider> GetCredentialsProvider() const {
304312
return m_signerProvider->GetCredentialsProvider();
305313
}
306-
protected:
307314

308315
/**
309316
* Creates an HttpRequest instance with the given URI and sets the proper headers from the

‎src/aws-cpp-sdk-core/source/client/AWSClient.cpp‎

Lines changed: 23 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -820,30 +820,7 @@ void AWSClient::AddContentBodyToRequest(const std::shared_ptr<Aws::Http::HttpReq
820820
httpRequest->DeleteHeader(Http::CONTENT_LENGTH_HEADER);
821821
}
822822
}
823-
824-
//Add transfer-encoding:chunked to header
825-
if (body && isChunked && !httpRequest->HasHeader(Http::CONTENT_LENGTH_HEADER))
826-
{
827-
httpRequest->SetTransferEncoding(CHUNKED_VALUE);
828-
}
829-
//in the scenario where we are adding a content body as a stream, the request object likely already
830-
//has a content-length header set and we don't want to seek the stream just to find this information.
831-
else if (body && !httpRequest->HasHeader(Http::CONTENT_LENGTH_HEADER))
832-
{
833-
if (!m_httpClient->SupportsChunkedTransferEncoding())
834-
{
835-
AWS_LOGSTREAM_WARN(AWS_CLIENT_LOG_TAG, "This http client doesn't support transfer-encoding:chunked. " <<
836-
"The request may fail if it's not a seekable stream.");
837-
}
838-
AWS_LOGSTREAM_TRACE(AWS_CLIENT_LOG_TAG, "Found body, but content-length has not been set, attempting to compute content-length");
839-
body->seekg(0, body->end);
840-
auto streamSize = body->tellg();
841-
body->seekg(0, body->beg);
842-
Aws::StringStream ss;
843-
ss << streamSize;
844-
httpRequest->SetContentLength(ss.str());
845-
}
846-
823+
AddContentLengthToRequest(httpRequest, body, isChunked);
847824
if (needsContentMd5 && body && !httpRequest->HasHeader(Http::CONTENT_MD5_HEADER))
848825
{
849826
AWS_LOGSTREAM_TRACE(AWS_CLIENT_LOG_TAG, "Found body, and content-md5 needs to be set" <<
@@ -862,6 +839,28 @@ void AWSClient::AddContentBodyToRequest(const std::shared_ptr<Aws::Http::HttpReq
862839
}
863840
}
864841

842+
void AWSClient::AddContentLengthToRequest(
843+
const std::shared_ptr<Aws::Http::HttpRequest>& httpRequest,
844+
const std::shared_ptr<Aws::IOStream>& body,
845+
bool isChunked) const
846+
{
847+
if (body && isChunked && !httpRequest->HasHeader(Http::CONTENT_LENGTH_HEADER)) {
848+
httpRequest->SetTransferEncoding(CHUNKED_VALUE);
849+
} else if (body && !httpRequest->HasHeader(Http::CONTENT_LENGTH_HEADER)) {
850+
if (!m_httpClient->SupportsChunkedTransferEncoding()) {
851+
AWS_LOGSTREAM_WARN(AWS_CLIENT_LOG_TAG, "This http client doesn't support transfer-encoding:chunked. " <<
852+
"The request may fail if it's not a seekable stream.");
853+
}
854+
AWS_LOGSTREAM_TRACE(AWS_CLIENT_LOG_TAG, "Found body, but content-length has not been set, attempting to compute content-length");
855+
body->seekg(0, body->end);
856+
auto streamSize = body->tellg();
857+
body->seekg(0, body->beg);
858+
Aws::StringStream ss;
859+
ss << streamSize;
860+
httpRequest->SetContentLength(ss.str());
861+
}
862+
}
863+
865864
Aws::String Aws::Client::GetAuthorizationHeader(const Aws::Http::HttpRequest& httpRequest)
866865
{
867866
// Extract the hex-encoded signature from the authorization header rather than recalculating it.

‎tests/aws-cpp-sdk-s3-crt-integration-tests/BucketAndObjectOperationTest.cpp‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1530,6 +1530,29 @@ namespace
15301530
}
15311531
}
15321532

1533+
TEST_F(BucketAndObjectOperationTest, ContentLengthMissingShouldWork) {
1534+
const Aws::String fullBucketName = CalculateBucketName(BASE_PUT_OBJECTS_BUCKET_NAME.c_str());
1535+
SCOPED_TRACE(Aws::String("FullBucketName ") + fullBucketName);
1536+
CreateBucketRequest createBucketRequest;
1537+
createBucketRequest.SetBucket(fullBucketName);
1538+
createBucketRequest.SetACL(BucketCannedACL::private_);
1539+
1540+
CreateBucketOutcome createBucketOutcome = Client->CreateBucket(createBucketRequest);
1541+
AWS_ASSERT_SUCCESS(createBucketOutcome);
1542+
const CreateBucketResult& createBucketResult = createBucketOutcome.GetResult();
1543+
ASSERT_TRUE(!createBucketResult.GetLocation().empty());
1544+
ASSERT_TRUE(WaitForBucketToPropagate(fullBucketName));
1545+
TagTestBucket(fullBucketName, Client);
1546+
1547+
S3CrtClientConfiguration configuration{};
1548+
configuration.contentLenghtConfiguration = S3CrtClientConfiguration::CONTENT_LENGTH_CONFIGURATION::SKIP_CONTENT_LENGTH;
1549+
const S3CrtClient client{configuration};
1550+
auto request = PutObjectRequest{}.WithBucket(fullBucketName).WithKey("sam");
1551+
request.SetBody(Aws::MakeShared<StringStream>(ALLOCATION_TAG, "bridges"));
1552+
const auto response = client.PutObject(request);
1553+
AWS_EXPECT_SUCCESS(response);
1554+
}
1555+
15331556
class TestMonitoring: public Aws::Monitoring::MonitoringInterface
15341557
{
15351558
mutable std::shared_ptr<Aws::Vector<Aws::String>> m_sequence;

‎tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/s3/S3ClientHeader.vm‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,10 @@ namespace ${rootNamespace}
212212
Aws::Client::XmlOutcome GenerateXmlOutcome(const std::shared_ptr<Http::HttpResponse>& response) const;
213213
Aws::Client::StreamOutcome GenerateStreamOutcome(const std::shared_ptr<Http::HttpResponse>& response) const;
214214

215+
protected:
216+
void AddContentLengthToRequest(const std::shared_ptr<Aws::Http::HttpRequest>& httpRequest,
217+
const std::shared_ptr<Aws::IOStream>& body,
218+
bool isChunked) const override;
215219
#end
216220
private:
217221
friend class Aws::Client::ClientWithAsyncTemplateMethods<${className}>;

‎tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/s3/S3ClientSource.vm‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,4 +356,15 @@ bool ${className}::MultipartUploadSupported() const
356356
{
357357
return true;
358358
}
359+
#if($serviceNamespace == "S3Crt")
360+
361+
void ${className}::AddContentLengthToRequest(const std::shared_ptr<Aws::Http::HttpRequest>& httpRequest,
362+
const std::shared_ptr<Aws::IOStream>& body,
363+
bool isChunked) const
364+
{
365+
if (m_clientConfiguration.contentLenghtConfiguration == ${className}Configuration::CONTENT_LENGTH_CONFIGURATION::SEEK_STREAM) {
366+
BASECLASS::AddContentLengthToRequest(httpRequest, body, isChunked);
367+
}
368+
}
369+
#end
359370
#end

‎tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/s3/s3-crt/S3CrtClientConfigHeader.vm‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,4 +93,14 @@
9393
* This setting maps to CRT's network_interface_names_array config.
9494
*/
9595
Aws::Vector<Aws::String> networkInterfaceNames;
96+
97+
/**
98+
* Configuration on how to handle missing content length. By default the SDK attempts to seek the stream
99+
* to find the total length of the stream before streaming. CRT will fallback to multipart upload if there
100+
* is no content length, creating multipart uploads of partSize before knowing the stream has ended.
101+
*/
102+
enum class CONTENT_LENGTH_CONFIGURATION {
103+
SEEK_STREAM,
104+
MULTIPART_UPLOAD,
105+
} contentLenghtConfiguration {CONTENT_LENGTH_CONFIGURATION::SEEK_STREAM};
96106
/* End of S3 CRT specifics */

‎tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/s3/s3-crt/S3CrtServiceClientSourceInit.vm‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -132,12 +132,14 @@ ${className}::${className}(const ${clientConfigurationNamespace}::ClientConfigur
132132
false),
133133
Aws::MakeShared<${metadata.classNamePrefix}ErrorMarshaller>(ALLOCATION_TAG)),
134134
#if($serviceModel.endpointRules)
135-
m_clientConfiguration(clientConfiguration${signPayloadsParam}${virtualAddressingInit}${USEast1RegionalEndpointInitString}),
135+
m_clientConfiguration(clientConfiguration),
136136
${defaultCredentialsProviderChainMember},
137137
#else${defaultCredentialsProviderChainMember}${virtualAddressingInit}${USEast1RegionalEndpointInitString},
138138
#end
139139
m_identityProvider(Aws::MakeShared<DefaultS3ExpressIdentityProvider>(ALLOCATION_TAG, *this))
140140
{
141+
AWS_UNREFERENCED_PARAM(useVirtualAddressing);
142+
AWS_UNREFERENCED_PARAM(USEast1RegionalEndPointOption);
141143
init(clientConfiguration${credentialsParam});
142144
}
143145

@@ -152,12 +154,14 @@ ${className}::${className}(const AWSCredentials& credentials, const ${clientConf
152154
false),
153155
Aws::MakeShared<${metadata.classNamePrefix}ErrorMarshaller>(ALLOCATION_TAG)),
154156
#if($serviceModel.endpointRules)
155-
m_clientConfiguration(clientConfiguration${signPayloadsParam}${virtualAddressingInit}${USEast1RegionalEndpointInitString}),
157+
m_clientConfiguration(clientConfiguration),
156158
${simpleCredentialsProviderMember},
157159
#else${simpleCredentialsProviderMember}${virtualAddressingInit}${USEast1RegionalEndpointInitString},
158160
#end
159161
m_identityProvider(Aws::MakeShared<DefaultS3ExpressIdentityProvider>(ALLOCATION_TAG, *this))
160162
{
163+
AWS_UNREFERENCED_PARAM(useVirtualAddressing);
164+
AWS_UNREFERENCED_PARAM(USEast1RegionalEndPointOption);
161165
init(clientConfiguration${credentialsParam});
162166
}
163167

@@ -173,13 +177,15 @@ ${className}::${className}(const std::shared_ptr<AWSCredentialsProvider>& creden
173177
false),
174178
Aws::MakeShared<${metadata.classNamePrefix}ErrorMarshaller>(ALLOCATION_TAG)),
175179
#if($serviceModel.endpointRules)
176-
m_clientConfiguration(clientConfiguration${signPayloadsParam}${virtualAddressingInit}${USEast1RegionalEndpointInitString}),
180+
m_clientConfiguration(clientConfiguration${signPayloadsParam}),
177181
${credentialsProviderMember},
178182
#else
179183
${credentialsProviderMember}${virtualAddressingInit}${USEast1RegionalEndpointInitString},
180184
#end
181185
m_identityProvider(Aws::MakeShared<DefaultS3ExpressIdentityProvider>(ALLOCATION_TAG, *this))
182186
{
187+
AWS_UNREFERENCED_PARAM(useVirtualAddressing);
188+
AWS_UNREFERENCED_PARAM(USEast1RegionalEndPointOption);
183189
init(clientConfiguration${credentialsParam});
184190
}
185191
#else
@@ -189,13 +195,15 @@ ${className}::${className}(const ${clientConfigurationNamespace}::ClientConfigur
189195
SERVICE_NAME, Aws::Region::ComputeSignerRegion(clientConfiguration.region)${signPayloadsParam}${doubleEncodeValue}),
190196
Aws::MakeShared<${metadata.classNamePrefix}ErrorMarshaller>(ALLOCATION_TAG)),
191197
#if($serviceModel.endpointRules)
192-
m_clientConfiguration(clientConfiguration${signPayloadsParam}${virtualAddressingInit}${USEast1RegionalEndpointInitString}),
198+
m_clientConfiguration(clientConfiguration${signPayloadsParam}),
193199
${defaultCredentialsProviderChainMember},
194200
#else
195201
${defaultCredentialsProviderChainMember}${virtualAddressingInit}${USEast1RegionalEndpointInitString},
196202
#end
197203
m_identityProvider(Aws::MakeShared<DefaultS3ExpressIdentityProvider>(ALLOCATION_TAG, *this))
198204
{
205+
AWS_UNREFERENCED_PARAM(useVirtualAddressing);
206+
AWS_UNREFERENCED_PARAM(USEast1RegionalEndPointOption);
199207
init(clientConfiguration${credentialsParam});
200208
}
201209

@@ -205,13 +213,15 @@ ${className}::${className}(const AWSCredentials& credentials, const ${clientConf
205213
SERVICE_NAME, Aws::Region::ComputeSignerRegion(clientConfiguration.region)${signPayloadsParam}${doubleEncodeValue}),
206214
Aws::MakeShared<${metadata.classNamePrefix}ErrorMarshaller>(ALLOCATION_TAG)),
207215
#if($serviceModel.endpointRules)
208-
m_clientConfiguration(clientConfiguration${signPayloadsParam}${virtualAddressingInit}${USEast1RegionalEndpointInitString}),
216+
m_clientConfiguration(clientConfiguration),
209217
${simpleCredentialsProviderMember},
210218
#else
211219
${simpleCredentialsProviderMember}${virtualAddressingInit}${USEast1RegionalEndpointInitString},
212220
#end
213221
m_identityProvider(Aws::MakeShared<DefaultS3ExpressIdentityProvider>(ALLOCATION_TAG, *this))
214222
{
223+
AWS_UNREFERENCED_PARAM(useVirtualAddressing);
224+
AWS_UNREFERENCED_PARAM(USEast1RegionalEndPointOption);
215225
init(clientConfiguration${credentialsParam});
216226
}
217227

@@ -222,13 +232,15 @@ ${className}::${className}(const std::shared_ptr<AWSCredentialsProvider>& creden
222232
SERVICE_NAME, Aws::Region::ComputeSignerRegion(clientConfiguration.region)${signPayloadsParam}${doubleEncodeValue}),
223233
Aws::MakeShared<${metadata.classNamePrefix}ErrorMarshaller>(ALLOCATION_TAG)),
224234
#if($serviceModel.endpointRules)
225-
m_clientConfiguration(clientConfiguration${signPayloadsParam}${virtualAddressingInit}${USEast1RegionalEndpointInitString}),
235+
m_clientConfiguration(clientConfiguration),
226236
${virtualAddressingInit},
227237
#else
228238
${virtualAddressingInit}${USEast1RegionalEndpointInitString}${credentialsProviderMember},
229239
#end
230240
m_identityProvider(Aws::MakeShared<DefaultS3ExpressIdentityProvider>(ALLOCATION_TAG, *this))
231241
{
242+
AWS_UNREFERENCED_PARAM(useVirtualAddressing);
243+
AWS_UNREFERENCED_PARAM(USEast1RegionalEndPointOption);
232244
init(clientConfiguration${credentialsParam});
233245
}
234246
#end
@@ -239,13 +251,15 @@ ${className}::${className}(const std::shared_ptr<Aws::Auth::AWSAuthSignerProvide
239251
BASECLASS(clientConfiguration, signerProvider,
240252
Aws::MakeShared<${metadata.classNamePrefix}ErrorMarshaller>(ALLOCATION_TAG)),
241253
#if($serviceModel.endpointRules)
242-
m_clientConfiguration(clientConfiguration${signPayloadsParam}${virtualAddressingInit}${USEast1RegionalEndpointInitString}),
254+
m_clientConfiguration(clientConfiguration),
243255
${virtualAddressingInit},
244256
#else
245257
${virtualAddressingInit},
246258
#end
247259
m_identityProvider(Aws::MakeShared<DefaultS3ExpressIdentityProvider>(ALLOCATION_TAG, *this))
248260
{
261+
AWS_UNREFERENCED_PARAM(useVirtualAddressing);
262+
AWS_UNREFERENCED_PARAM(USEast1RegionalEndPointOption);
249263
init(m_clientConfiguration);
250264
}
251265

0 commit comments

Comments
 (0)