Skip to content

Commit 534ea0b

Browse files
sandeepvinayaksandeepvinayak
authored
add more perf configs, upgrade AWS and fix conf tests (#78)
1 parent 64c785b commit 534ea0b

File tree

1,103 files changed

+13680
-14455
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,103 files changed

+13680
-14455
lines changed

‎blob/blob-aws/pom.xml‎

Lines changed: 33 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@
165165
<dependency>
166166
<groupId>software.amazon.awssdk</groupId>
167167
<artifactId>apache-client</artifactId>
168-
<version>2.29.47</version>
168+
<version>2.35.0</version>
169169
<exclusions>
170170
<exclusion>
171171
<groupId>*</groupId>
@@ -176,7 +176,7 @@
176176
<dependency>
177177
<groupId>software.amazon.awssdk</groupId>
178178
<artifactId>auth</artifactId>
179-
<version>2.29.47</version>
179+
<version>2.35.0</version>
180180
<exclusions>
181181
<exclusion>
182182
<groupId>*</groupId>
@@ -187,7 +187,7 @@
187187
<dependency>
188188
<groupId>software.amazon.awssdk</groupId>
189189
<artifactId>aws-core</artifactId>
190-
<version>2.29.47</version>
190+
<version>2.35.0</version>
191191
<exclusions>
192192
<exclusion>
193193
<groupId>*</groupId>
@@ -198,7 +198,7 @@
198198
<dependency>
199199
<groupId>software.amazon.awssdk</groupId>
200200
<artifactId>http-client-spi</artifactId>
201-
<version>2.29.47</version>
201+
<version>2.35.0</version>
202202
<exclusions>
203203
<exclusion>
204204
<groupId>*</groupId>
@@ -209,7 +209,7 @@
209209
<dependency>
210210
<groupId>software.amazon.awssdk</groupId>
211211
<artifactId>netty-nio-client</artifactId>
212-
<version>2.29.47</version>
212+
<version>2.35.0</version>
213213
<exclusions>
214214
<exclusion>
215215
<groupId>*</groupId>
@@ -220,7 +220,7 @@
220220
<dependency>
221221
<groupId>software.amazon.awssdk</groupId>
222222
<artifactId>regions</artifactId>
223-
<version>2.29.47</version>
223+
<version>2.35.0</version>
224224
<exclusions>
225225
<exclusion>
226226
<groupId>*</groupId>
@@ -231,7 +231,7 @@
231231
<dependency>
232232
<groupId>software.amazon.awssdk</groupId>
233233
<artifactId>s3</artifactId>
234-
<version>2.29.47</version>
234+
<version>2.35.0</version>
235235
<exclusions>
236236
<exclusion>
237237
<groupId>*</groupId>
@@ -242,7 +242,7 @@
242242
<dependency>
243243
<groupId>software.amazon.awssdk</groupId>
244244
<artifactId>sdk-core</artifactId>
245-
<version>2.29.47</version>
245+
<version>2.35.0</version>
246246
<exclusions>
247247
<exclusion>
248248
<groupId>*</groupId>
@@ -253,7 +253,7 @@
253253
<dependency>
254254
<groupId>software.amazon.awssdk</groupId>
255255
<artifactId>sts</artifactId>
256-
<version>2.29.47</version>
256+
<version>2.35.0</version>
257257
<exclusions>
258258
<exclusion>
259259
<groupId>*</groupId>
@@ -277,7 +277,7 @@
277277
<dependency>
278278
<groupId>com.google.errorprone</groupId>
279279
<artifactId>error_prone_annotations</artifactId>
280-
<version>2.31.0</version>
280+
<version>2.35.0</version>
281281
<exclusions>
282282
<exclusion>
283283
<groupId>*</groupId>
@@ -475,7 +475,7 @@
475475
<dependency>
476476
<groupId>software.amazon.awssdk</groupId>
477477
<artifactId>annotations</artifactId>
478-
<version>2.29.47</version>
478+
<version>2.35.0</version>
479479
<exclusions>
480480
<exclusion>
481481
<groupId>*</groupId>
@@ -486,7 +486,7 @@
486486
<dependency>
487487
<groupId>software.amazon.awssdk</groupId>
488488
<artifactId>arns</artifactId>
489-
<version>2.29.47</version>
489+
<version>2.35.0</version>
490490
<exclusions>
491491
<exclusion>
492492
<groupId>*</groupId>
@@ -497,7 +497,7 @@
497497
<dependency>
498498
<groupId>software.amazon.awssdk</groupId>
499499
<artifactId>aws-query-protocol</artifactId>
500-
<version>2.29.47</version>
500+
<version>2.35.0</version>
501501
<exclusions>
502502
<exclusion>
503503
<groupId>*</groupId>
@@ -508,7 +508,7 @@
508508
<dependency>
509509
<groupId>software.amazon.awssdk</groupId>
510510
<artifactId>aws-xml-protocol</artifactId>
511-
<version>2.29.47</version>
511+
<version>2.35.0</version>
512512
<exclusions>
513513
<exclusion>
514514
<groupId>*</groupId>
@@ -519,7 +519,7 @@
519519
<dependency>
520520
<groupId>software.amazon.awssdk</groupId>
521521
<artifactId>checksums</artifactId>
522-
<version>2.29.47</version>
522+
<version>2.35.0</version>
523523
<exclusions>
524524
<exclusion>
525525
<groupId>*</groupId>
@@ -530,7 +530,7 @@
530530
<dependency>
531531
<groupId>software.amazon.awssdk</groupId>
532532
<artifactId>checksums-spi</artifactId>
533-
<version>2.29.47</version>
533+
<version>2.35.0</version>
534534
<exclusions>
535535
<exclusion>
536536
<groupId>*</groupId>
@@ -541,7 +541,7 @@
541541
<dependency>
542542
<groupId>software.amazon.awssdk</groupId>
543543
<artifactId>crt-core</artifactId>
544-
<version>2.29.47</version>
544+
<version>2.35.0</version>
545545
<exclusions>
546546
<exclusion>
547547
<groupId>*</groupId>
@@ -564,7 +564,7 @@
564564
<dependency>
565565
<groupId>software.amazon.awssdk</groupId>
566566
<artifactId>endpoints-spi</artifactId>
567-
<version>2.29.47</version>
567+
<version>2.35.0</version>
568568
<exclusions>
569569
<exclusion>
570570
<groupId>*</groupId>
@@ -575,7 +575,7 @@
575575
<dependency>
576576
<groupId>software.amazon.awssdk</groupId>
577577
<artifactId>http-auth</artifactId>
578-
<version>2.29.47</version>
578+
<version>2.35.0</version>
579579
<exclusions>
580580
<exclusion>
581581
<groupId>*</groupId>
@@ -586,7 +586,7 @@
586586
<dependency>
587587
<groupId>software.amazon.awssdk</groupId>
588588
<artifactId>http-auth-aws</artifactId>
589-
<version>2.29.47</version>
589+
<version>2.35.0</version>
590590
<exclusions>
591591
<exclusion>
592592
<groupId>*</groupId>
@@ -597,7 +597,7 @@
597597
<dependency>
598598
<groupId>software.amazon.awssdk</groupId>
599599
<artifactId>http-auth-spi</artifactId>
600-
<version>2.29.47</version>
600+
<version>2.35.0</version>
601601
<exclusions>
602602
<exclusion>
603603
<groupId>*</groupId>
@@ -608,7 +608,7 @@
608608
<dependency>
609609
<groupId>software.amazon.awssdk</groupId>
610610
<artifactId>identity-spi</artifactId>
611-
<version>2.29.47</version>
611+
<version>2.35.0</version>
612612
<exclusions>
613613
<exclusion>
614614
<groupId>*</groupId>
@@ -619,7 +619,7 @@
619619
<dependency>
620620
<groupId>software.amazon.awssdk</groupId>
621621
<artifactId>json-utils</artifactId>
622-
<version>2.29.47</version>
622+
<version>2.35.0</version>
623623
<exclusions>
624624
<exclusion>
625625
<groupId>*</groupId>
@@ -630,7 +630,7 @@
630630
<dependency>
631631
<groupId>software.amazon.awssdk</groupId>
632632
<artifactId>metrics-spi</artifactId>
633-
<version>2.29.47</version>
633+
<version>2.35.0</version>
634634
<exclusions>
635635
<exclusion>
636636
<groupId>*</groupId>
@@ -641,7 +641,7 @@
641641
<dependency>
642642
<groupId>software.amazon.awssdk</groupId>
643643
<artifactId>profiles</artifactId>
644-
<version>2.29.47</version>
644+
<version>2.35.0</version>
645645
<exclusions>
646646
<exclusion>
647647
<groupId>*</groupId>
@@ -652,7 +652,7 @@
652652
<dependency>
653653
<groupId>software.amazon.awssdk</groupId>
654654
<artifactId>protocol-core</artifactId>
655-
<version>2.29.47</version>
655+
<version>2.35.0</version>
656656
<exclusions>
657657
<exclusion>
658658
<groupId>*</groupId>
@@ -663,7 +663,7 @@
663663
<dependency>
664664
<groupId>software.amazon.awssdk</groupId>
665665
<artifactId>retries</artifactId>
666-
<version>2.29.47</version>
666+
<version>2.35.0</version>
667667
<exclusions>
668668
<exclusion>
669669
<groupId>*</groupId>
@@ -674,7 +674,7 @@
674674
<dependency>
675675
<groupId>software.amazon.awssdk</groupId>
676676
<artifactId>retries-spi</artifactId>
677-
<version>2.29.47</version>
677+
<version>2.35.0</version>
678678
<exclusions>
679679
<exclusion>
680680
<groupId>*</groupId>
@@ -685,7 +685,7 @@
685685
<dependency>
686686
<groupId>software.amazon.awssdk</groupId>
687687
<artifactId>third-party-jackson-core</artifactId>
688-
<version>2.29.47</version>
688+
<version>2.35.0</version>
689689
<exclusions>
690690
<exclusion>
691691
<groupId>*</groupId>
@@ -696,7 +696,7 @@
696696
<dependency>
697697
<groupId>software.amazon.awssdk</groupId>
698698
<artifactId>utils</artifactId>
699-
<version>2.29.47</version>
699+
<version>2.35.0</version>
700700
<exclusions>
701701
<exclusion>
702702
<groupId>*</groupId>
@@ -724,9 +724,9 @@
724724
<scope>test</scope>
725725
</dependency>
726726
<dependency>
727-
<groupId>org.slf4j</groupId>
728-
<artifactId>slf4j-simple</artifactId>
729-
<version>2.0.15</version>
727+
<groupId>ch.qos.logback</groupId>
728+
<artifactId>logback-classic</artifactId>
729+
<version>1.5.19</version>
730730
<scope>test</scope>
731731
</dependency>
732732

‎blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStore.java‎

Lines changed: 55 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -443,6 +443,12 @@ private static S3AsyncClient buildCrtS3Client(Builder builder, Region regionObj)
443443
if (builder.getMaxNativeMemoryLimitInBytes() != null) {
444444
crtBuilder.maxNativeMemoryLimitInBytes(builder.getMaxNativeMemoryLimitInBytes());
445445
}
446+
if (builder.getInitialReadBufferSizeInBytes() != null) {
447+
crtBuilder.initialReadBufferSizeInBytes(builder.getInitialReadBufferSizeInBytes());
448+
}
449+
if (builder.getMaxConcurrency() != null) {
450+
crtBuilder.maxConcurrency(builder.getMaxConcurrency());
451+
}
446452

447453
// Apply common configuration (credentials, endpoint, proxy, part buffer size)
448454
applyCommonConfig(crtBuilder, builder, regionObj);
@@ -480,17 +486,39 @@ private static void applyCommonConfig(S3AsyncClientBuilder builder, Builder conf
480486
builder.endpointOverride(config.getEndpoint());
481487
}
482488

483-
// Configure proxy if specified
484-
if (config.getProxyEndpoint() != null) {
485-
ProxyConfiguration proxyConfig = ProxyConfiguration.builder()
486-
.scheme(config.getProxyEndpoint().getScheme())
487-
.host(config.getProxyEndpoint().getHost())
488-
.port(config.getProxyEndpoint().getPort())
489-
.build();
490-
SdkAsyncHttpClient httpClient = NettyNioAsyncHttpClient.builder()
491-
.proxyConfiguration(proxyConfig)
492-
.build();
493-
builder.httpClient(httpClient);
489+
// Configure HTTP client if any settings are specified
490+
if (config.getProxyEndpoint() != null || config.getMaxConnections() != null ||
491+
config.getSocketTimeout() != null || config.getIdleConnectionTimeout() != null) {
492+
493+
NettyNioAsyncHttpClient.Builder httpClientBuilder = NettyNioAsyncHttpClient.builder();
494+
495+
// Configure proxy if specified
496+
if (config.getProxyEndpoint() != null) {
497+
ProxyConfiguration proxyConfig = ProxyConfiguration.builder()
498+
.scheme(config.getProxyEndpoint().getScheme())
499+
.host(config.getProxyEndpoint().getHost())
500+
.port(config.getProxyEndpoint().getPort())
501+
.build();
502+
httpClientBuilder.proxyConfiguration(proxyConfig);
503+
}
504+
505+
// Configure max connections if specified
506+
if (config.getMaxConnections() != null) {
507+
httpClientBuilder.maxConcurrency(config.getMaxConnections());
508+
}
509+
510+
// Configure socket timeout if specified
511+
if (config.getSocketTimeout() != null) {
512+
httpClientBuilder.writeTimeout(config.getSocketTimeout());
513+
httpClientBuilder.readTimeout(config.getSocketTimeout());
514+
}
515+
516+
// Configure idle connection timeout if specified
517+
if (config.getIdleConnectionTimeout() != null) {
518+
httpClientBuilder.connectionMaxIdleTime(config.getIdleConnectionTimeout());
519+
}
520+
521+
builder.httpClient(httpClientBuilder.build());
494522
}
495523

496524
// Configure multipart configuration (common for both clients)
@@ -576,9 +604,25 @@ public AsyncBlobStore build() {
576604
if (tm == null) {
577605
var transferManagerBuilder = S3TransferManager.builder()
578606
.s3Client(client);
607+
608+
// Configure executor service for transfer manager
579609
if (getExecutorService() != null) {
610+
// Use custom executor service if provided
580611
transferManagerBuilder.executor(getExecutorService());
612+
} else {
613+
// Determine thread pool size for transfer manager
614+
Integer poolSize = getTransferManagerThreadPoolSize();
615+
616+
if (poolSize != null) {
617+
transferManagerBuilder.executor(Executors.newFixedThreadPool(poolSize));
618+
}
581619
}
620+
621+
// Configure transferDirectoryMaxConcurrency if specified
622+
if (getTransferDirectoryMaxConcurrency() != null) {
623+
transferManagerBuilder.transferDirectoryMaxConcurrency(getTransferDirectoryMaxConcurrency());
624+
}
625+
582626
tm = transferManagerBuilder.build();
583627
}
584628

‎blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/AwsBlobStoreIT.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import com.salesforce.multicloudj.sts.model.StsCredentials;
99
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
1010
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
11+
import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
1112
import software.amazon.awssdk.http.SdkHttpClient;
1213
import software.amazon.awssdk.regions.Region;
1314
import software.amazon.awssdk.services.s3.S3Client;
@@ -67,10 +68,12 @@ private AbstractBlobStore<?> createBlobStore(final String bucketName, final Cred
6768
client = S3Client.builder()
6869
.region(Region.US_WEST_2)
6970
.httpClient(httpClient)
71+
.requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
7072
.credentialsProvider(StaticCredentialsProvider.create(awsCredentials))
7173
.endpointOverride(URI.create(endpoint))
7274
.serviceConfiguration(S3Configuration.builder()
7375
.pathStyleAccessEnabled(true)
76+
.chunkedEncodingEnabled(false)
7477
.build())
7578
.build();
7679

0 commit comments

Comments
 (0)