add timeout. Add sync http client with higher conn pool manually and close s3 client

Author: L-Applin

test/s3-tests/pom.xml

@@ -176,6 +176,12 @@
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>apache-client</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/ControlPlaneOperationRegressionTesting.java

@@ -21,7 +21,9 @@
 
 import java.util.List;
 import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
 import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.Timeout;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 import software.amazon.awssdk.awscore.AwsClient;
@@ -45,6 +47,7 @@ public class ControlPlaneOperationRegressionTesting extends BaseS3RegressionTest
     // Request checksum required
     @ParameterizedTest
     @MethodSource("testConfigs")
+    @Timeout(value = 120, unit = TimeUnit.SECONDS)
     void deleteObject(TestConfig config) throws Exception {
         assumeNotAccessPointWithPathStyle(config);
 

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DownloadStreamingRegressionTesting.java

@@ -40,6 +40,7 @@
 import org.junit.jupiter.api.Assumptions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Timeout;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 import software.amazon.awssdk.core.ResponseBytes;
@@ -111,6 +112,7 @@ void testCleanup() {
 
     @ParameterizedTest
     @MethodSource("downloadConfigs")
+    @Timeout(value = 120, unit = TimeUnit.SECONDS)
     void downloadObject(DownloadConfig config) throws Exception {
         assumeNotAccessPointWithPathStyle(config.baseConfig());
 

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ChecksumsTestUtils.java

@@ -31,6 +31,8 @@
 import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm;
 import software.amazon.awssdk.checksums.SdkChecksum;
 import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
+import software.amazon.awssdk.http.SdkHttpClient;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3AsyncClient;
 import software.amazon.awssdk.services.s3.S3Client;
@@ -43,6 +45,7 @@
 import software.amazon.awssdk.services.s3.model.S3Exception;
 import software.amazon.awssdk.services.s3.regression.upload.FlattenUploadConfig;
 import software.amazon.awssdk.services.s3control.S3ControlClient;
+import software.amazon.awssdk.services.s3control.model.CloudWatchMetrics;
 import software.amazon.awssdk.services.s3control.model.CreateMultiRegionAccessPointRequest;
 import software.amazon.awssdk.services.s3control.model.GetMultiRegionAccessPointResponse;
 import software.amazon.awssdk.services.s3control.model.MultiRegionAccessPointStatus;
@@ -277,13 +280,20 @@ public static S3Client makeSyncClient(FlattenUploadConfig config, ClientOverride
                                           Region region, AwsCredentialsProvider provider) {
         return S3Client.builder()
                        .overrideConfiguration(overrideConfiguration)
+                       .httpClient(makeHttpClient())
                        .forcePathStyle(config.isForcePathStyle())
                        .requestChecksumCalculation(config.getRequestChecksumValidation())
                        .region(region)
                        .credentialsProvider(provider)
                        .build();
     }
 
+    private static SdkHttpClient makeHttpClient() {
+        return ApacheHttpClient.builder()
+            .maxConnections(10_000)
+            .build();
+    }
+
     public static S3AsyncClient makeAsyncClient(FlattenUploadConfig config,
                                                 S3ClientFlavor flavor,
                                                 ClientOverrideConfiguration overrideConfiguration,

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/upload/UploadAsyncRegressionTesting.java

@@ -21,7 +21,11 @@
 
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import org.junit.Test;
 import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Timeout;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
@@ -44,6 +48,7 @@ public static List<FlattenUploadConfig> testConfigs() {
 
     @ParameterizedTest
     @MethodSource("testConfigs")
+    @Timeout(value = 120, unit = TimeUnit.SECONDS)
     void putObject(FlattenUploadConfig config) throws Exception {
         assumeNotAccessPointWithPathStyle(config);
 

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/upload/UploadCrtRegressionTesting.java

@@ -20,7 +20,9 @@
 import static software.amazon.awssdk.services.s3.regression.S3ClientFlavor.CRT_BASED;
 
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.Timeout;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
@@ -40,6 +42,7 @@ public static List<FlattenUploadConfig> testConfigs() {
 
     @ParameterizedTest
     @MethodSource("testConfigs")
+    @Timeout(value = 120, unit = TimeUnit.SECONDS)
     void putObject(FlattenUploadConfig config) throws Exception {
         assumeNotAccessPointWithPathStyle(config);
 

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/upload/UploadStreamingRegressionTesting.java

@@ -130,6 +130,8 @@ protected TestCallable<PutObjectResponse> callPutObject(PutObjectRequest request
                 return s3Client.putObject(request, requestBody);
             } catch (Exception e) {
                 throw new RuntimeException(e);
+            } finally {
+                s3Client.close();
             }
         };
         return new TestCallable<>(s3Client, callable);
@@ -149,6 +151,8 @@ protected TestCallable<PutObjectResponse> callPutObject(PutObjectRequest request
                 return CompletableFutureUtils.joinLikeSync(future);
             } catch (Exception e) {
                 throw new RuntimeException(e);
+            } finally {
+                s3Client.close();
             }
         };
         return new TestCallable<>(s3Client, callable);
@@ -170,6 +174,8 @@ protected TestCallable<PutObjectResponse> callTmUpload(PutObjectRequest request,
                 return completedUpload.response();
             } catch (Exception e) {
                 throw new RuntimeException(e);
+            } finally {
+                transferManager.close();
             }
         };
         return new TestCallable<>(transferManager, callable);

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/upload/UploadSyncRegressionTesting.java

@@ -20,6 +20,8 @@
 
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import org.junit.jupiter.api.Timeout;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
@@ -42,6 +44,7 @@ public static List<FlattenUploadConfig> testConfigs() {
 
     @ParameterizedTest
     @MethodSource("testConfigs")
+    @Timeout(value = 120, unit = TimeUnit.SECONDS)
     void putObject(FlattenUploadConfig config) throws Exception {
         assumeNotAccessPointWithPathStyle(config);
 

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/upload/UploadTransferManagerRegressionTesting.java

@@ -19,7 +19,9 @@
 import static software.amazon.awssdk.services.s3.regression.S3ClientFlavor.MULTIPART_ENABLED;
 
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.Timeout;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
@@ -40,6 +42,7 @@ public static List<FlattenUploadConfig> testConfigs() {
 
     @ParameterizedTest
     @MethodSource("testConfigs")
+    @Timeout(value = 120, unit = TimeUnit.SECONDS)
     void putObject(FlattenUploadConfig config) throws Exception {
 
         assumeNotAccessPointWithPathStyle(config);