Lower large file and split codebuilds to reduce build time

Author: L-Applin

buildspecs/s3-regression-tests.yml

@@ -3,7 +3,7 @@ version: 0.2
 phases:
   build:
     commands:
-      - mvn clean install -P s3-regression-tests -pl :s3-tests -am -T1C $MAVEN_OPTIONS
+      - mvn clean install -P s3-regression-tests -pl :s3-tests -am -T1C Dregression.test=$REGRESSION_TEST $MAVEN_OPTIONS
       - echo $MAVEN_OPTIONS
     finally:
       - mkdir -p codebuild-test-reports

pom.xml

@@ -924,9 +924,9 @@
                                 </goals>
                                 <configuration>
                                     <!-- this test needs a lot of memory, we need to increase java heap size or we get OOM in codebuild -->
-                                    <argLine>-Xmx12g -Xms4g -XX:+AllowRedefinitionToAddDeleteMethods</argLine>
+                                    <argLine>-Xmx12g -Xms4g</argLine>
                                     <includes>
-                                        <include>**/*RegressionTesting.java</include>
+                                        <include>**/*${regression.test}.java</include>
                                     </includes>
                                     <trimStackTrace>false</trimStackTrace>
                                     <classesDirectory>${project.build.outputDirectory}</classesDirectory>

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DownloadStreamingRegressionTesting.java

@@ -79,8 +79,8 @@ public class DownloadStreamingRegressionTesting extends BaseS3RegressionTest {
     static void init() throws Exception {
         tempDirPath = createTempDir("DownloadStreamingIntegrationTesting");
         smallObject = uploadObjectSmall(); // 16 KiB
-        largeObject = uploadObjectLarge(); // 80 MiB
-        largeObjectMulti = uploadMultiPartObject(); // 80 MiB, default multipart config
+        largeObject = uploadObjectLarge(); // 60 MiB
+        largeObjectMulti = uploadMultiPartObject(); // 60 MiB, default multipart config
     }
 
     @AfterAll
@@ -179,13 +179,13 @@ static ObjectWithCRC uploadObjectSmall() throws IOException {
         return new ObjectWithCRC(name, crc32);
     }
 
-    // 80 MiB
+    // 60 MiB
     static ObjectWithCRC uploadObjectLarge() throws IOException {
         String name = String.format("%s-%s.dat", System.currentTimeMillis(), UUID.randomUUID());
         LOG.info(() -> "test setup - uploading large test object: " + name);
         ByteArrayOutputStream os = new ByteArrayOutputStream();
         byte[] rand = new byte[1024 * 1024];
-        for (int i = 0; i < 80; i++) {
+        for (int i = 0; i < 60; i++) {
             new Random().nextBytes(rand);
             os.write(rand);
         }
@@ -204,12 +204,12 @@ static ObjectWithCRC uploadObjectLarge() throws IOException {
         return new ObjectWithCRC(name, crc32);
     }
 
-    // 80MiB, multipart default config
+    // 60MiB, multipart default config
     static ObjectWithCRC uploadMultiPartObject() throws Exception {
         String name = String.format("%s-%s.dat", System.currentTimeMillis(), UUID.randomUUID());
         LOG.info(() -> "test setup - uploading large test object - multipart: " + name);
         ByteArrayOutputStream os = new ByteArrayOutputStream();
-        byte[] rand = new byte[8 * 1024 * 1024];
+        byte[] rand = new byte[6 * 1024 * 1024];
         for (int i = 0; i < 10; i++) {
             new Random().nextBytes(rand);
             os.write(rand);

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ChecksumsTestUtils.java

@@ -333,12 +333,12 @@ public static Path createRandomFile16KB() throws IOException {
         return tmp;
     }
 
-    public static Path createRandomFile80MB() throws IOException {
+    public static Path createRandomFile60MB() throws IOException {
         Path tmp = Files.createTempFile(null, null);
         byte[] randomBytes = new byte[1024 * 1024];
         new Random().nextBytes(randomBytes);
         try (OutputStream os = Files.newOutputStream(tmp)) {
-            for (int i = 0; i < 80; ++i) {
+            for (int i = 0; i < 60; ++i) {
                 os.write(randomBytes);
             }
         }

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java

@@ -93,7 +93,7 @@ public class UploadStreamingRegressionTesting extends BaseS3RegressionTest {
     @BeforeAll
     static void setupClass() throws IOException {
         testFileSmall = S3ChecksumsTestUtils.createRandomFile16KB();
-        testFileLarge = S3ChecksumsTestUtils.createRandomFile80MB();
+        testFileLarge = S3ChecksumsTestUtils.createRandomFile60MB();
 
         // used in RequestBody.*buffers(...) API
         // we calculate crc32 once to try to accelerate test execution