feat(storage): Customize minimum multipart upload size (#3039)

* Customize minimum upload part size (for client apps)

Co-authored-by: arunj-kp <[email protected]>

Author: tylerjroach

aws-android-sdk-s3/src/main/java/com/amazonaws/mobileconnectors/s3/transferutility/TransferUtility.java

@@ -43,6 +43,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import static com.amazonaws.services.s3.internal.Constants.GB;
 import static com.amazonaws.services.s3.internal.Constants.MAXIMUM_UPLOAD_PARTS;
 import static com.amazonaws.services.s3.internal.Constants.MB;
 
@@ -141,7 +142,9 @@ public class TransferUtility {
      * Default minimum part size for upload parts. Anything below this will use a
      * single upload
      */
-    static final int MINIMUM_UPLOAD_PART_SIZE = 5 * MB;
+    static final int DEFAULT_MINIMUM_UPLOAD_PART_SIZE_IN_BYTES = 5 * MB;
+    static final int MINIMUM_SUPPORTED_UPLOAD_PART_SIZE_IN_BYTES = 5 * MB;
+    static final long MAXIMUM_SUPPORTED_UPLOAD_PART_SIZE_IN_BYTES = 5 * GB;
 
     private static String userAgentFromConfig = "";
 
@@ -770,15 +773,16 @@ private List<Integer> getTransferIdsWithTypeAndStates(TransferType type, Transfe
      */
     private int createMultipartUploadRecords(String bucket, String key, File file, ObjectMetadata metadata,
             CannedAccessControlList cannedAcl) {
-        long remainingLenth = file.length();
-        double partSize = (double) remainingLenth / (double) MAXIMUM_UPLOAD_PARTS;
+        long remainingLength = file.length();
+        double partSize = (double) remainingLength / (double) MAXIMUM_UPLOAD_PARTS;
         partSize = Math.ceil(partSize);
-        final long optimalPartSize = (long) Math.max(partSize, MINIMUM_UPLOAD_PART_SIZE);
+        final long optimalPartSize = (long) Math.max(partSize,
+                transferUtilityOptions.getMinimumUploadPartSizeInBytes());
         long fileOffset = 0;
         int partNumber = 1;
 
         // the number of parts
-        final int partCount = (int) Math.ceil((double) remainingLenth / (double) optimalPartSize);
+        final int partCount = (int) Math.ceil((double) remainingLength / (double) optimalPartSize);
 
         /*
          * the size of valuesArray is partCount + 1, one for a multipart upload summary,
@@ -788,11 +792,11 @@ private int createMultipartUploadRecords(String bucket, String key, File file, O
         valuesArray[0] = dbUtil.generateContentValuesForMultiPartUpload(bucket, key, file, fileOffset, 0, "",
                 file.length(), 0, metadata, cannedAcl, transferUtilityOptions);
         for (int i = 1; i < partCount + 1; i++) {
-            final long bytesForPart = Math.min(optimalPartSize, remainingLenth);
+            final long bytesForPart = Math.min(optimalPartSize, remainingLength);
             valuesArray[i] = dbUtil.generateContentValuesForMultiPartUpload(bucket, key, file, fileOffset, partNumber,
-                    "", bytesForPart, remainingLenth - optimalPartSize <= 0 ? 1 : 0, metadata, cannedAcl, transferUtilityOptions);
+                    "", bytesForPart, remainingLength - optimalPartSize <= 0 ? 1 : 0, metadata, cannedAcl, transferUtilityOptions);
             fileOffset += optimalPartSize;
-            remainingLenth -= optimalPartSize;
+            remainingLength -= optimalPartSize;
             partNumber++;
         }
         return dbUtil.bulkInsertTransferRecords(valuesArray);
@@ -986,7 +990,10 @@ private synchronized void submitTransferJob(String action, int id) {
     }
 
     private boolean shouldUploadInMultipart(File file) {
-        return (file != null && file.length() > MINIMUM_UPLOAD_PART_SIZE);
+        return (
+                file != null &&
+                file.length() > transferUtilityOptions.getMinimumUploadPartSizeInBytes()
+        );
     }
 
     static <X extends AmazonWebServiceRequest> X appendTransferServiceUserAgentString(final X request) {

aws-android-sdk-s3/src/main/java/com/amazonaws/mobileconnectors/s3/transferutility/TransferUtilityOptions.java

@@ -15,6 +15,14 @@
 
 package com.amazonaws.mobileconnectors.s3.transferutility;
 
+import static com.amazonaws.mobileconnectors.s3.transferutility.TransferUtility.DEFAULT_MINIMUM_UPLOAD_PART_SIZE_IN_BYTES;
+import static com.amazonaws.mobileconnectors.s3.transferutility.TransferUtility.MAXIMUM_SUPPORTED_UPLOAD_PART_SIZE_IN_BYTES;
+import static com.amazonaws.mobileconnectors.s3.transferutility.TransferUtility.MINIMUM_SUPPORTED_UPLOAD_PART_SIZE_IN_BYTES;
+
+import com.amazonaws.logging.Log;
+import com.amazonaws.logging.LogFactory;
+import com.amazonaws.services.s3.internal.Constants;
+
 import java.io.Serializable;
 
 /**
@@ -39,6 +47,8 @@
  */
 public class TransferUtilityOptions implements Serializable {
 
+    private static final Log LOGGER = LogFactory.getLog(TransferUtilityOptions.class);
+
     /**
      * Default serial versionID.
      */
@@ -64,6 +74,12 @@ public class TransferUtilityOptions implements Serializable {
      */
     private int transferThreadPoolSize;
 
+    /**
+     * Minimum part size for upload parts. Anything below this will use a
+     * single upload
+     */
+    private long minimumUploadPartSizeInBytes;
+
     /**
      * Type of connection to use for transfers.
      */
@@ -78,6 +94,7 @@ public TransferUtilityOptions() {
         this.transferServiceCheckTimeInterval = getDefaultCheckTimeInterval();
         this.transferThreadPoolSize = getDefaultThreadPoolSize();
         this.transferNetworkConnectionType = getDefaultTransferNetworkConnectionType();
+        this.minimumUploadPartSizeInBytes = DEFAULT_MINIMUM_UPLOAD_PART_SIZE_IN_BYTES;
     }
 
     /**
@@ -93,6 +110,7 @@ public TransferUtilityOptions(int transferThreadPoolSize,
         this.transferServiceCheckTimeInterval = getDefaultCheckTimeInterval();
         this.transferThreadPoolSize = transferThreadPoolSize;
         this.transferNetworkConnectionType = transferNetworkConnectionType;
+        this.minimumUploadPartSizeInBytes = DEFAULT_MINIMUM_UPLOAD_PART_SIZE_IN_BYTES;
     }
 
     /**
@@ -162,6 +180,49 @@ static int getDefaultThreadPoolSize() {
         return 2 * (Runtime.getRuntime().availableProcessors() + 1);
     }
 
+    /**
+     * Retrieve minimum part size for upload parts in Bytes.
+     * @return the minimum upload part size in Bytes
+     */
+    protected long getMinimumUploadPartSizeInBytes() {
+        return minimumUploadPartSizeInBytes;
+    }
+
+    /**
+     * Retrieve minimum part size for upload parts in MB.
+     * @return the minimum upload part size in MB
+     */
+    public int getMinimumUploadPartSizeInMB() {
+        return (int) (minimumUploadPartSizeInBytes / Constants.MB);
+    }
+
+    /**
+     * Set the minimum part size in MB for upload parts.
+     * There maximum value allowed is 5GB. Anything higher will set minimum part size at a 5GB.
+     * There minimum value allowed is 5MB. Anything lower will set minimum part size at 5MB.
+     * @param minimumUploadPartSizeInMB the minimum part size to set in MB.
+     */
+    public void setMinimumUploadPartSizeInMB(final int minimumUploadPartSizeInMB) {
+        long minimumUploadPartSizeInBytes = minimumUploadPartSizeInMB * ((long) Constants.MB);
+        if (minimumUploadPartSizeInBytes > MAXIMUM_SUPPORTED_UPLOAD_PART_SIZE_IN_BYTES) {
+            LOGGER.warn(
+                    "The provided minimumUploadPartSize is greater than the maximum upload part " +
+                            "size limit. Setting upload part size to the maximum allowed value of"
+                            + (MINIMUM_SUPPORTED_UPLOAD_PART_SIZE_IN_BYTES / Constants.MB) + "MB."
+            );
+            this.minimumUploadPartSizeInBytes = MAXIMUM_SUPPORTED_UPLOAD_PART_SIZE_IN_BYTES;
+        } else if (minimumUploadPartSizeInBytes < MINIMUM_SUPPORTED_UPLOAD_PART_SIZE_IN_BYTES) {
+            LOGGER.warn(
+                    "The provided minimumUploadPartSize is less than the minimum upload part " +
+                            "size limit. Setting upload part size to the minimum allowed value of"
+                            + (MINIMUM_SUPPORTED_UPLOAD_PART_SIZE_IN_BYTES / Constants.MB) + "MB."
+            );
+            this.minimumUploadPartSizeInBytes = MINIMUM_SUPPORTED_UPLOAD_PART_SIZE_IN_BYTES;
+        } else {
+            this.minimumUploadPartSizeInBytes = minimumUploadPartSizeInBytes;
+        }
+    }
+
     /**
      * Return the default check time interval.
      *