Added local testing support for S3 (#1097)

* Added local testing support for S3

Author: raphkim

aws-android-sdk-s3/pom.xml

@@ -38,6 +38,12 @@
       <version>1.10.5</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.robolectric</groupId>
+      <artifactId>robolectric</artifactId>
+      <version>3.4</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
     	<groupId>org.hamcrest</groupId>
     	<artifactId>hamcrest-all</artifactId>

aws-android-sdk-s3/src/main/java/com/amazonaws/mobileconnectors/s3/transferutility/TransferUtility.java

@@ -23,6 +23,8 @@
 import android.database.Cursor;
 import android.net.ConnectivityManager;
 import android.net.Uri;
+import com.amazonaws.services.s3.S3ClientOptions;
+import com.amazonaws.services.s3.internal.Constants;
 import org.json.JSONObject;
 
 import com.amazonaws.AmazonWebServiceRequest;
@@ -264,6 +266,23 @@ public TransferUtility build() {
                     this.s3.setRegion(Region.getRegion(tuConfig.getString("Region")));
                     this.defaultBucket = tuConfig.getString("Bucket");
 
+                    // Checks if awsconfiguration.json has local testing flag to dangerously connect to HTTP endpoint.
+                    // Defaults to false unless specified.
+                    final boolean canConnectToHTTPEndpoint = tuConfig.has(Constants.LOCAL_TESTING_FLAG_NAME) ?
+                            tuConfig.getBoolean(Constants.LOCAL_TESTING_FLAG_NAME) : false;
+
+                    // Mutates AmazonS3Client object to have local endpoint
+                    if (canConnectToHTTPEndpoint) {
+                        this.s3.setEndpoint(Constants.LOCAL_TESTING_ENDPOINT);
+                        this.s3.setS3ClientOptions(S3ClientOptions.builder()
+                                // Prevents reformatting host address to accommodate AWS service hostname pattern
+                                .setPathStyleAccess(true)
+                                // Skips data integrity check after each transfer because correct
+                                // hashing algorithm isn't yet implemented in local storage server
+                                .skipContentMd5Check(true)
+                                .build());
+                    }
+
                     TransferUtility.setUserAgentFromConfig(this.awsConfig.getUserAgent());
                 } catch (Exception e) {
                     throw new IllegalArgumentException("Failed to read S3TransferUtility "

aws-android-sdk-s3/src/main/java/com/amazonaws/services/s3/AmazonS3Client.java

@@ -201,7 +201,7 @@ public class AmazonS3Client extends AmazonWebServiceClient implements AmazonS3 {
     private static final RequestPaymentConfigurationXmlFactory requestPaymentConfigurationXmlFactory = new RequestPaymentConfigurationXmlFactory();
 
     /** S3 specific client configuration options */
-    private S3ClientOptions clientOptions = new S3ClientOptions();
+    protected S3ClientOptions clientOptions = new S3ClientOptions();
 
     /** Provider for AWS credentials. */
     private final AWSCredentialsProvider awsCredentialsProvider;
@@ -1624,12 +1624,13 @@ public S3Object getObject(GetObjectRequest getObjectRequest)
                         ProgressEvent.STARTED_EVENT_CODE);
             }
 
+
             // The Etag header contains a server-side MD5 of the object. If
             // we're downloading the whole object, by default we wrap the
             // stream in a validator that calculates an MD5 of the downloaded
             // bytes and complains if what we received doesn't match the Etag.
-            if (!ServiceUtils.skipMd5CheckPerRequest(getObjectRequest)
-                    && !ServiceUtils.skipMd5CheckPerResponse(s3Object.getObjectMetadata())) {
+            if (!ServiceUtils.skipMd5CheckPerRequest(getObjectRequest, clientOptions)
+                    && !ServiceUtils.skipMd5CheckPerResponse(s3Object.getObjectMetadata(), clientOptions)) {
                 byte[] serverSideHash = null;
                 final String etag = s3Object.getObjectMetadata().getETag();
                 if (etag != null && ServiceUtils.isMultipartUploadETag(etag) == false) {
@@ -1706,7 +1707,7 @@ public S3Object getS3ObjectStream() {
 
                     @Override
                     public boolean needIntegrityCheck() {
-                        return !ServiceUtils.skipMd5CheckPerRequest(getObjectRequest);
+                        return !ServiceUtils.skipMd5CheckPerRequest(getObjectRequest, clientOptions);
                     }
 
                 }, mode);
@@ -1810,7 +1811,7 @@ public PutObjectResult putObject(PutObjectRequest putObjectRequest)
         assertParameterNotNull(key, "The key parameter must be specified when uploading an object");
 
         final boolean skipContentMd5Check = ServiceUtils
-                .skipMd5CheckPerRequest(putObjectRequest);
+                .skipMd5CheckPerRequest(putObjectRequest, clientOptions);
 
         // If a file is specified for upload, we need to pull some additional
         // information from it to auto-configure a few options
@@ -3854,7 +3855,7 @@ public UploadPartResult uploadPart(UploadPartRequest uploadPartRequest)
 
         MD5DigestCalculatingInputStream md5DigestStream = null;
         if (uploadPartRequest.getMd5Digest() == null
-                && !ServiceUtils.skipMd5CheckPerRequest(uploadPartRequest)) {
+                && !ServiceUtils.skipMd5CheckPerRequest(uploadPartRequest, clientOptions)) {
             /*
              * If the user hasn't set the content MD5, then we don't want to
              * buffer the whole stream in memory just to calculate it. Instead,
@@ -3885,7 +3886,7 @@ public UploadPartResult uploadPart(UploadPartRequest uploadPartRequest)
                     key);
 
             if (metadata != null && md5DigestStream != null
-                    && !ServiceUtils.skipMd5CheckPerResponse(metadata)) {
+                    && !ServiceUtils.skipMd5CheckPerResponse(metadata, clientOptions)) {
                 final byte[] clientSideHash = md5DigestStream.getMd5Digest();
                 final byte[] serverSideHash = BinaryUtils.fromHex(metadata.getETag());
 

aws-android-sdk-s3/src/main/java/com/amazonaws/services/s3/S3ClientOptions.java

@@ -19,6 +19,8 @@
  */
 public class S3ClientOptions {
 
+    /** The default setting for skipping content MD5 check */
+    public static final boolean DEFAULT_SKIP_CONTENT_MD5_CHECK = false;
     /** The default setting for use of path-style access */
     public static final boolean DEFAULT_PATH_STYLE_ACCESS = false;
     /** The default setting for use of chunked encoding */
@@ -30,6 +32,8 @@ public class S3ClientOptions {
     /** S3 dualstack endpoint is by default not enabled */
     public static final boolean DEFAULT_DUALSTACK_ENABLED = false;
 
+    /** Flag for skipping content MD5 check for local testing */
+    private boolean skipContentMd5Check;
     /** Flag for use of path-style access */
     private boolean pathStyleAccess;
     private final boolean chunkedEncodingDisabled;
@@ -48,6 +52,7 @@ public static Builder builder() {
      * Builder class for S3ClientOptions.
      */
     public static final class Builder {
+        private boolean skipContentMd5Check = DEFAULT_SKIP_CONTENT_MD5_CHECK;
         private boolean pathStyleAccess = DEFAULT_PATH_STYLE_ACCESS;
         /** Flag for user of chunked encoding */
         private boolean chunkedEncodingDisabled = DEFAULT_CHUNKED_ENCODING_DISABLED;
@@ -63,13 +68,34 @@ private Builder() {
          * @return S3ClientOptions instance.
          */
         public S3ClientOptions build() {
-            return new S3ClientOptions(pathStyleAccess,
+            return new S3ClientOptions(skipContentMd5Check,
+                    pathStyleAccess,
                     chunkedEncodingDisabled,
                     accelerateModeEnabled,
                     payloadSigningEnabled,
                     dualstackEnabled);
         }
 
+        /**
+         * <p>
+         * Configures the client to skip content MD5 check for all requests
+         * and responses.
+         * </p>
+         * <p>
+         * Setting this flag will allow the client to disregard data transfer
+         * integrity check by bypassing content MD5 check for all requests
+         * and responses.
+         * </p>
+         *
+         * @param skipContentMd5Check True to always skip content MD5 check.
+         * @return this Builder instance that can be used for method chaining
+         */
+        @SuppressWarnings("checkstyle:hiddenfield")
+        public Builder skipContentMd5Check(boolean skipContentMd5Check) {
+            this.skipContentMd5Check = skipContentMd5Check;
+            return this;
+        }
+
         /**
          * <p>
          * Configures the client to use path-style access for all requests.
@@ -186,6 +212,7 @@ public Builder enableDualstack() {
      */
     @Deprecated
     public S3ClientOptions() {
+        this.skipContentMd5Check = DEFAULT_SKIP_CONTENT_MD5_CHECK;
         this.pathStyleAccess = DEFAULT_PATH_STYLE_ACCESS;
         this.chunkedEncodingDisabled = DEFAULT_CHUNKED_ENCODING_DISABLED;
         this.accelerateModeEnabled = DEFAULT_ACCELERATE_MODE_ENABLED;
@@ -200,25 +227,45 @@ public S3ClientOptions() {
      */
     @Deprecated
     public S3ClientOptions(S3ClientOptions other) {
+        this.skipContentMd5Check = other.skipContentMd5Check;
         this.pathStyleAccess = other.pathStyleAccess;
         this.chunkedEncodingDisabled = other.chunkedEncodingDisabled;
         this.accelerateModeEnabled = other.accelerateModeEnabled;
         this.payloadSigningEnabled = other.payloadSigningEnabled;
         this.dualstackEnabled = other.dualstackEnabled;
     }
 
-    private S3ClientOptions(boolean pathStyleAccess,
+    private S3ClientOptions(boolean skipContentMd5Check,
+            boolean pathStyleAccess,
             boolean chunkedEncodingDisabled,
             boolean accelerateModeEnabled,
             boolean payloadSigningEnabled,
             boolean dualstackEnabled) {
+        this.skipContentMd5Check = skipContentMd5Check;
         this.pathStyleAccess = pathStyleAccess;
         this.chunkedEncodingDisabled = chunkedEncodingDisabled;
         this.accelerateModeEnabled = accelerateModeEnabled;
         this.payloadSigningEnabled = payloadSigningEnabled;
         this.dualstackEnabled = dualstackEnabled;
     }
 
+    /**
+     * <p>
+     * Returns whether the client skips content MD5 check for all requests
+     * and responses.
+     * </p>
+     * <p>
+     * Setting this flag will allow the client to disregard data transfer
+     * integrity check by bypassing content MD5 check for all requests and
+     * responses.
+     * </p>
+     *
+     * @return True if the client should always skip content MD5 check
+     */
+    public boolean isContentMd5CheckSkipped() {
+        return skipContentMd5Check;
+    }
+
     /**
      * <p>
      * Returns whether the client uses path-style access for all requests.
@@ -235,7 +282,7 @@ private S3ClientOptions(boolean pathStyleAccess,
      * this flag will result in path-style access being used for all requests.
      * </p>
      *
-     * @return True is the client should always use path-style access
+     * @return True if the client should always use path-style access
      */
     public boolean isPathStyleAccess() {
         return pathStyleAccess;
@@ -305,6 +352,22 @@ public boolean isPayloadSigningEnabled() {
         return payloadSigningEnabled;
     }
 
+    /**
+     * <p>
+     * Returns whether the client skips content MD5 check for all requests
+     * and responses.
+     * </p>
+     * <p>
+     * Setting this flag will allow the client to disregard data transfer
+     * integrity check by bypassing content MD5 check for all requests and
+     * responses.
+     * </p>
+     *
+     * @param skipContentMd5Check True to always skip content MD5 check
+     */
+    public void skipContentMd5Check(boolean skipContentMd5Check) {
+        this.skipContentMd5Check = skipContentMd5Check;
+    }
 
     /**
      * <p>

aws-android-sdk-s3/src/main/java/com/amazonaws/services/s3/internal/Constants.java

@@ -39,6 +39,16 @@ public class Constants {
     /** Service hostname for accessing dualstack accelerated S3 buckets */
     public static final String S3_ACCELERATE_DUALSTACK_HOSTNAME = "s3-accelerate.dualstack.amazonaws.com";
 
+    /** Parameter name to indicate permission to dangerously connect to HTTP endpoint */
+    public static final String LOCAL_TESTING_FLAG_NAME = "DangerouslyConnectToHTTPEndpointForTesting";
+
+    /**
+     * Default endpoint for the local S3 storage testing
+     *
+     * Android emulator uses 10.0.2.2 to connect to localhost instead of 127.0.0.1 (reserved for emulator itself)
+     */
+    public static final String LOCAL_TESTING_ENDPOINT = "http://10.0.2.2:20005";
+
     /** Dualstack qualifier for S3 */
     public static final String S3_DUALSTACK_QUALIFIER = "dualstack";
 

aws-android-sdk-s3/src/main/java/com/amazonaws/services/s3/internal/ServiceUtils.java

@@ -24,6 +24,7 @@
 import com.amazonaws.AmazonWebServiceRequest;
 import com.amazonaws.Request;
 import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.S3ClientOptions;
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.PutObjectRequest;
@@ -419,6 +420,29 @@ public static S3Object retryableDownloadS3ObjectToFile(File file,
      *         check on the requested object content.
      */
     public static boolean skipMd5CheckPerResponse(ObjectMetadata metadata) {
+        return skipMd5CheckPerResponse(metadata, null);
+    }
+
+    /**
+     * Based on the given metadata of an S3 response, Returns whether the
+     * specified request should skip MD5 check on the requested object content.
+     * Specifically, MD5 check should be skipped if either SSE-KMS or SSE-C is
+     * involved.
+     * <p>
+     * The reason is that when SSE-KMS or SSE-C is involved, the MD5 returned
+     * from the server side is the MD5 of the ciphertext, which will by
+     * definition mismatch the MD5 on the client side which is computed based on
+     * the plaintext.
+     * @param metadata the ObjectMetadata of an S3 response.
+     * @param clientOptions the S3 client options to see if check can be skipped
+     * @return true if the specified response should skip MD5
+     *         check on the requested object content.
+     */
+    public static boolean skipMd5CheckPerResponse(ObjectMetadata metadata, S3ClientOptions clientOptions) {
+        if (clientOptions != null && clientOptions.isContentMd5CheckSkipped()) {
+            return true;
+        }
+
         if (metadata == null) {
             return false;
         }
@@ -436,6 +460,22 @@ public static boolean skipMd5CheckPerResponse(ObjectMetadata metadata) {
      *         check on the requested object content.
      */
     public static boolean skipMd5CheckPerRequest(AmazonWebServiceRequest request) {
+        return skipMd5CheckPerRequest(request, null);
+    }
+
+    /**
+     * Returns whether the specified request should skip MD5 check on the
+     * requested object content.
+     * @param request the AmazonWebServiceRequest.
+     * @param clientOptions the S3 client options to see if check can be skipped
+     * @return true if the specified request should skip MD5
+     *         check on the requested object content.
+     */
+    public static boolean skipMd5CheckPerRequest(AmazonWebServiceRequest request, S3ClientOptions clientOptions) {
+        if (clientOptions != null && clientOptions.isContentMd5CheckSkipped()) {
+            return true;
+        }
+
         if (System.getProperty("com.amazonaws.services.s3.disableGetObjectMD5Validation") != null) {
             return true;
         }

aws-android-sdk-s3/src/test/java/com/amazonaws/services/s3/S3ClientOptionsTest.java

@@ -22,10 +22,14 @@
 public class S3ClientOptionsTest {
     @Test
     public void testBuilder() {
-        S3ClientOptions options = S3ClientOptions.builder().setAccelerateModeEnabled(true)
-                .setPathStyleAccess(true).build();
+        S3ClientOptions options = S3ClientOptions.builder()
+                .setAccelerateModeEnabled(true)
+                .setPathStyleAccess(true)
+                .skipContentMd5Check(true)
+                .build();
         assertTrue(options.isAccelerateModeEnabled());
         assertTrue(options.isPathStyleAccess());
+        assertTrue(options.isContentMd5CheckSkipped());
     }
 }