Merge pull request #169 from jofriedm-msft/dev

Fix transactional MD5 validations on recovery

Author: jofriedm-msft

ChangeLog.txt

@@ -1,3 +1,6 @@
+2017.XX.XX Version X.X.X
+ * Fixed a bug where the tranactional MD5 check would fail when downloading a range of blob or file and the recovery action is performed on a subsection of the range.
+
 2017.05.23 Version 5.2.0
  * Fixed Exists() calls on Shares and Directories to now populate metadata. This was already being done for Files.
  * Changed blob constants to support up to 256 MB on put blob for block blobs. The default value for put blob threshold has also been updated to half of the maximum, or 128 MB currently.

microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobEncryptStream.java

@@ -145,7 +145,7 @@ public void write(byte[] data, int offset, int length) throws IOException {
 
     @Override
     public void write(InputStream sourceStream, long writeLength) throws IOException, StorageException {
-        Utility.writeToOutputStream(sourceStream, this, writeLength, false, false, this.opContext, this.options, false);      
+        Utility.writeToOutputStream(sourceStream, this, writeLength, false, false, this.opContext, this.options, false, null /* descriptor */);
     }
 
     @Override

microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobOutputStreamInternal.java

@@ -641,7 +641,7 @@ public void write(final byte[] data, final int offset, final int length) throws
      */
     @DoesServiceRequest
     public void write(final InputStream sourceStream, final long writeLength) throws IOException, StorageException {
-        Utility.writeToOutputStream(sourceStream, this, writeLength, false, false, this.opContext, this.options, false);
+        Utility.writeToOutputStream(sourceStream, this, writeLength, false, false, this.opContext, this.options, false, null /* descriptor */);
     }
 
     /**

microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlob.java

@@ -1402,7 +1402,7 @@ public Integer postProcessResponse(HttpURLConnection connection, CloudBlob blob,
                     // writeToOutputStream will update the currentRequestByteCount on this request in case a retry
                     // is needed and download should resume from that point
                     final StreamMd5AndLength descriptor = Utility.writeToOutputStream(streamRef, outStream, -1, false,
-                            validateMD5, context, options, true, this);
+                            validateMD5, context, options, true, this, this.getCurrentDescriptor());
 
                     // length was already checked by the NetworkInputStream, now check Md5
                     if (validateMD5 && !this.getContentMD5().equals(descriptor.getMd5())) {

microsoft-azure-storage/src/com/microsoft/azure/storage/core/StorageRequest.java

@@ -99,6 +99,11 @@ public abstract class StorageRequest<C, P, R> {
      */
     private String contentMD5 = null;
 
+    /**
+     * Holds the descriptor which contains the stream length and MD5 hash.
+     */
+    private StreamMd5AndLength currentDescriptor = null;
+
     /**
      * Denotes the StorageUri of the request
      */
@@ -212,6 +217,11 @@ public final String getContentMD5() {
         return this.contentMD5;
     }
 
+    /**
+     * @return the current descriptor which contains the stream length and MD5 hash.
+     */
+    protected StreamMd5AndLength getCurrentDescriptor() { return this.currentDescriptor; }
+
     /**
      * @return the location mode used to decide which location the request should be sent to.
      */
@@ -457,6 +467,14 @@ public void setContentMD5(String contentMD5) {
         this.contentMD5 = contentMD5;
     }
 
+    /**
+     * @param currentDescriptor
+     *              the descriptor value
+     */
+    protected void setCurrentDescriptor(StreamMd5AndLength currentDescriptor) {
+        this.currentDescriptor = currentDescriptor;
+    }
+
     /**
      * @param etagLockCondition
      *            the locked ETag condition

microsoft-azure-storage/src/com/microsoft/azure/storage/core/Utility.java

@@ -1272,7 +1272,45 @@ public static StreamMd5AndLength writeToOutputStream(final InputStream sourceStr
             long writeLength, final boolean rewindSourceStream, final boolean calculateMD5, OperationContext opContext,
             final RequestOptions options, final Boolean shouldFlush) throws IOException, StorageException {
         return writeToOutputStream(sourceStream, outStream, writeLength, rewindSourceStream, calculateMD5, opContext,
-                options, shouldFlush, null /*StorageRequest*/);
+                options, shouldFlush, null /*StorageRequest*/, null /* descriptor */);
+    }
+
+    /**
+     * Reads data from an input stream and writes it to an output stream, calculates the length of the data written, and
+     * optionally calculates the MD5 hash for the data.
+     *
+     * @param sourceStream
+     *            An <code>InputStream</code> object that represents the input stream to use as the source.
+     * @param outStream
+     *            An <code>OutputStream</code> object that represents the output stream to use as the destination.
+     * @param writeLength
+     *            The number of bytes to read from the stream.
+     * @param rewindSourceStream
+     *            <code>true</code> if the input stream should be rewound <strong>before</strong> it is read; otherwise,
+     *            <code>false</code>
+     * @param calculateMD5
+     *            <code>true</code> if an MD5 hash will be calculated; otherwise, <code>false</code>.
+     * @param opContext
+     *            An {@link OperationContext} object that represents the context for the current operation. This object
+     *            is used to track requests to the storage service, and to provide additional runtime information about
+     *            the operation.
+     * @param options
+     *            A {@link RequestOptions} object that specifies any additional options for the request. Namely, the
+     *            maximum execution time.
+     * @param request
+     *            Used by download resume to set currentRequestByteCount on the request. Otherwise, null is always used.
+     * @return A {@link StreamMd5AndLength} object that contains the output stream length, and optionally the MD5 hash.
+     *
+     * @throws IOException
+     *             If an I/O error occurs.
+     * @throws StorageException
+     *             If a storage service error occurred.
+     */
+    public static StreamMd5AndLength writeToOutputStream(final InputStream sourceStream, final OutputStream outStream,
+                                                         long writeLength, final boolean rewindSourceStream, final boolean calculateMD5, OperationContext opContext,
+                                                         final RequestOptions options, final Boolean shouldFlush, StorageRequest<?, ?, Integer> request)
+            throws IOException, StorageException {
+        return writeToOutputStream(sourceStream, outStream, writeLength, rewindSourceStream, calculateMD5, opContext, options, shouldFlush, request, null /* descriptor */);
     }
 
     /**
@@ -1299,6 +1337,11 @@ public static StreamMd5AndLength writeToOutputStream(final InputStream sourceStr
      *            maximum execution time.
      * @param request
      *            Used by download resume to set currentRequestByteCount on the request. Otherwise, null is always used.
+     * @param descriptor
+     *                A {@Link StreamMd5AndLength} object to append to in the case of recovery action or null if this is not called
+     *                from a recovery. This value needs to be passed for recovery in case part of the body has already been read,
+     *                the recovery will attempt to download the remaining bytes but will do MD5 validation on the originally
+     *                requested range size.
      * @return A {@link StreamMd5AndLength} object that contains the output stream length, and optionally the MD5 hash.
      * 
      * @throws IOException
@@ -1308,24 +1351,28 @@ public static StreamMd5AndLength writeToOutputStream(final InputStream sourceStr
      */
     public static StreamMd5AndLength writeToOutputStream(final InputStream sourceStream, final OutputStream outStream,
             long writeLength, final boolean rewindSourceStream, final boolean calculateMD5, OperationContext opContext,
-            final RequestOptions options, final Boolean shouldFlush, StorageRequest<?, ?, Integer> request)
+            final RequestOptions options, final Boolean shouldFlush, StorageRequest<?, ?, Integer> request, StreamMd5AndLength descriptor)
             throws IOException, StorageException {
         if (rewindSourceStream && sourceStream.markSupported()) {
             sourceStream.reset();
             sourceStream.mark(Constants.MAX_MARK_LENGTH);
         }
 
-        final StreamMd5AndLength retVal = new StreamMd5AndLength();
-
-        if (calculateMD5) {
-            try {
-                retVal.setDigest(MessageDigest.getInstance("MD5"));
-            }
-            catch (final NoSuchAlgorithmException e) {
-                // This wont happen, throw fatal.
-                throw Utility.generateNewUnexpectedStorageException(e);
+        if (descriptor == null) {
+            descriptor = new StreamMd5AndLength();
+            if (calculateMD5) {
+                try {
+                    descriptor.setDigest(MessageDigest.getInstance("MD5"));
+                }
+                catch (final NoSuchAlgorithmException e) {
+                    // This wont happen, throw fatal.
+                    throw Utility.generateNewUnexpectedStorageException(e);
+                }
             }
         }
+        else {
+            descriptor.setMd5(null);
+        }
 
         if (writeLength < 0) {
             writeLength = Long.MAX_VALUE;
@@ -1349,25 +1396,26 @@ public static StreamMd5AndLength writeToOutputStream(final InputStream sourceStr
             }
 
             if (calculateMD5) {
-                retVal.getDigest().update(retrievedBuff, 0, count);
+                descriptor.getDigest().update(retrievedBuff, 0, count);
             }
 
-            retVal.setLength(retVal.getLength() + count);
-            retVal.setCurrentOperationByteCount(retVal.getCurrentOperationByteCount() + count);
+            descriptor.setLength(descriptor.getLength() + count);
+            descriptor.setCurrentOperationByteCount(descriptor.getCurrentOperationByteCount() + count);
 
             if (request != null) {
                 request.setCurrentRequestByteCount(request.getCurrentRequestByteCount() + count);
+                request.setCurrentDescriptor(descriptor);
             }
 
-            nextCopy = (int) Math.min(retrievedBuff.length, writeLength - retVal.getLength());
+            nextCopy = (int) Math.min(retrievedBuff.length, writeLength - descriptor.getLength());
             count = sourceStream.read(retrievedBuff, 0, nextCopy);
         }
 
         if (outStream != null && shouldFlush) {
             outStream.flush();
         }
 
-        return retVal;
+        return descriptor;
     }
 
     /**

microsoft-azure-storage/src/com/microsoft/azure/storage/file/CloudFile.java

@@ -1411,7 +1411,7 @@ public Integer postProcessResponse(HttpURLConnection connection, CloudFile file,
                     // writeToOutputStream will update the currentRequestByteCount on this request in case a retry
                     // is needed and download should resume from that point
                     final StreamMd5AndLength descriptor = Utility.writeToOutputStream(streamRef, outStream, -1, false,
-                            validateMD5, context, options, true, this);
+                            validateMD5, context, options, true, this, this.getCurrentDescriptor());
 
                     // length was already checked by the NetworkInputStream, now check Md5
                     if (validateMD5 && !this.getContentMD5().equals(descriptor.getMd5())) {

microsoft-azure-storage/src/com/microsoft/azure/storage/file/FileOutputStream.java

@@ -404,7 +404,7 @@ public void write(final byte[] data, final int offset, final int length) throws
      */
     @DoesServiceRequest
     public void write(final InputStream sourceStream, final long writeLength) throws IOException, StorageException {
-        Utility.writeToOutputStream(sourceStream, this, writeLength, false, false, this.opContext, this.options, false);
+        Utility.writeToOutputStream(sourceStream, this, writeLength, false, false, this.opContext, this.options, false, null /* descriptor */);
     }
 
     /**