Skip Etag Locking Request Option Added

Can now specify the BlobRequestOption skipEtagLocking, which tells blob
downloads to ignore changes in etags as they progress.

Author: jaschrep-msft

microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/BlobTestHelper.java

@@ -466,4 +466,17 @@ public static void disableSoftDelete() throws StorageException, URISyntaxExcepti
         bClient.uploadServiceProperties(serviceProperties);
         Thread.sleep(30000);
     }
+
+    public static HashMap<String, String> generateSampleMetadata(final int entries) {
+        if (entries < 0) {
+            throw new IllegalArgumentException();
+        }
+
+        HashMap<String, String> result = new HashMap<>(entries);
+        for (int i = 0; i < entries; i++) {
+            result.put("metadataKey_" + i, "metadataValue_" + i);
+        }
+
+        return result;
+    }
 }

microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudBlockBlobTests.java

@@ -2699,4 +2699,36 @@ public void testGetAccountInfo() throws StorageException, URISyntaxException, In
         assertNotNull(accountInformation.getAccountKind());
         assertNotNull(accountInformation.getSkuName());
     }
+
+    @Test
+    public void testSkipEtagCheck() throws StorageException, IOException, URISyntaxException {
+        final int blobSize = 2 * Constants.DEFAULT_MINIMUM_READ_SIZE_IN_BYTES; // so BlobInputStream doesn't read entire blob at once.
+
+        // setup
+        CloudBlockBlob blob = (CloudBlockBlob)BlobTestHelper.uploadNewBlob(
+                this.container, BlobType.BLOCK_BLOB, "testSkipEtagCheck", blobSize, null);
+
+        BlobRequestOptions options = new BlobRequestOptions();
+        options.setSkipEtagLocking(true); // Only request option to skip for these downloads. The rest is automatic.
+        BlobInputStream stream = blob.openInputStream(null, options, null);
+
+        // test
+        byte[] buffer = new byte[Constants.DEFAULT_MINIMUM_READ_SIZE_IN_BYTES];
+
+        assertEquals(Constants.DEFAULT_MINIMUM_READ_SIZE_IN_BYTES, stream.read(buffer)); // read 1st half of blob
+        blob.downloadAttributes();
+        String etag1 = blob.getProperties().getEtag();
+
+        blob.setMetadata(BlobTestHelper.generateSampleMetadata(1));
+        blob.uploadMetadata(); // change etag
+
+        assertEquals(Constants.DEFAULT_MINIMUM_READ_SIZE_IN_BYTES, stream.read(buffer)); // read 2nd half of blob
+        blob.downloadAttributes();
+        String etag2 = blob.getProperties().getEtag();
+
+        assertNotEquals(etag1, etag2); // assert etags were actually different if we get here without throwing
+
+        // cleanup
+        stream.close();
+    }
 }

microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobInputStream.java

@@ -222,8 +222,11 @@ protected BlobInputStream(long blobRangeOffset, Long blobRangeLength, final Clou
             }
         }
 
-        this.accessCondition = AccessCondition.generateIfMatchCondition(this.parentBlobRef.getProperties().getEtag());
+        this.accessCondition = new AccessCondition();
         this.accessCondition.setLeaseID(previousLeaseId);
+        if (!options.getSkipEtagLocking()) {
+            this.accessCondition.setIfMatch(this.parentBlobRef.getProperties().getEtag());
+        }
 
         this.streamLength = blobRangeLength == null
                 ? this.parentBlobRef.getProperties().getLength() - this.blobRangeOffset

microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobRequestOptions.java

@@ -71,6 +71,13 @@ public final class BlobRequestOptions extends RequestOptions {
      * Default is true.
      */
     private boolean validateEncryptionPolicy = true;
+
+    /**
+     * WARNING: etag locking is automatically used in blob downloads to ensure the blob does not change mid-download.
+     * Skip this validation at your own risk.
+     * Default is false.
+     */
+    private boolean skipEtagLocking = false;
 
     /**
      * Creates an instance of the <code>BlobRequestOptions</code> class.
@@ -97,6 +104,7 @@ public BlobRequestOptions(final BlobRequestOptions other) {
             this.setSingleBlobPutThresholdInBytes(other.getSingleBlobPutThresholdInBytes());
             this.setEncryptionPolicy(other.getEncryptionPolicy());
             this.setValidateEncryptionPolicy(other.getValidateEncryptionPolicy());
+            this.setSkipEtagLocking(other.getSkipEtagLocking());
         }
     }
 
@@ -300,6 +308,18 @@ protected boolean getValidateEncryptionPolicy() {
         return this.validateEncryptionPolicy;
     }
 
+    /**
+     * WARNING: etag locking is automatically used in blob downloads to ensure the blob does not change mid-download.
+     * Skip this validation at your own risk.
+     *
+     * Gets whether etag locking and validation on blob downloads should be skipped.
+     *
+     * @return <code>true</code> if skipping is enabled; otherwise, <code>false</code>.
+     */
+    public boolean getSkipEtagLocking() {
+        return this.skipEtagLocking;
+    }
+
     /**
      * Sets whether a conditional failure should be absorbed on a retry attempt for the request. This option 
      * is only used by {@link CloudAppendBlob} in upload and openWrite methods. By default, it is set to 
@@ -427,7 +447,20 @@ public void setEncryptionPolicy(BlobEncryptionPolicy encryptionPolicy) {
     protected void setValidateEncryptionPolicy(boolean validateEncryptionPolicy) {
         this.validateEncryptionPolicy = validateEncryptionPolicy;
     }
-    
+
+    /**
+     * WARNING: etag locking is automatically used in blob downloads to ensure the blob does not change mid-download.
+     * Skip this validation at your own risk.
+     *
+     * Sets whether etag locking and validation on blob downloads should be skipped.
+     *
+     * @param skipEtagLocking
+     *            Use <code>true</code> to skip etag locking and validation; otherwise, <code>false</code>.
+     */
+    public void setSkipEtagLocking(boolean skipEtagLocking) {
+        this.skipEtagLocking = skipEtagLocking;
+    }
+
     /**
      * Assert that if validation is on, an encryption policy is not specified.
      */

microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlob.java

@@ -1243,7 +1243,7 @@ public final void download(final OutputStream outStream, final AccessCondition a
         options = BlobRequestOptions.populateAndApplyDefaults(options, this.properties.getBlobType(), this.blobServiceClient);
 
         ExecutionEngine.executeWithRetry(this.blobServiceClient, this, this.downloadToStreamImpl(
-                null /* blobOffset */, null /* length */, outStream, accessCondition, options, opContext), options
+                null /* blobOffset */, null /* length */, outStream, accessCondition, options), options
                 .getRetryPolicyFactory(), opContext);
     }
 
@@ -1307,7 +1307,7 @@ public final void downloadRange(final long offset, final Long length, final Outp
         }
 
         ExecutionEngine.executeWithRetry(this.blobServiceClient, this,
-                this.downloadToStreamImpl(offset, length, outStream, accessCondition, options, opContext),
+                this.downloadToStreamImpl(offset, length, outStream, accessCondition, options),
                 options.getRetryPolicyFactory(), opContext);
     }
 
@@ -1415,7 +1415,7 @@ public Void preProcessResponse(CloudBlob blob, CloudBlobClient client, Operation
     @DoesServiceRequest
     private final StorageRequest<CloudBlobClient, CloudBlob, Integer> downloadToStreamImpl(Long offset,
             Long length, final OutputStream userStream, final AccessCondition accessCondition,
-            final BlobRequestOptions options, OperationContext opContext) {
+            final BlobRequestOptions options) {
         options.assertPolicyIfRequired();
 
         final Long userSpecifiedLength = length;
@@ -1588,7 +1588,7 @@ public Integer postProcessResponse(HttpURLConnection connection, CloudBlob blob,
 
             @Override
             public void recoveryAction(OperationContext context) throws IOException {
-                if (this.getETagLockCondition() == null && (!Utility.isNullOrEmpty(this.getLockedETag()))) {
+                if (!options.getSkipEtagLocking() && this.getETagLockCondition() == null && (!Utility.isNullOrEmpty(this.getLockedETag()))) {
                     AccessCondition etagLockCondition = new AccessCondition();
                     etagLockCondition.setIfMatch(this.getLockedETag());
                     if (accessCondition != null) {
@@ -1653,7 +1653,7 @@ protected final int downloadRangeInternal(final long blobOffset, final Long leng
 
         WrappedByteArrayOutputStream outputStream = new WrappedByteArrayOutputStream(buffer, bufferOffset);
         ExecutionEngine.executeWithRetry(this.blobServiceClient, this,
-                this.downloadToStreamImpl(blobOffset, length, outputStream, accessCondition, options, opContext),
+                this.downloadToStreamImpl(blobOffset, length, outputStream, accessCondition, options),
                 options.getRetryPolicyFactory(), opContext);
         return outputStream.getPosition();
     }
@@ -1791,7 +1791,7 @@ public final int downloadToByteArray(final byte[] buffer, final int bufferOffset
 
         WrappedByteArrayOutputStream outputStream = new WrappedByteArrayOutputStream(buffer, bufferOffset);
         ExecutionEngine.executeWithRetry(this.blobServiceClient, this,
-                this.downloadToStreamImpl(null, null, outputStream, accessCondition, options, opContext),
+                this.downloadToStreamImpl(null, null, outputStream, accessCondition, options),
                 options.getRetryPolicyFactory(), opContext);
         return outputStream.getPosition();
     }