Merge branch 'dev' into fileencryption

Author: jofriedm-msft

ChangeLog.txt

@@ -1,5 +1,18 @@
-2017.XX.XX Version X.X.0
+2017.XX.XX Version X.X.X
+ * Added ErrorReceivingResponseEvent which fires when a network error occurs before the responseReceivedEvent fires. If the responseReceivedEvent fires sucessfully, this new event will not fire.
+ * For Premium Accounts only, added support for getting and setting the tier on a page blob. The tier can also be set when creating or copying from an existing page blob.
  * Added support for server side encryption for File Service.
+
+2017.06.21 Version 5.3.1
+ * Fixed a bug in specific upload case for block blobs. This only affects uploads greater than the max put blob threshold, that have increased the streamWriteSizeInBytes beyond the 4 MB and storeBlobContentMD5 has been disabled.
+ * In some cases in the above mentioned upload path, fixed a bug where StorageExceptions were being thrown instead of IOExceptions.
+
+2017.06.13 Version 5.3.0
+ * Fixed a bug where the transactional MD5 check would fail when downloading a range of blob or file and the recovery action is performed on a subsection of the range.
+ * Fixed leaking connections for table requests.
+ * Fixed a bug where retries happened immediately when experiencing a network exception uploading data or getting the response.
+ * Fixed a bug where the response stream was not being closed on nonretryable exceptions.
+
 2017.05.23 Version 5.2.0
  * Fixed Exists() calls on Shares and Directories to now populate metadata. This was already being done for Files.
  * Changed blob constants to support up to 256 MB on put blob for block blobs. The default value for put blob threshold has also been updated to half of the maximum, or 128 MB currently.

README.md

@@ -30,7 +30,7 @@ To get the binaries of this library as distributed by Microsoft, ready for use w
 <dependency>
 	<groupId>com.microsoft.azure</groupId>
 	<artifactId>azure-storage</artifactId>
-	<version>5.2.0</version>
+	<version>5.3.1</version>
 </dependency>
 ```
 
@@ -127,6 +127,8 @@ If you would like to become an active contributor to this project please follow
 
 If you encounter any bugs with the library please file an issue in the [Issues](https://github.com/Azure/azure-storage-java/issues) section of the project.
 
+When sending pull requests, please send non-breaking PRs to the dev branch and breaking changes to the dev_breaking branch. Do not make PRs against master.
+
 # Learn More
 
 * [Azure Developer Center](http://azure.microsoft.com/en-us/develop/java/)

microsoft-azure-storage-samples/pom.xml

@@ -26,7 +26,7 @@
 		<dependency>
 			<groupId>com.microsoft.azure</groupId>
 			<artifactId>azure-storage</artifactId>
-			<version>5.2.0</version>
+			<version>5.3.1</version>
 		</dependency>
 		<dependency>
 			<groupId>com.microsoft.azure</groupId>

microsoft-azure-storage-test/res/TestConfigurations.xml

@@ -1,5 +1,6 @@
 <TestConfigurations>
 <TargetTestTenant>ProductionTenant</TargetTestTenant>
+<TargetPremiumBlobTenant>ProductionTenant</TargetPremiumBlobTenant>
 <TenantConfigurations>
   <TenantConfiguration>
     <TenantName>DevStore</TenantName>

microsoft-azure-storage-test/src/com/microsoft/azure/storage/EventFiringTests.java

@@ -14,18 +14,20 @@
  */
 package com.microsoft.azure.storage;
 
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.CloudBlobClient;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.microsoft.azure.storage.blob.*;
 import com.microsoft.azure.storage.core.SR;
 import com.microsoft.azure.storage.TestRunners.CloudTests;
 import com.microsoft.azure.storage.TestRunners.DevFabricTests;
 import com.microsoft.azure.storage.TestRunners.DevStoreTests;
 
+import org.apache.http.protocol.HTTP;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.net.HttpURLConnection;
+import java.net.SocketException;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 
@@ -111,6 +113,22 @@ public void eventOccurred(ResponseReceivedEvent eventArg) {
             }
         });
 
+        eventContext.getErrorReceivingResponseEventHandler().addListener(new StorageEvent<ErrorReceivingResponseEvent>() {
+
+            @Override
+            public void eventOccurred(ErrorReceivingResponseEvent eventArg) {
+                fail("This event should not trigger");
+            }
+        });
+
+        OperationContext.getGlobalErrorReceivingResponseEventHandler().addListener(new StorageEvent<ErrorReceivingResponseEvent>() {
+
+            @Override
+            public void eventOccurred(ErrorReceivingResponseEvent eventArg) {
+                fail("This event should not trigger");
+            }
+        });
+
         assertEquals(0, callList.size());
         assertEquals(0, globalCallList.size());
 
@@ -138,6 +156,85 @@ public void eventOccurred(ResponseReceivedEvent eventArg) {
         assertEquals(2, globalCallList.size());
     }
 
+    @Test
+    public void testErrorReceivingResponseEvent() throws URISyntaxException, StorageException {
+        final ArrayList<Boolean> callList = new ArrayList<Boolean>();
+        final ArrayList<Boolean> globalCallList = new ArrayList<Boolean>();
+
+        OperationContext eventContext = new OperationContext();
+        BlobRequestOptions options = new BlobRequestOptions();
+        options.setRetryPolicyFactory(new RetryNoRetry());
+
+        // setting the sending request event handler to trigger an exception.
+        // this is a retryable exception
+        eventContext.getSendingRequestEventHandler().addListener(new StorageEvent<SendingRequestEvent>() {
+            @Override
+            public void eventOccurred(SendingRequestEvent eventArg) {
+                HttpURLConnection connection = (HttpURLConnection) eventArg.getConnectionObject();
+                connection.setFixedLengthStreamingMode(0);
+            }
+        });
+
+        eventContext.getErrorReceivingResponseEventHandler().addListener(new StorageEvent<ErrorReceivingResponseEvent>() {
+            @Override
+            public void eventOccurred(ErrorReceivingResponseEvent eventArg) {
+                assertEquals(eventArg.getRequestResult(), eventArg.getOpContext().getLastResult());
+                callList.add(true);
+            }
+        });
+
+        OperationContext.getGlobalErrorReceivingResponseEventHandler().addListener(new StorageEvent<ErrorReceivingResponseEvent>() {
+            @Override
+            public void eventOccurred(ErrorReceivingResponseEvent eventArg) {
+                assertEquals(eventArg.getRequestResult(), eventArg.getOpContext().getLastResult());
+                globalCallList.add(true);
+            }
+        });
+
+        CloudBlobClient blobClient = TestHelper.createCloudBlobClient();
+        CloudBlobContainer container = blobClient.getContainerReference("container1");
+        container.createIfNotExists();
+
+        try {
+            CloudBlockBlob blob1 = container.getBlockBlobReference("blob1");
+            try {
+                String blockID = String.format("%08d", 1);
+                blob1.uploadBlock(blockID, BlobTestHelper.getRandomDataStream(10), 10, null, options, eventContext);
+            } catch (Exception e) { }
+
+            // make sure both the local and globab context update
+            assertEquals(1, callList.size());
+            assertEquals(1, globalCallList.size());
+
+            // make sure only global updates by replacing the local with a no-op event
+            eventContext
+                    .setErrorReceivingResponseEventHandler(new StorageEventMultiCaster<ErrorReceivingResponseEvent, StorageEvent<ErrorReceivingResponseEvent>>());
+            try {
+                String blockID2 = String.format("%08d", 2);
+                blob1.uploadBlock(blockID2, BlobTestHelper.getRandomDataStream(10), 10, null, options, eventContext);
+            } catch (Exception e) { }
+
+            assertEquals(1, callList.size());
+            assertEquals(2, globalCallList.size());
+
+            // make sure global does not update by replacing the global with a no-op
+            OperationContext
+                    .setGlobalErrorReceivingResponseEventHandler(new StorageEventMultiCaster<ErrorReceivingResponseEvent, StorageEvent<ErrorReceivingResponseEvent>>());
+
+            // make sure neither update
+            try {
+                String blockID3 = String.format("%08d", 3);
+                blob1.uploadBlock(blockID3, BlobTestHelper.getRandomDataStream(10), 10, null, options, eventContext);
+            } catch (Exception e) { }
+
+            assertEquals(1, callList.size());
+            assertEquals(2, globalCallList.size());
+        }
+        finally {
+            container.deleteIfExists();
+        }
+    }
+
     @Test
     public void testRequestCompletedEvents() throws URISyntaxException, StorageException {
         final ArrayList<Boolean> callList = new ArrayList<Boolean>();

microsoft-azure-storage-test/src/com/microsoft/azure/storage/TestHelper.java

@@ -15,6 +15,7 @@
 package com.microsoft.azure.storage;
 
 import static org.junit.Assert.*;
+import static org.junit.Assume.assumeNotNull;
 
 import java.io.ByteArrayInputStream;
 import java.io.File;
@@ -39,6 +40,7 @@
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 
+import org.junit.AssumptionViolatedException;
 import org.w3c.dom.DOMException;
 import org.w3c.dom.Document;
 import org.w3c.dom.Node;
@@ -58,6 +60,9 @@ public class TestHelper {
     private static Tenant tenant;
     private static StorageCredentialsAccountAndKey credentials;
     private static CloudStorageAccount account;
+    private static Tenant premiumBlobTenant;
+    private static StorageCredentialsAccountAndKey premiumBlobCredentials;
+    private static CloudStorageAccount premiumBlobAccount;
 
     private final static boolean enableFiddler = true;
     private final static boolean requireSecondaryEndpoint = false;
@@ -67,6 +72,11 @@ public static CloudBlobClient createCloudBlobClient() throws StorageException {
         return client;
     }
 
+    public static CloudBlobClient createPremiumCloudBlobClient() throws StorageException {
+        CloudBlobClient client = getPremiumBlobAccount().createCloudBlobClient();
+        return client;
+    }
+
     public static CloudBlobClient createCloudBlobClient(SharedAccessAccountPolicy policy, boolean useHttps)
             throws StorageException, InvalidKeyException, URISyntaxException {
 
@@ -328,12 +338,12 @@ private static CloudStorageAccount getAccount() throws StorageException {
                     account = CloudStorageAccount.parse(cloudAccount);
                 }
                 else if (accountConfig != null) {
-                    tenant = readTestConfigsFromXml(new File(accountConfig));
+                    readTestConfigsFromXml(new File(accountConfig), false);
                     setAccountAndCredentials();
                 }
                 else {
                     URL localTestConfig = TestHelper.class.getClassLoader().getResource("TestConfigurations.xml");
-                    tenant = readTestConfigsFromXml(new File(localTestConfig.getPath()));
+                    readTestConfigsFromXml(new File(localTestConfig.getPath()), false);
                     setAccountAndCredentials();
                 }
             }
@@ -344,6 +354,47 @@ else if (accountConfig != null) {
         return account;
     }
 
+    private static CloudStorageAccount getPremiumBlobAccount() throws StorageException {
+        // Only do this the first time TestBase is called as storage account is static
+        if (premiumBlobAccount == null) {
+            //enable fiddler
+            if (enableFiddler)
+                enableFiddler();
+
+            // try to get the environment variable with the test configuration file path
+            String accountConfig;
+            try {
+                accountConfig = System.getenv("storageTestConfiguration");
+            }
+            catch (SecurityException e) {
+                accountConfig = null;
+            }
+
+            // if storageConnection is set, use that as an account string
+            // if storageTestConfiguration is set, use that as a path to the configurations file
+            // if neither are set, use the local configurations file at TestConfigurations.xml
+            try {
+                if (accountConfig != null) {
+                    readTestConfigsFromXml(new File(accountConfig), true);
+                    setAccountAndCredentials();
+                }
+                else {
+                    URL localTestConfig = TestHelper.class.getClassLoader().getResource("TestConfigurations.xml");
+                    readTestConfigsFromXml(new File(localTestConfig.getPath()), true);
+                    setAccountAndCredentials();
+                }
+            }
+            catch (AssumptionViolatedException e) {
+                throw e;
+            }
+            catch (Exception e) {
+                throw StorageException.translateClientException(e);
+            }
+        }
+
+        return premiumBlobAccount;
+    }
+
     private static void setAccountAndCredentials() {
         if (requireSecondaryEndpoint)
             tenant.assertSecondaryEndpoint();
@@ -353,9 +404,17 @@ private static void setAccountAndCredentials() {
                 tenant.getQueueServiceSecondaryEndpoint()), new StorageUri(tenant.getTableServiceEndpoint(),
                 tenant.getTableServiceSecondaryEndpoint()), new StorageUri(tenant.getFileServiceEndpoint(),
                 tenant.getFileServiceSecondaryEndpoint()));
+
+        if (premiumBlobTenant != null) {
+            premiumBlobCredentials = new StorageCredentialsAccountAndKey(premiumBlobTenant.getAccountName(), premiumBlobTenant.getAccountKey());
+            premiumBlobAccount = new CloudStorageAccount(premiumBlobCredentials, new StorageUri(premiumBlobTenant.getBlobServiceEndpoint(), premiumBlobTenant.getBlobServiceSecondaryEndpoint()),
+                    null,
+                    null,
+                    null);
+        }
     }
 
-    private static Tenant readTestConfigsFromXml(File testConfigurations) throws ParserConfigurationException,
+    private static void readTestConfigsFromXml(File testConfigurations, boolean premiumBlob) throws ParserConfigurationException,
             SAXException, IOException, DOMException, URISyntaxException {
 
         DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
@@ -372,7 +431,14 @@ private static Tenant readTestConfigsFromXml(File testConfigurations) throws Par
             throw new IllegalArgumentException("No TargetTestTenant specified.");
         }
 
-        Tenant tenant = null;
+        Node premiumBlobTenantNode = testConfigs.getElementsByTagName("TargetPremiumBlobTenant").item(0);
+        String premiumBlobTenantName = null;
+        if (premiumBlobTenantNode != null) {
+            premiumBlobTenantName = premiumBlobTenantNode.getTextContent();
+        }
+
+        tenant = null;
+        premiumBlobTenant = null;
         final NodeList tenantNodes = testConfigs.getElementsByTagName("TenantName");
         for (int i = 0; i < tenantNodes.getLength(); i++) {
             if (tenantNodes.item(i).getTextContent().equals(targetTenant)) {
@@ -436,18 +502,50 @@ else if (name.equals("TableHttpsPortOverride")) {
                         else if (name.equals("FileHttpsPortOverride")) {
                             tenant.setFileHttpsPortOverride(Integer.parseInt(node.getTextContent()));
                         }
-                        else {
+                        else if (!premiumBlob){
                             throw new IllegalArgumentException(String.format(
                                     "Invalid child of TenantConfiguration with name: %s", name));
                         }
                     }
                 }
             }
+
+            if (tenantNodes.item(i).getTextContent().equals(premiumBlobTenantName)) {
+                premiumBlobTenant = new Tenant();
+                Node parent = tenantNodes.item(i).getParentNode();
+                final NodeList childNodes = parent.getChildNodes();
+                for (int j = 0; j < childNodes.getLength(); j++) {
+                    final Node node = childNodes.item(j);
+
+                    if (node.getNodeType() != Node.ELEMENT_NODE) {
+                        // do nothing
+                    } else {
+                        final String name = node.getNodeName();
+
+                        if (name.equals("TenantName")) {
+                            premiumBlobTenant.setTenantName(node.getTextContent());
+                        } else if (name.equals("TenantType")) {
+                            // do nothing, we don't track this field
+                        } else if (name.equals("AccountName")) {
+                            premiumBlobTenant.setAccountName(node.getTextContent());
+                        } else if (name.equals("AccountKey")) {
+                            premiumBlobTenant.setAccountKey(node.getTextContent());
+                        } else if (name.equals("BlobServiceEndpoint")) {
+                            premiumBlobTenant.setBlobServiceEndpoint(new URI(node.getTextContent()));
+                        } else if (name.equals("BlobServiceSecondaryEndpoint")) {
+                            premiumBlobTenant.setBlobServiceSecondaryEndpoint(new URI(node.getTextContent()));
+                        }
+                    }
+                }
+            }
         }
 
-        if (tenant == null) {
+        if (tenant == null && !premiumBlob) {
             throw new IllegalArgumentException("TargetTestTenant specified did not exist in TenantConfigurations.");
         }
-        return tenant;
+
+        if (premiumBlobTenant == null && premiumBlob) {
+            assumeNotNull(premiumBlobTenant);
+        }
     }
 }