Merge pull request #117 from jofriedm-msft/master

Azure Storage Client Library 4.4.0 Release

Author: mirobers

ChangeLog.txt

@@ -1,3 +1,6 @@
+2016.08.30 Version 4.4.0
+ * Fixed a bug in client-side encryption for tables that was preventing the Java client from decrypting entities encrypted with the .NET client, and vice versa.
+
 2016.07.06 Version 4.3.0
  * Added support for server-side encryption.
  * Added support for getBlobReferenceFromServer methods on CloudBlobContainer to support retrieving a blob without knowing its type.

microsoft-azure-storage-samples/pom.xml

@@ -26,7 +26,7 @@
 		<dependency>
 			<groupId>com.microsoft.azure</groupId>
 			<artifactId>azure-storage</artifactId>
-			<version>4.3.0</version>
+			<version>4.4.0</version>
 		</dependency>
 		<dependency>
 			<groupId>com.microsoft.azure</groupId>

microsoft-azure-storage-test/src/com/microsoft/azure/storage/GenericTests.java

@@ -30,6 +30,8 @@
 import java.util.HashMap;
 import java.util.UUID;
 
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
@@ -53,6 +55,16 @@
 @Category({ DevFabricTests.class, DevStoreTests.class, CloudTests.class })
 public class GenericTests {
 
+    @Before
+    public void genericTestMethodSetUp() {
+        OperationContext.setDefaultProxy(Proxy.NO_PROXY);
+    }
+
+    @After
+    public void genericTestMethodTearDown() {
+        OperationContext.setDefaultProxy(Proxy.NO_PROXY);
+    }
+
     /**
      * ReadTimeout must always be explicitly set on HttpUrlConnection to avoid a bug in JDK 6. In certain cases this
      * bug causes an immediate SocketException to be thrown indicating that the read timed out even if ReadTimeout was

microsoft-azure-storage-test/src/com/microsoft/azure/storage/TestRunners.java

@@ -138,7 +138,7 @@ public static class EncryptionTestSuite {
 
     @RunWith(Suite.class)
     @SuiteClasses({ CoreTestSuite.class, BlobTestSuite.class, QueueTestSuite.class, TableTestSuite.class,
-            FileTestSuite.class, AnalyticsTestSuite.class })
+            FileTestSuite.class, AnalyticsTestSuite.class, EncryptionTestSuite.class })
     public static class AllTestSuite {
     }
 

microsoft-azure-storage-test/src/com/microsoft/azure/storage/table/TableEncryptionTests.java

@@ -45,6 +45,7 @@
 import com.microsoft.azure.storage.table.TableTestHelper.Class1;
 import com.microsoft.azure.storage.table.TableTestHelper.EncryptedClass1;
 import com.microsoft.azure.storage.table.TableTestHelper.ComplexEntity;
+import com.sun.jersey.core.util.Base64;
 
 public class TableEncryptionTests {
     CloudTable table = null;
@@ -457,6 +458,18 @@ private void doTableQueryDTEProjectionEncryption(TablePayloadFormat format, Symm
             assertTrue(ent.getProperties().get("A").getValueAsString().equals("a")
                     || ent.getProperties().get("A").getValueAsString().equals(Constants.EMPTY_STRING));
         }
+        
+        // Test to make sure that we don't specify encryption columns when there aren't any columns specified at all.
+        query = TableQuery.from(DynamicTableEntity.class);
+
+        for (DynamicTableEntity ent : this.table.execute(query, options, null)) {
+            assertNotNull(ent.getPartitionKey());
+            assertNotNull(ent.getRowKey());
+            assertNotNull(ent.getTimestamp());
+
+            assertTrue(ent.getProperties().get("A").getValueAsString().equals("a")
+                    || ent.getProperties().get("A").getValueAsString().equals(Constants.EMPTY_STRING));
+        }
     }
 
     @Test
@@ -1022,8 +1035,7 @@ public void testTableOperationEncryptionWithStrictModeOnMerge() throws InvalidKe
     }
 
     @Test
-    public void testTableOperationsIgnoreEncryption() throws InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException, URISyntaxException, StorageException
-    {
+    public void testTableOperationsIgnoreEncryption() throws InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException, URISyntaxException, StorageException {
         SymmetricKey aesKey = TestHelper.getSymmetricKey();
         TableRequestOptions options = new TableRequestOptions();
         options.setEncryptionPolicy(new TableEncryptionPolicy(aesKey, null));
@@ -1072,6 +1084,65 @@ public void testTableOperationsIgnoreEncryption() throws InvalidKeyException, No
         {
             testTable.deleteIfExists();
         }
+    }    
+    
+    @Test
+    public void testCrossPlatformCompatibility() throws StorageException, URISyntaxException {
+        CloudTable testTable = TableTestHelper.getRandomTableReference();
+
+        try
+        {
+            testTable.createIfNotExists();
+            
+            // Hard code some sample data, then see if we can decrypt it.
+            // This key is used only for test, do not use to encrypt any sensitive data.
+            SymmetricKey sampleKEK = new SymmetricKey("key1", Base64.decode("rFz7+tv4hRiWdWUJMFlxl1xxtU/qFUeTriGaxwEcxjU="));
+
+            // This data here was created using Fiddler to capture the .NET library uploading an encrypted entity, encrypted with the specified KEK and CEK.
+            // Note that this data is lacking the library information in the KeyWrappingMetadata.
+            DynamicTableEntity dteNetOld = new DynamicTableEntity("pk", "netUp");
+            dteNetOld.getProperties().put("sampleProp", new EntityProperty(Base64.decode("27cLSlSFqy9C0xUCr57XAA==")));
+            dteNetOld.getProperties().put("sampleProp2", new EntityProperty(Base64.decode("pZR6Ln/DwbwyyOCEezL/hg==")));
+            dteNetOld.getProperties().put("sampleProp3", new EntityProperty(Base64.decode("JOix4N8eX/WuCtIvlD2QxQ==")));
+            dteNetOld.getProperties().put("_ClientEncryptionMetadata1", new EntityProperty("{\"WrappedContentKey\":{\"KeyId\":\"key1\",\"EncryptedKey\":\"pwSKxpJkwCS2zCaykh0m8e4OApeLuQ4FiahZ9zdwxaLL1HsWqQ4DSw==\",\"Algorithm\":\"A256KW\"},\"EncryptionAgent\":{\"Protocol\":\"1.0\",\"EncryptionAlgorithm\":\"AES_CBC_256\"},\"ContentEncryptionIV\":\"obTAQcYeFQ3IU7Jfcema7Q==\",\"KeyWrappingMetadata\":{}}"));
+            dteNetOld.getProperties().put("_ClientEncryptionMetadata2", new EntityProperty(Base64.decode("MWA7LlvXSJnKhf8f7MVhfjWECkxrCyCXGIlYY6ucpr34IVDU7fN6IHvKxV15WiXp")));
+
+            testTable.execute(TableOperation.insert(dteNetOld));
+
+            // This data here was created using Fiddler to capture the Java library uploading an encrypted entity, encrypted with the specified KEK and CEK.
+            // Note that this data is lacking the KeyWrappingMetadata.  It also constructs an IV with PK + RK + column name.
+            DynamicTableEntity dteJavaOld = new DynamicTableEntity("pk", "javaUp");
+            dteJavaOld.getProperties().put("sampleProp", new EntityProperty(Base64.decode("sa3bCvXq79ImSPveChS+cg==")));
+            dteJavaOld.getProperties().put("sampleProp2", new EntityProperty(Base64.decode("KXjuBNn9DesCmMcdVpamJw==")));
+            dteJavaOld.getProperties().put("sampleProp3", new EntityProperty(Base64.decode("wykVEni1rV+H6oNjoNml6A==")));
+            dteJavaOld.getProperties().put("_ClientEncryptionMetadata1", new EntityProperty("{\"WrappedContentKey\":{\"KeyId\":\"key1\",\"EncryptedKey\":\"2F4rIuDmGPgEmhpvTtE7x6281BetKz80EsgRwGxTjL8rRt7Z7GrOgg==\",\"Algorithm\":\"A256KW\"},\"EncryptionAgent\":{\"Protocol\":\"1.0\",\"EncryptionAlgorithm\":\"AES_CBC_256\"},\"ContentEncryptionIV\":\"8st/uXffG+6DxBhw4D1URw==\"}"));
+            dteJavaOld.getProperties().put("_ClientEncryptionMetadata2", new EntityProperty(Base64.decode("WznUoytxkvl9KhZ4mNlqkBvRTUHN/D5IgJmNl7kQBOtFBOSgZZrTfZXKH8GjmvKA")));
+
+            testTable.execute(TableOperation.insert(dteJavaOld));
+            
+            TableEncryptionPolicy policy = new TableEncryptionPolicy(sampleKEK, null);
+            TableRequestOptions options = new TableRequestOptions();
+            options.setEncryptionPolicy(policy);
+            options.setEncryptionResolver(new EncryptionResolver() {
+                public boolean encryptionResolver(String pk, String rk, String key) {
+                    return true;
+                }
+            });
+
+            for (DynamicTableEntity dte : testTable.execute(TableQuery.from(DynamicTableEntity.class), options, null))
+            {
+                assertTrue("String not properly decoded.", dte.getProperties().get("sampleProp").getValueAsString().equals("sampleValue"));
+                assertTrue("String not properly decoded.", dte.getProperties().get("sampleProp2").getValueAsString().equals("sampleValue"));
+                assertTrue("String not properly decoded.", dte.getProperties().get("sampleProp3").getValueAsString().equals("sampleValue"));
+                assertEquals("Incorrect number or properties", dte.getProperties().size(), 3);
+            }
+        }
+        finally
+        {
+            if (testTable != null) {
+                testTable.deleteIfExists();
+            }
+        }
     }
 
     private ArrayList<String> listAllTables(CloudTableClient tableClient, String prefix, TableRequestOptions options) throws StorageException
@@ -1096,6 +1167,4 @@ private static DynamicTableEntity generateRandomEntity(String pk) {
         ent.setRowKey(UUID.randomUUID().toString());
         return ent;
     }
-}
-
-
+}

microsoft-azure-storage/src/com/microsoft/azure/storage/Constants.java

@@ -254,6 +254,41 @@ public static class EncryptionConstants
          * Additional property name to store the encryption metadata.
          */
         public static final String TABLE_ENCRYPTION_PROPERTY_DETAILS = "_ClientEncryptionMetadata2";
+        
+        /**
+         * Constant for the key for the encryption mode.
+         */
+        public static final String ENCRYPTION_MODE = "EncryptionMode";
+        
+        /**
+         * FullBlob string constant for the encryption mode.
+         */
+        public static final String FULL_BLOB = "FullBlob";
+        
+        /**
+         * Constant for the key for the wrapped CEK.
+         */
+        public static final String WRAPPED_CONTENT_KEY = "WrappedContentKey";
+                
+        /**
+         * Constant for the key for the encryption agent.
+         */
+        public static final String ENCRYPTION_AGENT = "EncryptionAgent";
+        
+        /**
+         * Constant for the key for the IV.
+         */
+        public static final String CONTENT_ENCRYPTION_IV = "ContentEncryptionIV";
+        
+        /**
+         * Constant for the key wrapping metadata.
+         */
+        public static final String KEY_WRAPPING_METADATA = "KeyWrappingMetadata";
+        
+        /**
+         * Constant for the key for the encryption library in the key wrapping metadata.
+         */
+        public static final String ENCRYPTION_LIBRARY = "EncryptionLibrary";
     }
 
     /**
@@ -611,7 +646,7 @@ public static class HeaderConstants {
         /**
          * Specifies the value to use for UserAgent header.
          */
-        public static final String USER_AGENT_VERSION = "4.3.0";
+        public static final String USER_AGENT_VERSION = "4.4.0";
 
         /**
          * The default type for content-type and accept

microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobEncryptionData.java

@@ -21,6 +21,7 @@
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.JsonToken;
+import com.microsoft.azure.storage.Constants;
 import com.microsoft.azure.storage.core.EncryptionAgent;
 import com.microsoft.azure.storage.core.EncryptionData;
 import com.microsoft.azure.storage.core.JsonUtilities;
@@ -66,7 +67,7 @@ public String serialize() throws IOException {
             generator.writeStartObject();
 
             // write the encryption mode
-            generator.writeStringField("EncryptionMode", "FullBlob");
+            generator.writeStringField(Constants.EncryptionConstants.ENCRYPTION_MODE, Constants.EncryptionConstants.FULL_BLOB);
 
             // write the encryption data
             this.serialize(generator);
@@ -98,18 +99,24 @@ public static BlobEncryptionData deserialize(String inputData)
                 String name = parser.getCurrentName();
                 parser.nextToken();
 
-                if (name.equals("EncryptionMode")) {
+                if (name.equals(Constants.EncryptionConstants.ENCRYPTION_MODE)) {
                     data.setEncryptionMode(parser.getValueAsString());
                 }
-                else if (name.equals("WrappedContentKey")) {
+                else if (name.equals(Constants.EncryptionConstants.WRAPPED_CONTENT_KEY)) {
                     data.setWrappedContentKey(WrappedContentKey.deserialize(parser));
                 } 
-                else if (name.equals("EncryptionAgent")) {
+                else if (name.equals(Constants.EncryptionConstants.ENCRYPTION_AGENT)) {
                     data.setEncryptionAgent(EncryptionAgent.deserialize(parser));
                 }
-                else if (name.equals("ContentEncryptionIV")) {
+                else if (name.equals(Constants.EncryptionConstants.CONTENT_ENCRYPTION_IV)) {
                     data.setContentEncryptionIV(parser.getBinaryValue());
                 }
+                else if (name.equals(Constants.EncryptionConstants.KEY_WRAPPING_METADATA)) {
+                    data.setKeyWrappingMetadata(deserializeKeyWrappingMetadata(parser));
+                }
+                else {
+                    consumeJsonObject(parser);
+                }
                 parser.nextToken();
             }
 

microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobEncryptionPolicy.java

@@ -15,6 +15,7 @@
 package com.microsoft.azure.storage.blob;
 
 import java.io.OutputStream;
+import java.util.HashMap;
 import java.util.Map;
 
 import javax.crypto.Cipher;
@@ -288,6 +289,11 @@ Cipher createAndSetEncryptionContext(Map<String, String> metadata, boolean noPad
             myAes.init(Cipher.ENCRYPT_MODE, aesKey);
 
             BlobEncryptionData encryptionData = new BlobEncryptionData();
+            if (encryptionData.getKeyWrappingMetadata() == null) {
+                encryptionData.setKeyWrappingMetadata(new HashMap<String, String>());
+            }
+            
+            encryptionData.getKeyWrappingMetadata().put(Constants.EncryptionConstants.ENCRYPTION_LIBRARY, "Java " + Constants.HeaderConstants.USER_AGENT_VERSION);
             encryptionData.setEncryptionAgent(new EncryptionAgent(Constants.EncryptionConstants.ENCRYPTION_PROTOCOL_V1,
                     EncryptionAlgorithm.AES_CBC_256));