Skip to content
This repository was archived by the owner on Jul 19, 2024. It is now read-only.

Commit 2ab7205

Browse files
rickle-msftrickle-msft
committed
Finished tests for deep sync copy
1 parent 9548d0f commit 2ab7205

File tree

9 files changed

+219
-22
lines changed

9 files changed

+219
-22
lines changed

‎microsoft-azure-storage-test/res/TestConfigurations.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
<TestConfigurations>
22
<TargetTestTenant>ProductionTenant</TargetTestTenant>
33
<TargetPremiumBlobTenant>ProductionTenant</TargetPremiumBlobTenant>
4+
<TargetCopySourceTenant>ProductionTenant</TargetCopySourceTenant>
45
<TenantConfigurations>
56
<TenantConfiguration>
67
<TenantName>DevStore</TenantName>

‎microsoft-azure-storage-test/src/com/microsoft/azure/storage/TestHelper.java‎

Lines changed: 103 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -72,8 +72,11 @@ public class TestHelper {
7272
private static Tenant premiumBlobTenant;
7373
private static StorageCredentialsAccountAndKey premiumBlobCredentials;
7474
private static CloudStorageAccount premiumBlobAccount;
75+
private static Tenant copySourceTenant;
76+
private static StorageCredentialsAccountAndKey copySourceCredentials;
77+
private static CloudStorageAccount copySourceAccount;
7578

76-
private final static boolean enableFiddler = false;
79+
private final static boolean enableFiddler = true;
7780
private final static boolean requireSecondaryEndpoint = false;
7881

7982
public static String generateOAuthToken() throws MalformedURLException, InterruptedException, ExecutionException, ServiceUnavailableException, StorageException {
@@ -127,6 +130,11 @@ public static CloudBlobClient createPremiumCloudBlobClient() throws StorageExcep
127130
return client;
128131
}
129132

133+
public static CloudBlobClient createCopySourceBlobClient() throws StorageException {
134+
CloudBlobClient client = getCopySourceAccount().createCloudBlobClient();
135+
return client;
136+
}
137+
130138
public static CloudBlobClient createCloudBlobClient(SharedAccessAccountPolicy policy, boolean useHttps)
131139
throws StorageException, InvalidKeyException, URISyntaxException {
132140

@@ -388,12 +396,12 @@ public static CloudStorageAccount getAccount() throws StorageException {
388396
account = CloudStorageAccount.parse(cloudAccount);
389397
}
390398
else if (accountConfig != null) {
391-
readTestConfigsFromXml(new File(accountConfig), false);
399+
readTestConfigsFromXml(new File(accountConfig), false, false);
392400
setAccountAndCredentials();
393401
}
394402
else {
395403
URL localTestConfig = TestHelper.class.getClassLoader().getResource("TestConfigurations.xml");
396-
readTestConfigsFromXml(new File(localTestConfig.getPath()), false);
404+
readTestConfigsFromXml(new File(localTestConfig.getPath()), false, false);
397405
setAccountAndCredentials();
398406
}
399407
}
@@ -425,12 +433,12 @@ private static CloudStorageAccount getPremiumBlobAccount() throws StorageExcepti
425433
// if neither are set, use the local configurations file at TestConfigurations.xml
426434
try {
427435
if (accountConfig != null) {
428-
readTestConfigsFromXml(new File(accountConfig), true);
436+
readTestConfigsFromXml(new File(accountConfig), true, false);
429437
setAccountAndCredentials();
430438
}
431439
else {
432440
URL localTestConfig = TestHelper.class.getClassLoader().getResource("TestConfigurations.xml");
433-
readTestConfigsFromXml(new File(localTestConfig.getPath()), true);
441+
readTestConfigsFromXml(new File(localTestConfig.getPath()), true, false);
434442
setAccountAndCredentials();
435443
}
436444
}
@@ -445,6 +453,47 @@ private static CloudStorageAccount getPremiumBlobAccount() throws StorageExcepti
445453
return premiumBlobAccount;
446454
}
447455

456+
private static CloudStorageAccount getCopySourceAccount() throws StorageException {
457+
// Only do this the first time TestBase is called as storage account is static
458+
if (copySourceAccount == null) {
459+
//enable fiddler
460+
if (enableFiddler)
461+
enableFiddler();
462+
463+
// try to get the environment variable with the test configuration file path
464+
String accountConfig;
465+
try {
466+
accountConfig = System.getenv("storageTestConfiguration");
467+
}
468+
catch (SecurityException e) {
469+
accountConfig = null;
470+
}
471+
472+
// if storageConnection is set, use that as an account string
473+
// if storageTestConfiguration is set, use that as a path to the configurations file
474+
// if neither are set, use the local configurations file at TestConfigurations.xml
475+
try {
476+
if (accountConfig != null) {
477+
readTestConfigsFromXml(new File(accountConfig), false, true);
478+
setAccountAndCredentials();
479+
}
480+
else {
481+
URL localTestConfig = TestHelper.class.getClassLoader().getResource("TestConfigurations.xml");
482+
readTestConfigsFromXml(new File(localTestConfig.getPath()), false, true);
483+
setAccountAndCredentials();
484+
}
485+
}
486+
catch (AssumptionViolatedException e) {
487+
throw e;
488+
}
489+
catch (Exception e) {
490+
throw StorageException.translateClientException(e);
491+
}
492+
}
493+
494+
return copySourceAccount;
495+
}
496+
448497
private static void setAccountAndCredentials() {
449498
if (requireSecondaryEndpoint)
450499
tenant.assertSecondaryEndpoint();
@@ -462,9 +511,17 @@ private static void setAccountAndCredentials() {
462511
null,
463512
null);
464513
}
514+
515+
if (copySourceTenant != null) {
516+
copySourceCredentials = new StorageCredentialsAccountAndKey(copySourceTenant.getAccountName(), copySourceTenant.getAccountKey());
517+
copySourceAccount = new CloudStorageAccount(copySourceCredentials, new StorageUri(copySourceTenant.getBlobServiceEndpoint(), copySourceTenant.getBlobServiceSecondaryEndpoint()),
518+
null,
519+
null,
520+
null);
521+
}
465522
}
466523

467-
private static void readTestConfigsFromXml(File testConfigurations, boolean premiumBlob) throws ParserConfigurationException,
524+
private static void readTestConfigsFromXml(File testConfigurations, boolean premiumBlob, boolean copyBlob) throws ParserConfigurationException,
468525
SAXException, IOException, DOMException, URISyntaxException {
469526

470527
DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
@@ -487,8 +544,15 @@ private static void readTestConfigsFromXml(File testConfigurations, boolean prem
487544
premiumBlobTenantName = premiumBlobTenantNode.getTextContent();
488545
}
489546

547+
Node copySourceBlobTenantNode = testConfigs.getElementsByTagName("TargetCopySourceTenant").item(0);
548+
String copySourceBlobTenantName = null;
549+
if (copySourceBlobTenantNode != null) {
550+
copySourceBlobTenantName = copySourceBlobTenantNode.getTextContent();
551+
}
552+
490553
tenant = null;
491554
premiumBlobTenant = null;
555+
copySourceTenant = null;
492556
final NodeList tenantNodes = testConfigs.getElementsByTagName("TenantName");
493557
for (int i = 0; i < tenantNodes.getLength(); i++) {
494558
if (tenantNodes.item(i).getTextContent().equals(targetTenant)) {
@@ -597,6 +661,35 @@ else if (!premiumBlob){
597661
}
598662
}
599663
}
664+
665+
if (tenantNodes.item(i).getTextContent().equals(copySourceBlobTenantName)) {
666+
copySourceTenant = new Tenant();
667+
Node parent = tenantNodes.item(i).getParentNode();
668+
final NodeList childNodes = parent.getChildNodes();
669+
for (int j = 0; j < childNodes.getLength(); j++) {
670+
final Node node = childNodes.item(j);
671+
672+
if (node.getNodeType() != Node.ELEMENT_NODE) {
673+
// do nothing
674+
} else {
675+
final String name = node.getNodeName();
676+
677+
if (name.equals("TenantName")) {
678+
copySourceTenant.setTenantName(node.getTextContent());
679+
} else if (name.equals("TenantType")) {
680+
// do nothing, we don't track this field
681+
} else if (name.equals("AccountName")) {
682+
copySourceTenant.setAccountName(node.getTextContent());
683+
} else if (name.equals("AccountKey")) {
684+
copySourceTenant.setAccountKey(node.getTextContent());
685+
} else if (name.equals("BlobServiceEndpoint")) {
686+
copySourceTenant.setBlobServiceEndpoint(new URI(node.getTextContent()));
687+
} else if (name.equals("BlobServiceSecondaryEndpoint")) {
688+
copySourceTenant.setBlobServiceSecondaryEndpoint(new URI(node.getTextContent()));
689+
}
690+
}
691+
}
692+
}
600693
}
601694

602695
if (tenant == null && !premiumBlob) {
@@ -606,5 +699,9 @@ else if (!premiumBlob){
606699
if (premiumBlobTenant == null && premiumBlob) {
607700
assumeNotNull(premiumBlobTenant);
608701
}
702+
703+
if (copySourceTenant == null && copyBlob) {
704+
assumeNotNull(copySourceTenant);
705+
}
609706
}
610707
}

‎microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/BlobTestHelper.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,14 @@ public static CloudBlobContainer getRandomPremiumBlobContainerReference() throws
6060
return container;
6161
}
6262

63+
public static CloudBlobContainer getRandomCopySourceContainerReference() throws URISyntaxException, StorageException {
64+
String containerName = generateRandomContainerName();
65+
CloudBlobClient bClient = TestHelper.createCopySourceBlobClient();
66+
CloudBlobContainer container = bClient.getContainerReference(containerName);
67+
68+
return container;
69+
}
70+
6371
public static String generateRandomBlobNameWithPrefix(String prefix) {
6472
if (prefix == null) {
6573
prefix = "";

‎microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudBlockBlobTests.java‎

Lines changed: 96 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
package com.microsoft.azure.storage.blob;
1616

1717
import com.microsoft.azure.storage.*;
18+
import com.microsoft.azure.storage.core.Base64;
1819
import com.microsoft.azure.storage.core.Utility;
1920
import com.microsoft.azure.storage.file.CloudFile;
2021
import com.microsoft.azure.storage.file.CloudFileShare;
@@ -39,6 +40,8 @@
3940
import java.net.URI;
4041
import java.net.URISyntaxException;
4142
import java.security.InvalidKeyException;
43+
import java.security.MessageDigest;
44+
import java.security.NoSuchAlgorithmException;
4245
import java.util.*;
4346

4447
import com.microsoft.azure.storage.TestRunners.CloudTests;
@@ -605,6 +608,79 @@ public void testCopyBlockBlobSyncTest() throws InterruptedException, IOException
605608
this.doCloudBlockBlobCopy(false, false, true);
606609
}
607610

611+
@Test
612+
public void testCopyBlockBlobWithSyncMd5() throws URISyntaxException, StorageException, IOException, NoSuchAlgorithmException {
613+
// Create source on server. We must copy from a different account or the service will not validate the md5.
614+
CloudBlobContainer container = BlobTestHelper.getRandomCopySourceContainerReference();
615+
container.create(BlobContainerPublicAccessType.CONTAINER, null, null);
616+
CloudBlockBlob source = container.getBlockBlobReference("source");
617+
618+
String data = "String data";
619+
source.uploadText(data, Constants.UTF8_CHARSET, null, null, null);
620+
621+
source.getMetadata().put("Test", "value");
622+
source.uploadMetadata();
623+
624+
// Get destination reference
625+
CloudBlockBlob destination = this.container.getBlockBlobReference("destination");
626+
destination.commitBlockList(new ArrayList<BlockEntry>());
627+
628+
String copyId = null;
629+
boolean exceptionThrown = false;
630+
// Test that setting an md5 without sync copy is disallowed.
631+
try {
632+
destination.startCopy(source, "md5", false, null, null, null, null);
633+
}
634+
catch(StorageException e) {
635+
assertTrue(e.getCause() instanceof IllegalArgumentException);
636+
assertTrue(e.getMessage().contains("MD5"));
637+
exceptionThrown = true;
638+
}
639+
assertTrue(exceptionThrown);
640+
641+
// Test that the md5 header is set correctly by using an incorrect md5.
642+
exceptionThrown = false;
643+
try {
644+
destination.startCopy(source, Base64.encode(MessageDigest.getInstance("MD5").digest("garbage".getBytes())),
645+
true, null, null, null, null);
646+
}
647+
catch (StorageException e) {
648+
exceptionThrown = true;
649+
assertEquals("Md5Mismatch", e.getErrorCode());
650+
String md5 = Base64.encode(MessageDigest.getInstance("MD5").digest(data.getBytes()));
651+
652+
// Start a sync copy with a correct md5. Should succeed.
653+
copyId = destination.startCopy(source, md5,true, null, null, null, null);
654+
}
655+
assertTrue(exceptionThrown);
656+
657+
Calendar calendar = Calendar.getInstance(Utility.UTC_ZONE);
658+
destination.downloadAttributes();
659+
660+
source.downloadAttributes();
661+
assertNotNull(destination.getProperties().getEtag());
662+
assertFalse(source.getProperties().getEtag().equals(destination.getProperties().getEtag()));
663+
assertTrue(destination.getProperties().getLastModified().compareTo(new Date(calendar.get(Calendar.MINUTE) - 1)) > 0);
664+
665+
String copyData = destination.downloadText(Constants.UTF8_CHARSET, null, null, null);
666+
assertEquals(data, copyData);
667+
668+
BlobProperties prop1 = destination.getProperties();
669+
BlobProperties prop2 = source.getProperties();
670+
671+
assertEquals(prop1.getCacheControl(), prop2.getCacheControl());
672+
assertEquals(prop1.getContentEncoding(), prop2.getContentEncoding());
673+
assertEquals(prop1.getContentLanguage(), prop2.getContentLanguage());
674+
assertEquals(prop1.getContentMD5(), prop2.getContentMD5());
675+
assertEquals(prop1.getContentType(), prop2.getContentType());
676+
677+
assertEquals("value", destination.getMetadata().get("Test"));
678+
679+
destination.delete();
680+
source.delete();
681+
container.deleteIfExists();
682+
}
683+
608684
@Test
609685
public void testCopyWithChineseChars() throws StorageException, IOException, URISyntaxException {
610686
String data = "sample data chinese chars 阿䶵";
@@ -637,8 +713,8 @@ public void eventOccurred(SendingRequestEvent eventArg) {
637713
}
638714
});
639715

640-
copyDestination.startCopy(copySource.getUri(), false, null, null, null, ctx);
641-
copyDestination.startCopy(copySource, false, null, null, null, ctx);
716+
copyDestination.startCopy(copySource.getUri(), null, false, null, null, null, ctx);
717+
copyDestination.startCopy(copySource, null, false, null, null, null, ctx);
642718
}
643719

644720
@Test
@@ -1166,7 +1242,7 @@ public void testBlobDownloadRangeValidationTest() throws StorageException, URISy
11661242

11671243
@Test
11681244
@Category({ DevFabricTests.class, DevStoreTests.class })
1169-
public void testUploadBlockFromURI() throws URISyntaxException, StorageException, IOException {
1245+
public void testUploadBlockFromURI() throws URISyntaxException, StorageException, IOException, NoSuchAlgorithmException {
11701246
CloudBlobContainer container = BlobTestHelper.getRandomContainerReference();
11711247
container.create(BlobContainerPublicAccessType.CONTAINER, null, null);
11721248
final CloudBlockBlob blob = container.getBlockBlobReference(BlobTestHelper
@@ -1181,14 +1257,24 @@ public void testUploadBlockFromURI() throws URISyntaxException, StorageException
11811257
final CloudBlockBlob blob2 = container.getBlockBlobReference(BlobTestHelper
11821258
.generateRandomBlobNameWithPrefix("copyBlob"));
11831259
Map<String, BlockEntry> blocks = BlobTestHelper.getBlockEntryList(2);
1184-
int i=0;
1185-
for (BlockEntry block : blocks.values()) {
1186-
blob2.uploadBlockFromURI(block.getId(), blob.getUri(), i*(text.length()/blocks.values().size()),
1187-
(long) (text.length()/blocks.values().size()));
1188-
i++;
1260+
// Copy the first half of the blob.
1261+
blob2.uploadBlockFromURI(((BlockEntry)blocks.values().toArray()[0]).getId(), blob.getUri(), 0, 5L);
1262+
1263+
// Copy the second half of the blob, specifying the MD5 and setting null for the length to indicate the remainder of the blob.
1264+
String md5 = Base64.encode(MessageDigest.getInstance("MD5").digest(text.substring(5).getBytes()));
1265+
boolean exceptionThrown = false;
1266+
try {
1267+
blob2.uploadBlockFromURI(((BlockEntry) blocks.values().toArray()[1]).getId(), blob.getUri(), 5, null,
1268+
Base64.encode(MessageDigest.getInstance("MD5").digest("garbage".getBytes())), null, null, null);
11891269
}
1190-
blob2.commitBlockList(blocks.values());
1270+
catch (StorageException e) {
1271+
exceptionThrown = true;
1272+
assertEquals("Md5Mismatch", e.getErrorCode());
1273+
blob2.uploadBlockFromURI(((BlockEntry) blocks.values().toArray()[1]).getId(), blob.getUri(), 5, null, md5, null, null, null);
1274+
}
1275+
assertTrue(exceptionThrown);
11911276

1277+
blob2.commitBlockList(blocks.values());
11921278
assertEquals(blob2.downloadText(), text);
11931279

11941280
container.deleteIfExists();
@@ -2333,7 +2419,7 @@ private void doCloudBlockBlobCopy(boolean sourceIsSas, boolean destinationIsSas,
23332419
Thread.sleep(30000);
23342420

23352421
// Start copy and wait for completion
2336-
String copyId = copyDestination.startCopy(copySource, syncCopy, null, null, null, null);
2422+
String copyId = copyDestination.startCopy(copySource, null, syncCopy, null, null, null, null);
23372423
BlobTestHelper.waitForCopy(copyDestination);
23382424
Calendar calendar = Calendar.getInstance(Utility.UTC_ZONE);
23392425
destination.downloadAttributes();

‎microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudPageBlobTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1390,7 +1390,7 @@ public void testCloudPageBlobSetBlobTierOnCopy() throws URISyntaxException, Stor
13901390

13911391
// copy to larger disk
13921392
CloudPageBlob copy = container.getPageBlobReference("copy");
1393-
copy.startCopy(TestHelper.defiddler(source.getUri()), false, PremiumPageBlobTier.P30, null, null, null, null);
1393+
copy.startCopy(TestHelper.defiddler(source.getUri()), null,false, PremiumPageBlobTier.P30, null, null, null, null);
13941394
assertEquals(BlobType.PAGE_BLOB, copy.getProperties().getBlobType());
13951395
assertEquals(PremiumPageBlobTier.P30, copy.getProperties().getPremiumPageBlobTier());
13961396
assertEquals(PremiumPageBlobTier.P10, source.getProperties().getPremiumPageBlobTier());
@@ -1412,7 +1412,7 @@ public void testCloudPageBlobSetBlobTierOnCopy() throws URISyntaxException, Stor
14121412
source2.create(1024);
14131413

14141414
CloudPageBlob copy3 = container.getPageBlobReference("copy3");
1415-
copy3.startCopy(TestHelper.defiddler(source2.getUri()), false, PremiumPageBlobTier.P60, null ,null ,null, null);
1415+
copy3.startCopy(TestHelper.defiddler(source2.getUri()), null,false, PremiumPageBlobTier.P60, null ,null ,null, null);
14161416
assertEquals(BlobType.PAGE_BLOB, copy3.getProperties().getBlobType());
14171417
assertEquals(PremiumPageBlobTier.P60, copy3.getProperties().getPremiumPageBlobTier());
14181418
assertNull(source2.getProperties().getPremiumPageBlobTier());

‎microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobRequest.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,10 +219,15 @@ public static HttpURLConnection appendBlock(final URI uri, final BlobRequestOpti
219219
* An {@link AccessCondition} object that represents the access conditions for the destination blob.
220220
* @param source
221221
* The canonical path to the source blob, in the form /<account-name>/<container-name>/<blob-name>.
222+
* @param contentMd5
223+
* An optional hash value used to ensure transactional integrity for the operation. May be {@code null}
224+
* or an empty string.
222225
* @param sourceSnapshotID
223226
* The snapshot version, if the source blob is a snapshot.
224227
* @param incrementalCopy
225228
* A boolean indicating whether or not this is an incremental copy.
229+
* @param syncCopy
230+
* A boolean to enable synchronous server copy of blobs.
226231
* @param premiumPageBlobTier
227232
* A {@link PremiumPageBlobTier} object which represents the tier of the blob.
228233
* @return a HttpURLConnection configured for the operation.

0 commit comments

Comments
 (0)