Skip to content
This repository was archived by the owner on Jul 19, 2024. It is now read-only.

Commit f4150e6

Browse files
rickle-msftrickle-msft
committed
Deep sync copy changes made. Need to finish tests and add MD5 support
1 parent 0bd50d4 commit f4150e6

File tree

9 files changed

+348
-30
lines changed

9 files changed

+348
-30
lines changed

‎microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudBlockBlobTests.java‎

Lines changed: 45 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -574,28 +574,35 @@ public void testBlobUriOnlyConstructors() throws URISyntaxException, StorageExce
574574
@Category(SlowTests.class)
575575
public void testCopyBlockBlobSasToSasTest() throws InvalidKeyException, URISyntaxException, StorageException,
576576
IOException, InterruptedException {
577-
this.doCloudBlockBlobCopy(true, true);
577+
this.doCloudBlockBlobCopy(true, true, false);
578578
}
579579

580580
@Test
581581
@Category(SlowTests.class)
582582
public void testCopyBlockBlobToSasTest() throws InvalidKeyException, URISyntaxException, StorageException,
583583
IOException, InterruptedException {
584-
this.doCloudBlockBlobCopy(false, true);
584+
this.doCloudBlockBlobCopy(false, true, false);
585585
}
586586

587587
@Test
588588
@Category(SlowTests.class)
589589
public void testCopyBlockBlobSasTest() throws InvalidKeyException, URISyntaxException, StorageException,
590590
IOException, InterruptedException {
591-
this.doCloudBlockBlobCopy(true, false);
591+
this.doCloudBlockBlobCopy(true, false, false);
592592
}
593593

594594
@Test
595595
@Category({ DevFabricTests.class, DevStoreTests.class, SlowTests.class })
596596
public void testCopyBlockBlobTest() throws InvalidKeyException, URISyntaxException, StorageException, IOException,
597597
InterruptedException {
598-
this.doCloudBlockBlobCopy(false, false);
598+
this.doCloudBlockBlobCopy(false, false, false);
599+
}
600+
601+
@Test
602+
@Category({ DevFabricTests.class, DevStoreTests.class, SlowTests.class })
603+
public void testCopyBlockBlobSyncTest() throws InterruptedException, IOException, StorageException,
604+
InvalidKeyException, URISyntaxException {
605+
this.doCloudBlockBlobCopy(false, false, true);
599606
}
600607

601608
@Test
@@ -630,8 +637,8 @@ public void eventOccurred(SendingRequestEvent eventArg) {
630637
}
631638
});
632639

633-
copyDestination.startCopy(copySource.getUri(), null, null, null, ctx);
634-
copyDestination.startCopy(copySource, null, null, null, ctx);
640+
copyDestination.startCopy(copySource.getUri(), false, null, null, null, ctx);
641+
copyDestination.startCopy(copySource, false, null, null, null, ctx);
635642
}
636643

637644
@Test
@@ -1157,6 +1164,36 @@ public void testBlobDownloadRangeValidationTest() throws StorageException, URISy
11571164
assertEquals(length, blockBlobRef.getProperties().getLength());
11581165
}
11591166

1167+
@Test
1168+
@Category({ DevFabricTests.class, DevStoreTests.class })
1169+
public void testUploadBlockFromURI() throws URISyntaxException, StorageException, IOException {
1170+
CloudBlobContainer container = BlobTestHelper.getRandomContainerReference();
1171+
container.create(BlobContainerPublicAccessType.CONTAINER, null, null);
1172+
final CloudBlockBlob blob = container.getBlockBlobReference(BlobTestHelper
1173+
.generateRandomBlobNameWithPrefix("testBlob"));
1174+
1175+
String text = "Test data.";
1176+
1177+
// create
1178+
blob.uploadText(text);
1179+
assertTrue(blob.exists());
1180+
1181+
final CloudBlockBlob blob2 = container.getBlockBlobReference(BlobTestHelper
1182+
.generateRandomBlobNameWithPrefix("copyBlob"));
1183+
Map<String, BlockEntry> blocks = BlobTestHelper.getBlockEntryList(2);
1184+
int i=0;
1185+
for (BlockEntry block : blocks.values()) {
1186+
blob2.uploadBlockFromURI(block.getId(), blob.getUri(), i*(text.length()/blocks.values().size()),
1187+
(long) (text.length()/blocks.values().size()));
1188+
i++;
1189+
}
1190+
blob2.commitBlockList(blocks.values());
1191+
1192+
assertEquals(blob2.downloadText(), text);
1193+
1194+
container.deleteIfExists();
1195+
}
1196+
11601197
@Test
11611198
@Category({ DevFabricTests.class, DevStoreTests.class })
11621199
public void testCommitBlockListContentMd5() throws URISyntaxException, StorageException, IOException {
@@ -2230,7 +2267,7 @@ private String getRandomString(int length, int maxCodePoint) {
22302267
return new String(codePoints, 0, length);
22312268
}
22322269

2233-
private void doCloudBlockBlobCopy(boolean sourceIsSas, boolean destinationIsSas) throws URISyntaxException,
2270+
private void doCloudBlockBlobCopy(boolean sourceIsSas, boolean destinationIsSas, boolean syncCopy) throws URISyntaxException,
22342271
StorageException, IOException, InvalidKeyException, InterruptedException {
22352272

22362273
// Create source on server.
@@ -2296,7 +2333,7 @@ private void doCloudBlockBlobCopy(boolean sourceIsSas, boolean destinationIsSas)
22962333
Thread.sleep(30000);
22972334

22982335
// Start copy and wait for completion
2299-
String copyId = copyDestination.startCopy(copySource);
2336+
String copyId = copyDestination.startCopy(copySource, syncCopy, null, null, null, null);
23002337
BlobTestHelper.waitForCopy(copyDestination);
23012338
Calendar calendar = Calendar.getInstance(Utility.UTC_ZONE);
23022339
destination.downloadAttributes();

‎microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudPageBlobTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1390,7 +1390,7 @@ public void testCloudPageBlobSetBlobTierOnCopy() throws URISyntaxException, Stor
13901390

13911391
// copy to larger disk
13921392
CloudPageBlob copy = container.getPageBlobReference("copy");
1393-
copy.startCopy(TestHelper.defiddler(source.getUri()), PremiumPageBlobTier.P30, null, null, null, null);
1393+
copy.startCopy(TestHelper.defiddler(source.getUri()), false, PremiumPageBlobTier.P30, null, null, null, null);
13941394
assertEquals(BlobType.PAGE_BLOB, copy.getProperties().getBlobType());
13951395
assertEquals(PremiumPageBlobTier.P30, copy.getProperties().getPremiumPageBlobTier());
13961396
assertEquals(PremiumPageBlobTier.P10, source.getProperties().getPremiumPageBlobTier());
@@ -1412,7 +1412,7 @@ public void testCloudPageBlobSetBlobTierOnCopy() throws URISyntaxException, Stor
14121412
source2.create(1024);
14131413

14141414
CloudPageBlob copy3 = container.getPageBlobReference("copy3");
1415-
copy3.startCopy(TestHelper.defiddler(source2.getUri()), PremiumPageBlobTier.P60, null ,null ,null, null);
1415+
copy3.startCopy(TestHelper.defiddler(source2.getUri()), false, PremiumPageBlobTier.P60, null ,null ,null, null);
14161416
assertEquals(BlobType.PAGE_BLOB, copy3.getProperties().getBlobType());
14171417
assertEquals(PremiumPageBlobTier.P60, copy3.getProperties().getPremiumPageBlobTier());
14181418
assertNull(source2.getProperties().getPremiumPageBlobTier());

‎microsoft-azure-storage/src/com/microsoft/azure/storage/Constants.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -615,6 +615,11 @@ public static class HeaderConstants {
615615
*/
616616
public static final String REQUEST_ID_HEADER = PREFIX_FOR_STORAGE_HEADER + "request-id";
617617

618+
/**
619+
* The header for copy sync.
620+
*/
621+
public static final String REQUIRES_SYNC_HEADER = PREFIX_FOR_STORAGE_HEADER + "requires-sync";
622+
618623
/**
619624
* The header field value received that indicates which server was accessed
620625
*/
@@ -640,6 +645,11 @@ public static class HeaderConstants {
640645
*/
641646
public static final String SNAPSHOT_ID_HEADER = PREFIX_FOR_STORAGE_HEADER + "snapshot";
642647

648+
/**
649+
* The header that specifies source content MD5.
650+
*/
651+
public static final String SOURCE_CONTENT_MD5_HEADER = PREFIX_FOR_STORAGE_HEADER + "source-content-md5";
652+
643653
/**
644654
* The header for the If-Match condition.
645655
*/
@@ -672,6 +682,11 @@ public static class HeaderConstants {
672682
*/
673683
public static final String STORAGE_RANGE_HEADER = PREFIX_FOR_STORAGE_HEADER + "range";
674684

685+
/**
686+
* The header for data ranges.
687+
*/
688+
public static final String STORAGE_SOURCE_RANGE_HEADER = PREFIX_FOR_STORAGE_HEADER + "source-range";
689+
675690
/**
676691
* The header for storage version.
677692
*/

‎microsoft-azure-storage/src/com/microsoft/azure/storage/blob/BlobRequest.java‎

Lines changed: 101 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ public static HttpURLConnection appendBlock(final URI uri, final BlobRequestOpti
235235
public static HttpURLConnection copyFrom(final URI uri, final BlobRequestOptions blobOptions,
236236
final OperationContext opContext, final AccessCondition sourceAccessCondition,
237237
final AccessCondition destinationAccessCondition, String source, final String sourceSnapshotID,
238-
final boolean incrementalCopy, final PremiumPageBlobTier premiumPageBlobTier)
238+
final boolean incrementalCopy, final boolean syncCopy, final PremiumPageBlobTier premiumPageBlobTier)
239239
throws StorageException, IOException, URISyntaxException {
240240

241241
if (sourceSnapshotID != null) {
@@ -270,6 +270,10 @@ public static HttpURLConnection copyFrom(final URI uri, final BlobRequestOptions
270270
destinationAccessCondition.applyConditionToRequest(request);
271271
}
272272

273+
if (syncCopy) {
274+
request.setRequestProperty(HeaderConstants.REQUIRES_SYNC_HEADER, Constants.TRUE);
275+
}
276+
273277
return request;
274278
}
275279

@@ -549,7 +553,7 @@ public static HttpURLConnection getBlob(final URI uri, final BlobRequestOptions
549553

550554
/**
551555
* Adds the Range Header for Blob Service Operations.
552-
*
556+
*
553557
* @param request
554558
* The request to add the range header to.
555559
* @param offset
@@ -558,16 +562,44 @@ public static HttpURLConnection getBlob(final URI uri, final BlobRequestOptions
558562
* Number of bytes in the range.
559563
*/
560564
private static void addRange(HttpURLConnection request, Long offset, Long count) {
565+
addRangeImpl(Constants.HeaderConstants.STORAGE_RANGE_HEADER, request, offset, count);
566+
}
567+
568+
/**
569+
* Adds the Range Header for Blob Service Operations.
570+
*
571+
* @param request
572+
* The request to add the range header to.
573+
* @param offset
574+
* Starting byte of the range.
575+
* @param count
576+
* Number of bytes in the range.
577+
*/
578+
private static void addSourceRange(HttpURLConnection request, Long offset, Long count) {
579+
addRangeImpl(HeaderConstants.STORAGE_SOURCE_RANGE_HEADER, request, offset, count);
580+
}
581+
582+
/**
583+
* Adds the Range Header for Blob Service Operations.
584+
*
585+
* @param request
586+
* The request to add the range header to.
587+
* @param offset
588+
* Starting byte of the range.
589+
* @param count
590+
* Number of bytes in the range.
591+
*/
592+
private static void addRangeImpl(String header, HttpURLConnection request, Long offset, Long count) {
561593
if (offset != null) {
562594
long rangeStart = offset;
563595
long rangeEnd;
564596
if (count != null) {
565597
rangeEnd = offset + count - 1;
566-
request.setRequestProperty(Constants.HeaderConstants.STORAGE_RANGE_HEADER, String.format(
598+
request.setRequestProperty(header, String.format(
567599
Utility.LOCALE_US, Constants.HeaderConstants.RANGE_HEADER_FORMAT, rangeStart, rangeEnd));
568600
}
569601
else {
570-
request.setRequestProperty(Constants.HeaderConstants.STORAGE_RANGE_HEADER, String.format(
602+
request.setRequestProperty(header, String.format(
571603
Utility.LOCALE_US, Constants.HeaderConstants.BEGIN_RANGE_HEADER_FORMAT, rangeStart));
572604
}
573605
}
@@ -1309,6 +1341,71 @@ public static HttpURLConnection putBlock(final URI uri, final BlobRequestOptions
13091341

13101342
return request;
13111343
}
1344+
1345+
/**
1346+
* Constructs a HttpURLConnection to upload a block. Sign with length of block data.
1347+
*
1348+
* @param uri
1349+
* A <code>java.net.URI</code> object that specifies the absolute URI.
1350+
* @param source
1351+
* The <code>String</code> form of a URI to the source data. It can point to any Azure Blob or File that
1352+
* is public or the URL can include a shared access signature.
1353+
* @param offset
1354+
* A <code>long</code> which represents the offset to use as the starting point for the source.
1355+
* @param length
1356+
* A <code>Long</code> which represents the number of bytes to copy or <code>null</code> to copy until the
1357+
* end of the blob.
1358+
* @param md5
1359+
* A <code>String</code> which represents the MD5 caluclated for the range of bytes of the source.
1360+
* @param blobOptions
1361+
* A {@link BlobRequestOptions} object that specifies execution options such as retry policy and timeout
1362+
* settings for the operation. Specify <code>null</code> to use the request options specified on the
1363+
* {@link CloudBlobClient}.
1364+
* @param opContext
1365+
* An {@link OperationContext} object that represents the context for the current operation. This object
1366+
* is used to track requests to the storage service, and to provide additional runtime information about
1367+
* the operation.
1368+
* @param accessCondition
1369+
* An {@link AccessCondition} object that represents the access conditions for the blob.
1370+
* @param blockId
1371+
* the Base64 ID for the block
1372+
* @return a HttpURLConnection to use to perform the operation.
1373+
* @throws IOException
1374+
* if there is an error opening the connection
1375+
* @throws URISyntaxException
1376+
* if the resource URI is invalid
1377+
* @throws StorageException
1378+
* an exception representing any error which occurred during the operation.
1379+
* @throws IllegalArgumentException
1380+
*/
1381+
public static HttpURLConnection putBlock(final URI uri, final String source, long offset, Long length,
1382+
final BlobRequestOptions blobOptions, String md5,
1383+
final OperationContext opContext, final AccessCondition accessCondition,
1384+
final String blockId)
1385+
throws IOException, URISyntaxException, StorageException {
1386+
final UriQueryBuilder builder = new UriQueryBuilder();
1387+
builder.add(Constants.QueryConstants.COMPONENT, BLOCK_QUERY_ELEMENT_NAME);
1388+
builder.add(BLOCK_ID_QUERY_ELEMENT_NAME, blockId);
1389+
1390+
final HttpURLConnection request = createURLConnection(uri, builder, blobOptions, opContext);
1391+
1392+
request.setDoOutput(true);
1393+
request.setRequestMethod(Constants.HTTP_PUT);
1394+
1395+
request.setFixedLengthStreamingMode(0);
1396+
request.setRequestProperty(Constants.HeaderConstants.CONTENT_LENGTH, "0");
1397+
request.setRequestProperty(Constants.HeaderConstants.COPY_SOURCE_HEADER, source);
1398+
1399+
if (accessCondition != null) {
1400+
accessCondition.applyConditionToRequest(request);
1401+
}
1402+
1403+
addSourceRange(request, offset, length);
1404+
1405+
BaseRequest.addOptionalHeader(request, HeaderConstants.SOURCE_CONTENT_MD5_HEADER, md5);
1406+
1407+
return request;
1408+
}
13121409

13131410
/**
13141411
* Constructs a HttpURLConnection to set the the tier on a page blob.

‎microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudAppendBlob.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ public final String startCopy(final CloudAppendBlob sourceBlob, final AccessCond
224224
source = sourceBlob.getServiceClient().getCredentials().transformUri(sourceBlob.getSnapshotQualifiedUri());
225225
}
226226

227-
return this.startCopy(source, sourceAccessCondition, destinationAccessCondition, options, opContext);
227+
return this.startCopy(source, false, sourceAccessCondition, destinationAccessCondition, options, opContext);
228228
}
229229

230230
/**

‎microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlob.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -629,7 +629,7 @@ public String preProcessResponse(CloudBlob blob, CloudBlobClient client, Operati
629629
*/
630630
@DoesServiceRequest
631631
public final String startCopy(final URI source) throws StorageException {
632-
return this.startCopy(source, null /* sourceAccessCondition */, null /* destinationAccessCondition */,
632+
return this.startCopy(source, false /* sync copy */, null /* sourceAccessCondition */, null /* destinationAccessCondition */,
633633
null /* options */, null /* opContext */);
634634
}
635635

@@ -660,10 +660,10 @@ public final String startCopy(final URI source) throws StorageException {
660660
*
661661
*/
662662
@DoesServiceRequest
663-
public final String startCopy(final URI source, final AccessCondition sourceAccessCondition,
663+
public final String startCopy(final URI source, boolean syncCopy, final AccessCondition sourceAccessCondition,
664664
final AccessCondition destinationAccessCondition, BlobRequestOptions options, OperationContext opContext)
665665
throws StorageException {
666-
return this.startCopy(source, null /* premiumPageBlobTier */, sourceAccessCondition, destinationAccessCondition, options, opContext);
666+
return this.startCopy(source, syncCopy, null /* premiumPageBlobTier */, sourceAccessCondition, destinationAccessCondition, options, opContext);
667667
}
668668

669669
/**
@@ -675,6 +675,8 @@ public final String startCopy(final URI source, final AccessCondition sourceAcce
675675
* @param source
676676
* A <code>java.net.URI</code> The source URI. URIs for resources outside of Azure
677677
* may only be copied into block blobs.
678+
* @param syncCopy
679+
* A <code>boolean</code> which indicates if the copy should be done synchronously on the service.
678680
* @param premiumPageBlobTier
679681
* A {@link PremiumPageBlobTier} object which represents the tier of the blob.
680682
* @param sourceAccessCondition
@@ -697,7 +699,7 @@ public final String startCopy(final URI source, final AccessCondition sourceAcce
697699
*
698700
*/
699701
@DoesServiceRequest
700-
protected final String startCopy(final URI source, final PremiumPageBlobTier premiumPageBlobTier, final AccessCondition sourceAccessCondition,
702+
protected final String startCopy(final URI source, boolean syncCopy, final PremiumPageBlobTier premiumPageBlobTier, final AccessCondition sourceAccessCondition,
701703
final AccessCondition destinationAccessCondition, BlobRequestOptions options, OperationContext opContext)
702704
throws StorageException {
703705
if (opContext == null) {
@@ -708,12 +710,12 @@ protected final String startCopy(final URI source, final PremiumPageBlobTier pre
708710
options = BlobRequestOptions.populateAndApplyDefaults(options, this.properties.getBlobType(), this.blobServiceClient);
709711

710712
return ExecutionEngine.executeWithRetry(this.blobServiceClient, this,
711-
this.startCopyImpl(source, false /* incrementalCopy */, premiumPageBlobTier, sourceAccessCondition, destinationAccessCondition, options),
713+
this.startCopyImpl(source, syncCopy, false /* incrementalCopy */, premiumPageBlobTier, sourceAccessCondition, destinationAccessCondition, options),
712714
options.getRetryPolicyFactory(), opContext);
713715
}
714716

715717
protected StorageRequest<CloudBlobClient, CloudBlob, String> startCopyImpl(
716-
final URI source, final boolean incrementalCopy, final PremiumPageBlobTier premiumPageBlobTier, final AccessCondition sourceAccessCondition,
718+
final URI source, final boolean syncCopy, final boolean incrementalCopy, final PremiumPageBlobTier premiumPageBlobTier, final AccessCondition sourceAccessCondition,
717719
final AccessCondition destinationAccessCondition, final BlobRequestOptions options) {
718720

719721
final StorageRequest<CloudBlobClient, CloudBlob, String> putRequest =
@@ -725,7 +727,7 @@ public HttpURLConnection buildRequest(CloudBlobClient client, CloudBlob blob, Op
725727
// toASCIIString() must be used in order to appropriately encode the URI
726728
return BlobRequest.copyFrom(blob.getTransformedAddress(context).getUri(this.getCurrentLocation()),
727729
options, context, sourceAccessCondition, destinationAccessCondition, source.toASCIIString(),
728-
blob.snapshotID, incrementalCopy, premiumPageBlobTier);
730+
blob.snapshotID, incrementalCopy, syncCopy, premiumPageBlobTier);
729731
}
730732

731733
@Override

0 commit comments

Comments
 (0)