Skip to content

Commit 1888685

Browse files
raphkimKarthikeyan
raphkim
authored and
Karthikeyan
committed
Fix tagging for multipart upload (#1138)
* Added integration test for object tagging * Fixed bug where multipart upload fails to append tag * Added javadoc and added timeout to integration test * Updated CHANGELOG.md * Minor updates to CHANGELOG
1 parent e7c825d commit 1888685

File tree

6 files changed

+256
-3
lines changed

6 files changed

+256
-3
lines changed

‎CHANGELOG.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,23 @@
11
# Change Log - AWS SDK for Android
22

3+
## [Release 2.15.1](https://github.com/aws/aws-sdk-android/releases/tag/release_v2.15.1)
4+
5+
### Bug Fixes
6+
7+
- **Amazon S3**
8+
- Fixed a bug where multi-part uploads via `TransferUtility` would fail to propagate tags to `Amazon S3` from the `UserMetadata` passed through the `ObjectMetadata`. See [Issue#541](https://github.com/aws-amplify/aws-sdk-android/issues/541).
9+
- The following code should now attach a tag for both single-part and multi-part uploads:
10+
11+
```java
12+
ObjectMetadata metadata = new ObjectMetadata();
13+
metadata.addUserMetadata(Headers.S3_TAGGING, "key=value");
14+
TransferObserver observer = transferUtility.upload(
15+
file.getName(),
16+
file,
17+
metadata
18+
);
19+
```
20+
321
## [Release 2.15.0](https://github.com/aws/aws-sdk-android/releases/tag/release_v2.15.0)
422

523
### Bug Fixes

‎aws-android-sdk-s3-test/src/androidTest/java/com/amazonaws/services/s3/TransferUtilityIntegrationTest.java‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
/*
2+
* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at:
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
11+
* OR CONDITIONS OF ANY KIND, either express or implied. See the
12+
* License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
package com.amazonaws.services.s3;
17+
18+
import android.content.Context;
19+
import android.support.test.InstrumentationRegistry;
20+
21+
import com.amazonaws.mobileconnectors.s3.transferutility.TransferListener;
22+
import com.amazonaws.mobileconnectors.s3.transferutility.TransferNetworkLossHandler;
23+
import com.amazonaws.mobileconnectors.s3.transferutility.TransferState;
24+
import com.amazonaws.mobileconnectors.s3.transferutility.TransferUtility;
25+
import com.amazonaws.services.s3.model.GetObjectTaggingRequest;
26+
import com.amazonaws.services.s3.model.ObjectMetadata;
27+
import com.amazonaws.services.s3.model.Tag;
28+
29+
import org.junit.AfterClass;
30+
import org.junit.BeforeClass;
31+
import org.junit.Test;
32+
33+
import java.io.File;
34+
import java.io.RandomAccessFile;
35+
import java.util.Date;
36+
import java.util.List;
37+
import java.util.concurrent.CountDownLatch;
38+
import java.util.concurrent.TimeUnit;
39+
40+
import static org.hamcrest.CoreMatchers.hasItems;
41+
import static org.hamcrest.CoreMatchers.is;
42+
import static org.junit.Assert.assertThat;
43+
import static org.junit.Assert.fail;
44+
45+
public class TransferUtilityIntegrationTest extends S3IntegrationTestBase {
46+
47+
/** The bucket created and used by these tests */
48+
private static final String bucketName = "amazon-transfer-util-integ-test-" + new Date().getTime();
49+
50+
/** Instrumentation test context */
51+
private static Context context = InstrumentationRegistry.getContext();
52+
53+
/** Countdown latch for testing */
54+
private static CountDownLatch latch;
55+
56+
/** The transfer utility used to upload to S3 */
57+
private static TransferUtility util;
58+
59+
/** The file containing the test data uploaded to S3 */
60+
private static File file = null;
61+
62+
/** The metadata for the object. */
63+
private ObjectMetadata metadata;
64+
65+
/** Simple transfer listener to confirm upload completion before bucket deletion */
66+
private static final TransferListener listener = new TransferListener() {
67+
68+
public void onProgressChanged(int id, long bytesCurrent, long bytesTotal) {
69+
}
70+
71+
public void onStateChanged(int id, TransferState state) {
72+
if (state.equals(TransferState.COMPLETED)) { latch.countDown(); }
73+
}
74+
75+
public void onError(int id, Exception ex) {
76+
fail(ex.getMessage());
77+
}
78+
};
79+
80+
/**
81+
* Creates and initializes all the test resources needed for these tests.
82+
*/
83+
@BeforeClass
84+
public static void setUpBeforeClass() throws Exception {
85+
setUp();
86+
TransferNetworkLossHandler.getInstance(context);
87+
util = TransferUtility.builder()
88+
.context(context)
89+
.s3Client(s3)
90+
.build();
91+
92+
try {
93+
s3.createBucket(bucketName);
94+
waitForBucketCreation(bucketName);
95+
} catch (final Exception e) {
96+
System.out.println("Error in creating the bucket. "
97+
+ "Please manually create the bucket " + bucketName);
98+
}
99+
}
100+
101+
@AfterClass
102+
public static void tearDown() {
103+
try {
104+
deleteBucketAndAllContents(bucketName);
105+
} catch (final Exception e) {
106+
System.out.println("Error in deleting the bucket. "
107+
+ "Please manually delete the bucket " + bucketName);
108+
e.printStackTrace();
109+
}
110+
111+
if (file != null) {
112+
file.delete();
113+
}
114+
}
115+
116+
@Test
117+
public void testSingleUploadTagging() throws Exception {
118+
// Object metadata to add
119+
metadata = new ObjectMetadata();
120+
metadata.addUserMetadata("x-amz-tagging", "key1=value1&key2=value2");
121+
metadata.addUserMetadata("key3", "value3"); // Should not get added to tags
122+
123+
latch = new CountDownLatch(1);
124+
// Small (1KB) file upload
125+
file = getRandomTempFile("small", 1000L);
126+
util.upload(bucketName, file.getName(), file, metadata)
127+
.setTransferListener(listener);
128+
latch.await(300, TimeUnit.SECONDS);
129+
130+
List<Tag> tags = s3.getObjectTagging(new GetObjectTaggingRequest(
131+
bucketName,
132+
file.getName()
133+
)).getTagSet();
134+
assertThat(tags.size(), is(2));
135+
assertThat(tags, hasItems(
136+
new Tag("key1", "value1"),
137+
new Tag("key2", "value2")
138+
));
139+
}
140+
141+
@Test
142+
public void testMultiUploadTagging() throws Exception {
143+
// Object metadata to add
144+
metadata = new ObjectMetadata();
145+
metadata.addUserMetadata("x-amz-tagging", "key1=value1&key2=value2");
146+
metadata.addUserMetadata("key3", "value3"); // Should not get added to tags
147+
148+
latch = new CountDownLatch(1);
149+
// Large (5MB) file upload
150+
long size = 5 * 1024 * 1024 + 1;
151+
file = getRandomSparseFile("large", size);
152+
util.upload(bucketName, file.getName(), file, metadata)
153+
.setTransferListener(listener);
154+
latch.await(300, TimeUnit.SECONDS);
155+
156+
List<Tag> tags = s3.getObjectTagging(new GetObjectTaggingRequest(
157+
bucketName,
158+
file.getName()
159+
)).getTagSet();
160+
assertThat(tags.size(), is(2));
161+
assertThat(tags, hasItems(
162+
new Tag("key1", "value1"),
163+
new Tag("key2", "value2")
164+
));
165+
}
166+
167+
/*
168+
* Helper method to create a sparse file (Faster than writing a file with random bytes with getRandomTempFile)
169+
* Use for testing large uploads
170+
*/
171+
private File getRandomSparseFile(String filename, long contentLength) throws Exception {
172+
// Use the same storage path as getRandomTempFile()
173+
String pathPrefix = System.getProperty("java.io.tmpdir") + File.separator + System.currentTimeMillis() + "-";
174+
File tempFile = new File(pathPrefix + filename);
175+
RandomAccessFile raf = new RandomAccessFile(tempFile, "rw"); // Create a new random access file with read/write access
176+
raf.setLength(contentLength);
177+
return tempFile;
178+
}
179+
}

‎aws-android-sdk-s3/src/main/java/com/amazonaws/mobileconnectors/s3/transferutility/UploadTask.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -378,7 +378,8 @@ private String initiateMultipartUpload(PutObjectRequest putObjectRequest) {
378378
.withCannedACL(putObjectRequest.getCannedAcl())
379379
.withObjectMetadata(putObjectRequest.getMetadata())
380380
.withSSEAwsKeyManagementParams(
381-
putObjectRequest.getSSEAwsKeyManagementParams());
381+
putObjectRequest.getSSEAwsKeyManagementParams())
382+
.withTagging(putObjectRequest.getTagging());
382383
TransferUtility
383384
.appendMultipartTransferServiceUserAgentString(initiateMultipartUploadRequest);
384385
return s3.initiateMultipartUpload(initiateMultipartUploadRequest).getUploadId();

‎aws-android-sdk-s3/src/main/java/com/amazonaws/services/s3/AmazonS3Client.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3671,6 +3671,8 @@ public InitiateMultipartUploadResult initiateMultipartUpload(
36713671
populateRequestMetadata(request, initiateMultipartUploadRequest.objectMetadata);
36723672
}
36733673

3674+
addHeaderIfNotNull(request, Headers.S3_TAGGING, urlEncodeTags(initiateMultipartUploadRequest.getTagging()));
3675+
36743676
populateRequesterPaysHeader(request, initiateMultipartUploadRequest.isRequesterPays());
36753677

36763678
// Populate the SSE-C parameters to the request header
@@ -3825,7 +3827,6 @@ public UploadPartResult uploadPart(UploadPartRequest uploadPartRequest)
38253827
if (objectMetadata != null) {
38263828
populateRequestMetadata(request, objectMetadata);
38273829
}
3828-
38293830
addHeaderIfNotNull(request, Headers.CONTENT_MD5, uploadPartRequest.getMd5Digest());
38303831
request.addHeader(Headers.CONTENT_LENGTH, Long.toString(partSize));
38313832
/*
@@ -4409,7 +4410,9 @@ protected static void populateRequestMetadata(Request<?> request, ObjectMetadata
44094410
if (value != null) {
44104411
value = value.trim();
44114412
}
4412-
request.addHeader(Headers.S3_USER_METADATA_PREFIX + key, value);
4413+
if (!Headers.S3_TAGGING.equals(key)) {
4414+
request.addHeader(Headers.S3_USER_METADATA_PREFIX + key, value);
4415+
}
44134416
}
44144417
}
44154418
}

‎aws-android-sdk-s3/src/main/java/com/amazonaws/services/s3/model/AbstractPutObjectRequest.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -701,14 +701,32 @@ public <T extends AbstractPutObjectRequest> T withSSECustomerKey(
701701
return t;
702702
}
703703

704+
/**
705+
* Gets the object tagging associated with this request.
706+
*
707+
* @return Object tagging attached to this request
708+
*/
704709
public ObjectTagging getTagging() {
705710
return tagging;
706711
}
707712

713+
/**
714+
* Attaches object tagging to this request.
715+
*
716+
* @param tagging Object tagging to attach
717+
*/
708718
public void setTagging(ObjectTagging tagging) {
709719
this.tagging = tagging;
710720
}
711721

722+
/**
723+
* Sets tagging for this request. Returns the updated request object
724+
* so that additional method calls can be chained together.
725+
*
726+
* @param tagSet Object tagging to be attached to this request
727+
* @return This updated request object so that additional method calls can
728+
* be chained together.
729+
*/
712730
public <T extends AbstractPutObjectRequest> T withTagging(ObjectTagging tagSet) {
713731
setTagging(tagSet);
714732
@SuppressWarnings("unchecked")

‎aws-android-sdk-s3/src/main/java/com/amazonaws/services/s3/model/InitiateMultipartUploadRequest.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,9 @@ public class InitiateMultipartUploadRequest extends AmazonWebServiceRequest
9898
*/
9999
private boolean isRequesterPays;
100100

101+
/** Object tagging associated with this request */
102+
private ObjectTagging tagging;
103+
101104
/**
102105
* Constructs a request to initiate a new multipart upload in the specified
103106
* bucket, stored by the specified key.
@@ -543,4 +546,35 @@ public InitiateMultipartUploadRequest withRequesterPays(boolean isRequesterPays)
543546
setRequesterPays(isRequesterPays);
544547
return this;
545548
}
549+
550+
/**
551+
* Gets the object tagging associated with this request.
552+
*
553+
* @return Object tagging attached to this request
554+
*/
555+
public ObjectTagging getTagging() {
556+
return tagging;
557+
}
558+
559+
/**
560+
* Attaches object tagging to this request.
561+
*
562+
* @param tagging Object tagging to attach
563+
*/
564+
public void setTagging(ObjectTagging tagging) {
565+
this.tagging = tagging;
566+
}
567+
568+
/**
569+
* Sets tagging for this request. Returns updated
570+
* InitiateMultipartUploadRequest object {@link InitiateMultipartUploadRequest},
571+
* so that additional method calls can be chained together.
572+
*
573+
* @param tagSet Object tagging to be attached to this request
574+
* @return The updated InitiateMultipartUploadRequest object
575+
*/
576+
public InitiateMultipartUploadRequest withTagging(ObjectTagging tagSet) {
577+
setTagging(tagSet);
578+
return this;
579+
}
546580
}

0 commit comments

Comments
 (0)