Skip to content

Migration tool ObjectMetadata transforms #5970

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Mar 28, 2025
Merged

Migration tool ObjectMetadata transforms #5970

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b7bd9ff
Migration tool ObjectMetadata transforms
davidh44 Mar 18, 2025
13aa1f8
Add transform for httpExpiresDate
davidh44 Mar 24, 2025
35cdd26
Add null checks and use fromContentProvider if content length not set
davidh44 Mar 24, 2025
94a5bca
Address comments
davidh44 Mar 27, 2025
36a24d0
Merge branch 'master' into hdavidh/migration-tool-objectmetadata-tran…
davidh44 Mar 27, 2025
6e6f026
Merge branch 'master' into hdavidh/migration-tool-objectmetadata-tran…
davidh44 Mar 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 43 additions & 6 deletions ...zon/awssdk/v2migrationtests/maven-nocompile/after/src/main/java/foo/bar/S3Transforms.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,57 @@

package foo.bar;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.Date;
import software.amazon.awssdk.core.async.AsyncRequestBody;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
import software.amazon.awssdk.transfer.s3.model.UploadRequest;

import java.io.ByteArrayInputStream;
import java.io.InputStream;

public class S3Transforms {

void upload(S3TransferManager tm, String bucket, String key) {
void upload_streamWithLiteralLength(S3TransferManager tm, String bucket, String key) {
HeadObjectResponse metadata = HeadObjectResponse.builder()
.build();
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
PutObjectRequest requestWithStreamAndLiteralLength = PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location").contentLength(333L)
.build();
/*AWS SDK for Java v2 migration: When using InputStream to upload with TransferManager, you must specify Content-Length and ExecutorService.*/tm.upload(UploadRequest.builder().putObjectRequest(requestWithStreamAndLiteralLength).requestBody(AsyncRequestBody.fromInputStream(inputStream, 333, newExecutorServiceVariableToDefine)).build());
}

void upload_streamWithAssignedLength(S3TransferManager tm, String bucket, String key) {
HeadObjectResponse metadata = HeadObjectResponse.builder()
.build();
long contentLen = 777;
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
PutObjectRequest requestWithInputStream = PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location")
PutObjectRequest requestWithStreamAndAssignedLength = PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location").contentLength(contentLen)
.build();
/*AWS SDK for Java v2 migration: When using InputStream to upload with TransferManager, you must specify Content-Length and ExecutorService.*/tm.upload(UploadRequest.builder().putObjectRequest(requestWithInputStream).requestBody(AsyncRequestBody.fromInputStream(inputStream, -1L, newExecutorServiceVariableToDefine)).build());
/*AWS SDK for Java v2 migration: When using InputStream to upload with TransferManager, you must specify Content-Length and ExecutorService.*/tm.upload(UploadRequest.builder().putObjectRequest(requestWithStreamAndAssignedLength).requestBody(AsyncRequestBody.fromInputStream(inputStream, contentLen, newExecutorServiceVariableToDefine)).build());
}

void upload_streamWithoutLength(S3TransferManager tm, String bucket, String key) {
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
PutObjectRequest requestWithStreamAndNoLength = PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location")
.build();
/*AWS SDK for Java v2 migration: When using InputStream to upload with TransferManager, you must specify Content-Length and ExecutorService.*/tm.upload(UploadRequest.builder().putObjectRequest(requestWithStreamAndNoLength).requestBody(AsyncRequestBody.fromInputStream(inputStream, -1L, newExecutorServiceVariableToDefine)).build());
}

void objectmetadata_unsupportedSetters(Date dateVal) {
HeadObjectResponse metadata = HeadObjectResponse.builder()
.build();

/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - expirationTimeRuleId - is not supported, please manually migrate the code by setting it on the v2 PutObjectRequest.*/metadata.expirationTimeRuleId("expirationTimeRuleId");
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - ongoingRestore - is not supported, please manually migrate the code by setting it on the v2 PutObjectRequest.*/metadata.ongoingRestore(false);
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - requesterCharged - is not supported, please manually migrate the code by setting it on the v2 PutObjectRequest.*/metadata.requesterCharged(false);

/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - lastModified - is not supported, please manually migrate the code by setting it on the v2 PutObjectRequest.*/metadata.lastModified(dateVal);
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - httpExpiresDate - is not supported, please manually migrate the code by setting it on the v2 PutObjectRequest.*/metadata.httpExpiresDate(dateVal);
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - expirationTime - is not supported, please manually migrate the code by setting it on the v2 PutObjectRequest.*/metadata.expirationTime(dateVal);
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - restoreExpirationTime - is not supported, please manually migrate the code by setting it on the v2 PutObjectRequest.*/metadata.restoreExpirationTime(dateVal);

/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - header - is not supported, please manually migrate the code by setting it on the v2 PutObjectRequest.*/metadata.header("key", "val");
/*AWS SDK for Java v2 migration: Transform for ObjectMetadata setter - addUserMetadata - is not supported, please manually migrate the code by setting it on the v2 PutObjectRequest.*/metadata.addUserMetadata("a", "b");
}
}
45 changes: 41 additions & 4 deletions ...on/awssdk/v2migrationtests/maven-nocompile/before/src/main/java/foo/bar/S3Transforms.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,54 @@

package foo.bar;

import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.Date;

public class S3Transforms {

void upload(TransferManager tm, String bucket, String key) {
void upload_streamWithLiteralLength(TransferManager tm, String bucket, String key) {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(333);
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
PutObjectRequest requestWithInputStream = new PutObjectRequest(bucket, key, "location");
requestWithInputStream.setInputStream(inputStream);
tm.upload(requestWithInputStream);
PutObjectRequest requestWithStreamAndLiteralLength = new PutObjectRequest(bucket, key, "location").withMetadata(metadata);
requestWithStreamAndLiteralLength.setInputStream(inputStream);
tm.upload(requestWithStreamAndLiteralLength);
}

void upload_streamWithAssignedLength(TransferManager tm, String bucket, String key) {
ObjectMetadata metadata = new ObjectMetadata();
long contentLen = 777;
metadata.setContentLength(contentLen);
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
PutObjectRequest requestWithStreamAndAssignedLength = new PutObjectRequest(bucket, key, "location").withMetadata(metadata);
requestWithStreamAndAssignedLength.setInputStream(inputStream);
tm.upload(requestWithStreamAndAssignedLength);
}

void upload_streamWithoutLength(TransferManager tm, String bucket, String key) {
InputStream inputStream = new ByteArrayInputStream(("HelloWorld").getBytes());
PutObjectRequest requestWithStreamAndNoLength = new PutObjectRequest(bucket, key, "location");
requestWithStreamAndNoLength.setInputStream(inputStream);
tm.upload(requestWithStreamAndNoLength);
}

void objectmetadata_unsupportedSetters(Date dateVal) {
ObjectMetadata metadata = new ObjectMetadata();

metadata.setExpirationTimeRuleId("expirationTimeRuleId");
metadata.setOngoingRestore(false);
metadata.setRequesterCharged(false);

metadata.setLastModified(dateVal);
metadata.setHttpExpiresDate(dateVal);
metadata.setExpirationTime(dateVal);
metadata.setRestoreExpirationTime(dateVal);

metadata.setHeader("key", "val");
metadata.addUserMetadata("a", "b");
}
}
53 changes: 50 additions & 3 deletions ...oftware/amazon/awssdk/v2migrationtests/maven/after/src/main/java/foo/bar/S3Streaming.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,14 @@
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.ObjectCannedACL;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.RequestPayer;
Expand All @@ -47,6 +50,13 @@ void putObject_bucketKeyFile(String bucket, String key, File file) {
.build(), RequestBody.fromFile(file));
}

void putObject_bucketKeyStreamMetadata(String bucket, String key, InputStream stream) {
HeadObjectResponse metadata = HeadObjectResponse.builder()
.build();
s3.putObject(PutObjectRequest.builder().bucket(bucket).key(key).contentLength(22L)
.build(), RequestBody.fromInputStream(stream, 22L));
}

/**
* Mixed ordering to ensure the files are assigned correctly
*/
Expand Down Expand Up @@ -76,10 +86,12 @@ void putObject_requestPojoWithInputStream(String bucket, String key) {

PutObjectRequest request1 = PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location")
.build();
/*AWS SDK for Java v2 migration: When using InputStream to upload with S3Client, Content-Length should be specified and used with RequestBody.fromInputStream(). Otherwise, the entire stream will be buffered in memory.*/s3.putObject(request1, RequestBody.fromContentProvider(() -> inputStream1, "binary/octet-stream"));
/*AWS SDK for Java v2 migration: When using InputStream to upload with S3Client, Content-Length should be specified and used with RequestBody.fromInputStream(). Otherwise, the entire stream will be buffered in memory.*/s3.putObject(request1, RequestBody.fromContentProvider(() -> inputStream1, "application/octet-stream"));

/*AWS SDK for Java v2 migration: When using InputStream to upload with S3Client, Content-Length should be specified and used with RequestBody.fromInputStream(). Otherwise, the entire stream will be buffered in memory.*/s3.putObject(PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location")
.build(), RequestBody.fromContentProvider(() -> inputStream2, "binary/octet-stream"));
HeadObjectResponse metadata = HeadObjectResponse.builder()
.build();
s3.putObject(PutObjectRequest.builder().bucket(bucket).key(key).websiteRedirectLocation("location").contentLength(11L)
.build(), RequestBody.fromInputStream(inputStream2, 11L));
}

void putObject_requestPojoWithoutPayload(String bucket, String key) {
Expand All @@ -104,4 +116,39 @@ void putObjectRequesterPaysSetter() {
PutObjectRequest requestWithFalse =PutObjectRequest.builder().bucket("bucket").key("key").websiteRedirectLocation("location")
.build();
}

void putObjectRequest_setMetadata() {
HeadObjectResponse metadata = HeadObjectResponse.builder()
.build();

PutObjectRequest request = PutObjectRequest.builder().bucket("bucket").key("key").websiteRedirectLocation("location")
.build();
request = request.toBuilder().contentLength(66L)
.contentEncoding("UTF-8")
.contentType("text/plain")
.build();
}

void putObjectRequest_withMetadata() {
HeadObjectResponse metadata = HeadObjectResponse.builder()
.build();
long contentLen = 66;

Map<String, String> userMetadata = new HashMap<>();
userMetadata.put("key", "value");

PutObjectRequest request = PutObjectRequest.builder().bucket("bucket").key("key").websiteRedirectLocation("location").contentLength(contentLen)
.contentEncoding("UTF-8")
.contentType("text/plain")
.contentLanguage("en-US")
.cacheControl("must-revalidate")
.contentDisposition("inline")
.contentMD5("md5Val")
.serverSideEncryption("sseEncryptionVal")
.serverSideEncryption("sseAlgorithmVal")
.sseCustomerKeyMD5("sseCustomerKeyMd5Val")
.bucketKeyEnabled(true)
.metadata(userMetadata)
.build();
}
}
45 changes: 44 additions & 1 deletion ...ftware/amazon/awssdk/v2migrationtests/maven/before/src/main/java/foo/bar/S3Streaming.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,14 @@
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;

public class S3Streaming {

Expand All @@ -41,6 +44,12 @@ void putObject_bucketKeyFile(String bucket, String key, File file) {
s3.putObject(bucket, key, file);
}

void putObject_bucketKeyStreamMetadata(String bucket, String key, InputStream stream) {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(22);
s3.putObject(bucket, key, stream, metadata);
}

/**
* Mixed ordering to ensure the files are assigned correctly
*/
Expand Down Expand Up @@ -69,7 +78,9 @@ void putObject_requestPojoWithInputStream(String bucket, String key) {
request1.setInputStream(inputStream1);
s3.putObject(request1);

s3.putObject(new PutObjectRequest(bucket, key, "location").withInputStream(inputStream2));
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(11);
s3.putObject(new PutObjectRequest(bucket, key, "location").withInputStream(inputStream2).withMetadata(metadata));
}

void putObject_requestPojoWithoutPayload(String bucket, String key) {
Expand All @@ -90,4 +101,36 @@ void putObjectRequesterPaysSetter() {

PutObjectRequest requestWithFalse = new PutObjectRequest("bucket", "key", "location").withRequesterPays(false);
}

void putObjectRequest_setMetadata() {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(66);
metadata.setContentType("text/plain");
metadata.setContentEncoding("UTF-8");

PutObjectRequest request = new PutObjectRequest("bucket", "key", "location");
request.setMetadata(metadata);
}

void putObjectRequest_withMetadata() {
ObjectMetadata metadata = new ObjectMetadata();
long contentLen = 66;
metadata.setContentLength(contentLen);
metadata.setContentType("text/plain");
metadata.setContentEncoding("UTF-8");
metadata.setContentLanguage("en-US");
metadata.setCacheControl("must-revalidate");
metadata.setContentDisposition("inline");
metadata.setContentMD5("md5Val");
metadata.setSSEAlgorithm("sseAlgorithmVal");
metadata.setServerSideEncryption("sseEncryptionVal");
metadata.setSSECustomerKeyMd5("sseCustomerKeyMd5Val");
metadata.setBucketKeyEnabled(true);

Map<String, String> userMetadata = new HashMap<>();
userMetadata.put("key", "value");
metadata.setUserMetadata(userMetadata);

PutObjectRequest request = new PutObjectRequest("bucket", "key", "location").withMetadata(metadata);
}
}
Loading
Loading