Skip to content
This repository was archived by the owner on Aug 6, 2025. It is now read-only.

Commit dd6cb51

Browse files
gaulgaul
committed
JCLOUDS-1644: Create AWS S3 buckets with ownership and public access block
AWS changed the defaults when creating buckets to prevent public-read and other canned ACLs. Background: https://stackoverflow.com/a/76102067/2800111
1 parent bdfac92 commit dd6cb51

File tree

11 files changed

+258
-5
lines changed

11 files changed

+258
-5
lines changed

‎apis/s3/src/main/java/org/jclouds/s3/S3Client.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,10 @@
6666
import org.jclouds.s3.binders.BindIterableAsPayloadToDeleteRequest;
6767
import org.jclouds.s3.binders.BindNoBucketLoggingToXmlPayload;
6868
import org.jclouds.s3.binders.BindObjectMetadataToRequest;
69+
import org.jclouds.s3.binders.BindOwnershipControlsToXMLPayload;
6970
import org.jclouds.s3.binders.BindPartIdsAndETagsToRequest;
7071
import org.jclouds.s3.binders.BindPayerToXmlPayload;
72+
import org.jclouds.s3.binders.BindPublicAccessBlockConfigurationToXMLPayload;
7173
import org.jclouds.s3.binders.BindS3ObjectMetadataToRequest;
7274
import org.jclouds.s3.domain.AccessControlList;
7375
import org.jclouds.s3.domain.BucketLogging;
@@ -79,6 +81,7 @@
7981
import org.jclouds.s3.domain.ListMultipartUploadsResponse;
8082
import org.jclouds.s3.domain.ObjectMetadata;
8183
import org.jclouds.s3.domain.Payer;
84+
import org.jclouds.s3.domain.PublicAccessBlockConfiguration;
8285
import org.jclouds.s3.domain.S3Object;
8386
import org.jclouds.s3.fallbacks.FalseIfBucketAlreadyOwnedByYouOrOperationAbortedWhenBucketExists;
8487
import org.jclouds.s3.filters.RequestAuthorizeSignature;
@@ -813,4 +816,19 @@ ListMultipartUploadsResponse listMultipartUploads(@Bucket @EndpointParam(parser
813816
@QueryParam("delimiter") @Nullable String delimiter, @QueryParam("max-uploads") @Nullable Integer maxUploads,
814817
@QueryParam("key-marker") @Nullable String keyMarker, @QueryParam("prefix") @Nullable String prefix,
815818
@QueryParam("upload-id-marker") @Nullable String uploadIdMarker);
819+
820+
@Named("PutBucketOwnershipControls")
821+
@PUT
822+
@Path("/")
823+
@QueryParams(keys = "ownershipControls")
824+
void putBucketOwnershipControls(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
825+
// BucketOwnerPreferred | ObjectWriter | BucketOwnerEnforced
826+
@BinderParam(BindOwnershipControlsToXMLPayload.class) String objectOwnership);
827+
828+
@Named("PutPublicAccessBlock")
829+
@PUT
830+
@Path("/")
831+
@QueryParams(keys = "publicAccessBlock")
832+
void putPublicAccessBlock(@Bucket @EndpointParam(parser = AssignCorrectHostnameForBucket.class) @BinderParam(BindAsHostPrefixIfConfigured.class) @ParamValidators(BucketNameValidator.class) String bucketName,
833+
@BinderParam(BindPublicAccessBlockConfigurationToXMLPayload.class) PublicAccessBlockConfiguration configuration);
816834
}

‎apis/s3/src/main/java/org/jclouds/s3/binders/BindOwnershipControlsToXMLPayload.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.jclouds.s3.binders;
18+
19+
import static org.jclouds.s3.binders.XMLHelper.asString;
20+
import static org.jclouds.s3.binders.XMLHelper.createDocument;
21+
import static org.jclouds.s3.binders.XMLHelper.elem;
22+
import static org.jclouds.s3.binders.XMLHelper.elemWithText;
23+
24+
import jakarta.inject.Singleton;
25+
import jakarta.ws.rs.core.MediaType;
26+
import javax.xml.parsers.FactoryConfigurationError;
27+
import javax.xml.parsers.ParserConfigurationException;
28+
import javax.xml.transform.TransformerException;
29+
30+
import org.jclouds.http.HttpRequest;
31+
import org.jclouds.rest.Binder;
32+
import org.jclouds.s3.reference.S3Constants;
33+
import org.w3c.dom.Document;
34+
import org.w3c.dom.Element;
35+
36+
import com.google.common.base.Throwables;
37+
38+
@Singleton
39+
public final class BindOwnershipControlsToXMLPayload implements Binder {
40+
@Override
41+
public <R extends HttpRequest> R bindToRequest(R request, Object payload) {
42+
String from = (String) payload;
43+
try {
44+
request.setPayload(generatePayload(from));
45+
request.getPayload().getContentMetadata().setContentType(MediaType.TEXT_XML);
46+
return request;
47+
} catch (Exception e) {
48+
Throwables.propagateIfPossible(e);
49+
throw new RuntimeException("error transforming acl: " + from, e);
50+
}
51+
}
52+
53+
protected String generatePayload(String objectOwnership)
54+
throws ParserConfigurationException, FactoryConfigurationError, TransformerException {
55+
Document document = createDocument();
56+
Element rootNode = elem(document, "OwnershipControls", document);
57+
rootNode.setAttribute("xmlns", S3Constants.S3_REST_API_XML_NAMESPACE);
58+
Element ruleNode = elem(rootNode, "Rule", document);
59+
elemWithText(ruleNode, "ObjectOwnership", objectOwnership, document);
60+
return asString(document);
61+
}
62+
}

‎apis/s3/src/main/java/org/jclouds/s3/binders/BindPublicAccessBlockConfigurationToXMLPayload.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.jclouds.s3.binders;
18+
19+
import static org.jclouds.s3.binders.XMLHelper.asString;
20+
import static org.jclouds.s3.binders.XMLHelper.createDocument;
21+
import static org.jclouds.s3.binders.XMLHelper.elem;
22+
import static org.jclouds.s3.binders.XMLHelper.elemWithText;
23+
24+
import jakarta.inject.Singleton;
25+
import jakarta.ws.rs.core.MediaType;
26+
import javax.xml.parsers.FactoryConfigurationError;
27+
import javax.xml.parsers.ParserConfigurationException;
28+
import javax.xml.transform.TransformerException;
29+
30+
import org.jclouds.http.HttpRequest;
31+
import org.jclouds.rest.Binder;
32+
import org.jclouds.s3.domain.PublicAccessBlockConfiguration;
33+
import org.jclouds.s3.reference.S3Constants;
34+
import org.w3c.dom.Document;
35+
import org.w3c.dom.Element;
36+
37+
import com.google.common.base.Throwables;
38+
39+
@Singleton
40+
public final class BindPublicAccessBlockConfigurationToXMLPayload implements Binder {
41+
@Override
42+
public <R extends HttpRequest> R bindToRequest(R request, Object payload) {
43+
PublicAccessBlockConfiguration configuration = (PublicAccessBlockConfiguration) payload;
44+
try {
45+
request.setPayload(generatePayload(configuration));
46+
request.getPayload().getContentMetadata().setContentType(MediaType.TEXT_XML);
47+
return request;
48+
} catch (Exception e) {
49+
Throwables.propagateIfPossible(e);
50+
throw new RuntimeException("error transforming configuration: " + configuration, e);
51+
}
52+
}
53+
54+
protected String generatePayload(PublicAccessBlockConfiguration configuration)
55+
throws ParserConfigurationException, FactoryConfigurationError, TransformerException {
56+
Document document = createDocument();
57+
Element rootNode = elem(document, "PublicAccessBlockConfiguration", document);
58+
rootNode.setAttribute("xmlns", S3Constants.S3_REST_API_XML_NAMESPACE);
59+
elemWithText(rootNode, "BlockPublicAcls", String.valueOf(configuration.blockPublicAcls()), document);
60+
elemWithText(rootNode, "IgnorePublicAcls", String.valueOf(configuration.ignorePublicAcls()), document);
61+
elemWithText(rootNode, "BlockPublicPolicy", String.valueOf(configuration.blockPublicPolicy()), document);
62+
elemWithText(rootNode, "RestrictPublicBuckets", String.valueOf(configuration.restrictPublicBuckets()), document);
63+
return asString(document);
64+
}
65+
}

‎apis/s3/src/main/java/org/jclouds/s3/domain/PublicAccessBlockConfiguration.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.jclouds.s3.domain;
18+
19+
import com.google.auto.value.AutoValue;
20+
import com.google.common.annotations.Beta;
21+
22+
@AutoValue
23+
@Beta
24+
public abstract class PublicAccessBlockConfiguration {
25+
public abstract boolean blockPublicAcls();
26+
public abstract boolean ignorePublicAcls();
27+
public abstract boolean blockPublicPolicy();
28+
public abstract boolean restrictPublicBuckets();
29+
30+
public static PublicAccessBlockConfiguration create(boolean blockPublicAcls, boolean ignorePublicAcls, boolean blockPublicPolicy, boolean restrictPublicBuckets) {
31+
return new AutoValue_PublicAccessBlockConfiguration(blockPublicAcls, ignorePublicAcls, blockPublicPolicy, restrictPublicBuckets);
32+
}
33+
}

‎apis/s3/src/test/java/org/jclouds/s3/S3ClientLiveTest.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,11 +163,16 @@ public void testPutCannedAccessPolicyPublic() throws Exception {
163163

164164
}
165165

166+
protected void allowPublicReadable(String containerName) {
167+
}
168+
166169
@Test(groups = {"fails-on-s3proxy"})
167170
public void testCopyCannedAccessPolicyPublic() throws Exception {
168171
String containerName = getContainerName();
169172
String destinationContainer = getContainerName();
170173
try {
174+
allowPublicReadable(destinationContainer);
175+
171176
addBlobToContainer(containerName, sourceKey);
172177
validateContent(containerName, sourceKey);
173178

@@ -193,6 +198,7 @@ public void testPublicWriteOnObject() throws InterruptedException, ExecutionExce
193198
final String publicReadWriteObjectKey = "public-read-write-acl";
194199
final String containerName = getContainerName();
195200
try {
201+
allowPublicReadable(containerName);
196202
S3Object object = getApi().newS3Object();
197203
object.getMetadata().setKey(publicReadWriteObjectKey);
198204
object.setPayload("");
@@ -295,6 +301,7 @@ public void testPublicReadOnObject() throws InterruptedException, ExecutionExcep
295301
final String publicReadObjectKey = "public-read-acl";
296302
final String containerName = getContainerName();
297303
try {
304+
allowPublicReadable(containerName);
298305
S3Object object = getApi().newS3Object();
299306
object.getMetadata().setKey(publicReadObjectKey);
300307
object.setPayload("");
@@ -715,6 +722,7 @@ private void addGrantsToACL(AccessControlList acl) {
715722
public void testUpdateBucketCannedACL() throws Exception {
716723
String containerName = getContainerName();
717724
try {
725+
allowPublicReadable(containerName);
718726
getApi().updateBucketCannedACL(containerName, CannedAccessPolicy.PUBLIC_READ);
719727
AccessControlList acl = getApi().getBucketACL(containerName);
720728
assertThat(acl.hasPermission(GroupGranteeURI.ALL_USERS, Permission.READ)).isTrue();
@@ -730,6 +738,7 @@ public void testUpdateBucketCannedACL() throws Exception {
730738
public void testUpdateObjectCannedACL() throws Exception {
731739
String containerName = getContainerName();
732740
try {
741+
allowPublicReadable(containerName);
733742
String key = "testUpdateObjectCannedACL";
734743
S3Object object = getApi().newS3Object();
735744
object.getMetadata().setKey(key);

‎apis/s3/src/test/java/org/jclouds/s3/services/BucketsLiveTest.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,12 @@
2525
import static org.jclouds.s3.domain.AccessControlList.Permission.READ_ACP;
2626
import static org.jclouds.s3.domain.AccessControlList.Permission.WRITE;
2727
import static org.jclouds.s3.domain.AccessControlList.Permission.WRITE_ACP;
28-
import static org.jclouds.s3.domain.CannedAccessPolicy.PUBLIC_READ;
2928
import static org.jclouds.s3.domain.Payer.BUCKET_OWNER;
3029
import static org.jclouds.s3.domain.Payer.REQUESTER;
3130
import static org.jclouds.s3.options.ListBucketOptions.Builder.afterMarker;
3231
import static org.jclouds.s3.options.ListBucketOptions.Builder.delimiter;
3332
import static org.jclouds.s3.options.ListBucketOptions.Builder.maxResults;
3433
import static org.jclouds.s3.options.ListBucketOptions.Builder.withPrefix;
35-
import static org.jclouds.s3.options.PutBucketOptions.Builder.withBucketAcl;
3634
import static org.testng.Assert.assertEquals;
3735
import static org.testng.Assert.assertNotNull;
3836
import static org.testng.Assert.assertNull;
@@ -53,6 +51,7 @@
5351
import org.jclouds.s3.domain.AccessControlList.Grant;
5452
import org.jclouds.s3.domain.BucketLogging;
5553
import org.jclouds.s3.domain.BucketMetadata;
54+
import org.jclouds.s3.domain.CannedAccessPolicy;
5655
import org.jclouds.s3.domain.ListBucketResponse;
5756
import org.jclouds.s3.domain.S3Object;
5857
import org.jclouds.util.Strings2;
@@ -154,7 +153,9 @@ private void addGrantsToACL(AccessControlList acl) {
154153
public void testPublicReadAccessPolicy() throws Exception {
155154
String bucketName = getScratchContainerName();
156155
try {
157-
getApi().putBucketInRegion(null, bucketName, withBucketAcl(PUBLIC_READ));
156+
getApi().putBucketInRegion(/*region=*/ null, bucketName);
157+
allowPublicReadable(bucketName);
158+
getApi().updateBucketCannedACL(bucketName, CannedAccessPolicy.PUBLIC_READ);
158159
AccessControlList acl = getApi().getBucketACL(bucketName);
159160
assertTrue(acl.hasPermission(ALL_USERS, READ), acl.toString());
160161
// TODO: I believe that the following should work based on the above acl assertion passing.
@@ -209,11 +210,15 @@ public void run() {
209210
}
210211
}
211212

213+
protected void allowPublicReadable(String containerName) {
214+
}
215+
212216
@Test(groups = {"fails-on-s3proxy"})
213217
public void testBucketLogging() throws Exception {
214218
final String bucketName = getContainerName();
215219
final String targetBucket = getContainerName();
216220
try {
221+
allowPublicReadable(targetBucket);
217222
assertNull(getApi().getBucketLogging(bucketName));
218223

219224
setupAclForBucketLoggingTarget(targetBucket);

‎blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -729,12 +729,16 @@ public void testPutMultipartInputStream() throws Exception {
729729
testPut(payload, null, new ByteSourcePayload(byteSource), length, new PutOptions().multipart(true));
730730
}
731731

732+
protected void allowPublicReadable(String containerName) {
733+
}
734+
732735
@Test(groups = { "integration", "live" })
733736
public void testSetBlobAccess() throws Exception {
734737
BlobStore blobStore = view.getBlobStore();
735738
String containerName = getContainerName();
736739
String blobName = "set-access-blob-name";
737740
try {
741+
allowPublicReadable(containerName);
738742
addBlobToContainer(containerName, blobName, blobName, MediaType.TEXT_PLAIN);
739743

740744
assertThat(blobStore.getBlobAccess(containerName, blobName)).isEqualTo(BlobAccess.PRIVATE);
@@ -778,6 +782,7 @@ public void testPutBlobAccess() throws Exception {
778782
public void testPutBlobAccessMultipart() throws Exception {
779783
BlobStore blobStore = view.getBlobStore();
780784
String containerName = getContainerName();
785+
allowPublicReadable(containerName);
781786
ByteSource byteSource = TestUtils.randomByteSource().slice(0, getMinimumMultipartBlobSize());
782787
Payload payload = Payloads.newByteSourcePayload(byteSource);
783788
payload.getContentMetadata().setContentLength(byteSource.size());

‎providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions;
3030
import org.jclouds.blobstore.BlobStoreContext;
3131
import org.jclouds.blobstore.domain.Blob;
32+
import org.jclouds.blobstore.domain.ContainerAccess;
3233
import org.jclouds.blobstore.domain.PageSet;
3334
import org.jclouds.blobstore.domain.StorageMetadata;
3435
import org.jclouds.blobstore.functions.BlobToHttpGetOptions;
@@ -47,7 +48,9 @@
4748
import org.jclouds.s3.blobstore.functions.ObjectToBlob;
4849
import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata;
4950
import org.jclouds.s3.domain.BucketMetadata;
51+
import org.jclouds.s3.domain.CannedAccessPolicy;
5052
import org.jclouds.s3.domain.ObjectMetadata;
53+
import org.jclouds.s3.domain.PublicAccessBlockConfiguration;
5154

5255
import com.google.common.base.Function;
5356
import com.google.common.base.Supplier;
@@ -58,6 +61,7 @@
5861
public class AWSS3BlobStore extends S3BlobStore {
5962

6063
private final BlobToObject blob2Object;
64+
private final AWSS3Client awsSync;
6165

6266
@Inject
6367
AWSS3BlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation,
@@ -70,6 +74,7 @@ public class AWSS3BlobStore extends S3BlobStore {
7074
super(context, blobUtils, defaultLocation, locations, slicer, sync, convertBucketsToStorageMetadata,
7175
container2BucketListOptions, bucket2ResourceList, object2Blob, blob2ObjectGetOptions, blob2Object,
7276
blob2ObjectMetadata, object2BlobMd, fetchBlobMetadataProvider);
77+
this.awsSync = sync;
7378
this.blob2Object = blob2Object;
7479
}
7580

@@ -102,6 +107,30 @@ public boolean createContainerInLocation(Location location, String container,
102107
// JCLOUDS-334 for details.
103108
return false;
104109
}
110+
// AWS blocks creating buckets with public-read canned ACL by default since 25 April 2023. Instead create a bucket, override the block, and set the ACL.
111+
if (options.isPublicRead()) {
112+
boolean created = super.createContainerInLocation(location, container, new CreateContainerOptions());
113+
if (!created) {
114+
return false;
115+
}
116+
awsSync.putBucketOwnershipControls(container, "ObjectWriter");
117+
awsSync.putPublicAccessBlock(container, PublicAccessBlockConfiguration.create(
118+
/*blockPublicAcls=*/ false, /*ignorePublicAcls=*/ false, /*blockPublicPolicy=*/ false, /*restrictPublicBuckets=*/ false));
119+
awsSync.updateBucketCannedACL(container, CannedAccessPolicy.PUBLIC_READ);
120+
return true;
121+
}
105122
return super.createContainerInLocation(location, container, options);
106123
}
124+
125+
@Override
126+
public void setContainerAccess(String container, ContainerAccess access) {
127+
CannedAccessPolicy acl = CannedAccessPolicy.PRIVATE;
128+
if (access == ContainerAccess.PUBLIC_READ) {
129+
acl = CannedAccessPolicy.PUBLIC_READ;
130+
awsSync.putBucketOwnershipControls(container, "ObjectWriter");
131+
awsSync.putPublicAccessBlock(container, PublicAccessBlockConfiguration.create(
132+
/*blockPublicAcls=*/ false, /*ignorePublicAcls=*/ false, /*blockPublicPolicy=*/ false, /*restrictPublicBuckets=*/ false));
133+
}
134+
awsSync.updateBucketCannedACL(container, acl);
135+
}
107136
}

0 commit comments

Comments
 (0)