From e45c6b1b704c3943d491159ad16270d6cedefeea Mon Sep 17 00:00:00 2001 From: Pavel Popov Date: Sun, 19 Nov 2017 00:32:26 +0300 Subject: [PATCH 1/7] Add support for ceph --- modules/ceph/pom.xml | 30 ++++ .../containers/CephContainer.java | 133 ++++++++++++++++++ .../containers/CephContainerTest.java | 98 +++++++++++++ pom.xml | 1 + 4 files changed, 262 insertions(+) create mode 100644 modules/ceph/pom.xml create mode 100644 modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java create mode 100644 modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java diff --git a/modules/ceph/pom.xml b/modules/ceph/pom.xml new file mode 100644 index 00000000000..d933b9eabd7 --- /dev/null +++ b/modules/ceph/pom.xml @@ -0,0 +1,30 @@ + + + 4.0.0 + + + org.testcontainers + testcontainers-parent + 0-SNAPSHOT + ../../pom.xml + + + ceph + TestContainers :: ceph + + + + ${project.groupId} + testcontainers + ${project.version} + + + com.amazonaws + aws-java-sdk-s3 + 1.11.231 + + + + \ No newline at end of file diff --git a/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java b/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java new file mode 100644 index 00000000000..c892913f12d --- /dev/null +++ b/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java @@ -0,0 +1,133 @@ +package org.testcontainers.containers; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import org.testcontainers.containers.wait.LogMessageWaitStrategy; + +import java.time.Duration; + +public class CephContainer> extends GenericContainer { + + private static final String IMAGE = "ceph/demo"; + + private String awsAccessKey = "ceph"; + + private String awsSecretKey = "ceph"; + + private String bucketName = "CEPH"; + + private String rgwName = "localhost"; + + private String demoUid = "ceph"; + + private NetworkAutoDetectMode networkAutoDetectMode = NetworkAutoDetectMode.IPV4_ONLY; + + private Duration startupTimeout = Duration.ofSeconds(20); + + public CephContainer() { + super(IMAGE + ":latest"); + } + + public CephContainer(String dockerImageName) { + super(dockerImageName); + } + + @Override + protected void configure() { + withEnv("RGW_NAME", rgwName); + withEnv("NETWORK_AUTO_DETECT", networkAutoDetectMode.value); + withEnv("CEPH_DEMO_UID", demoUid); + withEnv("CEPH_DEMO_ACCESS_KEY", awsAccessKey); + withEnv("CEPH_DEMO_SECRET_KEY", awsSecretKey); + withEnv("CEPH_DEMO_BUCKET", bucketName); + withExposedPorts(6789, 6800, 6801, 6802, 6803, 6804, 6805, 80, 5000); + waitingFor( + new LogMessageWaitStrategy() + .withRegEx(".*\\/entrypoint.sh: SUCCESS\n") + .withStartupTimeout(startupTimeout) + ); + } + + public AWSCredentialsProvider getAWSCredentialsProvider() { + return new AWSStaticCredentialsProvider( + new BasicAWSCredentials(awsAccessKey, awsSecretKey) + ); + } + + public AwsClientBuilder.EndpointConfiguration getAWSEndpointConfiguration() { + return new AwsClientBuilder.EndpointConfiguration( + getContainerIpAddress() + ":" + getMappedPort(80), + "us-east-1" + ); + } + + public SELF withAwsAccessKey(String awsAccessKey) { + this.awsAccessKey = awsAccessKey; + return self(); + } + + public SELF withAwsSecretKey(String awsSecretKey) { + this.awsSecretKey = awsSecretKey; + return self(); + } + + public String getBucketName() { + return bucketName; + } + + public SELF withBucketName(String bucketName) { + //because s3cmd transforming bucket name to uppercase + this.bucketName = bucketName.toUpperCase(); + return self(); + } + + public String getRgwName() { + return rgwName; + } + + public SELF withRgwName(String rgwName) { + this.rgwName = rgwName; + return self(); + } + + public String getDemoUid() { + return demoUid; + } + + public SELF withDemoUid(String demoUid) { + this.demoUid = demoUid; + return self(); + } + + public NetworkAutoDetectMode getNetworkAutoDetectMode() { + return networkAutoDetectMode; + } + + public SELF withNetworkAutoDetectMode(NetworkAutoDetectMode networkAutoDetectMode) { + this.networkAutoDetectMode = networkAutoDetectMode; + return self(); + } + + public Duration getStartupTimeout() { + return startupTimeout; + } + + public SELF withStartupTimeout(Duration startupTimeout) { + this.startupTimeout = startupTimeout; + return self(); + } + + public enum NetworkAutoDetectMode { + IPV6_OR_IPV4("1"), + IPV4_ONLY("4"), + IPV6_ONLY("6"); + + private String value; + + NetworkAutoDetectMode(String value) { + this.value = value; + } + } +} diff --git a/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java b/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java new file mode 100644 index 00000000000..d69d519c8ff --- /dev/null +++ b/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java @@ -0,0 +1,98 @@ +package org.testcontainers.containers; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.Bucket; +import com.amazonaws.services.s3.model.S3Object; +import org.apache.commons.io.IOUtils; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.Date; +import java.util.List; + +import static org.junit.Assert.*; + +public class CephContainerTest { + + @Rule + public CephContainer cephContainer = new CephContainer() + .withAwsAccessKey("test") + .withAwsSecretKey("test") + .withBucketName("test") + .withStartupTimeout(Duration.ofMinutes(5)); + + private AmazonS3 amazonS3; + + @Before + public void setUp() { + amazonS3 = AmazonS3ClientBuilder.standard() + .withCredentials(cephContainer.getAWSCredentialsProvider()) + .withEndpointConfiguration(cephContainer.getAWSEndpointConfiguration()) + .withPathStyleAccessEnabled(true) + .withClientConfiguration( + new ClientConfiguration() + .withProtocol(Protocol.HTTP) + .withSignerOverride("S3SignerType") + ) + .build(); + } + + @Test + public void testS3Bucket() { + String bucketName = "test2"; + + //check current bucket + assertTrue(amazonS3.doesBucketExistV2(cephContainer.getBucketName())); + + //create another bucket + amazonS3.createBucket(bucketName); + assertTrue(amazonS3.doesBucketExistV2(bucketName)); + List buckets = amazonS3.listBuckets(); + assertEquals(2, buckets.size()); + assertTrue(buckets.stream().anyMatch( + bucket -> bucket.getName().equals(bucketName) + )); + + //remove bucket + amazonS3.deleteBucket(bucketName); + assertFalse(amazonS3.doesBucketExistV2(bucketName)); + buckets = amazonS3.listBuckets(); + assertEquals(1, buckets.size()); + assertFalse(buckets.stream().anyMatch( + bucket -> bucket.getName().equals(bucketName) + )); + } + + @Test + public void testS3Object() throws IOException { + String objectId = "test"; + String testData = "This is test data"; + + //put object + amazonS3.putObject(cephContainer.getBucketName(), objectId, testData); + assertEquals(1, amazonS3.listObjects(cephContainer.getBucketName()).getObjectSummaries().size()); + S3Object object = amazonS3.getObject(cephContainer.getBucketName(), objectId); + assertEquals(testData, IOUtils.toString(object.getObjectContent(), StandardCharsets.UTF_8)); + + //generate presigned url and download file + URL url = amazonS3.generatePresignedUrl( + cephContainer.getBucketName(), + objectId, + Date.from(Instant.now().plusSeconds(Duration.ofMinutes(5).toMillis()))); + + try (InputStream inputStream = url.openStream()) { + assertEquals(testData, IOUtils.toString(inputStream, StandardCharsets.UTF_8)); + } + } + +} diff --git a/pom.xml b/pom.xml index bf158718386..7aa61de3745 100644 --- a/pom.xml +++ b/pom.xml @@ -166,6 +166,7 @@ modules/selenium modules/nginx modules/jdbc-test + modules/ceph From ef38b4249b2d031ec90db09d6186a8b6567ce20e Mon Sep 17 00:00:00 2001 From: Pavel Popov Date: Sun, 19 Nov 2017 15:58:13 +0300 Subject: [PATCH 2/7] Review fixes --- modules/ceph/pom.xml | 1 + .../containers/CephContainer.java | 60 +++++++------------ .../containers/CephContainerTest.java | 3 +- 3 files changed, 23 insertions(+), 41 deletions(-) diff --git a/modules/ceph/pom.xml b/modules/ceph/pom.xml index d933b9eabd7..63c6ac43f1d 100644 --- a/modules/ceph/pom.xml +++ b/modules/ceph/pom.xml @@ -24,6 +24,7 @@ com.amazonaws aws-java-sdk-s3 1.11.231 + provided diff --git a/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java b/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java index c892913f12d..565f3ec0390 100644 --- a/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java +++ b/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java @@ -4,13 +4,18 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; -import org.testcontainers.containers.wait.LogMessageWaitStrategy; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.testcontainers.containers.wait.HttpWaitStrategy; import java.time.Duration; +@Getter public class CephContainer> extends GenericContainer { - private static final String IMAGE = "ceph/demo"; + public static final String IMAGE = "ceph/demo"; + public static final int S3_PORT = 80; + public static final int REST_API_PORT = 5000; private String awsAccessKey = "ceph"; @@ -24,10 +29,8 @@ public class CephContainer> extends GenericCont private NetworkAutoDetectMode networkAutoDetectMode = NetworkAutoDetectMode.IPV4_ONLY; - private Duration startupTimeout = Duration.ofSeconds(20); - public CephContainer() { - super(IMAGE + ":latest"); + super(IMAGE + ":tag-stable-3.0-jewel-ubuntu-16.04"); } public CephContainer(String dockerImageName) { @@ -42,14 +45,20 @@ protected void configure() { withEnv("CEPH_DEMO_ACCESS_KEY", awsAccessKey); withEnv("CEPH_DEMO_SECRET_KEY", awsSecretKey); withEnv("CEPH_DEMO_BUCKET", bucketName); - withExposedPorts(6789, 6800, 6801, 6802, 6803, 6804, 6805, 80, 5000); + withExposedPorts(REST_API_PORT, S3_PORT); waitingFor( - new LogMessageWaitStrategy() - .withRegEx(".*\\/entrypoint.sh: SUCCESS\n") - .withStartupTimeout(startupTimeout) + new HttpWaitStrategy() + .forPath("/api/v0.1/health") + .forStatusCode(200) + .withStartupTimeout(Duration.ofMinutes(5)) ); } + @Override + protected Integer getLivenessCheckPort() { + return getMappedPort(REST_API_PORT); + } + public AWSCredentialsProvider getAWSCredentialsProvider() { return new AWSStaticCredentialsProvider( new BasicAWSCredentials(awsAccessKey, awsSecretKey) @@ -58,7 +67,7 @@ public AWSCredentialsProvider getAWSCredentialsProvider() { public AwsClientBuilder.EndpointConfiguration getAWSEndpointConfiguration() { return new AwsClientBuilder.EndpointConfiguration( - getContainerIpAddress() + ":" + getMappedPort(80), + getContainerIpAddress() + ":" + getMappedPort(S3_PORT), "us-east-1" ); } @@ -73,61 +82,34 @@ public SELF withAwsSecretKey(String awsSecretKey) { return self(); } - public String getBucketName() { - return bucketName; - } - public SELF withBucketName(String bucketName) { //because s3cmd transforming bucket name to uppercase this.bucketName = bucketName.toUpperCase(); return self(); } - public String getRgwName() { - return rgwName; - } - public SELF withRgwName(String rgwName) { this.rgwName = rgwName; return self(); } - public String getDemoUid() { - return demoUid; - } - public SELF withDemoUid(String demoUid) { this.demoUid = demoUid; return self(); } - public NetworkAutoDetectMode getNetworkAutoDetectMode() { - return networkAutoDetectMode; - } - public SELF withNetworkAutoDetectMode(NetworkAutoDetectMode networkAutoDetectMode) { this.networkAutoDetectMode = networkAutoDetectMode; return self(); } - public Duration getStartupTimeout() { - return startupTimeout; - } - - public SELF withStartupTimeout(Duration startupTimeout) { - this.startupTimeout = startupTimeout; - return self(); - } - + @AllArgsConstructor public enum NetworkAutoDetectMode { IPV6_OR_IPV4("1"), IPV4_ONLY("4"), IPV6_ONLY("6"); - private String value; + private final String value; - NetworkAutoDetectMode(String value) { - this.value = value; - } } } diff --git a/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java b/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java index d69d519c8ff..71c1239c355 100644 --- a/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java +++ b/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java @@ -28,8 +28,7 @@ public class CephContainerTest { public CephContainer cephContainer = new CephContainer() .withAwsAccessKey("test") .withAwsSecretKey("test") - .withBucketName("test") - .withStartupTimeout(Duration.ofMinutes(5)); + .withBucketName("test"); private AmazonS3 amazonS3; From b5ab8cf7d2e731e4292f7ecf148afb5df9b46779 Mon Sep 17 00:00:00 2001 From: Pavel Popov Date: Mon, 26 Feb 2018 15:03:47 +0300 Subject: [PATCH 3/7] Fix deps --- modules/ceph/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/ceph/build.gradle b/modules/ceph/build.gradle index 721b2e57f78..9e5db0f67c0 100644 --- a/modules/ceph/build.gradle +++ b/modules/ceph/build.gradle @@ -1,5 +1,7 @@ description = "Testcontainers :: JDBC :: Ceph" dependencies { + compile project(':testcontainers') + provided 'com.amazonaws:aws-java-sdk-s3:1.11.231' } From 736437e4641c85385fbdf658391518cf9c4af810 Mon Sep 17 00:00:00 2001 From: Pavel Popov Date: Sun, 10 Feb 2019 04:23:05 +0300 Subject: [PATCH 4/7] Another review fixes; Switch to new ceph image --- modules/ceph/build.gradle | 4 +-- .../containers/CephContainer.java | 33 ++++++++++--------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/modules/ceph/build.gradle b/modules/ceph/build.gradle index 9e5db0f67c0..75a6a98c444 100644 --- a/modules/ceph/build.gradle +++ b/modules/ceph/build.gradle @@ -1,7 +1,7 @@ -description = "Testcontainers :: JDBC :: Ceph" +description = "Testcontainers :: Ceph" dependencies { compile project(':testcontainers') - provided 'com.amazonaws:aws-java-sdk-s3:1.11.231' + provided 'com.amazonaws:aws-java-sdk-s3:1.11.495' } diff --git a/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java b/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java index 565f3ec0390..547919d9e58 100644 --- a/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java +++ b/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java @@ -6,15 +6,18 @@ import com.amazonaws.client.builder.AwsClientBuilder; import lombok.AllArgsConstructor; import lombok.Getter; -import org.testcontainers.containers.wait.HttpWaitStrategy; +import lombok.extern.slf4j.Slf4j; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; import java.time.Duration; +@Slf4j @Getter -public class CephContainer> extends GenericContainer { +public class CephContainer extends GenericContainer { - public static final String IMAGE = "ceph/demo"; - public static final int S3_PORT = 80; + public static final String IMAGE = "ceph/daemon"; + public static final int S3_PORT = 8080; public static final int REST_API_PORT = 5000; private String awsAccessKey = "ceph"; @@ -30,7 +33,7 @@ public class CephContainer> extends GenericCont private NetworkAutoDetectMode networkAutoDetectMode = NetworkAutoDetectMode.IPV4_ONLY; public CephContainer() { - super(IMAGE + ":tag-stable-3.0-jewel-ubuntu-16.04"); + super(IMAGE + ":v3.0.5-stable-3.0-luminous-centos-7"); } public CephContainer(String dockerImageName) { @@ -41,6 +44,7 @@ public CephContainer(String dockerImageName) { protected void configure() { withEnv("RGW_NAME", rgwName); withEnv("NETWORK_AUTO_DETECT", networkAutoDetectMode.value); + withEnv("CEPH_DAEMON", "demo"); withEnv("CEPH_DEMO_UID", demoUid); withEnv("CEPH_DEMO_ACCESS_KEY", awsAccessKey); withEnv("CEPH_DEMO_SECRET_KEY", awsSecretKey); @@ -49,14 +53,11 @@ protected void configure() { waitingFor( new HttpWaitStrategy() .forPath("/api/v0.1/health") + .forPort(REST_API_PORT) .forStatusCode(200) .withStartupTimeout(Duration.ofMinutes(5)) ); - } - - @Override - protected Integer getLivenessCheckPort() { - return getMappedPort(REST_API_PORT); + withLogConsumer(new Slf4jLogConsumer(log)); } public AWSCredentialsProvider getAWSCredentialsProvider() { @@ -72,33 +73,33 @@ public AwsClientBuilder.EndpointConfiguration getAWSEndpointConfiguration() { ); } - public SELF withAwsAccessKey(String awsAccessKey) { + public CephContainer withAwsAccessKey(String awsAccessKey) { this.awsAccessKey = awsAccessKey; return self(); } - public SELF withAwsSecretKey(String awsSecretKey) { + public CephContainer withAwsSecretKey(String awsSecretKey) { this.awsSecretKey = awsSecretKey; return self(); } - public SELF withBucketName(String bucketName) { + public CephContainer withBucketName(String bucketName) { //because s3cmd transforming bucket name to uppercase this.bucketName = bucketName.toUpperCase(); return self(); } - public SELF withRgwName(String rgwName) { + public CephContainer withRgwName(String rgwName) { this.rgwName = rgwName; return self(); } - public SELF withDemoUid(String demoUid) { + public CephContainer withDemoUid(String demoUid) { this.demoUid = demoUid; return self(); } - public SELF withNetworkAutoDetectMode(NetworkAutoDetectMode networkAutoDetectMode) { + public CephContainer withNetworkAutoDetectMode(NetworkAutoDetectMode networkAutoDetectMode) { this.networkAutoDetectMode = networkAutoDetectMode; return self(); } From 2417ebc2149913090baab5f042c495c785c7fdb5 Mon Sep 17 00:00:00 2001 From: Richard North Date: Fri, 8 May 2020 16:21:59 +0100 Subject: [PATCH 5/7] Tweaks to bring up to date --- docs/modules/ceph.md | 46 +++++++++++++++++++ docs/modules/localstack.md | 9 ++++ mkdocs.yml | 1 + .../containers/{ => ceph}/CephContainer.java | 10 ++-- .../{ => ceph}/CephContainerTest.java | 23 ++++++---- 5 files changed, 76 insertions(+), 13 deletions(-) create mode 100644 docs/modules/ceph.md rename modules/ceph/src/main/java/org/testcontainers/containers/{ => ceph}/CephContainer.java (92%) rename modules/ceph/src/test/java/org/testcontainers/containers/{ => ceph}/CephContainerTest.java (89%) diff --git a/docs/modules/ceph.md b/docs/modules/ceph.md new file mode 100644 index 00000000000..4fa94dffe3d --- /dev/null +++ b/docs/modules/ceph.md @@ -0,0 +1,46 @@ +# Ceph Module + +!!! note + This module is INCUBATING. While it is ready for use and operational in the current version of Testcontainers, it is possible that it may receive breaking changes in the future. See [our contributing guidelines](/contributing/#incubating-modules) for more information on our incubating modules policy. + +Testcontainers module for [Ceph](https://ceph.io/). + +## Ceph vs Localstack for simulating S3 + +One possible usage for the Ceph module is to take advantage of its S3 compatibility. + +For simulating S3 you may wish to also consider the [Localstack](./localstack.md) module. +Ceph and Localstack provide good compatibility with the S3 API, and we encourage you to evaluate both. + +Localstack is likely to be a better choice if you require simulation of other AWS services along with S3. + +## Usage example + +Creating a Ceph container: + + +[Creating a Ceph container](../../modules/ceph/src/test/java/org/testcontainers/containers/ceph/CephContainerTest.java) inside_block:creating_container + + +Configuring an S3 Client to use Ceph: + + +[Configuring an S3 Client to use Ceph](../../modules/ceph/src/test/java/org/testcontainers/containers/ceph/CephContainerTest.java) inside_block:setting_up_s3_client + + +## Adding this module to your project dependencies + +Add the following dependency to your `pom.xml`/`build.gradle` file: + +```groovy tab='Gradle' +testCompile "org.testcontainers:ceph:{{latest_version}}" +``` + +```xml tab='Maven' + + org.testcontainers + ceph + {{latest_version}} + test + +``` diff --git a/docs/modules/localstack.md b/docs/modules/localstack.md index 4ff46233b05..b4cd3c5e7dd 100644 --- a/docs/modules/localstack.md +++ b/docs/modules/localstack.md @@ -2,6 +2,15 @@ Testcontainers module for the Atlassian's [LocalStack](https://github.com/localstack/localstack), 'a fully functional local AWS cloud stack'. +## Ceph vs Localstack for simulating S3 + +One possible usage for the Localstack module is to take advantage of its S3 compatibility. + +For simulating S3 you may wish to also consider the [Ceph](./ceph.md) module. +Ceph and Localstack provide good compatibility with the S3 API, and we encourage you to evaluate both. + +Localstack is likely to be a better choice if you require simulation of other AWS services along with S3. + ## Usage example Running LocalStack as a stand-in for AWS S3 during a test: diff --git a/mkdocs.yml b/mkdocs.yml index e1b21d80811..5c113bd4fde 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -58,6 +58,7 @@ nav: - modules/databases/orientdb.md - modules/databases/postgres.md - modules/databases/presto.md + - modules/ceph.md - modules/docker_compose.md - modules/elasticsearch.md - modules/kafka.md diff --git a/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java b/modules/ceph/src/main/java/org/testcontainers/containers/ceph/CephContainer.java similarity index 92% rename from modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java rename to modules/ceph/src/main/java/org/testcontainers/containers/ceph/CephContainer.java index 547919d9e58..3e3daeab015 100644 --- a/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java +++ b/modules/ceph/src/main/java/org/testcontainers/containers/ceph/CephContainer.java @@ -1,4 +1,4 @@ -package org.testcontainers.containers; +package org.testcontainers.containers.ceph; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; @@ -7,6 +7,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; @@ -17,6 +18,8 @@ public class CephContainer extends GenericContainer { public static final String IMAGE = "ceph/daemon"; + public static final String DEFAULT_TAG = "v3.2.13-stable-3.2-mimic-centos-7"; + public static final int S3_PORT = 8080; public static final int REST_API_PORT = 5000; @@ -33,7 +36,7 @@ public class CephContainer extends GenericContainer { private NetworkAutoDetectMode networkAutoDetectMode = NetworkAutoDetectMode.IPV4_ONLY; public CephContainer() { - super(IMAGE + ":v3.0.5-stable-3.0-luminous-centos-7"); + super(IMAGE + ":" + DEFAULT_TAG); } public CephContainer(String dockerImageName) { @@ -52,7 +55,7 @@ protected void configure() { withExposedPorts(REST_API_PORT, S3_PORT); waitingFor( new HttpWaitStrategy() - .forPath("/api/v0.1/health") + .forPath("/") .forPort(REST_API_PORT) .forStatusCode(200) .withStartupTimeout(Duration.ofMinutes(5)) @@ -111,6 +114,5 @@ public enum NetworkAutoDetectMode { IPV6_ONLY("6"); private final String value; - } } diff --git a/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java b/modules/ceph/src/test/java/org/testcontainers/containers/ceph/CephContainerTest.java similarity index 89% rename from modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java rename to modules/ceph/src/test/java/org/testcontainers/containers/ceph/CephContainerTest.java index 71c1239c355..8f317702269 100644 --- a/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java +++ b/modules/ceph/src/test/java/org/testcontainers/containers/ceph/CephContainerTest.java @@ -1,4 +1,4 @@ -package org.testcontainers.containers; +package org.testcontainers.containers.ceph; import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; @@ -8,7 +8,6 @@ import com.amazonaws.services.s3.model.S3Object; import org.apache.commons.io.IOUtils; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import java.io.IOException; @@ -20,20 +19,26 @@ import java.util.Date; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class CephContainerTest { - @Rule - public CephContainer cephContainer = new CephContainer() - .withAwsAccessKey("test") - .withAwsSecretKey("test") - .withBucketName("test"); + public CephContainer cephContainer; private AmazonS3 amazonS3; @Before public void setUp() { + // creating_container { + cephContainer = new CephContainer("v3.2.13-stable-3.2-mimic-centos-7") + .withAwsAccessKey("test") + .withAwsSecretKey("test") + .withBucketName("test"); + // } + + // setting_up_s3_client { amazonS3 = AmazonS3ClientBuilder.standard() .withCredentials(cephContainer.getAWSCredentialsProvider()) .withEndpointConfiguration(cephContainer.getAWSEndpointConfiguration()) @@ -44,6 +49,7 @@ public void setUp() { .withSignerOverride("S3SignerType") ) .build(); + // } } @Test @@ -93,5 +99,4 @@ public void testS3Object() throws IOException { assertEquals(testData, IOUtils.toString(inputStream, StandardCharsets.UTF_8)); } } - } From 480817ae6b91cbd28a42b1a102761c6b5adbb2b0 Mon Sep 17 00:00:00 2001 From: Richard North Date: Fri, 8 May 2020 20:17:44 +0100 Subject: [PATCH 6/7] Undo package move --- docs/modules/ceph.md | 4 ++-- .../testcontainers/containers/{ceph => }/CephContainer.java | 3 +-- .../containers/{ceph => }/CephContainerTest.java | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) rename modules/ceph/src/main/java/org/testcontainers/containers/{ceph => }/CephContainer.java (97%) rename modules/ceph/src/test/java/org/testcontainers/containers/{ceph => }/CephContainerTest.java (98%) diff --git a/docs/modules/ceph.md b/docs/modules/ceph.md index 4fa94dffe3d..0300339cfb5 100644 --- a/docs/modules/ceph.md +++ b/docs/modules/ceph.md @@ -19,13 +19,13 @@ Localstack is likely to be a better choice if you require simulation of other AW Creating a Ceph container: -[Creating a Ceph container](../../modules/ceph/src/test/java/org/testcontainers/containers/ceph/CephContainerTest.java) inside_block:creating_container +[Creating a Ceph container](../../modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java) inside_block:creating_container Configuring an S3 Client to use Ceph: -[Configuring an S3 Client to use Ceph](../../modules/ceph/src/test/java/org/testcontainers/containers/ceph/CephContainerTest.java) inside_block:setting_up_s3_client +[Configuring an S3 Client to use Ceph](../../modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java) inside_block:setting_up_s3_client ## Adding this module to your project dependencies diff --git a/modules/ceph/src/main/java/org/testcontainers/containers/ceph/CephContainer.java b/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java similarity index 97% rename from modules/ceph/src/main/java/org/testcontainers/containers/ceph/CephContainer.java rename to modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java index 3e3daeab015..013350ebdcc 100644 --- a/modules/ceph/src/main/java/org/testcontainers/containers/ceph/CephContainer.java +++ b/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java @@ -1,4 +1,4 @@ -package org.testcontainers.containers.ceph; +package org.testcontainers.containers; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; @@ -7,7 +7,6 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; diff --git a/modules/ceph/src/test/java/org/testcontainers/containers/ceph/CephContainerTest.java b/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java similarity index 98% rename from modules/ceph/src/test/java/org/testcontainers/containers/ceph/CephContainerTest.java rename to modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java index 8f317702269..361a1afd267 100644 --- a/modules/ceph/src/test/java/org/testcontainers/containers/ceph/CephContainerTest.java +++ b/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java @@ -1,4 +1,4 @@ -package org.testcontainers.containers.ceph; +package org.testcontainers.containers; import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; From 34f72fbeeac2304466406e72c9cf8793122f799d Mon Sep 17 00:00:00 2001 From: Richard North Date: Fri, 8 May 2020 20:43:04 +0100 Subject: [PATCH 7/7] Fix start/use ordering and adopt try-with-resources --- .../containers/CephContainerTest.java | 134 ++++++++++-------- 1 file changed, 72 insertions(+), 62 deletions(-) diff --git a/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java b/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java index 361a1afd267..620250bb3be 100644 --- a/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java +++ b/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java @@ -7,7 +7,6 @@ import com.amazonaws.services.s3.model.Bucket; import com.amazonaws.services.s3.model.S3Object; import org.apache.commons.io.IOUtils; -import org.junit.Before; import org.junit.Test; import java.io.IOException; @@ -24,79 +23,90 @@ import static org.junit.Assert.assertTrue; public class CephContainerTest { - - public CephContainer cephContainer; - - private AmazonS3 amazonS3; - - @Before - public void setUp() { - // creating_container { - cephContainer = new CephContainer("v3.2.13-stable-3.2-mimic-centos-7") - .withAwsAccessKey("test") - .withAwsSecretKey("test") - .withBucketName("test"); - // } - - // setting_up_s3_client { - amazonS3 = AmazonS3ClientBuilder.standard() - .withCredentials(cephContainer.getAWSCredentialsProvider()) - .withEndpointConfiguration(cephContainer.getAWSEndpointConfiguration()) - .withPathStyleAccessEnabled(true) - .withClientConfiguration( - new ClientConfiguration() - .withProtocol(Protocol.HTTP) - .withSignerOverride("S3SignerType") - ) - .build(); - // } - } - @Test public void testS3Bucket() { - String bucketName = "test2"; - - //check current bucket - assertTrue(amazonS3.doesBucketExistV2(cephContainer.getBucketName())); - - //create another bucket - amazonS3.createBucket(bucketName); - assertTrue(amazonS3.doesBucketExistV2(bucketName)); - List buckets = amazonS3.listBuckets(); - assertEquals(2, buckets.size()); - assertTrue(buckets.stream().anyMatch( + try ( + // creating_container { + CephContainer cephContainer = new CephContainer() + .withAwsAccessKey("test") + .withAwsSecretKey("test") + .withBucketName("test"); + // } + ) { + cephContainer.start(); + + final AmazonS3 amazonS3 = buildS3Client(cephContainer); + + String bucketName = "test2"; + + //check current bucket + assertTrue(amazonS3.doesBucketExistV2(cephContainer.getBucketName())); + + //create another bucket + amazonS3.createBucket(bucketName); + assertTrue(amazonS3.doesBucketExistV2(bucketName)); + List buckets = amazonS3.listBuckets(); + assertEquals(2, buckets.size()); + assertTrue(buckets.stream().anyMatch( bucket -> bucket.getName().equals(bucketName) - )); - - //remove bucket - amazonS3.deleteBucket(bucketName); - assertFalse(amazonS3.doesBucketExistV2(bucketName)); - buckets = amazonS3.listBuckets(); - assertEquals(1, buckets.size()); - assertFalse(buckets.stream().anyMatch( + )); + + //remove bucket + amazonS3.deleteBucket(bucketName); + assertFalse(amazonS3.doesBucketExistV2(bucketName)); + buckets = amazonS3.listBuckets(); + assertEquals(1, buckets.size()); + assertFalse(buckets.stream().anyMatch( bucket -> bucket.getName().equals(bucketName) - )); + )); + } } @Test public void testS3Object() throws IOException { - String objectId = "test"; - String testData = "This is test data"; - - //put object - amazonS3.putObject(cephContainer.getBucketName(), objectId, testData); - assertEquals(1, amazonS3.listObjects(cephContainer.getBucketName()).getObjectSummaries().size()); - S3Object object = amazonS3.getObject(cephContainer.getBucketName(), objectId); - assertEquals(testData, IOUtils.toString(object.getObjectContent(), StandardCharsets.UTF_8)); - - //generate presigned url and download file - URL url = amazonS3.generatePresignedUrl( + try ( + CephContainer cephContainer = new CephContainer() + .withAwsAccessKey("test") + .withAwsSecretKey("test") + .withBucketName("test"); + ) { + cephContainer.start(); + + final AmazonS3 amazonS3 = buildS3Client(cephContainer); + String objectId = "test"; + String testData = "This is test data"; + + //put object + amazonS3.putObject(cephContainer.getBucketName(), objectId, testData); + assertEquals(1, amazonS3.listObjects(cephContainer.getBucketName()).getObjectSummaries().size()); + S3Object object = amazonS3.getObject(cephContainer.getBucketName(), objectId); + assertEquals(testData, IOUtils.toString(object.getObjectContent(), StandardCharsets.UTF_8)); + + //generate presigned url and download file + URL url = amazonS3.generatePresignedUrl( cephContainer.getBucketName(), objectId, Date.from(Instant.now().plusSeconds(Duration.ofMinutes(5).toMillis()))); - try (InputStream inputStream = url.openStream()) { - assertEquals(testData, IOUtils.toString(inputStream, StandardCharsets.UTF_8)); + try (InputStream inputStream = url.openStream()) { + assertEquals(testData, IOUtils.toString(inputStream, StandardCharsets.UTF_8)); + } } } + + private AmazonS3 buildS3Client(final CephContainer cephContainer) { + // setting_up_s3_client { + final AmazonS3 amazonS3 = AmazonS3ClientBuilder.standard() + .withCredentials(cephContainer.getAWSCredentialsProvider()) + .withEndpointConfiguration(cephContainer.getAWSEndpointConfiguration()) + .withPathStyleAccessEnabled(true) + .withClientConfiguration( + new ClientConfiguration() + .withProtocol(Protocol.HTTP) + .withSignerOverride("S3SignerType") + ) + .build(); + // } + return amazonS3; + } }