Skip to content

Commit 2afe714

Browse files
nicoloboschinicoloboschi
authored
Pulsar: add flag to enable transactions and set configuration (#5479)
* Add `withTransactions()` method to enable transactions on Pulsar container * Set default version to latest released (2.10.0) * New docker command that enables the user to easily change configuration parameters * Added new tests to cover new methods * Improved the documentation with simple usage and new methods
1 parent 745bfc3 commit 2afe714

File tree

4 files changed

+195
-25
lines changed

4 files changed

+195
-25
lines changed

‎docs/modules/pulsar.md‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,51 @@
11
# Apache Pulsar Module
22

3+
Testcontainers can be used to automatically create [Apache Pulsar](https://pulsar.apache.org) containers without external services.
4+
5+
It's based on the official Apache Pulsar docker image, it is recommended to read the [official guide](https://pulsar.apache.org/docs/next/getting-started-docker/).
6+
7+
## Example
8+
9+
Create a `PulsarContainer` to use it in your tests:
10+
11+
<!--codeinclude-->
12+
[Create a Pulsar container](../../modules/pulsar/src/test/java/org/testcontainers/containers/PulsarContainerTest.java) inside_block:constructorWithVersion
13+
<!--/codeinclude-->
14+
15+
Then you can retrieve the broker and the admin url:
16+
17+
<!--codeinclude-->
18+
[Get broker and admin urls](../../modules/pulsar/src/test/java/org/testcontainers/containers/PulsarContainerTest.java) inside_block:coordinates
19+
<!--/codeinclude-->
20+
21+
## Options
22+
23+
### Configuration
24+
If you need to set Pulsar configuration variables you can use the native APIs and set each variable with `PULSAR_PREFIX_` as prefix.
25+
26+
For example, if you want to enable `brokerDeduplicationEnabled`:
27+
28+
<!--codeinclude-->
29+
[Set configuration variables](../../modules/pulsar/src/test/java/org/testcontainers/containers/PulsarContainerTest.java) inside_block:constructorWithEnv
30+
<!--/codeinclude-->
31+
32+
### Pulsar IO
33+
34+
If you need to test Pulsar IO framework you can enable the Pulsar Functions Worker:
35+
36+
<!--codeinclude-->
37+
[Create a Pulsar container with functions worker](../../modules/pulsar/src/test/java/org/testcontainers/containers/PulsarContainerTest.java) inside_block:constructorWithFunctionsWorker
38+
<!--/codeinclude-->
39+
40+
### Pulsar Transactions
41+
42+
If you need to test Pulsar Transactions you can enable the transactions feature:
43+
44+
<!--codeinclude-->
45+
[Create a Pulsar container with transactions](../../modules/pulsar/src/test/java/org/testcontainers/containers/PulsarContainerTest.java) inside_block:constructorWithTransactions
46+
<!--/codeinclude-->
47+
48+
349
## Adding this module to your project dependencies
450

551
Add the following dependency to your `pom.xml`/`build.gradle` file:

‎modules/pulsar/build.gradle‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ description = "Testcontainers :: Pulsar"
33
dependencies {
44
api project(':testcontainers')
55

6-
testImplementation group: 'org.apache.pulsar', name: 'pulsar-client', version: '2.7.4'
6+
testImplementation group: 'org.apache.pulsar', name: 'pulsar-client', version: '2.10.0'
77
testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.23.1'
8-
testImplementation group: 'org.apache.pulsar', name: 'pulsar-client-admin', version: '2.7.4'
8+
testImplementation group: 'org.apache.pulsar', name: 'pulsar-client-admin', version: '2.10.0'
99
}

‎modules/pulsar/src/main/java/org/testcontainers/containers/PulsarContainer.java‎

Lines changed: 44 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,12 @@
22

33
import org.testcontainers.containers.wait.strategy.Wait;
44
import org.testcontainers.containers.wait.strategy.WaitAllStrategy;
5+
import org.testcontainers.containers.wait.strategy.WaitStrategy;
56
import org.testcontainers.utility.DockerImageName;
67

8+
import java.util.ArrayList;
9+
import java.util.List;
10+
711
/**
812
* This container wraps Apache Pulsar running in standalone mode
913
*/
@@ -15,13 +19,21 @@ public class PulsarContainer extends GenericContainer<PulsarContainer> {
1519

1620
public static final String METRICS_ENDPOINT = "/metrics";
1721

22+
/**
23+
* See <a href="https://github.com/apache/pulsar/blob/master/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java">SystemTopicNames</a>.
24+
*/
25+
private static final String TRANSACTION_TOPIC_ENDPOINT =
26+
"/admin/v2/persistent/pulsar/system/transaction_coordinator_assign/partitions";
27+
1828
private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("apachepulsar/pulsar");
1929

2030
@Deprecated
21-
private static final String DEFAULT_TAG = "2.2.0";
31+
private static final String DEFAULT_TAG = "2.10.0";
2232

2333
private boolean functionsWorkerEnabled = false;
2434

35+
private boolean transactionsEnabled = false;
36+
2537
/**
2638
* @deprecated use {@link PulsarContainer(DockerImageName)} instead
2739
*/
@@ -41,36 +53,55 @@ public PulsarContainer(String pulsarVersion) {
4153
public PulsarContainer(final DockerImageName dockerImageName) {
4254
super(dockerImageName);
4355
dockerImageName.assertCompatibleWith(DockerImageName.parse("apachepulsar/pulsar"));
44-
4556
withExposedPorts(BROKER_PORT, BROKER_HTTP_PORT);
46-
withCommand("/pulsar/bin/pulsar", "standalone", "--no-functions-worker", "-nss");
47-
waitingFor(Wait.forHttp(METRICS_ENDPOINT).forStatusCode(200).forPort(BROKER_HTTP_PORT));
4857
}
4958

5059
@Override
5160
protected void configure() {
5261
super.configure();
53-
54-
if (functionsWorkerEnabled) {
55-
withCommand("/pulsar/bin/pulsar", "standalone");
56-
waitingFor(
57-
new WaitAllStrategy()
58-
.withStrategy(waitStrategy)
59-
.withStrategy(Wait.forLogMessage(".*Function worker service started.*", 1))
60-
);
61-
}
62+
setupCommandAndEnv();
6263
}
6364

6465
public PulsarContainer withFunctionsWorker() {
6566
functionsWorkerEnabled = true;
6667
return this;
6768
}
6869

70+
public PulsarContainer withTransactions() {
71+
transactionsEnabled = true;
72+
return this;
73+
}
74+
6975
public String getPulsarBrokerUrl() {
7076
return String.format("pulsar://%s:%s", getHost(), getMappedPort(BROKER_PORT));
7177
}
7278

7379
public String getHttpServiceUrl() {
7480
return String.format("http://%s:%s", getHost(), getMappedPort(BROKER_HTTP_PORT));
7581
}
82+
83+
protected void setupCommandAndEnv() {
84+
String standaloneBaseCommand =
85+
"/pulsar/bin/apply-config-from-env.py /pulsar/conf/standalone.conf " + "&& bin/pulsar standalone";
86+
87+
if (!functionsWorkerEnabled) {
88+
standaloneBaseCommand += " --no-functions-worker -nss";
89+
}
90+
91+
withCommand("/bin/bash", "-c", standaloneBaseCommand);
92+
93+
List<WaitStrategy> waitStrategies = new ArrayList<>();
94+
waitStrategies.add(Wait.defaultWaitStrategy());
95+
waitStrategies.add(Wait.forHttp(METRICS_ENDPOINT).forStatusCode(200).forPort(BROKER_HTTP_PORT));
96+
if (transactionsEnabled) {
97+
withEnv("PULSAR_PREFIX_transactionCoordinatorEnabled", "true");
98+
waitStrategies.add(Wait.forHttp(TRANSACTION_TOPIC_ENDPOINT).forStatusCode(200).forPort(BROKER_HTTP_PORT));
99+
}
100+
if (functionsWorkerEnabled) {
101+
waitStrategies.add(Wait.forLogMessage(".*Function worker service started.*", 1));
102+
}
103+
final WaitAllStrategy compoundedWaitStrategy = new WaitAllStrategy();
104+
waitStrategies.forEach(compoundedWaitStrategy::withStrategy);
105+
waitingFor(compoundedWaitStrategy);
106+
}
76107
}

‎modules/pulsar/src/test/java/org/testcontainers/containers/PulsarContainerTest.java‎

Lines changed: 103 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
import org.apache.pulsar.client.api.Message;
77
import org.apache.pulsar.client.api.Producer;
88
import org.apache.pulsar.client.api.PulsarClient;
9+
import org.apache.pulsar.client.api.Schema;
10+
import org.apache.pulsar.client.api.SubscriptionInitialPosition;
11+
import org.apache.pulsar.client.api.transaction.Transaction;
912
import org.junit.Test;
1013
import org.testcontainers.utility.DockerImageName;
1114

@@ -19,36 +22,103 @@ public class PulsarContainerTest {
1922

2023
public static final String TEST_TOPIC = "test_topic";
2124

22-
private static final DockerImageName PULSAR_IMAGE = DockerImageName.parse("apachepulsar/pulsar:2.2.0");
25+
private static final DockerImageName PULSAR_IMAGE = DockerImageName.parse("apachepulsar/pulsar:2.10.0");
2326

2427
@Test
2528
public void testUsage() throws Exception {
26-
try (PulsarContainer pulsar = new PulsarContainer(PULSAR_IMAGE)) {
29+
try (
30+
// do not use PULSAR_IMAGE to make the doc looks easier
31+
// constructorWithVersion {
32+
PulsarContainer pulsar = new PulsarContainer(DockerImageName.parse("apachepulsar/pulsar:2.10.0"));
33+
// }
34+
) {
35+
pulsar.start();
36+
// coordinates {
37+
final String pulsarBrokerUrl = pulsar.getPulsarBrokerUrl();
38+
final String httpServiceUrl = pulsar.getHttpServiceUrl();
39+
// }
40+
testPulsarFunctionality(pulsarBrokerUrl);
41+
}
42+
}
43+
44+
@Test
45+
public void envVarsUsage() throws Exception {
46+
try (
47+
// constructorWithEnv {
48+
PulsarContainer pulsar = new PulsarContainer(PULSAR_IMAGE)
49+
.withEnv("PULSAR_PREFIX_brokerDeduplicationEnabled", "true");
50+
// }
51+
) {
2752
pulsar.start();
2853
testPulsarFunctionality(pulsar.getPulsarBrokerUrl());
2954
}
3055
}
3156

3257
@Test
3358
public void shouldNotEnableFunctionsWorkerByDefault() throws Exception {
34-
try (PulsarContainer pulsar = new PulsarContainer("2.5.1")) {
59+
try (PulsarContainer pulsar = new PulsarContainer(PULSAR_IMAGE)) {
3560
pulsar.start();
3661

37-
PulsarAdmin pulsarAdmin = PulsarAdmin.builder().serviceHttpUrl(pulsar.getHttpServiceUrl()).build();
38-
39-
assertThatThrownBy(() -> pulsarAdmin.functions().getFunctions("public", "default"))
40-
.isInstanceOf(PulsarAdminException.class);
62+
try (PulsarAdmin pulsarAdmin = PulsarAdmin.builder().serviceHttpUrl(pulsar.getHttpServiceUrl()).build()) {
63+
assertThatThrownBy(() -> pulsarAdmin.functions().getFunctions("public", "default"))
64+
.isInstanceOf(PulsarAdminException.class);
65+
}
4166
}
4267
}
4368

4469
@Test
4570
public void shouldWaitForFunctionsWorkerStarted() throws Exception {
46-
try (PulsarContainer pulsar = new PulsarContainer("2.5.1").withFunctionsWorker()) {
71+
try (
72+
// constructorWithFunctionsWorker {
73+
PulsarContainer pulsar = new PulsarContainer(PULSAR_IMAGE).withFunctionsWorker();
74+
// }
75+
) {
76+
pulsar.start();
77+
78+
try (PulsarAdmin pulsarAdmin = PulsarAdmin.builder().serviceHttpUrl(pulsar.getHttpServiceUrl()).build()) {
79+
assertThat(pulsarAdmin.functions().getFunctions("public", "default")).hasSize(0);
80+
}
81+
}
82+
}
83+
84+
@Test
85+
public void testTransactions() throws Exception {
86+
try (
87+
// constructorWithTransactions {
88+
PulsarContainer pulsar = new PulsarContainer(PULSAR_IMAGE).withTransactions();
89+
// }
90+
) {
4791
pulsar.start();
4892

49-
PulsarAdmin pulsarAdmin = PulsarAdmin.builder().serviceHttpUrl(pulsar.getHttpServiceUrl()).build();
93+
try (PulsarAdmin pulsarAdmin = PulsarAdmin.builder().serviceHttpUrl(pulsar.getHttpServiceUrl()).build()) {
94+
assertThat(
95+
pulsarAdmin
96+
.topics()
97+
.getList("pulsar/system")
98+
.contains("persistent://pulsar/system/transaction_coordinator_assign-partition-0")
99+
)
100+
.isTrue();
101+
}
102+
testTransactionFunctionality(pulsar.getPulsarBrokerUrl());
103+
}
104+
}
105+
106+
@Test
107+
public void testTransactionsAndFunctionsWorker() throws Exception {
108+
try (PulsarContainer pulsar = new PulsarContainer(PULSAR_IMAGE).withTransactions().withFunctionsWorker()) {
109+
pulsar.start();
50110

51-
assertThat(pulsarAdmin.functions().getFunctions("public", "default")).hasSize(0);
111+
try (PulsarAdmin pulsarAdmin = PulsarAdmin.builder().serviceHttpUrl(pulsar.getHttpServiceUrl()).build();) {
112+
assertThat(
113+
pulsarAdmin
114+
.topics()
115+
.getList("pulsar/system")
116+
.contains("persistent://pulsar/system/transaction_coordinator_assign-partition-0")
117+
)
118+
.isTrue();
119+
assertThat(pulsarAdmin.functions().getFunctions("public", "default")).hasSize(0);
120+
}
121+
testTransactionFunctionality(pulsar.getPulsarBrokerUrl());
52122
}
53123
}
54124

@@ -65,4 +135,27 @@ protected void testPulsarFunctionality(String pulsarBrokerUrl) throws Exception
65135
assertThat(new String(message.getData())).isEqualTo("test containers");
66136
}
67137
}
138+
139+
protected void testTransactionFunctionality(String pulsarBrokerUrl) throws Exception {
140+
try (
141+
PulsarClient client = PulsarClient.builder().serviceUrl(pulsarBrokerUrl).enableTransaction(true).build();
142+
Consumer<String> consumer = client
143+
.newConsumer(Schema.STRING)
144+
.topic("transaction-topic")
145+
.subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
146+
.subscriptionName("test-transaction-sub")
147+
.subscribe();
148+
Producer<String> producer = client
149+
.newProducer(Schema.STRING)
150+
.sendTimeout(0, TimeUnit.SECONDS)
151+
.topic("transaction-topic")
152+
.create()
153+
) {
154+
final Transaction transaction = client.newTransaction().build().get();
155+
producer.newMessage(transaction).value("first").send();
156+
transaction.commit();
157+
Message<String> message = consumer.receive();
158+
assertThat(message.getValue()).isEqualTo("first");
159+
}
160+
}
68161
}

0 commit comments

Comments
 (0)