Skip to content

Commit f763945

Browse files
eddumelendezeddumelendez
authored
Fix register listeners in RedpandaContainer (#9247)
Previously, RedpandaContainer allowed to register a new listener using a Supplier and the new listener's host was also added as a network alias. However, if the listener's host was an IP the listener configuration was wrong and didn't allow to connect to the broker.
1 parent 031fd06 commit f763945

File tree

6 files changed

+197
-24
lines changed

6 files changed

+197
-24
lines changed

‎docs/modules/redpanda.md‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,26 @@ Client using the new registered listener:
6363
[Produce/Consume via new listener](../../modules/redpanda/src/test/java/org/testcontainers/redpanda/RedpandaContainerTest.java) inside_block:produceConsumeMessage
6464
<!--/codeinclude-->
6565

66+
The following examples shows how to register a proxy as a new listener in `RedpandaContainer`:
67+
68+
Use `SocatContainer` to create the proxy
69+
70+
<!--codeinclude-->
71+
[Create Proxy](../../modules/redpanda/src/test/java/org/testcontainers/redpanda/RedpandaContainerTest.java) inside_block:createProxy
72+
<!--/codeinclude-->
73+
74+
Register the listener and advertised listener
75+
76+
<!--codeinclude-->
77+
[Register Listener](../../modules/redpanda/src/test/java/org/testcontainers/redpanda/RedpandaContainerTest.java) inside_block:registerListenerAndAdvertisedListener
78+
<!--/codeinclude-->
79+
80+
Client using the new registered listener:
81+
82+
<!--codeinclude-->
83+
[Produce/Consume via new listener](../../modules/redpanda/src/test/java/org/testcontainers/redpanda/RedpandaContainerTest.java) inside_block:produceConsumeMessageFromProxy
84+
<!--/codeinclude-->
85+
6686
## Adding this module to your project dependencies
6787

6888
Add the following dependency to your `pom.xml`/`build.gradle` file:

‎modules/redpanda/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,5 @@ dependencies {
77
testImplementation 'org.apache.kafka:kafka-clients:3.8.0'
88
testImplementation 'org.assertj:assertj-core:3.26.3'
99
testImplementation 'io.rest-assured:rest-assured:5.5.0'
10+
testImplementation 'org.awaitility:awaitility:4.2.0'
1011
}

‎modules/redpanda/src/main/java/org/testcontainers/redpanda/RedpandaContainer.java‎

Lines changed: 102 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,11 @@ public class RedpandaContainer extends GenericContainer<RedpandaContainer> {
7171

7272
private final List<String> superusers = new ArrayList<>();
7373

74+
@Deprecated
7475
private final Set<Supplier<Listener>> listenersValueSupplier = new HashSet<>();
7576

77+
private final Map<String, Supplier<String>> listeners = new HashMap<>();
78+
7679
public RedpandaContainer(String image) {
7780
this(DockerImageName.parse(image));
7881
}
@@ -109,6 +112,7 @@ protected void configure() {
109112
.map(Supplier::get)
110113
.map(Listener::getAddress)
111114
.forEach(this::withNetworkAliases);
115+
this.listeners.keySet().stream().map(listener -> listener.split(":")[0]).forEach(this::withNetworkAliases);
112116
}
113117

114118
@SneakyThrows
@@ -212,13 +216,74 @@ public RedpandaContainer withSuperuser(String username) {
212216
* </ul>
213217
* @param listenerSupplier a supplier that will provide a listener
214218
* @return this {@link RedpandaContainer} instance
219+
* @deprecated use {@link #withListener(String, Supplier)} instead
215220
*/
221+
@Deprecated
216222
public RedpandaContainer withListener(Supplier<String> listenerSupplier) {
217223
String[] parts = listenerSupplier.get().split(":");
218224
this.listenersValueSupplier.add(() -> new Listener(parts[0], Integer.parseInt(parts[1])));
219225
return this;
220226
}
221227

228+
/**
229+
* Add a listener in the format {@code host:port}.
230+
* Host will be included as a network alias.
231+
* <p>
232+
* Use it to register additional connections to the Kafka broker within the same container network.
233+
* <p>
234+
* The listener will be added to the list of default listeners.
235+
* <p>
236+
* Default listeners:
237+
* <ul>
238+
* <li>0.0.0.0:9092</li>
239+
* <li>0.0.0.0:9093</li>
240+
* </ul>
241+
* <p>
242+
* The listener will be added to the list of default advertised listeners.
243+
* <p>
244+
* Default advertised listeners:
245+
* <ul>
246+
* <li>{@code container.getConfig().getHostName():9092}</li>
247+
* <li>{@code container.getHost():container.getMappedPort(9093)}</li>
248+
* </ul>
249+
* @param listener a listener with format {@code host:port}
250+
* @return this {@link RedpandaContainer} instance
251+
*/
252+
public RedpandaContainer withListener(String listener) {
253+
this.listeners.put(listener, () -> listener);
254+
return this;
255+
}
256+
257+
/**
258+
* Add a listener in the format {@code host:port} and a {@link Supplier} for the advertised listener.
259+
* Host from listener will be included as a network alias.
260+
* <p>
261+
* Use it to register additional connections to the Kafka broker from outside the container network
262+
* <p>
263+
* The listener will be added to the list of default listeners.
264+
* <p>
265+
* Default listeners:
266+
* <ul>
267+
* <li>0.0.0.0:9092</li>
268+
* <li>0.0.0.0:9093</li>
269+
* </ul>
270+
* <p>
271+
* The {@link Supplier} will be added to the list of default advertised listeners.
272+
* <p>
273+
* Default advertised listeners:
274+
* <ul>
275+
* <li>{@code container.getConfig().getHostName():9092}</li>
276+
* <li>{@code container.getHost():container.getMappedPort(9093)}</li>
277+
* </ul>
278+
* @param listener a supplier that will provide a listener
279+
* @param advertisedListener a supplier that will provide a listener
280+
* @return this {@link RedpandaContainer} instance
281+
*/
282+
public RedpandaContainer withListener(String listener, Supplier<String> advertisedListener) {
283+
this.listeners.put(listener, advertisedListener);
284+
return this;
285+
}
286+
222287
private Transferable getBootstrapFile(Configuration cfg) {
223288
Map<String, Object> kafkaApi = new HashMap<>();
224289
kafkaApi.put("enableAuthorization", this.enableAuthorization);
@@ -233,6 +298,12 @@ private Transferable getBootstrapFile(Configuration cfg) {
233298
}
234299

235300
private Transferable getRedpandaFile(Configuration cfg) {
301+
Map<String, Object> kafkaApi = new HashMap<>();
302+
kafkaApi.put("authenticationMethod", this.authenticationMethod);
303+
kafkaApi.put("enableAuthorization", this.enableAuthorization);
304+
kafkaApi.put("advertisedHost", getHost());
305+
kafkaApi.put("advertisedPort", getMappedPort(9092));
306+
236307
List<Map<String, Object>> listeners =
237308
this.listenersValueSupplier.stream()
238309
.map(Supplier::get)
@@ -244,19 +315,44 @@ private Transferable getRedpandaFile(Configuration cfg) {
244315
return listenerMap;
245316
})
246317
.collect(Collectors.toList());
247-
248-
Map<String, Object> kafkaApi = new HashMap<>();
249-
kafkaApi.put("authenticationMethod", this.authenticationMethod);
250-
kafkaApi.put("enableAuthorization", this.enableAuthorization);
251-
kafkaApi.put("advertisedHost", getHost());
252-
kafkaApi.put("advertisedPort", getMappedPort(9092));
253318
kafkaApi.put("listeners", listeners);
254319

320+
List<Map<String, Object>> kafkaListeners =
321+
this.listeners.keySet()
322+
.stream()
323+
.map(listener -> {
324+
Map<String, Object> listenerMap = new HashMap<>();
325+
listenerMap.put("name", listener.split(":")[0]);
326+
listenerMap.put("address", listener.split(":")[0]);
327+
listenerMap.put("port", listener.split(":")[1]);
328+
listenerMap.put("authentication_method", this.authenticationMethod);
329+
return listenerMap;
330+
})
331+
.collect(Collectors.toList());
332+
333+
List<Map<String, Object>> kafkaAdvertisedListeners =
334+
this.listeners.entrySet()
335+
.stream()
336+
.map(entry -> {
337+
String advertisedListener = entry.getValue().get();
338+
Map<String, Object> listenerMap = new HashMap<>();
339+
listenerMap.put("name", entry.getKey().split(":")[0]);
340+
listenerMap.put("address", advertisedListener.split(":")[0]);
341+
listenerMap.put("port", advertisedListener.split(":")[1]);
342+
return listenerMap;
343+
})
344+
.collect(Collectors.toList());
345+
346+
Map<String, Object> kafka = new HashMap<>();
347+
kafka.put("listeners", kafkaListeners);
348+
kafka.put("advertisedListeners", kafkaAdvertisedListeners);
349+
255350
Map<String, Object> schemaRegistry = new HashMap<>();
256351
schemaRegistry.put("authenticationMethod", this.schemaRegistryAuthenticationMethod);
257352

258353
Map<String, Object> root = new HashMap<>();
259354
root.put("kafkaApi", kafkaApi);
355+
root.put("kafka", kafka);
260356
root.put("schemaRegistry", schemaRegistry);
261357

262358
String file = resolveTemplate(cfg, "redpanda.yaml.ftl", root);

‎modules/redpanda/src/main/resources/testcontainers/redpanda.yaml.ftl‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,12 @@ redpanda:
2727
port: ${listener.port}
2828
authentication_method: ${listener.authentication_method}
2929
</#list>
30+
<#list kafka.listeners as listener>
31+
- address: ${listener.address}
32+
name: ${listener.name}
33+
port: ${listener.port}
34+
authentication_method: ${listener.authentication_method}
35+
</#list>
3036

3137
advertised_kafka_api:
3238
- address: ${ kafkaApi.advertisedHost }
@@ -40,6 +46,11 @@ redpanda:
4046
name: ${listener.address}
4147
port: ${listener.port}
4248
</#list>
49+
<#list kafka.advertisedListeners as listener>
50+
- address: ${listener.address}
51+
name: ${listener.name}
52+
port: ${listener.port}
53+
</#list>
4354

4455
schema_registry:
4556
schema_registry_api:

‎modules/redpanda/src/test/java/org/testcontainers/redpanda/AbstractRedpanda.java‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
import org.apache.kafka.clients.producer.ProducerRecord;
1414
import org.apache.kafka.common.serialization.StringDeserializer;
1515
import org.apache.kafka.common.serialization.StringSerializer;
16-
import org.rnorth.ducttape.unreliables.Unreliables;
16+
import org.awaitility.Awaitility;
1717

1818
import java.time.Duration;
1919
import java.util.Collection;
@@ -67,24 +67,17 @@ protected void testKafkaFunctionality(String bootstrapServers, int partitions, i
6767

6868
producer.send(new ProducerRecord<>(topicName, "testcontainers", "rulezzz")).get();
6969

70-
Unreliables.retryUntilTrue(
71-
10,
72-
TimeUnit.SECONDS,
73-
() -> {
70+
Awaitility
71+
.await()
72+
.atMost(Duration.ofSeconds(10))
73+
.untilAsserted(() -> {
7474
ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(100));
7575

76-
if (records.isEmpty()) {
77-
return false;
78-
}
79-
8076
assertThat(records)
8177
.hasSize(1)
8278
.extracting(ConsumerRecord::topic, ConsumerRecord::key, ConsumerRecord::value)
8379
.containsExactly(tuple(topicName, "testcontainers", "rulezzz"));
84-
85-
return true;
86-
}
87-
);
80+
});
8881

8982
consumer.unsubscribe();
9083
}

‎modules/redpanda/src/test/java/org/testcontainers/redpanda/RedpandaContainerTest.java‎

Lines changed: 57 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.junit.Test;
1616
import org.testcontainers.containers.GenericContainer;
1717
import org.testcontainers.containers.Network;
18+
import org.testcontainers.containers.SocatContainer;
1819
import org.testcontainers.images.builder.Transferable;
1920
import org.testcontainers.utility.DockerImageName;
2021

@@ -114,10 +115,37 @@ public void testSchemaRegistry() {
114115
public void testUsageWithListener() throws Exception {
115116
try (
116117
Network network = Network.newNetwork();
117-
// registerListener {
118118
RedpandaContainer redpanda = new RedpandaContainer("docker.redpanda.com/redpandadata/redpanda:v23.1.7")
119119
.withListener(() -> "redpanda:19092")
120120
.withNetwork(network);
121+
GenericContainer<?> kcat = new GenericContainer<>("confluentinc/cp-kcat:7.4.1")
122+
.withCreateContainerCmdModifier(cmd -> {
123+
cmd.withEntrypoint("sh");
124+
})
125+
.withCopyToContainer(Transferable.of("Message produced by kcat"), "/data/msgs.txt")
126+
.withNetwork(network)
127+
.withCommand("-c", "tail -f /dev/null")
128+
) {
129+
redpanda.start();
130+
kcat.start();
131+
132+
kcat.execInContainer("kcat", "-b", "redpanda:19092", "-t", "msgs", "-P", "-l", "/data/msgs.txt");
133+
String stdout = kcat
134+
.execInContainer("kcat", "-b", "redpanda:19092", "-C", "-t", "msgs", "-c", "1")
135+
.getStdout();
136+
137+
assertThat(stdout).contains("Message produced by kcat");
138+
}
139+
}
140+
141+
@Test
142+
public void testUsageWithListenerInTheSameNetwork() throws Exception {
143+
try (
144+
Network network = Network.newNetwork();
145+
// registerListener {
146+
RedpandaContainer kafka = new RedpandaContainer("docker.redpanda.com/redpandadata/redpanda:v23.1.7")
147+
.withListener("kafka:19092")
148+
.withNetwork(network);
121149
// }
122150
// createKCatContainer {
123151
GenericContainer<?> kcat = new GenericContainer<>("confluentinc/cp-kcat:7.4.1")
@@ -129,18 +157,42 @@ public void testUsageWithListener() throws Exception {
129157
.withCommand("-c", "tail -f /dev/null")
130158
// }
131159
) {
132-
redpanda.start();
160+
kafka.start();
133161
kcat.start();
162+
134163
// produceConsumeMessage {
135-
kcat.execInContainer("kcat", "-b", "redpanda:19092", "-t", "msgs", "-P", "-l", "/data/msgs.txt");
164+
kcat.execInContainer("kcat", "-b", "kafka:19092", "-t", "msgs", "-P", "-l", "/data/msgs.txt");
136165
String stdout = kcat
137-
.execInContainer("kcat", "-b", "redpanda:19092", "-C", "-t", "msgs", "-c", "1")
166+
.execInContainer("kcat", "-b", "kafka:19092", "-C", "-t", "msgs", "-c", "1")
138167
.getStdout();
139168
// }
169+
140170
assertThat(stdout).contains("Message produced by kcat");
141171
}
142172
}
143173

174+
@Test
175+
public void testUsageWithListenerFromProxy() throws Exception {
176+
try (
177+
Network network = Network.newNetwork();
178+
// createProxy {
179+
SocatContainer socat = new SocatContainer().withNetwork(network).withTarget(2000, "kafka", 19092);
180+
// }
181+
// registerListenerAndAdvertisedListener {
182+
RedpandaContainer kafka = new RedpandaContainer("docker.redpanda.com/redpandadata/redpanda:v23.1.7")
183+
.withListener("kafka:19092", () -> socat.getHost() + ":" + socat.getMappedPort(2000))
184+
.withNetwork(network)
185+
// }
186+
) {
187+
socat.start();
188+
kafka.start();
189+
// produceConsumeMessageFromProxy {
190+
String bootstrapServers = String.format("%s:%s", socat.getHost(), socat.getMappedPort(2000));
191+
testKafkaFunctionality(bootstrapServers);
192+
// }
193+
}
194+
}
195+
144196
@Test
145197
public void testUsageWithListenerAndSasl() throws Exception {
146198
final String username = "panda";
@@ -153,7 +205,7 @@ public void testUsageWithListenerAndSasl() throws Exception {
153205
.enableAuthorization()
154206
.enableSasl()
155207
.withSuperuser("panda")
156-
.withListener(() -> "my-panda:29092")
208+
.withListener("my-panda:29092")
157209
.withNetwork(network);
158210
GenericContainer<?> kcat = new GenericContainer<>("confluentinc/cp-kcat:7.4.1")
159211
.withCreateContainerCmdModifier(cmd -> {

0 commit comments

Comments
 (0)