Skip to content

Commit d01f509

Browse files
stefans-elasticstefans-elastic
authored
Kafka system tests variants (#15629)
* add kafka 4.0.0 test variant * kafka 3.6.0 variant * add more variants * cover kafka 0.10.2.1 and kafka 1.1.0 in system test variants * yaml format fix * yaml format fix * yaml format fix * oprimize docker image building
1 parent 8bf7fb5 commit d01f509

File tree

18 files changed

+344
-42
lines changed

18 files changed

+344
-42
lines changed

‎packages/kafka/_dev/deploy/docker/Dockerfile‎

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎packages/kafka/_dev/deploy/docker/Dockerfile-java8‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
FROM debian:stretch
2+
3+
USER root
4+
5+
ARG KAFKA_VERSION
6+
ARG SCALA_VERSION
7+
8+
ENV KAFKA_HOME=/kafka
9+
10+
ENV KAFKA_LOGS_DIR="/kafka-logs"
11+
ENV _JAVA_OPTIONS="-Djava.net.preferIPv4Stack=true"
12+
ENV TERM=linux
13+
14+
RUN sed -i s/deb.debian.org/archive.debian.org/g /etc/apt/sources.list \
15+
&& sed -i 's|security.debian.org|archive.debian.org/|g' /etc/apt/sources.list \
16+
&& sed -i '/stretch-updates/d' /etc/apt/sources.list \
17+
&& apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat-openbsd dnsutils procps
18+
19+
RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \
20+
curl -J -L -s -f -o - https://github.com/kadwanev/retry/releases/download/1.0.1/retry-1.0.1.tar.gz | tar xfz - -C /usr/local/bin && \
21+
retry --min 1 --max 180 -- curl -J -L -s -f --show-error -o $INSTALL_DIR/kafka.tgz \
22+
"https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" && \
23+
tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1
24+
25+
# Copy jolokia agent jar from host into container
26+
ADD jolokia/jolokia-jvm-1.5.0.jar /opt/jolokia/jolokia-jvm.jar
27+
28+
ADD java8/startup.sh /startup.sh
29+
ADD java8/healthcheck.sh /healthcheck.sh
30+
31+
ADD java8/server.properties ${KAFKA_HOME}/config/server_custom.properties
32+
33+
RUN chmod +x /startup.sh /healthcheck.sh && chown -R 1001:0 /opt ${KAFKA_HOME} ${KAFKA_LOGS_DIR}
34+
35+
EXPOSE 9092
36+
EXPOSE 8780
37+
EXPOSE 8775
38+
EXPOSE 8774
39+
EXPOSE 2181
40+
41+
USER 1001
42+
43+
ENTRYPOINT ["/startup.sh"]

‎packages/kafka/_dev/deploy/docker/Dockerfile-kraft‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
FROM apache/kafka:4.0.0
2+
3+
USER root
4+
5+
ENV KAFKA_NODE_ID=1
6+
ENV KAFKA_PROCESS_ROLES="broker,controller"
7+
ENV KAFKA_CONTROLLER_LISTENER_NAMES="CONTROLLER"
8+
ENV KAFKA_LISTENERS="PLAINTEXT://:9092,CONTROLLER://:9093"
9+
ENV KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://localhost:9092"
10+
ENV KAFKA_CONTROLLER_QUORUM_VOTERS="1@localhost:9093"
11+
ENV KAFKA_LOG_DIRS="/tmp/kraft-combined-logs"
12+
ENV KAFKA_JMX_PORT="9999"
13+
ENV KAFKA_LISTENER_SECURITY_PROTOCOL_MAP="PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT"
14+
ENV KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote \
15+
-Dcom.sun.management.jmxremote.local.only=false \
16+
-Dcom.sun.management.jmxremote.authenticate=false \
17+
-Dcom.sun.management.jmxremote.ssl=false \
18+
-Djava.rmi.server.hostname=localhost \
19+
-Dcom.sun.management.jmxremote.port=${KAFKA_JMX_PORT:-9999} \
20+
-Dcom.sun.management.jmxremote.rmi.port=${KAFKA_JMX_PORT:-9999}"
21+
22+
# Copy jolokia agent jar from host into container
23+
ADD jolokia/jolokia-jvm-2.3.0.jar /opt/jolokia/jolokia-jvm.jar
24+
25+
# Copy startup script
26+
ADD kraft/startup.sh /opt/startup.sh
27+
ADD kraft/healthcheck.sh /healthcheck.sh
28+
ADD kraft/server.properties /opt/kafka/config/kraft/server_custom.properties
29+
RUN chmod +x /opt/startup.sh /healthcheck.sh && chown -R 1001:0 /opt
30+
31+
USER 1001
32+
33+
ENTRYPOINT ["/opt/startup.sh"]

‎packages/kafka/_dev/deploy/docker/Dockerfile-zookeeper‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
FROM debian:stretch
2+
3+
USER root
4+
5+
ARG KAFKA_VERSION
6+
ARG SCALA_VERSION
7+
8+
ENV KAFKA_HOME=/kafka
9+
10+
ENV KAFKA_LOGS_DIR="/kafka-logs"
11+
ENV _JAVA_OPTIONS="-Djava.net.preferIPv4Stack=true"
12+
ENV TERM=linux
13+
14+
RUN sed -i s/deb.debian.org/archive.debian.org/g /etc/apt/sources.list \
15+
&& sed -i 's|security.debian.org|archive.debian.org/|g' /etc/apt/sources.list \
16+
&& sed -i '/stretch-updates/d' /etc/apt/sources.list \
17+
&& apt-get update && apt-get upgrade -y && apt-get install -y curl openjdk-8-jre-headless netcat-openbsd dnsutils procps
18+
19+
RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \
20+
curl -J -L -s -f -o - https://github.com/kadwanev/retry/releases/download/1.0.1/retry-1.0.1.tar.gz | tar xfz - -C /usr/local/bin && \
21+
retry --min 1 --max 180 -- curl -J -L -s -f --show-error \
22+
"https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" | \
23+
tar xzf - -C ${KAFKA_HOME} --strip-components 1
24+
25+
# Copy jolokia agent jar from host into container
26+
ADD jolokia/jolokia-jvm-1.5.0.jar /opt/jolokia/jolokia-jvm.jar
27+
28+
ADD zookeeper/startup.sh /startup.sh
29+
ADD zookeeper/healthcheck.sh /healthcheck.sh
30+
31+
ADD zookeeper/server.properties ${KAFKA_HOME}/config/server_custom.properties
32+
33+
RUN chmod +x /startup.sh /healthcheck.sh && chown -R 1001:0 /opt ${KAFKA_HOME} ${KAFKA_LOGS_DIR}
34+
35+
EXPOSE 9092
36+
EXPOSE 8780
37+
EXPOSE 8775
38+
EXPOSE 8774
39+
EXPOSE 2181
40+
41+
USER 1001
42+
43+
ENTRYPOINT ["/startup.sh"]

‎packages/kafka/_dev/deploy/docker/docker-compose.yml‎

Lines changed: 9 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,37 +4,22 @@ services:
44
default:
55
aliases:
66
- svc-kafka
7-
build: .
7+
build:
8+
context: .
9+
dockerfile: ${DOCKER_FILE}
10+
args:
11+
KAFKA_VERSION: ${KAFKA_VERSION:-4.0.0}
12+
SCALA_VERSION: ${SCALA_VERSION:-2.13}
813
ports:
914
- "9092:9092"
1015
- "9093:9093"
11-
- "9999:9999"
1216
- "8780:8780"
1317
- "8774:8774"
1418
- "8775:8775"
19+
- "2181:2181"
1520
healthcheck:
16-
test: [ "CMD-SHELL", "/opt/healthcheck.sh" ]
21+
test: [ "CMD-SHELL", "/healthcheck.sh" ]
1722
interval: 15s
1823
timeout: 10s
1924
retries: 5
20-
start_period: 60s
21-
environment:
22-
KAFKA_NODE_ID: 1
23-
KAFKA_PROCESS_ROLES: broker,controller
24-
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
25-
KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
26-
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
27-
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
28-
KAFKA_LOG_DIRS: /tmp/kraft-combined-logs
29-
KAFKA_JMX_PORT: 9999
30-
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
31-
KAFKA_JMX_OPTS: >
32-
-Dcom.sun.management.jmxremote
33-
-Dcom.sun.management.jmxremote.local.only=false
34-
-Dcom.sun.management.jmxremote.authenticate=false
35-
-Dcom.sun.management.jmxremote.ssl=false
36-
-Djava.rmi.server.hostname=localhost
37-
-Dcom.sun.management.jmxremote.port=9999
38-
-Dcom.sun.management.jmxremote.rmi.port=9999
39-
volumes:
40-
- ./server.properties:/opt/kafka/config/kraft/server.properties
25+
start_period: 60s

‎packages/kafka/_dev/deploy/docker/java8/healthcheck.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
# Exit immediately if a command exits with a non-zero status.
3+
set -e
4+
5+
# 1. Check for the Kafka Broker process
6+
pgrep -f 'kafka.Kafka' > /dev/null || { echo "Healthcheck failed: Kafka broker not running."; exit 1; }
7+
8+
# 2. Check for the Console Producer process
9+
pgrep -f 'kafka.tools.ConsoleProducer' > /dev/null || { echo "Healthcheck failed: Console producer not running."; exit 1; }
10+
11+
# 3. Check for the Console Consumer process
12+
pgrep -f 'kafka.tools.ConsoleConsumer' > /dev/null || { echo "Healthcheck failed: Console consumer not running."; exit 1; }
13+
14+
# If all checks pass, exit with 0 to indicate the container is healthy.
15+
exit 0

‎packages/kafka/_dev/deploy/docker/java8/server.properties‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
node.id=1
2+
listeners=PLAINTEXT://:9092,CONTROLLER://:9093
3+
listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
4+
inter.broker.listener.name=PLAINTEXT
5+
log.dirs=/kafka-logs
6+
controller.listener.names=CONTROLLER
7+
offsets.topic.replication.factor=1
8+
transaction.state.log.replication.factor=1
9+
zookeeper.set.acl=false
10+
zookeeper.connect=localhost:2181

‎packages/kafka/_dev/deploy/docker/java8/startup.sh‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/bin/bash
2+
3+
# --- Utility Functions ---
4+
5+
# Waits for a specified port to become available on localhost.
6+
wait_for_port() {
7+
count=20
8+
port=$1
9+
echo "Waiting for port ${port} to become available..."
10+
while ! nc -z localhost $port && [[ $count -ne 0 ]]; do
11+
count=$(( $count - 1 ))
12+
# If the count reaches 0, exit with an error.
13+
[[ $count -eq 0 ]] && echo "Error: Port ${port} did not become available." && return 1
14+
sleep 0.5
15+
done
16+
echo "Port ${port} is now available."
17+
nc -z localhost $port
18+
}
19+
20+
echo "Starting ZooKeeper"
21+
${KAFKA_HOME}/bin/zookeeper-server-start.sh ${KAFKA_HOME}/config/zookeeper.properties &
22+
wait_for_port 2181
23+
24+
# --- Kafka Setup ---
25+
26+
echo "Starting Kafka broker..."
27+
28+
export KAFKA_OPTS="-javaagent:/opt/jolokia/jolokia-jvm.jar=port=8780,host=0.0.0.0"
29+
30+
${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server_custom.properties &
31+
# Store the Process ID (PID) of the Kafka server
32+
kafka_pid=$!
33+
34+
# Wait for the Kafka broker and Jolokia agent ports to be available.
35+
wait_for_port 9092
36+
wait_for_port 8780
37+
38+
echo "Kafka broker started successfully."
39+
40+
echo "Creating 'test' topic..."
41+
KAFKA_OPTS="" ${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper localhost:2181 --create --if-not-exists --topic my-topic --partitions 1 --replication-factor 1
42+
43+
# --- Start Producer and Consumer ---
44+
45+
echo "Starting producer with Jolokia on port 8775..."
46+
export KAFKA_OPTS="-javaagent:/opt/jolokia/jolokia-jvm.jar=port=8775,host=0.0.0.0"
47+
(while true; do echo "Test msg"; sleep 5; done) | \
48+
${KAFKA_HOME}/bin/kafka-console-producer.sh \
49+
--broker-list localhost:9092 \
50+
--topic my-topic &
51+
52+
echo "Starting consumer with Jolokia on port 8774..."
53+
export KAFKA_OPTS="-javaagent:/opt/jolokia/jolokia-jvm.jar=port=8774,host=0.0.0.0"
54+
${KAFKA_HOME}/bin/kafka-console-consumer.sh \
55+
--bootstrap-server localhost:9092 \
56+
--topic my-topic \
57+
--from-beginning &
58+
59+
# --- Keep Container Running ---
60+
echo "Kafka setup complete. The container will now run in the background."
61+
62+
# The 'wait' command blocks the script until the Kafka server process (kafka_pid) exits.
63+
# This keeps the container running.
64+
wait $kafka_pid

‎packages/kafka/_dev/deploy/docker/jolokia/jolokia-jvm-1.5.0.jar‎

451 KB
Binary file not shown.

‎packages/kafka/_dev/deploy/docker/jolokia/jolokia-jvm-agent.jar‎ renamed to ‎packages/kafka/_dev/deploy/docker/jolokia/jolokia-jvm-2.3.0.jar‎

File renamed without changes.

0 commit comments

Comments
 (0)