Kafka system tests variants

packages/kafka/_dev/deploy/docker/Dockerfile-java8

@@ -0,0 +1,44 @@
+FROM debian:stretch
+
+USER root
+
+ARG KAFKA_VERSION
+ARG SCALA_VERSION
+
+ENV KAFKA_HOME=/kafka
+
+ENV KAFKA_LOGS_DIR="/kafka-logs"
+ENV _JAVA_OPTIONS="-Djava.net.preferIPv4Stack=true"
+ENV TERM=linux
+
+RUN sed -i s/deb.debian.org/archive.debian.org/g /etc/apt/sources.list \
+    && sed -i 's|security.debian.org|archive.debian.org/|g' /etc/apt/sources.list \
+    && sed -i '/stretch-updates/d' /etc/apt/sources.list
+
+RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat-openbsd dnsutils procps
+
+RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \
+    curl -J -L -s -f -o - https://github.com/kadwanev/retry/releases/download/1.0.1/retry-1.0.1.tar.gz | tar xfz - -C /usr/local/bin && \
+    retry --min 1 --max 180 -- curl -J -L -s -f --show-error -o $INSTALL_DIR/kafka.tgz \
+        "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" && \
+    tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1
+
+# Copy jolokia agent jar from host into container
+ADD jolokia/jolokia-jvm-1.5.0.jar /opt/jolokia/jolokia-jvm.jar
+
+ADD java8/startup.sh /startup.sh
+ADD java8/healthcheck.sh /healthcheck.sh
+
+ADD java8/server.properties ${KAFKA_HOME}/config/server_custom.properties
+
+RUN chmod +x /startup.sh /healthcheck.sh && chown -R 1001:0 /opt ${KAFKA_HOME} ${KAFKA_LOGS_DIR}
+
+EXPOSE 9092
+EXPOSE 8780
+EXPOSE 8775
+EXPOSE 8774
+EXPOSE 2181
+
+USER 1001
+
+ENTRYPOINT ["/startup.sh"]

packages/kafka/_dev/deploy/docker/Dockerfile-kraft

@@ -0,0 +1,33 @@
+FROM apache/kafka:4.0.0
+
+USER root
+
+ENV KAFKA_NODE_ID=1
+ENV KAFKA_PROCESS_ROLES="broker,controller"
+ENV KAFKA_CONTROLLER_LISTENER_NAMES="CONTROLLER"
+ENV KAFKA_LISTENERS="PLAINTEXT://:9092,CONTROLLER://:9093"
+ENV KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://localhost:9092"
+ENV KAFKA_CONTROLLER_QUORUM_VOTERS="1@localhost:9093"
+ENV KAFKA_LOG_DIRS="/tmp/kraft-combined-logs"
+ENV KAFKA_JMX_PORT="9999"
+ENV KAFKA_LISTENER_SECURITY_PROTOCOL_MAP="PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT"
+ENV KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote \
+    -Dcom.sun.management.jmxremote.local.only=false \
+    -Dcom.sun.management.jmxremote.authenticate=false \
+    -Dcom.sun.management.jmxremote.ssl=false \
+    -Djava.rmi.server.hostname=localhost \
+    -Dcom.sun.management.jmxremote.port=${KAFKA_JMX_PORT:-9999} \
+    -Dcom.sun.management.jmxremote.rmi.port=${KAFKA_JMX_PORT:-9999}"
+
+# Copy jolokia agent jar from host into container
+ADD jolokia/jolokia-jvm-2.3.0.jar /opt/jolokia/jolokia-jvm.jar
+
+# Copy startup script
+ADD kraft/startup.sh /opt/startup.sh
+ADD kraft/healthcheck.sh /healthcheck.sh
+ADD kraft/server.properties /opt/kafka/config/kraft/server_custom.properties
+RUN chmod +x /opt/startup.sh /healthcheck.sh && chown -R 1001:0 /opt
+
+USER 1001
+
+ENTRYPOINT ["/opt/startup.sh"]

packages/kafka/_dev/deploy/docker/Dockerfile-zookeeper

@@ -0,0 +1,44 @@
+FROM debian:stretch
+
+USER root
+
+ARG KAFKA_VERSION
+ARG SCALA_VERSION
+
+ENV KAFKA_HOME=/kafka
+
+ENV KAFKA_LOGS_DIR="/kafka-logs"
+ENV _JAVA_OPTIONS="-Djava.net.preferIPv4Stack=true"
+ENV TERM=linux
+
+RUN sed -i s/deb.debian.org/archive.debian.org/g /etc/apt/sources.list \
+    && sed -i 's|security.debian.org|archive.debian.org/|g' /etc/apt/sources.list \
+    && sed -i '/stretch-updates/d' /etc/apt/sources.list
+
+RUN apt-get update && apt-get upgrade -y && apt-get install -y curl openjdk-8-jre-headless netcat-openbsd dnsutils procps
+
+RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \
+    curl -J -L -s -f -o - https://github.com/kadwanev/retry/releases/download/1.0.1/retry-1.0.1.tar.gz | tar xfz - -C /usr/local/bin && \
+    retry --min 1 --max 180 -- curl -J -L -s -f --show-error \
+        "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" | \
+    tar xzf - -C ${KAFKA_HOME} --strip-components 1
+
+# Copy jolokia agent jar from host into container
+ADD jolokia/jolokia-jvm-1.5.0.jar /opt/jolokia/jolokia-jvm.jar
+
+ADD zookeeper/startup.sh /startup.sh
+ADD zookeeper/healthcheck.sh /healthcheck.sh
+
+ADD zookeeper/server.properties ${KAFKA_HOME}/config/server_custom.properties
+
+RUN chmod +x /startup.sh /healthcheck.sh && chown -R 1001:0 /opt ${KAFKA_HOME} ${KAFKA_LOGS_DIR}
+
+EXPOSE 9092
+EXPOSE 8780
+EXPOSE 8775
+EXPOSE 8774
+EXPOSE 2181
+
+USER 1001
+
+ENTRYPOINT ["/startup.sh"]

packages/kafka/_dev/deploy/docker/docker-compose.yml

@@ -4,37 +4,22 @@ services:
       default:
         aliases:
           - svc-kafka
-    build: .
+    build:
+      context: .
+      dockerfile: ${DOCKER_FILE}
+      args:
+        KAFKA_VERSION: ${KAFKA_VERSION:-4.0.0}
+        SCALA_VERSION: ${SCALA_VERSION:-2.13}
     ports:
       - "9092:9092"
       - "9093:9093"
-      - "9999:9999"
       - "8780:8780"
       - "8774:8774"
       - "8775:8775"
+      - "2181:2181"
     healthcheck:
-      test: [ "CMD-SHELL", "/opt/healthcheck.sh" ]
+      test: [ "CMD-SHELL", "/healthcheck.sh" ]
       interval: 15s
       timeout: 10s
       retries: 5
-      start_period: 60s
-    environment:
-      KAFKA_NODE_ID: 1
-      KAFKA_PROCESS_ROLES: broker,controller
-      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
-      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
-      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
-      KAFKA_LOG_DIRS: /tmp/kraft-combined-logs
-      KAFKA_JMX_PORT: 9999
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
-      KAFKA_JMX_OPTS: >
-        -Dcom.sun.management.jmxremote
-        -Dcom.sun.management.jmxremote.local.only=false
-        -Dcom.sun.management.jmxremote.authenticate=false
-        -Dcom.sun.management.jmxremote.ssl=false
-        -Djava.rmi.server.hostname=localhost
-        -Dcom.sun.management.jmxremote.port=9999
-        -Dcom.sun.management.jmxremote.rmi.port=9999
-    volumes:
-      - ./server.properties:/opt/kafka/config/kraft/server.properties
+      start_period: 60s

packages/kafka/_dev/deploy/docker/java8/healthcheck.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Exit immediately if a command exits with a non-zero status.
+set -e
+
+# 1. Check for the Kafka Broker process
+pgrep -f 'kafka.Kafka' > /dev/null || { echo "Healthcheck failed: Kafka broker not running."; exit 1; }
+
+# 2. Check for the Console Producer process
+pgrep -f 'kafka.tools.ConsoleProducer' > /dev/null || { echo "Healthcheck failed: Console producer not running."; exit 1; }
+
+# 3. Check for the Console Consumer process
+pgrep -f 'kafka.tools.ConsoleConsumer' > /dev/null || { echo "Healthcheck failed: Console consumer not running."; exit 1; }
+
+# If all checks pass, exit with 0 to indicate the container is healthy.
+exit 0

packages/kafka/_dev/deploy/docker/java8/server.properties

@@ -0,0 +1,10 @@
+node.id=1
+listeners=PLAINTEXT://:9092,CONTROLLER://:9093
+listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
+inter.broker.listener.name=PLAINTEXT
+log.dirs=/kafka-logs
+controller.listener.names=CONTROLLER
+offsets.topic.replication.factor=1
+transaction.state.log.replication.factor=1
+zookeeper.set.acl=false
+zookeeper.connect=localhost:2181

packages/kafka/_dev/deploy/docker/java8/startup.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# --- Utility Functions ---
+
+# Waits for a specified port to become available on localhost.
+wait_for_port() {
+    count=20
+    port=$1
+    echo "Waiting for port ${port} to become available..."
+    while ! nc -z localhost $port && [[ $count -ne 0 ]]; do
+        count=$(( $count - 1 ))
+        # If the count reaches 0, exit with an error.
+        [[ $count -eq 0 ]] && echo "Error: Port ${port} did not become available." && return 1
+        sleep 0.5
+    done
+    echo "Port ${port} is now available."
+    nc -z localhost $port
+}
+
+echo "Starting ZooKeeper"
+${KAFKA_HOME}/bin/zookeeper-server-start.sh ${KAFKA_HOME}/config/zookeeper.properties &
+wait_for_port 2181
+
+# --- Kafka Setup ---
+
+echo "Starting Kafka broker..."
+
+export KAFKA_OPTS="-javaagent:/opt/jolokia/jolokia-jvm.jar=port=8780,host=0.0.0.0"
+
+${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server_custom.properties &
+# Store the Process ID (PID) of the Kafka server
+kafka_pid=$!
+
+# Wait for the Kafka broker and Jolokia agent ports to be available.
+wait_for_port 9092
+wait_for_port 8780
+
+echo "Kafka broker started successfully."
+
+echo "Creating 'test' topic..."
+KAFKA_OPTS="" ${KAFKA_HOME}/bin/kafka-topics.sh --zookeeper localhost:2181 --create --if-not-exists --topic my-topic --partitions 1 --replication-factor 1
+
+# --- Start Producer and Consumer ---
+
+echo "Starting producer with Jolokia on port 8775..."
+export KAFKA_OPTS="-javaagent:/opt/jolokia/jolokia-jvm.jar=port=8775,host=0.0.0.0"
+(while true; do echo "Test msg"; sleep 5; done) | \
+${KAFKA_HOME}/bin/kafka-console-producer.sh \
+  --broker-list localhost:9092 \
+  --topic my-topic &
+
+echo "Starting consumer with Jolokia on port 8774..."
+export KAFKA_OPTS="-javaagent:/opt/jolokia/jolokia-jvm.jar=port=8774,host=0.0.0.0"
+${KAFKA_HOME}/bin/kafka-console-consumer.sh \
+  --bootstrap-server localhost:9092 \
+  --topic my-topic \
+  --from-beginning &
+
+# --- Keep Container Running ---
+echo "Kafka setup complete. The container will now run in the background."
+
+# The 'wait' command blocks the script until the Kafka server process (kafka_pid) exits.
+# This keeps the container running.
+wait $kafka_pid

packages/kafka/_dev/deploy/docker/server.properties → packages/kafka/_dev/deploy/docker/kraft/server.properties

@@ -5,4 +5,6 @@ listeners=PLAINTEXT://:9092,CONTROLLER://:9093
 listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
 inter.broker.listener.name=PLAINTEXT
 log.dirs=/tmp/kraft-combined-logs
-controller.listener.names=CONTROLLER
+controller.listener.names=CONTROLLER
+offsets.topic.replication.factor=1
+transaction.state.log.replication.factor=1

packages/kafka/_dev/deploy/docker/startup.sh → packages/kafka/_dev/deploy/docker/kraft/startup.sh

@@ -26,14 +26,14 @@ chown -R 1001:0 "$LOG_DIR"
 # Initialize storage if not formatted yet
 if [ ! -f "$META_FILE" ]; then
   echo "Formatting KRaft storage with Cluster ID: $CLUSTER_ID"
-  /opt/kafka/bin/kafka-storage.sh format -t "$CLUSTER_ID" -c /opt/kafka/config/kraft/server.properties
+  /opt/kafka/bin/kafka-storage.sh format -t "$CLUSTER_ID" -c /opt/kafka/config/kraft/server_custom.properties
 else
   echo "KRaft storage already formatted."
 fi
 
 # Start the Kafka server in the background
 echo "Starting Kafka server..."
-/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/kraft/server.properties &
+/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/kraft/server_custom.properties &
 # Store the Process ID (PID) of the Kafka server
 kafka_pid=$!
 

packages/kafka/_dev/deploy/docker/zookeeper/healthcheck.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Exit immediately if a command exits with a non-zero status.
+set -e
+
+# 1. Check for the Kafka Broker process
+ps -ef | grep '[k]afka.Kafka' | awk '{print $2}' > /dev/null || { echo "Healthcheck failed: Kafka broker not running."; exit 1; }
+
+# 2. Check for the Console Producer process
+ps -ef | grep '[k]afka.tools.ConsoleProducer' | awk '{print $2}' > /dev/null || { echo "Healthcheck failed: Console producer not running."; exit 1; }
+
+# 3. Check for the Console Consumer process
+ps -ef | grep '[k]afka.tools.ConsoleConsumer' | awk '{print $2}' > /dev/null || { echo "Healthcheck failed: Console consumer not running."; exit 1; }
+
+# If all checks pass, exit with 0 to indicate the container is healthy.
+exit 0

packages/kafka/_dev/deploy/docker/zookeeper/server.properties

@@ -0,0 +1,13 @@
+node.id=1
+
+listeners=PLAINTEXT://:9092
+advertised.listeners=PLAINTEXT://localhost:9092
+
+listener.security.protocol.map=PLAINTEXT:PLAINTEXT
+inter.broker.listener.name=PLAINTEXT
+log.dirs=/kafka-logs
+offsets.topic.replication.factor=1
+transaction.state.log.replication.factor=1
+zookeeper.set.acl=false
+
+zookeeper.connect=127.0.0.1:2181