Skip to content

Commit f6166bd

Browse files
1sonofqiu1sonofqiu
authored
feat(docker): kafka docker image adapt (#2828)
* feat(docker): add AutoMQ Kafka Docker release workflow and update Dockerfile
1 parent 7ea0cb6 commit f6166bd

File tree

4 files changed

+229
-59
lines changed

4 files changed

+229
-59
lines changed

‎.github/workflows/docker-kafka-release.yaml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: AutoMQ Kafka Docker Release
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
tag:
7+
description: 'AutoMQ Version Tag'
8+
required: false
9+
type: string
10+
workflow_run:
11+
workflows: ["GitHub Release"]
12+
types:
13+
- completed
14+
15+
env:
16+
KAFKA_VERSION: "3.9.0"
17+
18+
jobs:
19+
automq-kafka-release:
20+
name: AutoMQ Kafka Docker Image Release
21+
strategy:
22+
matrix:
23+
platform: [ "ubuntu-24.04" ]
24+
jdk: [ "17" ]
25+
runs-on: ${{ matrix.platform }}
26+
permissions:
27+
contents: write
28+
steps:
29+
- name: Checkout Code
30+
uses: actions/checkout@v4
31+
32+
- name: Get release tag
33+
run: |
34+
if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ github.event.inputs.tag }}" ]]; then
35+
TAG="${{ github.event.inputs.tag }}"
36+
# use the latest tag if not specified
37+
elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
38+
TAG=$(git ls-remote --tags https://github.com/AutoMQ/automq.git | grep -v '\^{}' | tail -1 | sed 's/.*refs\/tags\///')
39+
else
40+
TAG="${{ github.event.workflow_run.head_branch }}"
41+
fi
42+
43+
AUTOMQ_URL="https://github.com/AutoMQ/automq/releases/download/${TAG}/automq-${TAG}_kafka-${KAFKA_VERSION}.tgz"
44+
45+
{
46+
echo "AUTOMQ_VERSION=${TAG}-kafka"
47+
echo "AUTOMQ_URL=${AUTOMQ_URL}"
48+
} >> $GITHUB_ENV
49+
50+
- name: Set up Python 3.10
51+
uses: actions/setup-python@v5
52+
with:
53+
python-version: "3.10"
54+
55+
- name: Login to Docker Hub
56+
uses: docker/login-action@v3
57+
with:
58+
username: ${{ secrets.DOCKERHUB_USERNAME }}
59+
password: ${{ secrets.DOCKERHUB_READ_WRITE_TOKEN }}
60+
61+
- name: Build and Push Docker Image
62+
run: |
63+
python3 -m venv .venv
64+
source .venv/bin/activate
65+
.venv/bin/pip install setuptools
66+
67+
cd docker
68+
python3 docker_release.py \
69+
${{ secrets.DOCKERHUB_USERNAME }}:${AUTOMQ_VERSION} \
70+
--kafka-url ${AUTOMQ_URL}

‎docker/automq-feat-kafka-docker.patch‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
From a46795f12e3e012e6ec1ad08783f6f0428a3c034 Mon Sep 17 00:00:00 2001
2+
From: 1sonofqiu <zhangjianxin@automq.com>
3+
Date: Wed, 10 Sep 2025 16:30:39 +0800
4+
Subject: [PATCH 6/6] feat(docker): add AutoMQ Kafka Docker release workflow
5+
and update Dockerfile
6+
7+
---
8+
docker/jvm/Dockerfile | 58 +++++++++++--------------------------------
9+
docker/jvm/launch | 16 ------------
10+
2 files changed, 15 insertions(+), 59 deletions(-)
11+
12+
diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile
13+
index 72e35e63c0..de97104271 100644
14+
--- a/docker/jvm/Dockerfile
15+
+++ b/docker/jvm/Dockerfile
16+
@@ -16,60 +16,31 @@
17+
# limitations under the License.
18+
###############################################################################
19+
20+
-FROM eclipse-temurin:21-jre-alpine AS build-jsa
21+
-
22+
-USER root
23+
-
24+
-# Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments
25+
-ARG kafka_url
26+
-
27+
-COPY jsa_launch /etc/kafka/docker/jsa_launch
28+
-
29+
-RUN set -eux ; \
30+
- apk update ; \
31+
- apk upgrade ; \
32+
- apk add --no-cache wget gcompat gpg gpg-agent procps bash; \
33+
- mkdir opt/kafka; \
34+
- wget -nv -O kafka.tgz "$kafka_url"; \
35+
- wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \
36+
- tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
37+
- wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
38+
- gpg --import KEYS; \
39+
- gpg --batch --verify kafka.tgz.asc kafka.tgz
40+
-
41+
-# Generate jsa files using dynamic CDS for kafka server start command and kafka storage format command
42+
-RUN /etc/kafka/docker/jsa_launch
43+
-
44+
-
45+
-FROM eclipse-temurin:21-jre-alpine
46+
+FROM amazoncorretto:17-alpine
47+
48+
# exposed ports
49+
EXPOSE 9092
50+
51+
USER root
52+
53+
-# Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments
54+
+# Get AutoMQ URL from https://github.com/AutoMQ/automq/releases and pass the url passed through build arguments
55+
ARG kafka_url
56+
ARG build_date
57+
58+
59+
-LABEL org.label-schema.name="kafka" \
60+
- org.label-schema.description="Apache Kafka" \
61+
+LABEL org.label-schema.name="automq-kafka" \
62+
+ org.label-schema.description="AutoMQ for Kafka" \
63+
org.label-schema.build-date="${build_date}" \
64+
- org.label-schema.vcs-url="https://github.com/apache/kafka" \
65+
- maintainer="Apache Kafka"
66+
+ org.label-schema.vcs-url="https://github.com/AutoMQ/automq" \
67+
+ maintainer="AutoMQ"
68+
69+
RUN set -eux ; \
70+
apk update ; \
71+
apk upgrade ; \
72+
- apk add --no-cache wget gcompat gpg gpg-agent procps bash; \
73+
+ apk add --no-cache wget gcompat procps bash jemalloc; \
74+
mkdir opt/kafka; \
75+
- wget -nv -O kafka.tgz "$kafka_url"; \
76+
- wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \
77+
- tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
78+
- wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
79+
- gpg --import KEYS; \
80+
- gpg --batch --verify kafka.tgz.asc kafka.tgz; \
81+
+ wget -nv -O automq.tgz "$kafka_url"; \
82+
+ tar xfz automq.tgz -C /opt/kafka --strip-components 1; \
83+
mkdir -p /var/lib/kafka/data /etc/kafka/secrets; \
84+
mkdir -p /etc/kafka/docker /usr/logs /mnt/shared/config; \
85+
adduser -h /home/appuser -D --shell /bin/bash appuser; \
86+
@@ -78,16 +49,17 @@ RUN set -eux ; \
87+
chmod -R ug+w /etc/kafka /var/lib/kafka /etc/kafka/secrets; \
88+
cp /opt/kafka/config/log4j.properties /etc/kafka/docker/log4j.properties; \
89+
cp /opt/kafka/config/tools-log4j.properties /etc/kafka/docker/tools-log4j.properties; \
90+
- cp /opt/kafka/config/kraft/server.properties /etc/kafka/docker/server.properties; \
91+
- rm kafka.tgz kafka.tgz.asc KEYS; \
92+
- apk del wget gpg gpg-agent; \
93+
+ cp /opt/kafka/config/kraft/reconfig-server.properties /etc/kafka/docker/server.properties; \
94+
+ rm automq.tgz; \
95+
+ apk del wget; \
96+
apk cache clean;
97+
98+
-COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa
99+
-COPY --from=build-jsa storage.jsa /opt/kafka/storage.jsa
100+
COPY --chown=appuser:appuser resources/common-scripts /etc/kafka/docker
101+
COPY --chown=appuser:appuser launch /etc/kafka/docker/launch
102+
103+
+# Configure jemalloc as the memory allocator
104+
+ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
105+
+
106+
USER appuser
107+
108+
VOLUME ["/etc/kafka/secrets", "/var/lib/kafka/data", "/mnt/shared/config"]
109+
diff --git a/docker/jvm/launch b/docker/jvm/launch
110+
index 6c4ca1d2e0..fa0010ff03 100755
111+
--- a/docker/jvm/launch
112+
+++ b/docker/jvm/launch
113+
@@ -38,16 +38,6 @@ if [ "${KAFKA_JMX_PORT-}" ]; then
114+
-Dcom.sun.management.jmxremote.port=$JMX_PORT"
115+
fi
116+
117+
-# Make a temp env variable to store user provided performance otps
118+
-if [ -z "${KAFKA_JVM_PERFORMANCE_OPTS-}" ]; then
119+
- export TEMP_KAFKA_JVM_PERFORMANCE_OPTS=""
120+
-else
121+
- export TEMP_KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS"
122+
-fi
123+
-
124+
-# We will first use CDS for storage to format storage
125+
-export KAFKA_JVM_PERFORMANCE_OPTS="${KAFKA_JVM_PERFORMANCE_OPTS-} -XX:SharedArchiveFile=/opt/kafka/storage.jsa"
126+
-
127+
echo "===> Using provided cluster id $CLUSTER_ID ..."
128+
129+
# Invoke the docker wrapper to setup property files and format storage
130+
@@ -58,11 +48,5 @@ result=$(/opt/kafka/bin/kafka-run-class.sh kafka.docker.KafkaDockerWrapper setup
131+
echo $result | grep -i "already formatted" || \
132+
{ echo $result && (exit 1) }
133+
134+
-# Using temp env variable to get rid of storage CDS command
135+
-export KAFKA_JVM_PERFORMANCE_OPTS="$TEMP_KAFKA_JVM_PERFORMANCE_OPTS"
136+
-
137+
-# Now we will use CDS for kafka to start kafka server
138+
-export KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS -XX:SharedArchiveFile=/opt/kafka/kafka.jsa"
139+
-
140+
# Start kafka broker
141+
exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties
142+
--
143+
2.39.5 (Apple Git-154)
144+

‎docker/jvm/Dockerfile‎

Lines changed: 15 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -16,60 +16,31 @@
1616
# limitations under the License.
1717
###############################################################################
1818

19-
FROM eclipse-temurin:21-jre-alpine AS build-jsa
20-
21-
USER root
22-
23-
# Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments
24-
ARG kafka_url
25-
26-
COPY jsa_launch /etc/kafka/docker/jsa_launch
27-
28-
RUN set -eux ; \
29-
apk update ; \
30-
apk upgrade ; \
31-
apk add --no-cache wget gcompat gpg gpg-agent procps bash; \
32-
mkdir opt/kafka; \
33-
wget -nv -O kafka.tgz "$kafka_url"; \
34-
wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \
35-
tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
36-
wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
37-
gpg --import KEYS; \
38-
gpg --batch --verify kafka.tgz.asc kafka.tgz
39-
40-
# Generate jsa files using dynamic CDS for kafka server start command and kafka storage format command
41-
RUN /etc/kafka/docker/jsa_launch
42-
43-
44-
FROM eclipse-temurin:21-jre-alpine
19+
FROM amazoncorretto:17-alpine
4520

4621
# exposed ports
4722
EXPOSE 9092
4823

4924
USER root
5025

51-
# Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments
26+
# Get AutoMQ URL from https://github.com/AutoMQ/automq/releases and pass the url passed through build arguments
5227
ARG kafka_url
5328
ARG build_date
5429

5530

56-
LABEL org.label-schema.name="kafka" \
57-
org.label-schema.description="Apache Kafka" \
31+
LABEL org.label-schema.name="automq-kafka" \
32+
org.label-schema.description="AutoMQ for Kafka" \
5833
org.label-schema.build-date="${build_date}" \
59-
org.label-schema.vcs-url="https://github.com/apache/kafka" \
60-
maintainer="Apache Kafka"
34+
org.label-schema.vcs-url="https://github.com/AutoMQ/automq" \
35+
maintainer="AutoMQ"
6136

6237
RUN set -eux ; \
6338
apk update ; \
6439
apk upgrade ; \
65-
apk add --no-cache wget gcompat gpg gpg-agent procps bash; \
40+
apk add --no-cache wget gcompat procps bash jemalloc; \
6641
mkdir opt/kafka; \
67-
wget -nv -O kafka.tgz "$kafka_url"; \
68-
wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \
69-
tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
70-
wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
71-
gpg --import KEYS; \
72-
gpg --batch --verify kafka.tgz.asc kafka.tgz; \
42+
wget -nv -O automq.tgz "$kafka_url"; \
43+
tar xfz automq.tgz -C /opt/kafka --strip-components 1; \
7344
mkdir -p /var/lib/kafka/data /etc/kafka/secrets; \
7445
mkdir -p /etc/kafka/docker /usr/logs /mnt/shared/config; \
7546
adduser -h /home/appuser -D --shell /bin/bash appuser; \
@@ -78,16 +49,17 @@ RUN set -eux ; \
7849
chmod -R ug+w /etc/kafka /var/lib/kafka /etc/kafka/secrets; \
7950
cp /opt/kafka/config/log4j.properties /etc/kafka/docker/log4j.properties; \
8051
cp /opt/kafka/config/tools-log4j.properties /etc/kafka/docker/tools-log4j.properties; \
81-
cp /opt/kafka/config/kraft/server.properties /etc/kafka/docker/server.properties; \
82-
rm kafka.tgz kafka.tgz.asc KEYS; \
83-
apk del wget gpg gpg-agent; \
52+
cp /opt/kafka/config/kraft/reconfig-server.properties /etc/kafka/docker/server.properties; \
53+
rm automq.tgz; \
54+
apk del wget; \
8455
apk cache clean;
8556

86-
COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa
87-
COPY --from=build-jsa storage.jsa /opt/kafka/storage.jsa
8857
COPY --chown=appuser:appuser resources/common-scripts /etc/kafka/docker
8958
COPY --chown=appuser:appuser launch /etc/kafka/docker/launch
9059

60+
# Configure jemalloc as the memory allocator
61+
ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
62+
9163
USER appuser
9264

9365
VOLUME ["/etc/kafka/secrets", "/var/lib/kafka/data", "/mnt/shared/config"]

‎docker/jvm/launch‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -38,16 +38,6 @@ if [ "${KAFKA_JMX_PORT-}" ]; then
3838
-Dcom.sun.management.jmxremote.port=$JMX_PORT"
3939
fi
4040

41-
# Make a temp env variable to store user provided performance otps
42-
if [ -z "${KAFKA_JVM_PERFORMANCE_OPTS-}" ]; then
43-
export TEMP_KAFKA_JVM_PERFORMANCE_OPTS=""
44-
else
45-
export TEMP_KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS"
46-
fi
47-
48-
# We will first use CDS for storage to format storage
49-
export KAFKA_JVM_PERFORMANCE_OPTS="${KAFKA_JVM_PERFORMANCE_OPTS-} -XX:SharedArchiveFile=/opt/kafka/storage.jsa"
50-
5141
echo "===> Using provided cluster id $CLUSTER_ID ..."
5242

5343
# Invoke the docker wrapper to setup property files and format storage
@@ -58,11 +48,5 @@ result=$(/opt/kafka/bin/kafka-run-class.sh kafka.docker.KafkaDockerWrapper setup
5848
echo $result | grep -i "already formatted" || \
5949
{ echo $result && (exit 1) }
6050

61-
# Using temp env variable to get rid of storage CDS command
62-
export KAFKA_JVM_PERFORMANCE_OPTS="$TEMP_KAFKA_JVM_PERFORMANCE_OPTS"
63-
64-
# Now we will use CDS for kafka to start kafka server
65-
export KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS -XX:SharedArchiveFile=/opt/kafka/kafka.jsa"
66-
6751
# Start kafka broker
6852
exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties

0 commit comments

Comments
 (0)