[WIP]KAFKA-15444: Native docker image

core/src/main/scala/kafka/Kafka.scala

@@ -86,6 +86,10 @@ object Kafka extends Logging {
   }
 
   def main(args: Array[String]): Unit = {
+    process(args)
+  }
+
+  def process(args: Array[String]): Unit = {
     try {
       val serverProps = getPropsFromArgs(args)
       val server = buildServer(serverProps)

core/src/main/scala/kafka/KafkaNativeWrapper.scala

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka
+
+import kafka.tools.StorageTool
+import kafka.utils.Logging
+
+object KafkaNativeWrapper extends Logging {
+  def main(args: Array[String]): Unit = {
+    if (args.length == 0) {
+
+    }
+    val operation = args.head
+    val arguments = args.tail
+    operation match {
+      case "storage-tool" => StorageTool.process(arguments)
+      case "kafka" => Kafka.process(arguments)
+      case _ =>
+        throw new RuntimeException(s"Unknown operation $operation. " +
+          s"Please provide a valid operation: 'storage-tool' or 'kafka'.")
+    }
+  }
+}

core/src/main/scala/kafka/tools/StorageTool.scala

@@ -42,6 +42,10 @@ import scala.collection.mutable.ArrayBuffer
 
 object StorageTool extends Logging {
   def main(args: Array[String]): Unit = {
+    process(args)
+  }
+
+  def process(args: Array[String]): Unit = {
     try {
       val namespace = parseArguments(args)
       val command = namespace.getString("command")

docker/docker_build_test.py

@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+from datetime import date
+import argparse
+from distutils.dir_util import copy_tree
+import shutil
+
+
+def build_jvm(image, tag, kafka_url):
+    image = f'{image}:{tag}'
+    copy_tree("resources", "jvm/resources")
+    result = subprocess.run(
+        ["docker", "build", "-f", "jvm/Dockerfile", "-t", image, "--build-arg", f"kafka_url={kafka_url}",
+         "--build-arg", f'build_date={date.today()}', "jvm"])
+    if result.stderr:
+        print(result.stdout)
+        return
+    shutil.rmtree("jvm/resources")
+
+
+def build_native(image, tag):
+    image = f'{image}:{tag}'
+    copy_tree("resources", "native-image/resources")
+    result = subprocess.run(
+        ["docker", "build", "-f", "native-image/Dockerfile", "-t", image,
+         "--build-arg", f'build_date={date.today()}', "native-image"])
+    if result.stderr:
+        print(result.stdout)
+        return
+    shutil.rmtree("native-image/resources")
+
+
+def run_jvm_tests(image, tag):
+    subprocess.run(["python3", "docker_sanity_test.py", f"{image}:{tag}", "jvm"], cwd="test")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("image", help="Image name that you want to keep for the Docker image")
+    parser.add_argument("-tag", "--image-tag", default="latest", dest="tag", help="Image tag that you want to add to the image")
+    parser.add_argument("-type", "--image-type", default="all", dest="image_type", help="Image type you want to build. By default it's all")
+    parser.add_argument("-u", "--kafka-url", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image")
+    parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, help="Only build the image, don't run tests")
+    parser.add_argument("-t", "--test", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image")
+    args = parser.parse_args()
+
+    if args.image_type in ("all", "jvm") and (args.build_only or not (args.build_only or args.test_only)):
+        if args.kafka_url:
+            build_jvm(args.image, args.tag, args.kafka_url)
+        else:
+            raise ValueError("--kafka-url is a required argument for jvm image")
+
+    if args.image_type in ("all", "native-image") and (args.build_only or not (args.build_only or args.test_only)):
+        build_native(args.image, args.tag)
+
+    if args.image_type in ("all", "jvm") and (args.test_only or not (args.build_only or args.test_only)):
+        run_jvm_tests(args.image, args.tag)

docker/jvm/Dockerfile

@@ -0,0 +1,78 @@
+###############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+ARG GOLANG_VERSION=1.21.1
+
+FROM golang:${GOLANG_VERSION} AS build-ub
+WORKDIR /build
+RUN useradd --no-log-init --create-home --shell /bin/bash appuser
+COPY --chown=appuser:appuser resources/ub/ ./
+RUN go build -ldflags="-w -s" ./ub.go
+USER appuser
+RUN go test ./...
+
+
+FROM eclipse-temurin:17-jre
+
+# exposed ports
+EXPOSE 9092
+
+USER root
+
+# Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments
+ARG kafka_url
+ARG build_date
+
+
+LABEL org.label-schema.name="kafka" \
+      org.label-schema.description="Apache Kafka" \
+      org.label-schema.build-date="${build_date}" \
+      org.label-schema.vcs-url="https://github.com/apache/kafka" \
+      org.label-schema.schema-version="1.0" \
+      maintainer="apache"
+
+ENV KAFKA_URL=$kafka_url
+
+RUN set -eux ; \
+    apt-get update ; \
+    apt-get upgrade -y ; \
+    apt-get install -y --no-install-recommends curl wget gpg dirmngr gpg-agent; \
+    mkdir opt/kafka; \
+    wget -nv -O kafka.tgz "$KAFKA_URL"; \
+    wget -nv -O kafka.tgz.asc "$KAFKA_URL.asc"; \
+    tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \
+    wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \
+    gpg --import KEYS; \
+    gpg --batch --verify kafka.tgz.asc kafka.tgz; \
+    mkdir -p /var/lib/kafka/data /etc/kafka/secrets /var/log/kafka /var/lib/zookeeper; \
+    mkdir -p /etc/kafka/docker /usr/logs; \
+    useradd --no-log-init --create-home --shell /bin/bash appuser; \
+    chown appuser:appuser -R /etc/kafka/ /usr/logs /opt/kafka; \
+    chown appuser:root -R /etc/kafka /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka /var/lib/zookeeper; \
+    chmod -R ug+w /etc/kafka /var/lib/kafka /var/lib/kafka /etc/kafka/secrets /etc/kafka /var/log/kafka /var/lib/zookeeper; \
+    rm kafka.tgz;
+
+COPY --from=build-ub /build/ub /usr/bin
+COPY --chown=appuser:appuser resources/common-scripts /etc/kafka/docker
+COPY --chown=appuser:appuser launch /etc/kafka/docker/launch
+
+USER appuser
+
+VOLUME ["/etc/kafka/secrets", "/var/lib/kafka/data"]
+
+CMD ["/etc/kafka/docker/run"]

docker/jvm/launch

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+###############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+
+# Override this section from the script to include the com.sun.management.jmxremote.rmi.port property.
+if [ -z "$KAFKA_JMX_OPTS" ]; then
+  export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false  -Dcom.sun.management.jmxremote.ssl=false "
+fi
+
+# The JMX client needs to be able to connect to java.rmi.server.hostname.
+# The default for bridged n/w is the bridged IP so you will only be able to connect from another docker container.
+# For host n/w, this is the IP that the hostname on the host resolves to.
+
+# If you have more that one n/w configured, hostname -i gives you all the IPs,
+# the default is to pick the first IP (or network).
+export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)}
+
+if [ "$KAFKA_JMX_PORT" ]; then
+  # This ensures that the "if" section for JMX_PORT in kafka launch script does not trigger.
+  export JMX_PORT=$KAFKA_JMX_PORT
+  export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT"
+fi
+
+# KRaft required step: Format the storage directory with provided cluster ID unless it already exists.
+if [[ -n "${KAFKA_PROCESS_ROLES-}" ]]
+then
+    echo "===> Using provided cluster id $CLUSTER_ID ..."
+
+    # A bit of a hack to not error out if the storage is already formatted.  Need storage-tool to support this
+    result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /etc/kafka/kafka.properties 2>&1) || \
+        echo $result | grep -i "already formatted" || \
+        { echo $result && (exit 1) }
+fi
+
+# Start kafka broker
+exec /opt/kafka/bin/kafka-server-start.sh /etc/kafka/kafka.properties

docker/native-image/Dockerfile

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang:1.21-bullseye AS build-ub
+WORKDIR /build
+RUN useradd --no-log-init --create-home --shell /bin/bash appuser
+COPY --chown=appuser:appuser resources/ub/ ./
+RUN go build -ldflags="-w -s" ./ub.go
+USER appuser
+RUN go test ./...
+
+
+FROM ghcr.io/graalvm/graalvm-community:17 AS build-native-image
+
+ARG kafka_url
+
+WORKDIR /app
+
+ENV KAFKA_URL=$kafka_url
+COPY native-image-configs native-image-configs
+
+RUN wget -nv -O kafka.tgz "$KAFKA_URL"; \
+    tar xfz kafka.tgz -C /kafka --strip-components 1; \
+    rm kafka.tgz ; \
+    cd kafka ;  \
+    native-image --no-fallback \
+        --allow-incomplete-classpath \
+        --report-unsupported-elements-at-runtime \
+        --install-exit-handlers \
+        -H:+ReportExceptionStackTraces \
+        -H:ReflectionConfigurationFiles=/app/native-image-configs/reflect-config.json \
+        -H:JNIConfigurationFiles=/app/native-image-configs/jni-config.json \
+        -H:ResourceConfigurationFiles=/app/native-image-configs/resource-config.json \
+        -H:SerializationConfigurationFiles=/app/native-image-configs/serialization-config.json \
+        -H:PredefinedClassesConfigurationFiles=/app/native-image-configs/predefined-classes-config.json \
+        -H:DynamicProxyConfigurationFiles=/app/native-image-configs/proxy-config.json \
+        --verbose \
+        -cp "libs/*" kafka.KafkaNativeWrapper
+
+
+FROM alpine:latest
+RUN apk update && \
+    apk add --no-cache gcompat && \
+    mkdir -p /etc/kafka/config
+WORKDIR /app
+
+COPY --from=build-ub /build/ub /usr/bin
+COPY --from=build-native-image /app/kafka_2.13-3.7.0-SNAPSHOT/kafka.kafkanativewrapper .
+COPY resources/common-scripts /etc/kafka/docker
+COPY launch /etc/kafka/docker/
+
+CMD ["/etc/kafka/docker/launch"]

docker/native-image/launch

@@ -0,0 +1,23 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json > /etc/kafka/config/kafka.properties
+ub render-template /etc/kafka/docker/kafka-log4j.properties.template > /etc/kafka/config/log4j.properties
+ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template > /etc/kafka/config/tools-log4j.properties
+
+
+result=$(/app/kafka.kafkanativewrapper storage-tool format --cluster-id=$CLUSTER_ID -c /etc/kafka/config/kafka.properties 2>&1) || echo $result | grep -i "already formatted" || echo $result && (exit 1)
+exec /app/kafka.kafkanativewrapper kafka /etc/kafka/config/kafka.properties -Dkafka.logs.dir=logs/ -Dlog4j.configuration=file:/etc/kafka/config/log4j.properties

docker/native-image/native-image-configs/jni-config.json

@@ -0,0 +1,35 @@
+[
+{
+  "name":"[Lcom.sun.management.internal.DiagnosticCommandArgumentInfo;"
+},
+{
+  "name":"[Lcom.sun.management.internal.DiagnosticCommandInfo;"
+},
+{
+  "name":"com.github.luben.zstd.ZstdInputStreamNoFinalizer",
+  "fields":[{"name":"dstPos"}, {"name":"srcPos"}]
+},
+{
+  "name":"com.sun.management.internal.DiagnosticCommandArgumentInfo",
+  "methods":[{"name":"<init>","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","boolean","boolean","int"] }]
+},
+{
+  "name":"com.sun.management.internal.DiagnosticCommandInfo",
+  "methods":[{"name":"<init>","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","java.util.List"] }]
+},
+{
+  "name":"java.lang.Boolean",
+  "methods":[{"name":"getBoolean","parameterTypes":["java.lang.String"] }]
+},
+{
+  "name":"java.lang.OutOfMemoryError"
+},
+{
+  "name":"java.util.Arrays",
+  "methods":[{"name":"asList","parameterTypes":["java.lang.Object[]"] }]
+},
+{
+  "name":"sun.management.VMManagementImpl",
+  "fields":[{"name":"compTimeMonitoringSupport"}, {"name":"currentThreadCpuTimeSupport"}, {"name":"objectMonitorUsageSupport"}, {"name":"otherThreadCpuTimeSupport"}, {"name":"remoteDiagnosticCommandsSupport"}, {"name":"synchronizerUsageSupport"}, {"name":"threadAllocatedMemorySupport"}, {"name":"threadContentionMonitoringSupport"}]
+}
+]

docker/native-image/native-image-configs/predefined-classes-config.json

@@ -0,0 +1,7 @@
+[
+  {
+    "type":"agent-extracted",
+    "classes":[
+    ]
+  }
+]