[WIP]KAFKA-15444: Native docker image

.github/workflows/docker_build_and_test.yml

@@ -21,7 +21,7 @@ on:
       image_type:
         type: choice
         description: Docker image type to build and test
-        options: 
+        options:
           - "jvm"
       kafka_url:
         description: Kafka url to be used to build the docker image
@@ -31,36 +31,36 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.10"
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r docker/requirements.txt
-    - name: Build image and run tests
-      working-directory: ./docker
-      run: |
-        python docker_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -u=${{ github.event.inputs.kafka_url }}
-    - name: Run CVE scan
-      uses: aquasecurity/trivy-action@master
-      with:
-        image-ref: 'kafka/test:test'
-        format: 'table'
-        severity: 'CRITICAL,HIGH'
-        output: scan_report_${{ github.event.inputs.image_type }}.txt
-        exit-code: '1'
-    - name: Upload test report
-      if: always()
-      uses: actions/upload-artifact@v3
-      with:
-        name: report_${{ github.event.inputs.image_type }}.html
-        path: docker/test/report_${{ github.event.inputs.image_type }}.html
-    - name: Upload CVE scan report
-      if: always()
-      uses: actions/upload-artifact@v3
-      with:
-        name: scan_report_${{ github.event.inputs.image_type }}.txt
-        path: scan_report_${{ github.event.inputs.image_type }}.txt
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r docker/requirements.txt
+      - name: Build image and run tests
+        working-directory: ./docker
+        run: |
+          python docker_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -u=${{ github.event.inputs.kafka_url }}
+      - name: Run CVE scan
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: 'kafka/test:test'
+          format: 'table'
+          severity: 'CRITICAL,HIGH'
+          output: scan_report_${{ github.event.inputs.image_type }}.txt
+          exit-code: '1'
+      - name: Upload test report
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: report_${{ github.event.inputs.image_type }}.html
+          path: docker/test/report_${{ github.event.inputs.image_type }}.html
+      - name: Upload CVE scan report
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: scan_report_${{ github.event.inputs.image_type }}.txt
+          path: scan_report_${{ github.event.inputs.image_type }}.txt

core/src/main/scala/kafka/KafkaNativeWrapper.scala

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kafka
+
+import kafka.tools.StorageTool
+import kafka.utils.Logging
+
+object KafkaNativeWrapper extends Logging {
+  def main(args: Array[String]): Unit = {
+    if (args.length == 0) {
+      throw new RuntimeException(s"Error: No operation input provided. " +
+        s"Please provide a valid operation: 'storage-tool' or 'kafka'.")
+    }
+    val operation = args.head
+    val arguments = args.tail
+    operation match {
+      case "storage-tool" => StorageTool.main(arguments)
+      case "kafka" => Kafka.main(arguments)
+      case _ =>
+        throw new RuntimeException(s"Unknown operation $operation. " +
+          s"Please provide a valid operation: 'storage-tool' or 'kafka'.")
+    }
+  }
+}

docker/README.md

@@ -65,7 +65,7 @@ Using the image in a docker container
 -------------------------------------
 - The image uses the kafka downloaded from provided kafka url
 - The image can be run in a container in default mode by running
-`docker run -p 9092:9092 <image-name:tag>`
+  `docker run -p 9092:9092 <image-name:tag>`
 - Default configs run kafka in kraft mode with plaintext listners on 9092 port.
 - Once user provided config properties are provided default configs will get replaced.
 - User can provide kafka configs following two ways:-

docker/docker_build_test.py

@@ -79,6 +79,6 @@ def run_jvm_tests(image, tag, kafka_url):
             build_jvm(args.image, args.tag, args.kafka_url)
         else:
             raise ValueError("--kafka-url is a required argument for jvm image")
-    
+
     if args.image_type == "jvm" and (args.test_only or not (args.build_only or args.test_only)):
         run_jvm_tests(args.image, args.tag, args.kafka_url)

docker/native-image/Dockerfile

@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang:1.21-bullseye AS build-ub
+WORKDIR /build
+RUN useradd --no-log-init --create-home --shell /bin/bash appuser
+COPY --chown=appuser:appuser resources/ub/ ./
+RUN go build -ldflags="-w -s" ./ub.go
+USER appuser
+RUN go test ./...
+
+
+FROM ghcr.io/graalvm/graalvm-community:17 AS build-native-image
+
+ARG kafka_url
+
+WORKDIR /app
+
+ENV KAFKA_URL=$kafka_url
+COPY native-image-configs native-image-configs
+
+RUN mkdir kafka; \
+    microdnf install wget; \
+    wget -nv -O kafka.tgz "$KAFKA_URL"; \
+    tar xfz kafka.tgz -C kafka --strip-components 1; \
+    rm kafka.tgz ; \
+    cd kafka ;  \
+    native-image --no-fallback \
+        --allow-incomplete-classpath \
+        --report-unsupported-elements-at-runtime \
+        --install-exit-handlers \
+        -H:+ReportExceptionStackTraces \
+        -H:ReflectionConfigurationFiles=/app/native-image-configs/reflect-config.json \
+        -H:JNIConfigurationFiles=/app/native-image-configs/jni-config.json \
+        -H:ResourceConfigurationFiles=/app/native-image-configs/resource-config.json \
+        -H:SerializationConfigurationFiles=/app/native-image-configs/serialization-config.json \
+        -H:PredefinedClassesConfigurationFiles=/app/native-image-configs/predefined-classes-config.json \
+        -H:DynamicProxyConfigurationFiles=/app/native-image-configs/proxy-config.json \
+        --verbose \
+        -cp "libs/*" kafka.KafkaNativeWrapper
+
+
+FROM alpine:latest
+RUN apk update && \
+    apk add --no-cache gcompat && \
+    mkdir -p /etc/kafka/config
+WORKDIR /app
+
+COPY --from=build-ub /build/ub /usr/bin
+COPY --from=build-native-image /app/kafka/kafka.kafkanativewrapper .
+COPY resources/common-scripts /etc/kafka/docker
+COPY launch /etc/kafka/docker/
+
+CMD ["/etc/kafka/docker/launch"]

docker/native-image/Dockerfile2

@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang:1.21-bullseye AS build-ub
+WORKDIR /build
+RUN useradd --no-log-init --create-home --shell /bin/bash appuser
+COPY --chown=appuser:appuser resources/ub/ ./
+RUN go build -ldflags="-w -s" ./ub.go
+USER appuser
+RUN go test ./...
+
+
+FROM ghcr.io/graalvm/graalvm-community:17 AS build-native-image
+
+WORKDIR /app
+
+COPY kafka_2.13-3.7.0-SNAPSHOT.tgz kafka_2.13-3.7.0-SNAPSHOT.tgz
+COPY native-image-configs native-image-configs
+
+RUN tar -xzf kafka_2.13-3.7.0-SNAPSHOT.tgz ; \
+    rm kafka_2.13-3.7.0-SNAPSHOT.tgz ; \
+    cd kafka_2.13-3.7.0-SNAPSHOT ;  \
+    native-image --no-fallback \
+        --allow-incomplete-classpath \
+        --report-unsupported-elements-at-runtime \
+        --install-exit-handlers \
+        -H:+ReportExceptionStackTraces \
+        -H:ReflectionConfigurationFiles=/app/native-image-configs/reflect-config.json \
+        -H:JNIConfigurationFiles=/app/native-image-configs/jni-config.json \
+        -H:ResourceConfigurationFiles=/app/native-image-configs/resource-config.json \
+        -H:SerializationConfigurationFiles=/app/native-image-configs/serialization-config.json \
+        -H:PredefinedClassesConfigurationFiles=/app/native-image-configs/predefined-classes-config.json \
+        -H:DynamicProxyConfigurationFiles=/app/native-image-configs/proxy-config.json \
+        --verbose \
+        -cp "libs/*" kafka.KafkaNativeWrapper
+
+
+FROM alpine:latest
+RUN apk update && \
+    apk add --no-cache gcompat && \
+    apk add --no-cache bash && \
+    apk -v cache clean && \
+    mkdir -p /etc/kafka/config
+WORKDIR /app
+
+COPY --from=build-ub /build/ub /usr/bin
+COPY --from=build-native-image /app/kafka_2.13-3.7.0-SNAPSHOT/kafka.kafkanativewrapper .
+COPY resources/common-scripts /etc/kafka/docker
+COPY launch /etc/kafka/docker/
+
+CMD ["/etc/kafka/docker/launch"]

docker/native-image/launch

@@ -0,0 +1,23 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json > /etc/kafka/config/kafka.properties
+ub render-template /etc/kafka/docker/kafka-log4j.properties.template > /etc/kafka/config/log4j.properties
+ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template > /etc/kafka/config/tools-log4j.properties
+
+
+result=$(/app/kafka.kafkanativewrapper storage-tool format --cluster-id=$CLUSTER_ID -c /etc/kafka/config/kafka.properties 2>&1) || echo $result | grep -i "already formatted" || echo $result && (exit 1)
+exec /app/kafka.kafkanativewrapper kafka /etc/kafka/config/kafka.properties -Dkafka.logs.dir=logs/ -Dlog4j.configuration=file:/etc/kafka/config/log4j.properties

docker/native-image/native-image-configs/jni-config.json

@@ -0,0 +1,35 @@
+[
+{
+  "name":"[Lcom.sun.management.internal.DiagnosticCommandArgumentInfo;"
+},
+{
+  "name":"[Lcom.sun.management.internal.DiagnosticCommandInfo;"
+},
+{
+  "name":"com.github.luben.zstd.ZstdInputStreamNoFinalizer",
+  "fields":[{"name":"dstPos"}, {"name":"srcPos"}]
+},
+{
+  "name":"com.sun.management.internal.DiagnosticCommandArgumentInfo",
+  "methods":[{"name":"<init>","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","boolean","boolean","int"] }]
+},
+{
+  "name":"com.sun.management.internal.DiagnosticCommandInfo",
+  "methods":[{"name":"<init>","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","java.util.List"] }]
+},
+{
+  "name":"java.lang.Boolean",
+  "methods":[{"name":"getBoolean","parameterTypes":["java.lang.String"] }]
+},
+{
+  "name":"java.lang.OutOfMemoryError"
+},
+{
+  "name":"java.util.Arrays",
+  "methods":[{"name":"asList","parameterTypes":["java.lang.Object[]"] }]
+},
+{
+  "name":"sun.management.VMManagementImpl",
+  "fields":[{"name":"compTimeMonitoringSupport"}, {"name":"currentThreadCpuTimeSupport"}, {"name":"objectMonitorUsageSupport"}, {"name":"otherThreadCpuTimeSupport"}, {"name":"remoteDiagnosticCommandsSupport"}, {"name":"synchronizerUsageSupport"}, {"name":"threadAllocatedMemorySupport"}, {"name":"threadContentionMonitoringSupport"}]
+}
+]

docker/native-image/native-image-configs/predefined-classes-config.json

@@ -0,0 +1,7 @@
+[
+  {
+    "type":"agent-extracted",
+    "classes":[
+    ]
+  }
+]

docker/native-image/native-image-configs/proxy-config.json

@@ -0,0 +1,5 @@
+[
+  {
+    "interfaces":["sun.misc.SignalHandler"]
+  }
+]