diff --git a/.github/workflows/docker_build_and_test.yml b/.github/workflows/docker_build_and_test.yml index 695c08672fd87..5721333b14623 100644 --- a/.github/workflows/docker_build_and_test.yml +++ b/.github/workflows/docker_build_and_test.yml @@ -21,7 +21,7 @@ on: image_type: type: choice description: Docker image type to build and test - options: + options: - "jvm" kafka_url: description: Kafka url to be used to build the docker image @@ -31,36 +31,36 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r docker/requirements.txt - - name: Build image and run tests - working-directory: ./docker - run: | - python docker_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -u=${{ github.event.inputs.kafka_url }} - - name: Run CVE scan - uses: aquasecurity/trivy-action@master - with: - image-ref: 'kafka/test:test' - format: 'table' - severity: 'CRITICAL,HIGH' - output: scan_report_${{ github.event.inputs.image_type }}.txt - exit-code: '1' - - name: Upload test report - if: always() - uses: actions/upload-artifact@v3 - with: - name: report_${{ github.event.inputs.image_type }}.html - path: docker/test/report_${{ github.event.inputs.image_type }}.html - - name: Upload CVE scan report - if: always() - uses: actions/upload-artifact@v3 - with: - name: scan_report_${{ github.event.inputs.image_type }}.txt - path: scan_report_${{ github.event.inputs.image_type }}.txt + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r docker/requirements.txt + - name: Build image and run tests + working-directory: ./docker + run: | + python docker_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -u=${{ github.event.inputs.kafka_url }} + - name: Run CVE scan + uses: aquasecurity/trivy-action@master + with: + image-ref: 'kafka/test:test' + format: 'table' + severity: 'CRITICAL,HIGH' + output: scan_report_${{ github.event.inputs.image_type }}.txt + exit-code: '1' + - name: Upload test report + if: always() + uses: actions/upload-artifact@v3 + with: + name: report_${{ github.event.inputs.image_type }}.html + path: docker/test/report_${{ github.event.inputs.image_type }}.html + - name: Upload CVE scan report + if: always() + uses: actions/upload-artifact@v3 + with: + name: scan_report_${{ github.event.inputs.image_type }}.txt + path: scan_report_${{ github.event.inputs.image_type }}.txt diff --git a/core/src/main/scala/kafka/KafkaNativeWrapper.scala b/core/src/main/scala/kafka/KafkaNativeWrapper.scala new file mode 100644 index 0000000000000..83fb5960b593a --- /dev/null +++ b/core/src/main/scala/kafka/KafkaNativeWrapper.scala @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka + +import kafka.tools.StorageTool +import kafka.utils.Logging + +object KafkaNativeWrapper extends Logging { + def main(args: Array[String]): Unit = { + if (args.length == 0) { + throw new RuntimeException(s"Error: No operation input provided. " + + s"Please provide a valid operation: 'storage-tool' or 'kafka'.") + } + val operation = args.head + val arguments = args.tail + operation match { + case "storage-tool" => StorageTool.main(arguments) + case "kafka" => Kafka.main(arguments) + case _ => + throw new RuntimeException(s"Unknown operation $operation. " + + s"Please provide a valid operation: 'storage-tool' or 'kafka'.") + } + } +} diff --git a/docker/README.md b/docker/README.md index f4036b156c62b..05801053ce62a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -65,7 +65,7 @@ Using the image in a docker container ------------------------------------- - The image uses the kafka downloaded from provided kafka url - The image can be run in a container in default mode by running -`docker run -p 9092:9092 ` + `docker run -p 9092:9092 ` - Default configs run kafka in kraft mode with plaintext listners on 9092 port. - Once user provided config properties are provided default configs will get replaced. - User can provide kafka configs following two ways:- diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index e63d8746ac1bc..d376796d41f55 100755 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -79,6 +79,6 @@ def run_jvm_tests(image, tag, kafka_url): build_jvm(args.image, args.tag, args.kafka_url) else: raise ValueError("--kafka-url is a required argument for jvm image") - + if args.image_type == "jvm" and (args.test_only or not (args.build_only or args.test_only)): run_jvm_tests(args.image, args.tag, args.kafka_url) diff --git a/docker/native-image/Dockerfile b/docker/native-image/Dockerfile new file mode 100644 index 0000000000000..8439fa16a71c9 --- /dev/null +++ b/docker/native-image/Dockerfile @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.21-bullseye AS build-ub +WORKDIR /build +RUN useradd --no-log-init --create-home --shell /bin/bash appuser +COPY --chown=appuser:appuser resources/ub/ ./ +RUN go build -ldflags="-w -s" ./ub.go +USER appuser +RUN go test ./... + + +FROM ghcr.io/graalvm/graalvm-community:17 AS build-native-image + +ARG kafka_url + +WORKDIR /app + +ENV KAFKA_URL=$kafka_url +COPY native-image-configs native-image-configs + +RUN mkdir kafka; \ + microdnf install wget; \ + wget -nv -O kafka.tgz "$KAFKA_URL"; \ + tar xfz kafka.tgz -C kafka --strip-components 1; \ + rm kafka.tgz ; \ + cd kafka ; \ + native-image --no-fallback \ + --allow-incomplete-classpath \ + --report-unsupported-elements-at-runtime \ + --install-exit-handlers \ + -H:+ReportExceptionStackTraces \ + -H:ReflectionConfigurationFiles=/app/native-image-configs/reflect-config.json \ + -H:JNIConfigurationFiles=/app/native-image-configs/jni-config.json \ + -H:ResourceConfigurationFiles=/app/native-image-configs/resource-config.json \ + -H:SerializationConfigurationFiles=/app/native-image-configs/serialization-config.json \ + -H:PredefinedClassesConfigurationFiles=/app/native-image-configs/predefined-classes-config.json \ + -H:DynamicProxyConfigurationFiles=/app/native-image-configs/proxy-config.json \ + --verbose \ + -cp "libs/*" kafka.KafkaNativeWrapper + + +FROM alpine:latest +RUN apk update && \ + apk add --no-cache gcompat && \ + mkdir -p /etc/kafka/config +WORKDIR /app + +COPY --from=build-ub /build/ub /usr/bin +COPY --from=build-native-image /app/kafka/kafka.kafkanativewrapper . +COPY resources/common-scripts /etc/kafka/docker +COPY launch /etc/kafka/docker/ + +CMD ["/etc/kafka/docker/launch"] diff --git a/docker/native-image/Dockerfile2 b/docker/native-image/Dockerfile2 new file mode 100644 index 0000000000000..53d7fba76e8af --- /dev/null +++ b/docker/native-image/Dockerfile2 @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.21-bullseye AS build-ub +WORKDIR /build +RUN useradd --no-log-init --create-home --shell /bin/bash appuser +COPY --chown=appuser:appuser resources/ub/ ./ +RUN go build -ldflags="-w -s" ./ub.go +USER appuser +RUN go test ./... + + +FROM ghcr.io/graalvm/graalvm-community:17 AS build-native-image + +WORKDIR /app + +COPY kafka_2.13-3.7.0-SNAPSHOT.tgz kafka_2.13-3.7.0-SNAPSHOT.tgz +COPY native-image-configs native-image-configs + +RUN tar -xzf kafka_2.13-3.7.0-SNAPSHOT.tgz ; \ + rm kafka_2.13-3.7.0-SNAPSHOT.tgz ; \ + cd kafka_2.13-3.7.0-SNAPSHOT ; \ + native-image --no-fallback \ + --allow-incomplete-classpath \ + --report-unsupported-elements-at-runtime \ + --install-exit-handlers \ + -H:+ReportExceptionStackTraces \ + -H:ReflectionConfigurationFiles=/app/native-image-configs/reflect-config.json \ + -H:JNIConfigurationFiles=/app/native-image-configs/jni-config.json \ + -H:ResourceConfigurationFiles=/app/native-image-configs/resource-config.json \ + -H:SerializationConfigurationFiles=/app/native-image-configs/serialization-config.json \ + -H:PredefinedClassesConfigurationFiles=/app/native-image-configs/predefined-classes-config.json \ + -H:DynamicProxyConfigurationFiles=/app/native-image-configs/proxy-config.json \ + --verbose \ + -cp "libs/*" kafka.KafkaNativeWrapper + + +FROM alpine:latest +RUN apk update && \ + apk add --no-cache gcompat && \ + apk add --no-cache bash && \ + apk -v cache clean && \ + mkdir -p /etc/kafka/config +WORKDIR /app + +COPY --from=build-ub /build/ub /usr/bin +COPY --from=build-native-image /app/kafka_2.13-3.7.0-SNAPSHOT/kafka.kafkanativewrapper . +COPY resources/common-scripts /etc/kafka/docker +COPY launch /etc/kafka/docker/ + +CMD ["/etc/kafka/docker/launch"] diff --git a/docker/native-image/launch b/docker/native-image/launch new file mode 100755 index 0000000000000..a09fc492f11fe --- /dev/null +++ b/docker/native-image/launch @@ -0,0 +1,23 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json > /etc/kafka/config/kafka.properties +ub render-template /etc/kafka/docker/kafka-log4j.properties.template > /etc/kafka/config/log4j.properties +ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template > /etc/kafka/config/tools-log4j.properties + + +result=$(/app/kafka.kafkanativewrapper storage-tool format --cluster-id=$CLUSTER_ID -c /etc/kafka/config/kafka.properties 2>&1) || echo $result | grep -i "already formatted" || echo $result && (exit 1) +exec /app/kafka.kafkanativewrapper kafka /etc/kafka/config/kafka.properties -Dkafka.logs.dir=logs/ -Dlog4j.configuration=file:/etc/kafka/config/log4j.properties diff --git a/docker/native-image/native-image-configs/jni-config.json b/docker/native-image/native-image-configs/jni-config.json new file mode 100644 index 0000000000000..50b6bf1fd6dbb --- /dev/null +++ b/docker/native-image/native-image-configs/jni-config.json @@ -0,0 +1,35 @@ +[ +{ + "name":"[Lcom.sun.management.internal.DiagnosticCommandArgumentInfo;" +}, +{ + "name":"[Lcom.sun.management.internal.DiagnosticCommandInfo;" +}, +{ + "name":"com.github.luben.zstd.ZstdInputStreamNoFinalizer", + "fields":[{"name":"dstPos"}, {"name":"srcPos"}] +}, +{ + "name":"com.sun.management.internal.DiagnosticCommandArgumentInfo", + "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","boolean","boolean","int"] }] +}, +{ + "name":"com.sun.management.internal.DiagnosticCommandInfo", + "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","java.lang.String","boolean","java.util.List"] }] +}, +{ + "name":"java.lang.Boolean", + "methods":[{"name":"getBoolean","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"java.lang.OutOfMemoryError" +}, +{ + "name":"java.util.Arrays", + "methods":[{"name":"asList","parameterTypes":["java.lang.Object[]"] }] +}, +{ + "name":"sun.management.VMManagementImpl", + "fields":[{"name":"compTimeMonitoringSupport"}, {"name":"currentThreadCpuTimeSupport"}, {"name":"objectMonitorUsageSupport"}, {"name":"otherThreadCpuTimeSupport"}, {"name":"remoteDiagnosticCommandsSupport"}, {"name":"synchronizerUsageSupport"}, {"name":"threadAllocatedMemorySupport"}, {"name":"threadContentionMonitoringSupport"}] +} +] \ No newline at end of file diff --git a/docker/native-image/native-image-configs/predefined-classes-config.json b/docker/native-image/native-image-configs/predefined-classes-config.json new file mode 100644 index 0000000000000..847895071fbcb --- /dev/null +++ b/docker/native-image/native-image-configs/predefined-classes-config.json @@ -0,0 +1,7 @@ +[ + { + "type":"agent-extracted", + "classes":[ + ] + } +] diff --git a/docker/native-image/native-image-configs/proxy-config.json b/docker/native-image/native-image-configs/proxy-config.json new file mode 100644 index 0000000000000..2a8e3c8a13689 --- /dev/null +++ b/docker/native-image/native-image-configs/proxy-config.json @@ -0,0 +1,5 @@ +[ + { + "interfaces":["sun.misc.SignalHandler"] + } +] \ No newline at end of file diff --git a/docker/native-image/native-image-configs/reflect-config.json b/docker/native-image/native-image-configs/reflect-config.json new file mode 100644 index 0000000000000..30a90f34cea81 --- /dev/null +++ b/docker/native-image/native-image-configs/reflect-config.json @@ -0,0 +1,719 @@ +[ +{ + "name":"[B" +}, +{ + "name":"[C" +}, +{ + "name":"[D" +}, +{ + "name":"[F" +}, +{ + "name":"[I" +}, +{ + "name":"[J" +}, +{ + "name":"[Ljava.io.File;" +}, +{ + "name":"[Ljava.lang.Runnable;" +}, +{ + "name":"[Ljava.lang.String;" +}, +{ + "name":"[Ljava.util.concurrent.CompletableFuture;" +}, +{ + "name":"[Ljava.util.concurrent.Future;" +}, +{ + "name":"[Ljavax.management.openmbean.CompositeData;" +}, +{ + "name":"[Lkafka.server.DelayedOperationPurgatory$WatcherList;" +}, +{ + "name":"[Lorg.apache.kafka.common.resource.PatternType;" +}, +{ + "name":"[Lscala.Tuple2;" +}, +{ + "name":"[S" +}, +{ + "name":"[Z" +}, +{ + "name":"com.fasterxml.jackson.databind.ext.Java7SupportImpl", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"com.sun.management.GarbageCollectorMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"com.sun.management.GcInfo", + "queryAllPublicMethods":true +}, +{ + "name":"com.sun.management.HotSpotDiagnosticMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"com.sun.management.ThreadMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"com.sun.management.UnixOperatingSystemMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"com.sun.management.VMOption", + "queryAllPublicMethods":true +}, +{ + "name":"com.sun.management.internal.GarbageCollectorExtImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"com.sun.management.internal.HotSpotDiagnostic", + "queryAllPublicConstructors":true +}, +{ + "name":"com.sun.management.internal.HotSpotThreadImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"com.sun.management.internal.OperatingSystemImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$Gauge", + "queryAllPublicConstructors":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$GaugeMBean", + "queryAllPublicMethods":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$Histogram", + "queryAllPublicConstructors":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$HistogramMBean", + "queryAllPublicMethods":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$Meter", + "queryAllPublicConstructors":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$MeterMBean", + "queryAllPublicMethods":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$Timer", + "queryAllPublicConstructors":true +}, +{ + "name":"com.yammer.metrics.reporting.JmxReporter$TimerMBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.beans.PropertyVetoException" +}, +{ + "name":"java.io.Serializable", + "queryAllDeclaredMethods":true +}, +{ + "name":"java.lang.Boolean", + "fields":[{"name":"TYPE"}] +}, +{ + "name":"java.lang.Byte", + "fields":[{"name":"TYPE"}] +}, +{ + "name":"java.lang.Character", + "fields":[{"name":"TYPE"}] +}, +{ + "name":"java.lang.ClassValue" +}, +{ + "name":"java.lang.Deprecated", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.Double", + "fields":[{"name":"TYPE"}] +}, +{ + "name":"java.lang.Float", + "fields":[{"name":"TYPE"}] +}, +{ + "name":"java.lang.Integer", + "fields":[{"name":"TYPE"}], + "methods":[{"name":"valueOf","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"java.lang.Iterable", + "queryAllDeclaredMethods":true +}, +{ + "name":"java.lang.Long", + "fields":[{"name":"TYPE"}], + "methods":[{"name":"valueOf","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"java.lang.Object", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.ObjectBeanInfo" +}, +{ + "name":"java.lang.ObjectCustomizer" +}, +{ + "name":"java.lang.Short", + "fields":[{"name":"TYPE"}] +}, +{ + "name":"java.lang.StackTraceElement", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.String", + "fields":[{"name":"TYPE"}], + "methods":[{"name":"","parameterTypes":["java.lang.String"] }, {"name":"valueOf","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"java.lang.Thread", + "fields":[{"name":"threadLocalRandomProbe"}] +}, +{ + "name":"java.lang.Void", + "fields":[{"name":"TYPE"}] +}, +{ + "name":"java.lang.invoke.VarHandle", + "methods":[{"name":"releaseFence","parameterTypes":[] }] +}, +{ + "name":"java.lang.management.BufferPoolMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.ClassLoadingMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.CompilationMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.LockInfo", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.ManagementPermission", + "methods":[{"name":"","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"java.lang.management.MemoryMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.MemoryManagerMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.MemoryPoolMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.MemoryUsage", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.MonitorInfo", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.PlatformLoggingMXBean", + "queryAllPublicMethods":true, + "methods":[{"name":"getLoggerLevel","parameterTypes":["java.lang.String"] }, {"name":"getLoggerNames","parameterTypes":[] }, {"name":"getParentLoggerName","parameterTypes":["java.lang.String"] }, {"name":"setLoggerLevel","parameterTypes":["java.lang.String","java.lang.String"] }] +}, +{ + "name":"java.lang.management.RuntimeMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.lang.management.ThreadInfo", + "queryAllPublicMethods":true +}, +{ + "name":"java.math.BigDecimal" +}, +{ + "name":"java.math.BigInteger" +}, +{ + "name":"java.rmi.Remote", + "queryAllPublicMethods":true +}, +{ + "name":"java.rmi.registry.Registry", + "queryAllPublicMethods":true +}, +{ + "name":"java.security.SecureRandomParameters" +}, +{ + "name":"java.util.AbstractCollection", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true +}, +{ + "name":"java.util.AbstractList", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true +}, +{ + "name":"java.util.AbstractMap", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true +}, +{ + "name":"java.util.Collection", + "queryAllDeclaredMethods":true +}, +{ + "name":"java.util.Date" +}, +{ + "name":"java.util.List", + "queryAllDeclaredMethods":true +}, +{ + "name":"java.util.Map", + "queryAllDeclaredMethods":true +}, +{ + "name":"java.util.PropertyPermission", + "methods":[{"name":"","parameterTypes":["java.lang.String","java.lang.String"] }] +}, +{ + "name":"java.util.concurrent.ForkJoinTask", + "fields":[{"name":"aux"}, {"name":"status"}] +}, +{ + "name":"java.util.concurrent.atomic.AtomicBoolean", + "fields":[{"name":"value"}] +}, +{ + "name":"java.util.concurrent.atomic.AtomicReference", + "fields":[{"name":"value"}] +}, +{ + "name":"java.util.concurrent.atomic.Striped64", + "fields":[{"name":"base"}, {"name":"cellsBusy"}] +}, +{ + "name":"java.util.concurrent.atomic.Striped64$Cell", + "fields":[{"name":"value"}] +}, +{ + "name":"java.util.logging.LogManager", + "methods":[{"name":"getLoggingMXBean","parameterTypes":[] }] +}, +{ + "name":"java.util.logging.LoggingMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"java.util.zip.CRC32C", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"java.util.zip.Checksum", + "methods":[{"name":"update","parameterTypes":["java.nio.ByteBuffer"] }] +}, +{ + "name":"javax.management.MBeanOperationInfo", + "queryAllPublicMethods":true, + "methods":[{"name":"getSignature","parameterTypes":[] }] +}, +{ + "name":"javax.management.MBeanServerBuilder", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"javax.management.ObjectName" +}, +{ + "name":"javax.management.StandardEmitterMBean", + "methods":[{"name":"cacheMBeanInfo","parameterTypes":["javax.management.MBeanInfo"] }, {"name":"getCachedMBeanInfo","parameterTypes":[] }, {"name":"getMBeanInfo","parameterTypes":[] }] +}, +{ + "name":"javax.management.openmbean.CompositeData" +}, +{ + "name":"javax.management.openmbean.OpenMBeanOperationInfoSupport" +}, +{ + "name":"javax.management.openmbean.TabularData" +}, +{ + "name":"javax.management.remote.rmi.RMIServer", + "queryAllPublicMethods":true, + "methods":[{"name":"getVersion","parameterTypes":[] }, {"name":"newClient","parameterTypes":["java.lang.Object"] }] +}, +{ + "name":"javax.management.remote.rmi.RMIServerImpl_Skel" +}, +{ + "name":"javax.management.remote.rmi.RMIServerImpl_Stub", + "methods":[{"name":"","parameterTypes":["java.rmi.server.RemoteRef"] }] +}, +{ + "name":"jdk.management.jfr.ConfigurationInfo", + "queryAllPublicMethods":true +}, +{ + "name":"jdk.management.jfr.EventTypeInfo", + "queryAllPublicMethods":true +}, +{ + "name":"jdk.management.jfr.FlightRecorderMXBean", + "queryAllPublicMethods":true +}, +{ + "name":"jdk.management.jfr.FlightRecorderMXBeanImpl", + "queryAllPublicConstructors":true, + "methods":[{"name":"cacheMBeanInfo","parameterTypes":["javax.management.MBeanInfo"] }, {"name":"getCachedMBeanInfo","parameterTypes":[] }, {"name":"getMBeanInfo","parameterTypes":[] }, {"name":"getNotificationInfo","parameterTypes":[] }] +}, +{ + "name":"jdk.management.jfr.RecordingInfo", + "queryAllPublicMethods":true +}, +{ + "name":"jdk.management.jfr.SettingDescriptorInfo", + "queryAllPublicMethods":true +}, +{ + "name":"kafka.tools.ConsoleProducer$LineMessageReader", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"kafka.tools.DefaultMessageFormatter", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"kafka.utils.Log4jController", + "queryAllPublicConstructors":true, + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"kafka.utils.Log4jControllerMBean", + "queryAllPublicMethods":true +}, +{ + "name":"net.jpountz.lz4.LZ4JavaSafeCompressor", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.lz4.LZ4HCJavaSafeCompressor", + "fields":[{"name":"INSTANCE"}], + "methods":[{"name":"","parameterTypes":["int"] }] +}, +{ + "name":"net.jpountz.lz4.LZ4JavaSafeFastDecompressor", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.lz4.LZ4JavaSafeSafeDecompressor", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.XXHash32JavaSafe", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.XXHash64JavaSafe", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.StreamingXXHash32JavaSafe$Factory", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.StreamingXXHash64JavaSafe$Factory", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.lz4.LZ4HCJNICompressor", + "fields":[{"name":"INSTANCE"}], + "methods":[{"name":"","parameterTypes":["int"] }] +}, +{ + "name":"net.jpountz.lz4.LZ4JNICompressor", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.lz4.LZ4JNIFastDecompressor", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.lz4.LZ4JNISafeDecompressor", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.StreamingXXHash32JNI$Factory", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.StreamingXXHash64JNI$Factory", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.XXHash32JNI", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"net.jpountz.xxhash.XXHash64JNI", + "fields":[{"name":"INSTANCE"}] +}, +{ + "name":"org.apache.kafka.clients.consumer.CooperativeStickyAssignor", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"org.apache.kafka.clients.consumer.RangeAssignor", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"org.apache.kafka.common.metrics.JmxReporter", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"org.apache.kafka.common.serialization.ByteArraySerializer", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"org.apache.kafka.common.utils.AppInfoParser$AppInfo", + "queryAllPublicConstructors":true +}, +{ + "name":"org.apache.kafka.common.utils.AppInfoParser$AppInfoMBean", + "queryAllPublicMethods":true +}, +{ + "name":"org.apache.kafka.coordinator.group.assignor.RangeAssignor", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"org.apache.log4j.AppenderSkeleton", + "queryAllPublicMethods":true, + "methods":[{"name":"setThreshold","parameterTypes":["org.apache.log4j.Priority"] }] +}, +{ + "name":"org.apache.log4j.AppenderSkeletonBeanInfo" +}, +{ + "name":"org.apache.log4j.AppenderSkeletonCustomizer" +}, +{ + "name":"org.apache.log4j.ConsoleAppender", + "queryAllPublicMethods":true, + "methods":[{"name":"","parameterTypes":[] }, {"name":"setTarget","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"org.apache.log4j.ConsoleAppenderBeanInfo" +}, +{ + "name":"org.apache.log4j.ConsoleAppenderCustomizer" +}, +{ + "name":"org.apache.log4j.DailyRollingFileAppender", + "queryAllPublicMethods":true, + "methods":[{"name":"","parameterTypes":[] }, {"name":"setDatePattern","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"org.apache.log4j.DailyRollingFileAppenderBeanInfo" +}, +{ + "name":"org.apache.log4j.DailyRollingFileAppenderCustomizer" +}, +{ + "name":"org.apache.log4j.FileAppender", + "queryAllPublicMethods":true, + "methods":[{"name":"setFile","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"org.apache.log4j.FileAppenderBeanInfo" +}, +{ + "name":"org.apache.log4j.FileAppenderCustomizer" +}, +{ + "name":"org.apache.log4j.Layout", + "queryAllPublicMethods":true +}, +{ + "name":"org.apache.log4j.LayoutBeanInfo" +}, +{ + "name":"org.apache.log4j.LayoutCustomizer" +}, +{ + "name":"org.apache.log4j.Log4jLoggerFactory" +}, +{ + "name":"org.apache.log4j.PatternLayout", + "queryAllPublicMethods":true, + "methods":[{"name":"","parameterTypes":[] }, {"name":"setConversionPattern","parameterTypes":["java.lang.String"] }] +}, +{ + "name":"org.apache.log4j.PatternLayoutBeanInfo" +}, +{ + "name":"org.apache.log4j.PatternLayoutCustomizer" +}, +{ + "name":"org.apache.log4j.WriterAppender", + "queryAllPublicMethods":true +}, +{ + "name":"org.apache.log4j.WriterAppenderBeanInfo" +}, +{ + "name":"org.apache.log4j.WriterAppenderCustomizer" +}, +{ + "name":"org.apache.zookeeper.ClientCnxnSocketNIO", + "methods":[{"name":"","parameterTypes":["org.apache.zookeeper.client.ZKClientConfig"] }] +}, +{ + "name":"org.osgi.framework.BundleEvent" +}, +{ + "name":"scala.collection.convert.JavaCollectionWrappers$IterableWrapperTrait", + "queryAllDeclaredMethods":true +}, +{ + "name":"scala.collection.convert.JavaCollectionWrappers$MapWrapper", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true +}, +{ + "name":"scala.collection.convert.JavaCollectionWrappers$MutableBufferWrapper", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true +}, +{ + "name":"scala.collection.convert.JavaCollectionWrappers$MutableMapWrapper", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true +}, +{ + "name":"scala.collection.convert.JavaCollectionWrappers$SeqWrapper", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true, + "queryAllDeclaredConstructors":true +}, +{ + "name":"scala.reflect.ScalaSignature", + "queryAllPublicMethods":true +}, +{ + "name":"sun.management.ClassLoadingImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.management.CompilationImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.management.ManagementFactoryHelper$1", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.management.ManagementFactoryHelper$PlatformLoggingImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.management.MemoryImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.management.MemoryManagerImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.management.MemoryPoolImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.management.RuntimeImpl", + "queryAllPublicConstructors":true +}, +{ + "name":"sun.misc.Signal", + "methods":[{"name":"","parameterTypes":["java.lang.String"] }, {"name":"getName","parameterTypes":[] }, {"name":"handle","parameterTypes":["sun.misc.Signal","sun.misc.SignalHandler"] }] +}, +{ + "name":"sun.misc.SignalHandler", + "methods":[{"name":"handle","parameterTypes":["sun.misc.Signal"] }] +}, +{ + "name":"sun.misc.Unsafe", + "fields":[{"name":"theUnsafe"}], + "methods":[{"name":"invokeCleaner","parameterTypes":["java.nio.ByteBuffer"] }] +}, +{ + "name":"sun.rmi.registry.RegistryImpl_Skel", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.rmi.registry.RegistryImpl_Stub", + "methods":[{"name":"","parameterTypes":["java.rmi.server.RemoteRef"] }] +}, +{ + "name":"sun.rmi.transport.DGCImpl_Skel", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.rmi.transport.DGCImpl_Stub", + "methods":[{"name":"","parameterTypes":["java.rmi.server.RemoteRef"] }] +}, +{ + "name":"sun.security.provider.ConfigFile", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.MD5", + "methods":[{"name":"","parameterTypes":[] }] +}, +{ + "name":"sun.security.provider.NativePRNG", + "methods":[{"name":"","parameterTypes":[] }, {"name":"","parameterTypes":["java.security.SecureRandomParameters"] }] +}, +{ + "name":"sun.security.provider.SHA", + "methods":[{"name":"","parameterTypes":[] }] +} +] \ No newline at end of file diff --git a/docker/native-image/native-image-configs/resource-config.json b/docker/native-image/native-image-configs/resource-config.json new file mode 100644 index 0000000000000..8b489fd205938 --- /dev/null +++ b/docker/native-image/native-image-configs/resource-config.json @@ -0,0 +1,20 @@ +{ + "resources":{ + "includes":[{ + "pattern":"\\Qkafka/kafka-version.properties\\E" + }, { + "pattern":"\\Qlinux/amd64/libzstd-jni-1.5.5-6.so\\E" + }, { + "pattern":"\\Qnet/jpountz/util/linux/amd64/liblz4-java.so\\E" + }, { + "pattern":"\\Qorg/slf4j/impl/StaticLoggerBinder.class\\E" + }, { + "pattern":"\\Qorg/xerial/snappy/VERSION\\E" + }, { + "pattern":"\\Qorg/xerial/snappy/native/Linux/x86_64/libsnappyjava.so\\E" + }]}, + "bundles":[{ + "name":"net.sourceforge.argparse4j.internal.ArgumentParserImpl", + "locales":["und"] + }] +} \ No newline at end of file diff --git a/docker/native-image/native-image-configs/serialization-config.json b/docker/native-image/native-image-configs/serialization-config.json new file mode 100644 index 0000000000000..0d85b152974ca --- /dev/null +++ b/docker/native-image/native-image-configs/serialization-config.json @@ -0,0 +1,17 @@ +{ + "types":[ + { + "name":"java.rmi.server.RemoteObject" + }, + { + "name":"java.rmi.server.RemoteStub" + }, + { + "name":"javax.management.remote.rmi.RMIServerImpl_Stub" + } + ], + "lambdaCapturingTypes":[ + ], + "proxies":[ + ] +} \ No newline at end of file diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index eab86c0e9e265..d68b988f212b8 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -29,7 +29,7 @@ then # Unset in case env variable is set with empty value unset KAFKA_ADVERTISED_LISTENERS fi - fi + fi fi # By default, LISTENERS is derived from ADVERTISED_LISTENERS by replacing diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 3bebaefbfd4dd..b42c6fd00e237 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -24,7 +24,7 @@ class DockerSanityTest(unittest.TestCase): IMAGE="apache/kafka" FIXTURES_DIR="." - + def resume_container(self): subprocess.run(["docker", "start", constants.BROKER_CONTAINER]) @@ -42,7 +42,7 @@ def start_compose(self, filename) -> None: self.update_file(filename, "image: {$IMAGE}", f"image: {self.IMAGE}") self.update_file(f"{self.FIXTURES_DIR}/{constants.SSL_CLIENT_CONFIG}", "{$DIR}", self.FIXTURES_DIR) subprocess.run(["docker-compose", "-f", filename, "up", "-d"]) - + def destroy_compose(self, filename) -> None: subprocess.run(["docker-compose", "-f", filename, "down"]) self.update_file(filename, f"image: {self.IMAGE}", "image: {$IMAGE}") @@ -58,24 +58,24 @@ def create_topic(self, topic, topic_config): if topic in output.decode("utf-8"): return True return False - + def produce_message(self, topic, producer_config, key, value): command = ["echo", f'"{key}:{value}"', "|", f"{self.FIXTURES_DIR}/{constants.KAFKA_CONSOLE_PRODUCER}", "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'", "--timeout", f"{constants.CLIENT_TIMEOUT}"] command.extend(producer_config) subprocess.run(["bash", "-c", " ".join(command)]) - + def consume_message(self, topic, consumer_config): command = [f"{self.FIXTURES_DIR}/{constants.KAFKA_CONSOLE_CONSUMER}", "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1", "--timeout-ms", f"{constants.CLIENT_TIMEOUT}"] command.extend(consumer_config) message = subprocess.check_output(["bash", "-c", " ".join(command)]) return message.decode("utf-8").strip() - + def get_metrics(self, jmx_tool_config): command = [f"{self.FIXTURES_DIR}/{constants.KAFKA_RUN_CLASS}", constants.JMX_TOOL] command.extend(jmx_tool_config) message = subprocess.check_output(["bash", "-c", " ".join(command)]) return message.decode("utf-8").strip().split() - + def broker_metrics_flow(self): print(f"Running {constants.BROKER_METRICS_TESTS}") errors = [] @@ -102,7 +102,7 @@ def broker_metrics_flow(self): except AssertionError as e: errors.append(constants.BROKER_METRICS_ERROR_PREFIX + str(e)) return errors - + metrics_after_message = self.get_metrics(jmx_tool_config) try: self.assertEqual(len(metrics_before_message), 2) @@ -142,19 +142,19 @@ def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): self.assertEqual(message, "key:message") except AssertionError as e: errors.append(test_error_prefix + str(e)) - + return errors - + def broker_restart_flow(self): print(f"Running {constants.BROKER_RESTART_TESTS}") errors = [] - + try: self.assertTrue(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) except AssertionError as e: errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) return errors - + producer_config = ["--bootstrap-server", "localhost:9092", "--property", "client.id=host"] self.produce_message(constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") @@ -169,7 +169,7 @@ def broker_restart_flow(self): self.assertEqual(message, "key:message") except AssertionError as e: errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) - + return errors def execute(self): @@ -194,7 +194,7 @@ def execute(self): except Exception as e: print(constants.BROKER_RESTART_ERROR_PREFIX, str(e)) total_errors.append(str(e)) - + self.assertEqual(total_errors, []) class DockerSanityTestJVMCombinedMode(DockerSanityTest): @@ -220,7 +220,7 @@ def run_tests(image, mode, fixtures_dir): test_classes_to_run = [] if mode == "jvm": test_classes_to_run = [DockerSanityTestJVMCombinedMode, DockerSanityTestJVMIsolatedMode] - + loader = unittest.TestLoader() suites_list = [] for test_class in test_classes_to_run: @@ -230,9 +230,9 @@ def run_tests(image, mode, fixtures_dir): cur_directory = os.path.dirname(os.path.realpath(__file__)) outfile = open(f"{cur_directory}/report_{mode}.html", "w") runner = HTMLTestRunner.HTMLTestRunner( - stream=outfile, - title='Test Report', - description='This demonstrates the report output.' - ) + stream=outfile, + title='Test Report', + description='This demonstrates the report output.' + ) result = runner.run(combined_suite) return result.failure_count