From 5e92ed26465056c00f339fac086281f15b54013b Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Fri, 13 Oct 2023 17:55:27 +0530 Subject: [PATCH 01/66] Add JVM Docker image --- docker/jvm/Dockerfile | 89 ++++ .../jvm/__pycache__/constants.cpython-39.pyc | Bin 0 -> 951 bytes docker/jvm/bash-config | 26 + docker/jvm/constants.py | 19 + docker/jvm/docker-compose.yml | 25 + docker/jvm/docker_release.py | 15 + docker/jvm/docker_sanity_test.py | 312 +++++++++++ docker/jvm/fixtures/input.txt | 0 docker/jvm/fixtures/kraft/docker-compose.yml | 100 ++++ docker/jvm/fixtures/native/docker-compose.yml | 68 +++ docker/jvm/fixtures/output.txt | 0 docker/jvm/fixtures/schema.avro | 8 + .../secrets/broker_broker-ssl_cert-file | 17 + .../secrets/broker_broker-ssl_cert-signed | 20 + .../broker_broker-ssl_server.keystore.jks | Bin 0 -> 4750 bytes .../broker_broker-ssl_server.truststore.jks | Bin 0 -> 1238 bytes .../fixtures/secrets/broker_broker_cert-file | 17 + .../secrets/broker_broker_cert-signed | 20 + .../secrets/broker_broker_server.keystore.jks | Bin 0 -> 4750 bytes .../broker_broker_server.truststore.jks | Bin 0 -> 1238 bytes docker/jvm/fixtures/secrets/ca-cert | 20 + docker/jvm/fixtures/secrets/ca-cert.key | 30 ++ docker/jvm/fixtures/secrets/ca-cert.srl | 1 + .../fixtures/secrets/client_python_client.key | 30 ++ .../fixtures/secrets/client_python_client.pem | 20 + .../fixtures/secrets/client_python_client.req | 17 + .../fixtures/secrets/kafka-generate-ssl.sh | 164 ++++++ .../jvm/fixtures/secrets/kafka_keystore_creds | 1 + .../jvm/fixtures/secrets/kafka_ssl_key_creds | 1 + .../fixtures/secrets/kafka_truststore_creds | 1 + docker/jvm/fixtures/sink_connector.json | 11 + docker/jvm/fixtures/source_connector.json | 14 + .../jvm/fixtures/zookeeper/docker-compose.yml | 132 +++++ docker/jvm/include/etc/kafka/docker/configure | 152 ++++++ .../etc/kafka/docker/configureDefaults | 45 ++ docker/jvm/include/etc/kafka/docker/ensure | 35 ++ .../docker/kafka-log4j.properties.template | 11 + .../kafka/docker/kafka-propertiesSpec.json | 25 + .../kafka-tools-log4j.properties.template | 6 + docker/jvm/include/etc/kafka/docker/launch | 30 ++ docker/jvm/include/etc/kafka/docker/run | 50 ++ docker/jvm/requirements.txt | 6 + docker/jvm/ub/go.mod | 15 + docker/jvm/ub/go.sum | 14 + docker/jvm/ub/testResources/sampleFile | 0 docker/jvm/ub/testResources/sampleFile2 | 0 .../jvm/ub/testResources/sampleLog4j.template | 11 + docker/jvm/ub/ub.go | 496 ++++++++++++++++++ docker/jvm/ub/ub_test.go | 446 ++++++++++++++++ 49 files changed, 2520 insertions(+) create mode 100644 docker/jvm/Dockerfile create mode 100644 docker/jvm/__pycache__/constants.cpython-39.pyc create mode 100644 docker/jvm/bash-config create mode 100644 docker/jvm/constants.py create mode 100644 docker/jvm/docker-compose.yml create mode 100644 docker/jvm/docker_release.py create mode 100644 docker/jvm/docker_sanity_test.py create mode 100644 docker/jvm/fixtures/input.txt create mode 100644 docker/jvm/fixtures/kraft/docker-compose.yml create mode 100644 docker/jvm/fixtures/native/docker-compose.yml create mode 100644 docker/jvm/fixtures/output.txt create mode 100644 docker/jvm/fixtures/schema.avro create mode 100644 docker/jvm/fixtures/secrets/broker_broker-ssl_cert-file create mode 100644 docker/jvm/fixtures/secrets/broker_broker-ssl_cert-signed create mode 100644 docker/jvm/fixtures/secrets/broker_broker-ssl_server.keystore.jks create mode 100644 docker/jvm/fixtures/secrets/broker_broker-ssl_server.truststore.jks create mode 100644 docker/jvm/fixtures/secrets/broker_broker_cert-file create mode 100644 docker/jvm/fixtures/secrets/broker_broker_cert-signed create mode 100644 docker/jvm/fixtures/secrets/broker_broker_server.keystore.jks create mode 100644 docker/jvm/fixtures/secrets/broker_broker_server.truststore.jks create mode 100644 docker/jvm/fixtures/secrets/ca-cert create mode 100644 docker/jvm/fixtures/secrets/ca-cert.key create mode 100644 docker/jvm/fixtures/secrets/ca-cert.srl create mode 100644 docker/jvm/fixtures/secrets/client_python_client.key create mode 100644 docker/jvm/fixtures/secrets/client_python_client.pem create mode 100644 docker/jvm/fixtures/secrets/client_python_client.req create mode 100755 docker/jvm/fixtures/secrets/kafka-generate-ssl.sh create mode 100644 docker/jvm/fixtures/secrets/kafka_keystore_creds create mode 100644 docker/jvm/fixtures/secrets/kafka_ssl_key_creds create mode 100644 docker/jvm/fixtures/secrets/kafka_truststore_creds create mode 100644 docker/jvm/fixtures/sink_connector.json create mode 100644 docker/jvm/fixtures/source_connector.json create mode 100644 docker/jvm/fixtures/zookeeper/docker-compose.yml create mode 100755 docker/jvm/include/etc/kafka/docker/configure create mode 100755 docker/jvm/include/etc/kafka/docker/configureDefaults create mode 100755 docker/jvm/include/etc/kafka/docker/ensure create mode 100644 docker/jvm/include/etc/kafka/docker/kafka-log4j.properties.template create mode 100644 docker/jvm/include/etc/kafka/docker/kafka-propertiesSpec.json create mode 100644 docker/jvm/include/etc/kafka/docker/kafka-tools-log4j.properties.template create mode 100755 docker/jvm/include/etc/kafka/docker/launch create mode 100755 docker/jvm/include/etc/kafka/docker/run create mode 100644 docker/jvm/requirements.txt create mode 100644 docker/jvm/ub/go.mod create mode 100644 docker/jvm/ub/go.sum create mode 100755 docker/jvm/ub/testResources/sampleFile create mode 100755 docker/jvm/ub/testResources/sampleFile2 create mode 100644 docker/jvm/ub/testResources/sampleLog4j.template create mode 100644 docker/jvm/ub/ub.go create mode 100644 docker/jvm/ub/ub_test.go diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile new file mode 100644 index 0000000000000..ff54a3853f667 --- /dev/null +++ b/docker/jvm/Dockerfile @@ -0,0 +1,89 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +ARG GOLANG_VERSION=1.21.1 + +FROM golang:${GOLANG_VERSION} AS build-ub +WORKDIR /build +RUN useradd --no-log-init --create-home --shell /bin/bash appuser +COPY --chown=appuser:appuser ub/ ./ +RUN go build -ldflags="-w -s" ./ub.go +USER appuser +RUN go test ./... + + +FROM eclipse-temurin:17-jre + +COPY bash-config /etc/kafka/docker/bash-config + +# exposed ports +EXPOSE 9092 + +USER root + +ARG kafka_url=https://archive.apache.org/dist/kafka/3.5.1/kafka_2.13-3.5.1.tgz +ARG kafka_version=2.13-3.5.1 +ARG vcs_ref=unspecified +ARG build_date=unspecified + +LABEL org.label-schema.name="kafka" \ + org.label-schema.description="Apache Kafka" \ + org.label-schema.build-date="${build_date}" \ + org.label-schema.vcs-url="https://github.com/apache/kafka" \ + org.label-schema.vcs-ref="${vcs_ref}" \ + org.label-schema.version="${kafka_version}" \ + org.label-schema.schema-version="1.0" \ + maintainer="apache" + +ENV KAFKA_URL=$kafka_url + +# allow arg override of required env params +ARG KAFKA_ZOOKEEPER_CONNECT +ENV KAFKA_ZOOKEEPER_CONNECT=${KAFKA_ZOOKEEPER_CONNECT} +ARG KAFKA_ADVERTISED_LISTENERS +ENV KAFKA_ADVERTISED_LISTENERS=${KAFKA_ADVERTISED_LISTENERS} +ARG CLUSTER_ID +ENV CLUSTER_ID=${CLUSTER_ID} + +RUN set -eux ; \ + apt-get update ; \ + apt-get upgrade -y ; \ + apt-get install -y --no-install-recommends curl wget gpg dirmngr gpg-agent; \ + mkdir opt/kafka; \ + wget -nv -O kafka.tgz "$KAFKA_URL"; \ + wget -nv -O kafka.tgz.asc "$KAFKA_URL.asc"; \ + tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \ + wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \ + gpg --import KEYS; \ + gpg --batch --verify kafka.tgz.asc kafka.tgz; \ + mkdir -p /var/lib/kafka/data /etc/kafka/secrets /var/log/kafka /var/lib/zookeeper; \ + mkdir -p /etc/kafka/docker /usr/logs; \ + useradd --no-log-init --create-home --shell /bin/bash appuser; \ + chown appuser:appuser -R /etc/kafka/ /usr/logs /opt/kafka; \ + chown appuser:root -R /etc/kafka /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka /var/lib/zookeeper; \ + chmod -R ug+w /etc/kafka /var/lib/kafka /var/lib/kafka /etc/kafka/secrets /etc/kafka /var/log/kafka /var/lib/zookeeper; \ + rm kafka.tgz; + +COPY --from=build-ub /build/ub /usr/bin +COPY --chown=appuser:appuser include/etc/kafka/docker /etc/kafka/docker + +USER appuser + +VOLUME ["/etc/kafka/secrets", "/var/lib/kafka/data"] + +CMD ["/etc/kafka/docker/run"] diff --git a/docker/jvm/__pycache__/constants.cpython-39.pyc b/docker/jvm/__pycache__/constants.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1fca3f425054fd2ab00a8e3af2eac9a849d6efef GIT binary patch literal 951 zcma)5-EPw`6i&Nt{ab%lKw<+iZjh)DH3$h!6GAJQJEC<><)ni7#>#A$bZwI=b|=~^ zUWZrV5pv5FufP>fXuGWR0!KMI-*)W$B|B=?TpAkr|lmaK}6{pHD z4QnigQ!I_sYzuF(49>7^yv=s-4$I;!%i$c$<2)VRBcD1W*BA2m=+)8d`O{7G8$f-}<5c>A`1M?AzlgmlH`sjOGUoaGSwtz@-4xc}aO7=Px$JMg#6i zGMP!&_Xv$qW5UP(zj?@KIOkkZ#`|Afyr0l>75GCQ5G7tZfyDPL{Uf|+5(Li-%SM+3 zB7}>&+37&tjzQM4+ObpnfXs7<)K<%GqBz*8SHpiQ^iVroRfs@YZyT^<6Wh3eX3q`} z)_dBjBC*Y`p@+3)>jZbWvQ-b9lLd={7CvlZj5l{k1bER6Sllh5B+zQWwFN@j1M|)O?4HTi#(Ftu;sn zziUBFUCY?~ehYen2Ljx1rsn1f*9pcB@Pz None: + subprocess.run(["docker", "stop", self.CONTAINER_NAME]) + + def startCompose(self, filename) -> None: + old_string="image: {$IMAGE}" + new_string=f"image: {self.IMAGE}" + + with open(filename) as f: + s = f.read() + if old_string not in s: + print('"{old_string}" not found in {filename}.'.format(**locals())) + + with open(filename, 'w') as f: + print('Changing "{old_string}" to "{new_string}" in {filename}'.format(**locals())) + s = s.replace(old_string, new_string) + f.write(s) + + subprocess.run(["docker-compose", "-f", filename, "up", "-d"]) + time.sleep(25) + + def destroyCompose(self, filename) -> None: + old_string=f"image: {self.IMAGE}" + new_string="image: {$IMAGE}" + + subprocess.run(["docker-compose", "-f", filename, "down"]) + time.sleep(10) + with open(filename) as f: + s = f.read() + if old_string not in s: + print('"{old_string}" not found in {filename}.'.format(**locals())) + + with open(filename, 'w') as f: + print('Changing "{old_string}" to "{new_string}" in {filename}'.format(**locals())) + s = s.replace(old_string, new_string) + f.write(s) + + def create_topic(self, topic): + kafka_admin = confluent_kafka.admin.AdminClient({"bootstrap.servers": "localhost:9092"}) + new_topic = confluent_kafka.admin.NewTopic(topic, 1, 1) + kafka_admin.create_topics([new_topic,]) + timeout = constants.CLIENT_TIMEOUT + while timeout > 0: + timeout -= 1 + if topic not in kafka_admin.list_topics().topics: + time.sleep(1) + continue + return topic + return None + + def produce_message(self, topic, producer_config, key, value): + producer = Producer(producer_config) + producer.produce(topic, key=key, value=value) + producer.flush() + del producer + + def consume_message(self, topic, consumer_config): + consumer = Consumer(consumer_config) + consumer.subscribe([topic]) + timeout = constants.CLIENT_TIMEOUT + while timeout > 0: + message = consumer.poll(1) + if message is None: + time.sleep(1) + timeout -= 1 + continue + del consumer + return message + raise None + + def schema_registry_flow(self): + print("Running Schema Registry tests") + errors = [] + schema_registry_conf = {'url': constants.SCHEMA_REGISTRY_URL} + schema_registry_client = SchemaRegistryClient(schema_registry_conf) + avro_schema = "" + with open("fixtures/schema.avro") as f: + avro_schema = f.read() + avro_serializer = AvroSerializer(schema_registry_client=schema_registry_client, + schema_str=avro_schema) + producer_config = { + "bootstrap.servers": "localhost:9092", + } + + avro_deserializer = AvroDeserializer(schema_registry_client, avro_schema) + + key = {"key": "key", "value": ""} + value = {"value": "message", "key": ""} + self.produce_message(constants.SCHEMA_REGISTRY_TEST_TOPIC, producer_config, key=avro_serializer(key, SerializationContext(constants.SCHEMA_REGISTRY_TEST_TOPIC, MessageField.KEY)), value=avro_serializer(value, SerializationContext(constants.SCHEMA_REGISTRY_TEST_TOPIC, MessageField.VALUE))) + time.sleep(3) + + consumer_config = { + "bootstrap.servers": "localhost:9092", + "group.id": "test-group", + 'auto.offset.reset': "earliest" + } + + message = self.consume_message(constants.SCHEMA_REGISTRY_TEST_TOPIC, consumer_config) + + try: + self.assertIsNotNone(message) + except AssertionError as e: + errors.append(constants.SCHEMA_REGISTRY_ERROR_PREFIX + str(e)) + return + + deserialized_value = avro_deserializer(message.value(), SerializationContext(message.topic(), MessageField.VALUE)) + deserialized_key = avro_deserializer(message.key(), SerializationContext(message.topic(), MessageField.KEY)) + try: + self.assertEqual(deserialized_key, key) + except AssertionError as e: + errors.append(constants.SCHEMA_REGISTRY_ERROR_PREFIX + str(e)) + try: + self.assertEqual(deserialized_value, value) + except AssertionError as e: + errors.append(constants.SCHEMA_REGISTRY_ERROR_PREFIX + str(e)) + print("Errors in Schema Registry Test Flow:-", errors) + return errors + + def connect_flow(self): + print("Running Connect tests") + errors = [] + try: + self.assertEqual(self.create_topic(constants.CONNECT_TEST_TOPIC), constants.CONNECT_TEST_TOPIC) + except AssertionError as e: + errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) + return errors + subprocess.run(["curl", "-X", "POST", "-H", "Content-Type:application/json", "--data", constants.CONNECT_SOURCE_CONNECTOR_CONFIG, constants.CONNECT_URL]) + consumer_config = { + "bootstrap.servers": "localhost:9092", + "group.id": "test-group", + 'auto.offset.reset': "earliest" + } + message = self.consume_message(constants.CONNECT_TEST_TOPIC, consumer_config) + try: + self.assertIsNotNone(message) + except AssertionError as e: + errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) + return errors + try: + self.assertIn('User', message.key().decode('ascii')) + except AssertionError as e: + errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) + try: + self.assertIsNotNone(message.value()) + except AssertionError as e: + errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) + print("Errors in Connect Test Flow:-", errors) + return errors + + def ssl_flow(self): + print("Running SSL flow tests") + errors = [] + producer_config = {"bootstrap.servers": "localhost:9093", + "security.protocol": "SSL", + "ssl.ca.location": constants.SSL_CA_LOCATION, + "ssl.certificate.location": constants.SSL_CERTIFICATE_LOCATION, + "ssl.key.location": constants.SSL_KEY_LOCATION, + "ssl.endpoint.identification.algorithm": "none", + "ssl.key.password": constants.SSL_KEY_PASSWORD, + 'client.id': socket.gethostname() + '2'} + + self.produce_message(constants.SSL_TOPIC, producer_config, "key", "message") + + consumer_config = { + "bootstrap.servers": "localhost:9093", + "group.id": "test-group-5", + 'auto.offset.reset': "earliest", + "security.protocol": "SSL", + "ssl.ca.location": constants.SSL_CA_LOCATION, + "ssl.certificate.location": constants.SSL_CERTIFICATE_LOCATION, + "ssl.key.location": constants.SSL_KEY_LOCATION, + "ssl.endpoint.identification.algorithm": "none", + "ssl.key.password": constants.SSL_KEY_PASSWORD + } + message = self.consume_message(constants.SSL_TOPIC, consumer_config) + try: + self.assertIsNotNone(message) + except AssertionError as e: + errors.append(constants.SSL_ERROR_PREFIX + str(e)) + try: + self.assertEqual(message.key(), b'key') + except AssertionError as e: + errors.append(constants.SSL_ERROR_PREFIX + str(e)) + try: + self.assertEqual(message.value(), b'message') + except AssertionError as e: + errors.append(constants.SSL_ERROR_PREFIX + str(e)) + print("Errors in SSL Flow:-", errors) + return errors + + def broker_restart_flow(self): + print("Running broker restart tests") + errors = [] + try: + self.assertEqual(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC), constants.BROKER_RESTART_TEST_TOPIC) + except AssertionError as e: + errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + return errors + + producer_config = {"bootstrap.servers": "localhost:9092", 'client.id': socket.gethostname()} + self.produce_message(constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") + + print("Stopping Image") + self.stopImage() + time.sleep(15) + + print("Resuming Image") + self.resumeImage() + time.sleep(15) + consumer_config = {"bootstrap.servers": "localhost:9092", 'group.id': 'test-group-1', 'auto.offset.reset': 'smallest'} + message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) + try: + self.assertIsNotNone(message) + except AssertionError as e: + errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + return errors + try: + self.assertEqual(message.key(), b'key') + except AssertionError as e: + errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + try: + self.assertEqual(message.value(), b'message') + except AssertionError as e: + errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + print("Errors in Broker Restart Flow:-", errors) + return errors + + def execute(self): + total_errors = [] + try: + total_errors.extend(self.schema_registry_flow()) + except Exception as e: + print("Schema registry error") + total_errors.append(str(e)) + try: + total_errors.extend(self.connect_flow()) + except Exception as e: + print("Connect flow error") + total_errors.append(str(e)) + try: + total_errors.extend(self.ssl_flow()) + except Exception as e: + print("SSL flow error") + total_errors.append(str(e)) + try: + total_errors.extend(self.broker_restart_flow()) + except Exception as e: + print("Broker restart flow error") + total_errors.append(str(e)) + + self.assertEqual(total_errors, []) + +class DockerSanityTestKraftMode(DockerSanityTestCommon): + def setUp(self) -> None: + self.startCompose("fixtures/kraft/docker-compose.yml") + def tearDown(self) -> None: + self.destroyCompose("fixtures/kraft/docker-compose.yml") + def test_bed(self): + self.execute() + +class DockerSanityTestZookeeper(DockerSanityTestCommon): + def setUp(self) -> None: + self.startCompose("fixtures/zookeeper/docker-compose.yml") + def tearDown(self) -> None: + self.destroyCompose("fixtures/zookeeper/docker-compose.yml") + def test_bed(self): + self.execute() + +class DockerSanityTestNative(DockerSanityTestCommon): + def setUp(self) -> None: + self.startCompose("fixtures/native/docker-compose.yml") + def tearDown(self) -> None: + self.destroyCompose("fixtures/native/docker-compose.yml") + def test_bed(self): + self.execute() + +if __name__ == "__main__": + if len(sys.argv) > 0: + DockerSanityTestCommon.IMAGE = sys.argv.pop() + test_classes_to_run = [DockerSanityTestKraftMode, DockerSanityTestZookeeper, DockerSanityTestNative] + loader = unittest.TestLoader() + suites_list = [] + for test_class in test_classes_to_run: + suite = loader.loadTestsFromTestCase(test_class) + suites_list.append(suite) + big_suite = unittest.TestSuite(suites_list) + outfile = open("report.html", "w") + runner = HTMLTestRunner.HTMLTestRunner( + stream=outfile, + title='Test Report', + description='This demonstrates the report output.' + ) + runner.run(big_suite) \ No newline at end of file diff --git a/docker/jvm/fixtures/input.txt b/docker/jvm/fixtures/input.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/docker/jvm/fixtures/kraft/docker-compose.yml b/docker/jvm/fixtures/kraft/docker-compose.yml new file mode 100644 index 0000000000000..704280c290f0d --- /dev/null +++ b/docker/jvm/fixtures/kraft/docker-compose.yml @@ -0,0 +1,100 @@ +--- +version: '2' +services: + + broker: + image: {$IMAGE} + hostname: broker + container_name: broker + ports: + - "9092:9092" + environment: + KAFKA_NODE_ID: 1 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' + KAFKA_LISTENERS: 'PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + + broker-ssl: + image: {$IMAGE} + hostname: broker-ssl + container_name: broker-ssl + ports: + - "9093:9093" + volumes: + - ../secrets:/etc/kafka/secrets + environment: + KAFKA_NODE_ID: 2 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,SSL:SSL,SSL-INT:SSL,BROKER:PLAINTEXT,CONTROLLER:PLAINTEXT" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:19092,SSL://localhost:19093,SSL-INT://localhost:9093,BROKER://localhost:9092" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '2@broker-ssl:29093' + KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:19093,SSL-INT://0.0.0.0:9093,BROKER://0.0.0.0:9092,CONTROLLER://broker-ssl:29093" + KAFKA_INTER_BROKER_LISTENER_NAME: "BROKER" + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + KAFKA_SSL_KEYSTORE_FILENAME: "broker_broker-ssl_server.keystore.jks" + KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" + KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" + KAFKA_SSL_TRUSTSTORE_FILENAME: "broker_broker-ssl_server.truststore.jks" + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" + KAFKA_SSL_CLIENT_AUTH: "required" + KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + + schema-registry: + image: confluentinc/cp-schema-registry:latest + hostname: schema-registry + container_name: schema-registry + depends_on: + - broker + ports: + - "8081:8081" + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' + SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 + + connect: + image: cnfldemos/cp-server-connect-datagen:0.6.2-7.5.0 + hostname: connect + container_name: connect + depends_on: + - broker + - schema-registry + ports: + - "8083:8083" + environment: + CONNECT_BOOTSTRAP_SERVERS: broker:29092 + CONNECT_REST_ADVERTISED_HOST_NAME: connect + CONNECT_GROUP_ID: compose-connect-group + CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 + CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter + CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 + # CLASSPATH required due to CC-2422 + CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.4.1.jar + CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" + CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" + CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" + CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR diff --git a/docker/jvm/fixtures/native/docker-compose.yml b/docker/jvm/fixtures/native/docker-compose.yml new file mode 100644 index 0000000000000..0d4a874ee712d --- /dev/null +++ b/docker/jvm/fixtures/native/docker-compose.yml @@ -0,0 +1,68 @@ +--- +version: '2' +services: + + broker: + image: kafka-poc:latest + hostname: broker + container_name: broker + ports: + - "9092:9092" + environment: + KAFKA_NODE_ID: 1 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' + KAFKA_LISTENERS: 'PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + + schema-registry: + image: confluentinc/cp-schema-registry:latest + hostname: schema-registry + container_name: schema-registry + depends_on: + - broker + ports: + - "8081:8081" + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' + SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 + + connect: + image: cnfldemos/cp-server-connect-datagen:0.6.2-7.5.0 + hostname: connect + container_name: connect + depends_on: + - broker + - schema-registry + ports: + - "8083:8083" + environment: + CONNECT_BOOTSTRAP_SERVERS: broker:29092 + CONNECT_REST_ADVERTISED_HOST_NAME: connect + CONNECT_GROUP_ID: compose-connect-group + CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 + CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter + CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 + # CLASSPATH required due to CC-2422 + CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.4.1.jar + CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" + CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" + CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" + CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR diff --git a/docker/jvm/fixtures/output.txt b/docker/jvm/fixtures/output.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/docker/jvm/fixtures/schema.avro b/docker/jvm/fixtures/schema.avro new file mode 100644 index 0000000000000..d85b0e1204773 --- /dev/null +++ b/docker/jvm/fixtures/schema.avro @@ -0,0 +1,8 @@ +{ + "type": "record", + "name": "Message", + "fields": [ + {"name": "key", "type": "string"}, + {"name": "value", "type": "string"} + ] +} \ No newline at end of file diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-file b/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-file new file mode 100644 index 0000000000000..3a0c3c9ee9f72 --- /dev/null +++ b/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-file @@ -0,0 +1,17 @@ +-----BEGIN NEW CERTIFICATE REQUEST----- +MIICyzCCAbMCAQAwVjELMAkGA1UEBhMCTk4xCzAJBgNVBAgTAk5OMQswCQYDVQQH +EwJOTjELMAkGA1UEChMCTk4xCzAJBgNVBAsTAk5OMRMwEQYDVQQDEwpicm9rZXIt +c3NsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA8sIlIx37zD3Tz9eI ++RN1dlbWoFI94Tlj+1ReO62HrJZIO+UGDl9wR7WFb4lJWM7qol6RDXG/7aWXKyLK +1w9XF8QhRaKx+0gnhZCaeCnQ3Ne5VtK8a64tg7ZgVSzWHJDOnIGeE7sAR15v7w8z +tinteU+0wLu6lQXU2d0MHGY4CuBDp3VwtGNVoxZ86wxDE3fSTBwS+hjBrW+e7ajr +PMZ8Mp4fpERdblrXFZNyUnycMOhchAoDMdqDV2CgRv6z5I5vDEknlOSdiOhHHnI+ +55RCwD98uIs4C+ZNdUD91W2baXaYMXdUF7aqKW3P1uTXx+xi2VoWWTjB8cCN4T2r +FnPYxwIDAQABoDAwLgYJKoZIhvcNAQkOMSEwHzAdBgNVHQ4EFgQUCe7i0TB0oEfd +DmuM4WWcWgCxV+8wDQYJKoZIhvcNAQELBQADggEBAHFcgQDrj7F0Oi3CannGvOB6 +XLTf6S5+f7fd9aIkq+cRIVV7aIacu8xXmTKyLgbuJMN/AhPqzZwt79jnIm54/mWh +mTBM3B9BRQT4GreJ2b1xgb543JB85LyCU2eMxx5UxOvUV/7VMxee2mRcWQUPw6Jo +0YCJqeNFZwsg80MzuQMOPA6wmGPNvgJ8LmcwMMfUnnaUlnvYL1cdw9n79Ddkuvm+ +8I63wrws9ejuO45i6o4uIL7sy9n2egwZ85oz/8hboUQgaOs+V8A2LE8xLnoLUHAV +p5pvjlB3alfhxRJEhKf4W16i0CXT3tMBl/v1o9o7NA/CllfZeb0ElboBfZA2GpI= +-----END NEW CERTIFICATE REQUEST----- diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-signed b/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-signed new file mode 100644 index 0000000000000..0a5ccf415e8f6 --- /dev/null +++ b/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-signed @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQTCCAikCCQDO815g0gGg1DANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJO +TjELMAkGA1UECAwCTk4xCzAJBgNVBAcMAk5OMQswCQYDVQQKDAJOTjELMAkGA1UE +CwwCTk4xCjAIBgNVBAMMAS8xHjAcBgkqhkiG9w0BCQEWD3ZlZGFydGhzaGFybWFA +LzAgFw0yMzEwMTIxNzM2MzBaGA8yMDUxMDIyNzE3MzYzMFowVjELMAkGA1UEBhMC +Tk4xCzAJBgNVBAgTAk5OMQswCQYDVQQHEwJOTjELMAkGA1UEChMCTk4xCzAJBgNV +BAsTAk5OMRMwEQYDVQQDEwpicm9rZXItc3NsMIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEA8sIlIx37zD3Tz9eI+RN1dlbWoFI94Tlj+1ReO62HrJZIO+UG +Dl9wR7WFb4lJWM7qol6RDXG/7aWXKyLK1w9XF8QhRaKx+0gnhZCaeCnQ3Ne5VtK8 +a64tg7ZgVSzWHJDOnIGeE7sAR15v7w8ztinteU+0wLu6lQXU2d0MHGY4CuBDp3Vw +tGNVoxZ86wxDE3fSTBwS+hjBrW+e7ajrPMZ8Mp4fpERdblrXFZNyUnycMOhchAoD +MdqDV2CgRv6z5I5vDEknlOSdiOhHHnI+55RCwD98uIs4C+ZNdUD91W2baXaYMXdU +F7aqKW3P1uTXx+xi2VoWWTjB8cCN4T2rFnPYxwIDAQABMA0GCSqGSIb3DQEBCwUA +A4IBAQCYERHx3CzqOixzxtWZSugqRFahFdxWSFiuTTIv/3JhSpLjiMZGQt2YqX85 +YLnSvW0luChw8IW5S0Mtkn/Mgpnt9hzPsr1vY1aQ5By8PUXIqCNMmnIY8yC+HYNs +7DzTbU5Lin/YwRzMLnqq/9zvh+YBVdPhBrFpSXRjEkjqTXfs4fzNm8m1MsJifz3f +Q4t0iPOPjrbXrq1CQ+MstcpMwTi3eHHxcvyNHHlFLs10GH74NIymYKYwDG8fsatl +jScfxkn2rLuMFWuo42QqdHsPgS7QyTrZjvCM+5w1aUo6a35NPEcOqFD311/PJh/Y +vlSocIMIFklDRWFVh1D946t2Z42/ +-----END CERTIFICATE----- diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_server.keystore.jks b/docker/jvm/fixtures/secrets/broker_broker-ssl_server.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..a823cf3b7ca0fd48292238f98cf99d893e6aa067 GIT binary patch literal 4750 zcma)AWmFUlv)*00yBnly7o@uz=_Le~mX4*HB^8j65Rhg;KoD3`LIeaU>5vXdK|s2% z_uPBl@7`bE_hZg8^E@;2^Eqb*j39jl0-}Ktq)#z%c;eLKE(w5Gz#;@GJ34}t$W+{8w(|JrC~R7cqg@3nOY(ktEN*K!K_n}PyfsR9A` z`;tZ#bH?Bq+(hL+Bay47(T0bCeZrZJwti&jUk;;w>`&UO4i2A^ANHX(2LEgckmZid zzU&v0aZz47Z2IhPveMZ_|4=*mCifI4DDg_tRO07t3VyOe5`TH;&QZ0gMUEkR>7cG~ zm4KeP%#s{kWU2J4MQof9d=hf<=Qjnm9NNj2Aoc}|lm2)e0CyfV+GHr8zBji;m#XKrTC7*4EkH0>#s~ zXilCT@C04lALcg`q4qAzFWxpZ-j*xIoxwEkDPTh9nF)e=a;wqJrZWri^*2j;$b$<5 zmHW^x;}r2wYeK4V(cNABGQ-SK*LL6OZCw>p$)Z#+T4|X4i zP210c(536xa;P+$NYRA6U~_T-BlQ*!m2{XBCvm5~8i|{h9x?4(Y@+Rq*@2i=r0EV8 z&RQM`S@umi9f~4O%lolqn=HNqQo9@b6GM~%4esT=rMQO|TQc@+NrDHPPyg5w-$w&{ zx3Jr3z6Lt(rFS%SsJzVN$Zhv#sy6f(_ z0ml+K(U&jP48n%I|eF;yOYCVS#$o}&yWjerzEx?&6` zag~U@6-S#CdDm7FY#t09nbhinARAXmvbwj%8?cvAJaJbs_GJ2>bS3i__J^XLof|U2 zo5b~X3Z%~jE<(5$fw=2Gc1JXVP-}WbLwSwACYe!@|JD>PIZ%t{aP2q>EsyL9Zbp9 z3}?m&=G5lawL3^rlfso-1tNT&4{_jMK@aNEtB8^C150h*swD#=ogazuK^9xa3K9us$K?D|fk$QIii~Jh+`5$kd0m-c1paNhplCX2y&S*`bUAj~?}s zu(+D4SQ$uF{e+-%t{g5(JUHI2+LfBP3r@&JH5Hv^EkWVXZhJ4@o zRuKMWtBtUM+tH{Sdl@F0YO1*!B{Vi>`Qu}YX4L|%Ae+=E`}FVmK=(&??qkI5CLRL<6kgoL&+cV5uu)sLrB2u_1WRBIbAJKMaW#aQ~8q_S)*#E zp>JddGevqstn6-pPFIS>J6~6Q+zqG)gQp_Jjl!F+tRGuDW|;O`y_$UsGX3>oaf8;4 zopRD&y2gs_L-#Ot;g5XK)ICLFeXLv&u#SM&bo4uuS?O4;HPp`jnXm@wN88zsp zo+1A3;!7fD;<;#|KT$3Acfh_4TChyn-dSGRrdg5yw`W1Gr)th!_bslWGu0%NfzN=I!5&ip(iAv4c?H{b0v-v`vJCoRHCi-yNToT+HTa`Hup!ADXzBt-OS zmc5IYxG99VxYXt<-2~ea$;=>NnJKIX&9SIsfP6b{L@NZ%ObcELm_qgwZVD^(goh}d zq@lIkyfT~x6p4me&stEUF9#;Jy7*7IV*Mx$~di= z6!p6aZ#EWCblm_*y5)`hr^QFw%1{Th0~JdBh-dSwS*(;Yw)P>qba3H!%o(+U=ZQ2i zl5jWm*X?mt`#W)+qj~~G1)qJC&YGWxQx@1bkV|v(l&~QWtqeAzQsiUvd3@hSYWOlI zSsVxw>3y!zWwNx%tn58aswz&Aor$~l4YS){LGZcwF3%jd%#VbL$gz;9d3S%KrDwH( zy*l}U7u?HkBURdE_T*#=lI6x?~_A^q1UPN{e6y_4H1 zTXLf96`MU4G!BIN18kq!6um`6Y#SwnEDr8Ud70HQpq7k}NFDFgto*6HuAuz+rXT0& z?D)}E_S$#c^vftIdn?hz7vLlQ%k=PKE7ees4+-Jv0Y5vl$MaWWwiGdNCnWLWE_o=4+i#Li9pK|}n)Unxq|lDe zrmb7$t}6*Hkww@=jgnCwdp@9GkGA44XjAvShQW#_2`$iMeuc-rmpM}!&?1B3W~xn=N~A?3!czwxm>T_cl~(e zB4~Ge{3l2Cou$4&uSFBTBeY`jII6O&iWS((2ycJn3n1!YxEU}A*xTRy?Qr018CrmE zIHjW(O7xqquuah0=|HZU)4voOY;yX0vT2bssUIO1Zf+tb2xCTxE#CK-`<&;IeVgtIoRSs*3kg`^J}E{+Yu(NWSDexvQMJr; ztFSp-9>d5cMs8FkwKGqp|LWCTtgJzlJD{r>^KfM5Pez5)(7P774h_hoJ=eE}>J`%n z(T5+51)tP@9+#L{@EQNYwIz92@)4-v<86U0d{OZGel01gEfBM5EcoVx16HvLp4X}tiScZKOF6_@%o?ADd@BN%PCo4HLvuN%4vL;McneKqY2olC)U7BM6ldn%(f-3)qP$10#&AH_IADQ8vTmSYK&`mu=zdGPPtEq-|^HEa}L zc*d)oCJyb)b+eP4R^1KW%H)RGKUtS5$UQ27FD98pxf6oOPi^D7KlD=SE%ez85+JC~ z;+OSeI;3pLEdHka zj?8Vt1ax-r0wj0A=|d4`ct{)2QJk^kv3K-Hv6ytYIN+tV-gnc_-q3CILOEZbLrh3B z;=^&!vXeFYBUh)Rk*tdxt%Gap#Va3umEbXEfZkP0p7KJWhXH_K%l>Wf7jr}EDFPC_ zF^mrjb4fK#xMw|hi227X&F$iC8A{O<_xKsn>i{^|Qn=LJsA|GG?@a17%Co-uH-RdY zVkBSNR?SzXT|`LerP2Z;qmaHRgYhj5sVH{3cWlJ#SJY#Zr4pA=p?gKXE%X@A| zXm+C+>#wAd_b1fR&N)aF$~X7<1C49)mhlOW1q}3!Qzz!G3Ov%Q;yYH&pSK#D+z)I# zm*`Ur+v;cAGP3%DgIN7TU(2H<8xX{R&sd|Jgz%me36s0GDHDrg6~-2ANTHr*>imT= zlKe`p4|8ZNyG-F9AXzL zq72pYB#=&68CBpdCveDWQSCUd73NsVTBEW6>wqC(f`5KAARq<+o!Rx-vx&AZ7mdTK zECEq&ubrRCwqIMx?+}kGE3ToLB^#rYd(20o+(x`q9_{K5ti_RK*Jja4|9w*Y7a0`6 A#Q*>R literal 0 HcmV?d00001 diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_server.truststore.jks b/docker/jvm/fixtures/secrets/broker_broker-ssl_server.truststore.jks new file mode 100644 index 0000000000000000000000000000000000000000..b526dd149726d5b5f14d458b9855af077c197cd2 GIT binary patch literal 1238 zcmV;{1S$J4f&|h60Ru3C1bhYwDuzgg_YDCD0ic2eZ3Kb@X)uBWWiWySVFn2*hDe6@ z4FLxRpn?QaFoFb50s#Opf&@nf2`Yw2hW8Bt2LUi<1_>&LNQU+thDZTr0|Wso1Q6)P*k=lo4{ribAXc|spM-#d1JG88u`XIkMH^{>lLExi+PFW^ zB?pbblmwhZl95UB&R59%mp?wOpl^w#7q&J%nb0|CPS8hqe$+89sZf3p`|@?MQggrY z6-TNTHFq}tQ4)^G7zuG;HGzvqC@!I48tfp4qd)UU(Kj6NScU}N?a>JNhW1c+A13Kl zQ;Gkn-)StuoBuk`kEidVAYIfab!|_{$!9`==yeKm56pr&aUOF59w1g@6CtZ+DLoPF zue;~{vjWt-o$4Z?bJHvPRbqi_>=-n=5#sPVYd%jPe$41HQsXhU+Sz21vR+BYQO*og z{6yh#t00ubb!pV@S`;iS(~urt!&#{-R) zY62Zj_OkN8`>Va1GLM*izh!bI!4gaCL_1FLPyQ*?6v9jUuFUufcLBhlS}(A@IlKKu zxMC+TOY`5ZsY9tu+anTqy{8^YXu^z+V$7P}i;;7K^m9cb@c>Ck;2sj0qlWW5CJTMM z)X3$Z@7WeGAxL$Q2f6>!EJlv>3mN6Af6F_-2QmB=+FecQu=LTp8ZHfZKUQWYn&MqLMb6rJ=V3r9OK=UbbQmJ{-3l~6Cid$41Y%|!$t z>wLkiAQWfy;H)DG$1dv?2aHy|pZDjDvTa-LS`@X?N0$H7Ic%{;1PWaE^gcvROeZtW zg#_N~lac)VV99}H)q52!ikbFeIFi^-g`H$?$5hFDX6FYs(*p+)ZPdjrK^#l>yt8%| zP@5^P#;gUlV-@??hv}0MAKcMiwYbt`1+OL^`n`@aJZHb1n> zCeuxysJaF7)iW};l{)45V;DO9Wx(`wo;=YX!8E~;v4pWK4GjEI-@(LBYo_2SD9Nb7 z4BU-#b_sGTJarFflL<1_@w>NC9O71OfpC00bb=VYHg|wC=Y_ z>K$^fKz=nohNEb?2Wq)Xg7;r0_z`ae6bHMGdOoJ>5YY_$M+66>H8dkc51?leY6p)gJl@gF{L6Me@ z>pl0L_q+Gk_x+gj%skJ`{Cv)tfuhJGfmqm36!{|@e7;cS&~sueT&x@vITr{;&i0pP zf}+3;|4#xJf>7YxzjWr`NW~%i-zg#jEMN`_Ec=&~g4+Bu0VakTL&g795<&Svly0Zr zvpr2A6+;g#xs5AXJKP1FP(UEh10WVFln4j!e>MW~!2l=|4*u&8WP>$fB?tu?m+m8Nn=89iSLpKUz&~ij?5t76%_(NzbzwB%YQVh zwdy;plk(XZ3hlNz4nt;Ffg&GXnr+t^nE0v7Y#f=U+1}1)1q1}CzM>PRuk#fiBV{e# zhUag3xxA%cEFH`0@Hraj==T0WH_J)mNF*;%5$j4z*(`c}UrdgWaJ?&0DcC7#0P!HH zWRJ)3PB9dGi-Z{-HuMCfXDZ>;p2P{G?-jj*y^ZcszuU^ge=(1Ifsh(6y-XFPqnO9) z(@8fwqwgu-S6n|EZYaYl#{8oG`07aXeD<@~S=^cR4kdxWO4LK?$HgR`W~7Ez5tt<9 zM12j%2o3Mo7RL%t^!8f|3x_fUqpN}|9X_S696t4`o|;rXXv+++RH`y4q0j>}$XPPq zFv-rLcGg+fTN_UeONYgD%g<8Rt~_MY1CBZ^hIYO7-iZrQNEgdGr^@tDslHm~>h}y+ zS#o3vY(cML=JaS5)Uv4BoL05WMKe1>;R{@biNRR75h*7C>FX>$c!^6ma2&~hWm_$( z$q+Th#(1f`+2bpWEO=b7ZZ%9O=ppW8KCwn1!Pm#$5^d-SN{}tHu8r%%V;r!^U#*~p zG=0nP6*P4$)vJgHj*jgJldsVEx;sAne0_B)gMQK&ZUx{QTvhV=VC`grNT4kkdPesy z_GJannJ59tL2$m=#5@F7!dxQTrln60DJZz(j7J5ZY6ko5U?KVHRim*^+KUpj_5 zDf@B#W@M1~8Ct$jjp!zqXz;*Z?$34_$BLcoH+9`L+et>9G?mMl_=Mhc_s&-GeBeio zO)jO!y#ZaZi7lENy=mQt>7TEy37$my%$)nSaY0aQSj6b|6`^iuW7xb|N`nai#*urn z*PX#^zq4meWwd1>WD|^gom7RuB$EXG&h2G%waS30{O%5_U3+x>ZdCJdFuOllU%gd| zsG39SY%rDy5w(YW?wnwSRGR$B<8cJH17Q}X-&fOq&MS9R$9cisk0+3k)*T&JUs0FY zhlgA^<+-;OYh2+E=^$l(swe&Y&uuSxs`nkH>!eijTJ?|l_n|Bvc5gBKHDIjqWT&_= zTRtI22#!&df#WOPCx@90{aFl4Xqq?c2X(ixQNXRa5t5tb_Y~JC1^3w5^Zp{!#b}Cc z`2-s-FjEcDj6gFbgl;vvtb5d4z*(1}IIHY#fPek_+;w4!Ws<(EPg^Du=dYUaGos=`nap8z#sUQs z-vdMzFGC2P<~ZYsarx&SI1oIilKgTLA8FEi32xXt<~H3I!~3Pw$cH1Iry&yw z4t{bqEye?yeKpfLy<#k-`i*v*$rh@TD6c&9rm>#(j%WJ*q4_txsjP1M6=`(dB?b0} zS{vANDBC}FiBAcmWF`k70L}nQfcamv`UJUij?gZWaLOV1S z;r9JtZn4=%_vw9R#!W5sfhfZ9G77)pGTWI?eO#qj0#WJ%f|8Hv9X+_{Dc`D$zlb6+ zu@a6oiI6C$Nd)p=LK1tKaK)dxw(340C!{AFRG@<(u~$z3&P2WxMg?k;l&~?=d?U$V zUwFyplyZ9RqwlLL8#M!G{ey_K3!lhycg4n$4GpYBeiECdRggWkKQ8xjRf;PQG`*z_ zwkR$2aSn$P2YB*a;Lt9-rXx5RPDUuhiJtnpX=|-0A`h7`HSEU|Txm<9J4^QBf^&xY za-w?&j%KLrPkSkIQg{eO-CB^R+q5#b=hF!p>GpWyb3!}*Q=Vh+pB8wpxf7{Wu}W_C z@m45Gybfy932i_7$)nQEd|rSa>DY8qrfn<8N5qoqCwOM%YdrvqCE1>CU}22E#nq2| zoW=8qhu56F-efSeNE2*v^B2FdSicTah&U?uTAZkp`N<7@iV+wH@=uAJAEE1w)!bU0i zelT!Xz2LsP5LXL^o2(@`H)*A*@ny=5thuko7d5ZxWLhLTl!V_Dzf9H~P7n}9ZyaG{ zQzyjqlqmCs{xOkSE6mX6fJs+6OL>2RqE9ea@ny=LtDcxnkqj6RHg3m7coY${z9ypvVN^eu_(b=_R6-cq$BA7;PbmY7$EISZ{25J2$^_3J?F!4neEZNLsgR^YjFV4@hSzP8Ka z8#rKBHkPWsd8xJ_4a0x8O~>(P%)=;GWh`${dylKAxV0I=-4GpW$*glhKYgYs<^AkJ z&nD{&%C3`OAX5I<6K6onx#;E_iHB_iJRjHdn?vQRR?2!FFG>q67S(jUlZ=pQ&2{h} zp00_GHl@C;Y+YCZ&_1%BGBi4Nk<3;$7#c^Ue&KXY*shLoxCT z!u1-o65PoQIA>smfmX@Y)I%?m%_+I?5L9V00O9h;-quPg)3h+p78 zV)fje#nw_ts3a|1Dj38B7+LYb7_r}kc4F^H7(J~V3CT#RIGuhvCZsX8nY^L2YJNZ# zN%VXf%FRWrYn|E;_5O3OuvsIdP|P^ehfI2r-pS7LRr`QMvqJ!sY_4hBh&_=O)Ke6^ zR<#?l=pmsn-<8KkU}#;({l0-65fyh3Z`+#UpqQB*CE=r$(v5d1hP)ty4T_qfl>uLv zs$(jJEY{MZsvL$9NaDsuPKU#nr=wB`a}57I`a^)5`bO|TgmT0UN@eSaTg)`nvG>Va zgd-9u8Q^>QX0?QOoQJYVt_t?d!8a@yu{_)JgZC#xAC0{%+qmw=sjS=>KRY`sOJOnM`Z=zwM2vrm8u(#)X zFQT71^M|CCP*x)WyLqPs>;AhG|ETsWD{x~iEW3>`YmEB z2Nm3oG8XZN+m1JGf^35JhDxdL>}hH)a+KcbGS_GLGT@`X`LKj7^`JYgv_k~flfK}e zI8`b>DEri({^2m2RzR{{x2-)Vpm8mz1wAg0g#)iCT*f6TI=EGKT1ak7SN2@}u3p(0 zgVx|>T{9t7zk47I+cuOIw06Y@Xob*A+J3_4C@JoryeUAObGx@Sa0`jrkPg4B2FJ3l zAjZ@`RXIs=cp6(sKJoVN8)%rR2qQ>6EpC##m|$K8ZvDBy@eQ1(cOna!wjcf93|6@b@TisdL^kpAU&r?6!s zay_TZX;JH?z?!jaZ06hDn&%Ue@U0XELisIxhH^5SYqJl8w$~<*X%&nMZ8PXf&BeWoB_|&m&B$+&k#Jc2akQp^Q2My+HP-v?ro~s2 zEl*}inG}~qzW$n>VPi-axYbrFoWj7RPxgLLrk<>(Sa(i4Ui5IC1SJIK{p1m13BR2# z$2(wd&*C6)-sPYOC5>qs@f~dPiR9>yfgUpW&D^CgC!&cvQ6@zxk`EX3$z#ta_Qy@OjEUf!g^w_!gS!n79dosCH%gGeE%PA#`PzJRXWv}|SuPcW)Asq2wIumCa z_;z^M^;_cZqOc9#*Jf?GaFgVV!%eD+lb9_w2)?J9Xqp5fmNjdPxM@P)iG^lVZ?kjO zqo*gA4dFHR0%5(z1&!3}Y?evg$CTzcO z%Gz95Ig@UG1o@*@#NmXf!9HQH0KgMjH z;FC~FRbGg8pBY%u=3`k%&*xL;AvLL5qFLK3+)~l3Jq$Gb3VWOsb5*) znG`sC+SiP;$18c~D2^*;&1a0V$n?B@4&slq*#2VUB7}rfPuSoV`)Ba#z;0pzwgpZ^ zFtmS3DJfC#@uD~;-z3~L1`m63+9>()@bhitI%Tvzf#Y>maXjkeL6tD}q(HHN%*UXp z*{#ek4KA)_E?4r!s{v<*&rj%|Fk(MsAT}(;GN9MleA6Xv!1d6mrXz3NxuP)L6b=Du z?x5M@ao zn4;J`%P-xiY-`WXde5@&zGRcsoo0oz6@?)Yq3+?{ot_pq@^aM??=tp47moNoNQzC- zBcXeH$OI`k$lC93O<8#8dDQ735PDy3PDUN-DyN(gb?3+J#f4R3YQ}$&P}nA#T$?FJ zR=OuBbqJrx;hZkic$fwt(@#{Hs><#-2YhS@r89hKDf3+h)$-P=yEMZ>K#G8kzvt&w z{O&4Kys1{J`Y^6jsFW{u{cjG{Cdl>0%E&950Qopr-{ec6z7=Rx!($!#731jI37;f& z90+=yvy~oh^D%kineRO&LNQUCc5!rUdCMV@CO0{ClCSwATSID2r7n1hW8Bu2?YQ! z9R>+thDZTr0|Wso1Q5jYAgfh1*n&#?ebGsJHYtFD1JGFzj_}5R)2_-ag-=_~ny9w}M8;}F zDKA=%k;NKXcVz33DvTJqV2-?frBKif~zW%qBuAh6lp?@njnlUH;-<%m@7iIV63WOQ8hiY zZ0#38umsN<)9g#{C(4<@`8M^hLu6q)QIoR{FrtvATf+#49?+MB|Lwmu!$nvjMyY-O zk@sgxe1d&w`Xi-O=77DD)!LT9{svF#!e`tue(yujIOBOHJYrxbEJsQ?D}_9FlI0iIeIJOSKK;Twz2W8 zg5S85RYijnsXD?@af8pubU+qrw+anAL~axgG*T$8RaCYFe^} z%o)5U$KOAGzYU@`QbY)*TCCIvw6-qb9bA`Ky))GpTgLi!G$bcLe?B+*#YEID~)LQ4I{G;o7 zs}A$G>35i-vq1j_why`uBe$NsrxA3JoZZnJ(CmE4qHc~6cnGUeJ zJfYcIc!#@{pT;)dzIp(F?mXWEC8sjoGb>h^s63!~lNkDWtWFg%C(w}Rk|BW@-pbTN zI9E~88JAM|@2N$p3GY;2whx-8Y!!E0q_b${Y-B;}=yAKF^&$#L;oY7dg9Js-jrx#) zDxUBOGN_{BkV1JYZIHpk@yS&ooA^_*k&MKF=r!=Tv-wCn0!$FupntmE;oN4cMe2L- z?hnOTGrapoa+p(Yhb`<-kR=^*4}^HUA;z}hT;BDVcnqlK*)%{B%E+9r`#esnNRFflL<1_@w>NC9O71OfpC00baQabg^ag;b " + echo " $0 [-k] server|client " + echo "" + echo " -k = Use keytool/Java Keystore, else standard SSL keys" + exit 1 +fi \ No newline at end of file diff --git a/docker/jvm/fixtures/secrets/kafka_keystore_creds b/docker/jvm/fixtures/secrets/kafka_keystore_creds new file mode 100644 index 0000000000000..1656f9233d999 --- /dev/null +++ b/docker/jvm/fixtures/secrets/kafka_keystore_creds @@ -0,0 +1 @@ +abcdefgh \ No newline at end of file diff --git a/docker/jvm/fixtures/secrets/kafka_ssl_key_creds b/docker/jvm/fixtures/secrets/kafka_ssl_key_creds new file mode 100644 index 0000000000000..1656f9233d999 --- /dev/null +++ b/docker/jvm/fixtures/secrets/kafka_ssl_key_creds @@ -0,0 +1 @@ +abcdefgh \ No newline at end of file diff --git a/docker/jvm/fixtures/secrets/kafka_truststore_creds b/docker/jvm/fixtures/secrets/kafka_truststore_creds new file mode 100644 index 0000000000000..1656f9233d999 --- /dev/null +++ b/docker/jvm/fixtures/secrets/kafka_truststore_creds @@ -0,0 +1 @@ +abcdefgh \ No newline at end of file diff --git a/docker/jvm/fixtures/sink_connector.json b/docker/jvm/fixtures/sink_connector.json new file mode 100644 index 0000000000000..f22ef5495f0af --- /dev/null +++ b/docker/jvm/fixtures/sink_connector.json @@ -0,0 +1,11 @@ +{ + "name": "kafka-sink", + "config": { + "connector.class": "org.apache.kafka.connect.file.FileStreamSinkConnector", + "tasks.max": "1", + "file": "/tmp/output.txt", + "topics": "test_topic_connect", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.storage.StringConverter" + } +} \ No newline at end of file diff --git a/docker/jvm/fixtures/source_connector.json b/docker/jvm/fixtures/source_connector.json new file mode 100644 index 0000000000000..495bed2db8f1b --- /dev/null +++ b/docker/jvm/fixtures/source_connector.json @@ -0,0 +1,14 @@ +{ + "name": "datagen-users", + "config": { + "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector", + "kafka.topic": "test_topic_connect", + "quickstart": "users", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "false", + "max.interval": 1000, + "iterations": 10, + "tasks.max": "1" + } +} \ No newline at end of file diff --git a/docker/jvm/fixtures/zookeeper/docker-compose.yml b/docker/jvm/fixtures/zookeeper/docker-compose.yml new file mode 100644 index 0000000000000..1057375bf7b2c --- /dev/null +++ b/docker/jvm/fixtures/zookeeper/docker-compose.yml @@ -0,0 +1,132 @@ +--- +version: '2' +services: + + zookeeper-1: + image: confluentinc/cp-zookeeper:latest + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_SERVER_ID: 1 + ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 + + zookeeper-2: + image: confluentinc/cp-zookeeper:latest + ports: + - "2182:2182" + environment: + ZOOKEEPER_CLIENT_PORT: 2182 + ZOOKEEPER_SERVER_ID: 2 + ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 + + broker: + image: {$IMAGE} + hostname: broker + container_name: broker + ports: + - "9092:9092" + - "29092:29092" + - "39092:39092" + volumes: + - ../secrets:/etc/kafka/secrets + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker:19092,EXTERNAL://localhost:9092,SSL://localhost:39092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT,SSL:SSL + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_SSL_KEYSTORE_FILENAME: "broker_broker_server.keystore.jks" + KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" + KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" + KAFKA_SSL_TRUSTSTORE_FILENAME: "broker_broker_server.truststore.jks" + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" + KAFKA_SSL_CLIENT_AUTH: "required" + KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + depends_on: + - zookeeper-1 + - zookeeper-2 + + broker-ssl: + image: {$IMAGE} + hostname: broker-ssl + container_name: broker-ssl + ports: + - "29093:29093" + - "9093:9093" + - "39093:39093" + volumes: + - ../secrets:/etc/kafka/secrets + environment: + KAFKA_BROKER_ID: 2 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker-ssl:19093,EXTERNAL://127.0.0.1:39093,SSL://localhost:9093,DOCKER://host.docker.internal:29093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,SSL:SSL,DOCKER:PLAINTEXT,EXTERNAL:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_SSL_KEYSTORE_FILENAME: "broker_broker-ssl_server.keystore.jks" + KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" + KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" + KAFKA_SSL_TRUSTSTORE_FILENAME: "broker_broker-ssl_server.truststore.jks" + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" + KAFKA_SSL_CLIENT_AUTH: "required" + KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + KAFKA_ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + depends_on: + - zookeeper-1 + - zookeeper-2 + + schema-registry: + image: confluentinc/cp-schema-registry:latest + hostname: schema-registry + container_name: schema-registry + depends_on: + - zookeeper-1 + - zookeeper-2 + - broker + ports: + - "8081:8081" + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' + SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 + + connect: + image: cnfldemos/cp-server-connect-datagen:0.6.2-7.5.0 + hostname: connect + container_name: connect + depends_on: + - zookeeper-1 + - zookeeper-2 + - broker + - schema-registry + ports: + - "8083:8083" + environment: + CONNECT_BOOTSTRAP_SERVERS: broker:29092 + CONNECT_REST_ADVERTISED_HOST_NAME: connect + CONNECT_GROUP_ID: compose-connect-group + CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 + CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 + CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter + CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 + # CLASSPATH required due to CC-2422 + CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.4.1.jar + CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" + CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" + CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" + CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR diff --git a/docker/jvm/include/etc/kafka/docker/configure b/docker/jvm/include/etc/kafka/docker/configure new file mode 100755 index 0000000000000..059d03c0819ab --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/configure @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +. /etc/kafka/docker/bash-config + +# --- for broker + +# If KAFKA_PROCESS_ROLES is defined it means we are running in KRaft mode + +# unset KAFKA_ADVERTISED_LISTENERS from ENV in KRaft mode when running as controller only +if [[ -n "${KAFKA_PROCESS_ROLES-}" ]] +then + echo "Running in KRaft mode..." + ub ensure CLUSTER_ID + if [[ $KAFKA_PROCESS_ROLES == "controller" ]] + then + if [[ -n "${KAFKA_ADVERTISED_LISTENERS-}" ]] + then + echo "KAFKA_ADVERTISED_LISTENERS is not supported on a KRaft controller." + exit 1 + else + unset KAFKA_ADVERTISED_LISTENERS + fi + else + ub ensure KAFKA_ADVERTISED_LISTENERS + fi +else + echo "Running in Zookeeper mode..." + ub ensure KAFKA_ZOOKEEPER_CONNECT + ub ensure KAFKA_ADVERTISED_LISTENERS +fi + +# By default, LISTENERS is derived from ADVERTISED_LISTENERS by replacing +# hosts with 0.0.0.0. This is good default as it ensures that the broker +# process listens on all ports. +if [[ -z "${KAFKA_LISTENERS-}" ]] && ( [[ -z "${KAFKA_PROCESS_ROLES-}" ]] || [[ $KAFKA_PROCESS_ROLES != "controller" ]] ) +then + export KAFKA_LISTENERS + KAFKA_LISTENERS=$(echo "$KAFKA_ADVERTISED_LISTENERS" | sed -e 's|://[^:]*:|://0.0.0.0:|g') +fi + +ub path /etc/kafka/ writable + +if [[ -z "${KAFKA_LOG_DIRS-}" ]] +then + export KAFKA_LOG_DIRS + KAFKA_LOG_DIRS="/var/lib/kafka/data" +fi + +# advertised.host, advertised.port, host and port are deprecated. Exit if these properties are set. +if [[ -n "${KAFKA_ADVERTISED_PORT-}" ]] +then + echo "advertised.port is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." + exit 1 +fi + +if [[ -n "${KAFKA_ADVERTISED_HOST-}" ]] +then + echo "advertised.host is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." + exit 1 +fi + +if [[ -n "${KAFKA_HOST-}" ]] +then + echo "host is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." + exit 1 +fi + +if [[ -n "${KAFKA_PORT-}" ]] +then + echo "port is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." + exit 1 +fi + +# Set if ADVERTISED_LISTENERS has SSL:// or SASL_SSL:// endpoints. +if [[ -n "${KAFKA_ADVERTISED_LISTENERS-}" ]] && [[ $KAFKA_ADVERTISED_LISTENERS == *"SSL://"* ]] +then + echo "SSL is enabled." + + ub ensure KAFKA_SSL_KEYSTORE_FILENAME + export KAFKA_SSL_KEYSTORE_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_KEYSTORE_FILENAME" + ub path "$KAFKA_SSL_KEYSTORE_LOCATION" existence + + ub ensure KAFKA_SSL_KEY_CREDENTIALS + KAFKA_SSL_KEY_CREDENTIALS_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_KEY_CREDENTIALS" + ub path "$KAFKA_SSL_KEY_CREDENTIALS_LOCATION" existence + export KAFKA_SSL_KEY_PASSWORD + KAFKA_SSL_KEY_PASSWORD=$(cat "$KAFKA_SSL_KEY_CREDENTIALS_LOCATION") + + ub ensure KAFKA_SSL_KEYSTORE_CREDENTIALS + KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_KEYSTORE_CREDENTIALS" + ub path "$KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION" existence + export KAFKA_SSL_KEYSTORE_PASSWORD + KAFKA_SSL_KEYSTORE_PASSWORD=$(cat "$KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION") + + if [[ -n "${KAFKA_SSL_CLIENT_AUTH-}" ]] && ( [[ $KAFKA_SSL_CLIENT_AUTH == *"required"* ]] || [[ $KAFKA_SSL_CLIENT_AUTH == *"requested"* ]] ) + then + ub ensure KAFKA_SSL_TRUSTSTORE_FILENAME + export KAFKA_SSL_TRUSTSTORE_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_TRUSTSTORE_FILENAME" + ub path "$KAFKA_SSL_TRUSTSTORE_LOCATION" existence + + ub ensure KAFKA_SSL_TRUSTSTORE_CREDENTIALS + KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_TRUSTSTORE_CREDENTIALS" + ub path "$KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION" existence + export KAFKA_SSL_TRUSTSTORE_PASSWORD + KAFKA_SSL_TRUSTSTORE_PASSWORD=$(cat "$KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION") + fi + +fi + +# Set if KAFKA_ADVERTISED_LISTENERS has SASL_PLAINTEXT:// or SASL_SSL:// endpoints. +if [[ -n "${KAFKA_ADVERTISED_LISTENERS-}" ]] && [[ $KAFKA_ADVERTISED_LISTENERS =~ .*SASL_.*://.* ]] +then + echo "SASL" is enabled. + + ub ensure KAFKA_OPTS + + if [[ ! $KAFKA_OPTS == *"java.security.auth.login.config"* ]] + then + echo "KAFKA_OPTS should contain 'java.security.auth.login.config' property." + fi +fi + +if [[ -n "${KAFKA_JMX_OPTS-}" ]] +then + if [[ ! $KAFKA_JMX_OPTS == *"com.sun.management.jmxremote.rmi.port"* ]] + then + echo "KAFKA_OPTS should contain 'com.sun.management.jmxremote.rmi.port' property. It is required for accessing the JMX metrics externally." + fi +fi + + +# --- for broker +ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json > /etc/kafka/kafka.properties +ub render-template /etc/kafka/docker/kafka-log4j.properties.template > /etc/kafka/log4j.properties +ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template > /etc/kafka/tools-log4j.properties diff --git a/docker/jvm/include/etc/kafka/docker/configureDefaults b/docker/jvm/include/etc/kafka/docker/configureDefaults new file mode 100755 index 0000000000000..6868f14d220a1 --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/configureDefaults @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +declare -A env_defaults +env_defaults=( +# Replace CLUSTER_ID with a unique base64 UUID using "bin/kafka-storage.sh random-uuid" +# See https://docs.confluent.io/kafka/operations-tools/kafka-tools.html#kafka-storage-sh + ["CLUSTER_ID"]="5L6g3nShT-eMCtK--X86sw" + ["KAFKA_NODE_ID"]=1 + ["KAFKA_LISTENER_SECURITY_PROTOCOL_MAP"]="CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT" + ["KAFKA_LISTENERS"]="PLAINTEXT://localhost:29092,CONTROLLER://localhost:29093,PLAINTEXT_HOST://0.0.0.0:9092" + ["KAFKA_ADVERTISED_LISTENERS"]="PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092" + ["KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR"]=1 + ["KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS"]=0 + ["KAFKA_TRANSACTION_STATE_LOG_MIN_ISR"]=1 + ["KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR"]=1 + ["KAFKA_PROCESS_ROLES"]="broker,controller" + ["KAFKA_CONTROLLER_QUORUM_VOTERS"]="1@localhost:29093" + ["KAFKA_INTER_BROKER_LISTENER_NAME"]="PLAINTEXT" + ["KAFKA_CONTROLLER_LISTENER_NAMES"]="CONTROLLER" + ["KAFKA_LOG_DIRS"]="/tmp/kraft-combined-logs" +) + +for key in "${!env_defaults[@]}"; do + if [[ -z "${!key:-}" ]]; then + echo ${key} not set. Setting it to default value: \"${env_defaults[$key]}\" + export "$key"="${env_defaults[$key]}" + fi +done diff --git a/docker/jvm/include/etc/kafka/docker/ensure b/docker/jvm/include/etc/kafka/docker/ensure new file mode 100755 index 0000000000000..a50f7812ca863 --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/ensure @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +. /etc/kafka/docker/bash-config + +export KAFKA_DATA_DIRS=${KAFKA_DATA_DIRS:-"/var/lib/kafka/data"} +echo "===> Check if $KAFKA_DATA_DIRS is writable ..." +ub path "$KAFKA_DATA_DIRS" writable + +# KRaft required step: Format the storage directory with provided cluster ID unless it already exists. +if [[ -n "${KAFKA_PROCESS_ROLES-}" ]] +then + echo "===> Using provided cluster id $CLUSTER_ID ..." + + # A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this + result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /etc/kafka/kafka.properties 2>&1) || \ + echo $result | grep -i "already formatted" || \ + { echo $result && (exit 1) } +fi diff --git a/docker/jvm/include/etc/kafka/docker/kafka-log4j.properties.template b/docker/jvm/include/etc/kafka/docker/kafka-log4j.properties.template new file mode 100644 index 0000000000000..3a7b4744e34c4 --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/kafka-log4j.properties.template @@ -0,0 +1,11 @@ +log4j.rootLogger={{ getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +{{ $loggerDefaults := "kafka=INFO,kafka.network.RequestChannel$=WARN,kafka.producer.async.DefaultEventHandler=DEBUG,kafka.request.logger=WARN,kafka.controller=TRACE,kafka.log.LogCleaner=INFO,state.change.logger=TRACE,kafka.authorizer.logger=WARN"}} +{{ $loggers := getEnv "KAFKA_LOG4J_LOGGERS" "" -}} +{{ range $k, $v := splitToMapDefaults "," $loggerDefaults $loggers}} +log4j.logger.{{ $k }}={{ $v -}} +{{ end }} diff --git a/docker/jvm/include/etc/kafka/docker/kafka-propertiesSpec.json b/docker/jvm/include/etc/kafka/docker/kafka-propertiesSpec.json new file mode 100644 index 0000000000000..0e10d92d6473e --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/kafka-propertiesSpec.json @@ -0,0 +1,25 @@ +{ + "prefixes": { + "KAFKA": false, + "CONFLUENT": true + }, + "renamed": { + "KAFKA_ZOOKEEPER_CLIENT_CNXN_SOCKET": "zookeeper.clientCnxnSocket" + }, + "excludes": [ + "KAFKA_VERSION", + "KAFKA_HEAP_OPT", + "KAFKA_LOG4J_OPTS", + "KAFKA_OPTS", + "KAFKA_JMX_OPTS", + "KAFKA_JVM_PERFORMANCE_OPTS", + "KAFKA_GC_LOG_OPTS", + "KAFKA_LOG4J_ROOT_LOGLEVEL", + "KAFKA_LOG4J_LOGGERS", + "KAFKA_TOOLS_LOG4J_LOGLEVEL", + "KAFKA_ZOOKEEPER_CLIENT_CNXN_SOCKET" + ], + "defaults": { + }, + "excludeWithPrefix": "" +} diff --git a/docker/jvm/include/etc/kafka/docker/kafka-tools-log4j.properties.template b/docker/jvm/include/etc/kafka/docker/kafka-tools-log4j.properties.template new file mode 100644 index 0000000000000..c2df5bcf064a4 --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/kafka-tools-log4j.properties.template @@ -0,0 +1,6 @@ +log4j.rootLogger={{ getEnv "KAFKA_TOOLS_LOG4J_LOGLEVEL" "WARN" }}, stderr + +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n +log4j.appender.stderr.Target=System.err diff --git a/docker/jvm/include/etc/kafka/docker/launch b/docker/jvm/include/etc/kafka/docker/launch new file mode 100755 index 0000000000000..b29294011c780 --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/launch @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + + +# Start kafka broker +# TODO: REMOVE THIS +echo "$(date +"%H:%M:%S.%3N") ===> Launching kafka ... " +/opt/kafka/bin/kafka-server-start.sh /etc/kafka/kafka.properties & # your first application +P1=$! # capture PID of the process + +# Wait for process to exit +wait -n $P1 +# Exit with status of process that exited first +exit $? diff --git a/docker/jvm/include/etc/kafka/docker/run b/docker/jvm/include/etc/kafka/docker/run new file mode 100755 index 0000000000000..d32e8e59f4441 --- /dev/null +++ b/docker/jvm/include/etc/kafka/docker/run @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +. /etc/kafka/docker/bash-config + +#TODO: REMOVE THIS +echo $(date +"%H:%M:%S.%3N") + +# Set environment values if they exist as arguments +if [ $# -ne 0 ]; then + echo "===> Overriding env params with args ..." + for var in "$@" + do + export "$var" + done +fi + +echo "===> User" +id + +if [[ -z "${KAFKA_ZOOKEEPER_CONNECT}" ]] +then +echo "===> Setting default values of environment variables if not already set." +. /etc/kafka/docker/configureDefaults +fi + +echo "===> Configuring ..." +/etc/kafka/docker/configure + +echo "===> Running preflight checks ... " +/etc/kafka/docker/ensure + +echo "===> Launching ... " +exec /etc/kafka/docker/launch diff --git a/docker/jvm/requirements.txt b/docker/jvm/requirements.txt new file mode 100644 index 0000000000000..011440694c092 --- /dev/null +++ b/docker/jvm/requirements.txt @@ -0,0 +1,6 @@ +confluent_kafka +urllib3 +requests +fastavro +jsonschema +HTMLTestRunner-Python3 \ No newline at end of file diff --git a/docker/jvm/ub/go.mod b/docker/jvm/ub/go.mod new file mode 100644 index 0000000000000..1723614f1826f --- /dev/null +++ b/docker/jvm/ub/go.mod @@ -0,0 +1,15 @@ +//module base-lite +module ub + +go 1.19 + +require ( + github.com/spf13/cobra v1.7.0 + golang.org/x/exp v0.0.0-20230419192730-864b3d6c5c2c + golang.org/x/sys v0.7.0 +) + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect +) diff --git a/docker/jvm/ub/go.sum b/docker/jvm/ub/go.sum new file mode 100644 index 0000000000000..5f20e19b6de1e --- /dev/null +++ b/docker/jvm/ub/go.sum @@ -0,0 +1,14 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/exp v0.0.0-20230419192730-864b3d6c5c2c h1:HDdYQYKOkvJT/Plb5HwJJywTVyUnIctjQm6XSnZ/0CY= +golang.org/x/exp v0.0.0-20230419192730-864b3d6c5c2c/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/docker/jvm/ub/testResources/sampleFile b/docker/jvm/ub/testResources/sampleFile new file mode 100755 index 0000000000000..e69de29bb2d1d diff --git a/docker/jvm/ub/testResources/sampleFile2 b/docker/jvm/ub/testResources/sampleFile2 new file mode 100755 index 0000000000000..e69de29bb2d1d diff --git a/docker/jvm/ub/testResources/sampleLog4j.template b/docker/jvm/ub/testResources/sampleLog4j.template new file mode 100644 index 0000000000000..3aace55b81aba --- /dev/null +++ b/docker/jvm/ub/testResources/sampleLog4j.template @@ -0,0 +1,11 @@ +log4j.rootLogger={{ getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +{{ $loggerDefaults := "kafka=INFO,kafka.network.RequestChannel$=WARN,kafka.producer.async.DefaultEventHandler=DEBUG,kafka.request.logger=WARN,kafka.controller=TRACE,kafka.log.LogCleaner=INFO,state.change.logger=TRACE,kafka.authorizer.logger=WARN"}} +{{$loggers := getEnv "KAFKA_LOG4J_LOGGERS" "" -}} +{{ range $k, $v := splitToMapDefaults "," $loggerDefaults $loggers}} +log4j.logger.{{ $k }}={{ $v -}} +{{ end }} \ No newline at end of file diff --git a/docker/jvm/ub/ub.go b/docker/jvm/ub/ub.go new file mode 100644 index 0000000000000..47dec76966475 --- /dev/null +++ b/docker/jvm/ub/ub.go @@ -0,0 +1,496 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "os/signal" + "regexp" + "sort" + "strconv" + "strings" + "text/template" + "time" + + pt "path" + + "github.com/spf13/cobra" + "golang.org/x/exp/slices" + "golang.org/x/sys/unix" +) + +type ConfigSpec struct { + Prefixes map[string]bool `json:"prefixes"` + Excludes []string `json:"excludes"` + Renamed map[string]string `json:"renamed"` + Defaults map[string]string `json:"defaults"` + ExcludeWithPrefix string `json:"excludeWithPrefix"` +} + +var ( + bootstrapServers string + configFile string + zookeeperConnect string + security string + + re = regexp.MustCompile("[^_]_[^_]") + + ensureCmd = &cobra.Command{ + Use: "ensure ", + Short: "checks if environment variable is set or not", + Args: cobra.ExactArgs(1), + RunE: runEnsureCmd, + } + + pathCmd = &cobra.Command{ + Use: "path ", + Short: "checks if an operation is permitted on a file", + Args: cobra.ExactArgs(2), + RunE: runPathCmd, + } + + renderTemplateCmd = &cobra.Command{ + Use: "render-template ", + Short: "renders template to stdout", + Args: cobra.ExactArgs(1), + RunE: runRenderTemplateCmd, + } + + renderPropertiesCmd = &cobra.Command{ + Use: "render-properties ", + Short: "creates and renders properties to stdout using the json config spec.", + Args: cobra.ExactArgs(1), + RunE: runRenderPropertiesCmd, + } + + waitCmd = &cobra.Command{ + Use: "wait ", + Short: "waits for a service to start listening on a port", + Args: cobra.ExactArgs(3), + RunE: runWaitCmd, + } + + httpReadyCmd = &cobra.Command{ + Use: "http-ready ", + Short: "waits for an HTTP/HTTPS URL to be retrievable", + Args: cobra.ExactArgs(2), + RunE: runHttpReadyCmd, + } + + kafkaReadyCmd = &cobra.Command{ + Use: "kafka-ready ", + Short: "checks if kafka brokers are up and running", + Args: cobra.ExactArgs(2), + RunE: runKafkaReadyCmd, + } +) + +func ensure(envVar string) bool { + _, found := os.LookupEnv(envVar) + return found +} + +func path(filePath string, operation string) (bool, error) { + switch operation { + + case "readable": + err := unix.Access(filePath, unix.R_OK) + if err != nil { + return false, err + } + return true, nil + case "executable": + info, err := os.Stat(filePath) + if err != nil { + err = fmt.Errorf("error checking executable status of file %q: %w", filePath, err) + return false, err + } + return info.Mode()&0111 != 0, nil //check whether file is executable by anyone, use 0100 to check for execution rights for owner + case "existence": + if _, err := os.Stat(filePath); err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil + case "writable": + err := unix.Access(filePath, unix.W_OK) + if err != nil { + return false, err + } + return true, nil + default: + err := fmt.Errorf("unknown operation %q", operation) + return false, err + } +} + +func renderTemplate(templateFilePath string) error { + funcs := template.FuncMap{ + "getEnv": getEnvOrDefault, + "splitToMapDefaults": splitToMapDefaults, + } + t, err := template.New(pt.Base(templateFilePath)).Funcs(funcs).ParseFiles(templateFilePath) + if err != nil { + err = fmt.Errorf("error %q: %w", templateFilePath, err) + return err + } + return buildTemplate(os.Stdout, *t) +} + +func buildTemplate(writer io.Writer, template template.Template) error { + err := template.Execute(writer, GetEnvironment()) + if err != nil { + err = fmt.Errorf("error building template file : %w", err) + return err + } + return nil +} + +func renderConfig(writer io.Writer, configSpec ConfigSpec) error { + return writeConfig(writer, buildProperties(configSpec, GetEnvironment())) +} + +// ConvertKey Converts an environment variable name to a property-name according to the following rules: +// - a single underscore (_) is replaced with a . +// - a double underscore (__) is replaced with a single underscore +// - a triple underscore (___) is replaced with a dash +// Moreover, the whole string is converted to lower-case. +// The behavior of sequences of four or more underscores is undefined. +func ConvertKey(key string) string { + singleReplaced := re.ReplaceAllStringFunc(key, replaceUnderscores) + singleTripleReplaced := strings.ReplaceAll(singleReplaced, "___", "-") + return strings.ToLower(strings.ReplaceAll(singleTripleReplaced, "__", "_")) +} + +// replaceUnderscores replaces every underscore '_' by a dot '.' +func replaceUnderscores(s string) string { + return strings.ReplaceAll(s, "_", ".") +} + +// ListToMap splits each and entry of the kvList argument at '=' into a key/value pair and returns a map of all the k/v pair thus obtained. +// this method will only consider values in the list formatted as key=value +func ListToMap(kvList []string) map[string]string { + m := make(map[string]string, len(kvList)) + for _, l := range kvList { + parts := strings.Split(l, "=") + if len(parts) == 2 { + m[parts[0]] = parts[1] + } + } + return m +} + +func splitToMapDefaults(separator string, defaultValues string, value string) map[string]string { + values := KvStringToMap(defaultValues, separator) + for k, v := range KvStringToMap(value, separator) { + values[k] = v + } + return values +} + +func KvStringToMap(kvString string, sep string) map[string]string { + return ListToMap(strings.Split(kvString, sep)) +} + +// GetEnvironment returns the current environment as a map. +func GetEnvironment() map[string]string { + return ListToMap(os.Environ()) +} + +// buildProperties creates a map suitable to be output as Java properties from a ConfigSpec and a map representing an environment. +func buildProperties(spec ConfigSpec, environment map[string]string) map[string]string { + config := make(map[string]string) + for key, value := range spec.Defaults { + config[key] = value + } + + for envKey, envValue := range environment { + if newKey, found := spec.Renamed[envKey]; found { + config[newKey] = envValue + } else { + if !slices.Contains(spec.Excludes, envKey) && !(len(spec.ExcludeWithPrefix) > 0 && strings.HasPrefix(envKey, spec.ExcludeWithPrefix)) { + for prefix, keep := range spec.Prefixes { + if strings.HasPrefix(envKey, prefix) { + var effectiveKey string + if keep { + effectiveKey = envKey + } else { + effectiveKey = envKey[len(prefix)+1:] + } + config[ConvertKey(effectiveKey)] = envValue + } + } + } + } + } + return config +} + +func writeConfig(writer io.Writer, config map[string]string) error { + // Go randomizes iterations over map by design. We sort properties by name to ease debugging: + sortedNames := make([]string, 0, len(config)) + for name := range config { + sortedNames = append(sortedNames, name) + } + sort.Strings(sortedNames) + for _, n := range sortedNames { + _, err := fmt.Fprintf(writer, "%s=%s\n", n, config[n]) + if err != nil { + err = fmt.Errorf("error printing configs: %w", err) + return err + } + } + return nil +} + +func loadConfigSpec(path string) (ConfigSpec, error) { + var spec ConfigSpec + bytes, err := os.ReadFile(path) + if err != nil { + err = fmt.Errorf("error reading from json file %q : %w", path, err) + return spec, err + } + + errParse := json.Unmarshal(bytes, &spec) + if errParse != nil { + err = fmt.Errorf("error parsing json file %q : %w", path, errParse) + return spec, err + } + return spec, nil +} + +func invokeJavaCommand(className string, jvmOpts string, args []string) bool { + classPath := getEnvOrDefault("UB_CLASSPATH", "/usr/share/java/cp-base-lite/*") + + opts := []string{} + if jvmOpts != "" { + opts = append(opts, jvmOpts) + } + opts = append(opts, "-cp", classPath, className) + cmd := exec.Command("java", append(opts[:], args...)...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + var exitError *exec.ExitError + if errors.As(err, &exitError) { + return exitError.ExitCode() == 0 + } + return false + } + return true +} + +func getEnvOrDefault(envVar string, defaultValue string) string { + val := os.Getenv(envVar) + if len(val) == 0 { + return defaultValue + } + return val +} + +func checkKafkaReady(minNumBroker string, timeout string, bootstrapServers string, zookeeperConnect string, configFile string, security string) bool { + + opts := []string{minNumBroker, timeout + "000"} + if bootstrapServers != "" { + opts = append(opts, "-b", bootstrapServers) + } + if zookeeperConnect != "" { + opts = append(opts, "-z", zookeeperConnect) + } + if configFile != "" { + opts = append(opts, "-c", configFile) + } + if security != "" { + opts = append(opts, "-s", security) + } + jvmOpts := os.Getenv("KAFKA_OPTS") + return invokeJavaCommand("io.confluent.admin.utils.cli.KafkaReadyCommand", jvmOpts, opts) +} + +func waitForServer(host string, port int, timeout time.Duration) bool { + address := fmt.Sprintf("%s:%d", host, port) + startTime := time.Now() + connectTimeout := 5 * time.Second + + for { + conn, err := net.DialTimeout("tcp", address, connectTimeout) + if err == nil { + _ = conn.Close() + return true + } + if time.Since(startTime) >= timeout { + return false + } + time.Sleep(1 * time.Second) + } +} + +func waitForHttp(URL string, timeout time.Duration) error { + parsedURL, err := url.Parse(URL) + if err != nil { + return fmt.Errorf("error in parsing url %q: %w", URL, err) + } + + host := parsedURL.Hostname() + portStr := parsedURL.Port() + + if len(host) == 0 { + host = "localhost" + } + + if len(portStr) == 0 { + switch parsedURL.Scheme { + case "http": + portStr = "80" + case "https": + portStr = "443" + default: + return fmt.Errorf("no port specified and cannot infer port based on protocol (only http(s) supported)") + } + } + port, err := strconv.Atoi(portStr) + if err != nil { + return fmt.Errorf("error in parsing port %q: %w", portStr, err) + } + + if !waitForServer(host, port, timeout) { + return fmt.Errorf("service is unreachable on host = %q, port = %q", host, portStr) + } + + httpClient := &http.Client{ + Timeout: timeout * time.Second, + } + resp, err := httpClient.Get(URL) + if err != nil { + return fmt.Errorf("error retrieving url") + } + statusOK := resp.StatusCode >= 200 && resp.StatusCode < 300 + if !statusOK { + return fmt.Errorf("unexpected response for %q with code %d", URL, resp.StatusCode) + } + return nil +} + +func runEnsureCmd(_ *cobra.Command, args []string) error { + success := ensure(args[0]) + if !success { + err := fmt.Errorf("environment variable %q is not set", args[0]) + return err + } + return nil +} + +func runPathCmd(_ *cobra.Command, args []string) error { + success, err := path(args[0], args[1]) + if err != nil { + err = fmt.Errorf("error in checking operation %q on file %q: %w", args[1], args[0], err) + return err + } + if !success { + err = fmt.Errorf("operation %q on file %q is unsuccessful", args[1], args[0]) + return err + } + return nil +} + +func runRenderTemplateCmd(_ *cobra.Command, args []string) error { + err := renderTemplate(args[0]) + if err != nil { + err = fmt.Errorf("error in rendering template %q: %w", args[0], err) + return err + } + return nil +} + +func runRenderPropertiesCmd(_ *cobra.Command, args []string) error { + configSpec, err := loadConfigSpec(args[0]) + if err != nil { + err = fmt.Errorf("error in loading config from file %q: %w", args[0], err) + return err + } + err = renderConfig(os.Stdout, configSpec) + if err != nil { + err = fmt.Errorf("error in building properties from file %q: %w", args[0], err) + return err + } + return nil +} + +func runWaitCmd(_ *cobra.Command, args []string) error { + port, err := strconv.Atoi(args[1]) + if err != nil { + return fmt.Errorf("error in parsing port %q: %w", args[1], err) + } + + secs, err := strconv.Atoi(args[2]) + if err != nil { + return fmt.Errorf("error in parsing timeout seconds %q: %w", args[2], err) + } + timeout := time.Duration(secs) * time.Second + + success := waitForServer(args[0], port, timeout) + if !success { + return fmt.Errorf("service is unreachable for host %q and port %q", args[0], args[1]) + } + return nil +} + +func runHttpReadyCmd(_ *cobra.Command, args []string) error { + secs, err := strconv.Atoi(args[1]) + if err != nil { + return fmt.Errorf("error in parsing timeout seconds %q: %w", args[1], err) + } + timeout := time.Duration(secs) * time.Second + + success := waitForHttp(args[0], timeout) + if success != nil { + return fmt.Errorf("error in http-ready check for url %q: %w", args[0], success) + } + return nil +} + +func runKafkaReadyCmd(_ *cobra.Command, args []string) error { + success := checkKafkaReady(args[0], args[1], bootstrapServers, zookeeperConnect, configFile, security) + if !success { + err := fmt.Errorf("kafka-ready check failed") + return err + } + return nil +} + +func main() { + rootCmd := &cobra.Command{ + Use: "ub", + Short: "utility commands for cp docker images", + Run: func(cmd *cobra.Command, args []string) {}, + } + + kafkaReadyCmd.PersistentFlags().StringVarP(&bootstrapServers, "bootstrap-servers", "b", "", "comma-separated list of kafka brokers") + kafkaReadyCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "", "path to the config file") + kafkaReadyCmd.PersistentFlags().StringVarP(&zookeeperConnect, "zookeeper-connect", "z", "", "zookeeper connect string") + kafkaReadyCmd.PersistentFlags().StringVarP(&security, "security", "s", "", "security protocol to use when multiple listeners are enabled.") + + rootCmd.AddCommand(pathCmd) + rootCmd.AddCommand(ensureCmd) + rootCmd.AddCommand(renderTemplateCmd) + rootCmd.AddCommand(renderPropertiesCmd) + rootCmd.AddCommand(waitCmd) + rootCmd.AddCommand(httpReadyCmd) + rootCmd.AddCommand(kafkaReadyCmd) + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + if err := rootCmd.ExecuteContext(ctx); err != nil { + fmt.Fprintf(os.Stderr, "error in executing the command: %s", err) + os.Exit(1) + } +} diff --git a/docker/jvm/ub/ub_test.go b/docker/jvm/ub/ub_test.go new file mode 100644 index 0000000000000..8ec46258597cb --- /dev/null +++ b/docker/jvm/ub/ub_test.go @@ -0,0 +1,446 @@ +package main + +import ( + "net" + "net/http" + "net/http/httptest" + "os" + "reflect" + "testing" + "time" +) + +func assertEqual(a string, b string, t *testing.T) { + if a != b { + t.Error(a + " != " + b) + } +} + +func Test_ensure(t *testing.T) { + type args struct { + envVar string + } + err := os.Setenv("ENV_VAR", "value") + if err != nil { + t.Fatal("Unable to set ENV_VAR for the test") + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "should exist", + args: args{ + envVar: "ENV_VAR", + }, + want: true, + }, + { + name: "should not exist", + args: args{ + envVar: "RANDOM_ENV_VAR", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ensure(tt.args.envVar); got != tt.want { + t.Errorf("ensure() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_path(t *testing.T) { + type args struct { + filePath string + operation string + } + const ( + sampleFile = "testResources/sampleFile" + sampleFile2 = "testResources/sampleFile2" + fileDoesNotExist = "testResources/sampleFile3" + ) + err := os.Chmod(sampleFile, 0777) + if err != nil { + t.Error("Unable to set permissions for the file") + } + err = os.Chmod(sampleFile2, 0000) + if err != nil { + t.Error("Unable to set permissions for the file") + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: "file readable", + args: args{filePath: sampleFile, + operation: "readable"}, + want: true, + wantErr: false, + }, + { + name: "file writable", + args: args{filePath: sampleFile, + operation: "writable"}, + want: true, + wantErr: false, + }, + { + name: "file executable", + args: args{filePath: sampleFile, + operation: "executable"}, + want: true, + wantErr: false, + }, + { + name: "file existence", + args: args{filePath: sampleFile, + operation: "existence"}, + want: true, + wantErr: false, + }, + { + name: "file not readable", + args: args{filePath: sampleFile2, + operation: "readable"}, + want: false, + wantErr: true, + }, + { + name: "file not writable", + args: args{filePath: sampleFile2, + operation: "writable"}, + want: false, + wantErr: true, + }, + { + name: "file not executable", + args: args{filePath: sampleFile2, + operation: "executable"}, + want: false, + wantErr: false, + }, + { + name: "file does not exist", + args: args{filePath: fileDoesNotExist, + operation: "existence"}, + want: false, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := path(tt.args.filePath, tt.args.operation) + if (err != nil) != tt.wantErr { + t.Errorf("path() error = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Errorf("path() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_renderTemplate(t *testing.T) { + type args struct { + templateFilePath string + } + const ( + fileExistsAndRenderable = "testResources/sampleLog4j.template" + fileDoesNotExist = "testResources/RandomFileName" + ) + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "render template success", + args: args{templateFilePath: fileExistsAndRenderable}, + wantErr: false, + }, + { + name: "render template failure ", + args: args{templateFilePath: fileDoesNotExist}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := renderTemplate(tt.args.templateFilePath); (err != nil) != tt.wantErr { + t.Errorf("renderTemplate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} +func Test_convertKey(t *testing.T) { + type args struct { + key string + } + tests := []struct { + name string + args args + wantString string + }{ + { + name: "Capitals", + args: args{key: "KEY"}, + wantString: "key", + }, + { + name: "Capitals with underscore", + args: args{key: "KEY_FOO"}, + wantString: "key.foo", + }, + { + name: "Capitals with double underscore", + args: args{key: "KEY__UNDERSCORE"}, + wantString: "key_underscore", + }, + { + name: "Capitals with double and single underscore", + args: args{key: "KEY_WITH__UNDERSCORE_AND__MORE"}, + wantString: "key.with_underscore.and_more", + }, + { + name: "Capitals with triple underscore", + args: args{key: "KEY___DASH"}, + wantString: "key-dash", + }, + { + name: "capitals with double,triple and single underscore", + args: args{key: "KEY_WITH___DASH_AND___MORE__UNDERSCORE"}, + wantString: "key.with-dash.and-more_underscore", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if result := ConvertKey(tt.args.key); result != tt.wantString { + t.Errorf("ConvertKey() result = %v, wantStr %v", result, tt.wantString) + } + }) + } +} + +func Test_buildProperties(t *testing.T) { + type args struct { + spec ConfigSpec + environment map[string]string + } + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "only defaults", + args: args{ + spec: ConfigSpec{ + Defaults: map[string]string{ + "default.property.key": "default.property.value", + "bootstrap.servers": "unknown", + }, + }, + environment: map[string]string{ + "PATH": "thePath", + "KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", + "CONFLUENT_METRICS": "metricsValue", + "KAFKA_IGNORED": "ignored", + "KAFKA_EXCLUDE_PREFIX_PROPERTY": "ignored", + }, + }, + want: map[string]string{"bootstrap.servers": "unknown", "default.property.key": "default.property.value"}, + }, + { + name: "server properties", + args: args{ + spec: ConfigSpec{ + Prefixes: map[string]bool{"KAFKA": false, "CONFLUENT": true}, + Excludes: []string{"KAFKA_IGNORED"}, + Renamed: map[string]string{}, + Defaults: map[string]string{ + "default.property.key": "default.property.value", + "bootstrap.servers": "unknown", + }, + ExcludeWithPrefix: "KAFKA_EXCLUDE_PREFIX_", + }, + environment: map[string]string{ + "PATH": "thePath", + "KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", + "CONFLUENT_METRICS": "metricsValue", + "KAFKA_IGNORED": "ignored", + "KAFKA_EXCLUDE_PREFIX_PROPERTY": "ignored", + }, + }, + want: map[string]string{"bootstrap.servers": "localhost:9092", "confluent.metrics": "metricsValue", "default.property.key": "default.property.value"}, + }, + { + name: "kafka properties", + args: args{ + spec: ConfigSpec{ + Prefixes: map[string]bool{"KAFKA": false, "CONFLUENT": true}, + Excludes: []string{"KAFKA_IGNORED"}, + Renamed: map[string]string{}, + Defaults: map[string]string{ + "default.property.key": "default.property.value", + "bootstrap.servers": "unknown", + }, + ExcludeWithPrefix: "KAFKA_EXCLUDE_PREFIX_", + }, + environment: map[string]string{ + "KAFKA_FOO": "foo", + "KAFKA_FOO_BAR": "bar", + "KAFKA_IGNORED": "ignored", + "KAFKA_WITH__UNDERSCORE": "with underscore", + "KAFKA_WITH__UNDERSCORE_AND_MORE": "with underscore and more", + "KAFKA_WITH___DASH": "with dash", + "KAFKA_WITH___DASH_AND_MORE": "with dash and more", + }, + }, + want: map[string]string{"bootstrap.servers": "unknown", "default.property.key": "default.property.value", "foo": "foo", "foo.bar": "bar", "with-dash": "with dash", "with-dash.and.more": "with dash and more", "with_underscore": "with underscore", "with_underscore.and.more": "with underscore and more"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := buildProperties(tt.args.spec, tt.args.environment); !reflect.DeepEqual(got, tt.want) { + t.Errorf("buildProperties() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_splitToMapDefaults(t *testing.T) { + type args struct { + separator string + defaultValues string + value string + } + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "split to default", + args: args{ + separator: ",", + defaultValues: "kafka=INFO,kafka.producer.async.DefaultEventHandler=DEBUG,state.change.logger=TRACE", + value: "kafka.producer.async.DefaultEventHandler=ERROR,kafka.request.logger=WARN", + }, + want: map[string]string{"kafka": "INFO", "kafka.producer.async.DefaultEventHandler": "ERROR", "kafka.request.logger": "WARN", "state.change.logger": "TRACE"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := splitToMapDefaults(tt.args.separator, tt.args.defaultValues, tt.args.value); !reflect.DeepEqual(got, tt.want) { + t.Errorf("splitToMapDefaults() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_waitForServer(t *testing.T) { + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer mockServer.Close() + port := mockServer.Listener.Addr().(*net.TCPAddr).Port + + type args struct { + host string + port int + timeout time.Duration + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "invalid server address", + args: args{ + host: "localhost", + port: port + 1, + timeout: time.Duration(5) * time.Second, + }, + want: false, + }, + { + name: "valid server address", + args: args{ + host: "localhost", + port: port, + timeout: time.Duration(5) * time.Second, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := waitForServer(tt.args.host, tt.args.port, tt.args.timeout); !reflect.DeepEqual(got, tt.want) { + t.Errorf("waitForServer() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_waitForHttp(t *testing.T) { + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/names" { + w.WriteHeader(http.StatusOK) + } else { + http.NotFound(w, r) + } + })) + defer mockServer.Close() + + serverURL := mockServer.URL + + type args struct { + URL string + timeout time.Duration + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "valid server address, valid url", + args: args{ + URL: serverURL + "/names", + timeout: time.Duration(5) * time.Second, + }, + wantErr: false, + }, + { + name: "valid server address, invalid url", + args: args{ + URL: serverURL, + timeout: time.Duration(5) * time.Second, + }, + wantErr: true, + }, + { + name: "invalid server address", + args: args{ + URL: "http://invalidAddress:50111/names", + timeout: time.Duration(5) * time.Second, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := waitForHttp(tt.args.URL, tt.args.timeout); (err != nil) != tt.wantErr { + t.Errorf("waitForHttp() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} From 7e7f8e0bdc4f27ab2f8f8fa14e1b01a11e1cb5ae Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 16 Oct 2023 12:37:50 +0530 Subject: [PATCH 02/66] Refactor build test and setup proper directory structure --- docker/docker_build_test.py | 37 ++++ docker/jvm/Dockerfile | 14 +- docker/jvm/docker_release.py | 15 -- docker/jvm/fixtures/native/docker-compose.yml | 68 -------- .../fixtures/secrets/kafka-generate-ssl.sh | 164 ------------------ docker/jvm/fixtures/sink_connector.json | 11 -- .../docker_scripts}/bash-config | 0 .../include/etc/kafka/docker/configure | 0 .../etc/kafka/docker/configureDefaults | 0 .../include/etc/kafka/docker/ensure | 0 .../docker/kafka-log4j.properties.template | 0 .../kafka/docker/kafka-propertiesSpec.json | 0 .../kafka-tools-log4j.properties.template | 0 .../include/etc/kafka/docker/launch | 0 .../include/etc/kafka/docker/run | 0 docker/{jvm => resources}/ub/go.mod | 0 docker/{jvm => resources}/ub/go.sum | 0 .../ub/testResources/sampleFile | 0 .../ub/testResources/sampleFile2 | 0 .../ub/testResources/sampleLog4j.template | 0 docker/{jvm => resources}/ub/ub.go | 0 docker/{jvm => resources}/ub/ub_test.go | 0 .../__pycache__/constants.cpython-39.pyc | Bin 951 -> 952 bytes docker/{jvm => test}/constants.py | 0 docker/{jvm => test}/docker_sanity_test.py | 25 +-- docker/{jvm => test}/fixtures/input.txt | 0 .../fixtures/kraft/docker-compose.yml | 0 docker/{jvm => test}/fixtures/output.txt | 0 docker/{jvm => test}/fixtures/schema.avro | 0 .../secrets/broker_broker-ssl_cert-file | 0 .../secrets/broker_broker-ssl_cert-signed | 0 .../broker_broker-ssl_server.keystore.jks | Bin .../broker_broker-ssl_server.truststore.jks | Bin .../fixtures/secrets/broker_broker_cert-file | 0 .../secrets/broker_broker_cert-signed | 0 .../secrets/broker_broker_server.keystore.jks | Bin .../broker_broker_server.truststore.jks | Bin docker/{jvm => test}/fixtures/secrets/ca-cert | 0 .../fixtures/secrets/ca-cert.key | 0 .../fixtures/secrets/ca-cert.srl | 0 .../fixtures/secrets/client_python_client.key | 0 .../fixtures/secrets/client_python_client.pem | 0 .../fixtures/secrets/client_python_client.req | 0 .../fixtures/secrets/kafka_keystore_creds | 0 .../fixtures/secrets/kafka_ssl_key_creds | 0 .../fixtures/secrets/kafka_truststore_creds | 0 .../fixtures/source_connector.json | 0 .../fixtures/zookeeper/docker-compose.yml | 0 docker/{jvm => test}/requirements.txt | 0 49 files changed, 56 insertions(+), 278 deletions(-) create mode 100644 docker/docker_build_test.py delete mode 100644 docker/jvm/docker_release.py delete mode 100644 docker/jvm/fixtures/native/docker-compose.yml delete mode 100755 docker/jvm/fixtures/secrets/kafka-generate-ssl.sh delete mode 100644 docker/jvm/fixtures/sink_connector.json rename docker/{jvm => resources/docker_scripts}/bash-config (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/configure (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/configureDefaults (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/ensure (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/kafka-log4j.properties.template (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/kafka-propertiesSpec.json (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/kafka-tools-log4j.properties.template (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/launch (100%) rename docker/{jvm => resources/docker_scripts}/include/etc/kafka/docker/run (100%) rename docker/{jvm => resources}/ub/go.mod (100%) rename docker/{jvm => resources}/ub/go.sum (100%) rename docker/{jvm => resources}/ub/testResources/sampleFile (100%) rename docker/{jvm => resources}/ub/testResources/sampleFile2 (100%) rename docker/{jvm => resources}/ub/testResources/sampleLog4j.template (100%) rename docker/{jvm => resources}/ub/ub.go (100%) rename docker/{jvm => resources}/ub/ub_test.go (100%) rename docker/{jvm => test}/__pycache__/constants.cpython-39.pyc (83%) rename docker/{jvm => test}/constants.py (100%) rename docker/{jvm => test}/docker_sanity_test.py (95%) rename docker/{jvm => test}/fixtures/input.txt (100%) rename docker/{jvm => test}/fixtures/kraft/docker-compose.yml (100%) rename docker/{jvm => test}/fixtures/output.txt (100%) rename docker/{jvm => test}/fixtures/schema.avro (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker-ssl_cert-file (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker-ssl_cert-signed (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker-ssl_server.keystore.jks (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker-ssl_server.truststore.jks (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker_cert-file (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker_cert-signed (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker_server.keystore.jks (100%) rename docker/{jvm => test}/fixtures/secrets/broker_broker_server.truststore.jks (100%) rename docker/{jvm => test}/fixtures/secrets/ca-cert (100%) rename docker/{jvm => test}/fixtures/secrets/ca-cert.key (100%) rename docker/{jvm => test}/fixtures/secrets/ca-cert.srl (100%) rename docker/{jvm => test}/fixtures/secrets/client_python_client.key (100%) rename docker/{jvm => test}/fixtures/secrets/client_python_client.pem (100%) rename docker/{jvm => test}/fixtures/secrets/client_python_client.req (100%) rename docker/{jvm => test}/fixtures/secrets/kafka_keystore_creds (100%) rename docker/{jvm => test}/fixtures/secrets/kafka_ssl_key_creds (100%) rename docker/{jvm => test}/fixtures/secrets/kafka_truststore_creds (100%) rename docker/{jvm => test}/fixtures/source_connector.json (100%) rename docker/{jvm => test}/fixtures/zookeeper/docker-compose.yml (100%) rename docker/{jvm => test}/requirements.txt (100%) diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py new file mode 100644 index 0000000000000..1e9d5d9862b02 --- /dev/null +++ b/docker/docker_build_test.py @@ -0,0 +1,37 @@ +import subprocess +from datetime import date +import argparse +from distutils.dir_util import copy_tree +import shutil + +def build_jvm(image, tag, kafka_url): + image = f'{image}:{tag}' + copy_tree("resources", "jvm/resources") + result = subprocess.run(["docker", "build", "-f", "jvm/Dockerfile", "-t", image, "--build-arg", f"kafka_url={kafka_url}", + "--build-arg", f'build_date={date.today()}', "jvm"]) + if result.stderr: + print(result.stdout) + return + shutil.rmtree("jvm/resources") + +def run_jvm_tests(image, tag): + subprocess.Popen(["python", "docker_sanity_test.py", f"{image}:{tag}", "jvm"], cwd="test") + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("image") + parser.add_argument("tag") + parser.add_argument("image_type", default="all") + parser.add_argument("-ku", "--kafka-url", dest="kafka_url") + parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, help="Only builds the image, don't run tests") + parser.add_argument("-t", "--test", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image") + args = parser.parse_args() + + if args.image_type in ("all", "jvm") and (args.build_only or not (args.build_only or args.test_only)): + if args.kafka_url: + build_jvm(args.image, args.tag, args.kafka_url) + else: + raise ValueError("--kafka-url is a required argument for jvm image") + + if args.image_type in ("all", "jvm") and (args.test_only or not (args.build_only or args.test_only)): + run_jvm_tests(args.image, args.tag) \ No newline at end of file diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index ff54a3853f667..738dd6e62e766 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -21,7 +21,7 @@ ARG GOLANG_VERSION=1.21.1 FROM golang:${GOLANG_VERSION} AS build-ub WORKDIR /build RUN useradd --no-log-init --create-home --shell /bin/bash appuser -COPY --chown=appuser:appuser ub/ ./ +COPY --chown=appuser:appuser resources/ub/ ./ RUN go build -ldflags="-w -s" ./ub.go USER appuser RUN go test ./... @@ -29,24 +29,22 @@ RUN go test ./... FROM eclipse-temurin:17-jre -COPY bash-config /etc/kafka/docker/bash-config +COPY resources/docker_scripts/bash-config /etc/kafka/docker/bash-config # exposed ports EXPOSE 9092 USER root -ARG kafka_url=https://archive.apache.org/dist/kafka/3.5.1/kafka_2.13-3.5.1.tgz -ARG kafka_version=2.13-3.5.1 -ARG vcs_ref=unspecified +# Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments +ARG kafka_url=unspecified ARG build_date=unspecified + LABEL org.label-schema.name="kafka" \ org.label-schema.description="Apache Kafka" \ org.label-schema.build-date="${build_date}" \ org.label-schema.vcs-url="https://github.com/apache/kafka" \ - org.label-schema.vcs-ref="${vcs_ref}" \ - org.label-schema.version="${kafka_version}" \ org.label-schema.schema-version="1.0" \ maintainer="apache" @@ -80,7 +78,7 @@ RUN set -eux ; \ rm kafka.tgz; COPY --from=build-ub /build/ub /usr/bin -COPY --chown=appuser:appuser include/etc/kafka/docker /etc/kafka/docker +COPY --chown=appuser:appuser resources/docker_scripts/include/etc/kafka/docker /etc/kafka/docker USER appuser diff --git a/docker/jvm/docker_release.py b/docker/jvm/docker_release.py deleted file mode 100644 index 3f5ee911d7f99..0000000000000 --- a/docker/jvm/docker_release.py +++ /dev/null @@ -1,15 +0,0 @@ -import subprocess -from datetime import date - -def main(): - image = "apache/kafka:3.5.1" - # Refactor this and extract to constant - result = subprocess.run(["docker", "build", "-t", image, "--build-arg", "kafka_url=https://downloads.apache.org/kafka/3.5.1/kafka_2.13-3.5.1.tgz", - "--build-arg", f'build_date={date.today()}', "--build-arg", "kafka_version=2.13-3.5.1", "."]) - if result.stderr: - print(result.stdout) - return - subprocess.run(["python", "docker_sanity_test.py", image]) - -if __name__ == '__main__': - main() diff --git a/docker/jvm/fixtures/native/docker-compose.yml b/docker/jvm/fixtures/native/docker-compose.yml deleted file mode 100644 index 0d4a874ee712d..0000000000000 --- a/docker/jvm/fixtures/native/docker-compose.yml +++ /dev/null @@ -1,68 +0,0 @@ ---- -version: '2' -services: - - broker: - image: kafka-poc:latest - hostname: broker - container_name: broker - ports: - - "9092:9092" - environment: - KAFKA_NODE_ID: 1 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' - KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092' - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_PROCESS_ROLES: 'broker,controller' - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' - KAFKA_LISTENERS: 'PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092' - KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' - KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' - KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' - CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' - - schema-registry: - image: confluentinc/cp-schema-registry:latest - hostname: schema-registry - container_name: schema-registry - depends_on: - - broker - ports: - - "8081:8081" - environment: - SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' - SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 - - connect: - image: cnfldemos/cp-server-connect-datagen:0.6.2-7.5.0 - hostname: connect - container_name: connect - depends_on: - - broker - - schema-registry - ports: - - "8083:8083" - environment: - CONNECT_BOOTSTRAP_SERVERS: broker:29092 - CONNECT_REST_ADVERTISED_HOST_NAME: connect - CONNECT_GROUP_ID: compose-connect-group - CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 - CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter - CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 - # CLASSPATH required due to CC-2422 - CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.4.1.jar - CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" - CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" - CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" - CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR diff --git a/docker/jvm/fixtures/secrets/kafka-generate-ssl.sh b/docker/jvm/fixtures/secrets/kafka-generate-ssl.sh deleted file mode 100755 index 3cf17e305ab10..0000000000000 --- a/docker/jvm/fixtures/secrets/kafka-generate-ssl.sh +++ /dev/null @@ -1,164 +0,0 @@ -#!/bin/bash -# -# -# This scripts generates: -# - root CA certificate -# - server certificate and keystore -# - client keys -# -# https://cwiki.apache.org/confluence/display/KAFKA/Deploying+SSL+for+Kafka -# - - -if [[ "$1" == "-k" ]]; then - USE_KEYTOOL=1 - shift -else - USE_KEYTOOL=0 -fi - -OP="$1" -CA_CERT="$2" -PFX="$3" -HOST="$4" - -C=NN -ST=NN -L=NN -O=NN -OU=NN -CN="$HOST" - - -# Password -PASS="abcdefgh" - -# Cert validity, in days -VALIDITY=10000 - -set -e - -export LC_ALL=C - -if [[ $OP == "ca" && ! -z "$CA_CERT" && ! -z "$3" ]]; then - CN="$3" - openssl req -new -x509 -keyout ${CA_CERT}.key -out $CA_CERT -days $VALIDITY -passin "pass:$PASS" -passout "pass:$PASS" < " - echo " $0 [-k] server|client " - echo "" - echo " -k = Use keytool/Java Keystore, else standard SSL keys" - exit 1 -fi \ No newline at end of file diff --git a/docker/jvm/fixtures/sink_connector.json b/docker/jvm/fixtures/sink_connector.json deleted file mode 100644 index f22ef5495f0af..0000000000000 --- a/docker/jvm/fixtures/sink_connector.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "kafka-sink", - "config": { - "connector.class": "org.apache.kafka.connect.file.FileStreamSinkConnector", - "tasks.max": "1", - "file": "/tmp/output.txt", - "topics": "test_topic_connect", - "key.converter": "org.apache.kafka.connect.storage.StringConverter", - "value.converter": "org.apache.kafka.connect.storage.StringConverter" - } -} \ No newline at end of file diff --git a/docker/jvm/bash-config b/docker/resources/docker_scripts/bash-config similarity index 100% rename from docker/jvm/bash-config rename to docker/resources/docker_scripts/bash-config diff --git a/docker/jvm/include/etc/kafka/docker/configure b/docker/resources/docker_scripts/include/etc/kafka/docker/configure similarity index 100% rename from docker/jvm/include/etc/kafka/docker/configure rename to docker/resources/docker_scripts/include/etc/kafka/docker/configure diff --git a/docker/jvm/include/etc/kafka/docker/configureDefaults b/docker/resources/docker_scripts/include/etc/kafka/docker/configureDefaults similarity index 100% rename from docker/jvm/include/etc/kafka/docker/configureDefaults rename to docker/resources/docker_scripts/include/etc/kafka/docker/configureDefaults diff --git a/docker/jvm/include/etc/kafka/docker/ensure b/docker/resources/docker_scripts/include/etc/kafka/docker/ensure similarity index 100% rename from docker/jvm/include/etc/kafka/docker/ensure rename to docker/resources/docker_scripts/include/etc/kafka/docker/ensure diff --git a/docker/jvm/include/etc/kafka/docker/kafka-log4j.properties.template b/docker/resources/docker_scripts/include/etc/kafka/docker/kafka-log4j.properties.template similarity index 100% rename from docker/jvm/include/etc/kafka/docker/kafka-log4j.properties.template rename to docker/resources/docker_scripts/include/etc/kafka/docker/kafka-log4j.properties.template diff --git a/docker/jvm/include/etc/kafka/docker/kafka-propertiesSpec.json b/docker/resources/docker_scripts/include/etc/kafka/docker/kafka-propertiesSpec.json similarity index 100% rename from docker/jvm/include/etc/kafka/docker/kafka-propertiesSpec.json rename to docker/resources/docker_scripts/include/etc/kafka/docker/kafka-propertiesSpec.json diff --git a/docker/jvm/include/etc/kafka/docker/kafka-tools-log4j.properties.template b/docker/resources/docker_scripts/include/etc/kafka/docker/kafka-tools-log4j.properties.template similarity index 100% rename from docker/jvm/include/etc/kafka/docker/kafka-tools-log4j.properties.template rename to docker/resources/docker_scripts/include/etc/kafka/docker/kafka-tools-log4j.properties.template diff --git a/docker/jvm/include/etc/kafka/docker/launch b/docker/resources/docker_scripts/include/etc/kafka/docker/launch similarity index 100% rename from docker/jvm/include/etc/kafka/docker/launch rename to docker/resources/docker_scripts/include/etc/kafka/docker/launch diff --git a/docker/jvm/include/etc/kafka/docker/run b/docker/resources/docker_scripts/include/etc/kafka/docker/run similarity index 100% rename from docker/jvm/include/etc/kafka/docker/run rename to docker/resources/docker_scripts/include/etc/kafka/docker/run diff --git a/docker/jvm/ub/go.mod b/docker/resources/ub/go.mod similarity index 100% rename from docker/jvm/ub/go.mod rename to docker/resources/ub/go.mod diff --git a/docker/jvm/ub/go.sum b/docker/resources/ub/go.sum similarity index 100% rename from docker/jvm/ub/go.sum rename to docker/resources/ub/go.sum diff --git a/docker/jvm/ub/testResources/sampleFile b/docker/resources/ub/testResources/sampleFile similarity index 100% rename from docker/jvm/ub/testResources/sampleFile rename to docker/resources/ub/testResources/sampleFile diff --git a/docker/jvm/ub/testResources/sampleFile2 b/docker/resources/ub/testResources/sampleFile2 similarity index 100% rename from docker/jvm/ub/testResources/sampleFile2 rename to docker/resources/ub/testResources/sampleFile2 diff --git a/docker/jvm/ub/testResources/sampleLog4j.template b/docker/resources/ub/testResources/sampleLog4j.template similarity index 100% rename from docker/jvm/ub/testResources/sampleLog4j.template rename to docker/resources/ub/testResources/sampleLog4j.template diff --git a/docker/jvm/ub/ub.go b/docker/resources/ub/ub.go similarity index 100% rename from docker/jvm/ub/ub.go rename to docker/resources/ub/ub.go diff --git a/docker/jvm/ub/ub_test.go b/docker/resources/ub/ub_test.go similarity index 100% rename from docker/jvm/ub/ub_test.go rename to docker/resources/ub/ub_test.go diff --git a/docker/jvm/__pycache__/constants.cpython-39.pyc b/docker/test/__pycache__/constants.cpython-39.pyc similarity index 83% rename from docker/jvm/__pycache__/constants.cpython-39.pyc rename to docker/test/__pycache__/constants.cpython-39.pyc index 1fca3f425054fd2ab00a8e3af2eac9a849d6efef..75258fcc2b41dfd28c547d2ed05b9959b5c5ec66 100644 GIT binary patch delta 22 dcmdnazJq;(KQp8GZ$yLmD07(u8VE_OC diff --git a/docker/jvm/constants.py b/docker/test/constants.py similarity index 100% rename from docker/jvm/constants.py rename to docker/test/constants.py diff --git a/docker/jvm/docker_sanity_test.py b/docker/test/docker_sanity_test.py similarity index 95% rename from docker/jvm/docker_sanity_test.py rename to docker/test/docker_sanity_test.py index b1f4f76fe988c..7b9378299f9c5 100644 --- a/docker/jvm/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -10,6 +10,7 @@ from confluent_kafka.schema_registry import SchemaRegistryClient from HTMLTestRunner import HTMLTestRunner import constants +import argparse class DockerSanityTestCommon(unittest.TestCase): CONTAINER_NAME="broker" @@ -285,25 +286,25 @@ def tearDown(self) -> None: def test_bed(self): self.execute() -class DockerSanityTestNative(DockerSanityTestCommon): - def setUp(self) -> None: - self.startCompose("fixtures/native/docker-compose.yml") - def tearDown(self) -> None: - self.destroyCompose("fixtures/native/docker-compose.yml") - def test_bed(self): - self.execute() - if __name__ == "__main__": - if len(sys.argv) > 0: - DockerSanityTestCommon.IMAGE = sys.argv.pop() - test_classes_to_run = [DockerSanityTestKraftMode, DockerSanityTestZookeeper, DockerSanityTestNative] + parser = argparse.ArgumentParser() + parser.add_argument("image") + parser.add_argument("mode", default="all") + args = parser.parse_args() + + DockerSanityTestCommon.IMAGE = args.image + + test_classes_to_run = [] + if args.mode in ("all", "jvm"): + test_classes_to_run.extend([DockerSanityTestKraftMode, DockerSanityTestZookeeper]) + loader = unittest.TestLoader() suites_list = [] for test_class in test_classes_to_run: suite = loader.loadTestsFromTestCase(test_class) suites_list.append(suite) big_suite = unittest.TestSuite(suites_list) - outfile = open("report.html", "w") + outfile = open(f"report.html", "w") runner = HTMLTestRunner.HTMLTestRunner( stream=outfile, title='Test Report', diff --git a/docker/jvm/fixtures/input.txt b/docker/test/fixtures/input.txt similarity index 100% rename from docker/jvm/fixtures/input.txt rename to docker/test/fixtures/input.txt diff --git a/docker/jvm/fixtures/kraft/docker-compose.yml b/docker/test/fixtures/kraft/docker-compose.yml similarity index 100% rename from docker/jvm/fixtures/kraft/docker-compose.yml rename to docker/test/fixtures/kraft/docker-compose.yml diff --git a/docker/jvm/fixtures/output.txt b/docker/test/fixtures/output.txt similarity index 100% rename from docker/jvm/fixtures/output.txt rename to docker/test/fixtures/output.txt diff --git a/docker/jvm/fixtures/schema.avro b/docker/test/fixtures/schema.avro similarity index 100% rename from docker/jvm/fixtures/schema.avro rename to docker/test/fixtures/schema.avro diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-file b/docker/test/fixtures/secrets/broker_broker-ssl_cert-file similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker-ssl_cert-file rename to docker/test/fixtures/secrets/broker_broker-ssl_cert-file diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_cert-signed b/docker/test/fixtures/secrets/broker_broker-ssl_cert-signed similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker-ssl_cert-signed rename to docker/test/fixtures/secrets/broker_broker-ssl_cert-signed diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_server.keystore.jks b/docker/test/fixtures/secrets/broker_broker-ssl_server.keystore.jks similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker-ssl_server.keystore.jks rename to docker/test/fixtures/secrets/broker_broker-ssl_server.keystore.jks diff --git a/docker/jvm/fixtures/secrets/broker_broker-ssl_server.truststore.jks b/docker/test/fixtures/secrets/broker_broker-ssl_server.truststore.jks similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker-ssl_server.truststore.jks rename to docker/test/fixtures/secrets/broker_broker-ssl_server.truststore.jks diff --git a/docker/jvm/fixtures/secrets/broker_broker_cert-file b/docker/test/fixtures/secrets/broker_broker_cert-file similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker_cert-file rename to docker/test/fixtures/secrets/broker_broker_cert-file diff --git a/docker/jvm/fixtures/secrets/broker_broker_cert-signed b/docker/test/fixtures/secrets/broker_broker_cert-signed similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker_cert-signed rename to docker/test/fixtures/secrets/broker_broker_cert-signed diff --git a/docker/jvm/fixtures/secrets/broker_broker_server.keystore.jks b/docker/test/fixtures/secrets/broker_broker_server.keystore.jks similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker_server.keystore.jks rename to docker/test/fixtures/secrets/broker_broker_server.keystore.jks diff --git a/docker/jvm/fixtures/secrets/broker_broker_server.truststore.jks b/docker/test/fixtures/secrets/broker_broker_server.truststore.jks similarity index 100% rename from docker/jvm/fixtures/secrets/broker_broker_server.truststore.jks rename to docker/test/fixtures/secrets/broker_broker_server.truststore.jks diff --git a/docker/jvm/fixtures/secrets/ca-cert b/docker/test/fixtures/secrets/ca-cert similarity index 100% rename from docker/jvm/fixtures/secrets/ca-cert rename to docker/test/fixtures/secrets/ca-cert diff --git a/docker/jvm/fixtures/secrets/ca-cert.key b/docker/test/fixtures/secrets/ca-cert.key similarity index 100% rename from docker/jvm/fixtures/secrets/ca-cert.key rename to docker/test/fixtures/secrets/ca-cert.key diff --git a/docker/jvm/fixtures/secrets/ca-cert.srl b/docker/test/fixtures/secrets/ca-cert.srl similarity index 100% rename from docker/jvm/fixtures/secrets/ca-cert.srl rename to docker/test/fixtures/secrets/ca-cert.srl diff --git a/docker/jvm/fixtures/secrets/client_python_client.key b/docker/test/fixtures/secrets/client_python_client.key similarity index 100% rename from docker/jvm/fixtures/secrets/client_python_client.key rename to docker/test/fixtures/secrets/client_python_client.key diff --git a/docker/jvm/fixtures/secrets/client_python_client.pem b/docker/test/fixtures/secrets/client_python_client.pem similarity index 100% rename from docker/jvm/fixtures/secrets/client_python_client.pem rename to docker/test/fixtures/secrets/client_python_client.pem diff --git a/docker/jvm/fixtures/secrets/client_python_client.req b/docker/test/fixtures/secrets/client_python_client.req similarity index 100% rename from docker/jvm/fixtures/secrets/client_python_client.req rename to docker/test/fixtures/secrets/client_python_client.req diff --git a/docker/jvm/fixtures/secrets/kafka_keystore_creds b/docker/test/fixtures/secrets/kafka_keystore_creds similarity index 100% rename from docker/jvm/fixtures/secrets/kafka_keystore_creds rename to docker/test/fixtures/secrets/kafka_keystore_creds diff --git a/docker/jvm/fixtures/secrets/kafka_ssl_key_creds b/docker/test/fixtures/secrets/kafka_ssl_key_creds similarity index 100% rename from docker/jvm/fixtures/secrets/kafka_ssl_key_creds rename to docker/test/fixtures/secrets/kafka_ssl_key_creds diff --git a/docker/jvm/fixtures/secrets/kafka_truststore_creds b/docker/test/fixtures/secrets/kafka_truststore_creds similarity index 100% rename from docker/jvm/fixtures/secrets/kafka_truststore_creds rename to docker/test/fixtures/secrets/kafka_truststore_creds diff --git a/docker/jvm/fixtures/source_connector.json b/docker/test/fixtures/source_connector.json similarity index 100% rename from docker/jvm/fixtures/source_connector.json rename to docker/test/fixtures/source_connector.json diff --git a/docker/jvm/fixtures/zookeeper/docker-compose.yml b/docker/test/fixtures/zookeeper/docker-compose.yml similarity index 100% rename from docker/jvm/fixtures/zookeeper/docker-compose.yml rename to docker/test/fixtures/zookeeper/docker-compose.yml diff --git a/docker/jvm/requirements.txt b/docker/test/requirements.txt similarity index 100% rename from docker/jvm/requirements.txt rename to docker/test/requirements.txt From 06234352e0a318dc036e33f9b647e7702ee8622e Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 16 Oct 2023 15:51:14 +0530 Subject: [PATCH 03/66] Refactor scripts running in Dockerfile --- docker/docker_build_test.py | 10 +- docker/jvm/Dockerfile | 5 +- .../etc/kafka/docker/ensure => jvm/launch} | 25 +- .../{docker_scripts => }/bash-config | 0 .../kafka/docker => common_scripts}/configure | 2 - .../configureDefaults | 0 .../kafka-log4j.properties.template | 0 .../kafka-propertiesSpec.json | 0 .../kafka-tools-log4j.properties.template | 0 .../etc/kafka/docker => common_scripts}/run | 6 - .../include/etc/kafka/docker/launch | 30 -- docker/test/report.html | 327 ++++++++++++++++++ 12 files changed, 356 insertions(+), 49 deletions(-) rename docker/{resources/docker_scripts/include/etc/kafka/docker/ensure => jvm/launch} (54%) rename docker/resources/{docker_scripts => }/bash-config (100%) rename docker/resources/{docker_scripts/include/etc/kafka/docker => common_scripts}/configure (99%) rename docker/resources/{docker_scripts/include/etc/kafka/docker => common_scripts}/configureDefaults (100%) rename docker/resources/{docker_scripts/include/etc/kafka/docker => common_scripts}/kafka-log4j.properties.template (100%) rename docker/resources/{docker_scripts/include/etc/kafka/docker => common_scripts}/kafka-propertiesSpec.json (100%) rename docker/resources/{docker_scripts/include/etc/kafka/docker => common_scripts}/kafka-tools-log4j.properties.template (100%) rename docker/resources/{docker_scripts/include/etc/kafka/docker => common_scripts}/run (92%) delete mode 100755 docker/resources/docker_scripts/include/etc/kafka/docker/launch create mode 100644 docker/test/report.html diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 1e9d5d9862b02..7ae22188845d0 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -19,11 +19,11 @@ def run_jvm_tests(image, tag): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument("image") - parser.add_argument("tag") - parser.add_argument("image_type", default="all") - parser.add_argument("-ku", "--kafka-url", dest="kafka_url") - parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, help="Only builds the image, don't run tests") + parser.add_argument("image", help="Image name that you want to keep for the Docker image") + parser.add_argument("-tag", "--image-tag", default="latest", dest="tag", help="Image tag that you want to add to the image") + parser.add_argument("-type", "--image-type", default="all", dest="image_type", help="Image type you want to build. By default it's all") + parser.add_argument("-u", "--kafka-url", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image") + parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, help="Only build the image, don't run tests") parser.add_argument("-t", "--test", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image") args = parser.parse_args() diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 738dd6e62e766..80ca045a41b45 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -29,7 +29,7 @@ RUN go test ./... FROM eclipse-temurin:17-jre -COPY resources/docker_scripts/bash-config /etc/kafka/docker/bash-config +COPY resources/bash-config /etc/kafka/docker/bash-config # exposed ports EXPOSE 9092 @@ -78,7 +78,8 @@ RUN set -eux ; \ rm kafka.tgz; COPY --from=build-ub /build/ub /usr/bin -COPY --chown=appuser:appuser resources/docker_scripts/include/etc/kafka/docker /etc/kafka/docker +COPY --chown=appuser:appuser resources/common_scripts /etc/kafka/docker +COPY --chown=appuser:appuser launch /etc/kafka/docker/launch USER appuser diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/ensure b/docker/jvm/launch similarity index 54% rename from docker/resources/docker_scripts/include/etc/kafka/docker/ensure rename to docker/jvm/launch index a50f7812ca863..778313c7ed349 100755 --- a/docker/resources/docker_scripts/include/etc/kafka/docker/ensure +++ b/docker/jvm/launch @@ -17,11 +17,25 @@ # limitations under the License. ############################################################################### -. /etc/kafka/docker/bash-config -export KAFKA_DATA_DIRS=${KAFKA_DATA_DIRS:-"/var/lib/kafka/data"} -echo "===> Check if $KAFKA_DATA_DIRS is writable ..." -ub path "$KAFKA_DATA_DIRS" writable +# Override this section from the script to include the com.sun.management.jmxremote.rmi.port property. +if [ -z "$KAFKA_JMX_OPTS" ]; then + export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " +fi + +# The JMX client needs to be able to connect to java.rmi.server.hostname. +# The default for bridged n/w is the bridged IP so you will only be able to connect from another docker container. +# For host n/w, this is the IP that the hostname on the host resolves to. + +# If you have more that one n/w configured, hostname -i gives you all the IPs, +# the default is to pick the first IP (or network). +export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)} + +if [ "$KAFKA_JMX_PORT" ]; then + # This ensures that the "if" section for JMX_PORT in kafka launch script does not trigger. + export JMX_PORT=$KAFKA_JMX_PORT + export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" +fi # KRaft required step: Format the storage directory with provided cluster ID unless it already exists. if [[ -n "${KAFKA_PROCESS_ROLES-}" ]] @@ -33,3 +47,6 @@ then echo $result | grep -i "already formatted" || \ { echo $result && (exit 1) } fi + +# Start kafka broker +exec /opt/kafka/bin/kafka-server-start.sh /etc/kafka/kafka.properties diff --git a/docker/resources/docker_scripts/bash-config b/docker/resources/bash-config similarity index 100% rename from docker/resources/docker_scripts/bash-config rename to docker/resources/bash-config diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/configure b/docker/resources/common_scripts/configure similarity index 99% rename from docker/resources/docker_scripts/include/etc/kafka/docker/configure rename to docker/resources/common_scripts/configure index 059d03c0819ab..6793fbe3041ac 100755 --- a/docker/resources/docker_scripts/include/etc/kafka/docker/configure +++ b/docker/resources/common_scripts/configure @@ -17,8 +17,6 @@ # limitations under the License. ############################################################################### -. /etc/kafka/docker/bash-config - # --- for broker # If KAFKA_PROCESS_ROLES is defined it means we are running in KRaft mode diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/configureDefaults b/docker/resources/common_scripts/configureDefaults similarity index 100% rename from docker/resources/docker_scripts/include/etc/kafka/docker/configureDefaults rename to docker/resources/common_scripts/configureDefaults diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/kafka-log4j.properties.template b/docker/resources/common_scripts/kafka-log4j.properties.template similarity index 100% rename from docker/resources/docker_scripts/include/etc/kafka/docker/kafka-log4j.properties.template rename to docker/resources/common_scripts/kafka-log4j.properties.template diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/kafka-propertiesSpec.json b/docker/resources/common_scripts/kafka-propertiesSpec.json similarity index 100% rename from docker/resources/docker_scripts/include/etc/kafka/docker/kafka-propertiesSpec.json rename to docker/resources/common_scripts/kafka-propertiesSpec.json diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/kafka-tools-log4j.properties.template b/docker/resources/common_scripts/kafka-tools-log4j.properties.template similarity index 100% rename from docker/resources/docker_scripts/include/etc/kafka/docker/kafka-tools-log4j.properties.template rename to docker/resources/common_scripts/kafka-tools-log4j.properties.template diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/run b/docker/resources/common_scripts/run similarity index 92% rename from docker/resources/docker_scripts/include/etc/kafka/docker/run rename to docker/resources/common_scripts/run index d32e8e59f4441..ed794d0247b1b 100755 --- a/docker/resources/docker_scripts/include/etc/kafka/docker/run +++ b/docker/resources/common_scripts/run @@ -19,9 +19,6 @@ . /etc/kafka/docker/bash-config -#TODO: REMOVE THIS -echo $(date +"%H:%M:%S.%3N") - # Set environment values if they exist as arguments if [ $# -ne 0 ]; then echo "===> Overriding env params with args ..." @@ -43,8 +40,5 @@ fi echo "===> Configuring ..." /etc/kafka/docker/configure -echo "===> Running preflight checks ... " -/etc/kafka/docker/ensure - echo "===> Launching ... " exec /etc/kafka/docker/launch diff --git a/docker/resources/docker_scripts/include/etc/kafka/docker/launch b/docker/resources/docker_scripts/include/etc/kafka/docker/launch deleted file mode 100755 index b29294011c780..0000000000000 --- a/docker/resources/docker_scripts/include/etc/kafka/docker/launch +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -############################################################################### - - -# Start kafka broker -# TODO: REMOVE THIS -echo "$(date +"%H:%M:%S.%3N") ===> Launching kafka ... " -/opt/kafka/bin/kafka-server-start.sh /etc/kafka/kafka.properties & # your first application -P1=$! # capture PID of the process - -# Wait for process to exit -wait -n $P1 -# Exit with status of process that exited first -exit $? diff --git a/docker/test/report.html b/docker/test/report.html new file mode 100644 index 0000000000000..de108c5619f1f --- /dev/null +++ b/docker/test/report.html @@ -0,0 +1,327 @@ + + + + + Test Report + + + + + + + + + +
+

Test Report

+

Start Time: 2023-10-16 15:46:29

+

Duration: 0:04:16.391973

+

Status: Pass 2

+ +

This demonstrates the report output.

+
+ + + +

Show +Summary +Failed +All +

+ ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Test Group/Test caseCountPassFailErrorView
DockerSanityTestKraftMode1100Detail
test_bed
 + + + + pass + + + + +
DockerSanityTestZookeeper1100Detail
test_bed
 + + + + pass + + + + +
Total2200 
+ +
 
+ + + From eef401b26f992d59300265ec3938146317eeeca0 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 16 Oct 2023 17:41:58 +0530 Subject: [PATCH 04/66] Refactor bash scripts --- docker/docker_build_test.py | 2 +- docker/jvm/Dockerfile | 16 +- .../{ => common-scripts}/bash-config | 0 .../configure | 0 .../configureDefaults | 0 .../kafka-log4j.properties.template | 0 .../kafka-propertiesSpec.json | 0 .../kafka-tools-log4j.properties.template | 0 .../{common_scripts => common-scripts}/run | 2 +- .../test/__pycache__/constants.cpython-39.pyc | Bin 952 -> 0 bytes docker/test/fixtures/kraft/docker-compose.yml | 4 +- docker/test/report.html | 327 ------------------ 12 files changed, 7 insertions(+), 344 deletions(-) rename docker/resources/{ => common-scripts}/bash-config (100%) rename docker/resources/{common_scripts => common-scripts}/configure (100%) rename docker/resources/{common_scripts => common-scripts}/configureDefaults (100%) rename docker/resources/{common_scripts => common-scripts}/kafka-log4j.properties.template (100%) rename docker/resources/{common_scripts => common-scripts}/kafka-propertiesSpec.json (100%) rename docker/resources/{common_scripts => common-scripts}/kafka-tools-log4j.properties.template (100%) rename docker/resources/{common_scripts => common-scripts}/run (97%) delete mode 100644 docker/test/__pycache__/constants.cpython-39.pyc diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 7ae22188845d0..85b5e5e0a536f 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -15,7 +15,7 @@ def build_jvm(image, tag, kafka_url): shutil.rmtree("jvm/resources") def run_jvm_tests(image, tag): - subprocess.Popen(["python", "docker_sanity_test.py", f"{image}:{tag}", "jvm"], cwd="test") + subprocess.run(["python3", "docker_sanity_test.py", f"{image}:{tag}", "jvm"], cwd="test") if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 80ca045a41b45..ae1cff10a6dcb 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -29,16 +29,14 @@ RUN go test ./... FROM eclipse-temurin:17-jre -COPY resources/bash-config /etc/kafka/docker/bash-config - # exposed ports EXPOSE 9092 USER root # Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments -ARG kafka_url=unspecified -ARG build_date=unspecified +ARG kafka_url +ARG build_date LABEL org.label-schema.name="kafka" \ @@ -50,14 +48,6 @@ LABEL org.label-schema.name="kafka" \ ENV KAFKA_URL=$kafka_url -# allow arg override of required env params -ARG KAFKA_ZOOKEEPER_CONNECT -ENV KAFKA_ZOOKEEPER_CONNECT=${KAFKA_ZOOKEEPER_CONNECT} -ARG KAFKA_ADVERTISED_LISTENERS -ENV KAFKA_ADVERTISED_LISTENERS=${KAFKA_ADVERTISED_LISTENERS} -ARG CLUSTER_ID -ENV CLUSTER_ID=${CLUSTER_ID} - RUN set -eux ; \ apt-get update ; \ apt-get upgrade -y ; \ @@ -78,7 +68,7 @@ RUN set -eux ; \ rm kafka.tgz; COPY --from=build-ub /build/ub /usr/bin -COPY --chown=appuser:appuser resources/common_scripts /etc/kafka/docker +COPY --chown=appuser:appuser resources/common-scripts /etc/kafka/docker COPY --chown=appuser:appuser launch /etc/kafka/docker/launch USER appuser diff --git a/docker/resources/bash-config b/docker/resources/common-scripts/bash-config similarity index 100% rename from docker/resources/bash-config rename to docker/resources/common-scripts/bash-config diff --git a/docker/resources/common_scripts/configure b/docker/resources/common-scripts/configure similarity index 100% rename from docker/resources/common_scripts/configure rename to docker/resources/common-scripts/configure diff --git a/docker/resources/common_scripts/configureDefaults b/docker/resources/common-scripts/configureDefaults similarity index 100% rename from docker/resources/common_scripts/configureDefaults rename to docker/resources/common-scripts/configureDefaults diff --git a/docker/resources/common_scripts/kafka-log4j.properties.template b/docker/resources/common-scripts/kafka-log4j.properties.template similarity index 100% rename from docker/resources/common_scripts/kafka-log4j.properties.template rename to docker/resources/common-scripts/kafka-log4j.properties.template diff --git a/docker/resources/common_scripts/kafka-propertiesSpec.json b/docker/resources/common-scripts/kafka-propertiesSpec.json similarity index 100% rename from docker/resources/common_scripts/kafka-propertiesSpec.json rename to docker/resources/common-scripts/kafka-propertiesSpec.json diff --git a/docker/resources/common_scripts/kafka-tools-log4j.properties.template b/docker/resources/common-scripts/kafka-tools-log4j.properties.template similarity index 100% rename from docker/resources/common_scripts/kafka-tools-log4j.properties.template rename to docker/resources/common-scripts/kafka-tools-log4j.properties.template diff --git a/docker/resources/common_scripts/run b/docker/resources/common-scripts/run similarity index 97% rename from docker/resources/common_scripts/run rename to docker/resources/common-scripts/run index ed794d0247b1b..5104c1c11fa62 100755 --- a/docker/resources/common_scripts/run +++ b/docker/resources/common-scripts/run @@ -31,7 +31,7 @@ fi echo "===> User" id -if [[ -z "${KAFKA_ZOOKEEPER_CONNECT}" ]] +if [[ -z "${KAFKA_ZOOKEEPER_CONNECT-}" ]] then echo "===> Setting default values of environment variables if not already set." . /etc/kafka/docker/configureDefaults diff --git a/docker/test/__pycache__/constants.cpython-39.pyc b/docker/test/__pycache__/constants.cpython-39.pyc deleted file mode 100644 index 75258fcc2b41dfd28c547d2ed05b9959b5c5ec66..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 952 zcma)5-EPw`6i&Nt{ab%lKw<+iZjh)DHHZtQ389tD9nrdGa#F#3V`a8WN`G9%E~35S zb$AsXA-7!d3S8lYw#!N{aFnC-eaGjV&pD1)DisX9zCEe)A8Et*8G+g)nd8!RF zu)$I|#nL#?Sn z2p3Je(}AWNgRJAUW2g2G+2;`Ht&ZD7aj;XbhW}LPqIS5d5P`DUwqVC4u5|(Jt{Wb# z_jGkd;@X#%8P=Aq6WrmFbL=iMAz45hMPAD~3-gi0zxkEOHJhYun@!iUJK_D9fXKC4 zmZsoMq#8xe;nfD_vgtSf!n+^q13;E)i^5AH%y#&9~UNT^xS;ud%?&9LP{|3dIJwILG@fgH0kJG>*5#vT{x|tZn@no;q!Zr4{{y|LCH?>a diff --git a/docker/test/fixtures/kraft/docker-compose.yml b/docker/test/fixtures/kraft/docker-compose.yml index 704280c290f0d..5fe95ae3bc04e 100644 --- a/docker/test/fixtures/kraft/docker-compose.yml +++ b/docker/test/fixtures/kraft/docker-compose.yml @@ -3,7 +3,7 @@ version: '2' services: broker: - image: {$IMAGE} + image: apache/kafka:3.6.0 hostname: broker container_name: broker ports: @@ -25,7 +25,7 @@ services: CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' broker-ssl: - image: {$IMAGE} + image: apache/kafka:3.6.0 hostname: broker-ssl container_name: broker-ssl ports: diff --git a/docker/test/report.html b/docker/test/report.html index de108c5619f1f..e69de29bb2d1d 100644 --- a/docker/test/report.html +++ b/docker/test/report.html @@ -1,327 +0,0 @@ - - - - - Test Report - - - - - - - - - -
-

Test Report

-

Start Time: 2023-10-16 15:46:29

-

Duration: 0:04:16.391973

-

Status: Pass 2

- -

This demonstrates the report output.

-
- - - -

Show -Summary -Failed -All -

- -------- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Test Group/Test caseCountPassFailErrorView
DockerSanityTestKraftMode1100Detail
test_bed
 - - - - pass - - - - -
DockerSanityTestZookeeper1100Detail
test_bed
 - - - - pass - - - - -
Total2200 
- -
 
- - - From 45c2272c1a49536eebebbd050732b8efc162f47f Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 16 Oct 2023 18:00:28 +0530 Subject: [PATCH 05/66] Remove redundant files --- docker/jvm/docker-compose.yml | 25 ---------------- docker/test/fixtures/input.txt | 0 docker/test/fixtures/kraft/docker-compose.yml | 4 +-- docker/test/fixtures/output.txt | 0 .../secrets/broker_broker-ssl_cert-file | 17 ----------- .../secrets/broker_broker-ssl_cert-signed | 20 ------------- .../fixtures/secrets/broker_broker_cert-file | 17 ----------- .../secrets/broker_broker_cert-signed | 20 ------------- docker/test/fixtures/secrets/ca-cert.key | 30 ------------------- docker/test/fixtures/secrets/ca-cert.srl | 1 - .../fixtures/secrets/client_python_client.req | 17 ----------- docker/test/report.html | 0 12 files changed, 2 insertions(+), 149 deletions(-) delete mode 100644 docker/jvm/docker-compose.yml delete mode 100644 docker/test/fixtures/input.txt delete mode 100644 docker/test/fixtures/output.txt delete mode 100644 docker/test/fixtures/secrets/broker_broker-ssl_cert-file delete mode 100644 docker/test/fixtures/secrets/broker_broker-ssl_cert-signed delete mode 100644 docker/test/fixtures/secrets/broker_broker_cert-file delete mode 100644 docker/test/fixtures/secrets/broker_broker_cert-signed delete mode 100644 docker/test/fixtures/secrets/ca-cert.key delete mode 100644 docker/test/fixtures/secrets/ca-cert.srl delete mode 100644 docker/test/fixtures/secrets/client_python_client.req delete mode 100644 docker/test/report.html diff --git a/docker/jvm/docker-compose.yml b/docker/jvm/docker-compose.yml deleted file mode 100644 index 53386ea471184..0000000000000 --- a/docker/jvm/docker-compose.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -version: '2' -services: - - broker: - image: kafka/test:3.5.1 - hostname: broker - container_name: broker - ports: - - "9092:9092" - environment: - KAFKA_NODE_ID: 1 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' - KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092' - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_PROCESS_ROLES: 'broker,controller' - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' - KAFKA_LISTENERS: 'PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092' - KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' - KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' - KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' - CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' diff --git a/docker/test/fixtures/input.txt b/docker/test/fixtures/input.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/docker/test/fixtures/kraft/docker-compose.yml b/docker/test/fixtures/kraft/docker-compose.yml index 5fe95ae3bc04e..704280c290f0d 100644 --- a/docker/test/fixtures/kraft/docker-compose.yml +++ b/docker/test/fixtures/kraft/docker-compose.yml @@ -3,7 +3,7 @@ version: '2' services: broker: - image: apache/kafka:3.6.0 + image: {$IMAGE} hostname: broker container_name: broker ports: @@ -25,7 +25,7 @@ services: CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' broker-ssl: - image: apache/kafka:3.6.0 + image: {$IMAGE} hostname: broker-ssl container_name: broker-ssl ports: diff --git a/docker/test/fixtures/output.txt b/docker/test/fixtures/output.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/docker/test/fixtures/secrets/broker_broker-ssl_cert-file b/docker/test/fixtures/secrets/broker_broker-ssl_cert-file deleted file mode 100644 index 3a0c3c9ee9f72..0000000000000 --- a/docker/test/fixtures/secrets/broker_broker-ssl_cert-file +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN NEW CERTIFICATE REQUEST----- -MIICyzCCAbMCAQAwVjELMAkGA1UEBhMCTk4xCzAJBgNVBAgTAk5OMQswCQYDVQQH -EwJOTjELMAkGA1UEChMCTk4xCzAJBgNVBAsTAk5OMRMwEQYDVQQDEwpicm9rZXIt -c3NsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA8sIlIx37zD3Tz9eI -+RN1dlbWoFI94Tlj+1ReO62HrJZIO+UGDl9wR7WFb4lJWM7qol6RDXG/7aWXKyLK -1w9XF8QhRaKx+0gnhZCaeCnQ3Ne5VtK8a64tg7ZgVSzWHJDOnIGeE7sAR15v7w8z -tinteU+0wLu6lQXU2d0MHGY4CuBDp3VwtGNVoxZ86wxDE3fSTBwS+hjBrW+e7ajr -PMZ8Mp4fpERdblrXFZNyUnycMOhchAoDMdqDV2CgRv6z5I5vDEknlOSdiOhHHnI+ -55RCwD98uIs4C+ZNdUD91W2baXaYMXdUF7aqKW3P1uTXx+xi2VoWWTjB8cCN4T2r -FnPYxwIDAQABoDAwLgYJKoZIhvcNAQkOMSEwHzAdBgNVHQ4EFgQUCe7i0TB0oEfd -DmuM4WWcWgCxV+8wDQYJKoZIhvcNAQELBQADggEBAHFcgQDrj7F0Oi3CannGvOB6 -XLTf6S5+f7fd9aIkq+cRIVV7aIacu8xXmTKyLgbuJMN/AhPqzZwt79jnIm54/mWh -mTBM3B9BRQT4GreJ2b1xgb543JB85LyCU2eMxx5UxOvUV/7VMxee2mRcWQUPw6Jo -0YCJqeNFZwsg80MzuQMOPA6wmGPNvgJ8LmcwMMfUnnaUlnvYL1cdw9n79Ddkuvm+ -8I63wrws9ejuO45i6o4uIL7sy9n2egwZ85oz/8hboUQgaOs+V8A2LE8xLnoLUHAV -p5pvjlB3alfhxRJEhKf4W16i0CXT3tMBl/v1o9o7NA/CllfZeb0ElboBfZA2GpI= ------END NEW CERTIFICATE REQUEST----- diff --git a/docker/test/fixtures/secrets/broker_broker-ssl_cert-signed b/docker/test/fixtures/secrets/broker_broker-ssl_cert-signed deleted file mode 100644 index 0a5ccf415e8f6..0000000000000 --- a/docker/test/fixtures/secrets/broker_broker-ssl_cert-signed +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDQTCCAikCCQDO815g0gGg1DANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJO -TjELMAkGA1UECAwCTk4xCzAJBgNVBAcMAk5OMQswCQYDVQQKDAJOTjELMAkGA1UE -CwwCTk4xCjAIBgNVBAMMAS8xHjAcBgkqhkiG9w0BCQEWD3ZlZGFydGhzaGFybWFA -LzAgFw0yMzEwMTIxNzM2MzBaGA8yMDUxMDIyNzE3MzYzMFowVjELMAkGA1UEBhMC -Tk4xCzAJBgNVBAgTAk5OMQswCQYDVQQHEwJOTjELMAkGA1UEChMCTk4xCzAJBgNV -BAsTAk5OMRMwEQYDVQQDEwpicm9rZXItc3NsMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEA8sIlIx37zD3Tz9eI+RN1dlbWoFI94Tlj+1ReO62HrJZIO+UG -Dl9wR7WFb4lJWM7qol6RDXG/7aWXKyLK1w9XF8QhRaKx+0gnhZCaeCnQ3Ne5VtK8 -a64tg7ZgVSzWHJDOnIGeE7sAR15v7w8ztinteU+0wLu6lQXU2d0MHGY4CuBDp3Vw -tGNVoxZ86wxDE3fSTBwS+hjBrW+e7ajrPMZ8Mp4fpERdblrXFZNyUnycMOhchAoD -MdqDV2CgRv6z5I5vDEknlOSdiOhHHnI+55RCwD98uIs4C+ZNdUD91W2baXaYMXdU -F7aqKW3P1uTXx+xi2VoWWTjB8cCN4T2rFnPYxwIDAQABMA0GCSqGSIb3DQEBCwUA -A4IBAQCYERHx3CzqOixzxtWZSugqRFahFdxWSFiuTTIv/3JhSpLjiMZGQt2YqX85 -YLnSvW0luChw8IW5S0Mtkn/Mgpnt9hzPsr1vY1aQ5By8PUXIqCNMmnIY8yC+HYNs -7DzTbU5Lin/YwRzMLnqq/9zvh+YBVdPhBrFpSXRjEkjqTXfs4fzNm8m1MsJifz3f -Q4t0iPOPjrbXrq1CQ+MstcpMwTi3eHHxcvyNHHlFLs10GH74NIymYKYwDG8fsatl -jScfxkn2rLuMFWuo42QqdHsPgS7QyTrZjvCM+5w1aUo6a35NPEcOqFD311/PJh/Y -vlSocIMIFklDRWFVh1D946t2Z42/ ------END CERTIFICATE----- diff --git a/docker/test/fixtures/secrets/broker_broker_cert-file b/docker/test/fixtures/secrets/broker_broker_cert-file deleted file mode 100644 index 9c7787fd1cea3..0000000000000 --- a/docker/test/fixtures/secrets/broker_broker_cert-file +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN NEW CERTIFICATE REQUEST----- -MIICxzCCAa8CAQAwUjELMAkGA1UEBhMCTk4xCzAJBgNVBAgTAk5OMQswCQYDVQQH -EwJOTjELMAkGA1UEChMCTk4xCzAJBgNVBAsTAk5OMQ8wDQYDVQQDEwZicm9rZXIw -ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCxSel2eRmGXIC3j/SdGlXZ -LFdbuENPekTBfIKDeKCGni6wpcZQM6OC2GUX1GALLGm1slkHFVUyuN/YgG7lQ4OA -jGRagOY92HUN9QGbelXWzCQCj4e1AgPxJjW3pfpmP4LWDrLN/IE6ECZGO4QOXyeO -NOJARCn10e+RWERSFrAAdWIpySRtIcmcsj69I66/EEuqatqER8CSEhyfaEXi1lZN -1WXraC+i49qDTBSbEBBAdZed7V69R1JYTam43qMgqUcymLZzW38Uq18zc5fv6XoO -5CiHipkmEG1H8tv1XALNpGDPN8wdp1ylTi842N+noXDMimGgGBMFAzXPlJ/QrQF/ -AgMBAAGgMDAuBgkqhkiG9w0BCQ4xITAfMB0GA1UdDgQWBBSwXHUp4Hj8v70a+xBm -dwWZLGlIPzANBgkqhkiG9w0BAQsFAAOCAQEAHOj/IV6oJc8BFHv1EUC9SEdNU4S7 -PnsA3bmpZ/wM5SNnmiCMppeHRX5fY3ehW/kiTCadreLXz5fjrLW6xMOXEXlojb8S -12IHK/3qcB2W/9BTmHFcnijr7oXaBwi9OBTRF2U5hUZ6vlF63jMrP6+Kfa6S6ICx -a+6o57b2RdvBsTNrwT05IHHvm3fdjCjm1MrP1kpRvXsO1WzzU7fwvhYB8Ax8hl8a -FAqqFet+5w3iEyx7a/DwUrChkgrv3zCSYZpU1O8PjwmOwHyCU8o/p/qtSE4KTj8J -PK7CJKYT/0MhaEH4EN/XpzAExybQCuGsGbYnAvLrEkkQrDoM1IwE8yDEsA== ------END NEW CERTIFICATE REQUEST----- diff --git a/docker/test/fixtures/secrets/broker_broker_cert-signed b/docker/test/fixtures/secrets/broker_broker_cert-signed deleted file mode 100644 index 414a580f1400c..0000000000000 --- a/docker/test/fixtures/secrets/broker_broker_cert-signed +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDPTCCAiUCCQDO815g0gGg1zANBgkqhkiG9w0BAQsFADBtMQswCQYDVQQGEwJO -TjELMAkGA1UECAwCTk4xCzAJBgNVBAcMAk5OMQswCQYDVQQKDAJOTjELMAkGA1UE -CwwCTk4xCjAIBgNVBAMMAS8xHjAcBgkqhkiG9w0BCQEWD3ZlZGFydGhzaGFybWFA -LzAgFw0yMzEwMTMwNTA2MjJaGA8yMDUxMDIyODA1MDYyMlowUjELMAkGA1UEBhMC -Tk4xCzAJBgNVBAgTAk5OMQswCQYDVQQHEwJOTjELMAkGA1UEChMCTk4xCzAJBgNV -BAsTAk5OMQ8wDQYDVQQDEwZicm9rZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw -ggEKAoIBAQCxSel2eRmGXIC3j/SdGlXZLFdbuENPekTBfIKDeKCGni6wpcZQM6OC -2GUX1GALLGm1slkHFVUyuN/YgG7lQ4OAjGRagOY92HUN9QGbelXWzCQCj4e1AgPx -JjW3pfpmP4LWDrLN/IE6ECZGO4QOXyeONOJARCn10e+RWERSFrAAdWIpySRtIcmc -sj69I66/EEuqatqER8CSEhyfaEXi1lZN1WXraC+i49qDTBSbEBBAdZed7V69R1JY -Tam43qMgqUcymLZzW38Uq18zc5fv6XoO5CiHipkmEG1H8tv1XALNpGDPN8wdp1yl -Ti842N+noXDMimGgGBMFAzXPlJ/QrQF/AgMBAAEwDQYJKoZIhvcNAQELBQADggEB -ALVAXtrZgxHpsxvvD2DJR5QzTnD+XeNmP5NtEGak3XgEkeoLm5s/QDe9Xf/8HVKB -6X0FqNoL2M/cODY1+3SOmOOw62pt+8PL4enIwjT7Yz7qnCPYBK9kaGuEDeTcyGDl -LI3ZwXIdiXXGUTdb1I30lQ/FwQQE+7ICQcZ6qTzBFhkJMr8R8fEnOULITnZqhH2N -cgJuIythTVRK1Mzy84f26AYt9WYpH6EjCDk90hS8INGfHeEb6lJ863wq1ORcSDVW -jCBSE8giKYhDTES/ZglX9l1M4yeY7b9rsySHDByJcFmiwhwFLNp+2U48YPW7qwMv -8kQTCammGGThiweU3ZyUAmc= ------END CERTIFICATE----- diff --git a/docker/test/fixtures/secrets/ca-cert.key b/docker/test/fixtures/secrets/ca-cert.key deleted file mode 100644 index 1459db022ff46..0000000000000 --- a/docker/test/fixtures/secrets/ca-cert.key +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN ENCRYPTED PRIVATE KEY----- -MIIFHzBJBgkqhkiG9w0BBQ0wPDAbBgkqhkiG9w0BBQwwDgQIjq9pTch7ScICAggA -MB0GCWCGSAFlAwQBKgQQtPgFCeVYgKYeCBkdA6fcDQSCBNDYadryv5Lfz7KD4t0L -hN2b3/+o9zHWh9JvfPHp7No9QAnW+eP30CC3stnxQpMqrNU/fwaYPw4S7T5Iwlsj -zf1hmycxEdCnAHJeaw1EPmXBDGqRj9uu7CNo1T/Jgdv80uH3gcHKX4OSUWA5IlYh -5CFRLDW9NdjkCb/WuAKB4iF4mTVXU1tctedjbWmSZWKzX4IShe9zu0G6UZ9bbdSc -P2CRISo8MYEr8gTEOwOLrcL3nrgzpy1BcPjmANdb+6HfYjzYCARydgWvuAbTWcpR -ertpaYDEZHozr5CDOOVftlfsNm1nA6myU50t0KZLwOl9V6iES5kY4S7yAic4PSzx -TWM8hftocRVKUBnupugb6F+snlndgTEygakv/xndqtlTvQlrCz4jGGJ4ubVkg1rL -+rXN4ok7CRqxgYx6MgqKeYuy4+UK9EN+zvtOpznWac2B1K5vunLjZ5jg2OTnHe/d -FOBfNv9NONzAIu1FbRi8csO6fcGzTZ+JyP02EetPgv3WtGRzKUYmnuGDiNPZmZRZ -fSv8uGvl4Na7kqkeQUn8O4bh3uiThvfJXgmGHPzQTcTyXhhIeqzBIc4kqDjOY40G -do8fgmY8nBBGKc3DLBP85tu37dBMkXyxxacYnl+ustFbWvmpqnPBvxvKQ/whHj2z -s1T5SGBGqW5ubYgrbg3K5dsD7GBUiT+9uFEyH5W/mv09HLu3KXKoLL2+OBnsOQjE -DQvWjO4lF/lV11p7eoHJxpAd0KABbnQxWPRY9EdNmaQuNZfeaVx5Z/yDUa1rRkyJ -gJk6R/exbOn+SQV4CfThzgrFRfB9FUYAqD4QPPpbm04sGVDeUsX0vZB6rxHGKGYg -KoqZo6kXs6AGt2R+GovLDvTcaxWL6ksKa4SRtuthPd47goHECv7n/YG++XqPfYNv -aOIKs217fQ818NNYbSc0+Uji+EjYG3PxDR+gxwMpX8qnIW3HSX7J+0I0vkPqPNCq -2VksaR3NvmjmwaNt6jq6hlbXbmtWMgwiDuO/nCqG/5n7usPQdKCwlvZF8OZH86J6 -uZPquWVNU56RRJCjjrRyuYbRjd3TSTpSLxE4LL4pSAhPSfSb4/Z1cJ52DhCg3mk1 -xNyAJ5mvDhNfDsvV82UuYB4bRUJtGxKe47YceJl32v0hHGRqzR1xLGMH30aIy+SA -gpOFAzOOCCPkiCkRgutuxwOUJpQdhZnH7ufOsGysAIwB/BoUc/Z0gDI49g7+NZGi -AqJP+EnKHpnygHt6R1PfCd9xUKiv0/GQZchFCnxoqrI2gDH0tD7+aSkaARpbmABP -Pp8o4hCnp8wT2tMh+aW78esW9JaqKxJRYaqITVkm0UaRvP/+RI1EmZoeepszBGcA -KXRy2HgbtJLxZ4XhFyMw/+B8P6tgsAOIw8BRtw2zdKHMQoMEC8BzPGFQ0ixXA5W7 -5+KSoNveD5f8/eQlFKBo+7cyFzu7Ru+BgcymoX/TPU4tBCCPGzYz71cwPE2K+ElG -fvziAi6hvkj2lyEIsvyVxWIs5RaKpNzx3L+xZl/tNJtyON5CrNUMn1Pb1+7HkDOo -2Ak48dWk/8wuOV5l1mWgQNxFyq2WAWZIyB/9jNUkQAGUJ/ZCaV4KGPhMtZ5tH5zz -p4UcKVYiNyjEjDT5SJCv0gEk7w== ------END ENCRYPTED PRIVATE KEY----- diff --git a/docker/test/fixtures/secrets/ca-cert.srl b/docker/test/fixtures/secrets/ca-cert.srl deleted file mode 100644 index bdea1459911d0..0000000000000 --- a/docker/test/fixtures/secrets/ca-cert.srl +++ /dev/null @@ -1 +0,0 @@ -CEF35E60D201A0D7 diff --git a/docker/test/fixtures/secrets/client_python_client.req b/docker/test/fixtures/secrets/client_python_client.req deleted file mode 100644 index 09e134033148c..0000000000000 --- a/docker/test/fixtures/secrets/client_python_client.req +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIICsDCCAZgCAQAwUjELMAkGA1UEBhMCTk4xCzAJBgNVBAgMAk5OMQswCQYDVQQH -DAJOTjELMAkGA1UECgwCTk4xCzAJBgNVBAsMAk5OMQ8wDQYDVQQDDAZweXRob24w -ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCuluzHsu5QJA1a8qgilJok -NXcCfOhZWOXZcNDBleI/yiNcqFM6wF7ezXalOJMYVISIzuuh8KIqcCvlbmFD9n6g -1Qygub4Yaq1eczAk9VR/kPpgHf70UKhV0KRRJ0LqqSQOBOmvRYfTHAD0uW36q9vL -PEKsutrKK/8i4pSy54g6/ScSZ/k4XySDYbBmJxGzK45qAAyol84lThjlI19SA77S -g7PBcnirUZHqzJRa1iclTBZ6ypUC89OxbZnGSoxXrjl1drgwEPnIxn49QGKrbHDG -183sR+ObFjTQymLkvyvaZXRx609mSSuvsC8I7+jvnykqrO71qMoQFS7e9kBBijqz -AgMBAAGgGTAXBgkqhkiG9w0BCQcxCgwIYWJjZGVmZ2gwDQYJKoZIhvcNAQELBQAD -ggEBAFqbC3KtdSQmIUiAqhDHRdtNNkmJH9jWb6Czz4jgFST0vF9vjJTy/k4tWLg+ -irua3EE33qhQcae+/oYxf9r/F4Aqu11fobpJpX/KOSnLFzPBooB8Dv2FBzH3Hxka -35RSvFYaIj3SI+94rGY9TMCLeSiDSX5iq60m2J+WRWjI7FdW3ddvXAGaJiSG/5aB -Adv4jhN5/ZpmVbYKWo0sYf+w0yMmQSnII6TZt8uSw2t3o27JUMeAgIQFZ5UAWoIw -jF17pzxkTopPN0ecYCRFnQbVwgruW6+umqp8cU2zYpNLv3lIy+oQAsuKnFyuLz4S -XknoTUqNHPL3/2HCHAJyvkZvhTQ= ------END CERTIFICATE REQUEST----- diff --git a/docker/test/report.html b/docker/test/report.html deleted file mode 100644 index e69de29bb2d1d..0000000000000 From 12534a32c7160a463537edc95b5ca2fcbf7b75f4 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 16 Oct 2023 18:21:46 +0530 Subject: [PATCH 06/66] Add licence to the files --- docker/docker_build_test.py | 15 +++++++++++ docker/resources/common-scripts/bash-config | 25 ++++++++----------- docker/resources/common-scripts/configure | 25 ++++++++----------- .../common-scripts/configureDefaults | 25 ++++++++----------- docker/resources/common-scripts/run | 25 ++++++++----------- docker/resources/ub/go.mod | 16 +++++++++++- docker/resources/ub/ub.go | 15 +++++++++++ docker/resources/ub/ub_test.go | 15 +++++++++++ docker/test/constants.py | 15 +++++++++++ docker/test/docker_sanity_test.py | 16 +++++++++++- docker/test/fixtures/kraft/docker-compose.yml | 15 +++++++++++ .../fixtures/zookeeper/docker-compose.yml | 15 +++++++++++ 12 files changed, 164 insertions(+), 58 deletions(-) diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 85b5e5e0a536f..0c1850744fbb4 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import subprocess from datetime import date import argparse diff --git a/docker/resources/common-scripts/bash-config b/docker/resources/common-scripts/bash-config index 5cff1d24f6912..b6971610ef695 100644 --- a/docker/resources/common-scripts/bash-config +++ b/docker/resources/common-scripts/bash-config @@ -1,20 +1,17 @@ -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. -############################################################################### set -o nounset \ -o errexit diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index 6793fbe3041ac..b05d226a75913 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -1,21 +1,18 @@ #!/usr/bin/env bash -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. -############################################################################### # --- for broker diff --git a/docker/resources/common-scripts/configureDefaults b/docker/resources/common-scripts/configureDefaults index 6868f14d220a1..a732ad60c37fb 100755 --- a/docker/resources/common-scripts/configureDefaults +++ b/docker/resources/common-scripts/configureDefaults @@ -1,21 +1,18 @@ #!/usr/bin/env bash -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. -############################################################################### declare -A env_defaults env_defaults=( diff --git a/docker/resources/common-scripts/run b/docker/resources/common-scripts/run index 5104c1c11fa62..41273ea5b7a59 100755 --- a/docker/resources/common-scripts/run +++ b/docker/resources/common-scripts/run @@ -1,21 +1,18 @@ #!/usr/bin/env bash -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. -############################################################################### . /etc/kafka/docker/bash-config diff --git a/docker/resources/ub/go.mod b/docker/resources/ub/go.mod index 1723614f1826f..3ec7b827348d0 100644 --- a/docker/resources/ub/go.mod +++ b/docker/resources/ub/go.mod @@ -1,4 +1,18 @@ -//module base-lite +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + module ub go 1.19 diff --git a/docker/resources/ub/ub.go b/docker/resources/ub/ub.go index 47dec76966475..194b8480d53eb 100644 --- a/docker/resources/ub/ub.go +++ b/docker/resources/ub/ub.go @@ -1,3 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( diff --git a/docker/resources/ub/ub_test.go b/docker/resources/ub/ub_test.go index 8ec46258597cb..70aedba34c504 100644 --- a/docker/resources/ub/ub_test.go +++ b/docker/resources/ub/ub_test.go @@ -1,3 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( diff --git a/docker/test/constants.py b/docker/test/constants.py index 46010920ebc72..7b1787f93dab4 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + SCHEMA_REGISTRY_URL="http://localhost:8081" CONNECT_URL="http://localhost:8083/connectors" CLIENT_TIMEOUT=40 diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 7b9378299f9c5..0134b55352593 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -1,6 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest import subprocess -import sys from confluent_kafka import Producer, Consumer from confluent_kafka.schema_registry.avro import AvroSerializer, AvroDeserializer import confluent_kafka.admin diff --git a/docker/test/fixtures/kraft/docker-compose.yml b/docker/test/fixtures/kraft/docker-compose.yml index 704280c290f0d..0d14c21de57ff 100644 --- a/docker/test/fixtures/kraft/docker-compose.yml +++ b/docker/test/fixtures/kraft/docker-compose.yml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- version: '2' services: diff --git a/docker/test/fixtures/zookeeper/docker-compose.yml b/docker/test/fixtures/zookeeper/docker-compose.yml index 1057375bf7b2c..291ec7675b1af 100644 --- a/docker/test/fixtures/zookeeper/docker-compose.yml +++ b/docker/test/fixtures/zookeeper/docker-compose.yml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + --- version: '2' services: From 3eaef7a20ce8b1963cec5414e51ae44789e601ba Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Tue, 17 Oct 2023 09:41:37 +0530 Subject: [PATCH 07/66] Updated to 21-jre and fix typos --- docker/jvm/Dockerfile | 5 +++-- docker/test/requirements.txt | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index ae1cff10a6dcb..e9eea40877690 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -27,7 +27,7 @@ USER appuser RUN go test ./... -FROM eclipse-temurin:17-jre +FROM eclipse-temurin:21-jre # exposed ports EXPOSE 9092 @@ -65,7 +65,8 @@ RUN set -eux ; \ chown appuser:appuser -R /etc/kafka/ /usr/logs /opt/kafka; \ chown appuser:root -R /etc/kafka /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka /var/lib/zookeeper; \ chmod -R ug+w /etc/kafka /var/lib/kafka /var/lib/kafka /etc/kafka/secrets /etc/kafka /var/log/kafka /var/lib/zookeeper; \ - rm kafka.tgz; + rm kafka.tgz; \ + rm kafka.tgz.asc; COPY --from=build-ub /build/ub /usr/bin COPY --chown=appuser:appuser resources/common-scripts /etc/kafka/docker diff --git a/docker/test/requirements.txt b/docker/test/requirements.txt index 011440694c092..9bf7356f9bd38 100644 --- a/docker/test/requirements.txt +++ b/docker/test/requirements.txt @@ -1,4 +1,4 @@ -confluent_kafka +confluent-kafka urllib3 requests fastavro From 0c3b65aaa67cf3ee8340f52f289b0b0648a9b822 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 18 Oct 2023 16:00:30 +0530 Subject: [PATCH 08/66] Add a release script for pushing docker images to dockerhub --- docker/docker_build_test.py | 2 ++ docker/docker_release.py | 60 +++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 docker/docker_release.py diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 0c1850744fbb4..89701d8c93675 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. diff --git a/docker/docker_release.py b/docker/docker_release.py new file mode 100644 index 0000000000000..4135da8f38463 --- /dev/null +++ b/docker/docker_release.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Python script to build and push docker image +Usage: docker_release.py + +Interactive utility to push the docker image to dockerhub +""" + +import subprocess +from distutils.dir_util import copy_tree +from datetime import date +import shutil + +def push_jvm(docker_account, image_name, image_tag, kafka_url): + copy_tree("resources", "jvm/resources") + subprocess.run(["docker", "buildx", "build", "-f", "jvm/Dockerfile", "--build-arg", f"kafka_url={kafka_url}", "--build-arg", f"build_date={date.today()}", + "--push", + "--platform", "linux/amd64,linux/arm64", + "--tag", f"{docker_account}/{image_name}:{image_tag}", "jvm"]) + shutil.rmtree("jvm/resources") + +def login(): + subprocess.run(["docker", "login"]) + +def create(): + subprocess.run(["docker", "buildx", "create", "--name", "kafka-builder", "--use"]) + +def remove(): + subprocess.run(["docker", "buildx", "rm", "kafka-builder"]) + +if __name__ == "__main__": + print("\ + This script will build and push docker images of apache kafka.\n\ + Please ensure that image has been sanity tested before pushing the image") + login() + create() + docker_account = input("Enter the dockerhub account you want to push the image to: ") + image_name = input("Enter the image name: ") + image_tag = input("Enter the image tag for the image: ") + kafka_url = input("Enter the url for kafka binary tarball: ") + push_jvm(docker_account, image_name, image_tag, kafka_url) + remove() \ No newline at end of file From f75cc5fa1f0f0f0c9684057d3b51c66d57e367a9 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 18 Oct 2023 19:07:57 +0530 Subject: [PATCH 09/66] Update release script to support registry and add error handling --- docker/docker_release.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/docker/docker_release.py b/docker/docker_release.py index 4135da8f38463..344476a577bbc 100644 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -29,21 +29,24 @@ from datetime import date import shutil -def push_jvm(docker_account, image_name, image_tag, kafka_url): +def push_jvm(image, kafka_url): copy_tree("resources", "jvm/resources") subprocess.run(["docker", "buildx", "build", "-f", "jvm/Dockerfile", "--build-arg", f"kafka_url={kafka_url}", "--build-arg", f"build_date={date.today()}", "--push", "--platform", "linux/amd64,linux/arm64", - "--tag", f"{docker_account}/{image_name}:{image_tag}", "jvm"]) + "--tag", image, "jvm"]) shutil.rmtree("jvm/resources") def login(): - subprocess.run(["docker", "login"]) + status = subprocess.run(["docker", "login"]) + if status.returncode != 0: + print("Docker login failed, aborting the docker release") + raise PermissionError -def create(): +def create_builder(): subprocess.run(["docker", "buildx", "create", "--name", "kafka-builder", "--use"]) -def remove(): +def remove_builder(): subprocess.run(["docker", "buildx", "rm", "kafka-builder"]) if __name__ == "__main__": @@ -51,10 +54,27 @@ def remove(): This script will build and push docker images of apache kafka.\n\ Please ensure that image has been sanity tested before pushing the image") login() - create() - docker_account = input("Enter the dockerhub account you want to push the image to: ") + docker_registry = input("Enter the docker registry you want to push the image to [docker.io]: ") + if docker_registry == "": + docker_registry = "docker.io" + docker_namespace = input("Enter the docker namespace you want to push the image to: ") image_name = input("Enter the image name: ") + if image_name == "": + raise ValueError("image name cannot be empty") image_tag = input("Enter the image tag for the image: ") + if image_tag == "": + raise ValueError("image tag cannot be empty") kafka_url = input("Enter the url for kafka binary tarball: ") - push_jvm(docker_account, image_name, image_tag, kafka_url) - remove() \ No newline at end of file + if kafka_url == "": + raise ValueError("kafka url cannot be empty") + image = f"{docker_registry}/{docker_namespace}/{image_name}:{image_tag}" + print(f"Docker image containing kafka downloaded from {kafka_url} will be pushed to {image}") + proceed = input("Should we proceed? [y/N]: ") + if proceed == "y": + print("Building and pushing the image") + create_builder() + push_jvm(image, kafka_url) + remove_builder() + print(f"Image has been pushed to {image}") + else: + print("Image push aborted") From 6f64ce54254b985a50b3c5d8d63711cc6b71ab18 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 30 Oct 2023 09:33:59 +0530 Subject: [PATCH 10/66] Remove confluent artifacts from sanity test Remove confluent kafka python client as dependency Remove schema registry and connect from compose files Download kafka binary and use it's scripts to run tests --- docker/docker_build_test.py | 10 +- docker/test/constants.py | 13 +- docker/test/docker_sanity_test.py | 200 ++++-------------- docker/test/fixtures/kraft/docker-compose.yml | 48 +---- .../broker_broker-ssl_server.keystore.jks | Bin 4750 -> 0 bytes .../broker_broker-ssl_server.truststore.jks | Bin 1238 -> 0 bytes .../secrets/broker_broker_server.keystore.jks | Bin 4750 -> 0 bytes .../broker_broker_server.truststore.jks | Bin 1238 -> 0 bytes docker/test/fixtures/secrets/ca-cert | 20 -- .../fixtures/secrets/client-ssl.properties | 9 + .../test/fixtures/secrets/client.keystore.jks | Bin 0 -> 4382 bytes .../fixtures/secrets/client_python_client.key | 30 --- .../fixtures/secrets/client_python_client.pem | 20 -- .../fixtures/secrets/kafka.truststore.jks | Bin 0 -> 1126 bytes .../fixtures/secrets/kafka01.keystore.jks | Bin 0 -> 4382 bytes .../fixtures/secrets/kafka02.keystore.jks | Bin 0 -> 4382 bytes .../fixtures/zookeeper/docker-compose.yml | 56 +---- docker/test/requirements.txt | 3 - 18 files changed, 69 insertions(+), 340 deletions(-) delete mode 100644 docker/test/fixtures/secrets/broker_broker-ssl_server.keystore.jks delete mode 100644 docker/test/fixtures/secrets/broker_broker-ssl_server.truststore.jks delete mode 100644 docker/test/fixtures/secrets/broker_broker_server.keystore.jks delete mode 100644 docker/test/fixtures/secrets/broker_broker_server.truststore.jks delete mode 100644 docker/test/fixtures/secrets/ca-cert create mode 100644 docker/test/fixtures/secrets/client-ssl.properties create mode 100644 docker/test/fixtures/secrets/client.keystore.jks delete mode 100644 docker/test/fixtures/secrets/client_python_client.key delete mode 100644 docker/test/fixtures/secrets/client_python_client.pem create mode 100644 docker/test/fixtures/secrets/kafka.truststore.jks create mode 100644 docker/test/fixtures/secrets/kafka01.keystore.jks create mode 100644 docker/test/fixtures/secrets/kafka02.keystore.jks diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 89701d8c93675..a841e6f0ea548 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -31,8 +31,14 @@ def build_jvm(image, tag, kafka_url): return shutil.rmtree("jvm/resources") -def run_jvm_tests(image, tag): +def run_jvm_tests(image, tag, kafka_url): + subprocess.run(["wget", "-nv", "-O", "kafka.tgz", kafka_url]) + subprocess.run(["ls"]) + subprocess.run(["mkdir", "./test/fixtures/kafka"]) + subprocess.run(["tar", "xfz", "kafka.tgz", "-C", "./test/fixtures/kafka", "--strip-components", "1"]) subprocess.run(["python3", "docker_sanity_test.py", f"{image}:{tag}", "jvm"], cwd="test") + subprocess.run(["rm", "kafka.tgz"]) + shutil.rmtree("./test/fixtures/kafka") if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -51,4 +57,4 @@ def run_jvm_tests(image, tag): raise ValueError("--kafka-url is a required argument for jvm image") if args.image_type in ("all", "jvm") and (args.test_only or not (args.build_only or args.test_only)): - run_jvm_tests(args.image, args.tag) \ No newline at end of file + run_jvm_tests(args.image, args.tag, args.kafka_url) \ No newline at end of file diff --git a/docker/test/constants.py b/docker/test/constants.py index 7b1787f93dab4..621aa8f41e84b 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -13,6 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +KAFKA_TOPICS="./fixtures/kafka/bin/kafka-topics.sh" +KAFKA_CONSOLE_PRODUCER="./fixtures/kafka/bin/kafka-console-producer.sh" +KAFKA_CONSOLE_CONSUMER="./fixtures/kafka/bin/kafka-console-consumer.sh" + +KRAFT_COMPOSE="fixtures/kraft/docker-compose.yml" +ZOOKEEPER_COMPOSE="fixtures/zookeeper/docker-compose.yml" + SCHEMA_REGISTRY_URL="http://localhost:8081" CONNECT_URL="http://localhost:8083/connectors" CLIENT_TIMEOUT=40 @@ -20,11 +27,7 @@ CONNECT_TEST_TOPIC="test_topic_connect" CONNECT_SOURCE_CONNECTOR_CONFIG="@fixtures/source_connector.json" -SSL_TOPIC="test_topic_ssl" -SSL_CA_LOCATION="./fixtures/secrets/ca-cert" -SSL_CERTIFICATE_LOCATION="./fixtures/secrets/client_python_client.pem" -SSL_KEY_LOCATION="./fixtures/secrets/client_python_client.key" -SSL_KEY_PASSWORD="abcdefgh" +SSL_CLIENT_CONFIG="./fixtures/secrets/client-ssl.properties" BROKER_RESTART_TEST_TOPIC="test_topic_broker_restart" diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 0134b55352593..1022365cac8ac 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -15,13 +15,7 @@ import unittest import subprocess -from confluent_kafka import Producer, Consumer -from confluent_kafka.schema_registry.avro import AvroSerializer, AvroDeserializer -import confluent_kafka.admin import time -import socket -from confluent_kafka.serialization import SerializationContext, MessageField -from confluent_kafka.schema_registry import SchemaRegistryClient from HTMLTestRunner import HTMLTestRunner import constants import argparse @@ -69,154 +63,50 @@ def destroyCompose(self, filename) -> None: s = s.replace(old_string, new_string) f.write(s) - def create_topic(self, topic): - kafka_admin = confluent_kafka.admin.AdminClient({"bootstrap.servers": "localhost:9092"}) - new_topic = confluent_kafka.admin.NewTopic(topic, 1, 1) - kafka_admin.create_topics([new_topic,]) - timeout = constants.CLIENT_TIMEOUT - while timeout > 0: - timeout -= 1 - if topic not in kafka_admin.list_topics().topics: - time.sleep(1) - continue - return topic - return None - + def create_topic(self, topic, topic_config): + command = [constants.KAFKA_TOPICS, "--create", "--topic", topic] + command.extend(topic_config) + subprocess.run(command) + check_command = [constants.KAFKA_TOPICS, "--list"] + check_command.extend(topic_config) + output = subprocess.check_output(check_command, timeout=constants.CLIENT_TIMEOUT) + if topic in output.decode("utf-8"): + return True + return False + def produce_message(self, topic, producer_config, key, value): - producer = Producer(producer_config) - producer.produce(topic, key=key, value=value) - producer.flush() - del producer + command = ["echo", f'"{key}:{value}"', "|", constants.KAFKA_CONSOLE_PRODUCER, "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] + command.extend(producer_config) + print(" ".join(command)) + subprocess.run(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) def consume_message(self, topic, consumer_config): - consumer = Consumer(consumer_config) - consumer.subscribe([topic]) - timeout = constants.CLIENT_TIMEOUT - while timeout > 0: - message = consumer.poll(1) - if message is None: - time.sleep(1) - timeout -= 1 - continue - del consumer - return message - raise None - - def schema_registry_flow(self): - print("Running Schema Registry tests") - errors = [] - schema_registry_conf = {'url': constants.SCHEMA_REGISTRY_URL} - schema_registry_client = SchemaRegistryClient(schema_registry_conf) - avro_schema = "" - with open("fixtures/schema.avro") as f: - avro_schema = f.read() - avro_serializer = AvroSerializer(schema_registry_client=schema_registry_client, - schema_str=avro_schema) - producer_config = { - "bootstrap.servers": "localhost:9092", - } - - avro_deserializer = AvroDeserializer(schema_registry_client, avro_schema) + command = [constants.KAFKA_CONSOLE_CONSUMER, "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] + command.extend(consumer_config) + print(" ".join(command)) + message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) + return message.decode("utf-8").strip() - key = {"key": "key", "value": ""} - value = {"value": "message", "key": ""} - self.produce_message(constants.SCHEMA_REGISTRY_TEST_TOPIC, producer_config, key=avro_serializer(key, SerializationContext(constants.SCHEMA_REGISTRY_TEST_TOPIC, MessageField.KEY)), value=avro_serializer(value, SerializationContext(constants.SCHEMA_REGISTRY_TEST_TOPIC, MessageField.VALUE))) - time.sleep(3) - - consumer_config = { - "bootstrap.servers": "localhost:9092", - "group.id": "test-group", - 'auto.offset.reset': "earliest" - } - - message = self.consume_message(constants.SCHEMA_REGISTRY_TEST_TOPIC, consumer_config) - - try: - self.assertIsNotNone(message) - except AssertionError as e: - errors.append(constants.SCHEMA_REGISTRY_ERROR_PREFIX + str(e)) - return - - deserialized_value = avro_deserializer(message.value(), SerializationContext(message.topic(), MessageField.VALUE)) - deserialized_key = avro_deserializer(message.key(), SerializationContext(message.topic(), MessageField.KEY)) - try: - self.assertEqual(deserialized_key, key) - except AssertionError as e: - errors.append(constants.SCHEMA_REGISTRY_ERROR_PREFIX + str(e)) - try: - self.assertEqual(deserialized_value, value) - except AssertionError as e: - errors.append(constants.SCHEMA_REGISTRY_ERROR_PREFIX + str(e)) - print("Errors in Schema Registry Test Flow:-", errors) - return errors - - def connect_flow(self): - print("Running Connect tests") - errors = [] - try: - self.assertEqual(self.create_topic(constants.CONNECT_TEST_TOPIC), constants.CONNECT_TEST_TOPIC) - except AssertionError as e: - errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) - return errors - subprocess.run(["curl", "-X", "POST", "-H", "Content-Type:application/json", "--data", constants.CONNECT_SOURCE_CONNECTOR_CONFIG, constants.CONNECT_URL]) - consumer_config = { - "bootstrap.servers": "localhost:9092", - "group.id": "test-group", - 'auto.offset.reset': "earliest" - } - message = self.consume_message(constants.CONNECT_TEST_TOPIC, consumer_config) - try: - self.assertIsNotNone(message) - except AssertionError as e: - errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) - return errors - try: - self.assertIn('User', message.key().decode('ascii')) - except AssertionError as e: - errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) - try: - self.assertIsNotNone(message.value()) - except AssertionError as e: - errors.append(constants.CONNECT_ERROR_PREFIX + str(e)) - print("Errors in Connect Test Flow:-", errors) - return errors - def ssl_flow(self): print("Running SSL flow tests") errors = [] - producer_config = {"bootstrap.servers": "localhost:9093", - "security.protocol": "SSL", - "ssl.ca.location": constants.SSL_CA_LOCATION, - "ssl.certificate.location": constants.SSL_CERTIFICATE_LOCATION, - "ssl.key.location": constants.SSL_KEY_LOCATION, - "ssl.endpoint.identification.algorithm": "none", - "ssl.key.password": constants.SSL_KEY_PASSWORD, - 'client.id': socket.gethostname() + '2'} + producer_config = ["--bootstrap-server", "localhost:9093", + "--producer.config", constants.SSL_CLIENT_CONFIG] self.produce_message(constants.SSL_TOPIC, producer_config, "key", "message") - consumer_config = { - "bootstrap.servers": "localhost:9093", - "group.id": "test-group-5", - 'auto.offset.reset': "earliest", - "security.protocol": "SSL", - "ssl.ca.location": constants.SSL_CA_LOCATION, - "ssl.certificate.location": constants.SSL_CERTIFICATE_LOCATION, - "ssl.key.location": constants.SSL_KEY_LOCATION, - "ssl.endpoint.identification.algorithm": "none", - "ssl.key.password": constants.SSL_KEY_PASSWORD - } + consumer_config = [ + "--bootstrap-server", "localhost:9093", + "--property", "auto.offset.reset=earliest", + "--consumer.config", constants.SSL_CLIENT_CONFIG, + ] message = self.consume_message(constants.SSL_TOPIC, consumer_config) try: self.assertIsNotNone(message) except AssertionError as e: errors.append(constants.SSL_ERROR_PREFIX + str(e)) try: - self.assertEqual(message.key(), b'key') - except AssertionError as e: - errors.append(constants.SSL_ERROR_PREFIX + str(e)) - try: - self.assertEqual(message.value(), b'message') + self.assertEqual(message, "key:message") except AssertionError as e: errors.append(constants.SSL_ERROR_PREFIX + str(e)) print("Errors in SSL Flow:-", errors) @@ -226,12 +116,12 @@ def broker_restart_flow(self): print("Running broker restart tests") errors = [] try: - self.assertEqual(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC), constants.BROKER_RESTART_TEST_TOPIC) + self.assertTrue(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) except AssertionError as e: errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) return errors - producer_config = {"bootstrap.servers": "localhost:9092", 'client.id': socket.gethostname()} + producer_config = ["--bootstrap-server", "localhost:9092", "--property", "client.id=host"] self.produce_message(constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") print("Stopping Image") @@ -241,7 +131,7 @@ def broker_restart_flow(self): print("Resuming Image") self.resumeImage() time.sleep(15) - consumer_config = {"bootstrap.servers": "localhost:9092", 'group.id': 'test-group-1', 'auto.offset.reset': 'smallest'} + consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) try: self.assertIsNotNone(message) @@ -249,11 +139,7 @@ def broker_restart_flow(self): errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) return errors try: - self.assertEqual(message.key(), b'key') - except AssertionError as e: - errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) - try: - self.assertEqual(message.value(), b'message') + self.assertEqual(message, "key:message") except AssertionError as e: errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) print("Errors in Broker Restart Flow:-", errors) @@ -261,42 +147,32 @@ def broker_restart_flow(self): def execute(self): total_errors = [] - try: - total_errors.extend(self.schema_registry_flow()) - except Exception as e: - print("Schema registry error") - total_errors.append(str(e)) - try: - total_errors.extend(self.connect_flow()) - except Exception as e: - print("Connect flow error") - total_errors.append(str(e)) try: total_errors.extend(self.ssl_flow()) except Exception as e: - print("SSL flow error") + print("SSL flow error", str(e)) total_errors.append(str(e)) try: total_errors.extend(self.broker_restart_flow()) except Exception as e: - print("Broker restart flow error") + print("Broker restart flow error", str(e)) total_errors.append(str(e)) self.assertEqual(total_errors, []) class DockerSanityTestKraftMode(DockerSanityTestCommon): def setUp(self) -> None: - self.startCompose("fixtures/kraft/docker-compose.yml") + self.startCompose(constants.KRAFT_COMPOSE) def tearDown(self) -> None: - self.destroyCompose("fixtures/kraft/docker-compose.yml") + self.destroyCompose(constants.KRAFT_COMPOSE) def test_bed(self): self.execute() class DockerSanityTestZookeeper(DockerSanityTestCommon): def setUp(self) -> None: - self.startCompose("fixtures/zookeeper/docker-compose.yml") + self.startCompose(constants.ZOOKEEPER_COMPOSE) def tearDown(self) -> None: - self.destroyCompose("fixtures/zookeeper/docker-compose.yml") + self.destroyCompose(constants.ZOOKEEPER_COMPOSE) def test_bed(self): self.execute() diff --git a/docker/test/fixtures/kraft/docker-compose.yml b/docker/test/fixtures/kraft/docker-compose.yml index 0d14c21de57ff..ed06c819917d0 100644 --- a/docker/test/fixtures/kraft/docker-compose.yml +++ b/docker/test/fixtures/kraft/docker-compose.yml @@ -16,7 +16,6 @@ --- version: '2' services: - broker: image: {$IMAGE} hostname: broker @@ -62,54 +61,11 @@ services: KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' - KAFKA_SSL_KEYSTORE_FILENAME: "broker_broker-ssl_server.keystore.jks" + KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks" KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" - KAFKA_SSL_TRUSTSTORE_FILENAME: "broker_broker-ssl_server.truststore.jks" + KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" KAFKA_SSL_CLIENT_AUTH: "required" KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - - schema-registry: - image: confluentinc/cp-schema-registry:latest - hostname: schema-registry - container_name: schema-registry - depends_on: - - broker - ports: - - "8081:8081" - environment: - SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' - SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 - - connect: - image: cnfldemos/cp-server-connect-datagen:0.6.2-7.5.0 - hostname: connect - container_name: connect - depends_on: - - broker - - schema-registry - ports: - - "8083:8083" - environment: - CONNECT_BOOTSTRAP_SERVERS: broker:29092 - CONNECT_REST_ADVERTISED_HOST_NAME: connect - CONNECT_GROUP_ID: compose-connect-group - CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 - CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter - CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 - # CLASSPATH required due to CC-2422 - CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.4.1.jar - CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" - CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" - CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" - CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR diff --git a/docker/test/fixtures/secrets/broker_broker-ssl_server.keystore.jks b/docker/test/fixtures/secrets/broker_broker-ssl_server.keystore.jks deleted file mode 100644 index a823cf3b7ca0fd48292238f98cf99d893e6aa067..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4750 zcma)AWmFUlv)*00yBnly7o@uz=_Le~mX4*HB^8j65Rhg;KoD3`LIeaU>5vXdK|s2% z_uPBl@7`bE_hZg8^E@;2^Eqb*j39jl0-}Ktq)#z%c;eLKE(w5Gz#;@GJ34}t$W+{8w(|JrC~R7cqg@3nOY(ktEN*K!K_n}PyfsR9A` z`;tZ#bH?Bq+(hL+Bay47(T0bCeZrZJwti&jUk;;w>`&UO4i2A^ANHX(2LEgckmZid zzU&v0aZz47Z2IhPveMZ_|4=*mCifI4DDg_tRO07t3VyOe5`TH;&QZ0gMUEkR>7cG~ zm4KeP%#s{kWU2J4MQof9d=hf<=Qjnm9NNj2Aoc}|lm2)e0CyfV+GHr8zBji;m#XKrTC7*4EkH0>#s~ zXilCT@C04lALcg`q4qAzFWxpZ-j*xIoxwEkDPTh9nF)e=a;wqJrZWri^*2j;$b$<5 zmHW^x;}r2wYeK4V(cNABGQ-SK*LL6OZCw>p$)Z#+T4|X4i zP210c(536xa;P+$NYRA6U~_T-BlQ*!m2{XBCvm5~8i|{h9x?4(Y@+Rq*@2i=r0EV8 z&RQM`S@umi9f~4O%lolqn=HNqQo9@b6GM~%4esT=rMQO|TQc@+NrDHPPyg5w-$w&{ zx3Jr3z6Lt(rFS%SsJzVN$Zhv#sy6f(_ z0ml+K(U&jP48n%I|eF;yOYCVS#$o}&yWjerzEx?&6` zag~U@6-S#CdDm7FY#t09nbhinARAXmvbwj%8?cvAJaJbs_GJ2>bS3i__J^XLof|U2 zo5b~X3Z%~jE<(5$fw=2Gc1JXVP-}WbLwSwACYe!@|JD>PIZ%t{aP2q>EsyL9Zbp9 z3}?m&=G5lawL3^rlfso-1tNT&4{_jMK@aNEtB8^C150h*swD#=ogazuK^9xa3K9us$K?D|fk$QIii~Jh+`5$kd0m-c1paNhplCX2y&S*`bUAj~?}s zu(+D4SQ$uF{e+-%t{g5(JUHI2+LfBP3r@&JH5Hv^EkWVXZhJ4@o zRuKMWtBtUM+tH{Sdl@F0YO1*!B{Vi>`Qu}YX4L|%Ae+=E`}FVmK=(&??qkI5CLRL<6kgoL&+cV5uu)sLrB2u_1WRBIbAJKMaW#aQ~8q_S)*#E zp>JddGevqstn6-pPFIS>J6~6Q+zqG)gQp_Jjl!F+tRGuDW|;O`y_$UsGX3>oaf8;4 zopRD&y2gs_L-#Ot;g5XK)ICLFeXLv&u#SM&bo4uuS?O4;HPp`jnXm@wN88zsp zo+1A3;!7fD;<;#|KT$3Acfh_4TChyn-dSGRrdg5yw`W1Gr)th!_bslWGu0%NfzN=I!5&ip(iAv4c?H{b0v-v`vJCoRHCi-yNToT+HTa`Hup!ADXzBt-OS zmc5IYxG99VxYXt<-2~ea$;=>NnJKIX&9SIsfP6b{L@NZ%ObcELm_qgwZVD^(goh}d zq@lIkyfT~x6p4me&stEUF9#;Jy7*7IV*Mx$~di= z6!p6aZ#EWCblm_*y5)`hr^QFw%1{Th0~JdBh-dSwS*(;Yw)P>qba3H!%o(+U=ZQ2i zl5jWm*X?mt`#W)+qj~~G1)qJC&YGWxQx@1bkV|v(l&~QWtqeAzQsiUvd3@hSYWOlI zSsVxw>3y!zWwNx%tn58aswz&Aor$~l4YS){LGZcwF3%jd%#VbL$gz;9d3S%KrDwH( zy*l}U7u?HkBURdE_T*#=lI6x?~_A^q1UPN{e6y_4H1 zTXLf96`MU4G!BIN18kq!6um`6Y#SwnEDr8Ud70HQpq7k}NFDFgto*6HuAuz+rXT0& z?D)}E_S$#c^vftIdn?hz7vLlQ%k=PKE7ees4+-Jv0Y5vl$MaWWwiGdNCnWLWE_o=4+i#Li9pK|}n)Unxq|lDe zrmb7$t}6*Hkww@=jgnCwdp@9GkGA44XjAvShQW#_2`$iMeuc-rmpM}!&?1B3W~xn=N~A?3!czwxm>T_cl~(e zB4~Ge{3l2Cou$4&uSFBTBeY`jII6O&iWS((2ycJn3n1!YxEU}A*xTRy?Qr018CrmE zIHjW(O7xqquuah0=|HZU)4voOY;yX0vT2bssUIO1Zf+tb2xCTxE#CK-`<&;IeVgtIoRSs*3kg`^J}E{+Yu(NWSDexvQMJr; ztFSp-9>d5cMs8FkwKGqp|LWCTtgJzlJD{r>^KfM5Pez5)(7P774h_hoJ=eE}>J`%n z(T5+51)tP@9+#L{@EQNYwIz92@)4-v<86U0d{OZGel01gEfBM5EcoVx16HvLp4X}tiScZKOF6_@%o?ADd@BN%PCo4HLvuN%4vL;McneKqY2olC)U7BM6ldn%(f-3)qP$10#&AH_IADQ8vTmSYK&`mu=zdGPPtEq-|^HEa}L zc*d)oCJyb)b+eP4R^1KW%H)RGKUtS5$UQ27FD98pxf6oOPi^D7KlD=SE%ez85+JC~ z;+OSeI;3pLEdHka zj?8Vt1ax-r0wj0A=|d4`ct{)2QJk^kv3K-Hv6ytYIN+tV-gnc_-q3CILOEZbLrh3B z;=^&!vXeFYBUh)Rk*tdxt%Gap#Va3umEbXEfZkP0p7KJWhXH_K%l>Wf7jr}EDFPC_ zF^mrjb4fK#xMw|hi227X&F$iC8A{O<_xKsn>i{^|Qn=LJsA|GG?@a17%Co-uH-RdY zVkBSNR?SzXT|`LerP2Z;qmaHRgYhj5sVH{3cWlJ#SJY#Zr4pA=p?gKXE%X@A| zXm+C+>#wAd_b1fR&N)aF$~X7<1C49)mhlOW1q}3!Qzz!G3Ov%Q;yYH&pSK#D+z)I# zm*`Ur+v;cAGP3%DgIN7TU(2H<8xX{R&sd|Jgz%me36s0GDHDrg6~-2ANTHr*>imT= zlKe`p4|8ZNyG-F9AXzL zq72pYB#=&68CBpdCveDWQSCUd73NsVTBEW6>wqC(f`5KAARq<+o!Rx-vx&AZ7mdTK zECEq&ubrRCwqIMx?+}kGE3ToLB^#rYd(20o+(x`q9_{K5ti_RK*Jja4|9w*Y7a0`6 A#Q*>R diff --git a/docker/test/fixtures/secrets/broker_broker-ssl_server.truststore.jks b/docker/test/fixtures/secrets/broker_broker-ssl_server.truststore.jks deleted file mode 100644 index b526dd149726d5b5f14d458b9855af077c197cd2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1238 zcmV;{1S$J4f&|h60Ru3C1bhYwDuzgg_YDCD0ic2eZ3Kb@X)uBWWiWySVFn2*hDe6@ z4FLxRpn?QaFoFb50s#Opf&@nf2`Yw2hW8Bt2LUi<1_>&LNQU+thDZTr0|Wso1Q6)P*k=lo4{ribAXc|spM-#d1JG88u`XIkMH^{>lLExi+PFW^ zB?pbblmwhZl95UB&R59%mp?wOpl^w#7q&J%nb0|CPS8hqe$+89sZf3p`|@?MQggrY z6-TNTHFq}tQ4)^G7zuG;HGzvqC@!I48tfp4qd)UU(Kj6NScU}N?a>JNhW1c+A13Kl zQ;Gkn-)StuoBuk`kEidVAYIfab!|_{$!9`==yeKm56pr&aUOF59w1g@6CtZ+DLoPF zue;~{vjWt-o$4Z?bJHvPRbqi_>=-n=5#sPVYd%jPe$41HQsXhU+Sz21vR+BYQO*og z{6yh#t00ubb!pV@S`;iS(~urt!&#{-R) zY62Zj_OkN8`>Va1GLM*izh!bI!4gaCL_1FLPyQ*?6v9jUuFUufcLBhlS}(A@IlKKu zxMC+TOY`5ZsY9tu+anTqy{8^YXu^z+V$7P}i;;7K^m9cb@c>Ck;2sj0qlWW5CJTMM z)X3$Z@7WeGAxL$Q2f6>!EJlv>3mN6Af6F_-2QmB=+FecQu=LTp8ZHfZKUQWYn&MqLMb6rJ=V3r9OK=UbbQmJ{-3l~6Cid$41Y%|!$t z>wLkiAQWfy;H)DG$1dv?2aHy|pZDjDvTa-LS`@X?N0$H7Ic%{;1PWaE^gcvROeZtW zg#_N~lac)VV99}H)q52!ikbFeIFi^-g`H$?$5hFDX6FYs(*p+)ZPdjrK^#l>yt8%| zP@5^P#;gUlV-@??hv}0MAKcMiwYbt`1+OL^`n`@aJZHb1n> zCeuxysJaF7)iW};l{)45V;DO9Wx(`wo;=YX!8E~;v4pWK4GjEI-@(LBYo_2SD9Nb7 z4BU-#b_sGTJarFflL<1_@w>NC9O71OfpC00bb=VYHg|wC=Y_ z>K$^fKz=nohNEb?2Wq)Xg7;r0_z`ae6bHMGdOoJ>5YY_$M+66>H8dkc51?leY6p)gJl@gF{L6Me@ z>pl0L_q+Gk_x+gj%skJ`{Cv)tfuhJGfmqm36!{|@e7;cS&~sueT&x@vITr{;&i0pP zf}+3;|4#xJf>7YxzjWr`NW~%i-zg#jEMN`_Ec=&~g4+Bu0VakTL&g795<&Svly0Zr zvpr2A6+;g#xs5AXJKP1FP(UEh10WVFln4j!e>MW~!2l=|4*u&8WP>$fB?tu?m+m8Nn=89iSLpKUz&~ij?5t76%_(NzbzwB%YQVh zwdy;plk(XZ3hlNz4nt;Ffg&GXnr+t^nE0v7Y#f=U+1}1)1q1}CzM>PRuk#fiBV{e# zhUag3xxA%cEFH`0@Hraj==T0WH_J)mNF*;%5$j4z*(`c}UrdgWaJ?&0DcC7#0P!HH zWRJ)3PB9dGi-Z{-HuMCfXDZ>;p2P{G?-jj*y^ZcszuU^ge=(1Ifsh(6y-XFPqnO9) z(@8fwqwgu-S6n|EZYaYl#{8oG`07aXeD<@~S=^cR4kdxWO4LK?$HgR`W~7Ez5tt<9 zM12j%2o3Mo7RL%t^!8f|3x_fUqpN}|9X_S696t4`o|;rXXv+++RH`y4q0j>}$XPPq zFv-rLcGg+fTN_UeONYgD%g<8Rt~_MY1CBZ^hIYO7-iZrQNEgdGr^@tDslHm~>h}y+ zS#o3vY(cML=JaS5)Uv4BoL05WMKe1>;R{@biNRR75h*7C>FX>$c!^6ma2&~hWm_$( z$q+Th#(1f`+2bpWEO=b7ZZ%9O=ppW8KCwn1!Pm#$5^d-SN{}tHu8r%%V;r!^U#*~p zG=0nP6*P4$)vJgHj*jgJldsVEx;sAne0_B)gMQK&ZUx{QTvhV=VC`grNT4kkdPesy z_GJannJ59tL2$m=#5@F7!dxQTrln60DJZz(j7J5ZY6ko5U?KVHRim*^+KUpj_5 zDf@B#W@M1~8Ct$jjp!zqXz;*Z?$34_$BLcoH+9`L+et>9G?mMl_=Mhc_s&-GeBeio zO)jO!y#ZaZi7lENy=mQt>7TEy37$my%$)nSaY0aQSj6b|6`^iuW7xb|N`nai#*urn z*PX#^zq4meWwd1>WD|^gom7RuB$EXG&h2G%waS30{O%5_U3+x>ZdCJdFuOllU%gd| zsG39SY%rDy5w(YW?wnwSRGR$B<8cJH17Q}X-&fOq&MS9R$9cisk0+3k)*T&JUs0FY zhlgA^<+-;OYh2+E=^$l(swe&Y&uuSxs`nkH>!eijTJ?|l_n|Bvc5gBKHDIjqWT&_= zTRtI22#!&df#WOPCx@90{aFl4Xqq?c2X(ixQNXRa5t5tb_Y~JC1^3w5^Zp{!#b}Cc z`2-s-FjEcDj6gFbgl;vvtb5d4z*(1}IIHY#fPek_+;w4!Ws<(EPg^Du=dYUaGos=`nap8z#sUQs z-vdMzFGC2P<~ZYsarx&SI1oIilKgTLA8FEi32xXt<~H3I!~3Pw$cH1Iry&yw z4t{bqEye?yeKpfLy<#k-`i*v*$rh@TD6c&9rm>#(j%WJ*q4_txsjP1M6=`(dB?b0} zS{vANDBC}FiBAcmWF`k70L}nQfcamv`UJUij?gZWaLOV1S z;r9JtZn4=%_vw9R#!W5sfhfZ9G77)pGTWI?eO#qj0#WJ%f|8Hv9X+_{Dc`D$zlb6+ zu@a6oiI6C$Nd)p=LK1tKaK)dxw(340C!{AFRG@<(u~$z3&P2WxMg?k;l&~?=d?U$V zUwFyplyZ9RqwlLL8#M!G{ey_K3!lhycg4n$4GpYBeiECdRggWkKQ8xjRf;PQG`*z_ zwkR$2aSn$P2YB*a;Lt9-rXx5RPDUuhiJtnpX=|-0A`h7`HSEU|Txm<9J4^QBf^&xY za-w?&j%KLrPkSkIQg{eO-CB^R+q5#b=hF!p>GpWyb3!}*Q=Vh+pB8wpxf7{Wu}W_C z@m45Gybfy932i_7$)nQEd|rSa>DY8qrfn<8N5qoqCwOM%YdrvqCE1>CU}22E#nq2| zoW=8qhu56F-efSeNE2*v^B2FdSicTah&U?uTAZkp`N<7@iV+wH@=uAJAEE1w)!bU0i zelT!Xz2LsP5LXL^o2(@`H)*A*@ny=5thuko7d5ZxWLhLTl!V_Dzf9H~P7n}9ZyaG{ zQzyjqlqmCs{xOkSE6mX6fJs+6OL>2RqE9ea@ny=LtDcxnkqj6RHg3m7coY${z9ypvVN^eu_(b=_R6-cq$BA7;PbmY7$EISZ{25J2$^_3J?F!4neEZNLsgR^YjFV4@hSzP8Ka z8#rKBHkPWsd8xJ_4a0x8O~>(P%)=;GWh`${dylKAxV0I=-4GpW$*glhKYgYs<^AkJ z&nD{&%C3`OAX5I<6K6onx#;E_iHB_iJRjHdn?vQRR?2!FFG>q67S(jUlZ=pQ&2{h} zp00_GHl@C;Y+YCZ&_1%BGBi4Nk<3;$7#c^Ue&KXY*shLoxCT z!u1-o65PoQIA>smfmX@Y)I%?m%_+I?5L9V00O9h;-quPg)3h+p78 zV)fje#nw_ts3a|1Dj38B7+LYb7_r}kc4F^H7(J~V3CT#RIGuhvCZsX8nY^L2YJNZ# zN%VXf%FRWrYn|E;_5O3OuvsIdP|P^ehfI2r-pS7LRr`QMvqJ!sY_4hBh&_=O)Ke6^ zR<#?l=pmsn-<8KkU}#;({l0-65fyh3Z`+#UpqQB*CE=r$(v5d1hP)ty4T_qfl>uLv zs$(jJEY{MZsvL$9NaDsuPKU#nr=wB`a}57I`a^)5`bO|TgmT0UN@eSaTg)`nvG>Va zgd-9u8Q^>QX0?QOoQJYVt_t?d!8a@yu{_)JgZC#xAC0{%+qmw=sjS=>KRY`sOJOnM`Z=zwM2vrm8u(#)X zFQT71^M|CCP*x)WyLqPs>;AhG|ETsWD{x~iEW3>`YmEB z2Nm3oG8XZN+m1JGf^35JhDxdL>}hH)a+KcbGS_GLGT@`X`LKj7^`JYgv_k~flfK}e zI8`b>DEri({^2m2RzR{{x2-)Vpm8mz1wAg0g#)iCT*f6TI=EGKT1ak7SN2@}u3p(0 zgVx|>T{9t7zk47I+cuOIw06Y@Xob*A+J3_4C@JoryeUAObGx@Sa0`jrkPg4B2FJ3l zAjZ@`RXIs=cp6(sKJoVN8)%rR2qQ>6EpC##m|$K8ZvDBy@eQ1(cOna!wjcf93|6@b@TisdL^kpAU&r?6!s zay_TZX;JH?z?!jaZ06hDn&%Ue@U0XELisIxhH^5SYqJl8w$~<*X%&nMZ8PXf&BeWoB_|&m&B$+&k#Jc2akQp^Q2My+HP-v?ro~s2 zEl*}inG}~qzW$n>VPi-axYbrFoWj7RPxgLLrk<>(Sa(i4Ui5IC1SJIK{p1m13BR2# z$2(wd&*C6)-sPYOC5>qs@f~dPiR9>yfgUpW&D^CgC!&cvQ6@zxk`EX3$z#ta_Qy@OjEUf!g^w_!gS!n79dosCH%gGeE%PA#`PzJRXWv}|SuPcW)Asq2wIumCa z_;z^M^;_cZqOc9#*Jf?GaFgVV!%eD+lb9_w2)?J9Xqp5fmNjdPxM@P)iG^lVZ?kjO zqo*gA4dFHR0%5(z1&!3}Y?evg$CTzcO z%Gz95Ig@UG1o@*@#NmXf!9HQH0KgMjH z;FC~FRbGg8pBY%u=3`k%&*xL;AvLL5qFLK3+)~l3Jq$Gb3VWOsb5*) znG`sC+SiP;$18c~D2^*;&1a0V$n?B@4&slq*#2VUB7}rfPuSoV`)Ba#z;0pzwgpZ^ zFtmS3DJfC#@uD~;-z3~L1`m63+9>()@bhitI%Tvzf#Y>maXjkeL6tD}q(HHN%*UXp z*{#ek4KA)_E?4r!s{v<*&rj%|Fk(MsAT}(;GN9MleA6Xv!1d6mrXz3NxuP)L6b=Du z?x5M@ao zn4;J`%P-xiY-`WXde5@&zGRcsoo0oz6@?)Yq3+?{ot_pq@^aM??=tp47moNoNQzC- zBcXeH$OI`k$lC93O<8#8dDQ735PDy3PDUN-DyN(gb?3+J#f4R3YQ}$&P}nA#T$?FJ zR=OuBbqJrx;hZkic$fwt(@#{Hs><#-2YhS@r89hKDf3+h)$-P=yEMZ>K#G8kzvt&w z{O&4Kys1{J`Y^6jsFW{u{cjG{Cdl>0%E&950Qopr-{ec6z7=Rx!($!#731jI37;f& z90+=yvy~oh^D%kineRO&LNQUCc5!rUdCMV@CO0{ClCSwATSID2r7n1hW8Bu2?YQ! z9R>+thDZTr0|Wso1Q5jYAgfh1*n&#?ebGsJHYtFD1JGFzj_}5R)2_-ag-=_~ny9w}M8;}F zDKA=%k;NKXcVz33DvTJqV2-?frBKif~zW%qBuAh6lp?@njnlUH;-<%m@7iIV63WOQ8hiY zZ0#38umsN<)9g#{C(4<@`8M^hLu6q)QIoR{FrtvATf+#49?+MB|Lwmu!$nvjMyY-O zk@sgxe1d&w`Xi-O=77DD)!LT9{svF#!e`tue(yujIOBOHJYrxbEJsQ?D}_9FlI0iIeIJOSKK;Twz2W8 zg5S85RYijnsXD?@af8pubU+qrw+anAL~axgG*T$8RaCYFe^} z%o)5U$KOAGzYU@`QbY)*TCCIvw6-qb9bA`Ky))GpTgLi!G$bcLe?B+*#YEID~)LQ4I{G;o7 zs}A$G>35i-vq1j_why`uBe$NsrxA3JoZZnJ(CmE4qHc~6cnGUeJ zJfYcIc!#@{pT;)dzIp(F?mXWEC8sjoGb>h^s63!~lNkDWtWFg%C(w}Rk|BW@-pbTN zI9E~88JAM|@2N$p3GY;2whx-8Y!!E0q_b${Y-B;}=yAKF^&$#L;oY7dg9Js-jrx#) zDxUBOGN_{BkV1JYZIHpk@yS&ooA^_*k&MKF=r!=Tv-wCn0!$FupntmE;oN4cMe2L- z?hnOTGrapoa+p(Yhb`<-kR=^*4}^HUA;z}hT;BDVcnqlK*)%{B%E+9r`#esnNRFflL<1_@w>NC9O71OfpC00baQabg^ag;b(MyQlTa@T5qW7p#5~96&lno(> z&gwO*-`sQWdB1yqecz8cGxN+m^YfWGgA9h!0|^L`!IV89Qm$~7@ZS&u5&}#xWg}5A zW!)dR3K>k?@c&B0MMS~G`G4S?Kb8R^`(GA0m;i_gCYJdFqL8-#Fo+>YBc#~BQgS3W z5uA`<=iSaI+|Qpn91^w9;H9$eWC{dwPyz{9kmR5{|1%LtN(?|Uf=JE7RS0Ydi3qre zD8Jlla?S>EyGhg2GIXmgdJ_c`eWS7U_!%FstKy<*e=&#d63~S6q0`!_6)3{SU+sM8 z{LX<7N<+n|;v?;>Owp@L)p{@nB_=7BEG~_=FCpmFzg?!@n&H!>ENl>#R||}H1KYZ1 zjKZY|&g1oHB}whXyH{mEbx%TQJFje_`OL2zYGE7~oYn5GhTy(+?28)jum4K?0SY%+4X zYKU+1SFK|vnbV)Qdt2J5DUsSX%}hgHi~Cw9ptt;e)3fy~ zST`~Uz6aPg{F)^aVERfG7Du$5FtKT;sW!v(d}Z*Mu=}-UCf%vHG44Tieda4vOSUX+ zD-rudX9-hsslf(j@<4cZ`4mfHa~cxR<-}EEEoaS0#Zap(pHZQ@5&YzOC|9<8%`;Zv z$uKjs>=t7(hM4uM?aNp|o!`7Esi6be2E+5@)(|Z@e9q@g>F|vesGuXYKsV@rp8f{*V#pUhXwG1y#eb2=$iE9{bgi>qs z8fCC{Mbxx|UetA_(IR^o0vcrxX zdGq0|tNq%Sd&XsX{$|yew!0Wt`W}mHMuT~Z* zHqFOXk0WNT7q7KLqBo@hz9ukp$l0YhGrQIcSdM*#@%^S*RNhT6 zFx{Im9_BRnvJ{tS=JcWq-$N0laG2kolZ}?NLd;u7qFo@Iu3;w8h5Q|fd}C8!Ku1oB zX#;JLmUW5Yxf>U!J1v58zvZOFoxzNo7$|@%(s4SU-~>PynA1>CMM`enETSf1m{*0pO3D{#S+I---1xND#7-j zh;?&#&2bIG15Ot0KGB@OudRW>U~|~^Ng^}>6xYFV0I;aN*b$}&BPyIELfh96f>Er- z{uFmRH|m2VE-B8wzTub*B@-eA8LFS|ziN}AGJ3drcFGue())b9MruK)zfjiMZmpiS z)L-E*Ogvy-ePnC~$)wmRB-Uc_Sb=xOr{;OSjUL!Z#{KV>SQDgQLyom?#c2q2h0Lfh zag5#wpH*_4zaVesS0Gxyxez!ni46|$${&*Helu55q8r6SWp+oew}g&^D6EzahrNor zZ{fNJ)G5dgcd44B9QcB>(N8mZLRGGl8i8){kqc)PuD>?iy_`({TL(0`>hXBzZZ|Yn zUoV^0AayC?D>^smsK%blqg$aHiEaGVoWZ%HbZk-915 z>R0!i6XrTIF>W33mW&H~hgBWPO^P0J@HnI>U_pyui^eMFe?gmNV;WS z*3sxsGlke`bM-V@dvSlU0r#*ezU-3%Y5;09IK=4?(j2tA(`M`OU4F=U2=f_(z_Hp#V`{GSd?A1CH*&aMkP`9yy zyuQa;d|9$0TAyFS^IVulerppGCqKI?VZT zn~<&J&Ws+T|C3=DbD8BsS;^7UvR#c&S>uv{S7YmlqxOrQ#NG~-Hz+8$hfs7JS}()2 zo#-qKWgmoX0H_i7GE&pBTo2Nyb0%H+QZ&9rhYzs-O_Qyk{y-hQ^+9xjj3RWX7ayb* zv8uS@Q=+j;Jpw5731#{=9Q9O7xf{s{A8zm zW$Jl#B|Y{XXrnaHN3Va(c0oxN@!pS$!T;%xTmd6&@0|iF%BqvQ&{j6|V_6!lL9Zu~ zMbxa=1ehWXnl_8NvuLVO|1VQr&lAFV--nl9oTCPfzu;8Dj88GkKNip{5Cv4WG%e>LT7z9b#1Da z7tUkd)1M1oVq*qd;!lVxtixNKn?~JadrG4Iap*i}5hgsbuH6b3^2<#a%M_0rbY=h-5X4Ctn};1 zrL1HJb}i=egf)#r{vFkN=a2;Xrx){j#lfHl?wcx^9fDH{!VW*?yl8(c%ByX^J~rWK zHZ6Ycxf+w_YTElltz#?hH}9On2zfv}l^0vn)@)x_buNJu6XoorQPi%v3CkJDq=ndq zhGq7U*WSpEW*#i39QW7Wk&MScv) zw*WC~cDWGIra>KOyR0{@?lhAM382z zu&xKq<)`mb-F+vxrKi2><&Ll-jRDnxMXDt~FfWfiHVb&U+h@Ja>@->+9$pZ&n&12buM_;}(#3mEAL!=nH zAur~j8b|>o&LNQU+thDZTr0|Wso1Q6P#-ID$KntFh&I*VUrbi;sx17MWv7jr3h*nh0lMGhuY4o^29 z@Y$tU;vmV6U@Aoo`nii%ITe%x2B9$+n+0pVr!;SWnQL3wi}J2`Ves>mGtl50reI>8 zQoIa@tkx`gd7F0B9LLuFm|_+PZ6xQtp!dBXiV1e*I!P?UAnr-XfTps;$^ZA@e95~M z#6~`;_V1rSvqvk0cY8@yKg89TlqbuDdOnvILbJpX+>kV5)dD$%eOxZ%`xV_)Gh%hS zI1B(;MyianAp3Z{rEf`mylz_V4SzC8r(krK1g~Y0PouSZ?ou@{Mc&u_cGF7K>U2;N z46+Fi><%2!voa)&)Xl8`JC>LcU^!4E%=&@c)z@{&W-WjNvVQ~G983uO`O^l>C>`|v z^d%QqhP!RE4?b{6cL)_`Jc{tb75z3ns6ll-h^jaho_JkC_*}1wI%nR`@-I{|kv|e? z?SVDE@tU8W#S#pr?da3vlxRUHJD@PAy{T`an$_czlDnDD!D(eNDF7Cp(LEV`4-(Pk zP}&Se(j+3Q2`qhU-XJf!vjN6{FKzSQAdh#afc3zQYW+0tx>Tc}jG{MZR<6ood6lm7 z`%9|u)wX1&*kVzfrvyTdw1(2W>rW>Vm_7a`^vSD06tVE;6_KX7K$8{IrYhqvq4jMeO@I`eY)xC9J6o357PJ*W-T)F8yb2!5j!#)5P|FB@1yqtZO752@XgDIR48D(rT24<0%EOCIWDJm9k&??MfWDNC9O71OfpC00bb^Rm3+UnTTn#OHPE%<$O)N sHV-%uzF?3Jgj~ii%ky~z6v}D!W84x~O;Cx#kpePducmy3&H@4_5QdEUu>b%7 literal 0 HcmV?d00001 diff --git a/docker/test/fixtures/secrets/kafka01.keystore.jks b/docker/test/fixtures/secrets/kafka01.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..953c3355b6e6d0524ed8029abaf8572b93de7416 GIT binary patch literal 4382 zcma)AWmFUlvu1$>mfB@OM35GdUZlGlTm_`Nq)}=WB!#70LVD?1Vd)fUq&p;}LvjgG z@_Ns`=l$;e^?g6)%*-?M%#Zo;oH+vvrl7?I;K70+-2@;`q$=`)3_u9T4TjX=2SYG_ zVKgilSo{A1E77=MiDt|#Ss?BX;x=&bo*8l~$KE_~dq?McPUiDlwYv|LpB z`8+PlUoGc7xK;e~N)-WIx0Ea}dQU)k4nke?>63l5_|9+2h#ALMFRed=j5A~wiBxUF zVc>UHHI>lUlB2q9S85HXpCSv?ZIp&maEClSkk{+L8;7Z;*!cy4ae|T(Y{NkPv-d)I zfz#^Wq|m1}V7zjNaxi0ah1qVq+G6c#H9raoe26g-#LeHS6DibTXr`72H52ypDkpqr zJm4&}ae1_)DkjiYfn}DSQ%Qf*Z#Dnu(7Fq8n`?6JwBWMwlkOc6u+2cBfSJf=+R~Mh z=^e10+lsP|g5N)~c@Zi?WdwSd6|-^uGjmlc>$6;_W=2o{rFTGYm7*PlYqcf(G||-}SoWqaH}e z7d}`|6O~IB(&Ksh#Mo=Y-rGXi_D~w-BXiLw`{pn%UZO{gb9ToG>ZNlfYsPE4C`Ki9 z(P_{FCi51kDf#K;l_ZdBmV2f`3G0>=CniXIhI#!yl&lz zm_5|F1R^LXmvJT-V?eAbc|hXN?(A779^741*;tbbzK>om$CyyN@|&`9Xg~|9aK-%Q zJU`odP4Ht1$ix8p&%|)dw?f;izbcgPWu7hbwW&vBaU)^AT~Hg?9-pczHo*o1Cc09m5IhXL~K;cwzHf3f05~9X6LYv8*_c#i`kV26D&Z z&a1|aS%Lf2&#N=)uFNRy+CK zA0j=S=;qz(T*zLyhHfpVx!>TpPM#63aO{vf-xdLebDZx2}^ zqS3AIPx+JVb)i;e*(8Hka<1KF0}*1ba6N}XJCdW8MCBg1?xy+r91(6bCW(79 z%HANM7y0^5KxZ>Zzk?k`SmZWm@`=JEg3#`sDeoBM zWxv(v7TignSCM+=MSRF6)sbvkko&yx^bbdOQMJ^wYs4&fwGqAv>q-{d^fuk_(+uNS z3BNLo`5(CiQSnnTKyjRL+;FUMEdP4@zmW%y=l^ZF$4`Y1)pc;OW#ShW6BQB_6c!c{ zc={9;OpN_!4k1BqF!9=7Xc-rP^ViM(s{;6UV!gnmYpJ$zVl$XER8y8+d0~U2O8B3M z^%;nPtt{+7(6o`u$djdqGBB9fVvMLnt0Gm#vHqEL#m(u*wLt1pG{Q_q%I@`wM$gM)m0%p(S(?3_7@o_^s4~fy-YZ*W@Ztj(?q&{mY#DR3Ycn?Uvi{lLOy`5;us`CY{>V1~ z=88kxQaEL|0@-1x%(jPjT3_g)oyKcEzO(K8U%esan^*lCUsrFoL}?2y35^xBoAzRT zBYeRuC%L^*G>=6NZid_+qDE-v><wBg{f*GRn zvm4P8B&ev!z`eIaU3OEEfT)JjX{K==VU`qcdFV%k?vg##nUW#W!&($Xj1_1+N5RGI zU#Nsbkjx6Sa_SA-1e&H0ee6qPfdA2^sKskouV*hGMT=u}t3~8sU(;%6#+*sy9i zgoj8BNDV$XI1nRkD}WVl8c>zLe8;sd9yfl?Z{%!*KjCwC+Vv*0_S?q2$R~jDxjRbp&j63ERR)X#10-2QJTXV5Lw?m!E*{z)5dBJjjqx8L3GrqNb z?1kHRGoB%;C( z^??*G`g97NFs%|BJ>+jQ6L-#h*Nr5w_a}Wf{JuEXb-oVqRE-lkkn zk62V%gsYWaTe{bNZz$43>rSGK8-jtE`^inIy$O{|kmkP9i7U}SXbw+UA>BniDqzkm zwi~j-KJsjt?(~l3YfLoW&59(c*o#@0bD^|;&@3DzZ_uyi(dyI4)JIJs9O3YFSiEvN zQ^!tm*H7ZWV+6nc4Zrp^Z_pFHBgEPQUw&nR8D(sQmcB;~l|=nCfTnrv%P2V|Gcbd} zmfNU@O8M7;h%r~)X(G6`7tk<4tsHjM64ddGb&_9Tw85!er&BCy3;f52dEeLzh4=lC zX=f+q6cKvWuv2qif=+2=h3scy$3rrQpT+Hd<3{;3WsPe&P*1@b4-$?z?qQp#*TlO- z!PIt+yIU#~;f=3CNB`VD^EmfNrUYr1?Sdvh$J~;xwMavueu>OyY%Ky=7O&P_#zisfLMUi)*6=^29tVAAd4(0#ffHw7dLhxfokdz&LK7sAWz=epb*F zyexevC?Wd#DAd&S4B;CXnfKJtE-CD*>~O~$pV1he{q`hzAqGWD1k_AMV*YRjZy8E* zD4R_`%Ydaf+~VuWGZ~Z@nyvcyAr&v}#w*_?Q?Br^_V+_ykmIe)dc?NnbMv#%J}z#F zT77a_JH~C1*ye|WvTd}A-cyS%zs0L-iu9k+&!1o{tQ&eAn|}b`AmSRavxM{T7r(S8 z&U0_`xI53Bp^c0MqQotBf{j{_i>EXgFJondc^}~ta+V9-d3NPn>q!tl$75cgOJP8# z=c1Ch%x&ggE7;KN_NL@tgc~?Xlz_7kO{^Iry^Az&HYOtSTk^IYD=?0^EY{3kFwcrl z%Ml&ZQa$;vTr;{!GQkwO&M+&o-E2WzzHyw$R`^--&wB9rdqn0%y z5XJoJI_7zm&YEI#-Onk|L?(e{)+1xtH7hD4rE9&#>5`Du$-tvIEX zd>mJzV6!S(T{N>mhllnUZ?#h-vLFR}r9a?lZbOILUoe(*tf06rmV37a!;b>L0>8DZ zKGY)VR7gB^Y7}111uyT&?;w=uUwNOGWhf}C5Fiw(M~tO3rEod})j|Y4Wn7KGJh_`s?@&oX_P*z;uKht%8xok` zrmSBA3s3et(hR&Q-OFXA-xNiK9sUUy8O``nSj}V~H^(y;Mf6is7+LUUuSG%RSog`1 zf8FcWF3L3PioK#u?3HVlh%oASrb6B9XwH+8qjtUhZS!Yml_jOTh2{kXM)d5>y=E!f z?bh^QaaS6GbX=DrmAQ0{ctH$UKx>87aVu@5J_|J(+GYCKI*6WJLsAbZ)$3y&AUh@} z59u};1Nze{MVnT4W+e=VlNT5fyJwwW`VR@l-TB%P#Qt$u=Bz3(r0yh6_+a$m*4@DH z9mf$ZuNDXF?DkZ>etM0Y@#yHIB*jOCzy$zt@R?%1-bI|h4LJ{AaDaOE w1TbiNf|J`G2Pq#JlVoS}%Yg8y{4r~NEpfk9XGRIB445e6qbQc#vT59PAk#3L{DX9fST$XMD5v36c=@eKRq@+Ux z=?;nOJ@=mXyZ6`k{h0I2JkQMhe9oMKg;Cwd1LDKND7y&3{81`VXJo)Tz;|JkHJ~uc zs=ss@ER5jG|C0y`L16@Wf9dSMkxoeRzf+{dK)iQh1hRigB+Txg2?8>hF%15%k`x97 z(b)Sk^GDEiFQ7D1!=p9;_pD`tf8ycsP~rhuVWfma|FaPfOaOo}5rQqFRDiblARs@8 za&x*0Qi=6+>XbC+RGX3*0fm7^bb1HH4YP^!+rn+vEnbgA;92fAYEc~VFZljZ*sD-i z`nvXqNC1wjy)oH-ubvUHZhymTPe`m6d!DR1cI^5Lk+ zavjve>z7GjBe1EYUgMpd@TmxIYnW@bVmD{;5tEw0vK*7n<~9{|1M!+nHs$NexbqKe zG5SP7)mzX$Klt@ykA<-0g>==mh{DHq#_x;V?vEjUou$PhwNO`-AxuRaerDe@Wu>^d z2YW~*@Ha|yxC*#=lF5mLLc+rCzt)Zpl6SDY5&zZ508E#Sh%Wx)xz(l*~ch} zr$qy9kPy)e!w*28E_}3oT z{JSKvsc|f(cTrNKG7pdEYr!b|jTukF$6xg@b?_A?;29XkeKz7@?L1otCzRW68|fRY z^k^;aEX{Yx=#Cx)j6vHig%X&lfl`CU6B@!f)>`ek1 zV`ob`8hv(9ISo_ui`Ing92~<#1!Q+xjV=#|Vu?6~BwLagDL?tg7)-L=19_#1`+4v~ z*E$|H0IgGAaAIh0xBwNPJGB1zF*=XzDDI~pw-Z|JtUXP3^Z$f(shhHK@%GDO!DrqU zw5o7S*L7Y#>^Qnh@+m-j42^!9$KSw!CU3PJ?$Z$RmSnU_PpKFR5>S|eLU!#y)r&FByA4);ez z>BkLj1`Nau79il*Mjt04)quC3jpxtK+&*UV=X_+6@3xZ=LHNVEG1m6l6M}7uqucT} zi>WxUiE90wf$7zVRxbE0BuWX8g8^>AmfqTHOE+p3txv+SYW+Ct{LmgH^N0hzS0)@r zDiS&>5>rhySJT$GaI|r@UI8HVRTD{eAWf8Ii~0+(k>d=xmu&jB{ER)?rGW;4l#VRN zLm8aPe5o%P5n#0zMy?2W|Zx41BRkxoG5b@jBp3sTRLMif}sn zw916;*vW8ro5cOgC5@g@Ty={@OK<44;p+1XyzUkcD3eDUes}GI7m_jrPj2<0oXj_+ zAfKOS0j+*g+BcY#~tBaRe%F+cVi7YaYPv9KPJ{)}Mn zusnfT1@MJw#H>*=W*W3jxTRQtyNg_G=AQ<%_O_-ymwn;yyT30ls*z)^p0?|u#JyXe zCVNwVcmCvG6NrV%*n zErBK)+rA^>{V4Nk+_HyLGgJoSqcX-xEsR?&w^lSs=59G9Hp+rKCI&Iui$3oQa(7wvcx> z$5jc8eq?KUC%FIox3j0{F0#&?B?!AW$D-oLz?VkW|3g8VioM3+BIT7?-yAi4jujR1rMtppB&C$Iiq5#Iq6dh?0 z*BOpL!Cv9khrL0Qf!Y4NHBl?j*$#a=pOC9cCN&jcW_}#O^hjE;ea`dzpdea%TcN`X z;?w-YMQj(*iN|>71WT_<$d%FQM~x=DSoiK&u0R$(i(~lV-VwPe-Bm0=g-&HZ z>l;1>HF5why$kf<)XCCPjVM2K&6Xe-%sk)VGAYj!2DNzez9my2()fPVHt*V?%QNQO z&9RlX;tzh04O-l{->fR&uL&A<%g&3dhw3t5q65u?6HeUtX4IY_oT@nmUOwHMs0F$U z*d5X@+{2nU_)ZCXvRKx9@?P$7wx2h`(O8PvcRqh3;sf@z4<<@*E2<1+H7+4J=UcEt zyk}!YI8JFkGBzLg8=XI`5Q;g4QEJb0y>qu4xW9h13tC{(ay}D1VxY81?CLmr=vpwu z-;hQ&3v?5TR>e%dvkPM%&k|?5fd)N2@0o2cJ;5ZM47FCv|K>0wZ`(PlWrA{LB6XrG zmK3wJuIw1wcn3zs9A>b*M^jB~*$Y<$(&&Ei&-6-^zn*h%&82N^rQQDgz^>`DCu8Yi zc;d8EWZEt$0N|*-J2;5f$dTDYu=k1SP6>IzbYaX9A+%5oTTvwJ3}MeOoFp+I43EYe zK2MB6w{=yzKZ==doOg7|Vy;744VqvvsRAyj(_4aXUrJ0dK0G-V@82?b3uU-`UbR;; z?O45edEQ*=$1@iiExm5(y+1pO#Gn0+L>l()X}K1jVmuyhPdKOGdrDK zTio6O`i4>4(G8!vcwdZ?ZojdnjNV)P*S^B@C1sl4F}_5s`nDbFWVbjIGhDs{xmbZF zS2p_QTLgucP&~fMWJNx@yMs8XA~Df_I{VohU8W<^cL!;xNVSt}$WT$)TIZny(9a;`@Cr$fGjUO4&!rDQb16>O1MEp;W5Pc-PT{ahkEa&fuH)g$xT zdPP|-U^q}vbcO0P58sE!>0`a;d&OH=oimNqx0&rZ$`*Zk1!^|4BW1tSdV1?L;l~EP z_HVp@@9*JXPtNgBP%ulV@G{7sT~zjv8}dWmR~n1;8`!9pq~_gh^0U{zco-Sm)Y*HJ z*h2d4_1ihK9Ya3NBIjT?nx3^}5pgAg(NJsBV9l@=h0}EQ&L(smVcMj>@>efhhFwql%`cVKg ztNoZ;Wgd8`cM-5%7_;MhbOxmPraf>?;c0FlesXHy^SJD%cKb{cQoP)dWn!4(bSD3) z^p#{Ka-fVxe6&xb$iiGdg}ShEcbaEGO43~BLWB4cOxI6?v|eD2TZfYCUBotR{szTX z_Y?Z@)v7(_+uq7el?#w$FI=lOEwA*UQ{G5C-B;JobTO!(e?Oh$dT1W2&ZZF1FRO|# z%H5hG*3}W(f^P0;!7&8iWAbC*Vbz;{&y{Ve;5CsdgN0oE!kM?NC;|@Tx|?nt=}r&B z)mN)KEj>GBFCqtNf{2P|2<>>RIItdRS}eZCUN6~4?g|Q{93p9EY_zo(WWT)!f-9i50Yq1%8ej$y;ghR0VrgRrbp<1b-R7puO2$^(poe)m*PA zl}=euS1y~U=7hX9$ZeCm=4k$D`(|A{yJn>mCvbqML-?tc+xjSfp6rqS5$CZ)Ol5_| zIca!anBd;r(kWWdt-FduCmsk*9vP^3Dlg)(V_13x1SC$FpNLs7?>yC1(p(@-Dr-$@eUS)JHDf%Yd?IasMnK5u?S`JZ(V{Z)z(#o#mXS=k6s_FmR0 z<|Oxu+5Jnn?~+t?piwTF8~c+^ElnNoT;p3RTl3_Zjra=BNU>&&(?j0cAGm;iSm;F0 z!QyPD#;$Ifpu_Mb>Z_-$(wRXnS2_gO;?Z>-q{mqd@!XBvGNH)G#jTK_BFKEi7VY=H zTi;Agc^N3_EQNDN5(C;#QBTd>u84d2f0wWIZuHUER_C~7)@r#lU0$AbI&Qx@ZL$*D z$rgG~eX5WG5JUl5V{%WDsABOUhw<;ejK8>pH5ipCFo`CTa&Pi1)$H_f)Cg|g& z%k$qlyd&D2{>UC zq~yORMNOfoTcmh9?rhCDg5Cl-pvTOh*(#JSdvQ;FkVkgP#8y_T)sJ}Yua$^4h|Us? zkdqt0ah#+Lb);ftzEnQeF;$aCH~oVb7pzG-$r&lB4TwG^z+N6NApO@{EHl@$M!yKOTF zxC$s8OKR6=Z8h#?=zPxM-5Pd9ZpTn6C$7_-_-xxagsyQb!H1*N#&1fRF zmDli*lvRF_Uh@DES{4Y@ik25%CZtZ3f9c^B6Ax4ZLk1?)&FLY(Nc|ia2YM=BbQ0b! zKD2J&az6g*rQRN2l&A%maJDSc6sn{g(9Oog5TTyY2e&K0&M)cn&?tcLSNEjdh literal 0 HcmV?d00001 diff --git a/docker/test/fixtures/zookeeper/docker-compose.yml b/docker/test/fixtures/zookeeper/docker-compose.yml index 291ec7675b1af..9c8f0d9c61147 100644 --- a/docker/test/fixtures/zookeeper/docker-compose.yml +++ b/docker/test/fixtures/zookeeper/docker-compose.yml @@ -16,7 +16,6 @@ --- version: '2' services: - zookeeper-1: image: confluentinc/cp-zookeeper:latest ports: @@ -55,10 +54,10 @@ services: KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_SSL_KEYSTORE_FILENAME: "broker_broker_server.keystore.jks" + KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks" KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" - KAFKA_SSL_TRUSTSTORE_FILENAME: "broker_broker_server.truststore.jks" + KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" KAFKA_SSL_CLIENT_AUTH: "required" KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" @@ -82,10 +81,10 @@ services: KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker-ssl:19093,EXTERNAL://127.0.0.1:39093,SSL://localhost:9093,DOCKER://host.docker.internal:29093 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,SSL:SSL,DOCKER:PLAINTEXT,EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_SSL_KEYSTORE_FILENAME: "broker_broker-ssl_server.keystore.jks" + KAFKA_SSL_KEYSTORE_FILENAME: "kafka02.keystore.jks" KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" - KAFKA_SSL_TRUSTSTORE_FILENAME: "broker_broker-ssl_server.truststore.jks" + KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" KAFKA_SSL_CLIENT_AUTH: "required" KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" @@ -98,50 +97,3 @@ services: depends_on: - zookeeper-1 - zookeeper-2 - - schema-registry: - image: confluentinc/cp-schema-registry:latest - hostname: schema-registry - container_name: schema-registry - depends_on: - - zookeeper-1 - - zookeeper-2 - - broker - ports: - - "8081:8081" - environment: - SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' - SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 - - connect: - image: cnfldemos/cp-server-connect-datagen:0.6.2-7.5.0 - hostname: connect - container_name: connect - depends_on: - - zookeeper-1 - - zookeeper-2 - - broker - - schema-registry - ports: - - "8083:8083" - environment: - CONNECT_BOOTSTRAP_SERVERS: broker:29092 - CONNECT_REST_ADVERTISED_HOST_NAME: connect - CONNECT_GROUP_ID: compose-connect-group - CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 - CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter - CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081 - # CLASSPATH required due to CC-2422 - CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.4.1.jar - CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" - CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" - CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components" - CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR diff --git a/docker/test/requirements.txt b/docker/test/requirements.txt index 9bf7356f9bd38..401ad63aa6179 100644 --- a/docker/test/requirements.txt +++ b/docker/test/requirements.txt @@ -1,6 +1,3 @@ -confluent-kafka urllib3 requests -fastavro -jsonschema HTMLTestRunner-Python3 \ No newline at end of file From 72a9765adb58f1a08db7ac5e1dcef9daff505705 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 30 Oct 2023 09:40:30 +0530 Subject: [PATCH 11/66] Remove redundant files --- docker/test/fixtures/schema.avro | 8 -------- docker/test/fixtures/source_connector.json | 14 -------------- 2 files changed, 22 deletions(-) delete mode 100644 docker/test/fixtures/schema.avro delete mode 100644 docker/test/fixtures/source_connector.json diff --git a/docker/test/fixtures/schema.avro b/docker/test/fixtures/schema.avro deleted file mode 100644 index d85b0e1204773..0000000000000 --- a/docker/test/fixtures/schema.avro +++ /dev/null @@ -1,8 +0,0 @@ -{ - "type": "record", - "name": "Message", - "fields": [ - {"name": "key", "type": "string"}, - {"name": "value", "type": "string"} - ] -} \ No newline at end of file diff --git a/docker/test/fixtures/source_connector.json b/docker/test/fixtures/source_connector.json deleted file mode 100644 index 495bed2db8f1b..0000000000000 --- a/docker/test/fixtures/source_connector.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "datagen-users", - "config": { - "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector", - "kafka.topic": "test_topic_connect", - "quickstart": "users", - "key.converter": "org.apache.kafka.connect.storage.StringConverter", - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "value.converter.schemas.enable": "false", - "max.interval": 1000, - "iterations": 10, - "tasks.max": "1" - } -} \ No newline at end of file From 82775b36625dedba52073bbe647bc0d31f873514 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 30 Oct 2023 09:55:23 +0530 Subject: [PATCH 12/66] Fix tests Add SSL_TOPIC again in constants.py --- docker/test/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/constants.py b/docker/test/constants.py index 621aa8f41e84b..0a55731af90e8 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -28,6 +28,7 @@ CONNECT_SOURCE_CONNECTOR_CONFIG="@fixtures/source_connector.json" SSL_CLIENT_CONFIG="./fixtures/secrets/client-ssl.properties" +SSL_TOPIC="test_topic_ssl" BROKER_RESTART_TEST_TOPIC="test_topic_broker_restart" From 1c0d3c512c5462e6c4039dad690c9fc6d7bb70d1 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 30 Oct 2023 14:59:56 +0530 Subject: [PATCH 13/66] Use CDS to start kafka --- docker/jvm/Dockerfile | 40 +++++++++++++++++++++++++++++++--------- docker/jvm/jsa_launch | 15 +++++++++++++++ docker/jvm/launch | 12 +++++++++--- 3 files changed, 55 insertions(+), 12 deletions(-) create mode 100755 docker/jvm/jsa_launch diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index e9eea40877690..44010e026f587 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -16,9 +16,7 @@ # limitations under the License. ############################################################################### -ARG GOLANG_VERSION=1.21.1 - -FROM golang:${GOLANG_VERSION} AS build-ub +FROM golang:latest AS build-ub WORKDIR /build RUN useradd --no-log-init --create-home --shell /bin/bash appuser COPY --chown=appuser:appuser resources/ub/ ./ @@ -27,7 +25,29 @@ USER appuser RUN go test ./... -FROM eclipse-temurin:21-jre +FROM eclipse-temurin:21-jre-alpine AS build-jsa + +USER root + +# Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments +ARG kafka_url + +ENV KAFKA_URL=$kafka_url + +COPY jsa_launch /etc/kafka/docker/jsa_launch + +RUN set -eux ; \ + apk update ; \ + apk upgrade ; \ + apk add --no-cache wget gcompat; \ + mkdir opt/kafka; \ + wget -nv -O kafka.tgz "$KAFKA_URL"; \ + tar xfz kafka.tgz -C /opt/kafka --strip-components 1; + +RUN /etc/kafka/docker/jsa_launch + + +FROM eclipse-temurin:21-jre-alpine # exposed ports EXPOSE 9092 @@ -49,9 +69,9 @@ LABEL org.label-schema.name="kafka" \ ENV KAFKA_URL=$kafka_url RUN set -eux ; \ - apt-get update ; \ - apt-get upgrade -y ; \ - apt-get install -y --no-install-recommends curl wget gpg dirmngr gpg-agent; \ + apk update ; \ + apk upgrade ; \ + apk add --no-cache curl wget gpg dirmngr gpg-agent gcompat; \ mkdir opt/kafka; \ wget -nv -O kafka.tgz "$KAFKA_URL"; \ wget -nv -O kafka.tgz.asc "$KAFKA_URL.asc"; \ @@ -61,14 +81,16 @@ RUN set -eux ; \ gpg --batch --verify kafka.tgz.asc kafka.tgz; \ mkdir -p /var/lib/kafka/data /etc/kafka/secrets /var/log/kafka /var/lib/zookeeper; \ mkdir -p /etc/kafka/docker /usr/logs; \ - useradd --no-log-init --create-home --shell /bin/bash appuser; \ + adduser -h /home/appuser -D --shell /bin/bash appuser; \ chown appuser:appuser -R /etc/kafka/ /usr/logs /opt/kafka; \ chown appuser:root -R /etc/kafka /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka /var/lib/zookeeper; \ chmod -R ug+w /etc/kafka /var/lib/kafka /var/lib/kafka /etc/kafka/secrets /etc/kafka /var/log/kafka /var/lib/zookeeper; \ rm kafka.tgz; \ + apk cache clean; \ rm kafka.tgz.asc; -COPY --from=build-ub /build/ub /usr/bin +COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa +COPY --chown=appuser:appuser --from=build-ub /build/ub /usr/bin COPY --chown=appuser:appuser resources/common-scripts /etc/kafka/docker COPY --chown=appuser:appuser launch /etc/kafka/docker/launch diff --git a/docker/jvm/jsa_launch b/docker/jvm/jsa_launch new file mode 100755 index 0000000000000..eced8a568792b --- /dev/null +++ b/docker/jvm/jsa_launch @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +KAFKA_CLUSTER_ID="$(opt/kafka/bin/kafka-storage.sh random-uuid)" +opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c opt/kafka/config/kraft/server.properties +KAFKA_JVM_PERFORMANCE_OPTS="-XX:ArchiveClassesAtExit=kafka.jsa" opt/kafka/bin/kafka-server-start.sh opt/kafka/config/kraft/server.properties & +PIDS=$! + +sleep 10 +echo "test" | opt/kafka/bin/kafka-console-producer.sh --topic test-topic --bootstrap-server localhost:9092 +sleep 5 +echo $(opt/kafka/bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server localhost:9092 --max-messages 1) +sleep 5 + +kill -s TERM $PIDS +sleep 10 \ No newline at end of file diff --git a/docker/jvm/launch b/docker/jvm/launch index 778313c7ed349..33b8333c756ed 100755 --- a/docker/jvm/launch +++ b/docker/jvm/launch @@ -20,7 +20,7 @@ # Override this section from the script to include the com.sun.management.jmxremote.rmi.port property. if [ -z "$KAFKA_JMX_OPTS" ]; then - export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " + export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " fi # The JMX client needs to be able to connect to java.rmi.server.hostname. @@ -33,8 +33,8 @@ export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)} if [ "$KAFKA_JMX_PORT" ]; then # This ensures that the "if" section for JMX_PORT in kafka launch script does not trigger. - export JMX_PORT=$KAFKA_JMX_PORT - export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" + export JMX_PORT=$KAFKA_JMX_PORT + export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" fi # KRaft required step: Format the storage directory with provided cluster ID unless it already exists. @@ -48,5 +48,11 @@ then { echo $result && (exit 1) } fi +if [ -z "$KAFKA_JVM_PERFORMANCE_OPTS" ]; then + export KAFKA_JVM_PERFORMANCE_OPTS="-XX:SharedArchiveFile=/opt/kafka/kafka.jsa" +else + export KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS -XX:SharedArchiveFile=/opt/kafka/kafka.jsa" +fi + # Start kafka broker exec /opt/kafka/bin/kafka-server-start.sh /etc/kafka/kafka.properties From a9faadb3c2aadff2a4b1436d8a81d495fddf19cc Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 30 Oct 2023 20:47:05 +0530 Subject: [PATCH 14/66] Resolve PR comments - Remove zookeeper support & tests - Optimise image by removing redundant packages - Minor code fixes --- docker/jvm/Dockerfile | 12 +-- docker/jvm/jsa_launch | 4 +- docker/resources/common-scripts/configure | 5 +- .../common-scripts/kafka-propertiesSpec.json | 7 +- docker/resources/common-scripts/run | 3 - docker/test/docker_sanity_test.py | 12 +-- .../fixtures/zookeeper/docker-compose.yml | 99 ------------------- 7 files changed, 13 insertions(+), 129 deletions(-) delete mode 100644 docker/test/fixtures/zookeeper/docker-compose.yml diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 44010e026f587..b2044d1e7d5bf 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -39,7 +39,7 @@ COPY jsa_launch /etc/kafka/docker/jsa_launch RUN set -eux ; \ apk update ; \ apk upgrade ; \ - apk add --no-cache wget gcompat; \ + apk add --no-cache wget gcompat procps; \ mkdir opt/kafka; \ wget -nv -O kafka.tgz "$KAFKA_URL"; \ tar xfz kafka.tgz -C /opt/kafka --strip-components 1; @@ -82,12 +82,12 @@ RUN set -eux ; \ mkdir -p /var/lib/kafka/data /etc/kafka/secrets /var/log/kafka /var/lib/zookeeper; \ mkdir -p /etc/kafka/docker /usr/logs; \ adduser -h /home/appuser -D --shell /bin/bash appuser; \ - chown appuser:appuser -R /etc/kafka/ /usr/logs /opt/kafka; \ - chown appuser:root -R /etc/kafka /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka /var/lib/zookeeper; \ + chown appuser:appuser -R /usr/logs /opt/kafka; \ + chown appuser:root -R /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka /var/lib/zookeeper; \ chmod -R ug+w /etc/kafka /var/lib/kafka /var/lib/kafka /etc/kafka/secrets /etc/kafka /var/log/kafka /var/lib/zookeeper; \ - rm kafka.tgz; \ - apk cache clean; \ - rm kafka.tgz.asc; + rm kafka.tgz kafka.tgz.asc KEYS; \ + apk del curl wget gpg dirmngr gpg-agent; \ + apk cache clean; COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa COPY --chown=appuser:appuser --from=build-ub /build/ub /usr/bin diff --git a/docker/jvm/jsa_launch b/docker/jvm/jsa_launch index eced8a568792b..e9b24a7cbdf21 100755 --- a/docker/jvm/jsa_launch +++ b/docker/jvm/jsa_launch @@ -11,5 +11,5 @@ sleep 5 echo $(opt/kafka/bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server localhost:9092 --max-messages 1) sleep 5 -kill -s TERM $PIDS -sleep 10 \ No newline at end of file +opt/kafka/bin/kafka-server-stop.sh +sleep 5 \ No newline at end of file diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index b05d226a75913..7819c2dce127a 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -36,9 +36,8 @@ then ub ensure KAFKA_ADVERTISED_LISTENERS fi else - echo "Running in Zookeeper mode..." - ub ensure KAFKA_ZOOKEEPER_CONNECT - ub ensure KAFKA_ADVERTISED_LISTENERS + echo "Only KRaft mode is supported KAFKA_PROCESS_ROLES is a mandatory config" + exit 1 fi # By default, LISTENERS is derived from ADVERTISED_LISTENERS by replacing diff --git a/docker/resources/common-scripts/kafka-propertiesSpec.json b/docker/resources/common-scripts/kafka-propertiesSpec.json index 0e10d92d6473e..b67200c5fb305 100644 --- a/docker/resources/common-scripts/kafka-propertiesSpec.json +++ b/docker/resources/common-scripts/kafka-propertiesSpec.json @@ -1,10 +1,8 @@ { "prefixes": { - "KAFKA": false, - "CONFLUENT": true + "KAFKA": false }, "renamed": { - "KAFKA_ZOOKEEPER_CLIENT_CNXN_SOCKET": "zookeeper.clientCnxnSocket" }, "excludes": [ "KAFKA_VERSION", @@ -16,8 +14,7 @@ "KAFKA_GC_LOG_OPTS", "KAFKA_LOG4J_ROOT_LOGLEVEL", "KAFKA_LOG4J_LOGGERS", - "KAFKA_TOOLS_LOG4J_LOGLEVEL", - "KAFKA_ZOOKEEPER_CLIENT_CNXN_SOCKET" + "KAFKA_TOOLS_LOG4J_LOGLEVEL" ], "defaults": { }, diff --git a/docker/resources/common-scripts/run b/docker/resources/common-scripts/run index 41273ea5b7a59..4ff47c85558b9 100755 --- a/docker/resources/common-scripts/run +++ b/docker/resources/common-scripts/run @@ -28,11 +28,8 @@ fi echo "===> User" id -if [[ -z "${KAFKA_ZOOKEEPER_CONNECT-}" ]] -then echo "===> Setting default values of environment variables if not already set." . /etc/kafka/docker/configureDefaults -fi echo "===> Configuring ..." /etc/kafka/docker/configure diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 1022365cac8ac..12652112e139b 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -77,13 +77,11 @@ def create_topic(self, topic, topic_config): def produce_message(self, topic, producer_config, key, value): command = ["echo", f'"{key}:{value}"', "|", constants.KAFKA_CONSOLE_PRODUCER, "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] command.extend(producer_config) - print(" ".join(command)) subprocess.run(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) def consume_message(self, topic, consumer_config): command = [constants.KAFKA_CONSOLE_CONSUMER, "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] command.extend(consumer_config) - print(" ".join(command)) message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip() @@ -168,14 +166,6 @@ def tearDown(self) -> None: def test_bed(self): self.execute() -class DockerSanityTestZookeeper(DockerSanityTestCommon): - def setUp(self) -> None: - self.startCompose(constants.ZOOKEEPER_COMPOSE) - def tearDown(self) -> None: - self.destroyCompose(constants.ZOOKEEPER_COMPOSE) - def test_bed(self): - self.execute() - if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("image") @@ -186,7 +176,7 @@ def test_bed(self): test_classes_to_run = [] if args.mode in ("all", "jvm"): - test_classes_to_run.extend([DockerSanityTestKraftMode, DockerSanityTestZookeeper]) + test_classes_to_run.extend([DockerSanityTestKraftMode]) loader = unittest.TestLoader() suites_list = [] diff --git a/docker/test/fixtures/zookeeper/docker-compose.yml b/docker/test/fixtures/zookeeper/docker-compose.yml deleted file mode 100644 index 9c8f0d9c61147..0000000000000 --- a/docker/test/fixtures/zookeeper/docker-compose.yml +++ /dev/null @@ -1,99 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -version: '2' -services: - zookeeper-1: - image: confluentinc/cp-zookeeper:latest - ports: - - "2181:2181" - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_SERVER_ID: 1 - ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 - - zookeeper-2: - image: confluentinc/cp-zookeeper:latest - ports: - - "2182:2182" - environment: - ZOOKEEPER_CLIENT_PORT: 2182 - ZOOKEEPER_SERVER_ID: 2 - ZOOKEEPER_SERVERS: zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888 - - broker: - image: {$IMAGE} - hostname: broker - container_name: broker - ports: - - "9092:9092" - - "29092:29092" - - "39092:39092" - volumes: - - ../secrets:/etc/kafka/secrets - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker:19092,EXTERNAL://localhost:9092,SSL://localhost:39092,DOCKER://host.docker.internal:29092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT,SSL:SSL - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks" - KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" - KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" - KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" - KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" - KAFKA_SSL_CLIENT_AUTH: "required" - KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - depends_on: - - zookeeper-1 - - zookeeper-2 - - broker-ssl: - image: {$IMAGE} - hostname: broker-ssl - container_name: broker-ssl - ports: - - "29093:29093" - - "9093:9093" - - "39093:39093" - volumes: - - ../secrets:/etc/kafka/secrets - environment: - KAFKA_BROKER_ID: 2 - KAFKA_ADVERTISED_LISTENERS: INTERNAL://broker-ssl:19093,EXTERNAL://127.0.0.1:39093,SSL://localhost:9093,DOCKER://host.docker.internal:29093 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,SSL:SSL,DOCKER:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_SSL_KEYSTORE_FILENAME: "kafka02.keystore.jks" - KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" - KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" - KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" - KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" - KAFKA_SSL_CLIENT_AUTH: "required" - KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - KAFKA_ZOOKEEPER_CONNECT: "zookeeper-1:2181,zookeeper-2:2181" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - depends_on: - - zookeeper-1 - - zookeeper-2 From 6289c19984b787cc034ddb21aadc8d366e0e7064 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 2 Nov 2023 09:44:43 +0530 Subject: [PATCH 15/66] Add github actions workflow for build and test of jvm docker image --- .github/workflows/docker_build_and_test.yml | 47 +++++++++++++++++++++ docker/docker_build_test.py | 7 ++- docker/test/docker_sanity_test.py | 13 +++--- 3 files changed, 57 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/docker_build_and_test.yml diff --git a/.github/workflows/docker_build_and_test.yml b/.github/workflows/docker_build_and_test.yml new file mode 100644 index 0000000000000..0cbc2881d7e2a --- /dev/null +++ b/.github/workflows/docker_build_and_test.yml @@ -0,0 +1,47 @@ +name: Docker build test + +on: + workflow_dispatch: + inputs: + image_type: + type: choice + description: Docker image type to build and test + options: + - "jvm" + kafka_url: + required: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r docker/test/requirements.txt + - name: Run tests + working-directory: ./docker + run: | + python docker_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -u=${{ github.event.inputs.kafka_url }} + - name: Run Vulnerability scan + uses: aquasecurity/trivy-action@master + with: + image-ref: 'kafka/test:test' + format: 'table' + ignore-unfixed: true + vuln-type: 'os,library' + severity: 'CRITICAL,HIGH' + output: scan_${{ github.event.inputs.image_type }}.txt + - uses: actions/upload-artifact@v3 + with: + name: report_${{ github.event.inputs.image_type }}.html + path: docker/test/report_${{ github.event.inputs.image_type }}.html + - uses: actions/upload-artifact@v3 + with: + name: scan_${{ github.event.inputs.image_type }}.txt + path: scan_${{ github.event.inputs.image_type }}.txt \ No newline at end of file diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index a841e6f0ea548..5cefa723a2f4f 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -33,7 +33,6 @@ def build_jvm(image, tag, kafka_url): def run_jvm_tests(image, tag, kafka_url): subprocess.run(["wget", "-nv", "-O", "kafka.tgz", kafka_url]) - subprocess.run(["ls"]) subprocess.run(["mkdir", "./test/fixtures/kafka"]) subprocess.run(["tar", "xfz", "kafka.tgz", "-C", "./test/fixtures/kafka", "--strip-components", "1"]) subprocess.run(["python3", "docker_sanity_test.py", f"{image}:{tag}", "jvm"], cwd="test") @@ -44,17 +43,17 @@ def run_jvm_tests(image, tag, kafka_url): parser = argparse.ArgumentParser() parser.add_argument("image", help="Image name that you want to keep for the Docker image") parser.add_argument("-tag", "--image-tag", default="latest", dest="tag", help="Image tag that you want to add to the image") - parser.add_argument("-type", "--image-type", default="all", dest="image_type", help="Image type you want to build. By default it's all") + parser.add_argument("-type", "--image-type", choices=["jvm"], dest="image_type", help="Image type you want to build") parser.add_argument("-u", "--kafka-url", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image") parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, help="Only build the image, don't run tests") parser.add_argument("-t", "--test", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image") args = parser.parse_args() - if args.image_type in ("all", "jvm") and (args.build_only or not (args.build_only or args.test_only)): + if args.image_type == "jvm" and (args.build_only or not (args.build_only or args.test_only)): if args.kafka_url: build_jvm(args.image, args.tag, args.kafka_url) else: raise ValueError("--kafka-url is a required argument for jvm image") - if args.image_type in ("all", "jvm") and (args.test_only or not (args.build_only or args.test_only)): + if args.image_type == "jvm" and (args.test_only or not (args.build_only or args.test_only)): run_jvm_tests(args.image, args.tag, args.kafka_url) \ No newline at end of file diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 12652112e139b..d891af6391a8e 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -19,8 +19,9 @@ from HTMLTestRunner import HTMLTestRunner import constants import argparse +import socket -class DockerSanityTestCommon(unittest.TestCase): +class DockerSanityTest(unittest.TestCase): CONTAINER_NAME="broker" IMAGE="apache/kafka" @@ -158,7 +159,7 @@ def execute(self): self.assertEqual(total_errors, []) -class DockerSanityTestKraftMode(DockerSanityTestCommon): +class DockerSanityTestKraftMode(DockerSanityTest): def setUp(self) -> None: self.startCompose(constants.KRAFT_COMPOSE) def tearDown(self) -> None: @@ -172,11 +173,11 @@ def test_bed(self): parser.add_argument("mode", default="all") args = parser.parse_args() - DockerSanityTestCommon.IMAGE = args.image + DockerSanityTest.IMAGE = args.image test_classes_to_run = [] - if args.mode in ("all", "jvm"): - test_classes_to_run.extend([DockerSanityTestKraftMode]) + if args.mode == "jvm": + test_classes_to_run = [DockerSanityTestKraftMode] loader = unittest.TestLoader() suites_list = [] @@ -184,7 +185,7 @@ def test_bed(self): suite = loader.loadTestsFromTestCase(test_class) suites_list.append(suite) big_suite = unittest.TestSuite(suites_list) - outfile = open(f"report.html", "w") + outfile = open(f"report_{args.mode}.html", "w") runner = HTMLTestRunner.HTMLTestRunner( stream=outfile, title='Test Report', From c018f54089fbffa60ab56646ac68c94f83c34006 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 2 Nov 2023 09:54:09 +0530 Subject: [PATCH 16/66] Add description to kafka url link --- .github/workflows/docker_build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker_build_and_test.yml b/.github/workflows/docker_build_and_test.yml index 0cbc2881d7e2a..82a9a81a19ed2 100644 --- a/.github/workflows/docker_build_and_test.yml +++ b/.github/workflows/docker_build_and_test.yml @@ -9,6 +9,7 @@ on: options: - "jvm" kafka_url: + description: Kafka url to be used to build the docker image required: true jobs: From 8df9b59f8f535d1091c1bbd51934dab5fc9989d8 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 2 Nov 2023 12:38:44 +0530 Subject: [PATCH 17/66] Refactors jsa launch script with error handling and timeouts --- docker/jvm/Dockerfile | 2 +- docker/jvm/jsa_launch | 33 +++++++++++++++++++++++++++------ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index b2044d1e7d5bf..7cbfe38455cc0 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -39,7 +39,7 @@ COPY jsa_launch /etc/kafka/docker/jsa_launch RUN set -eux ; \ apk update ; \ apk upgrade ; \ - apk add --no-cache wget gcompat procps; \ + apk add --no-cache wget gcompat procps netcat-openbsd; \ mkdir opt/kafka; \ wget -nv -O kafka.tgz "$KAFKA_URL"; \ tar xfz kafka.tgz -C /opt/kafka --strip-components 1; diff --git a/docker/jvm/jsa_launch b/docker/jvm/jsa_launch index e9b24a7cbdf21..4962f97937832 100755 --- a/docker/jvm/jsa_launch +++ b/docker/jvm/jsa_launch @@ -3,13 +3,34 @@ KAFKA_CLUSTER_ID="$(opt/kafka/bin/kafka-storage.sh random-uuid)" opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c opt/kafka/config/kraft/server.properties KAFKA_JVM_PERFORMANCE_OPTS="-XX:ArchiveClassesAtExit=kafka.jsa" opt/kafka/bin/kafka-server-start.sh opt/kafka/config/kraft/server.properties & -PIDS=$! -sleep 10 +check_timeout() { + if [ $TIMEOUT -eq 0 ]; then + echo "Server startup timed out" + exit 1 + fi + echo "Check will timeout in $(( TIMEOUT-- )) seconds" + sleep 1 +} + +TIMEOUT=20 +while ! nc -z localhost 9092; do + check_timeout +done + +opt/kafka/bin/kafka-topics.sh --create --topic test-topic --bootstrap-server localhost:9092 +[ $? -eq 0 ] || exit 1 + echo "test" | opt/kafka/bin/kafka-console-producer.sh --topic test-topic --bootstrap-server localhost:9092 -sleep 5 -echo $(opt/kafka/bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server localhost:9092 --max-messages 1) -sleep 5 +[ $? -eq 0 ] || exit 1 + +opt/kafka/bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server localhost:9092 --max-messages 1 +[ $? -eq 0 ] || exit 1 opt/kafka/bin/kafka-server-stop.sh -sleep 5 \ No newline at end of file + +TIMEOUT=20 +until [ -f /kafka.jsa ] +do + check_timeout +done \ No newline at end of file From ee53281078c3448caa99b1f91a1dc420fff08f03 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 2 Nov 2023 16:19:18 +0530 Subject: [PATCH 18/66] Remove static sleep from sanity tests --- docker/test/constants.py | 14 ++---- docker/test/docker_sanity_test.py | 76 +++++++++++++++++++++---------- 2 files changed, 57 insertions(+), 33 deletions(-) diff --git a/docker/test/constants.py b/docker/test/constants.py index 0a55731af90e8..ace6ff9d66efa 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -20,19 +20,15 @@ KRAFT_COMPOSE="fixtures/kraft/docker-compose.yml" ZOOKEEPER_COMPOSE="fixtures/zookeeper/docker-compose.yml" -SCHEMA_REGISTRY_URL="http://localhost:8081" -CONNECT_URL="http://localhost:8083/connectors" CLIENT_TIMEOUT=40 -SCHEMA_REGISTRY_TEST_TOPIC="test_topic_schema" -CONNECT_TEST_TOPIC="test_topic_connect" -CONNECT_SOURCE_CONNECTOR_CONFIG="@fixtures/source_connector.json" +SSL_FLOW_TESTS="SSL Flow Tests" SSL_CLIENT_CONFIG="./fixtures/secrets/client-ssl.properties" -SSL_TOPIC="test_topic_ssl" +SSL_TOPIC="test-topic-ssl" -BROKER_RESTART_TEST_TOPIC="test_topic_broker_restart" +BROKER_RESTART_TESTS="Broker Restart Tests" +BROKER_CONTAINER="broker" +BROKER_RESTART_TEST_TOPIC="test-topic-broker-restart" -SCHEMA_REGISTRY_ERROR_PREFIX="SCHEMA_REGISTRY_ERR" -CONNECT_ERROR_PREFIX="CONNECT_ERR" SSL_ERROR_PREFIX="SSL_ERR" BROKER_RESTART_ERROR_PREFIX="BROKER_RESTART_ERR" \ No newline at end of file diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index d891af6391a8e..49655998c6c09 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -22,45 +22,32 @@ import socket class DockerSanityTest(unittest.TestCase): - CONTAINER_NAME="broker" IMAGE="apache/kafka" def resumeImage(self): - subprocess.run(["docker", "start", self.CONTAINER_NAME]) + subprocess.run(["docker", "start", constants.BROKER_CONTAINER]) def stopImage(self) -> None: - subprocess.run(["docker", "stop", self.CONTAINER_NAME]) + subprocess.run(["docker", "stop", constants.BROKER_CONTAINER]) def startCompose(self, filename) -> None: old_string="image: {$IMAGE}" new_string=f"image: {self.IMAGE}" - with open(filename) as f: s = f.read() - if old_string not in s: - print('"{old_string}" not found in {filename}.'.format(**locals())) - with open(filename, 'w') as f: - print('Changing "{old_string}" to "{new_string}" in {filename}'.format(**locals())) s = s.replace(old_string, new_string) f.write(s) subprocess.run(["docker-compose", "-f", filename, "up", "-d"]) - time.sleep(25) def destroyCompose(self, filename) -> None: old_string=f"image: {self.IMAGE}" new_string="image: {$IMAGE}" - subprocess.run(["docker-compose", "-f", filename, "down"]) - time.sleep(10) with open(filename) as f: s = f.read() - if old_string not in s: - print('"{old_string}" not found in {filename}.'.format(**locals())) - with open(filename, 'w') as f: - print('Changing "{old_string}" to "{new_string}" in {filename}'.format(**locals())) s = s.replace(old_string, new_string) f.write(s) @@ -85,13 +72,35 @@ def consume_message(self, topic, consumer_config): command.extend(consumer_config) message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip() + + def wait_for_port(self, host, port, open, timeout): + start_time = time.perf_counter() + while time.perf_counter() - start_time < timeout: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + status = sock.connect_ex((host, port)) + sock.close() + if (open and status == 0) or (not open and status != 0): + return + else: + time.sleep(1) + raise TimeoutError("Timed out while waiting for the port", host, port) def ssl_flow(self): - print("Running SSL flow tests") + print(f"Running {constants.SSL_FLOW_TESTS}") errors = [] + try: + self.wait_for_port('localhost', 9093, True, constants.CLIENT_TIMEOUT) + except e: + errors.append(str(e)) + return errors + try: + self.assertTrue(self.create_topic(constants.SSL_TOPIC, ["--bootstrap-server", "localhost:9093", "--command-config", constants.SSL_CLIENT_CONFIG])) + except AssertionError as e: + errors.append(constants.SSL_ERROR_PREFIX + str(e)) + return errors + producer_config = ["--bootstrap-server", "localhost:9093", "--producer.config", constants.SSL_CLIENT_CONFIG] - self.produce_message(constants.SSL_TOPIC, producer_config, "key", "message") consumer_config = [ @@ -108,12 +117,21 @@ def ssl_flow(self): self.assertEqual(message, "key:message") except AssertionError as e: errors.append(constants.SSL_ERROR_PREFIX + str(e)) - print("Errors in SSL Flow:-", errors) + if errors: + print(f"Errors in {constants.SSL_FLOW_TESTS}:- {errors}") + else: + print(f"No errors in {constants.SSL_FLOW_TESTS}") return errors def broker_restart_flow(self): - print("Running broker restart tests") + print(f"Running {constants.BROKER_RESTART_TESTS}") errors = [] + try: + self.wait_for_port('localhost', 9092, True, constants.CLIENT_TIMEOUT) + except e: + errors.append(str(e)) + return errors + try: self.assertTrue(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) except AssertionError as e: @@ -123,13 +141,20 @@ def broker_restart_flow(self): producer_config = ["--bootstrap-server", "localhost:9092", "--property", "client.id=host"] self.produce_message(constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") - print("Stopping Image") + print("Stopping Container") self.stopImage() - time.sleep(15) - + try: + self.wait_for_port('localhost', 9092, False, constants.CLIENT_TIMEOUT) + except e: + errors.append(str(e)) + return errors print("Resuming Image") self.resumeImage() - time.sleep(15) + try: + self.wait_for_port('localhost', 9092, True, constants.CLIENT_TIMEOUT) + except e: + errors.append(str(e)) + return errors consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) try: @@ -141,7 +166,10 @@ def broker_restart_flow(self): self.assertEqual(message, "key:message") except AssertionError as e: errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) - print("Errors in Broker Restart Flow:-", errors) + if errors: + print(f"Errors in {constants.BROKER_RESTART_TESTS}:- {errors}") + else: + print(f"No errors in {constants.BROKER_RESTART_TESTS}") return errors def execute(self): From 58aa37ea1219850c4164a8c75503d6ed4b52f7b9 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 2 Nov 2023 16:30:05 +0530 Subject: [PATCH 19/66] Rely on scripts to detect when server is up --- docker/test/docker_sanity_test.py | 35 +------------------------------ 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 49655998c6c09..5571bc125bb06 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -15,11 +15,9 @@ import unittest import subprocess -import time from HTMLTestRunner import HTMLTestRunner import constants import argparse -import socket class DockerSanityTest(unittest.TestCase): IMAGE="apache/kafka" @@ -72,27 +70,10 @@ def consume_message(self, topic, consumer_config): command.extend(consumer_config) message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip() - - def wait_for_port(self, host, port, open, timeout): - start_time = time.perf_counter() - while time.perf_counter() - start_time < timeout: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - status = sock.connect_ex((host, port)) - sock.close() - if (open and status == 0) or (not open and status != 0): - return - else: - time.sleep(1) - raise TimeoutError("Timed out while waiting for the port", host, port) def ssl_flow(self): print(f"Running {constants.SSL_FLOW_TESTS}") errors = [] - try: - self.wait_for_port('localhost', 9093, True, constants.CLIENT_TIMEOUT) - except e: - errors.append(str(e)) - return errors try: self.assertTrue(self.create_topic(constants.SSL_TOPIC, ["--bootstrap-server", "localhost:9093", "--command-config", constants.SSL_CLIENT_CONFIG])) except AssertionError as e: @@ -126,11 +107,6 @@ def ssl_flow(self): def broker_restart_flow(self): print(f"Running {constants.BROKER_RESTART_TESTS}") errors = [] - try: - self.wait_for_port('localhost', 9092, True, constants.CLIENT_TIMEOUT) - except e: - errors.append(str(e)) - return errors try: self.assertTrue(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) @@ -143,18 +119,9 @@ def broker_restart_flow(self): print("Stopping Container") self.stopImage() - try: - self.wait_for_port('localhost', 9092, False, constants.CLIENT_TIMEOUT) - except e: - errors.append(str(e)) - return errors print("Resuming Image") self.resumeImage() - try: - self.wait_for_port('localhost', 9092, True, constants.CLIENT_TIMEOUT) - except e: - errors.append(str(e)) - return errors + consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) try: From 24863bba82606df733f038b67a0f5c4c546d2741 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 2 Nov 2023 16:42:31 +0530 Subject: [PATCH 20/66] Removed redundant wait in jsa generation --- docker/jvm/jsa_launch | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docker/jvm/jsa_launch b/docker/jvm/jsa_launch index 4962f97937832..7f40f86f97058 100755 --- a/docker/jvm/jsa_launch +++ b/docker/jvm/jsa_launch @@ -13,22 +13,18 @@ check_timeout() { sleep 1 } -TIMEOUT=20 -while ! nc -z localhost 9092; do - check_timeout -done - opt/kafka/bin/kafka-topics.sh --create --topic test-topic --bootstrap-server localhost:9092 [ $? -eq 0 ] || exit 1 echo "test" | opt/kafka/bin/kafka-console-producer.sh --topic test-topic --bootstrap-server localhost:9092 [ $? -eq 0 ] || exit 1 -opt/kafka/bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server localhost:9092 --max-messages 1 +opt/kafka/bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server localhost:9092 --max-messages 1 --timeout-ms 20000 [ $? -eq 0 ] || exit 1 opt/kafka/bin/kafka-server-stop.sh +# Wait until jsa file is generated TIMEOUT=20 until [ -f /kafka.jsa ] do From ff310624628138ec2b93753feecd80ffcaae98c4 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 6 Nov 2023 14:16:02 +0530 Subject: [PATCH 21/66] Add promotion script --- docker/docker_promote.py | 85 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 docker/docker_promote.py diff --git a/docker/docker_promote.py b/docker/docker_promote.py new file mode 100644 index 0000000000000..ef8c8421ebf60 --- /dev/null +++ b/docker/docker_promote.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Python script to promote an rc image. + +Follow the interactive guide to pull an RC image and promote it desired dockerhub repository. + +Usage: docker_promote.py + +Interactive utility to promote a docker image +""" + +import subprocess +import requests +from getpass import getpass + +def execute(command): + if subprocess.run(command).returncode != 0: + raise SystemError("Failure in executing following command:- ", " ".join(command)) + +def login(): + execute(["docker", "login"]) + +def pull(rc_image, promotion_image): + execute(["docker", "pull", "--platform=linux/amd64", rc_image]) + execute(["docker", "tag", rc_image, f"{promotion_image}-amd64"]) + execute(["docker", "pull", "--platform=linux/arm64", rc_image]) + execute(["docker", "tag", rc_image, f"{promotion_image}-arm64"]) + +def push(promotion_image): + execute(["docker", "push", f"{promotion_image}-amd64"]) + execute(["docker", "push", f"{promotion_image}-arm64"]) + +def push_manifest(promotion_image): + execute(["docker", "manifest", "create", promotion_image, + "--amend", f"{promotion_image}-amd64", + "--amend", f"{promotion_image}-arm64"]) + + execute(["docker", "manifest", "push", promotion_image]) + +def remove(promotion_image_namespace, promotion_image_name, promotion_image_tag, token): + if requests.delete(f"https://hub.docker.com/v2/repositories/{promotion_image_namespace}/{promotion_image_name}/tags/{promotion_image_tag}-amd64", headers={"Authorization": f"JWT {token}"}).status_code != 204: + raise SystemError(f"Failed to delete redundant images from dockerhub. Please make sure {promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-amd64 is removed from dockerhub") + if requests.delete(f"https://hub.docker.com/v2/repositories/{promotion_image_namespace}/{promotion_image_name}/tags/{promotion_image_tag}-arm64", headers={"Authorization": f"JWT {token}"}).status_code != 204: + raise SystemError(f"Failed to delete redundant images from dockerhub. Please make sure {promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-arm64 is removed from dockerhub") + subprocess.run(["docker", "rmi", f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-amd64"]) + subprocess.run(["docker", "rmi", f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-arm64"]) + +if __name__ == "__main__": + login() + username = input("Enter dockerhub username: ") + password = getpass("Enter dockerhub password: ") + + token = (requests.post("https://hub.docker.com/v2/users/login/", json={"username": username, "password": password})).json()['token'] + if len(token) == 0: + raise PermissionError("Dockerhub login failed") + + rc_image = input("Enter the RC docker image that you want to pull (in the format ::): ") + promotion_image_namespace = input("Enter the dockerhub namespace that the rc image needs to be promoted to [example: apache]: ") + promotion_image_name = input("Enter the dockerhub image name that the rc image needs to be promoted to [example: kafka]: ") + promotion_image_tag = input("Enter the dockerhub image tag that the rc image needs to be promoted to [example: 4.0.0]: ") + promotion_image = f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}" + + pull(rc_image, promotion_image) + push(promotion_image) + push_manifest(promotion_image) + remove(promotion_image_namespace, promotion_image_name, promotion_image_tag, token) + print("The image has been promoted successfully. The promoted image should be accessible in dockerhub") \ No newline at end of file From 9a24709b3ebd43ece84c9d0897314ebe0af04a2f Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 6 Nov 2023 14:20:12 +0530 Subject: [PATCH 22/66] Add requirements.txt for promotion script --- docker/requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 docker/requirements.txt diff --git a/docker/requirements.txt b/docker/requirements.txt new file mode 100644 index 0000000000000..663bd1f6a2ae0 --- /dev/null +++ b/docker/requirements.txt @@ -0,0 +1 @@ +requests \ No newline at end of file From e6582989c58b7f412e1e16b13fd91c637a450c6f Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 6 Nov 2023 21:33:04 +0530 Subject: [PATCH 23/66] Fix property file location --- docker/jvm/launch | 4 ++-- docker/resources/common-scripts/configure | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/jvm/launch b/docker/jvm/launch index 33b8333c756ed..e01c5810a9248 100755 --- a/docker/jvm/launch +++ b/docker/jvm/launch @@ -43,7 +43,7 @@ then echo "===> Using provided cluster id $CLUSTER_ID ..." # A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this - result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /etc/kafka/kafka.properties 2>&1) || \ + result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /opt/kafka/config/kafka.properties 2>&1) || \ echo $result | grep -i "already formatted" || \ { echo $result && (exit 1) } fi @@ -55,4 +55,4 @@ else fi # Start kafka broker -exec /opt/kafka/bin/kafka-server-start.sh /etc/kafka/kafka.properties +exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/kafka.properties diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index 7819c2dce127a..33bb195a137ff 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -36,7 +36,7 @@ then ub ensure KAFKA_ADVERTISED_LISTENERS fi else - echo "Only KRaft mode is supported KAFKA_PROCESS_ROLES is a mandatory config" + echo "Only KRaft mode is supported. KAFKA_PROCESS_ROLES is a mandatory config" exit 1 fi @@ -49,7 +49,7 @@ then KAFKA_LISTENERS=$(echo "$KAFKA_ADVERTISED_LISTENERS" | sed -e 's|://[^:]*:|://0.0.0.0:|g') fi -ub path /etc/kafka/ writable +ub path /opt/kafka/config/ writable if [[ -z "${KAFKA_LOG_DIRS-}" ]] then @@ -141,6 +141,6 @@ fi # --- for broker -ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json > /etc/kafka/kafka.properties -ub render-template /etc/kafka/docker/kafka-log4j.properties.template > /etc/kafka/log4j.properties -ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template > /etc/kafka/tools-log4j.properties +ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json > /opt/kafka/config/kafka.properties +ub render-template /etc/kafka/docker/kafka-log4j.properties.template > /opt/kafka/config/log4j.properties +ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template > /opt/kafka/config/tools-log4j.properties From 9cbec87b114c3c2da8a889de23b886a92451b676 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 8 Nov 2023 14:41:19 +0530 Subject: [PATCH 24/66] Add support for supplying properties through file mounting Changes to add support for file mounting of properties. Env variables, if defined, will override the file input properties. --- docker/jvm/Dockerfile | 22 ++++++++---------- docker/jvm/launch | 15 ++++-------- docker/resources/common-scripts/configure | 23 ++++++++----------- .../common-scripts/configureDefaults | 13 ----------- .../kafka-log4j.properties.template | 12 ++++------ .../kafka-tools-log4j.properties.template | 10 ++++---- .../resources/common-scripts/kafka.properties | 13 +++++++++++ .../resources/common-scripts/log4j.properties | 15 ++++++++++++ .../common-scripts/tools-log4j.properties | 6 +++++ .../ub/testResources/sampleLog4j.template | 7 +----- docker/test/requirements.txt | 2 -- 11 files changed, 67 insertions(+), 71 deletions(-) create mode 100644 docker/resources/common-scripts/kafka.properties create mode 100644 docker/resources/common-scripts/log4j.properties create mode 100644 docker/resources/common-scripts/tools-log4j.properties diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 7cbfe38455cc0..0e0fd6ff623db 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -32,8 +32,6 @@ USER root # Get kafka from https://archive.apache.org/dist/kafka and pass the url through build arguments ARG kafka_url -ENV KAFKA_URL=$kafka_url - COPY jsa_launch /etc/kafka/docker/jsa_launch RUN set -eux ; \ @@ -41,7 +39,7 @@ RUN set -eux ; \ apk upgrade ; \ apk add --no-cache wget gcompat procps netcat-openbsd; \ mkdir opt/kafka; \ - wget -nv -O kafka.tgz "$KAFKA_URL"; \ + wget -nv -O kafka.tgz "$kafka_url"; \ tar xfz kafka.tgz -C /opt/kafka --strip-components 1; RUN /etc/kafka/docker/jsa_launch @@ -66,25 +64,23 @@ LABEL org.label-schema.name="kafka" \ org.label-schema.schema-version="1.0" \ maintainer="apache" -ENV KAFKA_URL=$kafka_url - RUN set -eux ; \ apk update ; \ apk upgrade ; \ apk add --no-cache curl wget gpg dirmngr gpg-agent gcompat; \ mkdir opt/kafka; \ - wget -nv -O kafka.tgz "$KAFKA_URL"; \ - wget -nv -O kafka.tgz.asc "$KAFKA_URL.asc"; \ + wget -nv -O kafka.tgz "$kafka_url"; \ + wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \ tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \ wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \ gpg --import KEYS; \ gpg --batch --verify kafka.tgz.asc kafka.tgz; \ - mkdir -p /var/lib/kafka/data /etc/kafka/secrets /var/log/kafka /var/lib/zookeeper; \ - mkdir -p /etc/kafka/docker /usr/logs; \ + mkdir -p /var/lib/kafka/data /etc/kafka/secrets /var/log/kafka; \ + mkdir -p /etc/kafka/docker /usr/logs /mnt/shared/config; \ adduser -h /home/appuser -D --shell /bin/bash appuser; \ - chown appuser:appuser -R /usr/logs /opt/kafka; \ - chown appuser:root -R /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka /var/lib/zookeeper; \ - chmod -R ug+w /etc/kafka /var/lib/kafka /var/lib/kafka /etc/kafka/secrets /etc/kafka /var/log/kafka /var/lib/zookeeper; \ + chown appuser:appuser -R /usr/logs /opt/kafka /mnt/shared/config; \ + chown appuser:root -R /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka; \ + chmod -R ug+w /etc/kafka /var/lib/kafka /var/lib/kafka /etc/kafka/secrets /etc/kafka /var/log/kafka; \ rm kafka.tgz kafka.tgz.asc KEYS; \ apk del curl wget gpg dirmngr gpg-agent; \ apk cache clean; @@ -96,6 +92,6 @@ COPY --chown=appuser:appuser launch /etc/kafka/docker/launch USER appuser -VOLUME ["/etc/kafka/secrets", "/var/lib/kafka/data"] +VOLUME ["/etc/kafka/secrets", "/var/lib/kafka/data", "/mnt/shared/config"] CMD ["/etc/kafka/docker/run"] diff --git a/docker/jvm/launch b/docker/jvm/launch index e01c5810a9248..a9844b60953e1 100755 --- a/docker/jvm/launch +++ b/docker/jvm/launch @@ -37,16 +37,11 @@ if [ "$KAFKA_JMX_PORT" ]; then export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" fi -# KRaft required step: Format the storage directory with provided cluster ID unless it already exists. -if [[ -n "${KAFKA_PROCESS_ROLES-}" ]] -then - echo "===> Using provided cluster id $CLUSTER_ID ..." - - # A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this - result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /opt/kafka/config/kafka.properties 2>&1) || \ - echo $result | grep -i "already formatted" || \ - { echo $result && (exit 1) } -fi +echo "===> Using provided cluster id $CLUSTER_ID ..." +# A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this +result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /opt/kafka/config/kafka.properties 2>&1) || \ + echo $result | grep -i "already formatted" || \ + { echo $result && (exit 1) } if [ -z "$KAFKA_JVM_PERFORMANCE_OPTS" ]; then export KAFKA_JVM_PERFORMANCE_OPTS="-XX:SharedArchiveFile=/opt/kafka/kafka.jsa" diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index 33bb195a137ff..2d29322298d89 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -35,15 +35,12 @@ then else ub ensure KAFKA_ADVERTISED_LISTENERS fi -else - echo "Only KRaft mode is supported. KAFKA_PROCESS_ROLES is a mandatory config" - exit 1 fi # By default, LISTENERS is derived from ADVERTISED_LISTENERS by replacing # hosts with 0.0.0.0. This is good default as it ensures that the broker # process listens on all ports. -if [[ -z "${KAFKA_LISTENERS-}" ]] && ( [[ -z "${KAFKA_PROCESS_ROLES-}" ]] || [[ $KAFKA_PROCESS_ROLES != "controller" ]] ) +if [[ -z "${KAFKA_LISTENERS-}" ]] && ( [[ -z "${KAFKA_PROCESS_ROLES-}" ]] || [[ $KAFKA_PROCESS_ROLES != "controller" ]] ) && [[ -n "${KAFKA_ADVERTISED_LISTENERS}" ]] then export KAFKA_LISTENERS KAFKA_LISTENERS=$(echo "$KAFKA_ADVERTISED_LISTENERS" | sed -e 's|://[^:]*:|://0.0.0.0:|g') @@ -51,12 +48,6 @@ fi ub path /opt/kafka/config/ writable -if [[ -z "${KAFKA_LOG_DIRS-}" ]] -then - export KAFKA_LOG_DIRS - KAFKA_LOG_DIRS="/var/lib/kafka/data" -fi - # advertised.host, advertised.port, host and port are deprecated. Exit if these properties are set. if [[ -n "${KAFKA_ADVERTISED_PORT-}" ]] then @@ -139,8 +130,12 @@ then fi fi +mv /etc/kafka/docker/kafka.properties /opt/kafka/config/kafka.properties +mv /etc/kafka/docker/log4j.properties /opt/kafka/config/log4j.properties +mv /etc/kafka/docker/tools-log4j.properties /opt/kafka/config/tools-log4j.properties -# --- for broker -ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json > /opt/kafka/config/kafka.properties -ub render-template /etc/kafka/docker/kafka-log4j.properties.template > /opt/kafka/config/log4j.properties -ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template > /opt/kafka/config/tools-log4j.properties +cp -R /mnt/shared/config/. /opt/kafka/config/ + +ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json >> /opt/kafka/config/kafka.properties +ub render-template /etc/kafka/docker/kafka-log4j.properties.template >> /opt/kafka/config/log4j.properties +ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template >> /opt/kafka/config/tools-log4j.properties diff --git a/docker/resources/common-scripts/configureDefaults b/docker/resources/common-scripts/configureDefaults index a732ad60c37fb..6f5bfa47bf4ea 100755 --- a/docker/resources/common-scripts/configureDefaults +++ b/docker/resources/common-scripts/configureDefaults @@ -19,19 +19,6 @@ env_defaults=( # Replace CLUSTER_ID with a unique base64 UUID using "bin/kafka-storage.sh random-uuid" # See https://docs.confluent.io/kafka/operations-tools/kafka-tools.html#kafka-storage-sh ["CLUSTER_ID"]="5L6g3nShT-eMCtK--X86sw" - ["KAFKA_NODE_ID"]=1 - ["KAFKA_LISTENER_SECURITY_PROTOCOL_MAP"]="CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT" - ["KAFKA_LISTENERS"]="PLAINTEXT://localhost:29092,CONTROLLER://localhost:29093,PLAINTEXT_HOST://0.0.0.0:9092" - ["KAFKA_ADVERTISED_LISTENERS"]="PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092" - ["KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR"]=1 - ["KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS"]=0 - ["KAFKA_TRANSACTION_STATE_LOG_MIN_ISR"]=1 - ["KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR"]=1 - ["KAFKA_PROCESS_ROLES"]="broker,controller" - ["KAFKA_CONTROLLER_QUORUM_VOTERS"]="1@localhost:29093" - ["KAFKA_INTER_BROKER_LISTENER_NAME"]="PLAINTEXT" - ["KAFKA_CONTROLLER_LISTENER_NAMES"]="CONTROLLER" - ["KAFKA_LOG_DIRS"]="/tmp/kraft-combined-logs" ) for key in "${!env_defaults[@]}"; do diff --git a/docker/resources/common-scripts/kafka-log4j.properties.template b/docker/resources/common-scripts/kafka-log4j.properties.template index 3a7b4744e34c4..5241723d8cc7b 100644 --- a/docker/resources/common-scripts/kafka-log4j.properties.template +++ b/docker/resources/common-scripts/kafka-log4j.properties.template @@ -1,11 +1,9 @@ -log4j.rootLogger={{ getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}, stdout +{{ with $value := getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }} +{{ if ne $value "INFO" }} +log4j.rootLogger=$value, stdout +{{ end }}{{ end }} -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - -{{ $loggerDefaults := "kafka=INFO,kafka.network.RequestChannel$=WARN,kafka.producer.async.DefaultEventHandler=DEBUG,kafka.request.logger=WARN,kafka.controller=TRACE,kafka.log.LogCleaner=INFO,state.change.logger=TRACE,kafka.authorizer.logger=WARN"}} {{ $loggers := getEnv "KAFKA_LOG4J_LOGGERS" "" -}} -{{ range $k, $v := splitToMapDefaults "," $loggerDefaults $loggers}} +{{ range $k, $v := splitToMapDefaults "," "" $loggers}} log4j.logger.{{ $k }}={{ $v -}} {{ end }} diff --git a/docker/resources/common-scripts/kafka-tools-log4j.properties.template b/docker/resources/common-scripts/kafka-tools-log4j.properties.template index c2df5bcf064a4..69fd05d437351 100644 --- a/docker/resources/common-scripts/kafka-tools-log4j.properties.template +++ b/docker/resources/common-scripts/kafka-tools-log4j.properties.template @@ -1,6 +1,4 @@ -log4j.rootLogger={{ getEnv "KAFKA_TOOLS_LOG4J_LOGLEVEL" "WARN" }}, stderr - -log4j.appender.stderr=org.apache.log4j.ConsoleAppender -log4j.appender.stderr.layout=org.apache.log4j.PatternLayout -log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n -log4j.appender.stderr.Target=System.err +{{ with $value := getEnv "KAFKA_TOOLS_LOG4J_LOGLEVEL" "WARN"}} +{{if ne $value "WARN"}} +log4j.rootLogger=$value, stderr +{{ end }}{{ end }} diff --git a/docker/resources/common-scripts/kafka.properties b/docker/resources/common-scripts/kafka.properties new file mode 100644 index 0000000000000..e9f40eddc1727 --- /dev/null +++ b/docker/resources/common-scripts/kafka.properties @@ -0,0 +1,13 @@ +advertised.listeners=PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092 +controller.listener.names=CONTROLLER +controller.quorum.voters=1@localhost:29093 +group.initial.rebalance.delay.ms=0 +inter.broker.listener.name=PLAINTEXT +listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT +listeners=PLAINTEXT://localhost:29092,CONTROLLER://localhost:29093,PLAINTEXT_HOST://0.0.0.0:9092 +log.dirs=/tmp/kraft-combined-logs +node.id=1 +offsets.topic.replication.factor=1 +process.roles=broker,controller +transaction.state.log.min.isr=1 +transaction.state.log.replication.factor=1 diff --git a/docker/resources/common-scripts/log4j.properties b/docker/resources/common-scripts/log4j.properties new file mode 100644 index 0000000000000..148bd53b664b6 --- /dev/null +++ b/docker/resources/common-scripts/log4j.properties @@ -0,0 +1,15 @@ +log4j.rootLogger=INFO, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + + +log4j.logger.kafka=INFO +log4j.logger.kafka.authorizer.logger=WARN +log4j.logger.kafka.controller=TRACE +log4j.logger.kafka.log.LogCleaner=INFO +log4j.logger.kafka.network.RequestChannel$=WARN +log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG +log4j.logger.kafka.request.logger=WARN +log4j.logger.state.change.logger=TRACE diff --git a/docker/resources/common-scripts/tools-log4j.properties b/docker/resources/common-scripts/tools-log4j.properties new file mode 100644 index 0000000000000..27d9fbee48bf5 --- /dev/null +++ b/docker/resources/common-scripts/tools-log4j.properties @@ -0,0 +1,6 @@ +log4j.rootLogger=WARN, stderr + +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n +log4j.appender.stderr.Target=System.err diff --git a/docker/resources/ub/testResources/sampleLog4j.template b/docker/resources/ub/testResources/sampleLog4j.template index 3aace55b81aba..6a6a8630b2c10 100644 --- a/docker/resources/ub/testResources/sampleLog4j.template +++ b/docker/resources/ub/testResources/sampleLog4j.template @@ -1,11 +1,6 @@ log4j.rootLogger={{ getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}, stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - -{{ $loggerDefaults := "kafka=INFO,kafka.network.RequestChannel$=WARN,kafka.producer.async.DefaultEventHandler=DEBUG,kafka.request.logger=WARN,kafka.controller=TRACE,kafka.log.LogCleaner=INFO,state.change.logger=TRACE,kafka.authorizer.logger=WARN"}} {{$loggers := getEnv "KAFKA_LOG4J_LOGGERS" "" -}} -{{ range $k, $v := splitToMapDefaults "," $loggerDefaults $loggers}} +{{ range $k, $v := splitToMapDefaults "," "" $loggers}} log4j.logger.{{ $k }}={{ $v -}} {{ end }} \ No newline at end of file diff --git a/docker/test/requirements.txt b/docker/test/requirements.txt index 401ad63aa6179..cb2ef3545ecf1 100644 --- a/docker/test/requirements.txt +++ b/docker/test/requirements.txt @@ -1,3 +1 @@ -urllib3 -requests HTMLTestRunner-Python3 \ No newline at end of file From 108ab8595f979cbc8855e181f7aa87926e10109b Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 8 Nov 2023 14:56:42 +0530 Subject: [PATCH 25/66] Ensure that environment variable configs are always appended in newline --- docker/resources/common-scripts/configure | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index 2d29322298d89..f3db5a2188b10 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -136,6 +136,6 @@ mv /etc/kafka/docker/tools-log4j.properties /opt/kafka/config/tools-log4j.proper cp -R /mnt/shared/config/. /opt/kafka/config/ -ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json >> /opt/kafka/config/kafka.properties -ub render-template /etc/kafka/docker/kafka-log4j.properties.template >> /opt/kafka/config/log4j.properties -ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template >> /opt/kafka/config/tools-log4j.properties +echo -e "\n$(ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json)" >> /opt/kafka/config/kafka.properties +echo -e "\n$(ub render-template /etc/kafka/docker/kafka-log4j.properties.template)" >> /opt/kafka/config/log4j.properties +echo -e "\n$(ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template)" >> /opt/kafka/config/tools-log4j.properties From ed2f94fc1ba5f8757b25f6d1d7a25a43d964614f Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 8 Nov 2023 15:04:21 +0530 Subject: [PATCH 26/66] Add missing brackets in template --- .../resources/common-scripts/kafka-log4j.properties.template | 5 ++--- .../common-scripts/kafka-tools-log4j.properties.template | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docker/resources/common-scripts/kafka-log4j.properties.template b/docker/resources/common-scripts/kafka-log4j.properties.template index 5241723d8cc7b..f9da46cc7443d 100644 --- a/docker/resources/common-scripts/kafka-log4j.properties.template +++ b/docker/resources/common-scripts/kafka-log4j.properties.template @@ -1,6 +1,5 @@ -{{ with $value := getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }} -{{ if ne $value "INFO" }} -log4j.rootLogger=$value, stdout +{{ with $value := getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}{{ if ne $value "INFO" }} +log4j.rootLogger={{ $value }}, stdout {{ end }}{{ end }} {{ $loggers := getEnv "KAFKA_LOG4J_LOGGERS" "" -}} diff --git a/docker/resources/common-scripts/kafka-tools-log4j.properties.template b/docker/resources/common-scripts/kafka-tools-log4j.properties.template index 69fd05d437351..4c55b9bb9a2ae 100644 --- a/docker/resources/common-scripts/kafka-tools-log4j.properties.template +++ b/docker/resources/common-scripts/kafka-tools-log4j.properties.template @@ -1,4 +1,3 @@ -{{ with $value := getEnv "KAFKA_TOOLS_LOG4J_LOGLEVEL" "WARN"}} -{{if ne $value "WARN"}} -log4j.rootLogger=$value, stderr +{{ with $value := getEnv "KAFKA_TOOLS_LOG4J_LOGLEVEL" "WARN"}} {{if ne $value "WARN"}} +log4j.rootLogger={{ $value }}, stderr {{ end }}{{ end }} From 7a7c33c8a828695878b1ed964553ccf6eab5fc09 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 9 Nov 2023 10:38:14 +0530 Subject: [PATCH 27/66] Add test for file input --- docker/jvm/launch | 4 +- docker/resources/common-scripts/configure | 4 +- .../{kafka.properties => server.properties} | 0 docker/test/constants.py | 8 ++- docker/test/docker_sanity_test.py | 59 ++++++++++--------- .../fixtures/file-input/server.properties | 26 ++++++++ .../{kraft => jvm}/docker-compose.yml | 11 ++++ 7 files changed, 79 insertions(+), 33 deletions(-) rename docker/resources/common-scripts/{kafka.properties => server.properties} (100%) create mode 100644 docker/test/fixtures/file-input/server.properties rename docker/test/fixtures/{kraft => jvm}/docker-compose.yml (91%) diff --git a/docker/jvm/launch b/docker/jvm/launch index a9844b60953e1..5d821d1e43773 100755 --- a/docker/jvm/launch +++ b/docker/jvm/launch @@ -39,7 +39,7 @@ fi echo "===> Using provided cluster id $CLUSTER_ID ..." # A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this -result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /opt/kafka/config/kafka.properties 2>&1) || \ +result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /opt/kafka/config/server.properties 2>&1) || \ echo $result | grep -i "already formatted" || \ { echo $result && (exit 1) } @@ -50,4 +50,4 @@ else fi # Start kafka broker -exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/kafka.properties +exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index f3db5a2188b10..4e812ad7cd731 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -130,12 +130,12 @@ then fi fi -mv /etc/kafka/docker/kafka.properties /opt/kafka/config/kafka.properties +mv /etc/kafka/docker/server.properties /opt/kafka/config/server.properties mv /etc/kafka/docker/log4j.properties /opt/kafka/config/log4j.properties mv /etc/kafka/docker/tools-log4j.properties /opt/kafka/config/tools-log4j.properties cp -R /mnt/shared/config/. /opt/kafka/config/ -echo -e "\n$(ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json)" >> /opt/kafka/config/kafka.properties +echo -e "\n$(ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json)" >> /opt/kafka/config/server.properties echo -e "\n$(ub render-template /etc/kafka/docker/kafka-log4j.properties.template)" >> /opt/kafka/config/log4j.properties echo -e "\n$(ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template)" >> /opt/kafka/config/tools-log4j.properties diff --git a/docker/resources/common-scripts/kafka.properties b/docker/resources/common-scripts/server.properties similarity index 100% rename from docker/resources/common-scripts/kafka.properties rename to docker/resources/common-scripts/server.properties diff --git a/docker/test/constants.py b/docker/test/constants.py index ace6ff9d66efa..adc88a44bce8a 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -17,7 +17,7 @@ KAFKA_CONSOLE_PRODUCER="./fixtures/kafka/bin/kafka-console-producer.sh" KAFKA_CONSOLE_CONSUMER="./fixtures/kafka/bin/kafka-console-consumer.sh" -KRAFT_COMPOSE="fixtures/kraft/docker-compose.yml" +JVM_COMPOSE="fixtures/jvm/docker-compose.yml" ZOOKEEPER_COMPOSE="fixtures/zookeeper/docker-compose.yml" CLIENT_TIMEOUT=40 @@ -26,9 +26,13 @@ SSL_CLIENT_CONFIG="./fixtures/secrets/client-ssl.properties" SSL_TOPIC="test-topic-ssl" +FILE_INPUT_FLOW_TESTS="File Input Flow Tests" +FILE_INPUT_TOPIC="test-topic-file-input" + BROKER_RESTART_TESTS="Broker Restart Tests" BROKER_CONTAINER="broker" BROKER_RESTART_TEST_TOPIC="test-topic-broker-restart" SSL_ERROR_PREFIX="SSL_ERR" -BROKER_RESTART_ERROR_PREFIX="BROKER_RESTART_ERR" \ No newline at end of file +BROKER_RESTART_ERROR_PREFIX="BROKER_RESTART_ERR" +FILE_INPUT_ERROR_PREFIX="FILE_INPUT_ERR" \ No newline at end of file diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 5571bc125bb06..1afb599e3759e 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -22,13 +22,13 @@ class DockerSanityTest(unittest.TestCase): IMAGE="apache/kafka" - def resumeImage(self): + def resume_container(self): subprocess.run(["docker", "start", constants.BROKER_CONTAINER]) - def stopImage(self) -> None: + def stop_container(self) -> None: subprocess.run(["docker", "stop", constants.BROKER_CONTAINER]) - def startCompose(self, filename) -> None: + def start_compose(self, filename) -> None: old_string="image: {$IMAGE}" new_string=f"image: {self.IMAGE}" with open(filename) as f: @@ -39,7 +39,7 @@ def startCompose(self, filename) -> None: subprocess.run(["docker-compose", "-f", filename, "up", "-d"]) - def destroyCompose(self, filename) -> None: + def destroy_compose(self, filename) -> None: old_string=f"image: {self.IMAGE}" new_string="image: {$IMAGE}" subprocess.run(["docker-compose", "-f", filename, "down"]) @@ -71,37 +71,37 @@ def consume_message(self, topic, consumer_config): message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip() - def ssl_flow(self): - print(f"Running {constants.SSL_FLOW_TESTS}") + def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): + print(f"Running {test_name}") errors = [] try: - self.assertTrue(self.create_topic(constants.SSL_TOPIC, ["--bootstrap-server", "localhost:9093", "--command-config", constants.SSL_CLIENT_CONFIG])) + self.assertTrue(self.create_topic(topic, ["--bootstrap-server", ssl_broker_port, "--command-config", constants.SSL_CLIENT_CONFIG])) except AssertionError as e: - errors.append(constants.SSL_ERROR_PREFIX + str(e)) + errors.append(test_error_prefix + str(e)) return errors - producer_config = ["--bootstrap-server", "localhost:9093", + producer_config = ["--bootstrap-server", ssl_broker_port, "--producer.config", constants.SSL_CLIENT_CONFIG] - self.produce_message(constants.SSL_TOPIC, producer_config, "key", "message") + self.produce_message(topic, producer_config, "key", "message") consumer_config = [ - "--bootstrap-server", "localhost:9093", + "--bootstrap-server", ssl_broker_port, "--property", "auto.offset.reset=earliest", "--consumer.config", constants.SSL_CLIENT_CONFIG, ] - message = self.consume_message(constants.SSL_TOPIC, consumer_config) + message = self.consume_message(topic, consumer_config) try: self.assertIsNotNone(message) except AssertionError as e: - errors.append(constants.SSL_ERROR_PREFIX + str(e)) + errors.append(test_error_prefix + str(e)) try: self.assertEqual(message, "key:message") except AssertionError as e: - errors.append(constants.SSL_ERROR_PREFIX + str(e)) + errors.append(test_error_prefix + str(e)) if errors: - print(f"Errors in {constants.SSL_FLOW_TESTS}:- {errors}") + print(f"Errors in {test_name}:- {errors}") else: - print(f"No errors in {constants.SSL_FLOW_TESTS}") + print(f"No errors in {test_name}") return errors def broker_restart_flow(self): @@ -118,9 +118,9 @@ def broker_restart_flow(self): self.produce_message(constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") print("Stopping Container") - self.stopImage() - print("Resuming Image") - self.resumeImage() + self.stop_container() + print("Resuming Container") + self.resume_container() consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) @@ -142,37 +142,42 @@ def broker_restart_flow(self): def execute(self): total_errors = [] try: - total_errors.extend(self.ssl_flow()) + total_errors.extend(self.ssl_flow('localhost:9093', constants.SSL_FLOW_TESTS, constants.SSL_ERROR_PREFIX, constants.SSL_TOPIC)) except Exception as e: - print("SSL flow error", str(e)) + print(constants.SSL_ERROR_PREFIX, str(e)) + total_errors.append(str(e)) + try: + total_errors.extend(self.ssl_flow('localhost:9094', constants.FILE_INPUT_FLOW_TESTS, constants.FILE_INPUT_ERROR_PREFIX, constants.FILE_INPUT_TOPIC)) + except Exception as e: + print(constants.FILE_INPUT_ERROR_PREFIX, str(e)) total_errors.append(str(e)) try: total_errors.extend(self.broker_restart_flow()) except Exception as e: - print("Broker restart flow error", str(e)) + print(constants.BROKER_RESTART_ERROR_PREFIX, str(e)) total_errors.append(str(e)) self.assertEqual(total_errors, []) -class DockerSanityTestKraftMode(DockerSanityTest): +class DockerSanityTestJVM(DockerSanityTest): def setUp(self) -> None: - self.startCompose(constants.KRAFT_COMPOSE) + self.start_compose(constants.JVM_COMPOSE) def tearDown(self) -> None: - self.destroyCompose(constants.KRAFT_COMPOSE) + self.destroy_compose(constants.JVM_COMPOSE) def test_bed(self): self.execute() if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("image") - parser.add_argument("mode", default="all") + parser.add_argument("mode") args = parser.parse_args() DockerSanityTest.IMAGE = args.image test_classes_to_run = [] if args.mode == "jvm": - test_classes_to_run = [DockerSanityTestKraftMode] + test_classes_to_run = [DockerSanityTestJVM] loader = unittest.TestLoader() suites_list = [] diff --git a/docker/test/fixtures/file-input/server.properties b/docker/test/fixtures/file-input/server.properties new file mode 100644 index 0000000000000..0ae8fa924f44a --- /dev/null +++ b/docker/test/fixtures/file-input/server.properties @@ -0,0 +1,26 @@ +advertised.listeners=PLAINTEXT://localhost:19092,SSL://localhost:19093,SSL-INT://localhost:9093,BROKER://localhost:9092 +controller.listener.names=CONTROLLER +controller.quorum.voters=3@broker-ssl-file-input:29093 +group.initial.rebalance.delay.ms=0 +inter.broker.listener.name=BROKER +listener.name.internal.ssl.endpoint.identification.algorithm= +listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SSL-INT:SSL,BROKER:PLAINTEXT,CONTROLLER:PLAINTEXT +listeners=PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:19093,SSL-INT://0.0.0.0:9093,BROKER://0.0.0.0:9092,CONTROLLER://broker-ssl-file-input:29093 +log.dirs=/tmp/kraft-combined-logs +node.id=3 +offsets.topic.replication.factor=1 +process.roles=broker,controller +ssl.client.auth=required +ssl.endpoint.identification.algorithm= +ssl.key.credentials=kafka_ssl_key_creds +ssl.key.password=abcdefgh +ssl.keystore.credentials=kafka_keystore_creds +ssl.keystore.filename=kafka01.keystore.jks +ssl.keystore.location=/etc/kafka/secrets/kafka01.keystore.jks +ssl.keystore.password=abcdefgh +ssl.truststore.credentials=kafka_truststore_creds +ssl.truststore.filename=kafka.truststore.jks +ssl.truststore.location=/etc/kafka/secrets/kafka.truststore.jks +ssl.truststore.password=abcdefgh +transaction.state.log.min.isr=1 +transaction.state.log.replication.factor=1 diff --git a/docker/test/fixtures/kraft/docker-compose.yml b/docker/test/fixtures/jvm/docker-compose.yml similarity index 91% rename from docker/test/fixtures/kraft/docker-compose.yml rename to docker/test/fixtures/jvm/docker-compose.yml index ed06c819917d0..de89fbaf3206f 100644 --- a/docker/test/fixtures/kraft/docker-compose.yml +++ b/docker/test/fixtures/jvm/docker-compose.yml @@ -69,3 +69,14 @@ services: KAFKA_SSL_CLIENT_AUTH: "required" KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + broker-ssl-file-input: + image: {$IMAGE} + hostname: broker-ssl-file-input + container_name: broker-ssl-file-input + ports: + - "9094:9093" + volumes: + - ../secrets:/etc/kafka/secrets + - ../file-input:/mnt/shared/config + environment: + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' From 1c8ada44d5a838c7a63851997efd00d766c2091e Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 9 Nov 2023 16:35:52 +0530 Subject: [PATCH 28/66] Bubble up test errors to root build test script --- .github/workflows/docker_build_and_test.yml | 24 +++--- docker/docker_build_test.py | 5 +- docker/requirements.txt | 3 +- .../common-scripts/configureDefaults | 1 - docker/test/__init__.py | 0 docker/test/constants.py | 11 ++- docker/test/docker_sanity_test.py | 75 +++++++++---------- .../fixtures/file-input/server.properties | 5 -- .../fixtures/secrets/client-ssl.properties | 4 +- docker/test/requirements.txt | 1 - 10 files changed, 63 insertions(+), 66 deletions(-) create mode 100644 docker/test/__init__.py delete mode 100644 docker/test/requirements.txt diff --git a/.github/workflows/docker_build_and_test.yml b/.github/workflows/docker_build_and_test.yml index 82a9a81a19ed2..af2aa121a3679 100644 --- a/.github/workflows/docker_build_and_test.yml +++ b/.github/workflows/docker_build_and_test.yml @@ -24,12 +24,13 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r docker/test/requirements.txt - - name: Run tests + pip install -r docker/requirements.txt + - name: Build image and run tests working-directory: ./docker run: | python docker_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -u=${{ github.event.inputs.kafka_url }} - - name: Run Vulnerability scan + - name: Run CVE scan + if: always() uses: aquasecurity/trivy-action@master with: image-ref: 'kafka/test:test' @@ -37,12 +38,17 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' - output: scan_${{ github.event.inputs.image_type }}.txt - - uses: actions/upload-artifact@v3 + output: scan_report_${{ github.event.inputs.image_type }}.txt + exit-code: '1' + - name: Upload test report + if: always() + uses: actions/upload-artifact@v3 with: name: report_${{ github.event.inputs.image_type }}.html - path: docker/test/report_${{ github.event.inputs.image_type }}.html - - uses: actions/upload-artifact@v3 + path: docker/report_${{ github.event.inputs.image_type }}.html + - name: Upload CVE scan report + if: always() + uses: actions/upload-artifact@v3 with: - name: scan_${{ github.event.inputs.image_type }}.txt - path: scan_${{ github.event.inputs.image_type }}.txt \ No newline at end of file + name: scan_report_${{ github.event.inputs.image_type }}.txt + path: scan_report_${{ github.event.inputs.image_type }}.txt diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 5cefa723a2f4f..25704711d7617 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -20,6 +20,7 @@ import argparse from distutils.dir_util import copy_tree import shutil +from test.docker_sanity_test import run_tests def build_jvm(image, tag, kafka_url): image = f'{image}:{tag}' @@ -35,9 +36,11 @@ def run_jvm_tests(image, tag, kafka_url): subprocess.run(["wget", "-nv", "-O", "kafka.tgz", kafka_url]) subprocess.run(["mkdir", "./test/fixtures/kafka"]) subprocess.run(["tar", "xfz", "kafka.tgz", "-C", "./test/fixtures/kafka", "--strip-components", "1"]) - subprocess.run(["python3", "docker_sanity_test.py", f"{image}:{tag}", "jvm"], cwd="test") + failure_count = run_tests(f"{image}:{tag}", "jvm") subprocess.run(["rm", "kafka.tgz"]) shutil.rmtree("./test/fixtures/kafka") + if failure_count != 0: + raise SystemError("Test Failure. Error count is non 0") if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/docker/requirements.txt b/docker/requirements.txt index 663bd1f6a2ae0..bc4fdd44eaf21 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1 +1,2 @@ -requests \ No newline at end of file +requests +HTMLTestRunner-Python3 \ No newline at end of file diff --git a/docker/resources/common-scripts/configureDefaults b/docker/resources/common-scripts/configureDefaults index 6f5bfa47bf4ea..14d28548a83eb 100755 --- a/docker/resources/common-scripts/configureDefaults +++ b/docker/resources/common-scripts/configureDefaults @@ -17,7 +17,6 @@ declare -A env_defaults env_defaults=( # Replace CLUSTER_ID with a unique base64 UUID using "bin/kafka-storage.sh random-uuid" -# See https://docs.confluent.io/kafka/operations-tools/kafka-tools.html#kafka-storage-sh ["CLUSTER_ID"]="5L6g3nShT-eMCtK--X86sw" ) diff --git a/docker/test/__init__.py b/docker/test/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/docker/test/constants.py b/docker/test/constants.py index adc88a44bce8a..89c27e0c3baf6 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -13,17 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -KAFKA_TOPICS="./fixtures/kafka/bin/kafka-topics.sh" -KAFKA_CONSOLE_PRODUCER="./fixtures/kafka/bin/kafka-console-producer.sh" -KAFKA_CONSOLE_CONSUMER="./fixtures/kafka/bin/kafka-console-consumer.sh" +KAFKA_TOPICS="./test/fixtures/kafka/bin/kafka-topics.sh" +KAFKA_CONSOLE_PRODUCER="./test/fixtures/kafka/bin/kafka-console-producer.sh" +KAFKA_CONSOLE_CONSUMER="./test/fixtures/kafka/bin/kafka-console-consumer.sh" -JVM_COMPOSE="fixtures/jvm/docker-compose.yml" -ZOOKEEPER_COMPOSE="fixtures/zookeeper/docker-compose.yml" +JVM_COMPOSE="./test/fixtures/jvm/docker-compose.yml" CLIENT_TIMEOUT=40 SSL_FLOW_TESTS="SSL Flow Tests" -SSL_CLIENT_CONFIG="./fixtures/secrets/client-ssl.properties" +SSL_CLIENT_CONFIG="./test/fixtures/secrets/client-ssl.properties" SSL_TOPIC="test-topic-ssl" FILE_INPUT_FLOW_TESTS="File Input Flow Tests" diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 1afb599e3759e..242ab37de28f1 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -16,17 +16,16 @@ import unittest import subprocess from HTMLTestRunner import HTMLTestRunner -import constants -import argparse +import test.constants class DockerSanityTest(unittest.TestCase): IMAGE="apache/kafka" def resume_container(self): - subprocess.run(["docker", "start", constants.BROKER_CONTAINER]) + subprocess.run(["docker", "start", test.constants.BROKER_CONTAINER]) def stop_container(self) -> None: - subprocess.run(["docker", "stop", constants.BROKER_CONTAINER]) + subprocess.run(["docker", "stop", test.constants.BROKER_CONTAINER]) def start_compose(self, filename) -> None: old_string="image: {$IMAGE}" @@ -50,44 +49,44 @@ def destroy_compose(self, filename) -> None: f.write(s) def create_topic(self, topic, topic_config): - command = [constants.KAFKA_TOPICS, "--create", "--topic", topic] + command = [test.constants.KAFKA_TOPICS, "--create", "--topic", topic] command.extend(topic_config) subprocess.run(command) - check_command = [constants.KAFKA_TOPICS, "--list"] + check_command = [test.constants.KAFKA_TOPICS, "--list"] check_command.extend(topic_config) - output = subprocess.check_output(check_command, timeout=constants.CLIENT_TIMEOUT) + output = subprocess.check_output(check_command, timeout=test.constants.CLIENT_TIMEOUT) if topic in output.decode("utf-8"): return True return False def produce_message(self, topic, producer_config, key, value): - command = ["echo", f'"{key}:{value}"', "|", constants.KAFKA_CONSOLE_PRODUCER, "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] + command = ["echo", f'"{key}:{value}"', "|", test.constants.KAFKA_CONSOLE_PRODUCER, "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] command.extend(producer_config) - subprocess.run(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) + subprocess.run(["bash", "-c", " ".join(command)], timeout=test.constants.CLIENT_TIMEOUT) def consume_message(self, topic, consumer_config): - command = [constants.KAFKA_CONSOLE_CONSUMER, "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] + command = [test.constants.KAFKA_CONSOLE_CONSUMER, "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] command.extend(consumer_config) - message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) + message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=test.constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip() def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): print(f"Running {test_name}") errors = [] try: - self.assertTrue(self.create_topic(topic, ["--bootstrap-server", ssl_broker_port, "--command-config", constants.SSL_CLIENT_CONFIG])) + self.assertTrue(self.create_topic(topic, ["--bootstrap-server", ssl_broker_port, "--command-config", test.constants.SSL_CLIENT_CONFIG])) except AssertionError as e: errors.append(test_error_prefix + str(e)) return errors producer_config = ["--bootstrap-server", ssl_broker_port, - "--producer.config", constants.SSL_CLIENT_CONFIG] + "--producer.config", test.constants.SSL_CLIENT_CONFIG] self.produce_message(topic, producer_config, "key", "message") consumer_config = [ "--bootstrap-server", ssl_broker_port, "--property", "auto.offset.reset=earliest", - "--consumer.config", constants.SSL_CLIENT_CONFIG, + "--consumer.config", test.constants.SSL_CLIENT_CONFIG, ] message = self.consume_message(topic, consumer_config) try: @@ -105,17 +104,17 @@ def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): return errors def broker_restart_flow(self): - print(f"Running {constants.BROKER_RESTART_TESTS}") + print(f"Running {test.constants.BROKER_RESTART_TESTS}") errors = [] try: - self.assertTrue(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) + self.assertTrue(self.create_topic(test.constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) except AssertionError as e: - errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + errors.append(test.constants.BROKER_RESTART_ERROR_PREFIX + str(e)) return errors producer_config = ["--bootstrap-server", "localhost:9092", "--property", "client.id=host"] - self.produce_message(constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") + self.produce_message(test.constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") print("Stopping Container") self.stop_container() @@ -123,60 +122,55 @@ def broker_restart_flow(self): self.resume_container() consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] - message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) + message = self.consume_message(test.constants.BROKER_RESTART_TEST_TOPIC, consumer_config) try: self.assertIsNotNone(message) except AssertionError as e: - errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + errors.append(test.constants.BROKER_RESTART_ERROR_PREFIX + str(e)) return errors try: self.assertEqual(message, "key:message") except AssertionError as e: - errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + errors.append(test.constants.BROKER_RESTART_ERROR_PREFIX + str(e)) if errors: - print(f"Errors in {constants.BROKER_RESTART_TESTS}:- {errors}") + print(f"Errors in {test.constants.BROKER_RESTART_TESTS}:- {errors}") else: - print(f"No errors in {constants.BROKER_RESTART_TESTS}") + print(f"No errors in {test.constants.BROKER_RESTART_TESTS}") return errors def execute(self): total_errors = [] try: - total_errors.extend(self.ssl_flow('localhost:9093', constants.SSL_FLOW_TESTS, constants.SSL_ERROR_PREFIX, constants.SSL_TOPIC)) + total_errors.extend(self.ssl_flow('localhost:9093', test.constants.SSL_FLOW_TESTS, test.constants.SSL_ERROR_PREFIX, test.constants.SSL_TOPIC)) except Exception as e: - print(constants.SSL_ERROR_PREFIX, str(e)) + print(test.constants.SSL_ERROR_PREFIX, str(e)) total_errors.append(str(e)) try: - total_errors.extend(self.ssl_flow('localhost:9094', constants.FILE_INPUT_FLOW_TESTS, constants.FILE_INPUT_ERROR_PREFIX, constants.FILE_INPUT_TOPIC)) + total_errors.extend(self.ssl_flow('localhost:9094', test.constants.FILE_INPUT_FLOW_TESTS, test.constants.FILE_INPUT_ERROR_PREFIX, test.constants.FILE_INPUT_TOPIC)) except Exception as e: - print(constants.FILE_INPUT_ERROR_PREFIX, str(e)) + print(test.constants.FILE_INPUT_ERROR_PREFIX, str(e)) total_errors.append(str(e)) try: total_errors.extend(self.broker_restart_flow()) except Exception as e: - print(constants.BROKER_RESTART_ERROR_PREFIX, str(e)) + print(test.constants.BROKER_RESTART_ERROR_PREFIX, str(e)) total_errors.append(str(e)) self.assertEqual(total_errors, []) class DockerSanityTestJVM(DockerSanityTest): def setUp(self) -> None: - self.start_compose(constants.JVM_COMPOSE) + self.start_compose(test.constants.JVM_COMPOSE) def tearDown(self) -> None: - self.destroy_compose(constants.JVM_COMPOSE) + self.destroy_compose(test.constants.JVM_COMPOSE) def test_bed(self): self.execute() -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("image") - parser.add_argument("mode") - args = parser.parse_args() - - DockerSanityTest.IMAGE = args.image +def run_tests(image, mode): + DockerSanityTest.IMAGE = image test_classes_to_run = [] - if args.mode == "jvm": + if mode == "jvm": test_classes_to_run = [DockerSanityTestJVM] loader = unittest.TestLoader() @@ -185,10 +179,11 @@ def test_bed(self): suite = loader.loadTestsFromTestCase(test_class) suites_list.append(suite) big_suite = unittest.TestSuite(suites_list) - outfile = open(f"report_{args.mode}.html", "w") + outfile = open(f"report_{mode}.html", "w") runner = HTMLTestRunner.HTMLTestRunner( stream=outfile, title='Test Report', description='This demonstrates the report output.' ) - runner.run(big_suite) \ No newline at end of file + result = runner.run(big_suite) + return result.failure_count diff --git a/docker/test/fixtures/file-input/server.properties b/docker/test/fixtures/file-input/server.properties index 0ae8fa924f44a..9647fcb7f2455 100644 --- a/docker/test/fixtures/file-input/server.properties +++ b/docker/test/fixtures/file-input/server.properties @@ -12,14 +12,9 @@ offsets.topic.replication.factor=1 process.roles=broker,controller ssl.client.auth=required ssl.endpoint.identification.algorithm= -ssl.key.credentials=kafka_ssl_key_creds ssl.key.password=abcdefgh -ssl.keystore.credentials=kafka_keystore_creds -ssl.keystore.filename=kafka01.keystore.jks ssl.keystore.location=/etc/kafka/secrets/kafka01.keystore.jks ssl.keystore.password=abcdefgh -ssl.truststore.credentials=kafka_truststore_creds -ssl.truststore.filename=kafka.truststore.jks ssl.truststore.location=/etc/kafka/secrets/kafka.truststore.jks ssl.truststore.password=abcdefgh transaction.state.log.min.isr=1 diff --git a/docker/test/fixtures/secrets/client-ssl.properties b/docker/test/fixtures/secrets/client-ssl.properties index 2ee9218a3fe16..5ecd5582f4e69 100644 --- a/docker/test/fixtures/secrets/client-ssl.properties +++ b/docker/test/fixtures/secrets/client-ssl.properties @@ -1,7 +1,7 @@ security.protocol=SSL -ssl.truststore.location=./fixtures/secrets/kafka.truststore.jks +ssl.truststore.location=./test/fixtures/secrets/kafka.truststore.jks ssl.truststore.password=abcdefgh -ssl.keystore.location=./fixtures/secrets/client.keystore.jks +ssl.keystore.location=./test/fixtures/secrets/client.keystore.jks ssl.keystore.password=abcdefgh ssl.key.password=abcdefgh ssl.enabled.protocols=TLSv1.2,TLSv1.1,TLSv1 diff --git a/docker/test/requirements.txt b/docker/test/requirements.txt deleted file mode 100644 index cb2ef3545ecf1..0000000000000 --- a/docker/test/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -HTMLTestRunner-Python3 \ No newline at end of file From 10b85b9578458c674c20abd895a79f0f8fa1ea2b Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Fri, 10 Nov 2023 14:00:22 +0530 Subject: [PATCH 29/66] Add license comment and refactor ub scripts to remove redundant code --- docker/docker_promote.py | 2 - docker/docker_release.py | 2 - .../resources/common-scripts/log4j.properties | 15 ++ .../common-scripts/server.properties | 14 ++ .../common-scripts/tools-log4j.properties | 14 ++ docker/resources/ub/ub.go | 190 +----------------- docker/resources/ub/ub_test.go | 104 ---------- docker/test/docker_sanity_test.py | 60 +++--- .../fixtures/file-input/server.properties | 15 ++ .../fixtures/secrets/client-ssl.properties | 15 ++ 10 files changed, 105 insertions(+), 326 deletions(-) diff --git a/docker/docker_promote.py b/docker/docker_promote.py index ef8c8421ebf60..45e14932f7d53 100644 --- a/docker/docker_promote.py +++ b/docker/docker_promote.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -15,7 +14,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# """ Python script to promote an rc image. diff --git a/docker/docker_release.py b/docker/docker_release.py index 344476a577bbc..b92aa64bd85e4 100644 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -15,7 +14,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# """ Python script to build and push docker image diff --git a/docker/resources/common-scripts/log4j.properties b/docker/resources/common-scripts/log4j.properties index 148bd53b664b6..7621ac44f42b8 100644 --- a/docker/resources/common-scripts/log4j.properties +++ b/docker/resources/common-scripts/log4j.properties @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + log4j.rootLogger=INFO, stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender diff --git a/docker/resources/common-scripts/server.properties b/docker/resources/common-scripts/server.properties index e9f40eddc1727..caa597ad96df5 100644 --- a/docker/resources/common-scripts/server.properties +++ b/docker/resources/common-scripts/server.properties @@ -1,3 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. advertised.listeners=PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092 controller.listener.names=CONTROLLER controller.quorum.voters=1@localhost:29093 diff --git a/docker/resources/common-scripts/tools-log4j.properties b/docker/resources/common-scripts/tools-log4j.properties index 27d9fbee48bf5..84f0e09405ddd 100644 --- a/docker/resources/common-scripts/tools-log4j.properties +++ b/docker/resources/common-scripts/tools-log4j.properties @@ -1,3 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. log4j.rootLogger=WARN, stderr log4j.appender.stderr=org.apache.log4j.ConsoleAppender diff --git a/docker/resources/ub/ub.go b/docker/resources/ub/ub.go index 194b8480d53eb..6c84fd2c35c90 100644 --- a/docker/resources/ub/ub.go +++ b/docker/resources/ub/ub.go @@ -18,23 +18,15 @@ package main import ( "context" "encoding/json" - "errors" "fmt" "io" - "net" - "net/http" - "net/url" "os" - "os/exec" "os/signal" + pt "path" "regexp" "sort" - "strconv" "strings" "text/template" - "time" - - pt "path" "github.com/spf13/cobra" "golang.org/x/exp/slices" @@ -50,11 +42,6 @@ type ConfigSpec struct { } var ( - bootstrapServers string - configFile string - zookeeperConnect string - security string - re = regexp.MustCompile("[^_]_[^_]") ensureCmd = &cobra.Command{ @@ -84,27 +71,6 @@ var ( Args: cobra.ExactArgs(1), RunE: runRenderPropertiesCmd, } - - waitCmd = &cobra.Command{ - Use: "wait ", - Short: "waits for a service to start listening on a port", - Args: cobra.ExactArgs(3), - RunE: runWaitCmd, - } - - httpReadyCmd = &cobra.Command{ - Use: "http-ready ", - Short: "waits for an HTTP/HTTPS URL to be retrievable", - Args: cobra.ExactArgs(2), - RunE: runHttpReadyCmd, - } - - kafkaReadyCmd = &cobra.Command{ - Use: "kafka-ready ", - Short: "checks if kafka brokers are up and running", - Args: cobra.ExactArgs(2), - RunE: runKafkaReadyCmd, - } ) func ensure(envVar string) bool { @@ -283,27 +249,6 @@ func loadConfigSpec(path string) (ConfigSpec, error) { return spec, nil } -func invokeJavaCommand(className string, jvmOpts string, args []string) bool { - classPath := getEnvOrDefault("UB_CLASSPATH", "/usr/share/java/cp-base-lite/*") - - opts := []string{} - if jvmOpts != "" { - opts = append(opts, jvmOpts) - } - opts = append(opts, "-cp", classPath, className) - cmd := exec.Command("java", append(opts[:], args...)...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - var exitError *exec.ExitError - if errors.As(err, &exitError) { - return exitError.ExitCode() == 0 - } - return false - } - return true -} - func getEnvOrDefault(envVar string, defaultValue string) string { val := os.Getenv(envVar) if len(val) == 0 { @@ -312,89 +257,6 @@ func getEnvOrDefault(envVar string, defaultValue string) string { return val } -func checkKafkaReady(minNumBroker string, timeout string, bootstrapServers string, zookeeperConnect string, configFile string, security string) bool { - - opts := []string{minNumBroker, timeout + "000"} - if bootstrapServers != "" { - opts = append(opts, "-b", bootstrapServers) - } - if zookeeperConnect != "" { - opts = append(opts, "-z", zookeeperConnect) - } - if configFile != "" { - opts = append(opts, "-c", configFile) - } - if security != "" { - opts = append(opts, "-s", security) - } - jvmOpts := os.Getenv("KAFKA_OPTS") - return invokeJavaCommand("io.confluent.admin.utils.cli.KafkaReadyCommand", jvmOpts, opts) -} - -func waitForServer(host string, port int, timeout time.Duration) bool { - address := fmt.Sprintf("%s:%d", host, port) - startTime := time.Now() - connectTimeout := 5 * time.Second - - for { - conn, err := net.DialTimeout("tcp", address, connectTimeout) - if err == nil { - _ = conn.Close() - return true - } - if time.Since(startTime) >= timeout { - return false - } - time.Sleep(1 * time.Second) - } -} - -func waitForHttp(URL string, timeout time.Duration) error { - parsedURL, err := url.Parse(URL) - if err != nil { - return fmt.Errorf("error in parsing url %q: %w", URL, err) - } - - host := parsedURL.Hostname() - portStr := parsedURL.Port() - - if len(host) == 0 { - host = "localhost" - } - - if len(portStr) == 0 { - switch parsedURL.Scheme { - case "http": - portStr = "80" - case "https": - portStr = "443" - default: - return fmt.Errorf("no port specified and cannot infer port based on protocol (only http(s) supported)") - } - } - port, err := strconv.Atoi(portStr) - if err != nil { - return fmt.Errorf("error in parsing port %q: %w", portStr, err) - } - - if !waitForServer(host, port, timeout) { - return fmt.Errorf("service is unreachable on host = %q, port = %q", host, portStr) - } - - httpClient := &http.Client{ - Timeout: timeout * time.Second, - } - resp, err := httpClient.Get(URL) - if err != nil { - return fmt.Errorf("error retrieving url") - } - statusOK := resp.StatusCode >= 200 && resp.StatusCode < 300 - if !statusOK { - return fmt.Errorf("unexpected response for %q with code %d", URL, resp.StatusCode) - } - return nil -} - func runEnsureCmd(_ *cobra.Command, args []string) error { success := ensure(args[0]) if !success { @@ -440,48 +302,6 @@ func runRenderPropertiesCmd(_ *cobra.Command, args []string) error { return nil } -func runWaitCmd(_ *cobra.Command, args []string) error { - port, err := strconv.Atoi(args[1]) - if err != nil { - return fmt.Errorf("error in parsing port %q: %w", args[1], err) - } - - secs, err := strconv.Atoi(args[2]) - if err != nil { - return fmt.Errorf("error in parsing timeout seconds %q: %w", args[2], err) - } - timeout := time.Duration(secs) * time.Second - - success := waitForServer(args[0], port, timeout) - if !success { - return fmt.Errorf("service is unreachable for host %q and port %q", args[0], args[1]) - } - return nil -} - -func runHttpReadyCmd(_ *cobra.Command, args []string) error { - secs, err := strconv.Atoi(args[1]) - if err != nil { - return fmt.Errorf("error in parsing timeout seconds %q: %w", args[1], err) - } - timeout := time.Duration(secs) * time.Second - - success := waitForHttp(args[0], timeout) - if success != nil { - return fmt.Errorf("error in http-ready check for url %q: %w", args[0], success) - } - return nil -} - -func runKafkaReadyCmd(_ *cobra.Command, args []string) error { - success := checkKafkaReady(args[0], args[1], bootstrapServers, zookeeperConnect, configFile, security) - if !success { - err := fmt.Errorf("kafka-ready check failed") - return err - } - return nil -} - func main() { rootCmd := &cobra.Command{ Use: "ub", @@ -489,18 +309,10 @@ func main() { Run: func(cmd *cobra.Command, args []string) {}, } - kafkaReadyCmd.PersistentFlags().StringVarP(&bootstrapServers, "bootstrap-servers", "b", "", "comma-separated list of kafka brokers") - kafkaReadyCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "", "path to the config file") - kafkaReadyCmd.PersistentFlags().StringVarP(&zookeeperConnect, "zookeeper-connect", "z", "", "zookeeper connect string") - kafkaReadyCmd.PersistentFlags().StringVarP(&security, "security", "s", "", "security protocol to use when multiple listeners are enabled.") - rootCmd.AddCommand(pathCmd) rootCmd.AddCommand(ensureCmd) rootCmd.AddCommand(renderTemplateCmd) rootCmd.AddCommand(renderPropertiesCmd) - rootCmd.AddCommand(waitCmd) - rootCmd.AddCommand(httpReadyCmd) - rootCmd.AddCommand(kafkaReadyCmd) ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() diff --git a/docker/resources/ub/ub_test.go b/docker/resources/ub/ub_test.go index 70aedba34c504..b5031a2b86692 100644 --- a/docker/resources/ub/ub_test.go +++ b/docker/resources/ub/ub_test.go @@ -16,13 +16,9 @@ package main import ( - "net" - "net/http" - "net/http/httptest" "os" "reflect" "testing" - "time" ) func assertEqual(a string, b string, t *testing.T) { @@ -359,103 +355,3 @@ func Test_splitToMapDefaults(t *testing.T) { }) } } - -func Test_waitForServer(t *testing.T) { - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer mockServer.Close() - port := mockServer.Listener.Addr().(*net.TCPAddr).Port - - type args struct { - host string - port int - timeout time.Duration - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "invalid server address", - args: args{ - host: "localhost", - port: port + 1, - timeout: time.Duration(5) * time.Second, - }, - want: false, - }, - { - name: "valid server address", - args: args{ - host: "localhost", - port: port, - timeout: time.Duration(5) * time.Second, - }, - want: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := waitForServer(tt.args.host, tt.args.port, tt.args.timeout); !reflect.DeepEqual(got, tt.want) { - t.Errorf("waitForServer() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_waitForHttp(t *testing.T) { - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/names" { - w.WriteHeader(http.StatusOK) - } else { - http.NotFound(w, r) - } - })) - defer mockServer.Close() - - serverURL := mockServer.URL - - type args struct { - URL string - timeout time.Duration - } - tests := []struct { - name string - args args - wantErr bool - }{ - { - name: "valid server address, valid url", - args: args{ - URL: serverURL + "/names", - timeout: time.Duration(5) * time.Second, - }, - wantErr: false, - }, - { - name: "valid server address, invalid url", - args: args{ - URL: serverURL, - timeout: time.Duration(5) * time.Second, - }, - wantErr: true, - }, - { - name: "invalid server address", - args: args{ - URL: "http://invalidAddress:50111/names", - timeout: time.Duration(5) * time.Second, - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := waitForHttp(tt.args.URL, tt.args.timeout); (err != nil) != tt.wantErr { - t.Errorf("waitForHttp() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 242ab37de28f1..217649d506c18 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -16,16 +18,16 @@ import unittest import subprocess from HTMLTestRunner import HTMLTestRunner -import test.constants +import test.constants as constants class DockerSanityTest(unittest.TestCase): IMAGE="apache/kafka" def resume_container(self): - subprocess.run(["docker", "start", test.constants.BROKER_CONTAINER]) + subprocess.run(["docker", "start", constants.BROKER_CONTAINER]) def stop_container(self) -> None: - subprocess.run(["docker", "stop", test.constants.BROKER_CONTAINER]) + subprocess.run(["docker", "stop", constants.BROKER_CONTAINER]) def start_compose(self, filename) -> None: old_string="image: {$IMAGE}" @@ -49,44 +51,44 @@ def destroy_compose(self, filename) -> None: f.write(s) def create_topic(self, topic, topic_config): - command = [test.constants.KAFKA_TOPICS, "--create", "--topic", topic] + command = [constants.KAFKA_TOPICS, "--create", "--topic", topic] command.extend(topic_config) subprocess.run(command) - check_command = [test.constants.KAFKA_TOPICS, "--list"] + check_command = [constants.KAFKA_TOPICS, "--list"] check_command.extend(topic_config) - output = subprocess.check_output(check_command, timeout=test.constants.CLIENT_TIMEOUT) + output = subprocess.check_output(check_command, timeout=constants.CLIENT_TIMEOUT) if topic in output.decode("utf-8"): return True return False def produce_message(self, topic, producer_config, key, value): - command = ["echo", f'"{key}:{value}"', "|", test.constants.KAFKA_CONSOLE_PRODUCER, "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] + command = ["echo", f'"{key}:{value}"', "|", constants.KAFKA_CONSOLE_PRODUCER, "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] command.extend(producer_config) - subprocess.run(["bash", "-c", " ".join(command)], timeout=test.constants.CLIENT_TIMEOUT) + subprocess.run(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) def consume_message(self, topic, consumer_config): - command = [test.constants.KAFKA_CONSOLE_CONSUMER, "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] + command = [constants.KAFKA_CONSOLE_CONSUMER, "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] command.extend(consumer_config) - message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=test.constants.CLIENT_TIMEOUT) + message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip() def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): print(f"Running {test_name}") errors = [] try: - self.assertTrue(self.create_topic(topic, ["--bootstrap-server", ssl_broker_port, "--command-config", test.constants.SSL_CLIENT_CONFIG])) + self.assertTrue(self.create_topic(topic, ["--bootstrap-server", ssl_broker_port, "--command-config", constants.SSL_CLIENT_CONFIG])) except AssertionError as e: errors.append(test_error_prefix + str(e)) return errors producer_config = ["--bootstrap-server", ssl_broker_port, - "--producer.config", test.constants.SSL_CLIENT_CONFIG] + "--producer.config", constants.SSL_CLIENT_CONFIG] self.produce_message(topic, producer_config, "key", "message") consumer_config = [ "--bootstrap-server", ssl_broker_port, "--property", "auto.offset.reset=earliest", - "--consumer.config", test.constants.SSL_CLIENT_CONFIG, + "--consumer.config", constants.SSL_CLIENT_CONFIG, ] message = self.consume_message(topic, consumer_config) try: @@ -104,17 +106,17 @@ def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): return errors def broker_restart_flow(self): - print(f"Running {test.constants.BROKER_RESTART_TESTS}") + print(f"Running {constants.BROKER_RESTART_TESTS}") errors = [] try: - self.assertTrue(self.create_topic(test.constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) + self.assertTrue(self.create_topic(constants.BROKER_RESTART_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) except AssertionError as e: - errors.append(test.constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) return errors producer_config = ["--bootstrap-server", "localhost:9092", "--property", "client.id=host"] - self.produce_message(test.constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") + self.produce_message(constants.BROKER_RESTART_TEST_TOPIC, producer_config, "key", "message") print("Stopping Container") self.stop_container() @@ -122,47 +124,47 @@ def broker_restart_flow(self): self.resume_container() consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] - message = self.consume_message(test.constants.BROKER_RESTART_TEST_TOPIC, consumer_config) + message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) try: self.assertIsNotNone(message) except AssertionError as e: - errors.append(test.constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) return errors try: self.assertEqual(message, "key:message") except AssertionError as e: - errors.append(test.constants.BROKER_RESTART_ERROR_PREFIX + str(e)) + errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) if errors: - print(f"Errors in {test.constants.BROKER_RESTART_TESTS}:- {errors}") + print(f"Errors in {constants.BROKER_RESTART_TESTS}:- {errors}") else: - print(f"No errors in {test.constants.BROKER_RESTART_TESTS}") + print(f"No errors in {constants.BROKER_RESTART_TESTS}") return errors def execute(self): total_errors = [] try: - total_errors.extend(self.ssl_flow('localhost:9093', test.constants.SSL_FLOW_TESTS, test.constants.SSL_ERROR_PREFIX, test.constants.SSL_TOPIC)) + total_errors.extend(self.ssl_flow('localhost:9093', constants.SSL_FLOW_TESTS, constants.SSL_ERROR_PREFIX, constants.SSL_TOPIC)) except Exception as e: - print(test.constants.SSL_ERROR_PREFIX, str(e)) + print(constants.SSL_ERROR_PREFIX, str(e)) total_errors.append(str(e)) try: - total_errors.extend(self.ssl_flow('localhost:9094', test.constants.FILE_INPUT_FLOW_TESTS, test.constants.FILE_INPUT_ERROR_PREFIX, test.constants.FILE_INPUT_TOPIC)) + total_errors.extend(self.ssl_flow('localhost:9094', constants.FILE_INPUT_FLOW_TESTS, constants.FILE_INPUT_ERROR_PREFIX, constants.FILE_INPUT_TOPIC)) except Exception as e: - print(test.constants.FILE_INPUT_ERROR_PREFIX, str(e)) + print(constants.FILE_INPUT_ERROR_PREFIX, str(e)) total_errors.append(str(e)) try: total_errors.extend(self.broker_restart_flow()) except Exception as e: - print(test.constants.BROKER_RESTART_ERROR_PREFIX, str(e)) + print(constants.BROKER_RESTART_ERROR_PREFIX, str(e)) total_errors.append(str(e)) self.assertEqual(total_errors, []) class DockerSanityTestJVM(DockerSanityTest): def setUp(self) -> None: - self.start_compose(test.constants.JVM_COMPOSE) + self.start_compose(constants.JVM_COMPOSE) def tearDown(self) -> None: - self.destroy_compose(test.constants.JVM_COMPOSE) + self.destroy_compose(constants.JVM_COMPOSE) def test_bed(self): self.execute() diff --git a/docker/test/fixtures/file-input/server.properties b/docker/test/fixtures/file-input/server.properties index 9647fcb7f2455..17c5a5938a85b 100644 --- a/docker/test/fixtures/file-input/server.properties +++ b/docker/test/fixtures/file-input/server.properties @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + advertised.listeners=PLAINTEXT://localhost:19092,SSL://localhost:19093,SSL-INT://localhost:9093,BROKER://localhost:9092 controller.listener.names=CONTROLLER controller.quorum.voters=3@broker-ssl-file-input:29093 diff --git a/docker/test/fixtures/secrets/client-ssl.properties b/docker/test/fixtures/secrets/client-ssl.properties index 5ecd5582f4e69..df1b20f259e3a 100644 --- a/docker/test/fixtures/secrets/client-ssl.properties +++ b/docker/test/fixtures/secrets/client-ssl.properties @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + security.protocol=SSL ssl.truststore.location=./test/fixtures/secrets/kafka.truststore.jks ssl.truststore.password=abcdefgh From 625376e3b4d42f6c0d399a3eaf13177465747a52 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Fri, 10 Nov 2023 17:25:14 +0530 Subject: [PATCH 30/66] Add readme file and refactor the python scripts --- docker/README.md | 54 ++++++++++++++++++++++++++++++++++ docker/common.py | 11 +++++++ docker/docker_build_test.py | 18 +++++------- docker/docker_promote.py | 36 ++++++++++++----------- docker/docker_release.py | 32 ++++++++++---------- docker/resources/ub/ub_test.go | 8 ++--- 6 files changed, 110 insertions(+), 49 deletions(-) create mode 100644 docker/README.md create mode 100644 docker/common.py diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000000..1b26da8505f80 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,54 @@ +Docker Images +============= + +This directory contains docker image for Kafka. +The scripts take a url containing kafka as input and generate the respective docker image. +There are interactive python scripts to release the docker image and promote a release candidate. + +Bulding image and running tests locally +--------------------------------------- +- `docker_build_test.py` script builds and tests the docker image. +- kafka binary tarball url along with image name, tag and type is needed to build the image. For detailed usage description check `python docker_build_test.py --help`. +- Sanity tests for the docker image are present in test/docker_sanity_test.py. +- By default image will be built and tested, but if only build is required pass `-b` flag and if only testing the given image is required pass `-t` flag. +- An html test report will be generated after the tests are executed containing the results. + +Bulding image and running tests using github actions +---------------------------------------------------- +This is the recommended way to build, test and get a CVE report for the docker image. +Just choose the image type and provide kafka url to Docker build test workflow. It will generate a test report and CVE report that can be shared to the community. + +Creating a release +------------------ +`docker_release.py` provides an interactive way to build multi arch image and publish it a docker registry. + +Promoting a release +------------------- +`docker_promote.py` provides an interactive way to pull an RC Docker image and promote it to required dockerhub repo. + + +Using the image in a docker container +------------------------------------- +- The image uses the kafka downloaded from provided kafka url +- The image can be run in a container in default mode by running +`docker run -p 9092:9092` +- Default configs run kafka in kraft mode with plaintext listners on 9092 port. +- Default configs can be overriden by user using 2 ways:- + - By mounting folder containing property files + - Mount the folder containing kafka property files to `/mnt/shared/config` + - These files will override the default config files + - Using environment variables + - Kafka properties defined via env variables will override properties defined in file input + - Replace . with _ + - Replace _ with __(double underscore) + - Replace - with ___(triple underscore) + - Prefix the result with KAFKA_ + - Examples: + - For abc.def, use KAFKA_ABC_DEF + - For abc-def, use KAFKA_ABC___DEF + - For abc_def, use KAFKA_ABC__DEF +- Hence order of precedence of properties is the follwing:- + - Env variable (highest) + - File input + - Default (lowest) +- Any env variable that is commonly used in starting kafka(for example, CLUSTER_ID) can be supplied to docker container and it will be available when kafka starts \ No newline at end of file diff --git a/docker/common.py b/docker/common.py new file mode 100644 index 0000000000000..450d97b33a5e2 --- /dev/null +++ b/docker/common.py @@ -0,0 +1,11 @@ +import subprocess + +def execute(command): + if subprocess.run(command).returncode != 0: + raise SystemError("Failure in executing following command:- ", " ".join(command)) + +def get_input(message): + value = input(message) + if value == "": + raise ValueError("This field cannot be empty") + return value \ No newline at end of file diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 25704711d7617..ddc900670bfcc 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -15,29 +15,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -import subprocess from datetime import date import argparse from distutils.dir_util import copy_tree import shutil from test.docker_sanity_test import run_tests +from common import execute def build_jvm(image, tag, kafka_url): image = f'{image}:{tag}' copy_tree("resources", "jvm/resources") - result = subprocess.run(["docker", "build", "-f", "jvm/Dockerfile", "-t", image, "--build-arg", f"kafka_url={kafka_url}", + execute(["docker", "build", "-f", "jvm/Dockerfile", "-t", image, "--build-arg", f"kafka_url={kafka_url}", "--build-arg", f'build_date={date.today()}', "jvm"]) - if result.stderr: - print(result.stdout) - return + shutil.rmtree("jvm/resources") def run_jvm_tests(image, tag, kafka_url): - subprocess.run(["wget", "-nv", "-O", "kafka.tgz", kafka_url]) - subprocess.run(["mkdir", "./test/fixtures/kafka"]) - subprocess.run(["tar", "xfz", "kafka.tgz", "-C", "./test/fixtures/kafka", "--strip-components", "1"]) + execute(["wget", "-nv", "-O", "kafka.tgz", kafka_url]) + execute(["mkdir", "./test/fixtures/kafka"]) + execute(["tar", "xfz", "kafka.tgz", "-C", "./test/fixtures/kafka", "--strip-components", "1"]) failure_count = run_tests(f"{image}:{tag}", "jvm") - subprocess.run(["rm", "kafka.tgz"]) + execute(["rm", "kafka.tgz"]) shutil.rmtree("./test/fixtures/kafka") if failure_count != 0: raise SystemError("Test Failure. Error count is non 0") @@ -46,7 +44,7 @@ def run_jvm_tests(image, tag, kafka_url): parser = argparse.ArgumentParser() parser.add_argument("image", help="Image name that you want to keep for the Docker image") parser.add_argument("-tag", "--image-tag", default="latest", dest="tag", help="Image tag that you want to add to the image") - parser.add_argument("-type", "--image-type", choices=["jvm"], dest="image_type", help="Image type you want to build") + parser.add_argument("-type", "--image-type", choices=["jvm"], default="jvm", dest="image_type", help="Image type you want to build") parser.add_argument("-u", "--kafka-url", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image") parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, help="Only build the image, don't run tests") parser.add_argument("-t", "--test", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image") diff --git a/docker/docker_promote.py b/docker/docker_promote.py index 45e14932f7d53..75cd2fc06da1f 100644 --- a/docker/docker_promote.py +++ b/docker/docker_promote.py @@ -25,13 +25,9 @@ Interactive utility to promote a docker image """ -import subprocess import requests from getpass import getpass - -def execute(command): - if subprocess.run(command).returncode != 0: - raise SystemError("Failure in executing following command:- ", " ".join(command)) +from common import execute, get_input def login(): execute(["docker", "login"]) @@ -58,26 +54,32 @@ def remove(promotion_image_namespace, promotion_image_name, promotion_image_tag, raise SystemError(f"Failed to delete redundant images from dockerhub. Please make sure {promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-amd64 is removed from dockerhub") if requests.delete(f"https://hub.docker.com/v2/repositories/{promotion_image_namespace}/{promotion_image_name}/tags/{promotion_image_tag}-arm64", headers={"Authorization": f"JWT {token}"}).status_code != 204: raise SystemError(f"Failed to delete redundant images from dockerhub. Please make sure {promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-arm64 is removed from dockerhub") - subprocess.run(["docker", "rmi", f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-amd64"]) - subprocess.run(["docker", "rmi", f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-arm64"]) + execute(["docker", "rmi", f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-amd64"]) + execute(["docker", "rmi", f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}-arm64"]) if __name__ == "__main__": login() - username = input("Enter dockerhub username: ") + username = get_input("Enter dockerhub username: ") password = getpass("Enter dockerhub password: ") token = (requests.post("https://hub.docker.com/v2/users/login/", json={"username": username, "password": password})).json()['token'] if len(token) == 0: raise PermissionError("Dockerhub login failed") - rc_image = input("Enter the RC docker image that you want to pull (in the format ::): ") - promotion_image_namespace = input("Enter the dockerhub namespace that the rc image needs to be promoted to [example: apache]: ") - promotion_image_name = input("Enter the dockerhub image name that the rc image needs to be promoted to [example: kafka]: ") - promotion_image_tag = input("Enter the dockerhub image tag that the rc image needs to be promoted to [example: 4.0.0]: ") + rc_image = get_input("Enter the RC docker image that you want to pull (in the format //:): ") + promotion_image_namespace = get_input("Enter the dockerhub namespace that the rc image needs to be promoted to [example: apache]: ") + promotion_image_name = get_input("Enter the dockerhub image name that the rc image needs to be promoted to [example: kafka]: ") + promotion_image_tag = get_input("Enter the dockerhub image tag that the rc image needs to be promoted to [example: latest]: ") promotion_image = f"{promotion_image_namespace}/{promotion_image_name}:{promotion_image_tag}" - pull(rc_image, promotion_image) - push(promotion_image) - push_manifest(promotion_image) - remove(promotion_image_namespace, promotion_image_name, promotion_image_tag, token) - print("The image has been promoted successfully. The promoted image should be accessible in dockerhub") \ No newline at end of file + print(f"Docker image {rc_image} will be pulled and pushed to {promotion_image}") + + proceed = input("Should we proceed? [y/N]: ") + if proceed == "y": + pull(rc_image, promotion_image) + push(promotion_image) + push_manifest(promotion_image) + remove(promotion_image_namespace, promotion_image_name, promotion_image_tag, token) + print("The image has been promoted successfully. The promoted image should be accessible in dockerhub") + else: + print("Image promotion aborted") \ No newline at end of file diff --git a/docker/docker_release.py b/docker/docker_release.py index b92aa64bd85e4..1c55d0779bd3b 100644 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -22,30 +22,34 @@ Interactive utility to push the docker image to dockerhub """ -import subprocess from distutils.dir_util import copy_tree from datetime import date import shutil +from common import execute, get_input + def push_jvm(image, kafka_url): copy_tree("resources", "jvm/resources") - subprocess.run(["docker", "buildx", "build", "-f", "jvm/Dockerfile", "--build-arg", f"kafka_url={kafka_url}", "--build-arg", f"build_date={date.today()}", + execute(["docker", "buildx", "build", "-f", "jvm/Dockerfile", "--build-arg", f"kafka_url={kafka_url}", "--build-arg", f"build_date={date.today()}", "--push", "--platform", "linux/amd64,linux/arm64", "--tag", image, "jvm"]) shutil.rmtree("jvm/resources") def login(): - status = subprocess.run(["docker", "login"]) - if status.returncode != 0: - print("Docker login failed, aborting the docker release") - raise PermissionError + execute(["docker", "login"]) def create_builder(): - subprocess.run(["docker", "buildx", "create", "--name", "kafka-builder", "--use"]) + execute(["docker", "buildx", "create", "--name", "kafka-builder", "--use"]) def remove_builder(): - subprocess.run(["docker", "buildx", "rm", "kafka-builder"]) + execute(["docker", "buildx", "rm", "kafka-builder"]) + +def get_input(message): + value = input(message) + if value == "": + raise ValueError("This field cannot be empty") + return value if __name__ == "__main__": print("\ @@ -56,15 +60,9 @@ def remove_builder(): if docker_registry == "": docker_registry = "docker.io" docker_namespace = input("Enter the docker namespace you want to push the image to: ") - image_name = input("Enter the image name: ") - if image_name == "": - raise ValueError("image name cannot be empty") - image_tag = input("Enter the image tag for the image: ") - if image_tag == "": - raise ValueError("image tag cannot be empty") - kafka_url = input("Enter the url for kafka binary tarball: ") - if kafka_url == "": - raise ValueError("kafka url cannot be empty") + image_name = get_input("Enter the image name: ") + image_tag = get_input("Enter the image tag for the image: ") + kafka_url = get_input("Enter the url for kafka binary tarball: ") image = f"{docker_registry}/{docker_namespace}/{image_name}:{image_tag}" print(f"Docker image containing kafka downloaded from {kafka_url} will be pushed to {image}") proceed = input("Should we proceed? [y/N]: ") diff --git a/docker/resources/ub/ub_test.go b/docker/resources/ub/ub_test.go index b5031a2b86692..9d09c5c413e27 100644 --- a/docker/resources/ub/ub_test.go +++ b/docker/resources/ub/ub_test.go @@ -261,7 +261,6 @@ func Test_buildProperties(t *testing.T) { environment: map[string]string{ "PATH": "thePath", "KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", - "CONFLUENT_METRICS": "metricsValue", "KAFKA_IGNORED": "ignored", "KAFKA_EXCLUDE_PREFIX_PROPERTY": "ignored", }, @@ -272,7 +271,7 @@ func Test_buildProperties(t *testing.T) { name: "server properties", args: args{ spec: ConfigSpec{ - Prefixes: map[string]bool{"KAFKA": false, "CONFLUENT": true}, + Prefixes: map[string]bool{"KAFKA": false}, Excludes: []string{"KAFKA_IGNORED"}, Renamed: map[string]string{}, Defaults: map[string]string{ @@ -284,18 +283,17 @@ func Test_buildProperties(t *testing.T) { environment: map[string]string{ "PATH": "thePath", "KAFKA_BOOTSTRAP_SERVERS": "localhost:9092", - "CONFLUENT_METRICS": "metricsValue", "KAFKA_IGNORED": "ignored", "KAFKA_EXCLUDE_PREFIX_PROPERTY": "ignored", }, }, - want: map[string]string{"bootstrap.servers": "localhost:9092", "confluent.metrics": "metricsValue", "default.property.key": "default.property.value"}, + want: map[string]string{"bootstrap.servers": "localhost:9092", "default.property.key": "default.property.value"}, }, { name: "kafka properties", args: args{ spec: ConfigSpec{ - Prefixes: map[string]bool{"KAFKA": false, "CONFLUENT": true}, + Prefixes: map[string]bool{"KAFKA": false}, Excludes: []string{"KAFKA_IGNORED"}, Renamed: map[string]string{}, Defaults: map[string]string{ From d9cbdd85480b7cfb856b84ecd5e42cffa68e2af1 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 15 Nov 2023 10:40:27 +0530 Subject: [PATCH 31/66] Add local setup section in Readme --- docker/README.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docker/README.md b/docker/README.md index 1b26da8505f80..98730bb70c3a0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -5,22 +5,29 @@ This directory contains docker image for Kafka. The scripts take a url containing kafka as input and generate the respective docker image. There are interactive python scripts to release the docker image and promote a release candidate. +Local Setup +----------- +Make sure you have python (>= 3.7.x) and java (>= 17) installed before running the tests and scripts. + +Run `pip install -r requirements.txt` to get all the requirements for running the scripts. + + Bulding image and running tests locally --------------------------------------- - `docker_build_test.py` script builds and tests the docker image. - kafka binary tarball url along with image name, tag and type is needed to build the image. For detailed usage description check `python docker_build_test.py --help`. - Sanity tests for the docker image are present in test/docker_sanity_test.py. -- By default image will be built and tested, but if only build is required pass `-b` flag and if only testing the given image is required pass `-t` flag. +- By default image will be built and tested, but if you only want to build the image, pass `-b` flag and if you only want to test the given image pass `-t` flag. - An html test report will be generated after the tests are executed containing the results. Bulding image and running tests using github actions ---------------------------------------------------- This is the recommended way to build, test and get a CVE report for the docker image. -Just choose the image type and provide kafka url to Docker build test workflow. It will generate a test report and CVE report that can be shared to the community. +Just choose the image type and provide kafka url to `Docker build test` workflow. It will generate a test report and CVE report that can be shared to the community. Creating a release ------------------ -`docker_release.py` provides an interactive way to build multi arch image and publish it a docker registry. +`docker_release.py` provides an interactive way to build multi arch image and publish it to a docker registry. Promoting a release ------------------- @@ -51,4 +58,4 @@ Using the image in a docker container - Env variable (highest) - File input - Default (lowest) -- Any env variable that is commonly used in starting kafka(for example, CLUSTER_ID) can be supplied to docker container and it will be available when kafka starts \ No newline at end of file +- Any env variable that is commonly used in starting kafka(for example, CLUSTER_ID) can be supplied to docker container and it will be available when kafka starts From 3dcdf2d22892f7c21f6da4a2f257aa1364c30fa9 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 15 Nov 2023 15:50:57 +0530 Subject: [PATCH 32/66] Update file input test and readme file --- .gitignore | 2 ++ docker/README.md | 2 -- docker/test/fixtures/file-input/server.properties | 3 +-- docker/test/fixtures/jvm/docker-compose.yml | 4 ++++ 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index f466af2c59828..7bf18c57cc6b0 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,5 @@ jmh-benchmarks/src/main/generated **/src/generated-test storage/kafka-tiered-storage/ + +docker/report_*.html diff --git a/docker/README.md b/docker/README.md index 98730bb70c3a0..2b20cc6164101 100644 --- a/docker/README.md +++ b/docker/README.md @@ -11,7 +11,6 @@ Make sure you have python (>= 3.7.x) and java (>= 17) installed before running t Run `pip install -r requirements.txt` to get all the requirements for running the scripts. - Bulding image and running tests locally --------------------------------------- - `docker_build_test.py` script builds and tests the docker image. @@ -33,7 +32,6 @@ Promoting a release ------------------- `docker_promote.py` provides an interactive way to pull an RC Docker image and promote it to required dockerhub repo. - Using the image in a docker container ------------------------------------- - The image uses the kafka downloaded from provided kafka url diff --git a/docker/test/fixtures/file-input/server.properties b/docker/test/fixtures/file-input/server.properties index 17c5a5938a85b..cb5d420c76e6a 100644 --- a/docker/test/fixtures/file-input/server.properties +++ b/docker/test/fixtures/file-input/server.properties @@ -22,9 +22,8 @@ listener.name.internal.ssl.endpoint.identification.algorithm= listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SSL-INT:SSL,BROKER:PLAINTEXT,CONTROLLER:PLAINTEXT listeners=PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:19093,SSL-INT://0.0.0.0:9093,BROKER://0.0.0.0:9092,CONTROLLER://broker-ssl-file-input:29093 log.dirs=/tmp/kraft-combined-logs -node.id=3 offsets.topic.replication.factor=1 -process.roles=broker,controller +process.roles=invalid,value ssl.client.auth=required ssl.endpoint.identification.algorithm= ssl.key.password=abcdefgh diff --git a/docker/test/fixtures/jvm/docker-compose.yml b/docker/test/fixtures/jvm/docker-compose.yml index de89fbaf3206f..73238f951be34 100644 --- a/docker/test/fixtures/jvm/docker-compose.yml +++ b/docker/test/fixtures/jvm/docker-compose.yml @@ -80,3 +80,7 @@ services: - ../file-input:/mnt/shared/config environment: CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + # Set a property absent from the file + KAFKA_NODE_ID: 3 + # Override an existing property + KAFKA_PROCESS_ROLES: 'broker,controller' \ No newline at end of file From 01619a0510dcaaebc33fa6d994e4c7ca1d21d472 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 16 Nov 2023 19:04:06 +0530 Subject: [PATCH 33/66] Add test for broker metrics --- docker/test/constants.py | 9 ++- docker/test/docker_sanity_test.py | 73 ++++++++++++++++----- docker/test/fixtures/jvm/docker-compose.yml | 3 + 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/docker/test/constants.py b/docker/test/constants.py index 89c27e0c3baf6..4f1fc3ce59ecf 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -16,6 +16,7 @@ KAFKA_TOPICS="./test/fixtures/kafka/bin/kafka-topics.sh" KAFKA_CONSOLE_PRODUCER="./test/fixtures/kafka/bin/kafka-console-producer.sh" KAFKA_CONSOLE_CONSUMER="./test/fixtures/kafka/bin/kafka-console-consumer.sh" +KAFKA_RUN_CLASS="./test/fixtures/kafka/bin/kafka-run-class.sh" JVM_COMPOSE="./test/fixtures/jvm/docker-compose.yml" @@ -32,6 +33,12 @@ BROKER_CONTAINER="broker" BROKER_RESTART_TEST_TOPIC="test-topic-broker-restart" +BROKER_METRICS_TESTS="Broker Metrics Tests" +BROKER_METRICS_TEST_TOPIC="test-topic-broker-metrics" +JMX_TOOL="org.apache.kafka.tools.JmxTool" +BROKER_METRICS_HEADING='"time","kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec:Count","kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec:EventType","kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec:FifteenMinuteRate","kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec:FiveMinuteRate","kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec:MeanRate","kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec:OneMinuteRate","kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec:RateUnit"' + SSL_ERROR_PREFIX="SSL_ERR" BROKER_RESTART_ERROR_PREFIX="BROKER_RESTART_ERR" -FILE_INPUT_ERROR_PREFIX="FILE_INPUT_ERR" \ No newline at end of file +FILE_INPUT_ERROR_PREFIX="FILE_INPUT_ERR" +BROKER_METRICS_ERROR_PREFIX="BROKER_METRICS_ERR" \ No newline at end of file diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 217649d506c18..b35ba0123a198 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -71,6 +71,55 @@ def consume_message(self, topic, consumer_config): command.extend(consumer_config) message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip() + + def get_metrics(self, jmx_tool_config): + command = [constants.KAFKA_RUN_CLASS, constants.JMX_TOOL] + command.extend(jmx_tool_config) + message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) + return message.decode("utf-8").strip().split() + + def broker_metrics_flow(self): + print(f"Running {constants.BROKER_METRICS_TESTS}") + errors = [] + try: + self.assertTrue(self.create_topic(constants.BROKER_METRICS_TEST_TOPIC, ["--bootstrap-server", "localhost:9092"])) + except AssertionError as e: + errors.append(constants.BROKER_METRICS_ERROR_PREFIX + str(e)) + return errors + jmx_tool_config = ["--one-time", "--object-name", "kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec", "--jmx-url", "service:jmx:rmi:///jndi/rmi://:9101/jmxrmi"] + metrics_before_message = self.get_metrics(jmx_tool_config) + try: + self.assertEqual(len(metrics_before_message), 2) + self.assertEqual(metrics_before_message[0], constants.BROKER_METRICS_HEADING) + except AssertionError as e: + errors.append(constants.BROKER_METRICS_ERROR_PREFIX + str(e)) + return errors + + producer_config = ["--bootstrap-server", "localhost:9092", "--property", "client.id=host"] + self.produce_message(constants.BROKER_METRICS_TEST_TOPIC, producer_config, "key", "message") + consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] + message = self.consume_message(constants.BROKER_METRICS_TEST_TOPIC, consumer_config) + try: + self.assertEqual(message, "key:message") + except AssertionError as e: + errors.append(constants.BROKER_METRICS_ERROR_PREFIX + str(e)) + return errors + + metrics_after_message = self.get_metrics(jmx_tool_config) + try: + self.assertEqual(len(metrics_before_message), 2) + self.assertEqual(metrics_after_message[0], constants.BROKER_METRICS_HEADING) + before_metrics_data, after_metrics_data = metrics_before_message[1].split(","), metrics_after_message[1].split(",") + self.assertEqual(len(before_metrics_data), len(after_metrics_data)) + for i in range(len(before_metrics_data)): + if after_metrics_data[i].replace(".", "").isnumeric(): + self.assertGreater(float(after_metrics_data[i]), float(before_metrics_data[i])) + else: + self.assertEqual(after_metrics_data[i], before_metrics_data[i]) + except AssertionError as e: + errors.append(constants.BROKER_METRICS_ERROR_PREFIX + str(e)) + + return errors def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): print(f"Running {test_name}") @@ -91,18 +140,11 @@ def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): "--consumer.config", constants.SSL_CLIENT_CONFIG, ] message = self.consume_message(topic, consumer_config) - try: - self.assertIsNotNone(message) - except AssertionError as e: - errors.append(test_error_prefix + str(e)) try: self.assertEqual(message, "key:message") except AssertionError as e: errors.append(test_error_prefix + str(e)) - if errors: - print(f"Errors in {test_name}:- {errors}") - else: - print(f"No errors in {test_name}") + return errors def broker_restart_flow(self): @@ -125,23 +167,20 @@ def broker_restart_flow(self): consumer_config = ["--bootstrap-server", "localhost:9092", "--property", "auto.offset.reset=earliest"] message = self.consume_message(constants.BROKER_RESTART_TEST_TOPIC, consumer_config) - try: - self.assertIsNotNone(message) - except AssertionError as e: - errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) - return errors try: self.assertEqual(message, "key:message") except AssertionError as e: errors.append(constants.BROKER_RESTART_ERROR_PREFIX + str(e)) - if errors: - print(f"Errors in {constants.BROKER_RESTART_TESTS}:- {errors}") - else: - print(f"No errors in {constants.BROKER_RESTART_TESTS}") + return errors def execute(self): total_errors = [] + try: + total_errors.extend(self.broker_metrics_flow()) + except Exception as e: + print(constants.BROKER_METRICS_ERROR_PREFIX, str(e)) + total_errors.append(str(e)) try: total_errors.extend(self.ssl_flow('localhost:9093', constants.SSL_FLOW_TESTS, constants.SSL_ERROR_PREFIX, constants.SSL_TOPIC)) except Exception as e: diff --git a/docker/test/fixtures/jvm/docker-compose.yml b/docker/test/fixtures/jvm/docker-compose.yml index 73238f951be34..64a89967fdf28 100644 --- a/docker/test/fixtures/jvm/docker-compose.yml +++ b/docker/test/fixtures/jvm/docker-compose.yml @@ -22,6 +22,7 @@ services: container_name: broker ports: - "9092:9092" + - "9101:9101" environment: KAFKA_NODE_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' @@ -37,6 +38,8 @@ services: KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + KAFKA_JMX_PORT: 9101 + KAFKA_JMX_HOSTNAME: localhost broker-ssl: image: {$IMAGE} From 6a1f038ae7a49481b06034f9a9215738c979fc78 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 16 Nov 2023 19:17:42 +0530 Subject: [PATCH 34/66] Remove potential flakiness from the test --- docker/test/docker_sanity_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index b35ba0123a198..522927545de78 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -113,7 +113,7 @@ def broker_metrics_flow(self): self.assertEqual(len(before_metrics_data), len(after_metrics_data)) for i in range(len(before_metrics_data)): if after_metrics_data[i].replace(".", "").isnumeric(): - self.assertGreater(float(after_metrics_data[i]), float(before_metrics_data[i])) + self.assertGreaterEqual(float(after_metrics_data[i]), float(before_metrics_data[i])) else: self.assertEqual(after_metrics_data[i], before_metrics_data[i]) except AssertionError as e: From c81e63a5ea255afb40e8ccb3b7ee25d53149881e Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 20 Nov 2023 09:53:40 +0530 Subject: [PATCH 35/66] Remove redundant paths in chown and chmod --- docker/jvm/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 0e0fd6ff623db..e6135ef249f76 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -75,12 +75,12 @@ RUN set -eux ; \ wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \ gpg --import KEYS; \ gpg --batch --verify kafka.tgz.asc kafka.tgz; \ - mkdir -p /var/lib/kafka/data /etc/kafka/secrets /var/log/kafka; \ + mkdir -p /var/lib/kafka/data /etc/kafka/secrets; \ mkdir -p /etc/kafka/docker /usr/logs /mnt/shared/config; \ adduser -h /home/appuser -D --shell /bin/bash appuser; \ chown appuser:appuser -R /usr/logs /opt/kafka /mnt/shared/config; \ - chown appuser:root -R /var/lib/kafka /etc/kafka/secrets /var/lib/kafka /etc/kafka /var/log/kafka; \ - chmod -R ug+w /etc/kafka /var/lib/kafka /var/lib/kafka /etc/kafka/secrets /etc/kafka /var/log/kafka; \ + chown appuser:root -R /var/lib/kafka /etc/kafka/secrets /etc/kafka; \ + chmod -R ug+w /etc/kafka /var/lib/kafka /etc/kafka/secrets; \ rm kafka.tgz kafka.tgz.asc KEYS; \ apk del curl wget gpg dirmngr gpg-agent; \ apk cache clean; From 3677b08039387594308314493475d709a372b002 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 20 Nov 2023 11:57:57 +0530 Subject: [PATCH 36/66] Fix build by adding license and excluding files where it cannot be added --- .github/workflows/docker_build_and_test.yml | 15 +++++++++++ build.gradle | 4 ++- docker/common.py | 17 +++++++++++++ docker/jvm/jsa_launch | 14 +++++++++++ docker/jvm/launch | 25 ++++++++----------- docker/requirements.txt | 14 +++++++++++ .../kafka-log4j.properties.template | 14 +++++++++++ .../kafka-tools-log4j.properties.template | 14 +++++++++++ docker/resources/ub/testResources/sampleFile | 14 +++++++++++ docker/resources/ub/testResources/sampleFile2 | 14 +++++++++++ .../ub/testResources/sampleLog4j.template | 14 +++++++++++ docker/test/__init__.py | 16 ++++++++++++ 12 files changed, 160 insertions(+), 15 deletions(-) diff --git a/.github/workflows/docker_build_and_test.yml b/.github/workflows/docker_build_and_test.yml index af2aa121a3679..47fe4d0929a9e 100644 --- a/.github/workflows/docker_build_and_test.yml +++ b/.github/workflows/docker_build_and_test.yml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Docker build test on: diff --git a/build.gradle b/build.gradle index 3b7878a3e8955..1e1ba0b546e3a 100644 --- a/build.gradle +++ b/build.gradle @@ -207,7 +207,9 @@ if (repo != null) { 'streams/streams-scala/logs/*', 'licenses/*', '**/generated/**', - 'clients/src/test/resources/serializedData/*' + 'clients/src/test/resources/serializedData/*', + 'docker/resources/ub/go.sum', + 'docker/test/fixtures/secrets/*' ]) } } else { diff --git a/docker/common.py b/docker/common.py index 450d97b33a5e2..66c4888c17853 100644 --- a/docker/common.py +++ b/docker/common.py @@ -1,3 +1,20 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import subprocess def execute(command): diff --git a/docker/jvm/jsa_launch b/docker/jvm/jsa_launch index 7f40f86f97058..eac2188f272aa 100755 --- a/docker/jvm/jsa_launch +++ b/docker/jvm/jsa_launch @@ -1,4 +1,18 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. KAFKA_CLUSTER_ID="$(opt/kafka/bin/kafka-storage.sh random-uuid)" opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c opt/kafka/config/kraft/server.properties diff --git a/docker/jvm/launch b/docker/jvm/launch index 5d821d1e43773..8c5fec1bfc568 100755 --- a/docker/jvm/launch +++ b/docker/jvm/launch @@ -1,21 +1,18 @@ #!/usr/bin/env bash -############################################################################### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and # limitations under the License. -############################################################################### # Override this section from the script to include the com.sun.management.jmxremote.rmi.port property. diff --git a/docker/requirements.txt b/docker/requirements.txt index bc4fdd44eaf21..f2854bc69ced5 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,2 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. requests HTMLTestRunner-Python3 \ No newline at end of file diff --git a/docker/resources/common-scripts/kafka-log4j.properties.template b/docker/resources/common-scripts/kafka-log4j.properties.template index f9da46cc7443d..7bbc37353e9de 100644 --- a/docker/resources/common-scripts/kafka-log4j.properties.template +++ b/docker/resources/common-scripts/kafka-log4j.properties.template @@ -1,3 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. {{ with $value := getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}{{ if ne $value "INFO" }} log4j.rootLogger={{ $value }}, stdout {{ end }}{{ end }} diff --git a/docker/resources/common-scripts/kafka-tools-log4j.properties.template b/docker/resources/common-scripts/kafka-tools-log4j.properties.template index 4c55b9bb9a2ae..9a7acb94c0438 100644 --- a/docker/resources/common-scripts/kafka-tools-log4j.properties.template +++ b/docker/resources/common-scripts/kafka-tools-log4j.properties.template @@ -1,3 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. {{ with $value := getEnv "KAFKA_TOOLS_LOG4J_LOGLEVEL" "WARN"}} {{if ne $value "WARN"}} log4j.rootLogger={{ $value }}, stderr {{ end }}{{ end }} diff --git a/docker/resources/ub/testResources/sampleFile b/docker/resources/ub/testResources/sampleFile index e69de29bb2d1d..91eacc92e8be9 100755 --- a/docker/resources/ub/testResources/sampleFile +++ b/docker/resources/ub/testResources/sampleFile @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/docker/resources/ub/testResources/sampleFile2 b/docker/resources/ub/testResources/sampleFile2 index e69de29bb2d1d..91eacc92e8be9 100755 --- a/docker/resources/ub/testResources/sampleFile2 +++ b/docker/resources/ub/testResources/sampleFile2 @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/docker/resources/ub/testResources/sampleLog4j.template b/docker/resources/ub/testResources/sampleLog4j.template index 6a6a8630b2c10..8bc1f5e3dbd4d 100644 --- a/docker/resources/ub/testResources/sampleLog4j.template +++ b/docker/resources/ub/testResources/sampleLog4j.template @@ -1,3 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. log4j.rootLogger={{ getEnv "KAFKA_LOG4J_ROOT_LOGLEVEL" "INFO" }}, stdout {{$loggers := getEnv "KAFKA_LOG4J_LOGGERS" "" -}} diff --git a/docker/test/__init__.py b/docker/test/__init__.py index e69de29bb2d1d..977976beec24a 100644 --- a/docker/test/__init__.py +++ b/docker/test/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file From 5193ca1d2872438f086412790f554682a70752cb Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 20 Nov 2023 17:40:46 +0530 Subject: [PATCH 37/66] Update documentation to include docker image in the release process --- docker/README.md | 11 +++++++++++ docker/docker_release.py | 2 ++ release.py | 5 +++++ 3 files changed, 18 insertions(+) diff --git a/docker/README.md b/docker/README.md index 2b20cc6164101..005ce4bec1741 100644 --- a/docker/README.md +++ b/docker/README.md @@ -57,3 +57,14 @@ Using the image in a docker container - File input - Default (lowest) - Any env variable that is commonly used in starting kafka(for example, CLUSTER_ID) can be supplied to docker container and it will be available when kafka starts + +Steps to release docker image +----------------------------- +- Make sure you have executed release.py script to prepare RC tarball in apache sftp server. +- Use the RC tarball url as input kafka url to build docker image and run sanity tests. +- Trigger github actions workflow using the RC branch, provide RC tarball url as kafka url. +- This will generate test report and CVE report for docker images. +- If the reports look fine, RC docker image can be built and published. +- Execute `docker_release.py` script to build and publish RC docker image in your own dockerhub account. +- Share the RC docker image, test report and CVE report with the community in RC vote email. +- Once approved and ready, take help from someone in PMC to trigger `docker_promote.py` script and promote the RC docker image to apache/kafka dockerhub repo diff --git a/docker/docker_release.py b/docker/docker_release.py index 1c55d0779bd3b..3a774ef11f9e4 100644 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -17,6 +17,8 @@ """ Python script to build and push docker image +This script is used to prepare and publish docker release candidate + Usage: docker_release.py Interactive utility to push the docker image to dockerhub diff --git a/release.py b/release.py index 8458323ea9d5e..ca9af6c86b7d7 100755 --- a/release.py +++ b/release.py @@ -759,6 +759,10 @@ def select_gpg_key(): * Release artifacts to be voted upon (source and binary): https://home.apache.org/~%(apache_id)s/kafka-%(rc_tag)s/ + +* Docker release artifact to be voted upon: +/: + * Maven artifacts to be voted upon: https://repository.apache.org/content/groups/staging/org/apache/kafka/ @@ -777,6 +781,7 @@ def select_gpg_key(): * Successful Jenkins builds for the %(dev_branch)s branch: Unit/integration tests: https://ci-builds.apache.org/job/Kafka/job/kafka/job/%(dev_branch)s// System tests: https://jenkins.confluent.io/job/system-test-kafka/job/%(dev_branch)s// +Docker Build Test Pipeline: https://github.com/apache/kafka/actions/runs/ /************************************** From d155e3c5d50c34ea801fbef2e727868c58f1fc74 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Tue, 21 Nov 2023 19:59:47 +0530 Subject: [PATCH 38/66] Add initial documentation and quick start --- docs/docker.html | 75 +++++++++++++++++++++++++++++++++++++++++ docs/documentation.html | 4 ++- docs/quickstart.html | 10 ++++++ docs/toc.html | 1 + 4 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 docs/docker.html diff --git a/docs/docker.html b/docs/docker.html new file mode 100644 index 0000000000000..195d1caecefa8 --- /dev/null +++ b/docs/docker.html @@ -0,0 +1,75 @@ + + + + + + +
diff --git a/docs/documentation.html b/docs/documentation.html index 57707f1be697b..d86efaa61c6bf 100644 --- a/docs/documentation.html +++ b/docs/documentation.html @@ -70,8 +70,10 @@

1.4 Ecosystem

-

1.5 Upgrading From Previous Versions

+

1.5 Upgrading From Previous Versions

+

1.6 Docker

+

2. APIs

diff --git a/docs/quickstart.html b/docs/quickstart.html index f396be4f5d594..811d2f1280101 100644 --- a/docs/quickstart.html +++ b/docs/quickstart.html @@ -76,6 +76,10 @@
Kafka with KRaft
+

Kafka can be run using KRaft mode using local scripts and downloaded files or the docker image. Follow one of the sections below but not both to start the kafka server.

+ +
Using downloaded files
+

Generate a Cluster UUID

@@ -94,6 +98,12 @@ 
$ bin/kafka-server-start.sh config/kraft/server.properties
+
Using docker image
+ +

Start the kafka docker container

+ + 
$ docker run -p 9092:9092 apache/kafka:{{fullDotVersion}}
+

Once the Kafka server has successfully launched, you will have a basic Kafka environment running and ready to use.

diff --git a/docs/toc.html b/docs/toc.html index 73bd66ae41c0f..127905209929d 100644 --- a/docs/toc.html +++ b/docs/toc.html @@ -27,6 +27,7 @@
  • 1.3 Quick Start
  • 1.4 Ecosystem
  • 1.5 Upgrading +
  • 1.6 Docker
  • 2. APIs From ad3687b77482c6ed3aa5f6beb9ce50b8440b51f4 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 22 Nov 2023 17:59:47 +0530 Subject: [PATCH 39/66] Add documentation for ssl and example compose files --- docs/docker.html | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/docs/docker.html b/docs/docker.html index 195d1caecefa8..1b1181b5e3771 100644 --- a/docs/docker.html +++ b/docs/docker.html @@ -20,8 +20,7 @@ From da0604312215d80229991a2ede00203bde3b53a5 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 23 Nov 2023 15:39:04 +0530 Subject: [PATCH 40/66] Add CDS for storage format --- docker/jvm/Dockerfile | 1 + docker/jvm/jsa_launch | 6 ++++-- docker/jvm/launch | 16 +++++++++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index e6135ef249f76..e5c9d7a47cb9a 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -86,6 +86,7 @@ RUN set -eux ; \ apk cache clean; COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa +COPY --from=build-jsa storage.jsa /opt/kafka/storage.jsa COPY --chown=appuser:appuser --from=build-ub /build/ub /usr/bin COPY --chown=appuser:appuser resources/common-scripts /etc/kafka/docker COPY --chown=appuser:appuser launch /etc/kafka/docker/launch diff --git a/docker/jvm/jsa_launch b/docker/jvm/jsa_launch index eac2188f272aa..50d620d6724ac 100755 --- a/docker/jvm/jsa_launch +++ b/docker/jvm/jsa_launch @@ -14,8 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -KAFKA_CLUSTER_ID="$(opt/kafka/bin/kafka-storage.sh random-uuid)" -opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c opt/kafka/config/kraft/server.properties +KAFKA_CLUSTER_ID="5L6g3nShT-eMCtK--X86sw" + +KAFKA_JVM_PERFORMANCE_OPTS="-XX:ArchiveClassesAtExit=storage.jsa" opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c opt/kafka/config/kraft/server.properties + KAFKA_JVM_PERFORMANCE_OPTS="-XX:ArchiveClassesAtExit=kafka.jsa" opt/kafka/bin/kafka-server-start.sh opt/kafka/config/kraft/server.properties & check_timeout() { diff --git a/docker/jvm/launch b/docker/jvm/launch index 8c5fec1bfc568..ccab3e02cda7a 100755 --- a/docker/jvm/launch +++ b/docker/jvm/launch @@ -34,17 +34,23 @@ if [ "$KAFKA_JMX_PORT" ]; then export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" fi +if [ -z "$KAFKA_JVM_PERFORMANCE_OPTS" ]; then + export TEMP_KAFKA_JVM_PERFORMANCE_OPTS="" +else + export TEMP_KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS" +fi + +export KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS -XX:SharedArchiveFile=/opt/kafka/storage.jsa" + echo "===> Using provided cluster id $CLUSTER_ID ..." # A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /opt/kafka/config/server.properties 2>&1) || \ echo $result | grep -i "already formatted" || \ { echo $result && (exit 1) } -if [ -z "$KAFKA_JVM_PERFORMANCE_OPTS" ]; then - export KAFKA_JVM_PERFORMANCE_OPTS="-XX:SharedArchiveFile=/opt/kafka/kafka.jsa" -else - export KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS -XX:SharedArchiveFile=/opt/kafka/kafka.jsa" -fi +export KAFKA_JVM_PERFORMANCE_OPTS="$TEMP_KAFKA_JVM_PERFORMANCE_OPTS" + +export KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS -XX:SharedArchiveFile=/opt/kafka/kafka.jsa" # Start kafka broker exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties From d5f2d559fc2b4867d9bac1646c2f03aaba4f5ab0 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Fri, 24 Nov 2023 16:03:27 +0530 Subject: [PATCH 41/66] Resolve comments - Update label values and remove redundand packages - Allow build and test script to be executable from anywhere --- .gitignore | 3 +- docker/docker_build_test.py | 32 +++++++++----- docker/jvm/Dockerfile | 10 +++-- docker/test/constants.py | 12 ++--- docker/test/docker_sanity_test.py | 44 +++++++++---------- .../fixtures/secrets/client-ssl.properties | 4 +- 6 files changed, 59 insertions(+), 46 deletions(-) diff --git a/.gitignore b/.gitignore index 7bf18c57cc6b0..4ac36a815ba54 100644 --- a/.gitignore +++ b/.gitignore @@ -60,4 +60,5 @@ jmh-benchmarks/src/main/generated storage/kafka-tiered-storage/ -docker/report_*.html +docker/test/report_*.html +__pycache__ diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index ddc900670bfcc..c05abbab94fc8 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -21,22 +21,32 @@ import shutil from test.docker_sanity_test import run_tests from common import execute +import tempfile +import os def build_jvm(image, tag, kafka_url): image = f'{image}:{tag}' - copy_tree("resources", "jvm/resources") - execute(["docker", "build", "-f", "jvm/Dockerfile", "-t", image, "--build-arg", f"kafka_url={kafka_url}", - "--build-arg", f'build_date={date.today()}', "jvm"]) - - shutil.rmtree("jvm/resources") + temp_dir_path = tempfile.mkdtemp() + current_dir = os.path.dirname(os.path.realpath(__file__)) + copy_tree(f"{current_dir}/jvm", f"{temp_dir_path}/jvm") + copy_tree(f"{current_dir}/resources", f"{temp_dir_path}/jvm/resources") + try: + execute(["docker", "build", "-f", f"{temp_dir_path}/jvm/Dockerfile", "-t", image, "--build-arg", f"kafka_url={kafka_url}", + "--build-arg", f'build_date={date.today()}', f"{temp_dir_path}/jvm"]) + except: + print("Docker Image Build failed") + finally: + shutil.rmtree(temp_dir_path) def run_jvm_tests(image, tag, kafka_url): - execute(["wget", "-nv", "-O", "kafka.tgz", kafka_url]) - execute(["mkdir", "./test/fixtures/kafka"]) - execute(["tar", "xfz", "kafka.tgz", "-C", "./test/fixtures/kafka", "--strip-components", "1"]) - failure_count = run_tests(f"{image}:{tag}", "jvm") - execute(["rm", "kafka.tgz"]) - shutil.rmtree("./test/fixtures/kafka") + temp_dir_path = tempfile.mkdtemp() + current_dir = os.path.dirname(os.path.realpath(__file__)) + copy_tree(f"{current_dir}/test/fixtures", f"{temp_dir_path}/fixtures") + execute(["wget", "-nv", "-O", f"{temp_dir_path}/kafka.tgz", kafka_url]) + execute(["mkdir", f"{temp_dir_path}/fixtures/kafka"]) + execute(["tar", "xfz", f"{temp_dir_path}/kafka.tgz", "-C", f"{temp_dir_path}/fixtures/kafka", "--strip-components", "1"]) + failure_count = run_tests(f"{image}:{tag}", "jvm", temp_dir_path) + shutil.rmtree(temp_dir_path) if failure_count != 0: raise SystemError("Test Failure. Error count is non 0") diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index e5c9d7a47cb9a..2e95564984912 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -20,6 +20,8 @@ FROM golang:latest AS build-ub WORKDIR /build RUN useradd --no-log-init --create-home --shell /bin/bash appuser COPY --chown=appuser:appuser resources/ub/ ./ + +# Generate utility belt (ub executable) for dealing with env variables RUN go build -ldflags="-w -s" ./ub.go USER appuser RUN go test ./... @@ -42,6 +44,7 @@ RUN set -eux ; \ wget -nv -O kafka.tgz "$kafka_url"; \ tar xfz kafka.tgz -C /opt/kafka --strip-components 1; +# Generate jsa files using dynamic CDS for kafka server start command and kafka storage format command RUN /etc/kafka/docker/jsa_launch @@ -61,13 +64,12 @@ LABEL org.label-schema.name="kafka" \ org.label-schema.description="Apache Kafka" \ org.label-schema.build-date="${build_date}" \ org.label-schema.vcs-url="https://github.com/apache/kafka" \ - org.label-schema.schema-version="1.0" \ - maintainer="apache" + maintainer="Apache Kafka" RUN set -eux ; \ apk update ; \ apk upgrade ; \ - apk add --no-cache curl wget gpg dirmngr gpg-agent gcompat; \ + apk add --no-cache wget gpg gpg-agent gcompat; \ mkdir opt/kafka; \ wget -nv -O kafka.tgz "$kafka_url"; \ wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \ @@ -82,7 +84,7 @@ RUN set -eux ; \ chown appuser:root -R /var/lib/kafka /etc/kafka/secrets /etc/kafka; \ chmod -R ug+w /etc/kafka /var/lib/kafka /etc/kafka/secrets; \ rm kafka.tgz kafka.tgz.asc KEYS; \ - apk del curl wget gpg dirmngr gpg-agent; \ + apk del wget gpg gpg-agent; \ apk cache clean; COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa diff --git a/docker/test/constants.py b/docker/test/constants.py index 4f1fc3ce59ecf..84c7961f721f3 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -13,17 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -KAFKA_TOPICS="./test/fixtures/kafka/bin/kafka-topics.sh" -KAFKA_CONSOLE_PRODUCER="./test/fixtures/kafka/bin/kafka-console-producer.sh" -KAFKA_CONSOLE_CONSUMER="./test/fixtures/kafka/bin/kafka-console-consumer.sh" -KAFKA_RUN_CLASS="./test/fixtures/kafka/bin/kafka-run-class.sh" +KAFKA_TOPICS="fixtures/kafka/bin/kafka-topics.sh" +KAFKA_CONSOLE_PRODUCER="fixtures/kafka/bin/kafka-console-producer.sh" +KAFKA_CONSOLE_CONSUMER="fixtures/kafka/bin/kafka-console-consumer.sh" +KAFKA_RUN_CLASS="fixtures/kafka/bin/kafka-run-class.sh" -JVM_COMPOSE="./test/fixtures/jvm/docker-compose.yml" +JVM_COMPOSE="fixtures/jvm/docker-compose.yml" CLIENT_TIMEOUT=40 SSL_FLOW_TESTS="SSL Flow Tests" -SSL_CLIENT_CONFIG="./test/fixtures/secrets/client-ssl.properties" +SSL_CLIENT_CONFIG="fixtures/secrets/client-ssl.properties" SSL_TOPIC="test-topic-ssl" FILE_INPUT_FLOW_TESTS="File Input Flow Tests" diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index 522927545de78..c11e1a1fc32ae 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -19,9 +19,11 @@ import subprocess from HTMLTestRunner import HTMLTestRunner import test.constants as constants +import os class DockerSanityTest(unittest.TestCase): IMAGE="apache/kafka" + FIXTURES_DIR="." def resume_container(self): subprocess.run(["docker", "start", constants.BROKER_CONTAINER]) @@ -29,32 +31,28 @@ def resume_container(self): def stop_container(self) -> None: subprocess.run(["docker", "stop", constants.BROKER_CONTAINER]) - def start_compose(self, filename) -> None: - old_string="image: {$IMAGE}" - new_string=f"image: {self.IMAGE}" + def update_file(self, filename, old_string, new_string): with open(filename) as f: s = f.read() with open(filename, 'w') as f: s = s.replace(old_string, new_string) f.write(s) + def start_compose(self, filename) -> None: + self.update_file(filename, "image: {$IMAGE}", f"image: {self.IMAGE}") + self.update_file(f"{self.FIXTURES_DIR}/{constants.SSL_CLIENT_CONFIG}", "{$DIR}", self.FIXTURES_DIR) subprocess.run(["docker-compose", "-f", filename, "up", "-d"]) def destroy_compose(self, filename) -> None: - old_string=f"image: {self.IMAGE}" - new_string="image: {$IMAGE}" subprocess.run(["docker-compose", "-f", filename, "down"]) - with open(filename) as f: - s = f.read() - with open(filename, 'w') as f: - s = s.replace(old_string, new_string) - f.write(s) + self.update_file(filename, f"image: {self.IMAGE}", "image: {$IMAGE}") + self.update_file(f"{self.FIXTURES_DIR}/{constants.SSL_CLIENT_CONFIG}", self.FIXTURES_DIR, "{$DIR}") def create_topic(self, topic, topic_config): - command = [constants.KAFKA_TOPICS, "--create", "--topic", topic] + command = [f"{self.FIXTURES_DIR}/{constants.KAFKA_TOPICS}", "--create", "--topic", topic] command.extend(topic_config) subprocess.run(command) - check_command = [constants.KAFKA_TOPICS, "--list"] + check_command = [f"{self.FIXTURES_DIR}/{constants.KAFKA_TOPICS}", "--list"] check_command.extend(topic_config) output = subprocess.check_output(check_command, timeout=constants.CLIENT_TIMEOUT) if topic in output.decode("utf-8"): @@ -62,18 +60,18 @@ def create_topic(self, topic, topic_config): return False def produce_message(self, topic, producer_config, key, value): - command = ["echo", f'"{key}:{value}"', "|", constants.KAFKA_CONSOLE_PRODUCER, "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] + command = ["echo", f'"{key}:{value}"', "|", f"{self.FIXTURES_DIR}/{constants.KAFKA_CONSOLE_PRODUCER}", "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] command.extend(producer_config) subprocess.run(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) def consume_message(self, topic, consumer_config): - command = [constants.KAFKA_CONSOLE_CONSUMER, "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] + command = [f"{self.FIXTURES_DIR}/{constants.KAFKA_CONSOLE_CONSUMER}", "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] command.extend(consumer_config) message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip() def get_metrics(self, jmx_tool_config): - command = [constants.KAFKA_RUN_CLASS, constants.JMX_TOOL] + command = [f"{self.FIXTURES_DIR}/{constants.KAFKA_RUN_CLASS}", constants.JMX_TOOL] command.extend(jmx_tool_config) message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) return message.decode("utf-8").strip().split() @@ -125,19 +123,19 @@ def ssl_flow(self, ssl_broker_port, test_name, test_error_prefix, topic): print(f"Running {test_name}") errors = [] try: - self.assertTrue(self.create_topic(topic, ["--bootstrap-server", ssl_broker_port, "--command-config", constants.SSL_CLIENT_CONFIG])) + self.assertTrue(self.create_topic(topic, ["--bootstrap-server", ssl_broker_port, "--command-config", f"{self.FIXTURES_DIR}/{constants.SSL_CLIENT_CONFIG}"])) except AssertionError as e: errors.append(test_error_prefix + str(e)) return errors producer_config = ["--bootstrap-server", ssl_broker_port, - "--producer.config", constants.SSL_CLIENT_CONFIG] + "--producer.config", f"{self.FIXTURES_DIR}/{constants.SSL_CLIENT_CONFIG}"] self.produce_message(topic, producer_config, "key", "message") consumer_config = [ "--bootstrap-server", ssl_broker_port, "--property", "auto.offset.reset=earliest", - "--consumer.config", constants.SSL_CLIENT_CONFIG, + "--consumer.config", f"{self.FIXTURES_DIR}/{constants.SSL_CLIENT_CONFIG}", ] message = self.consume_message(topic, consumer_config) try: @@ -201,14 +199,15 @@ def execute(self): class DockerSanityTestJVM(DockerSanityTest): def setUp(self) -> None: - self.start_compose(constants.JVM_COMPOSE) + self.start_compose(f"{self.FIXTURES_DIR}/{constants.JVM_COMPOSE}") def tearDown(self) -> None: - self.destroy_compose(constants.JVM_COMPOSE) + self.destroy_compose(f"{self.FIXTURES_DIR}/{constants.JVM_COMPOSE}") def test_bed(self): self.execute() -def run_tests(image, mode): +def run_tests(image, mode, fixtures_dir): DockerSanityTest.IMAGE = image + DockerSanityTest.FIXTURES_DIR = fixtures_dir test_classes_to_run = [] if mode == "jvm": @@ -220,7 +219,8 @@ def run_tests(image, mode): suite = loader.loadTestsFromTestCase(test_class) suites_list.append(suite) big_suite = unittest.TestSuite(suites_list) - outfile = open(f"report_{mode}.html", "w") + cur_directory = os.path.dirname(os.path.realpath(__file__)) + outfile = open(f"{cur_directory}/report_{mode}.html", "w") runner = HTMLTestRunner.HTMLTestRunner( stream=outfile, title='Test Report', diff --git a/docker/test/fixtures/secrets/client-ssl.properties b/docker/test/fixtures/secrets/client-ssl.properties index df1b20f259e3a..7467ceac5c64e 100644 --- a/docker/test/fixtures/secrets/client-ssl.properties +++ b/docker/test/fixtures/secrets/client-ssl.properties @@ -14,9 +14,9 @@ # limitations under the License. security.protocol=SSL -ssl.truststore.location=./test/fixtures/secrets/kafka.truststore.jks +ssl.truststore.location={$DIR}/fixtures/secrets/kafka.truststore.jks ssl.truststore.password=abcdefgh -ssl.keystore.location=./test/fixtures/secrets/client.keystore.jks +ssl.keystore.location={$DIR}/fixtures/secrets/client.keystore.jks ssl.keystore.password=abcdefgh ssl.key.password=abcdefgh ssl.enabled.protocols=TLSv1.2,TLSv1.1,TLSv1 From 29251ff82df964a98cb994ca4a9a98ff5d48fb11 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Fri, 24 Nov 2023 16:46:25 +0530 Subject: [PATCH 42/66] Refactor release script --- docker/common.py | 20 +++++++++++++++++++- docker/docker_build_test.py | 14 ++------------ docker/docker_release.py | 20 +++++++++----------- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/docker/common.py b/docker/common.py index 66c4888c17853..196ea2ea01fd3 100644 --- a/docker/common.py +++ b/docker/common.py @@ -16,6 +16,10 @@ # limitations under the License. import subprocess +import tempfile +import os +from distutils.dir_util import copy_tree +import shutil def execute(command): if subprocess.run(command).returncode != 0: @@ -25,4 +29,18 @@ def get_input(message): value = input(message) if value == "": raise ValueError("This field cannot be empty") - return value \ No newline at end of file + return value + +def jvm_image(command): + temp_dir_path = tempfile.mkdtemp() + current_dir = os.path.dirname(os.path.realpath(__file__)) + copy_tree(f"{current_dir}/jvm", f"{temp_dir_path}/jvm") + copy_tree(f"{current_dir}/resources", f"{temp_dir_path}/jvm/resources") + command = command.replace("$DOCKER_FILE", f"{temp_dir_path}/jvm/Dockerfile") + command = command.replace("$DOCKER_DIR", f"{temp_dir_path}/jvm") + try: + execute(command.split()) + except: + raise SystemError("Docker Image Build failed") + finally: + shutil.rmtree(temp_dir_path) \ No newline at end of file diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index c05abbab94fc8..ab1a0352036a2 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -20,23 +20,13 @@ from distutils.dir_util import copy_tree import shutil from test.docker_sanity_test import run_tests -from common import execute +from common import execute, jvm_image import tempfile import os def build_jvm(image, tag, kafka_url): image = f'{image}:{tag}' - temp_dir_path = tempfile.mkdtemp() - current_dir = os.path.dirname(os.path.realpath(__file__)) - copy_tree(f"{current_dir}/jvm", f"{temp_dir_path}/jvm") - copy_tree(f"{current_dir}/resources", f"{temp_dir_path}/jvm/resources") - try: - execute(["docker", "build", "-f", f"{temp_dir_path}/jvm/Dockerfile", "-t", image, "--build-arg", f"kafka_url={kafka_url}", - "--build-arg", f'build_date={date.today()}', f"{temp_dir_path}/jvm"]) - except: - print("Docker Image Build failed") - finally: - shutil.rmtree(temp_dir_path) + jvm_image(f"docker build -f $DOCKER_FILE -t {image} --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} $DOCKER_DIR") def run_jvm_tests(image, tag, kafka_url): temp_dir_path = tempfile.mkdtemp() diff --git a/docker/docker_release.py b/docker/docker_release.py index 3a774ef11f9e4..3c33b92457aa5 100644 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -26,17 +26,12 @@ from distutils.dir_util import copy_tree from datetime import date -import shutil -from common import execute, get_input +from common import execute, get_input, jvm_image -def push_jvm(image, kafka_url): - copy_tree("resources", "jvm/resources") - execute(["docker", "buildx", "build", "-f", "jvm/Dockerfile", "--build-arg", f"kafka_url={kafka_url}", "--build-arg", f"build_date={date.today()}", - "--push", - "--platform", "linux/amd64,linux/arm64", - "--tag", image, "jvm"]) - shutil.rmtree("jvm/resources") +def build_push_jvm(image, kafka_url): + jvm_image(f"docker buildx build -f $DOCKER_FILE --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --push \ + --platform linux/amd64,linux/arm64 --tag {image} $DOCKER_DIR") def login(): execute(["docker", "login"]) @@ -57,10 +52,13 @@ def get_input(message): print("\ This script will build and push docker images of apache kafka.\n\ Please ensure that image has been sanity tested before pushing the image") - login() docker_registry = input("Enter the docker registry you want to push the image to [docker.io]: ") if docker_registry == "": docker_registry = "docker.io" + if docker_registry == "docker.io": + login() + else: + print("Please make sure you are logged in to your docker registry and continue") docker_namespace = input("Enter the docker namespace you want to push the image to: ") image_name = get_input("Enter the image name: ") image_tag = get_input("Enter the image tag for the image: ") @@ -71,7 +69,7 @@ def get_input(message): if proceed == "y": print("Building and pushing the image") create_builder() - push_jvm(image, kafka_url) + build_push_jvm(image, kafka_url) remove_builder() print(f"Image has been pushed to {image}") else: From de4bf5bbfd4d58634cd219f6e91818f5c83d28ba Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 27 Nov 2023 15:59:34 +0530 Subject: [PATCH 43/66] Make RC release script non interactive --- docker/docker_build_test.py | 22 ++++++++++++- docker/docker_release.py | 65 ++++++++++++++++++------------------- 2 files changed, 52 insertions(+), 35 deletions(-) diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index ab1a0352036a2..5bc6475c1b4fa 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -15,6 +15,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Python script to build and test a docker image +This script is used to generate a test report + +Usage: + docker_build_test.py --help + Get detailed description of each option + + Example command:- + docker_build_test.py --image-tag --image-type --kafka-url + + This command will build an image with as image name, as image_tag (it will be latest by default), + as image type (jvm by default), for the kafka inside the image and run tests on the image. + -b can be passed as additional argument if you just want to build the image. + -t can be passed if you just want to run tests on the image. +""" + from datetime import date import argparse from distutils.dir_util import copy_tree @@ -37,8 +54,11 @@ def run_jvm_tests(image, tag, kafka_url): execute(["tar", "xfz", f"{temp_dir_path}/kafka.tgz", "-C", f"{temp_dir_path}/fixtures/kafka", "--strip-components", "1"]) failure_count = run_tests(f"{image}:{tag}", "jvm", temp_dir_path) shutil.rmtree(temp_dir_path) + test_report_location_text = f"To view test report please check {current_dir}/test/report_jvm.html" if failure_count != 0: - raise SystemError("Test Failure. Error count is non 0") + raise SystemError(f"{failure_count} tests have failed. {test_report_location_text}") + else: + print(f"All tests passed successfully. {test_report_location_text}") if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/docker/docker_release.py b/docker/docker_release.py index 3c33b92457aa5..4e49edc92ceb1 100644 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -16,25 +16,34 @@ # limitations under the License. """ -Python script to build and push docker image +Python script to build and push a multiarch docker image This script is used to prepare and publish docker release candidate -Usage: docker_release.py +Pre requisites: +Ensure that you are logged in the docker registry and you have access to push to that registry. -Interactive utility to push the docker image to dockerhub +Usage: + docker_release.py --help + Get detailed description of argument + + Example command:- + docker_release --kafka-url -t + This command will build the multiarch image of type (jvm by default), named using to download kafka and push it to the docker image name provided. + Make sure image is in the format of //:. """ -from distutils.dir_util import copy_tree from datetime import date +import argparse -from common import execute, get_input, jvm_image +from common import execute, jvm_image def build_push_jvm(image, kafka_url): - jvm_image(f"docker buildx build -f $DOCKER_FILE --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --push \ + try: + create_builder() + jvm_image(f"docker buildx build -f $DOCKER_FILE --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --push \ --platform linux/amd64,linux/arm64 --tag {image} $DOCKER_DIR") - -def login(): - execute(["docker", "login"]) + finally: + remove_builder() def create_builder(): execute(["docker", "buildx", "create", "--name", "kafka-builder", "--use"]) @@ -42,35 +51,23 @@ def create_builder(): def remove_builder(): execute(["docker", "buildx", "rm", "kafka-builder"]) -def get_input(message): - value = input(message) - if value == "": - raise ValueError("This field cannot be empty") - return value - if __name__ == "__main__": print("\ - This script will build and push docker images of apache kafka.\n\ - Please ensure that image has been sanity tested before pushing the image") - docker_registry = input("Enter the docker registry you want to push the image to [docker.io]: ") - if docker_registry == "": - docker_registry = "docker.io" - if docker_registry == "docker.io": - login() - else: - print("Please make sure you are logged in to your docker registry and continue") - docker_namespace = input("Enter the docker namespace you want to push the image to: ") - image_name = get_input("Enter the image name: ") - image_tag = get_input("Enter the image tag for the image: ") - kafka_url = get_input("Enter the url for kafka binary tarball: ") - image = f"{docker_registry}/{docker_namespace}/{image_name}:{image_tag}" - print(f"Docker image containing kafka downloaded from {kafka_url} will be pushed to {image}") + This script will build and push docker images of apache kafka.\n \ + Please ensure that image has been sanity tested before pushing the image. \n \ + Please ensure you are logged in the docker registry that you are trying to push to.") + parser = argparse.ArgumentParser() + parser.add_argument("image", help="Dockerhub image that you want to push to (in the format //:)") + parser.add_argument("-type", "--image-type", choices=["jvm"], default="jvm", dest="image_type", help="Image type you want to build") + parser.add_argument("-u", "--kafka-url", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image") + args = parser.parse_args() + + print(f"Docker image of type {args.image_type} containing kafka downloaded from {args.kafka_url} will be pushed to {args.image}") proceed = input("Should we proceed? [y/N]: ") if proceed == "y": print("Building and pushing the image") - create_builder() - build_push_jvm(image, kafka_url) - remove_builder() - print(f"Image has been pushed to {image}") + if args.image_type == "jvm": + build_push_jvm(args.image, args.kafka_url) + print(f"Image has been pushed to {args.image}") else: print("Image push aborted") From bc2237c5d52f1b995d48c697fa61384c10005df3 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 27 Nov 2023 16:02:21 +0530 Subject: [PATCH 44/66] Remove the confirmation check from RC release script --- docker/docker_release.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/docker/docker_release.py b/docker/docker_release.py index 4e49edc92ceb1..105582a940cad 100644 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -63,11 +63,8 @@ def remove_builder(): args = parser.parse_args() print(f"Docker image of type {args.image_type} containing kafka downloaded from {args.kafka_url} will be pushed to {args.image}") - proceed = input("Should we proceed? [y/N]: ") - if proceed == "y": - print("Building and pushing the image") - if args.image_type == "jvm": - build_push_jvm(args.image, args.kafka_url) - print(f"Image has been pushed to {args.image}") - else: - print("Image push aborted") + + print("Building and pushing the image") + if args.image_type == "jvm": + build_push_jvm(args.image, args.kafka_url) + print(f"Image has been pushed to {args.image}") From a4694f82968f4b03f7bc3d62381205eb3b009ca6 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 27 Nov 2023 16:41:59 +0530 Subject: [PATCH 45/66] Fix test report path in workflow and fix inline docs --- .github/workflows/docker_build_and_test.yml | 2 +- docker/docker_release.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker_build_and_test.yml b/.github/workflows/docker_build_and_test.yml index 47fe4d0929a9e..d12558e952614 100644 --- a/.github/workflows/docker_build_and_test.yml +++ b/.github/workflows/docker_build_and_test.yml @@ -60,7 +60,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: report_${{ github.event.inputs.image_type }}.html - path: docker/report_${{ github.event.inputs.image_type }}.html + path: docker/test/report_${{ github.event.inputs.image_type }}.html - name: Upload CVE scan report if: always() uses: actions/upload-artifact@v3 diff --git a/docker/docker_release.py b/docker/docker_release.py index 105582a940cad..6d2edcf7b1394 100644 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -27,8 +27,10 @@ Get detailed description of argument Example command:- - docker_release --kafka-url -t - This command will build the multiarch image of type (jvm by default), named using to download kafka and push it to the docker image name provided. + docker_release --kafka-url --image-type + + This command will build the multiarch image of type (jvm by default), + named using to download kafka and push it to the docker image name provided. Make sure image is in the format of //:. """ From 26f460fd5d0dcba494841e45ca7eba7e262e09e1 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 27 Nov 2023 17:16:57 +0530 Subject: [PATCH 46/66] Handle error to ensure files get deleted while running tests --- docker/docker_build_test.py | 18 +++++++++++------- docker/docker_release.py | 5 ++++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index 5bc6475c1b4fa..73022027a6eb7 100644 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -47,13 +47,17 @@ def build_jvm(image, tag, kafka_url): def run_jvm_tests(image, tag, kafka_url): temp_dir_path = tempfile.mkdtemp() - current_dir = os.path.dirname(os.path.realpath(__file__)) - copy_tree(f"{current_dir}/test/fixtures", f"{temp_dir_path}/fixtures") - execute(["wget", "-nv", "-O", f"{temp_dir_path}/kafka.tgz", kafka_url]) - execute(["mkdir", f"{temp_dir_path}/fixtures/kafka"]) - execute(["tar", "xfz", f"{temp_dir_path}/kafka.tgz", "-C", f"{temp_dir_path}/fixtures/kafka", "--strip-components", "1"]) - failure_count = run_tests(f"{image}:{tag}", "jvm", temp_dir_path) - shutil.rmtree(temp_dir_path) + try: + current_dir = os.path.dirname(os.path.realpath(__file__)) + copy_tree(f"{current_dir}/test/fixtures", f"{temp_dir_path}/fixtures") + execute(["wget", "-nv", "-O", f"{temp_dir_path}/kafka.tgz", kafka_url]) + execute(["mkdir", f"{temp_dir_path}/fixtures/kafka"]) + execute(["tar", "xfz", f"{temp_dir_path}/kafka.tgz", "-C", f"{temp_dir_path}/fixtures/kafka", "--strip-components", "1"]) + failure_count = run_tests(f"{image}:{tag}", "jvm", temp_dir_path) + except: + raise SystemError("Failed to run the tests") + finally: + shutil.rmtree(temp_dir_path) test_report_location_text = f"To view test report please check {current_dir}/test/report_jvm.html" if failure_count != 0: raise SystemError(f"{failure_count} tests have failed. {test_report_location_text}") diff --git a/docker/docker_release.py b/docker/docker_release.py index 6d2edcf7b1394..67433ef0fa27e 100644 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -20,7 +20,8 @@ This script is used to prepare and publish docker release candidate Pre requisites: -Ensure that you are logged in the docker registry and you have access to push to that registry. + Ensure that you are logged in the docker registry and you have access to push to that registry. + Ensure that docker buildx is enabled for you. Usage: docker_release.py --help @@ -44,6 +45,8 @@ def build_push_jvm(image, kafka_url): create_builder() jvm_image(f"docker buildx build -f $DOCKER_FILE --build-arg kafka_url={kafka_url} --build-arg build_date={date.today()} --push \ --platform linux/amd64,linux/arm64 --tag {image} $DOCKER_DIR") + except: + raise SystemError("Docker image push failed") finally: remove_builder() From b262984ef9a01a5c2af0499104db1e430e161c4d Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 27 Nov 2023 17:52:42 +0530 Subject: [PATCH 47/66] Update readme file to reflect the change --- docker/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 005ce4bec1741..6fc144b1a0598 100644 --- a/docker/README.md +++ b/docker/README.md @@ -26,7 +26,9 @@ Just choose the image type and provide kafka url to `Docker build test` workflow Creating a release ------------------ -`docker_release.py` provides an interactive way to build multi arch image and publish it to a docker registry. +- `docker_release.py` script builds a multi architecture image and pushes it to provided docker registry. +- Ensure you are logged in to the docker registry before triggering the script. +- kafka binary tarball url along with image name (in the format `//:`) and type is needed to build the image. For detailed usage description check `python docker_release.py --help`. Promoting a release ------------------- From b536088850df09c5579be061cd05373eb6929da5 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Tue, 28 Nov 2023 14:57:04 +0530 Subject: [PATCH 48/66] Remove redundant configs from tests --- .../test/fixtures/file-input/server.properties | 10 ++++------ docker/test/fixtures/jvm/docker-compose.yml | 16 +++++++--------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/docker/test/fixtures/file-input/server.properties b/docker/test/fixtures/file-input/server.properties index cb5d420c76e6a..0e29ff0931770 100644 --- a/docker/test/fixtures/file-input/server.properties +++ b/docker/test/fixtures/file-input/server.properties @@ -13,19 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -advertised.listeners=PLAINTEXT://localhost:19092,SSL://localhost:19093,SSL-INT://localhost:9093,BROKER://localhost:9092 +advertised.listeners=PLAINTEXT://localhost:19092,SSL://localhost:9093 controller.listener.names=CONTROLLER controller.quorum.voters=3@broker-ssl-file-input:29093 group.initial.rebalance.delay.ms=0 -inter.broker.listener.name=BROKER -listener.name.internal.ssl.endpoint.identification.algorithm= -listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SSL-INT:SSL,BROKER:PLAINTEXT,CONTROLLER:PLAINTEXT -listeners=PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:19093,SSL-INT://0.0.0.0:9093,BROKER://0.0.0.0:9092,CONTROLLER://broker-ssl-file-input:29093 +inter.broker.listener.name=PLAINTEXT +listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,CONTROLLER:PLAINTEXT +listeners=PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:9093,CONTROLLER://broker-ssl-file-input:29093 log.dirs=/tmp/kraft-combined-logs offsets.topic.replication.factor=1 process.roles=invalid,value ssl.client.auth=required -ssl.endpoint.identification.algorithm= ssl.key.password=abcdefgh ssl.keystore.location=/etc/kafka/secrets/kafka01.keystore.jks ssl.keystore.password=abcdefgh diff --git a/docker/test/fixtures/jvm/docker-compose.yml b/docker/test/fixtures/jvm/docker-compose.yml index 64a89967fdf28..86a3aed35aae8 100644 --- a/docker/test/fixtures/jvm/docker-compose.yml +++ b/docker/test/fixtures/jvm/docker-compose.yml @@ -25,15 +25,15 @@ services: - "9101:9101" environment: KAFKA_NODE_ID: 1 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' - KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' - KAFKA_LISTENERS: 'PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092' + KAFKA_LISTENERS: 'CONTROLLER://broker:29093,PLAINTEXT://0.0.0.0:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' @@ -51,16 +51,16 @@ services: - ../secrets:/etc/kafka/secrets environment: KAFKA_NODE_ID: 2 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,SSL:SSL,SSL-INT:SSL,BROKER:PLAINTEXT,CONTROLLER:PLAINTEXT" - KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:19092,SSL://localhost:19093,SSL-INT://localhost:9093,BROKER://localhost:9092" + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,SSL:SSL,CONTROLLER:PLAINTEXT" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:19092,SSL://localhost:9093" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_PROCESS_ROLES: 'broker,controller' KAFKA_CONTROLLER_QUORUM_VOTERS: '2@broker-ssl:29093' - KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:19093,SSL-INT://0.0.0.0:9093,BROKER://0.0.0.0:9092,CONTROLLER://broker-ssl:29093" - KAFKA_INTER_BROKER_LISTENER_NAME: "BROKER" + KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:9093,CONTROLLER://broker-ssl:29093" + KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT" KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' @@ -70,8 +70,6 @@ services: KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" KAFKA_SSL_CLIENT_AUTH: "required" - KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - KAFKA_LISTENER_NAME_INTERNAL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" broker-ssl-file-input: image: {$IMAGE} hostname: broker-ssl-file-input From bdb31aa4c0f3615b8ceb25362a795865d17ce21f Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Tue, 28 Nov 2023 17:19:38 +0530 Subject: [PATCH 49/66] Remove advertised.listeners from server properties in controller mode --- docker/resources/common-scripts/configure | 4 ++++ docker/resources/common-scripts/server.properties | 1 + 2 files changed, 5 insertions(+) diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index 4e812ad7cd731..8b16564614641 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -30,6 +30,10 @@ then echo "KAFKA_ADVERTISED_LISTENERS is not supported on a KRaft controller." exit 1 else + # Remove advertised.listeners from default configs + sed '/advertised.listeners=/d' /etc/kafka/docker/server.properties > /etc/kafka/docker/server.properties.temp + mv /etc/kafka/docker/server.properties.temp /etc/kafka/docker/server.properties + # Unset in case env variable is set with empty value unset KAFKA_ADVERTISED_LISTENERS fi else diff --git a/docker/resources/common-scripts/server.properties b/docker/resources/common-scripts/server.properties index caa597ad96df5..7a15f887d0fee 100644 --- a/docker/resources/common-scripts/server.properties +++ b/docker/resources/common-scripts/server.properties @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + advertised.listeners=PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092 controller.listener.names=CONTROLLER controller.quorum.voters=1@localhost:29093 From 1ac48a125e71b630a0f089aff940b1073aa36d42 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Tue, 28 Nov 2023 23:07:33 +0530 Subject: [PATCH 50/66] Copy property files instead of move to fix errors on restart --- docker/resources/common-scripts/configure | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index 8b16564614641..e9e1a9d5bb4f4 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -134,9 +134,9 @@ then fi fi -mv /etc/kafka/docker/server.properties /opt/kafka/config/server.properties -mv /etc/kafka/docker/log4j.properties /opt/kafka/config/log4j.properties -mv /etc/kafka/docker/tools-log4j.properties /opt/kafka/config/tools-log4j.properties +cp /etc/kafka/docker/server.properties /opt/kafka/config/server.properties +cp /etc/kafka/docker/log4j.properties /opt/kafka/config/log4j.properties +cp /etc/kafka/docker/tools-log4j.properties /opt/kafka/config/tools-log4j.properties cp -R /mnt/shared/config/. /opt/kafka/config/ From ed418cf797f45bdd84e9aa29bb53b3675448d065 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 29 Nov 2023 15:56:11 +0530 Subject: [PATCH 51/66] Remove redundant checks and improve script documentation --- docker/jvm/launch | 10 ++++++-- docker/resources/common-scripts/configure | 30 ----------------------- 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/docker/jvm/launch b/docker/jvm/launch index ccab3e02cda7a..37425745df396 100755 --- a/docker/jvm/launch +++ b/docker/jvm/launch @@ -24,7 +24,7 @@ fi # The default for bridged n/w is the bridged IP so you will only be able to connect from another docker container. # For host n/w, this is the IP that the hostname on the host resolves to. -# If you have more that one n/w configured, hostname -i gives you all the IPs, +# If you have more than one n/w configured, hostname -i gives you all the IPs, # the default is to pick the first IP (or network). export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)} @@ -34,23 +34,29 @@ if [ "$KAFKA_JMX_PORT" ]; then export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" fi +# Make a temp env variable to store user provided performance otps if [ -z "$KAFKA_JVM_PERFORMANCE_OPTS" ]; then export TEMP_KAFKA_JVM_PERFORMANCE_OPTS="" else export TEMP_KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS" fi +# We will first use CDS for storage to format storage export KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS -XX:SharedArchiveFile=/opt/kafka/storage.jsa" echo "===> Using provided cluster id $CLUSTER_ID ..." -# A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this +# A bit of a hack to not error out if the storage is already formatted. Need storage-tool to support this result=$(/opt/kafka/bin/kafka-storage.sh format --cluster-id=$CLUSTER_ID -c /opt/kafka/config/server.properties 2>&1) || \ echo $result | grep -i "already formatted" || \ { echo $result && (exit 1) } +# Using temp env variable to get rid of storage CDS command export KAFKA_JVM_PERFORMANCE_OPTS="$TEMP_KAFKA_JVM_PERFORMANCE_OPTS" +# Now we will use CDS for kafka to start kafka server export KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS -XX:SharedArchiveFile=/opt/kafka/kafka.jsa" +echo "===> KAFKA_JVM_PERFORMANCE_OPTS = $KAFKA_JVM_PERFORMANCE_OPTS" + # Start kafka broker exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index e9e1a9d5bb4f4..87438ff7f4f57 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# --- for broker - # If KAFKA_PROCESS_ROLES is defined it means we are running in KRaft mode # unset KAFKA_ADVERTISED_LISTENERS from ENV in KRaft mode when running as controller only @@ -36,8 +34,6 @@ then # Unset in case env variable is set with empty value unset KAFKA_ADVERTISED_LISTENERS fi - else - ub ensure KAFKA_ADVERTISED_LISTENERS fi fi @@ -52,31 +48,6 @@ fi ub path /opt/kafka/config/ writable -# advertised.host, advertised.port, host and port are deprecated. Exit if these properties are set. -if [[ -n "${KAFKA_ADVERTISED_PORT-}" ]] -then - echo "advertised.port is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." - exit 1 -fi - -if [[ -n "${KAFKA_ADVERTISED_HOST-}" ]] -then - echo "advertised.host is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." - exit 1 -fi - -if [[ -n "${KAFKA_HOST-}" ]] -then - echo "host is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." - exit 1 -fi - -if [[ -n "${KAFKA_PORT-}" ]] -then - echo "port is deprecated. Please use KAFKA_ADVERTISED_LISTENERS instead." - exit 1 -fi - # Set if ADVERTISED_LISTENERS has SSL:// or SASL_SSL:// endpoints. if [[ -n "${KAFKA_ADVERTISED_LISTENERS-}" ]] && [[ $KAFKA_ADVERTISED_LISTENERS == *"SSL://"* ]] then @@ -110,7 +81,6 @@ then export KAFKA_SSL_TRUSTSTORE_PASSWORD KAFKA_SSL_TRUSTSTORE_PASSWORD=$(cat "$KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION") fi - fi # Set if KAFKA_ADVERTISED_LISTENERS has SASL_PLAINTEXT:// or SASL_SSL:// endpoints. From fa387335d89d0301f4b691a9ece9f752f898e09f Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 30 Nov 2023 15:37:27 +0530 Subject: [PATCH 52/66] Add test for isolated mode kafka cluster --- docker/jvm/launch | 2 - docker/test/constants.py | 5 +- docker/test/docker_sanity_test.py | 32 +++-- .../fixtures/file-input/server.properties | 4 +- .../jvm/{ => combined}/docker-compose.yml | 9 +- .../jvm/single-controller/docker-compose.yml | 121 ++++++++++++++++++ 6 files changed, 151 insertions(+), 22 deletions(-) rename docker/test/fixtures/jvm/{ => combined}/docker-compose.yml (95%) create mode 100644 docker/test/fixtures/jvm/single-controller/docker-compose.yml diff --git a/docker/jvm/launch b/docker/jvm/launch index 37425745df396..de5e6e008065f 100755 --- a/docker/jvm/launch +++ b/docker/jvm/launch @@ -56,7 +56,5 @@ export KAFKA_JVM_PERFORMANCE_OPTS="$TEMP_KAFKA_JVM_PERFORMANCE_OPTS" # Now we will use CDS for kafka to start kafka server export KAFKA_JVM_PERFORMANCE_OPTS="$KAFKA_JVM_PERFORMANCE_OPTS -XX:SharedArchiveFile=/opt/kafka/kafka.jsa" -echo "===> KAFKA_JVM_PERFORMANCE_OPTS = $KAFKA_JVM_PERFORMANCE_OPTS" - # Start kafka broker exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties diff --git a/docker/test/constants.py b/docker/test/constants.py index 84c7961f721f3..cd80636ecaba3 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -18,9 +18,10 @@ KAFKA_CONSOLE_CONSUMER="fixtures/kafka/bin/kafka-console-consumer.sh" KAFKA_RUN_CLASS="fixtures/kafka/bin/kafka-run-class.sh" -JVM_COMPOSE="fixtures/jvm/docker-compose.yml" +JVM_COMBINED_MODE_COMPOSE="fixtures/jvm/combined/docker-compose.yml" +JVM_ISOLATED_COMPOSE="fixtures/jvm/isolated/docker-compose.yml" -CLIENT_TIMEOUT=40 +CLIENT_TIMEOUT=40000 SSL_FLOW_TESTS="SSL Flow Tests" SSL_CLIENT_CONFIG="fixtures/secrets/client-ssl.properties" diff --git a/docker/test/docker_sanity_test.py b/docker/test/docker_sanity_test.py index c11e1a1fc32ae..3bebaefbfd4dd 100644 --- a/docker/test/docker_sanity_test.py +++ b/docker/test/docker_sanity_test.py @@ -54,26 +54,26 @@ def create_topic(self, topic, topic_config): subprocess.run(command) check_command = [f"{self.FIXTURES_DIR}/{constants.KAFKA_TOPICS}", "--list"] check_command.extend(topic_config) - output = subprocess.check_output(check_command, timeout=constants.CLIENT_TIMEOUT) + output = subprocess.check_output(check_command) if topic in output.decode("utf-8"): return True return False def produce_message(self, topic, producer_config, key, value): - command = ["echo", f'"{key}:{value}"', "|", f"{self.FIXTURES_DIR}/{constants.KAFKA_CONSOLE_PRODUCER}", "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'"] + command = ["echo", f'"{key}:{value}"', "|", f"{self.FIXTURES_DIR}/{constants.KAFKA_CONSOLE_PRODUCER}", "--topic", topic, "--property", "'parse.key=true'", "--property", "'key.separator=:'", "--timeout", f"{constants.CLIENT_TIMEOUT}"] command.extend(producer_config) - subprocess.run(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) + subprocess.run(["bash", "-c", " ".join(command)]) def consume_message(self, topic, consumer_config): - command = [f"{self.FIXTURES_DIR}/{constants.KAFKA_CONSOLE_CONSUMER}", "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1"] + command = [f"{self.FIXTURES_DIR}/{constants.KAFKA_CONSOLE_CONSUMER}", "--topic", topic, "--property", "'print.key=true'", "--property", "'key.separator=:'", "--from-beginning", "--max-messages", "1", "--timeout-ms", f"{constants.CLIENT_TIMEOUT}"] command.extend(consumer_config) - message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) + message = subprocess.check_output(["bash", "-c", " ".join(command)]) return message.decode("utf-8").strip() def get_metrics(self, jmx_tool_config): command = [f"{self.FIXTURES_DIR}/{constants.KAFKA_RUN_CLASS}", constants.JMX_TOOL] command.extend(jmx_tool_config) - message = subprocess.check_output(["bash", "-c", " ".join(command)], timeout=constants.CLIENT_TIMEOUT) + message = subprocess.check_output(["bash", "-c", " ".join(command)]) return message.decode("utf-8").strip().split() def broker_metrics_flow(self): @@ -197,11 +197,19 @@ def execute(self): self.assertEqual(total_errors, []) -class DockerSanityTestJVM(DockerSanityTest): +class DockerSanityTestJVMCombinedMode(DockerSanityTest): def setUp(self) -> None: - self.start_compose(f"{self.FIXTURES_DIR}/{constants.JVM_COMPOSE}") + self.start_compose(f"{self.FIXTURES_DIR}/{constants.JVM_COMBINED_MODE_COMPOSE}") def tearDown(self) -> None: - self.destroy_compose(f"{self.FIXTURES_DIR}/{constants.JVM_COMPOSE}") + self.destroy_compose(f"{self.FIXTURES_DIR}/{constants.JVM_COMBINED_MODE_COMPOSE}") + def test_bed(self): + self.execute() + +class DockerSanityTestJVMIsolatedMode(DockerSanityTest): + def setUp(self) -> None: + self.start_compose(f"{self.FIXTURES_DIR}/{constants.JVM_ISOLATED_COMPOSE}") + def tearDown(self) -> None: + self.destroy_compose(f"{self.FIXTURES_DIR}/{constants.JVM_ISOLATED_COMPOSE}") def test_bed(self): self.execute() @@ -211,14 +219,14 @@ def run_tests(image, mode, fixtures_dir): test_classes_to_run = [] if mode == "jvm": - test_classes_to_run = [DockerSanityTestJVM] + test_classes_to_run = [DockerSanityTestJVMCombinedMode, DockerSanityTestJVMIsolatedMode] loader = unittest.TestLoader() suites_list = [] for test_class in test_classes_to_run: suite = loader.loadTestsFromTestCase(test_class) suites_list.append(suite) - big_suite = unittest.TestSuite(suites_list) + combined_suite = unittest.TestSuite(suites_list) cur_directory = os.path.dirname(os.path.realpath(__file__)) outfile = open(f"{cur_directory}/report_{mode}.html", "w") runner = HTMLTestRunner.HTMLTestRunner( @@ -226,5 +234,5 @@ def run_tests(image, mode, fixtures_dir): title='Test Report', description='This demonstrates the report output.' ) - result = runner.run(big_suite) + result = runner.run(combined_suite) return result.failure_count diff --git a/docker/test/fixtures/file-input/server.properties b/docker/test/fixtures/file-input/server.properties index 0e29ff0931770..9965958123fe2 100644 --- a/docker/test/fixtures/file-input/server.properties +++ b/docker/test/fixtures/file-input/server.properties @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -advertised.listeners=PLAINTEXT://localhost:19092,SSL://localhost:9093 +advertised.listeners=PLAINTEXT://localhost:19093,SSL://localhost:9094 controller.listener.names=CONTROLLER controller.quorum.voters=3@broker-ssl-file-input:29093 group.initial.rebalance.delay.ms=0 @@ -25,7 +25,7 @@ offsets.topic.replication.factor=1 process.roles=invalid,value ssl.client.auth=required ssl.key.password=abcdefgh -ssl.keystore.location=/etc/kafka/secrets/kafka01.keystore.jks +ssl.keystore.location=/etc/kafka/secrets/kafka02.keystore.jks ssl.keystore.password=abcdefgh ssl.truststore.location=/etc/kafka/secrets/kafka.truststore.jks ssl.truststore.password=abcdefgh diff --git a/docker/test/fixtures/jvm/docker-compose.yml b/docker/test/fixtures/jvm/combined/docker-compose.yml similarity index 95% rename from docker/test/fixtures/jvm/docker-compose.yml rename to docker/test/fixtures/jvm/combined/docker-compose.yml index 86a3aed35aae8..fbc53fe2dabde 100644 --- a/docker/test/fixtures/jvm/docker-compose.yml +++ b/docker/test/fixtures/jvm/combined/docker-compose.yml @@ -25,7 +25,7 @@ services: - "9101:9101" environment: KAFKA_NODE_ID: 1 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 @@ -48,7 +48,7 @@ services: ports: - "9093:9093" volumes: - - ../secrets:/etc/kafka/secrets + - ../../secrets:/etc/kafka/secrets environment: KAFKA_NODE_ID: 2 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,SSL:SSL,CONTROLLER:PLAINTEXT" @@ -70,6 +70,7 @@ services: KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" KAFKA_SSL_CLIENT_AUTH: "required" + broker-ssl-file-input: image: {$IMAGE} hostname: broker-ssl-file-input @@ -77,8 +78,8 @@ services: ports: - "9094:9093" volumes: - - ../secrets:/etc/kafka/secrets - - ../file-input:/mnt/shared/config + - ../../secrets:/etc/kafka/secrets + - ../../file-input:/mnt/shared/config environment: CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' # Set a property absent from the file diff --git a/docker/test/fixtures/jvm/single-controller/docker-compose.yml b/docker/test/fixtures/jvm/single-controller/docker-compose.yml new file mode 100644 index 0000000000000..1aa4ad30e9cd6 --- /dev/null +++ b/docker/test/fixtures/jvm/single-controller/docker-compose.yml @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +version: '2' +services: + controller: + image: {$IMAGE} + hostname: controller + container_name: controller + environment: + KAFKA_NODE_ID: 1 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller:29093' + KAFKA_LISTENERS: 'CONTROLLER://controller:29093' + KAFKA_INTER_BROKER_LISTENER_NAME: 'CONTROLLER' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + + broker: + image: {$IMAGE} + hostname: broker + container_name: broker + ports: + - "9092:9092" + - "19091:19091" + - "9101:9101" + volumes: + - ../../secrets:/etc/kafka/secrets + environment: + KAFKA_NODE_ID: 2 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,SSL:SSL,PLAINTEXT:PLAINTEXT' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092,SSL://localhost:19091' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'broker' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller:29093' + KAFKA_LISTENERS: 'PLAINTEXT://0.0.0.0:9092,SSL://0.0.0.0:19091' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + KAFKA_JMX_PORT: 9101 + KAFKA_JMX_HOSTNAME: localhost + KAFKA_SSL_KEYSTORE_FILENAME: "kafka02.keystore.jks" + KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" + KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" + KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" + KAFKA_SSL_CLIENT_AUTH: "required" + + broker-ssl: + image: {$IMAGE} + hostname: broker-ssl + container_name: broker-ssl + ports: + - "9093:9093" + - "19092:19092" + - "19092" + volumes: + - ../../secrets:/etc/kafka/secrets + environment: + KAFKA_NODE_ID: 3 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,SSL:SSL,CONTROLLER:PLAINTEXT" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:19092,SSL://localhost:9093" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'broker' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller:29093' + KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:9093" + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks" + KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" + KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" + KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" + KAFKA_SSL_CLIENT_AUTH: "required" + + broker-ssl-file-input: + image: {$IMAGE} + hostname: broker-ssl-file-input + container_name: broker-ssl-file-input + ports: + - "19093:19093" + - "9094:9094" + volumes: + - ../../secrets:/etc/kafka/secrets + - ../../file-input:/mnt/shared/config + environment: + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + # Set a property absent from the file + KAFKA_NODE_ID: 4 + # Override existing properties + KAFKA_PROCESS_ROLES: 'broker' + KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19093,SSL://0.0.0.0:9094" + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller:29093' \ No newline at end of file From 7cefb7f50c0533c7cc64787c68c087380eeab30e Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 30 Nov 2023 15:46:04 +0530 Subject: [PATCH 53/66] Fix directory naming --- .../{single-controller => isolated}/docker-compose.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) rename docker/test/fixtures/jvm/{single-controller => isolated}/docker-compose.yml (96%) diff --git a/docker/test/fixtures/jvm/single-controller/docker-compose.yml b/docker/test/fixtures/jvm/isolated/docker-compose.yml similarity index 96% rename from docker/test/fixtures/jvm/single-controller/docker-compose.yml rename to docker/test/fixtures/jvm/isolated/docker-compose.yml index 1aa4ad30e9cd6..fefb495017936 100644 --- a/docker/test/fixtures/jvm/single-controller/docker-compose.yml +++ b/docker/test/fixtures/jvm/isolated/docker-compose.yml @@ -68,6 +68,8 @@ services: KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" KAFKA_SSL_CLIENT_AUTH: "required" + depends_on: + - controller broker-ssl: image: {$IMAGE} @@ -100,6 +102,8 @@ services: KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" KAFKA_SSL_CLIENT_AUTH: "required" + depends_on: + - controller broker-ssl-file-input: image: {$IMAGE} @@ -118,4 +122,6 @@ services: # Override existing properties KAFKA_PROCESS_ROLES: 'broker' KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19093,SSL://0.0.0.0:9094" - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller:29093' \ No newline at end of file + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller:29093' + depends_on: + - controller \ No newline at end of file From 0e2f29d7295bcaaeb85dc402305ec6d8868dbb9c Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Thu, 30 Nov 2023 20:28:54 +0530 Subject: [PATCH 54/66] Add examples --- build.gradle | 1 + docker/examples/README.md | 31 +++++++ .../client-secrets/client-ssl.properties | 26 ++++++ .../client-secrets/client.keystore.jks | Bin 0 -> 4382 bytes .../fixtures/file-input/server.properties | 32 +++++++ .../fixtures/secrets/kafka.truststore.jks | Bin 0 -> 1126 bytes .../fixtures/secrets/kafka01.keystore.jks | Bin 0 -> 4382 bytes .../fixtures/secrets/kafka_keystore_creds | 1 + .../fixtures/secrets/kafka_ssl_key_creds | 1 + .../fixtures/secrets/kafka_truststore_creds | 1 + .../examples/jvm/cluster/docker-compose.yml | 78 ++++++++++++++++++ .../jvm/file-input/docker-compose.yml | 35 ++++++++ .../jvm/single-node/docker-compose.yml | 34 ++++++++ docker/examples/jvm/ssl/docker-compose.yml | 47 +++++++++++ 14 files changed, 287 insertions(+) create mode 100644 docker/examples/README.md create mode 100644 docker/examples/fixtures/client-secrets/client-ssl.properties create mode 100644 docker/examples/fixtures/client-secrets/client.keystore.jks create mode 100644 docker/examples/fixtures/file-input/server.properties create mode 100644 docker/examples/fixtures/secrets/kafka.truststore.jks create mode 100644 docker/examples/fixtures/secrets/kafka01.keystore.jks create mode 100644 docker/examples/fixtures/secrets/kafka_keystore_creds create mode 100644 docker/examples/fixtures/secrets/kafka_ssl_key_creds create mode 100644 docker/examples/fixtures/secrets/kafka_truststore_creds create mode 100644 docker/examples/jvm/cluster/docker-compose.yml create mode 100644 docker/examples/jvm/file-input/docker-compose.yml create mode 100644 docker/examples/jvm/single-node/docker-compose.yml create mode 100644 docker/examples/jvm/ssl/docker-compose.yml diff --git a/build.gradle b/build.gradle index 1e1ba0b546e3a..f96acbcd6e0b0 100644 --- a/build.gradle +++ b/build.gradle @@ -210,6 +210,7 @@ if (repo != null) { 'clients/src/test/resources/serializedData/*', 'docker/resources/ub/go.sum', 'docker/test/fixtures/secrets/*' + 'docker/examples/fixtures/secrets/*' ]) } } else { diff --git a/docker/examples/README.md b/docker/examples/README.md new file mode 100644 index 0000000000000..6fd5cba17c73b --- /dev/null +++ b/docker/examples/README.md @@ -0,0 +1,31 @@ +Kafka Docker Image Examples +--------------------------- + +- This directory contains docker compose files for some example configs to run docker image. + +- Run the commands from root of the repository. + +- To bring up the docker compose examples, use docker compose command. + +For example:- +``` +# This command brings up JVM cluster +$ docker compose -f docker/examples/jvm/cluster/docker-compose.yml up +``` + +- Kafka server can be accessed using cli scripts or your own client code. +Make sure jars are built for the code, if cli scripts are being used. + +For example:- +``` +# Produce messages to kafka broker +$ bin/kafka-console-producer.sh --topic quickstart-events --bootstrap-server localhost:29092 +``` + +- Use `./docker/examples/fixtures/client-secrets` for connecting with Kafka when running SSL example. `./docker/examples/fixtures/client-secrets/client-ssl.properties` file can be used as client config. + +For example:- +``` +# Produce message to SSL Kafka example +$ bin/kafka-console-producer.sh --topic test_topic_ssl --bootstrap-server localhost:9093 --producer.config ./docker/examples/fixtures/client-secrets/client-ssl.properties +``` diff --git a/docker/examples/fixtures/client-secrets/client-ssl.properties b/docker/examples/fixtures/client-secrets/client-ssl.properties new file mode 100644 index 0000000000000..39e57d8ba4d15 --- /dev/null +++ b/docker/examples/fixtures/client-secrets/client-ssl.properties @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +security.protocol=SSL +# Provide path to the truststore file +ssl.truststore.location=./docker/examples/fixtures/secrets/kafka.truststore.jks +# This the password for the example keystores and truststores +ssl.truststore.password=abcdefgh +# Provide path to the keystore file +ssl.keystore.location=./docker/examples/fixtures/client-secrets/client.keystore.jks +ssl.keystore.password=abcdefgh +ssl.key.password=abcdefgh +ssl.client.auth=required +ssl.endpoint.identification.algorithm= \ No newline at end of file diff --git a/docker/examples/fixtures/client-secrets/client.keystore.jks b/docker/examples/fixtures/client-secrets/client.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..7873e20ed170dc2a188857df6895245270c6ca55 GIT binary patch literal 4382 zcma)AXEYoPvtEl8%Z6PWHEO&o`idT_cY+W#tG5s>(MyQlTa@T5qW7p#5~96&lno(> z&gwO*-`sQWdB1yqecz8cGxN+m^YfWGgA9h!0|^L`!IV89Qm$~7@ZS&u5&}#xWg}5A zW!)dR3K>k?@c&B0MMS~G`G4S?Kb8R^`(GA0m;i_gCYJdFqL8-#Fo+>YBc#~BQgS3W z5uA`<=iSaI+|Qpn91^w9;H9$eWC{dwPyz{9kmR5{|1%LtN(?|Uf=JE7RS0Ydi3qre zD8Jlla?S>EyGhg2GIXmgdJ_c`eWS7U_!%FstKy<*e=&#d63~S6q0`!_6)3{SU+sM8 z{LX<7N<+n|;v?;>Owp@L)p{@nB_=7BEG~_=FCpmFzg?!@n&H!>ENl>#R||}H1KYZ1 zjKZY|&g1oHB}whXyH{mEbx%TQJFje_`OL2zYGE7~oYn5GhTy(+?28)jum4K?0SY%+4X zYKU+1SFK|vnbV)Qdt2J5DUsSX%}hgHi~Cw9ptt;e)3fy~ zST`~Uz6aPg{F)^aVERfG7Du$5FtKT;sW!v(d}Z*Mu=}-UCf%vHG44Tieda4vOSUX+ zD-rudX9-hsslf(j@<4cZ`4mfHa~cxR<-}EEEoaS0#Zap(pHZQ@5&YzOC|9<8%`;Zv z$uKjs>=t7(hM4uM?aNp|o!`7Esi6be2E+5@)(|Z@e9q@g>F|vesGuXYKsV@rp8f{*V#pUhXwG1y#eb2=$iE9{bgi>qs z8fCC{Mbxx|UetA_(IR^o0vcrxX zdGq0|tNq%Sd&XsX{$|yew!0Wt`W}mHMuT~Z* zHqFOXk0WNT7q7KLqBo@hz9ukp$l0YhGrQIcSdM*#@%^S*RNhT6 zFx{Im9_BRnvJ{tS=JcWq-$N0laG2kolZ}?NLd;u7qFo@Iu3;w8h5Q|fd}C8!Ku1oB zX#;JLmUW5Yxf>U!J1v58zvZOFoxzNo7$|@%(s4SU-~>PynA1>CMM`enETSf1m{*0pO3D{#S+I---1xND#7-j zh;?&#&2bIG15Ot0KGB@OudRW>U~|~^Ng^}>6xYFV0I;aN*b$}&BPyIELfh96f>Er- z{uFmRH|m2VE-B8wzTub*B@-eA8LFS|ziN}AGJ3drcFGue())b9MruK)zfjiMZmpiS z)L-E*Ogvy-ePnC~$)wmRB-Uc_Sb=xOr{;OSjUL!Z#{KV>SQDgQLyom?#c2q2h0Lfh zag5#wpH*_4zaVesS0Gxyxez!ni46|$${&*Helu55q8r6SWp+oew}g&^D6EzahrNor zZ{fNJ)G5dgcd44B9QcB>(N8mZLRGGl8i8){kqc)PuD>?iy_`({TL(0`>hXBzZZ|Yn zUoV^0AayC?D>^smsK%blqg$aHiEaGVoWZ%HbZk-915 z>R0!i6XrTIF>W33mW&H~hgBWPO^P0J@HnI>U_pyui^eMFe?gmNV;WS z*3sxsGlke`bM-V@dvSlU0r#*ezU-3%Y5;09IK=4?(j2tA(`M`OU4F=U2=f_(z_Hp#V`{GSd?A1CH*&aMkP`9yy zyuQa;d|9$0TAyFS^IVulerppGCqKI?VZT zn~<&J&Ws+T|C3=DbD8BsS;^7UvR#c&S>uv{S7YmlqxOrQ#NG~-Hz+8$hfs7JS}()2 zo#-qKWgmoX0H_i7GE&pBTo2Nyb0%H+QZ&9rhYzs-O_Qyk{y-hQ^+9xjj3RWX7ayb* zv8uS@Q=+j;Jpw5731#{=9Q9O7xf{s{A8zm zW$Jl#B|Y{XXrnaHN3Va(c0oxN@!pS$!T;%xTmd6&@0|iF%BqvQ&{j6|V_6!lL9Zu~ zMbxa=1ehWXnl_8NvuLVO|1VQr&lAFV--nl9oTCPfzu;8Dj88GkKNip{5Cv4WG%e>LT7z9b#1Da z7tUkd)1M1oVq*qd;!lVxtixNKn?~JadrG4Iap*i}5hgsbuH6b3^2<#a%M_0rbY=h-5X4Ctn};1 zrL1HJb}i=egf)#r{vFkN=a2;Xrx){j#lfHl?wcx^9fDH{!VW*?yl8(c%ByX^J~rWK zHZ6Ycxf+w_YTElltz#?hH}9On2zfv}l^0vn)@)x_buNJu6XoorQPi%v3CkJDq=ndq zhGq7U*WSpEW*#i39QW7Wk&MScv) zw*WC~cDWGIra>KOyR0{@?lhAM382z zu&xKq<)`mb-F+vxrKi2><&Ll-jRDnxMXDt~FfWfiHVb&U+h@Ja>@->+9$pZ&n&12buM_;}(#3mEAL!=nH zAur~j8b|>o&LNQU+thDZTr0|Wso1Q6P#-ID$KntFh&I*VUrbi;sx17MWv7jr3h*nh0lMGhuY4o^29 z@Y$tU;vmV6U@Aoo`nii%ITe%x2B9$+n+0pVr!;SWnQL3wi}J2`Ves>mGtl50reI>8 zQoIa@tkx`gd7F0B9LLuFm|_+PZ6xQtp!dBXiV1e*I!P?UAnr-XfTps;$^ZA@e95~M z#6~`;_V1rSvqvk0cY8@yKg89TlqbuDdOnvILbJpX+>kV5)dD$%eOxZ%`xV_)Gh%hS zI1B(;MyianAp3Z{rEf`mylz_V4SzC8r(krK1g~Y0PouSZ?ou@{Mc&u_cGF7K>U2;N z46+Fi><%2!voa)&)Xl8`JC>LcU^!4E%=&@c)z@{&W-WjNvVQ~G983uO`O^l>C>`|v z^d%QqhP!RE4?b{6cL)_`Jc{tb75z3ns6ll-h^jaho_JkC_*}1wI%nR`@-I{|kv|e? z?SVDE@tU8W#S#pr?da3vlxRUHJD@PAy{T`an$_czlDnDD!D(eNDF7Cp(LEV`4-(Pk zP}&Se(j+3Q2`qhU-XJf!vjN6{FKzSQAdh#afc3zQYW+0tx>Tc}jG{MZR<6ood6lm7 z`%9|u)wX1&*kVzfrvyTdw1(2W>rW>Vm_7a`^vSD06tVE;6_KX7K$8{IrYhqvq4jMeO@I`eY)xC9J6o357PJ*W-T)F8yb2!5j!#)5P|FB@1yqtZO752@XgDIR48D(rT24<0%EOCIWDJm9k&??MfWDNC9O71OfpC00bb^Rm3+UnTTn#OHPE%<$O)N sHV-%uzF?3Jgj~ii%ky~z6v}D!W84x~O;Cx#kpePducmy3&H@4_5QdEUu>b%7 literal 0 HcmV?d00001 diff --git a/docker/examples/fixtures/secrets/kafka01.keystore.jks b/docker/examples/fixtures/secrets/kafka01.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..953c3355b6e6d0524ed8029abaf8572b93de7416 GIT binary patch literal 4382 zcma)AWmFUlvu1$>mfB@OM35GdUZlGlTm_`Nq)}=WB!#70LVD?1Vd)fUq&p;}LvjgG z@_Ns`=l$;e^?g6)%*-?M%#Zo;oH+vvrl7?I;K70+-2@;`q$=`)3_u9T4TjX=2SYG_ zVKgilSo{A1E77=MiDt|#Ss?BX;x=&bo*8l~$KE_~dq?McPUiDlwYv|LpB z`8+PlUoGc7xK;e~N)-WIx0Ea}dQU)k4nke?>63l5_|9+2h#ALMFRed=j5A~wiBxUF zVc>UHHI>lUlB2q9S85HXpCSv?ZIp&maEClSkk{+L8;7Z;*!cy4ae|T(Y{NkPv-d)I zfz#^Wq|m1}V7zjNaxi0ah1qVq+G6c#H9raoe26g-#LeHS6DibTXr`72H52ypDkpqr zJm4&}ae1_)DkjiYfn}DSQ%Qf*Z#Dnu(7Fq8n`?6JwBWMwlkOc6u+2cBfSJf=+R~Mh z=^e10+lsP|g5N)~c@Zi?WdwSd6|-^uGjmlc>$6;_W=2o{rFTGYm7*PlYqcf(G||-}SoWqaH}e z7d}`|6O~IB(&Ksh#Mo=Y-rGXi_D~w-BXiLw`{pn%UZO{gb9ToG>ZNlfYsPE4C`Ki9 z(P_{FCi51kDf#K;l_ZdBmV2f`3G0>=CniXIhI#!yl&lz zm_5|F1R^LXmvJT-V?eAbc|hXN?(A779^741*;tbbzK>om$CyyN@|&`9Xg~|9aK-%Q zJU`odP4Ht1$ix8p&%|)dw?f;izbcgPWu7hbwW&vBaU)^AT~Hg?9-pczHo*o1Cc09m5IhXL~K;cwzHf3f05~9X6LYv8*_c#i`kV26D&Z z&a1|aS%Lf2&#N=)uFNRy+CK zA0j=S=;qz(T*zLyhHfpVx!>TpPM#63aO{vf-xdLebDZx2}^ zqS3AIPx+JVb)i;e*(8Hka<1KF0}*1ba6N}XJCdW8MCBg1?xy+r91(6bCW(79 z%HANM7y0^5KxZ>Zzk?k`SmZWm@`=JEg3#`sDeoBM zWxv(v7TignSCM+=MSRF6)sbvkko&yx^bbdOQMJ^wYs4&fwGqAv>q-{d^fuk_(+uNS z3BNLo`5(CiQSnnTKyjRL+;FUMEdP4@zmW%y=l^ZF$4`Y1)pc;OW#ShW6BQB_6c!c{ zc={9;OpN_!4k1BqF!9=7Xc-rP^ViM(s{;6UV!gnmYpJ$zVl$XER8y8+d0~U2O8B3M z^%;nPtt{+7(6o`u$djdqGBB9fVvMLnt0Gm#vHqEL#m(u*wLt1pG{Q_q%I@`wM$gM)m0%p(S(?3_7@o_^s4~fy-YZ*W@Ztj(?q&{mY#DR3Ycn?Uvi{lLOy`5;us`CY{>V1~ z=88kxQaEL|0@-1x%(jPjT3_g)oyKcEzO(K8U%esan^*lCUsrFoL}?2y35^xBoAzRT zBYeRuC%L^*G>=6NZid_+qDE-v><wBg{f*GRn zvm4P8B&ev!z`eIaU3OEEfT)JjX{K==VU`qcdFV%k?vg##nUW#W!&($Xj1_1+N5RGI zU#Nsbkjx6Sa_SA-1e&H0ee6qPfdA2^sKskouV*hGMT=u}t3~8sU(;%6#+*sy9i zgoj8BNDV$XI1nRkD}WVl8c>zLe8;sd9yfl?Z{%!*KjCwC+Vv*0_S?q2$R~jDxjRbp&j63ERR)X#10-2QJTXV5Lw?m!E*{z)5dBJjjqx8L3GrqNb z?1kHRGoB%;C( z^??*G`g97NFs%|BJ>+jQ6L-#h*Nr5w_a}Wf{JuEXb-oVqRE-lkkn zk62V%gsYWaTe{bNZz$43>rSGK8-jtE`^inIy$O{|kmkP9i7U}SXbw+UA>BniDqzkm zwi~j-KJsjt?(~l3YfLoW&59(c*o#@0bD^|;&@3DzZ_uyi(dyI4)JIJs9O3YFSiEvN zQ^!tm*H7ZWV+6nc4Zrp^Z_pFHBgEPQUw&nR8D(sQmcB;~l|=nCfTnrv%P2V|Gcbd} zmfNU@O8M7;h%r~)X(G6`7tk<4tsHjM64ddGb&_9Tw85!er&BCy3;f52dEeLzh4=lC zX=f+q6cKvWuv2qif=+2=h3scy$3rrQpT+Hd<3{;3WsPe&P*1@b4-$?z?qQp#*TlO- z!PIt+yIU#~;f=3CNB`VD^EmfNrUYr1?Sdvh$J~;xwMavueu>OyY%Ky=7O&P_#zisfLMUi)*6=^29tVAAd4(0#ffHw7dLhxfokdz&LK7sAWz=epb*F zyexevC?Wd#DAd&S4B;CXnfKJtE-CD*>~O~$pV1he{q`hzAqGWD1k_AMV*YRjZy8E* zD4R_`%Ydaf+~VuWGZ~Z@nyvcyAr&v}#w*_?Q?Br^_V+_ykmIe)dc?NnbMv#%J}z#F zT77a_JH~C1*ye|WvTd}A-cyS%zs0L-iu9k+&!1o{tQ&eAn|}b`AmSRavxM{T7r(S8 z&U0_`xI53Bp^c0MqQotBf{j{_i>EXgFJondc^}~ta+V9-d3NPn>q!tl$75cgOJP8# z=c1Ch%x&ggE7;KN_NL@tgc~?Xlz_7kO{^Iry^Az&HYOtSTk^IYD=?0^EY{3kFwcrl z%Ml&ZQa$;vTr;{!GQkwO&M+&o-E2WzzHyw$R`^--&wB9rdqn0%y z5XJoJI_7zm&YEI#-Onk|L?(e{)+1xtH7hD4rE9&#>5`Du$-tvIEX zd>mJzV6!S(T{N>mhllnUZ?#h-vLFR}r9a?lZbOILUoe(*tf06rmV37a!;b>L0>8DZ zKGY)VR7gB^Y7}111uyT&?;w=uUwNOGWhf}C5Fiw(M~tO3rEod})j|Y4Wn7KGJh_`s?@&oX_P*z;uKht%8xok` zrmSBA3s3et(hR&Q-OFXA-xNiK9sUUy8O``nSj}V~H^(y;Mf6is7+LUUuSG%RSog`1 zf8FcWF3L3PioK#u?3HVlh%oASrb6B9XwH+8qjtUhZS!Yml_jOTh2{kXM)d5>y=E!f z?bh^QaaS6GbX=DrmAQ0{ctH$UKx>87aVu@5J_|J(+GYCKI*6WJLsAbZ)$3y&AUh@} z59u};1Nze{MVnT4W+e=VlNT5fyJwwW`VR@l-TB%P#Qt$u=Bz3(r0yh6_+a$m*4@DH z9mf$ZuNDXF?DkZ>etM0Y@#yHIB*jOCzy$zt@R?%1-bI|h4LJ{AaDaOE w1TbiNf|J`G2Pq#JlVoS}%Yg8y{4r~NEpfk9XGRIB44 Date: Fri, 1 Dec 2023 14:57:08 +0530 Subject: [PATCH 55/66] Rename ub to utility and fix test compose files --- docker/jvm/Dockerfile | 10 +-- docker/resources/common-scripts/configure | 32 +++---- docker/resources/{ub => utility}/go.mod | 0 docker/resources/{ub => utility}/go.sum | 0 .../{ub => utility}/testResources/sampleFile | 0 .../{ub => utility}/testResources/sampleFile2 | 0 .../testResources/sampleLog4j.template | 0 .../{ub/ub.go => utility/utility.go} | 4 +- .../ub_test.go => utility/utility_test.go} | 0 docker/test/constants.py | 2 +- .../fixtures/file-input/server.properties | 4 +- .../fixtures/jvm/combined/docker-compose.yml | 49 +++++++---- .../fixtures/jvm/isolated/docker-compose.yml | 87 +++++++++++++------ 13 files changed, 118 insertions(+), 70 deletions(-) rename docker/resources/{ub => utility}/go.mod (100%) rename docker/resources/{ub => utility}/go.sum (100%) rename docker/resources/{ub => utility}/testResources/sampleFile (100%) rename docker/resources/{ub => utility}/testResources/sampleFile2 (100%) rename docker/resources/{ub => utility}/testResources/sampleLog4j.template (100%) rename docker/resources/{ub/ub.go => utility/utility.go} (99%) rename docker/resources/{ub/ub_test.go => utility/utility_test.go} (100%) diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 2e95564984912..46eb5c9a85ffd 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -16,13 +16,13 @@ # limitations under the License. ############################################################################### -FROM golang:latest AS build-ub +FROM golang:latest AS build-utility WORKDIR /build RUN useradd --no-log-init --create-home --shell /bin/bash appuser -COPY --chown=appuser:appuser resources/ub/ ./ +COPY --chown=appuser:appuser resources/utility/ ./ -# Generate utility belt (ub executable) for dealing with env variables -RUN go build -ldflags="-w -s" ./ub.go +# Generate utility executable for dealing with env variables +RUN go build -ldflags="-w -s" ./utility.go USER appuser RUN go test ./... @@ -89,7 +89,7 @@ RUN set -eux ; \ COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa COPY --from=build-jsa storage.jsa /opt/kafka/storage.jsa -COPY --chown=appuser:appuser --from=build-ub /build/ub /usr/bin +COPY --chown=appuser:appuser --from=build-utility /build/utility /usr/bin COPY --chown=appuser:appuser resources/common-scripts /etc/kafka/docker COPY --chown=appuser:appuser launch /etc/kafka/docker/launch diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index 87438ff7f4f57..cbc85ed658c6c 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -20,7 +20,7 @@ if [[ -n "${KAFKA_PROCESS_ROLES-}" ]] then echo "Running in KRaft mode..." - ub ensure CLUSTER_ID + utility ensure CLUSTER_ID if [[ $KAFKA_PROCESS_ROLES == "controller" ]] then if [[ -n "${KAFKA_ADVERTISED_LISTENERS-}" ]] @@ -46,38 +46,38 @@ then KAFKA_LISTENERS=$(echo "$KAFKA_ADVERTISED_LISTENERS" | sed -e 's|://[^:]*:|://0.0.0.0:|g') fi -ub path /opt/kafka/config/ writable +utility path /opt/kafka/config/ writable # Set if ADVERTISED_LISTENERS has SSL:// or SASL_SSL:// endpoints. if [[ -n "${KAFKA_ADVERTISED_LISTENERS-}" ]] && [[ $KAFKA_ADVERTISED_LISTENERS == *"SSL://"* ]] then echo "SSL is enabled." - ub ensure KAFKA_SSL_KEYSTORE_FILENAME + utility ensure KAFKA_SSL_KEYSTORE_FILENAME export KAFKA_SSL_KEYSTORE_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_KEYSTORE_FILENAME" - ub path "$KAFKA_SSL_KEYSTORE_LOCATION" existence + utility path "$KAFKA_SSL_KEYSTORE_LOCATION" existence - ub ensure KAFKA_SSL_KEY_CREDENTIALS + utility ensure KAFKA_SSL_KEY_CREDENTIALS KAFKA_SSL_KEY_CREDENTIALS_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_KEY_CREDENTIALS" - ub path "$KAFKA_SSL_KEY_CREDENTIALS_LOCATION" existence + utility path "$KAFKA_SSL_KEY_CREDENTIALS_LOCATION" existence export KAFKA_SSL_KEY_PASSWORD KAFKA_SSL_KEY_PASSWORD=$(cat "$KAFKA_SSL_KEY_CREDENTIALS_LOCATION") - ub ensure KAFKA_SSL_KEYSTORE_CREDENTIALS + utility ensure KAFKA_SSL_KEYSTORE_CREDENTIALS KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_KEYSTORE_CREDENTIALS" - ub path "$KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION" existence + utility path "$KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION" existence export KAFKA_SSL_KEYSTORE_PASSWORD KAFKA_SSL_KEYSTORE_PASSWORD=$(cat "$KAFKA_SSL_KEYSTORE_CREDENTIALS_LOCATION") if [[ -n "${KAFKA_SSL_CLIENT_AUTH-}" ]] && ( [[ $KAFKA_SSL_CLIENT_AUTH == *"required"* ]] || [[ $KAFKA_SSL_CLIENT_AUTH == *"requested"* ]] ) then - ub ensure KAFKA_SSL_TRUSTSTORE_FILENAME + utility ensure KAFKA_SSL_TRUSTSTORE_FILENAME export KAFKA_SSL_TRUSTSTORE_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_TRUSTSTORE_FILENAME" - ub path "$KAFKA_SSL_TRUSTSTORE_LOCATION" existence + utility path "$KAFKA_SSL_TRUSTSTORE_LOCATION" existence - ub ensure KAFKA_SSL_TRUSTSTORE_CREDENTIALS + utility ensure KAFKA_SSL_TRUSTSTORE_CREDENTIALS KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION="/etc/kafka/secrets/$KAFKA_SSL_TRUSTSTORE_CREDENTIALS" - ub path "$KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION" existence + utility path "$KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION" existence export KAFKA_SSL_TRUSTSTORE_PASSWORD KAFKA_SSL_TRUSTSTORE_PASSWORD=$(cat "$KAFKA_SSL_TRUSTSTORE_CREDENTIALS_LOCATION") fi @@ -88,7 +88,7 @@ if [[ -n "${KAFKA_ADVERTISED_LISTENERS-}" ]] && [[ $KAFKA_ADVERTISED_LISTENERS = then echo "SASL" is enabled. - ub ensure KAFKA_OPTS + utility ensure KAFKA_OPTS if [[ ! $KAFKA_OPTS == *"java.security.auth.login.config"* ]] then @@ -110,6 +110,6 @@ cp /etc/kafka/docker/tools-log4j.properties /opt/kafka/config/tools-log4j.proper cp -R /mnt/shared/config/. /opt/kafka/config/ -echo -e "\n$(ub render-properties /etc/kafka/docker/kafka-propertiesSpec.json)" >> /opt/kafka/config/server.properties -echo -e "\n$(ub render-template /etc/kafka/docker/kafka-log4j.properties.template)" >> /opt/kafka/config/log4j.properties -echo -e "\n$(ub render-template /etc/kafka/docker/kafka-tools-log4j.properties.template)" >> /opt/kafka/config/tools-log4j.properties +echo -e "\n$(utility render-properties /etc/kafka/docker/kafka-propertiesSpec.json)" >> /opt/kafka/config/server.properties +echo -e "\n$(utility render-template /etc/kafka/docker/kafka-log4j.properties.template)" >> /opt/kafka/config/log4j.properties +echo -e "\n$(utility render-template /etc/kafka/docker/kafka-tools-log4j.properties.template)" >> /opt/kafka/config/tools-log4j.properties diff --git a/docker/resources/ub/go.mod b/docker/resources/utility/go.mod similarity index 100% rename from docker/resources/ub/go.mod rename to docker/resources/utility/go.mod diff --git a/docker/resources/ub/go.sum b/docker/resources/utility/go.sum similarity index 100% rename from docker/resources/ub/go.sum rename to docker/resources/utility/go.sum diff --git a/docker/resources/ub/testResources/sampleFile b/docker/resources/utility/testResources/sampleFile similarity index 100% rename from docker/resources/ub/testResources/sampleFile rename to docker/resources/utility/testResources/sampleFile diff --git a/docker/resources/ub/testResources/sampleFile2 b/docker/resources/utility/testResources/sampleFile2 similarity index 100% rename from docker/resources/ub/testResources/sampleFile2 rename to docker/resources/utility/testResources/sampleFile2 diff --git a/docker/resources/ub/testResources/sampleLog4j.template b/docker/resources/utility/testResources/sampleLog4j.template similarity index 100% rename from docker/resources/ub/testResources/sampleLog4j.template rename to docker/resources/utility/testResources/sampleLog4j.template diff --git a/docker/resources/ub/ub.go b/docker/resources/utility/utility.go similarity index 99% rename from docker/resources/ub/ub.go rename to docker/resources/utility/utility.go index 6c84fd2c35c90..521b837b41662 100644 --- a/docker/resources/ub/ub.go +++ b/docker/resources/utility/utility.go @@ -304,8 +304,8 @@ func runRenderPropertiesCmd(_ *cobra.Command, args []string) error { func main() { rootCmd := &cobra.Command{ - Use: "ub", - Short: "utility commands for cp docker images", + Use: "utility", + Short: "utility commands for kafka docker images", Run: func(cmd *cobra.Command, args []string) {}, } diff --git a/docker/resources/ub/ub_test.go b/docker/resources/utility/utility_test.go similarity index 100% rename from docker/resources/ub/ub_test.go rename to docker/resources/utility/utility_test.go diff --git a/docker/test/constants.py b/docker/test/constants.py index cd80636ecaba3..e42861e2b928c 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -31,7 +31,7 @@ FILE_INPUT_TOPIC="test-topic-file-input" BROKER_RESTART_TESTS="Broker Restart Tests" -BROKER_CONTAINER="broker" +BROKER_CONTAINER="broker1" BROKER_RESTART_TEST_TOPIC="test-topic-broker-restart" BROKER_METRICS_TESTS="Broker Metrics Tests" diff --git a/docker/test/fixtures/file-input/server.properties b/docker/test/fixtures/file-input/server.properties index 9965958123fe2..781f058650ec4 100644 --- a/docker/test/fixtures/file-input/server.properties +++ b/docker/test/fixtures/file-input/server.properties @@ -15,14 +15,12 @@ advertised.listeners=PLAINTEXT://localhost:19093,SSL://localhost:9094 controller.listener.names=CONTROLLER -controller.quorum.voters=3@broker-ssl-file-input:29093 group.initial.rebalance.delay.ms=0 inter.broker.listener.name=PLAINTEXT listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,CONTROLLER:PLAINTEXT -listeners=PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:9093,CONTROLLER://broker-ssl-file-input:29093 log.dirs=/tmp/kraft-combined-logs offsets.topic.replication.factor=1 -process.roles=invalid,value +process.roles=to be overridden ssl.client.auth=required ssl.key.password=abcdefgh ssl.keystore.location=/etc/kafka/secrets/kafka02.keystore.jks diff --git a/docker/test/fixtures/jvm/combined/docker-compose.yml b/docker/test/fixtures/jvm/combined/docker-compose.yml index fbc53fe2dabde..b62f7a84529f3 100644 --- a/docker/test/fixtures/jvm/combined/docker-compose.yml +++ b/docker/test/fixtures/jvm/combined/docker-compose.yml @@ -16,37 +16,47 @@ --- version: '2' services: - broker: + broker1: image: {$IMAGE} - hostname: broker - container_name: broker + hostname: broker1 + container_name: broker1 ports: - "9092:9092" - "9101:9101" + - "19091:19091" + volumes: + - ../../secrets:/etc/kafka/secrets environment: KAFKA_NODE_ID: 1 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT' - KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092,SSL://localhost:19091' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_PROCESS_ROLES: 'broker,controller' - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' - KAFKA_LISTENERS: 'CONTROLLER://broker:29093,PLAINTEXT://0.0.0.0:9092' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker1:29093' + KAFKA_LISTENERS: 'CONTROLLER://broker1:29093,PLAINTEXT://0.0.0.0:9092,SSL://0.0.0.0:19091' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' KAFKA_JMX_PORT: 9101 KAFKA_JMX_HOSTNAME: localhost + KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks" + KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" + KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" + KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" + KAFKA_SSL_CLIENT_AUTH: "required" - broker-ssl: + broker2: image: {$IMAGE} - hostname: broker-ssl - container_name: broker-ssl + hostname: broker2 + container_name: broker2 ports: - "9093:9093" + - "19092:19092" volumes: - ../../secrets:/etc/kafka/secrets environment: @@ -58,8 +68,8 @@ services: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_PROCESS_ROLES: 'broker,controller' - KAFKA_CONTROLLER_QUORUM_VOTERS: '2@broker-ssl:29093' - KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:9093,CONTROLLER://broker-ssl:29093" + KAFKA_CONTROLLER_QUORUM_VOTERS: '2@broker2:29093' + KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:9093,CONTROLLER://broker2:29093" KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT" KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' @@ -71,18 +81,21 @@ services: KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" KAFKA_SSL_CLIENT_AUTH: "required" - broker-ssl-file-input: + broker3: image: {$IMAGE} - hostname: broker-ssl-file-input - container_name: broker-ssl-file-input + hostname: broker3 + container_name: broker3 ports: - - "9094:9093" + - "19093:19093" + - "9094:9094" volumes: - ../../secrets:/etc/kafka/secrets - ../../file-input:/mnt/shared/config environment: CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' - # Set a property absent from the file + # Set properties absent from the file KAFKA_NODE_ID: 3 + KAFKA_CONTROLLER_QUORUM_VOTERS: '3@broker3:29093' + KAFKA_LISTENERS: 'PLAINTEXT://0.0.0.0:19093,SSL://0.0.0.0:9094,CONTROLLER://broker3:29093' # Override an existing property - KAFKA_PROCESS_ROLES: 'broker,controller' \ No newline at end of file + KAFKA_PROCESS_ROLES: 'broker,controller' diff --git a/docker/test/fixtures/jvm/isolated/docker-compose.yml b/docker/test/fixtures/jvm/isolated/docker-compose.yml index fefb495017936..f5882d46bdc0d 100644 --- a/docker/test/fixtures/jvm/isolated/docker-compose.yml +++ b/docker/test/fixtures/jvm/isolated/docker-compose.yml @@ -16,10 +16,10 @@ --- version: '2' services: - controller: + controller1: image: {$IMAGE} - hostname: controller - container_name: controller + hostname: controller1 + container_name: controller1 environment: KAFKA_NODE_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT' @@ -28,17 +28,55 @@ services: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_PROCESS_ROLES: 'controller' - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller:29093' - KAFKA_LISTENERS: 'CONTROLLER://controller:29093' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:29093,2@controller2:39093,3@controller3:49093' + KAFKA_LISTENERS: 'CONTROLLER://controller1:29093' KAFKA_INTER_BROKER_LISTENER_NAME: 'CONTROLLER' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' - broker: + controller2: image: {$IMAGE} - hostname: broker - container_name: broker + hostname: controller2 + container_name: controller2 + environment: + KAFKA_NODE_ID: 2 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:29093,2@controller2:39093,3@controller3:49093' + KAFKA_LISTENERS: 'CONTROLLER://controller2:39093' + KAFKA_INTER_BROKER_LISTENER_NAME: 'CONTROLLER' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + + controller3: + image: {$IMAGE} + hostname: controller3 + container_name: controller3 + environment: + KAFKA_NODE_ID: 3 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:29093,2@controller2:39093,3@controller3:49093' + KAFKA_LISTENERS: 'CONTROLLER://controller3:49093' + KAFKA_INTER_BROKER_LISTENER_NAME: 'CONTROLLER' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + + broker1: + image: {$IMAGE} + hostname: broker1 + container_name: broker1 ports: - "9092:9092" - "19091:19091" @@ -46,7 +84,7 @@ services: volumes: - ../../secrets:/etc/kafka/secrets environment: - KAFKA_NODE_ID: 2 + KAFKA_NODE_ID: 4 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,SSL:SSL,PLAINTEXT:PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092,SSL://localhost:19091' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 @@ -54,7 +92,7 @@ services: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_PROCESS_ROLES: 'broker' - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller:29093' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:29093,2@controller2:39093,3@controller3:49093' KAFKA_LISTENERS: 'PLAINTEXT://0.0.0.0:9092,SSL://0.0.0.0:19091' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' @@ -62,27 +100,26 @@ services: CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' KAFKA_JMX_PORT: 9101 KAFKA_JMX_HOSTNAME: localhost - KAFKA_SSL_KEYSTORE_FILENAME: "kafka02.keystore.jks" + KAFKA_SSL_KEYSTORE_FILENAME: "kafka01.keystore.jks" KAFKA_SSL_KEYSTORE_CREDENTIALS: "kafka_keystore_creds" KAFKA_SSL_KEY_CREDENTIALS: "kafka_ssl_key_creds" KAFKA_SSL_TRUSTSTORE_FILENAME: "kafka.truststore.jks" KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" KAFKA_SSL_CLIENT_AUTH: "required" depends_on: - - controller + - controller1 - broker-ssl: + broker2: image: {$IMAGE} - hostname: broker-ssl - container_name: broker-ssl + hostname: broker2 + container_name: broker2 ports: - "9093:9093" - "19092:19092" - - "19092" volumes: - ../../secrets:/etc/kafka/secrets environment: - KAFKA_NODE_ID: 3 + KAFKA_NODE_ID: 5 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,SSL:SSL,CONTROLLER:PLAINTEXT" KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:19092,SSL://localhost:9093" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 @@ -90,7 +127,7 @@ services: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_PROCESS_ROLES: 'broker' - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller:29093' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:29093,2@controller2:39093,3@controller3:49093' KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19092,SSL://0.0.0.0:9093" KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' @@ -103,12 +140,12 @@ services: KAFKA_SSL_TRUSTSTORE_CREDENTIALS: "kafka_truststore_creds" KAFKA_SSL_CLIENT_AUTH: "required" depends_on: - - controller + - controller1 - broker-ssl-file-input: + broker3: image: {$IMAGE} - hostname: broker-ssl-file-input - container_name: broker-ssl-file-input + hostname: broker3 + container_name: broker3 ports: - "19093:19093" - "9094:9094" @@ -118,10 +155,10 @@ services: environment: CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' # Set a property absent from the file - KAFKA_NODE_ID: 4 + KAFKA_NODE_ID: 6 # Override existing properties KAFKA_PROCESS_ROLES: 'broker' KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19093,SSL://0.0.0.0:9094" - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller:29093' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:29093,2@controller2:39093,3@controller3:49093' depends_on: - - controller \ No newline at end of file + - controller1 \ No newline at end of file From 8b4997852fb05402ff03688cb1f7d537c3cf1547 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Fri, 1 Dec 2023 19:09:17 +0530 Subject: [PATCH 56/66] Fix gradle --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f96acbcd6e0b0..24178d80bfb13 100644 --- a/build.gradle +++ b/build.gradle @@ -209,7 +209,7 @@ if (repo != null) { '**/generated/**', 'clients/src/test/resources/serializedData/*', 'docker/resources/ub/go.sum', - 'docker/test/fixtures/secrets/*' + 'docker/test/fixtures/secrets/*', 'docker/examples/fixtures/secrets/*' ]) } From c44b1c4205f508b6851bd15878b4bb4ca5222e38 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Fri, 1 Dec 2023 19:15:04 +0530 Subject: [PATCH 57/66] Fix build error --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 1e1ba0b546e3a..c0d9fa9d3abbf 100644 --- a/build.gradle +++ b/build.gradle @@ -208,7 +208,7 @@ if (repo != null) { 'licenses/*', '**/generated/**', 'clients/src/test/resources/serializedData/*', - 'docker/resources/ub/go.sum', + 'docker/resources/utility/go.sum', 'docker/test/fixtures/secrets/*' ]) } From a5cc72425aa2140dd5cd733608cf5acdca0544a0 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Sat, 2 Dec 2023 14:20:32 +0530 Subject: [PATCH 58/66] Fix typo errors and remove redundant client config --- docker/README.md | 4 ++-- docker/test/fixtures/secrets/client-ssl.properties | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/README.md b/docker/README.md index 6fc144b1a0598..abf522170e22d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -38,7 +38,7 @@ Using the image in a docker container ------------------------------------- - The image uses the kafka downloaded from provided kafka url - The image can be run in a container in default mode by running -`docker run -p 9092:9092` +`docker run -p 9092:9092 ` - Default configs run kafka in kraft mode with plaintext listners on 9092 port. - Default configs can be overriden by user using 2 ways:- - By mounting folder containing property files @@ -54,7 +54,7 @@ Using the image in a docker container - For abc.def, use KAFKA_ABC_DEF - For abc-def, use KAFKA_ABC___DEF - For abc_def, use KAFKA_ABC__DEF -- Hence order of precedence of properties is the follwing:- +- Hence order of precedence of properties is the following:- - Env variable (highest) - File input - Default (lowest) diff --git a/docker/test/fixtures/secrets/client-ssl.properties b/docker/test/fixtures/secrets/client-ssl.properties index 7467ceac5c64e..57718c26c0c76 100644 --- a/docker/test/fixtures/secrets/client-ssl.properties +++ b/docker/test/fixtures/secrets/client-ssl.properties @@ -19,6 +19,5 @@ ssl.truststore.password=abcdefgh ssl.keystore.location={$DIR}/fixtures/secrets/client.keystore.jks ssl.keystore.password=abcdefgh ssl.key.password=abcdefgh -ssl.enabled.protocols=TLSv1.2,TLSv1.1,TLSv1 ssl.client.auth=required ssl.endpoint.identification.algorithm= \ No newline at end of file From 93bbf1afce95cdc84b4030159c023e4524d5197e Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 4 Dec 2023 14:12:36 +0530 Subject: [PATCH 59/66] Resolve comments - Add gpg verification in jsa file generation - Add details in README for image type and kafka url --- docker/README.md | 5 +++++ docker/jvm/Dockerfile | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index abf522170e22d..73403f7470756 100644 --- a/docker/README.md +++ b/docker/README.md @@ -24,6 +24,11 @@ Bulding image and running tests using github actions This is the recommended way to build, test and get a CVE report for the docker image. Just choose the image type and provide kafka url to `Docker build test` workflow. It will generate a test report and CVE report that can be shared to the community. +kafka-url - This is the url to download kafka tarball from. For example kafka tarball url from (https://archive.apache.org/dist/kafka). For building RC image this will be an RC tarball url. + +image-type - This is the type of image that we intend to build. This will be dropdown menu type selection in the workflow. `jvm` image type is for official docker image (to be hosted on apache/kafka) as described in [KIP-975](https://cwiki.apache.org/confluence/display/KAFKA/KIP-975%3A+Docker+Image+for+Apache+Kafka) + + Creating a release ------------------ - `docker_release.py` script builds a multi architecture image and pushes it to provided docker registry. diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 46eb5c9a85ffd..349d23c00b83d 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -39,10 +39,14 @@ COPY jsa_launch /etc/kafka/docker/jsa_launch RUN set -eux ; \ apk update ; \ apk upgrade ; \ - apk add --no-cache wget gcompat procps netcat-openbsd; \ + apk add --no-cache wget gcompat gpg gpg-agent procps netcat-openbsd; \ mkdir opt/kafka; \ wget -nv -O kafka.tgz "$kafka_url"; \ - tar xfz kafka.tgz -C /opt/kafka --strip-components 1; + wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \ + tar xfz kafka.tgz -C /opt/kafka --strip-components 1; \ + wget -nv -O KEYS https://downloads.apache.org/kafka/KEYS; \ + gpg --import KEYS; \ + gpg --batch --verify kafka.tgz.asc kafka.tgz # Generate jsa files using dynamic CDS for kafka server start command and kafka storage format command RUN /etc/kafka/docker/jsa_launch From 228c9e7c089d59afe2def321d422292ff043c855 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 4 Dec 2023 16:18:08 +0530 Subject: [PATCH 60/66] Update config handling logic - Remove all default property files from docker folder - Use default files from kafka binary - Do not use default kafka configs if any user provided config is found --- docker/jvm/Dockerfile | 2 ++ docker/resources/common-scripts/configure | 26 +++++++++++++--- .../resources/common-scripts/log4j.properties | 30 ------------------- .../common-scripts/server.properties | 28 ----------------- .../common-scripts/tools-log4j.properties | 20 ------------- 5 files changed, 24 insertions(+), 82 deletions(-) delete mode 100644 docker/resources/common-scripts/log4j.properties delete mode 100644 docker/resources/common-scripts/server.properties delete mode 100644 docker/resources/common-scripts/tools-log4j.properties diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 349d23c00b83d..5d152cb48397e 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -87,6 +87,8 @@ RUN set -eux ; \ chown appuser:appuser -R /usr/logs /opt/kafka /mnt/shared/config; \ chown appuser:root -R /var/lib/kafka /etc/kafka/secrets /etc/kafka; \ chmod -R ug+w /etc/kafka /var/lib/kafka /etc/kafka/secrets; \ + cp /opt/kafka/config/log4j.properties /etc/kafka/docker/log4j.properties; \ + cp /opt/kafka/config/tools-log4j.properties /etc/kafka/docker/tools-log4j.properties; \ rm kafka.tgz kafka.tgz.asc KEYS; \ apk del wget gpg gpg-agent; \ apk cache clean; diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index cbc85ed658c6c..e92a6f644d2d3 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# If KAFKA_PROCESS_ROLES is defined it means we are running in KRaft mode - # unset KAFKA_ADVERTISED_LISTENERS from ENV in KRaft mode when running as controller only if [[ -n "${KAFKA_PROCESS_ROLES-}" ]] then @@ -104,12 +102,32 @@ then fi fi -cp /etc/kafka/docker/server.properties /opt/kafka/config/server.properties +# Copy the bundled log4j.properties and tools-log4j.properties. This is done to handle property modification during container restart cp /etc/kafka/docker/log4j.properties /opt/kafka/config/log4j.properties cp /etc/kafka/docker/tools-log4j.properties /opt/kafka/config/tools-log4j.properties +# Copy all the user provided property files through file input cp -R /mnt/shared/config/. /opt/kafka/config/ -echo -e "\n$(utility render-properties /etc/kafka/docker/kafka-propertiesSpec.json)" >> /opt/kafka/config/server.properties +# Check the presence of user provided kafka configs via file input +if [ -e "/mnt/shared/config/server.properties" ] +then + echo "User provided kafka configs found via file input. Any properties provided via env variables will be appended to this." + # Append configs provided via env variables. + echo -e "\n$(utility render-properties /etc/kafka/docker/kafka-propertiesSpec.json)" >> /opt/kafka/config/server.properties +else + # Create the kafka config property file using user provided environment variables. + echo -e "\n$(utility render-properties /etc/kafka/docker/kafka-propertiesSpec.json)" > /opt/kafka/config/server.properties + if grep -q '[^[:space:]]' "/opt/kafka/config/server.properties"; then + echo "User provided kafka configs found via environment variables." + fi +fi + +# If no user provided kafka configs found, use default configs +if ! grep -q '[^[:space:]]' "/opt/kafka/config/server.properties"; then + echo "User provided kafka configs not found (neither via file input nor via environment variables). Falling back to default configs." + cp /opt/kafka/config/kraft/server.properties /opt/kafka/config/server.properties +fi + echo -e "\n$(utility render-template /etc/kafka/docker/kafka-log4j.properties.template)" >> /opt/kafka/config/log4j.properties echo -e "\n$(utility render-template /etc/kafka/docker/kafka-tools-log4j.properties.template)" >> /opt/kafka/config/tools-log4j.properties diff --git a/docker/resources/common-scripts/log4j.properties b/docker/resources/common-scripts/log4j.properties deleted file mode 100644 index 7621ac44f42b8..0000000000000 --- a/docker/resources/common-scripts/log4j.properties +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -log4j.rootLogger=INFO, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - - -log4j.logger.kafka=INFO -log4j.logger.kafka.authorizer.logger=WARN -log4j.logger.kafka.controller=TRACE -log4j.logger.kafka.log.LogCleaner=INFO -log4j.logger.kafka.network.RequestChannel$=WARN -log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG -log4j.logger.kafka.request.logger=WARN -log4j.logger.state.change.logger=TRACE diff --git a/docker/resources/common-scripts/server.properties b/docker/resources/common-scripts/server.properties deleted file mode 100644 index 7a15f887d0fee..0000000000000 --- a/docker/resources/common-scripts/server.properties +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -advertised.listeners=PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092 -controller.listener.names=CONTROLLER -controller.quorum.voters=1@localhost:29093 -group.initial.rebalance.delay.ms=0 -inter.broker.listener.name=PLAINTEXT -listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT -listeners=PLAINTEXT://localhost:29092,CONTROLLER://localhost:29093,PLAINTEXT_HOST://0.0.0.0:9092 -log.dirs=/tmp/kraft-combined-logs -node.id=1 -offsets.topic.replication.factor=1 -process.roles=broker,controller -transaction.state.log.min.isr=1 -transaction.state.log.replication.factor=1 diff --git a/docker/resources/common-scripts/tools-log4j.properties b/docker/resources/common-scripts/tools-log4j.properties deleted file mode 100644 index 84f0e09405ddd..0000000000000 --- a/docker/resources/common-scripts/tools-log4j.properties +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -log4j.rootLogger=WARN, stderr - -log4j.appender.stderr=org.apache.log4j.ConsoleAppender -log4j.appender.stderr.layout=org.apache.log4j.PatternLayout -log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n -log4j.appender.stderr.Target=System.err From 4e6ed3d5583fc7b8e3e364c5cf15a8b5fa6a5bb1 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Mon, 4 Dec 2023 18:31:59 +0530 Subject: [PATCH 61/66] Make python scripts executable and add detailed examples in README --- .github/workflows/docker_build_and_test.yml | 4 +-- docker/README.md | 30 ++++++++++++++++++--- docker/docker_build_test.py | 10 +++---- docker/docker_promote.py | 0 docker/docker_release.py | 0 docker/jvm/Dockerfile | 2 +- docker/jvm/jsa_launch | 7 ++--- 7 files changed, 38 insertions(+), 15 deletions(-) mode change 100644 => 100755 docker/docker_build_test.py mode change 100644 => 100755 docker/docker_promote.py mode change 100644 => 100755 docker/docker_release.py diff --git a/.github/workflows/docker_build_and_test.yml b/.github/workflows/docker_build_and_test.yml index d12558e952614..657d32b3b792e 100644 --- a/.github/workflows/docker_build_and_test.yml +++ b/.github/workflows/docker_build_and_test.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Docker build test +name: Docker Build Test on: workflow_dispatch: @@ -50,8 +50,6 @@ jobs: with: image-ref: 'kafka/test:test' format: 'table' - ignore-unfixed: true - vuln-type: 'os,library' severity: 'CRITICAL,HIGH' output: scan_report_${{ github.event.inputs.image_type }}.txt exit-code: '1' diff --git a/docker/README.md b/docker/README.md index 73403f7470756..0ed40b3a7ae7b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -7,10 +7,12 @@ There are interactive python scripts to release the docker image and promote a r Local Setup ----------- -Make sure you have python (>= 3.7.x) and java (>= 17) installed before running the tests and scripts. +Make sure you have python (>= 3.7.x) and java (>= 17) (java needed only for running tests) installed before running the tests and scripts. Run `pip install -r requirements.txt` to get all the requirements for running the scripts. +Make sure you have docker installed with support for buildx enabled. (For pushing multi-architecture image to docker registry) + Bulding image and running tests locally --------------------------------------- - `docker_build_test.py` script builds and tests the docker image. @@ -19,6 +21,12 @@ Bulding image and running tests locally - By default image will be built and tested, but if you only want to build the image, pass `-b` flag and if you only want to test the given image pass `-t` flag. - An html test report will be generated after the tests are executed containing the results. +Example command:- +To build and test an image named test under kafka namespace with 3.6.0 tag and jvm image type ensuring kafka to be containerised should be https://downloads.apache.org/kafka/3.6.0/kafka_2.13-3.6.0.tgz (it is recommended to use scala 2.13 binary tarball), following command can be used +``` +python docker_build_test.py kafka/test --image-tag=3.6.0 --image-type=jvm --kafka-url=https://archive.apache.org/dist/kafka/3.6.0/kafka_2.13-3.6.0.tgz +``` + Bulding image and running tests using github actions ---------------------------------------------------- This is the recommended way to build, test and get a CVE report for the docker image. @@ -28,13 +36,29 @@ kafka-url - This is the url to download kafka tarball from. For example kafka ta image-type - This is the type of image that we intend to build. This will be dropdown menu type selection in the workflow. `jvm` image type is for official docker image (to be hosted on apache/kafka) as described in [KIP-975](https://cwiki.apache.org/confluence/display/KAFKA/KIP-975%3A+Docker+Image+for+Apache+Kafka) +Example command:- +To build and test a jvm image type ensuring kafka to be containerised should be https://archive.apache.org/dist/kafka/3.6.0/kafka_2.13-3.6.0.tgz (it is recommended to use scala 2.13 binary tarball), following inputs in github actions workflow are recommended. +``` +image_type: jvm +kafka_url: https://archive.apache.org/dist/kafka/3.6.0/kafka_2.13-3.6.0.tgz +``` Creating a release ------------------ -- `docker_release.py` script builds a multi architecture image and pushes it to provided docker registry. +- `docker_release.py` script builds a multi-architecture image and pushes it to provided docker registry. - Ensure you are logged in to the docker registry before triggering the script. - kafka binary tarball url along with image name (in the format `//:`) and type is needed to build the image. For detailed usage description check `python docker_release.py --help`. +Example command:- +To push an image named test under kafka dockerhub namespace with 3.6.0 tag and jvm image type ensuring kafka to be containerised should be https://archive.apache.org/dist/kafka/3.6.0/kafka_2.13-3.6.0.tgz (it is recommended to use scala 2.13 binary tarball), following command can be used. (Make sure you have push access to the docker repo) +``` +# kafka/test is an example repo. Please replace with the docker hub repo you have push access to. + +python docker_release.py kafka/test:3.6.0 --kafka-url https://archive.apache.org/dist/kafka/3.6.0/kafka_2.13-3.6.0.tgz +``` + +Please note that we used docker buildx for preparing the multi-architecture image and pushing it to docker registry. It's possible to encounter build failures because of buildx. Please retry the command in case some buildx related error occurs. + Promoting a release ------------------- `docker_promote.py` provides an interactive way to pull an RC Docker image and promote it to required dockerhub repo. @@ -68,7 +92,7 @@ Using the image in a docker container Steps to release docker image ----------------------------- - Make sure you have executed release.py script to prepare RC tarball in apache sftp server. -- Use the RC tarball url as input kafka url to build docker image and run sanity tests. +- Use the RC tarball url (make sure you choose scala 2.13 version) as input kafka url to build docker image and run sanity tests. - Trigger github actions workflow using the RC branch, provide RC tarball url as kafka url. - This will generate test report and CVE report for docker images. - If the reports look fine, RC docker image can be built and published. diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py old mode 100644 new mode 100755 index 73022027a6eb7..f1623a6b681e3 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -67,11 +67,11 @@ def run_jvm_tests(image, tag, kafka_url): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("image", help="Image name that you want to keep for the Docker image") - parser.add_argument("-tag", "--image-tag", default="latest", dest="tag", help="Image tag that you want to add to the image") - parser.add_argument("-type", "--image-type", choices=["jvm"], default="jvm", dest="image_type", help="Image type you want to build") - parser.add_argument("-u", "--kafka-url", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image") - parser.add_argument("-b", "--build", action="store_true", dest="build_only", default=False, help="Only build the image, don't run tests") - parser.add_argument("-t", "--test", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image") + parser.add_argument("--image-tag", "-tag", default="latest", dest="tag", help="Image tag that you want to add to the image") + parser.add_argument("--image-type", "-type", choices=["jvm"], default="jvm", dest="image_type", help="Image type you want to build") + parser.add_argument("--kafka-url", "-u", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image") + parser.add_argument("--build", "-b", action="store_true", dest="build_only", default=False, help="Only build the image, don't run tests") + parser.add_argument("--test", "-t", action="store_true", dest="test_only", default=False, help="Only run the tests, don't build the image") args = parser.parse_args() if args.image_type == "jvm" and (args.build_only or not (args.build_only or args.test_only)): diff --git a/docker/docker_promote.py b/docker/docker_promote.py old mode 100644 new mode 100755 diff --git a/docker/docker_release.py b/docker/docker_release.py old mode 100644 new mode 100755 diff --git a/docker/jvm/Dockerfile b/docker/jvm/Dockerfile index 5d152cb48397e..9a74e96125359 100644 --- a/docker/jvm/Dockerfile +++ b/docker/jvm/Dockerfile @@ -39,7 +39,7 @@ COPY jsa_launch /etc/kafka/docker/jsa_launch RUN set -eux ; \ apk update ; \ apk upgrade ; \ - apk add --no-cache wget gcompat gpg gpg-agent procps netcat-openbsd; \ + apk add --no-cache wget gcompat gpg gpg-agent procps netcat-openbsd uuidgen; \ mkdir opt/kafka; \ wget -nv -O kafka.tgz "$kafka_url"; \ wget -nv -O kafka.tgz.asc "$kafka_url.asc"; \ diff --git a/docker/jvm/jsa_launch b/docker/jvm/jsa_launch index 50d620d6724ac..d7efe5845c7dc 100755 --- a/docker/jvm/jsa_launch +++ b/docker/jvm/jsa_launch @@ -15,6 +15,7 @@ # limitations under the License. KAFKA_CLUSTER_ID="5L6g3nShT-eMCtK--X86sw" +TOPIC="$(uuidgen)" KAFKA_JVM_PERFORMANCE_OPTS="-XX:ArchiveClassesAtExit=storage.jsa" opt/kafka/bin/kafka-storage.sh format -t $KAFKA_CLUSTER_ID -c opt/kafka/config/kraft/server.properties @@ -29,13 +30,13 @@ check_timeout() { sleep 1 } -opt/kafka/bin/kafka-topics.sh --create --topic test-topic --bootstrap-server localhost:9092 +opt/kafka/bin/kafka-topics.sh --create --topic $TOPIC --bootstrap-server localhost:9092 [ $? -eq 0 ] || exit 1 -echo "test" | opt/kafka/bin/kafka-console-producer.sh --topic test-topic --bootstrap-server localhost:9092 +echo "test" | opt/kafka/bin/kafka-console-producer.sh --topic $TOPIC --bootstrap-server localhost:9092 [ $? -eq 0 ] || exit 1 -opt/kafka/bin/kafka-console-consumer.sh --topic test-topic --from-beginning --bootstrap-server localhost:9092 --max-messages 1 --timeout-ms 20000 +opt/kafka/bin/kafka-console-consumer.sh --topic $TOPIC --from-beginning --bootstrap-server localhost:9092 --max-messages 1 --timeout-ms 20000 [ $? -eq 0 ] || exit 1 opt/kafka/bin/kafka-server-stop.sh From ce2d921259dc399004b97c41192c1ddfe5a04283 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Tue, 5 Dec 2023 00:15:54 +0530 Subject: [PATCH 62/66] Remove redundant fix for controller flow --- docker/resources/common-scripts/configure | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/resources/common-scripts/configure b/docker/resources/common-scripts/configure index e92a6f644d2d3..eab86c0e9e265 100755 --- a/docker/resources/common-scripts/configure +++ b/docker/resources/common-scripts/configure @@ -26,9 +26,6 @@ then echo "KAFKA_ADVERTISED_LISTENERS is not supported on a KRaft controller." exit 1 else - # Remove advertised.listeners from default configs - sed '/advertised.listeners=/d' /etc/kafka/docker/server.properties > /etc/kafka/docker/server.properties.temp - mv /etc/kafka/docker/server.properties.temp /etc/kafka/docker/server.properties # Unset in case env variable is set with empty value unset KAFKA_ADVERTISED_LISTENERS fi From 3936397518ed8de6a897afd3f4d1037dcd0909cf Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Tue, 5 Dec 2023 15:51:45 +0530 Subject: [PATCH 63/66] Update readme docs and minor change in github actions workflow --- .github/workflows/docker_build_and_test.yml | 1 - docker/README.md | 39 ++++++++++--------- docker/common.py | 2 +- docker/docker_build_test.py | 2 +- docker/docker_promote.py | 2 +- docker/docker_release.py | 4 +- docker/test/__init__.py | 2 +- docker/test/constants.py | 2 +- .../fixtures/jvm/isolated/docker-compose.yml | 2 +- .../fixtures/secrets/client-ssl.properties | 2 +- 10 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/docker_build_and_test.yml b/.github/workflows/docker_build_and_test.yml index 657d32b3b792e..695c08672fd87 100644 --- a/.github/workflows/docker_build_and_test.yml +++ b/.github/workflows/docker_build_and_test.yml @@ -45,7 +45,6 @@ jobs: run: | python docker_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -u=${{ github.event.inputs.kafka_url }} - name: Run CVE scan - if: always() uses: aquasecurity/trivy-action@master with: image-ref: 'kafka/test:test' diff --git a/docker/README.md b/docker/README.md index 0ed40b3a7ae7b..f4036b156c62b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,9 +1,7 @@ Docker Images ============= -This directory contains docker image for Kafka. -The scripts take a url containing kafka as input and generate the respective docker image. -There are interactive python scripts to release the docker image and promote a release candidate. +This directory contains scripts to build, test, push and promote docker image for kafka. Local Setup ----------- @@ -18,7 +16,7 @@ Bulding image and running tests locally - `docker_build_test.py` script builds and tests the docker image. - kafka binary tarball url along with image name, tag and type is needed to build the image. For detailed usage description check `python docker_build_test.py --help`. - Sanity tests for the docker image are present in test/docker_sanity_test.py. -- By default image will be built and tested, but if you only want to build the image, pass `-b` flag and if you only want to test the given image pass `-t` flag. +- By default image will be built and tested, but if you only want to build the image, pass `--build` (or `-b`) flag and if you only want to test the given image pass `--test` (or `-t`) flag. - An html test report will be generated after the tests are executed containing the results. Example command:- @@ -30,7 +28,7 @@ python docker_build_test.py kafka/test --image-tag=3.6.0 --image-type=jvm --kafk Bulding image and running tests using github actions ---------------------------------------------------- This is the recommended way to build, test and get a CVE report for the docker image. -Just choose the image type and provide kafka url to `Docker build test` workflow. It will generate a test report and CVE report that can be shared to the community. +Just choose the image type and provide kafka url to `Docker Build Test` workflow. It will generate a test report and CVE report that can be shared with the community. kafka-url - This is the url to download kafka tarball from. For example kafka tarball url from (https://archive.apache.org/dist/kafka). For building RC image this will be an RC tarball url. @@ -57,7 +55,7 @@ To push an image named test under kafka dockerhub namespace with 3.6.0 tag and j python docker_release.py kafka/test:3.6.0 --kafka-url https://archive.apache.org/dist/kafka/3.6.0/kafka_2.13-3.6.0.tgz ``` -Please note that we used docker buildx for preparing the multi-architecture image and pushing it to docker registry. It's possible to encounter build failures because of buildx. Please retry the command in case some buildx related error occurs. +Please note that we use docker buildx for preparing the multi-architecture image and pushing it to docker registry. It's possible to encounter build failures because of buildx. Please retry the command in case some buildx related error occurs. Promoting a release ------------------- @@ -69,33 +67,36 @@ Using the image in a docker container - The image can be run in a container in default mode by running `docker run -p 9092:9092 ` - Default configs run kafka in kraft mode with plaintext listners on 9092 port. -- Default configs can be overriden by user using 2 ways:- +- Once user provided config properties are provided default configs will get replaced. +- User can provide kafka configs following two ways:- - By mounting folder containing property files - Mount the folder containing kafka property files to `/mnt/shared/config` - - These files will override the default config files + - These files will replace the default config files - Using environment variables - Kafka properties defined via env variables will override properties defined in file input - - Replace . with _ - - Replace _ with __(double underscore) - - Replace - with ___(triple underscore) - - Prefix the result with KAFKA_ - - Examples: - - For abc.def, use KAFKA_ABC_DEF - - For abc-def, use KAFKA_ABC___DEF - - For abc_def, use KAFKA_ABC__DEF + - If properties are provided via environment variables only, default configs will be replaced by user provided properties + - Input format for env variables:- + - Replace . with _ + - Replace _ with __(double underscore) + - Replace - with ___(triple underscore) + - Prefix the result with KAFKA_ + - Examples: + - For abc.def, use KAFKA_ABC_DEF + - For abc-def, use KAFKA_ABC___DEF + - For abc_def, use KAFKA_ABC__DEF - Hence order of precedence of properties is the following:- - Env variable (highest) - File input - - Default (lowest) + - Default configs (only when there is no user provided config) - Any env variable that is commonly used in starting kafka(for example, CLUSTER_ID) can be supplied to docker container and it will be available when kafka starts Steps to release docker image ----------------------------- -- Make sure you have executed release.py script to prepare RC tarball in apache sftp server. +- Make sure you have executed `release.py` script to prepare RC tarball in apache sftp server. - Use the RC tarball url (make sure you choose scala 2.13 version) as input kafka url to build docker image and run sanity tests. - Trigger github actions workflow using the RC branch, provide RC tarball url as kafka url. - This will generate test report and CVE report for docker images. - If the reports look fine, RC docker image can be built and published. -- Execute `docker_release.py` script to build and publish RC docker image in your own dockerhub account. +- Execute `docker_release.py` script to build and publish RC docker image in your dockerhub account. - Share the RC docker image, test report and CVE report with the community in RC vote email. - Once approved and ready, take help from someone in PMC to trigger `docker_promote.py` script and promote the RC docker image to apache/kafka dockerhub repo diff --git a/docker/common.py b/docker/common.py index 196ea2ea01fd3..1c94e173eec19 100644 --- a/docker/common.py +++ b/docker/common.py @@ -43,4 +43,4 @@ def jvm_image(command): except: raise SystemError("Docker Image Build failed") finally: - shutil.rmtree(temp_dir_path) \ No newline at end of file + shutil.rmtree(temp_dir_path) diff --git a/docker/docker_build_test.py b/docker/docker_build_test.py index f1623a6b681e3..e63d8746ac1bc 100755 --- a/docker/docker_build_test.py +++ b/docker/docker_build_test.py @@ -81,4 +81,4 @@ def run_jvm_tests(image, tag, kafka_url): raise ValueError("--kafka-url is a required argument for jvm image") if args.image_type == "jvm" and (args.test_only or not (args.build_only or args.test_only)): - run_jvm_tests(args.image, args.tag, args.kafka_url) \ No newline at end of file + run_jvm_tests(args.image, args.tag, args.kafka_url) diff --git a/docker/docker_promote.py b/docker/docker_promote.py index 75cd2fc06da1f..1074947174df5 100755 --- a/docker/docker_promote.py +++ b/docker/docker_promote.py @@ -82,4 +82,4 @@ def remove(promotion_image_namespace, promotion_image_name, promotion_image_tag, remove(promotion_image_namespace, promotion_image_name, promotion_image_tag, token) print("The image has been promoted successfully. The promoted image should be accessible in dockerhub") else: - print("Image promotion aborted") \ No newline at end of file + print("Image promotion aborted") diff --git a/docker/docker_release.py b/docker/docker_release.py index 67433ef0fa27e..50eee56a490a1 100755 --- a/docker/docker_release.py +++ b/docker/docker_release.py @@ -63,8 +63,8 @@ def remove_builder(): Please ensure you are logged in the docker registry that you are trying to push to.") parser = argparse.ArgumentParser() parser.add_argument("image", help="Dockerhub image that you want to push to (in the format //:)") - parser.add_argument("-type", "--image-type", choices=["jvm"], default="jvm", dest="image_type", help="Image type you want to build") - parser.add_argument("-u", "--kafka-url", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image") + parser.add_argument("--image-type", "-type", choices=["jvm"], default="jvm", dest="image_type", help="Image type you want to build") + parser.add_argument("--kafka-url", "-u", dest="kafka_url", help="Kafka url to be used to download kafka binary tarball in the docker image") args = parser.parse_args() print(f"Docker image of type {args.image_type} containing kafka downloaded from {args.kafka_url} will be pushed to {args.image}") diff --git a/docker/test/__init__.py b/docker/test/__init__.py index 977976beec24a..8f97ef9f46252 100644 --- a/docker/test/__init__.py +++ b/docker/test/__init__.py @@ -13,4 +13,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. diff --git a/docker/test/constants.py b/docker/test/constants.py index e42861e2b928c..cc8e82490662f 100644 --- a/docker/test/constants.py +++ b/docker/test/constants.py @@ -42,4 +42,4 @@ SSL_ERROR_PREFIX="SSL_ERR" BROKER_RESTART_ERROR_PREFIX="BROKER_RESTART_ERR" FILE_INPUT_ERROR_PREFIX="FILE_INPUT_ERR" -BROKER_METRICS_ERROR_PREFIX="BROKER_METRICS_ERR" \ No newline at end of file +BROKER_METRICS_ERROR_PREFIX="BROKER_METRICS_ERR" diff --git a/docker/test/fixtures/jvm/isolated/docker-compose.yml b/docker/test/fixtures/jvm/isolated/docker-compose.yml index f5882d46bdc0d..768f81b0d0c60 100644 --- a/docker/test/fixtures/jvm/isolated/docker-compose.yml +++ b/docker/test/fixtures/jvm/isolated/docker-compose.yml @@ -161,4 +161,4 @@ services: KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:19093,SSL://0.0.0.0:9094" KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller1:29093,2@controller2:39093,3@controller3:49093' depends_on: - - controller1 \ No newline at end of file + - controller1 diff --git a/docker/test/fixtures/secrets/client-ssl.properties b/docker/test/fixtures/secrets/client-ssl.properties index 57718c26c0c76..ad4f886555106 100644 --- a/docker/test/fixtures/secrets/client-ssl.properties +++ b/docker/test/fixtures/secrets/client-ssl.properties @@ -20,4 +20,4 @@ ssl.keystore.location={$DIR}/fixtures/secrets/client.keystore.jks ssl.keystore.password=abcdefgh ssl.key.password=abcdefgh ssl.client.auth=required -ssl.endpoint.identification.algorithm= \ No newline at end of file +ssl.endpoint.identification.algorithm= From e4cebd73dc7ba1eba70e0d45c5b49bdc8697cf80 Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Tue, 5 Dec 2023 16:10:44 +0530 Subject: [PATCH 64/66] Add empty line at end of all files --- docker/test/fixtures/secrets/kafka_keystore_creds | 2 +- docker/test/fixtures/secrets/kafka_ssl_key_creds | 2 +- docker/test/fixtures/secrets/kafka_truststore_creds | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/test/fixtures/secrets/kafka_keystore_creds b/docker/test/fixtures/secrets/kafka_keystore_creds index 1656f9233d999..0e5c23f7f91ea 100644 --- a/docker/test/fixtures/secrets/kafka_keystore_creds +++ b/docker/test/fixtures/secrets/kafka_keystore_creds @@ -1 +1 @@ -abcdefgh \ No newline at end of file +abcdefgh diff --git a/docker/test/fixtures/secrets/kafka_ssl_key_creds b/docker/test/fixtures/secrets/kafka_ssl_key_creds index 1656f9233d999..0e5c23f7f91ea 100644 --- a/docker/test/fixtures/secrets/kafka_ssl_key_creds +++ b/docker/test/fixtures/secrets/kafka_ssl_key_creds @@ -1 +1 @@ -abcdefgh \ No newline at end of file +abcdefgh diff --git a/docker/test/fixtures/secrets/kafka_truststore_creds b/docker/test/fixtures/secrets/kafka_truststore_creds index 1656f9233d999..0e5c23f7f91ea 100644 --- a/docker/test/fixtures/secrets/kafka_truststore_creds +++ b/docker/test/fixtures/secrets/kafka_truststore_creds @@ -1 +1 @@ -abcdefgh \ No newline at end of file +abcdefgh From d0c8d456b923e84a1c6b86b1ca9f731c55efff4e Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Tue, 5 Dec 2023 17:18:07 +0530 Subject: [PATCH 65/66] Fix examples --- .../client-secrets/client-ssl.properties | 2 +- .../fixtures/file-input/server.properties | 1 - .../fixtures/secrets/kafka_keystore_creds | 2 +- .../fixtures/secrets/kafka_ssl_key_creds | 2 +- .../fixtures/secrets/kafka_truststore_creds | 2 +- .../examples/jvm/cluster/docker-compose.yml | 48 +++++++++++++++---- .../jvm/file-input/docker-compose.yml | 14 +++--- .../jvm/single-node/docker-compose.yml | 2 +- docker/examples/jvm/ssl/docker-compose.yml | 10 ++-- 9 files changed, 58 insertions(+), 25 deletions(-) diff --git a/docker/examples/fixtures/client-secrets/client-ssl.properties b/docker/examples/fixtures/client-secrets/client-ssl.properties index 39e57d8ba4d15..00b7c801fdc5d 100644 --- a/docker/examples/fixtures/client-secrets/client-ssl.properties +++ b/docker/examples/fixtures/client-secrets/client-ssl.properties @@ -23,4 +23,4 @@ ssl.keystore.location=./docker/examples/fixtures/client-secrets/client.keystore. ssl.keystore.password=abcdefgh ssl.key.password=abcdefgh ssl.client.auth=required -ssl.endpoint.identification.algorithm= \ No newline at end of file +ssl.endpoint.identification.algorithm= diff --git a/docker/examples/fixtures/file-input/server.properties b/docker/examples/fixtures/file-input/server.properties index 7b92a9d94811d..9d36d7c9a2833 100644 --- a/docker/examples/fixtures/file-input/server.properties +++ b/docker/examples/fixtures/file-input/server.properties @@ -18,7 +18,6 @@ controller.listener.names=CONTROLLER group.initial.rebalance.delay.ms=0 inter.broker.listener.name=PLAINTEXT listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,CONTROLLER:PLAINTEXT -listeners=PLAINTEXT://0.0.0.0:9092,SSL://0.0.0.0:9093,CONTROLLER://broker-ssl-file-input:29093 log.dirs=/tmp/kraft-combined-logs offsets.topic.replication.factor=1 process.roles=broker diff --git a/docker/examples/fixtures/secrets/kafka_keystore_creds b/docker/examples/fixtures/secrets/kafka_keystore_creds index 1656f9233d999..0e5c23f7f91ea 100644 --- a/docker/examples/fixtures/secrets/kafka_keystore_creds +++ b/docker/examples/fixtures/secrets/kafka_keystore_creds @@ -1 +1 @@ -abcdefgh \ No newline at end of file +abcdefgh diff --git a/docker/examples/fixtures/secrets/kafka_ssl_key_creds b/docker/examples/fixtures/secrets/kafka_ssl_key_creds index 1656f9233d999..0e5c23f7f91ea 100644 --- a/docker/examples/fixtures/secrets/kafka_ssl_key_creds +++ b/docker/examples/fixtures/secrets/kafka_ssl_key_creds @@ -1 +1 @@ -abcdefgh \ No newline at end of file +abcdefgh diff --git a/docker/examples/fixtures/secrets/kafka_truststore_creds b/docker/examples/fixtures/secrets/kafka_truststore_creds index 1656f9233d999..0e5c23f7f91ea 100644 --- a/docker/examples/fixtures/secrets/kafka_truststore_creds +++ b/docker/examples/fixtures/secrets/kafka_truststore_creds @@ -1 +1 @@ -abcdefgh \ No newline at end of file +abcdefgh diff --git a/docker/examples/jvm/cluster/docker-compose.yml b/docker/examples/jvm/cluster/docker-compose.yml index ef61fcabcfa35..276b94b1760a6 100644 --- a/docker/examples/jvm/cluster/docker-compose.yml +++ b/docker/examples/jvm/cluster/docker-compose.yml @@ -23,20 +23,46 @@ services: environment: KAFKA_NODE_ID: 1 KAFKA_PROCESS_ROLES: 'controller' - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LISTENERS: 'CONTROLLER://controller-1:9093' CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + + controller-2: + image: apache/kafka:latest + ports: + - 29093:9093 + environment: + KAFKA_NODE_ID: 2 + KAFKA_PROCESS_ROLES: 'controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LISTENERS: 'CONTROLLER://controller-2:9093' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + + controller-3: + image: apache/kafka:latest + ports: + - 39093:9093 + environment: + KAFKA_NODE_ID: 3 + KAFKA_PROCESS_ROLES: 'controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LISTENERS: 'CONTROLLER://controller-3:9093' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' kafka-1: image: apache/kafka:latest ports: - 29092:9092 environment: - KAFKA_NODE_ID: 2 + KAFKA_NODE_ID: 4 KAFKA_PROCESS_ROLES: 'broker' - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093' KAFKA_LISTENERS: 'PLAINTEXT://kafka-1:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092 @@ -44,15 +70,17 @@ services: CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' depends_on: - controller-1 + - controller-2 + - controller-3 kafka-2: image: apache/kafka:latest ports: - 39092:9092 environment: - KAFKA_NODE_ID: 3 + KAFKA_NODE_ID: 5 KAFKA_PROCESS_ROLES: 'broker' - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093' KAFKA_LISTENERS: 'PLAINTEXT://kafka-2:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:39092 @@ -60,19 +88,23 @@ services: CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' depends_on: - controller-1 + - controller-2 + - controller-3 kafka-3: image: apache/kafka:latest ports: - 49092:9092 environment: - KAFKA_NODE_ID: 4 + KAFKA_NODE_ID: 6 KAFKA_PROCESS_ROLES: 'broker' - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093' KAFKA_LISTENERS: 'PLAINTEXT://kafka-3:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:49092 KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' depends_on: - - controller-1 \ No newline at end of file + - controller-1 + - controller-2 + - controller-3 diff --git a/docker/examples/jvm/file-input/docker-compose.yml b/docker/examples/jvm/file-input/docker-compose.yml index b9667f63ae765..aec88af18805a 100644 --- a/docker/examples/jvm/file-input/docker-compose.yml +++ b/docker/examples/jvm/file-input/docker-compose.yml @@ -16,11 +16,12 @@ --- version: '2' services: - broker-ssl-file-input: + broker: image: apache/kafka:latest - hostname: broker-ssl-file-input - container_name: broker-ssl-file-input + hostname: broker + container_name: broker ports: + - "9092:9092" - "9093:9093" volumes: - ../../fixtures/secrets:/etc/kafka/secrets @@ -28,8 +29,9 @@ services: environment: # Environment variables used by kafka scripts will be needed in case of File input. CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' - # Set properties absent from the file + # Set properties not provided in the file input KAFKA_NODE_ID: 1 - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker-ssl-file-input:29093' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' + KAFKA_LISTENERS: 'PLAINTEXT://0.0.0.0:19093,SSL://0.0.0.0:9094,CONTROLLER://broker:29093' # Override an existing property - KAFKA_PROCESS_ROLES: 'broker,controller' \ No newline at end of file + KAFKA_PROCESS_ROLES: 'broker,controller' diff --git a/docker/examples/jvm/single-node/docker-compose.yml b/docker/examples/jvm/single-node/docker-compose.yml index 98cce532d85e2..34507bec23b6a 100644 --- a/docker/examples/jvm/single-node/docker-compose.yml +++ b/docker/examples/jvm/single-node/docker-compose.yml @@ -31,4 +31,4 @@ services: KAFKA_LISTENERS: 'CONTROLLER://broker:29093,PLAINTEXT://0.0.0.0:9092' KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' - CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' \ No newline at end of file + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' diff --git a/docker/examples/jvm/ssl/docker-compose.yml b/docker/examples/jvm/ssl/docker-compose.yml index 488666ce38cd4..a27a9214c8e08 100644 --- a/docker/examples/jvm/ssl/docker-compose.yml +++ b/docker/examples/jvm/ssl/docker-compose.yml @@ -16,10 +16,10 @@ --- version: '2' services: - broker-ssl: + broker: image: apache/kafka:latest - hostname: broker-ssl - container_name: broker-ssl + hostname: broker + container_name: broker ports: - '9093:9093' volumes: @@ -33,8 +33,8 @@ services: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_PROCESS_ROLES: 'broker,controller' - KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker-ssl:29093' - KAFKA_LISTENERS: 'SSL://0.0.0.0:9093,CONTROLLER://broker-ssl:29093' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' + KAFKA_LISTENERS: 'SSL://0.0.0.0:9093,CONTROLLER://broker:29093' KAFKA_INTER_BROKER_LISTENER_NAME: 'SSL' KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' From 3dd5b08ce399895b00a5cc078d5bafe550ab251b Mon Sep 17 00:00:00 2001 From: Vedarth Sharma Date: Wed, 6 Dec 2023 00:59:52 +0530 Subject: [PATCH 66/66] Update the docs with updated config handling --- docs/docker.html | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/docker.html b/docs/docker.html index 1b1181b5e3771..9b6cd817df890 100644 --- a/docs/docker.html +++ b/docs/docker.html @@ -29,16 +29,14 @@

    Start kafka with default configs

    Passing configs to kafka in docker container

    - Docker image comes with default configs for KRaft mode and can be used out of the box as mentioned in quickstart. - If default configs need to be overriden, it can be achieved by providing configs to the docker container. + When no user provided input is provided, default configs will be used to start the kafka server, as mentioned in quickstart. Configs can be provided to kafka running inside the container either by environment variables or by providing property files through file input. - Property files provided through file input will replace default property files. - Configs provided through environment variables will have the highest priority. + Property files provided through file input will replace default property files. Configs provided through environment variables will have the highest priority.

    File input
      -
    • This method requires users to get path to a local folder containing kafka property files and mount it to docker container using docker volume.
      • +
    • This method requires users to provide path to a local folder which contains kafka property files and mount it to docker container using docker volume.
    • It replaces the default config file present in docker container.
    • Mount the folder containing kafka property files to /mnt/shared/config in docker container.
    • Command docker run --volume path/to/property/folder:/mnt/shared/config -p 9092:9092 apache/kafka:{{fullDotVersion}} can be used to mount the folder containing property files.
      • @@ -47,7 +45,8 @@
      File input
      Using Environment Variables
        -
      • Kafka property defined via env variables will override the value of that property defined in file input and default config
        • +
      • Kafka property defined via env variables will override the value of that property defined in file input and default config.
        • +
      • If properties are provided via environment variables only, default configs will be replaced by user provided properties.
      • To construct the environment key variable name for server.properties configs, following steps can be followed:-
        • Replace . with _
          • @@ -73,7 +72,6 @@
          Using Environment Variables
      • Environment variables commonly used in Kafka can be provided via environment variables as well, for example CLUSTER_ID
      • Command docker run --env CONFIG_NAME=CONFIG_VALUE -p 9092:9092 apache/kafka:{{fullDotVersion}} can be used to provide environment variables to docker container
        • -
      • One thing to note is that, when using environment variables to provide configs, default value will be used for the configs that are not getting overriden by the environment variable

      Running in SSL mode