diff --git a/build.gradle b/build.gradle index 080daf28ac9fd..72e588ee72d26 100644 --- a/build.gradle +++ b/build.gradle @@ -209,7 +209,8 @@ if (repo != null) { '**/generated/**', 'clients/src/test/resources/serializedData/*', 'docker/resources/utility/go.sum', - 'docker/test/fixtures/secrets/*' + 'docker/test/fixtures/secrets/*', + 'docker/examples/fixtures/secrets/*' ]) } } else { diff --git a/docker/examples/README.md b/docker/examples/README.md new file mode 100644 index 0000000000000..6fd5cba17c73b --- /dev/null +++ b/docker/examples/README.md @@ -0,0 +1,31 @@ +Kafka Docker Image Examples +--------------------------- + +- This directory contains docker compose files for some example configs to run docker image. + +- Run the commands from root of the repository. + +- To bring up the docker compose examples, use docker compose command. + +For example:- +``` +# This command brings up JVM cluster +$ docker compose -f docker/examples/jvm/cluster/docker-compose.yml up +``` + +- Kafka server can be accessed using cli scripts or your own client code. +Make sure jars are built for the code, if cli scripts are being used. + +For example:- +``` +# Produce messages to kafka broker +$ bin/kafka-console-producer.sh --topic quickstart-events --bootstrap-server localhost:29092 +``` + +- Use `./docker/examples/fixtures/client-secrets` for connecting with Kafka when running SSL example. `./docker/examples/fixtures/client-secrets/client-ssl.properties` file can be used as client config. + +For example:- +``` +# Produce message to SSL Kafka example +$ bin/kafka-console-producer.sh --topic test_topic_ssl --bootstrap-server localhost:9093 --producer.config ./docker/examples/fixtures/client-secrets/client-ssl.properties +``` diff --git a/docker/examples/fixtures/client-secrets/client-ssl.properties b/docker/examples/fixtures/client-secrets/client-ssl.properties new file mode 100644 index 0000000000000..00b7c801fdc5d --- /dev/null +++ b/docker/examples/fixtures/client-secrets/client-ssl.properties @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +security.protocol=SSL +# Provide path to the truststore file +ssl.truststore.location=./docker/examples/fixtures/secrets/kafka.truststore.jks +# This the password for the example keystores and truststores +ssl.truststore.password=abcdefgh +# Provide path to the keystore file +ssl.keystore.location=./docker/examples/fixtures/client-secrets/client.keystore.jks +ssl.keystore.password=abcdefgh +ssl.key.password=abcdefgh +ssl.client.auth=required +ssl.endpoint.identification.algorithm= diff --git a/docker/examples/fixtures/client-secrets/client.keystore.jks b/docker/examples/fixtures/client-secrets/client.keystore.jks new file mode 100644 index 0000000000000..7873e20ed170d Binary files /dev/null and b/docker/examples/fixtures/client-secrets/client.keystore.jks differ diff --git a/docker/examples/fixtures/file-input/server.properties b/docker/examples/fixtures/file-input/server.properties new file mode 100644 index 0000000000000..9d36d7c9a2833 --- /dev/null +++ b/docker/examples/fixtures/file-input/server.properties @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +advertised.listeners=PLAINTEXT://localhost:9092,SSL://localhost:9093 +controller.listener.names=CONTROLLER +group.initial.rebalance.delay.ms=0 +inter.broker.listener.name=PLAINTEXT +listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,CONTROLLER:PLAINTEXT +log.dirs=/tmp/kraft-combined-logs +offsets.topic.replication.factor=1 +process.roles=broker +ssl.client.auth=required +ssl.key.password=abcdefgh +ssl.keystore.location=/etc/kafka/secrets/kafka01.keystore.jks +ssl.keystore.password=abcdefgh +ssl.truststore.location=/etc/kafka/secrets/kafka.truststore.jks +ssl.truststore.password=abcdefgh +transaction.state.log.min.isr=1 +transaction.state.log.replication.factor=1 diff --git a/docker/examples/fixtures/secrets/kafka.truststore.jks b/docker/examples/fixtures/secrets/kafka.truststore.jks new file mode 100644 index 0000000000000..ff1a5e8cd19f9 Binary files /dev/null and b/docker/examples/fixtures/secrets/kafka.truststore.jks differ diff --git a/docker/examples/fixtures/secrets/kafka01.keystore.jks b/docker/examples/fixtures/secrets/kafka01.keystore.jks new file mode 100644 index 0000000000000..953c3355b6e6d Binary files /dev/null and b/docker/examples/fixtures/secrets/kafka01.keystore.jks differ diff --git a/docker/examples/fixtures/secrets/kafka_keystore_creds b/docker/examples/fixtures/secrets/kafka_keystore_creds new file mode 100644 index 0000000000000..0e5c23f7f91ea --- /dev/null +++ b/docker/examples/fixtures/secrets/kafka_keystore_creds @@ -0,0 +1 @@ +abcdefgh diff --git a/docker/examples/fixtures/secrets/kafka_ssl_key_creds b/docker/examples/fixtures/secrets/kafka_ssl_key_creds new file mode 100644 index 0000000000000..0e5c23f7f91ea --- /dev/null +++ b/docker/examples/fixtures/secrets/kafka_ssl_key_creds @@ -0,0 +1 @@ +abcdefgh diff --git a/docker/examples/fixtures/secrets/kafka_truststore_creds b/docker/examples/fixtures/secrets/kafka_truststore_creds new file mode 100644 index 0000000000000..0e5c23f7f91ea --- /dev/null +++ b/docker/examples/fixtures/secrets/kafka_truststore_creds @@ -0,0 +1 @@ +abcdefgh diff --git a/docker/examples/jvm/cluster/docker-compose.yml b/docker/examples/jvm/cluster/docker-compose.yml new file mode 100644 index 0000000000000..276b94b1760a6 --- /dev/null +++ b/docker/examples/jvm/cluster/docker-compose.yml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +version: '1' +services: + controller-1: + image: apache/kafka:latest + ports: + - 19093:9093 + environment: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: 'controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LISTENERS: 'CONTROLLER://controller-1:9093' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + + controller-2: + image: apache/kafka:latest + ports: + - 29093:9093 + environment: + KAFKA_NODE_ID: 2 + KAFKA_PROCESS_ROLES: 'controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LISTENERS: 'CONTROLLER://controller-2:9093' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + + controller-3: + image: apache/kafka:latest + ports: + - 39093:9093 + environment: + KAFKA_NODE_ID: 3 + KAFKA_PROCESS_ROLES: 'controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LISTENERS: 'CONTROLLER://controller-3:9093' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + + kafka-1: + image: apache/kafka:latest + ports: + - 29092:9092 + environment: + KAFKA_NODE_ID: 4 + KAFKA_PROCESS_ROLES: 'broker' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093' + KAFKA_LISTENERS: 'PLAINTEXT://kafka-1:9092' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092 + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + depends_on: + - controller-1 + - controller-2 + - controller-3 + + kafka-2: + image: apache/kafka:latest + ports: + - 39092:9092 + environment: + KAFKA_NODE_ID: 5 + KAFKA_PROCESS_ROLES: 'broker' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093' + KAFKA_LISTENERS: 'PLAINTEXT://kafka-2:9092' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:39092 + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + depends_on: + - controller-1 + - controller-2 + - controller-3 + + kafka-3: + image: apache/kafka:latest + ports: + - 49092:9092 + environment: + KAFKA_NODE_ID: 6 + KAFKA_PROCESS_ROLES: 'broker' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:9093,2@controller-2:9093,3@controller-3:9093' + KAFKA_LISTENERS: 'PLAINTEXT://kafka-3:9092' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:49092 + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + depends_on: + - controller-1 + - controller-2 + - controller-3 diff --git a/docker/examples/jvm/file-input/docker-compose.yml b/docker/examples/jvm/file-input/docker-compose.yml new file mode 100644 index 0000000000000..aec88af18805a --- /dev/null +++ b/docker/examples/jvm/file-input/docker-compose.yml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +version: '2' +services: + broker: + image: apache/kafka:latest + hostname: broker + container_name: broker + ports: + - "9092:9092" + - "9093:9093" + volumes: + - ../../fixtures/secrets:/etc/kafka/secrets + - ../../fixtures/file-input:/mnt/shared/config + environment: + # Environment variables used by kafka scripts will be needed in case of File input. + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + # Set properties not provided in the file input + KAFKA_NODE_ID: 1 + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' + KAFKA_LISTENERS: 'PLAINTEXT://0.0.0.0:19093,SSL://0.0.0.0:9094,CONTROLLER://broker:29093' + # Override an existing property + KAFKA_PROCESS_ROLES: 'broker,controller' diff --git a/docker/examples/jvm/single-node/docker-compose.yml b/docker/examples/jvm/single-node/docker-compose.yml new file mode 100644 index 0000000000000..34507bec23b6a --- /dev/null +++ b/docker/examples/jvm/single-node/docker-compose.yml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +version: '2' +services: + broker: + image: apache/kafka:latest + hostname: broker + container_name: broker + ports: + - '9092:9092' + environment: + KAFKA_NODE_ID: 1 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092' + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' + KAFKA_LISTENERS: 'CONTROLLER://broker:29093,PLAINTEXT://0.0.0.0:9092' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' diff --git a/docker/examples/jvm/ssl/docker-compose.yml b/docker/examples/jvm/ssl/docker-compose.yml new file mode 100644 index 0000000000000..a27a9214c8e08 --- /dev/null +++ b/docker/examples/jvm/ssl/docker-compose.yml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +version: '2' +services: + broker: + image: apache/kafka:latest + hostname: broker + container_name: broker + ports: + - '9093:9093' + volumes: + - ../../fixtures/secrets:/etc/kafka/secrets + environment: + KAFKA_NODE_ID: 1 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'SSL:SSL,CONTROLLER:PLAINTEXT' + KAFKA_ADVERTISED_LISTENERS: 'SSL://localhost:9093' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093' + KAFKA_LISTENERS: 'SSL://0.0.0.0:9093,CONTROLLER://broker:29093' + KAFKA_INTER_BROKER_LISTENER_NAME: 'SSL' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs' + CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw' + KAFKA_SSL_KEYSTORE_FILENAME: 'kafka01.keystore.jks' + KAFKA_SSL_KEYSTORE_CREDENTIALS: 'kafka_keystore_creds' + KAFKA_SSL_KEY_CREDENTIALS: 'kafka_ssl_key_creds' + KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.truststore.jks' + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'kafka_truststore_creds' + KAFKA_SSL_CLIENT_AUTH: 'required' diff --git a/docs/docker.html b/docs/docker.html new file mode 100644 index 0000000000000..9b6cd817df890 --- /dev/null +++ b/docs/docker.html @@ -0,0 +1,104 @@ + + + + + + +
diff --git a/docs/documentation.html b/docs/documentation.html index 57707f1be697b..d86efaa61c6bf 100644 --- a/docs/documentation.html +++ b/docs/documentation.html @@ -70,8 +70,10 @@

1.4 Ecosystem

-

1.5 Upgrading From Previous Versions

+

1.5 Upgrading From Previous Versions

+

1.6 Docker

+

2. APIs

diff --git a/docs/quickstart.html b/docs/quickstart.html index 72ff7b818267e..c81d4f7fc5ba3 100644 --- a/docs/quickstart.html +++ b/docs/quickstart.html @@ -76,6 +76,10 @@
Kafka with KRaft
+

Kafka can be run using KRaft mode using local scripts and downloaded files or the docker image. Follow one of the sections below but not both to start the kafka server.

+ +
Using downloaded files
+

Generate a Cluster UUID

@@ -94,6 +98,12 @@
$ bin/kafka-server-start.sh config/kraft/server.properties
+
Using docker image
+ +

Start the kafka docker container

+ +
$ docker run -p 9092:9092 apache/kafka:{{fullDotVersion}}
+

Once the Kafka server has successfully launched, you will have a basic Kafka environment running and ready to use.

diff --git a/docs/toc.html b/docs/toc.html index d2902d8f164f2..2413491148860 100644 --- a/docs/toc.html +++ b/docs/toc.html @@ -27,6 +27,7 @@
  • 1.3 Quick Start
  • 1.4 Ecosystem
  • 1.5 Upgrading +
  • 1.6 Docker
  • 2. APIs