From bc1ef2c35d6db6a4cd6a87a4cf487eaf8dc8adb3 Mon Sep 17 00:00:00 2001 From: Harshil Nori Date: Tue, 15 Jun 2021 11:49:12 +0530 Subject: [PATCH 1/3] Added support for iam enabled msk using custom role attached to k8s pod Dockerfile added Renamed Dockerfile Removed unwanted client.properties. Additional formatting changes Additional formatting changes Pom changes Updated dockerfile Updated pom properties. Updated yml with is secured Updated yml with is secured Updated yml with updated properties file Added warn log --- pom.xml | 12 ++++++++++++ .../java/kafdrop/config/KafkaConfiguration.java | 17 +++++++++++++++-- src/main/resources/application.yml | 9 ++++++--- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 43146eb6..8d93efc6 100644 --- a/pom.xml +++ b/pom.xml @@ -22,6 +22,7 @@ UTF-8 3.22.3 1.18.0 + 1.1.6 7.3.3 @@ -76,6 +77,11 @@ org.projectlombok lombok + + software.amazon.msk + aws-msk-iam-auth + ${msk.auth.version} + io.confluent kafka-avro-serializer @@ -143,6 +149,12 @@ org.springframework.kafka spring-kafka + + + com.amazonaws + aws-java-sdk-sts + ${sts.sdk.version} + org.springframework.boot spring-boot-starter-undertow diff --git a/src/main/java/kafdrop/config/KafkaConfiguration.java b/src/main/java/kafdrop/config/KafkaConfiguration.java index bb6e137e..53ad76e0 100644 --- a/src/main/java/kafdrop/config/KafkaConfiguration.java +++ b/src/main/java/kafdrop/config/KafkaConfiguration.java @@ -28,15 +28,22 @@ public final class KafkaConfiguration { private String truststoreFile; private String propertiesFile; private String keystoreFile; + private String jaasConfig; + private String clientCallback; + private String iamEnabled; public void applyCommon(Properties properties) { properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerConnect); + if (isSecured) { LOG.warn("The 'isSecured' property is deprecated; consult README.md on the preferred way to configure security"); - properties.put(SaslConfigs.SASL_MECHANISM, saslMechanism); } - if (isSecured || securityProtocol.equals("SSL")) { + LOG.info("Setting sasl mechanism to {}", saslMechanism); + properties.put(SaslConfigs.SASL_MECHANISM, saslMechanism); + + if (isSecured || securityProtocol.equals("SSL") || securityProtocol.equals("SASL_SSL")) { + LOG.info("Setting security protocol to {}", securityProtocol); properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, securityProtocol); } @@ -45,6 +52,12 @@ public void applyCommon(Properties properties) { LOG.info("Assigning truststore location to {}", truststoreFile); properties.put("ssl.truststore.location", truststoreFile); } + LOG.info("Is iam enabled : {}", iamEnabled); + if (Boolean.parseBoolean(iamEnabled)) { + LOG.info("Setting sasl.jaas.config {} and sasl and callback callback properties {}", jaasConfig, clientCallback); + properties.put(SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS, clientCallback); + properties.put(SaslConfigs.SASL_JAAS_CONFIG, jaasConfig); + } LOG.info("Checking keystore file {}", keystoreFile); if (new File(keystoreFile).isFile()) { diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index c7d3a4df..203c4443 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -38,9 +38,12 @@ kafdrop.monitor: kafka: brokerConnect: localhost:9092 - isSecured: false - saslMechanism: "PLAIN" - securityProtocol: "SASL_PLAINTEXT" + isSecured: "${KAFKA_IS_SECURED:false}" + saslMechanism: "${KAFKA_SASL_MECHANISM:PLAIN}" + securityProtocol: "${KAFKA_SECURITY_PROTOCOL:SASL_PLAINTEXT}" truststoreFile: "${KAFKA_TRUSTSTORE_FILE:kafka.truststore.jks}" propertiesFile : "${KAFKA_PROPERTIES_FILE:kafka.properties}" keystoreFile: "${KAFKA_KEYSTORE_FILE:kafka.keystore.jks}" + iamEnabled: "${KAFKA_IAM_ENABLED:false}" + jaasConfig: "${KAFKA_JAAS_CONFIG}" + clientCallback: "software.amazon.msk.auth.iam.IAMClientCallbackHandler" From 4d8e5d1199f90714703730f1987243ebf53d8b87 Mon Sep 17 00:00:00 2001 From: Richard Wu Date: Fri, 21 Apr 2023 14:40:47 -0400 Subject: [PATCH 2/3] Add sts version. --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8d93efc6..1c7db9b3 100644 --- a/pom.xml +++ b/pom.xml @@ -22,8 +22,9 @@ UTF-8 3.22.3 1.18.0 - 1.1.6 7.3.3 + 1.0.0 + 1.11.704 From 98fab1766f65a853984c8937e86605b8c2a95567 Mon Sep 17 00:00:00 2001 From: Richard Wu Date: Tue, 25 Apr 2023 10:37:54 -0400 Subject: [PATCH 3/3] Comments. --- README.md | 28 +++++++++++++------ .../kafdrop/config/KafkaConfiguration.java | 16 +++++------ src/main/resources/application.yml | 4 +-- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 94c27413..a6bf8f8b 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Language grade: Java](https://img.shields.io/lgtm/grade/java/g/obsidiandynamics/kafdrop.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/obsidiandynamics/kafdrop/context:java) -Kafdrop is a web UI for viewing Kafka topics and browsing consumer groups. The tool displays information such as brokers, topics, partitions, consumers, and lets you view messages. +Kafdrop is a web UI for viewing Kafka topics and browsing consumer groups. The tool displays information such as brokers, topics, partitions, consumers, and lets you view messages. ![Overview Screenshot](docs/images/overview.png?raw=true) @@ -65,14 +65,14 @@ Finally, a default message and key format (e.g. to deserialize Avro messages or --message.format=AVRO --message.keyFormat=DEFAULT ``` -Valid format values are `DEFAULT`, `AVRO`, `PROTOBUF`. This can also be configured at the topic level via dropdown when viewing messages. +Valid format values are `DEFAULT`, `AVRO`, `PROTOBUF`. This can also be configured at the topic level via dropdown when viewing messages. If key format is unspecified, message format will be used for key too. ## Configure Protobuf message type -### Option 1: Using Protobuf Descriptor -In case of protobuf message type, the definition of a message could be compiled and transmitted using a descriptor file. -Thus, in order for kafdrop to recognize the message, the application will need to access to the descriptor file(s). -Kafdrop will allow user to select descriptor and well as specifying name of one of the message type provided by the descriptor at runtime. +### Option 1: Using Protobuf Descriptor +In case of protobuf message type, the definition of a message could be compiled and transmitted using a descriptor file. +Thus, in order for kafdrop to recognize the message, the application will need to access to the descriptor file(s). +Kafdrop will allow user to select descriptor and well as specifying name of one of the message type provided by the descriptor at runtime. To configure a folder with protobuf descriptor file(s) (.desc), follow: ``` @@ -289,7 +289,7 @@ docker run -d --rm -p 9000:9000 \ |`SERVER_PORT` |The web server port to listen on. Defaults to `9000`. |`SCHEMAREGISTRY_CONNECT `|The endpoint of Schema Registry for Avro or Protobuf message |`SCHEMAREGISTRY_AUTH` |Optional basic auth credentials in the form `username:password`. -|`CMD_ARGS` |Command line arguments to Kafdrop, e.g. `--message.format` or `--protobufdesc.directory` or `--server.port`. +|`CMD_ARGS` |Command line arguments to Kafdrop, e.g. `--message.format` or `--protobufdesc.directory` or `--server.port`. ##### Advanced configuration | Name |Description @@ -306,6 +306,16 @@ docker run -d --rm -p 9000:9000 \ | `SSL_KEY_STORE_PASSWORD` | Keystore password | `SSL_KEY_ALIAS` | Key alias +##### AWS SASL IAM +| Name |Description +|--------------------------|------------------------------- +| `KAFKA_IAM_ENABLED` |Set to `true` to use AWS SASL IAM +| `KAFKA_SASL_MECHANISM` |Set to `AWS_MSK_IAM` +| `KAFKA_SECURITY_PROTOCOL`|Set to `SASL_SSL` +| `KAFKA_SASL_JAAS_CONFIG` |Set to `software.amazon.msk.auth.iam.IAMLoginModule;` +| `KAFKA_SASL_CLIENT_CALLBACK`|Set to `software.amazon.msk.auth.iam.IAMClientCallbackHandler` +| `KAFKA_IS_SECURED` |Set to `true` + ### Using Helm Like in the Docker example, supply the files in base-64 form: @@ -342,7 +352,7 @@ Add a logout page in `/usr/local/opt/nginx/html/401.html`: Use the following snippet for `/usr/local/etc/nginx/nginx.conf`: ``` worker_processes 4; - + events { worker_connections 1024; } @@ -402,7 +412,7 @@ See [here](CONTRIBUTING.md). To cut an official release, these are the steps: -1. Commit a new version on master that has the `-SNAPSHOT` suffix stripped (see `pom.xml`). Once the commit is merged, the CI will treat it as a release build, and will end up publishing more artifacts than the regular (non-release/snapshot) build. One of those will be a dockerhub push to the specific version and "latest" tags. (The regular build doesn't update "latest"). +1. Commit a new version on master that has the `-SNAPSHOT` suffix stripped (see `pom.xml`). Once the commit is merged, the CI will treat it as a release build, and will end up publishing more artifacts than the regular (non-release/snapshot) build. One of those will be a dockerhub push to the specific version and "latest" tags. (The regular build doesn't update "latest"). 2. You can then edit the release description in GitHub to describe what went into the release. diff --git a/src/main/java/kafdrop/config/KafkaConfiguration.java b/src/main/java/kafdrop/config/KafkaConfiguration.java index 53ad76e0..73ab1caf 100644 --- a/src/main/java/kafdrop/config/KafkaConfiguration.java +++ b/src/main/java/kafdrop/config/KafkaConfiguration.java @@ -28,9 +28,9 @@ public final class KafkaConfiguration { private String truststoreFile; private String propertiesFile; private String keystoreFile; - private String jaasConfig; - private String clientCallback; - private String iamEnabled; + private String saslJaasConfig; + private String saslClientCallback; + private boolean iamEnabled = false; public void applyCommon(Properties properties) { properties.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerConnect); @@ -52,11 +52,11 @@ public void applyCommon(Properties properties) { LOG.info("Assigning truststore location to {}", truststoreFile); properties.put("ssl.truststore.location", truststoreFile); } - LOG.info("Is iam enabled : {}", iamEnabled); - if (Boolean.parseBoolean(iamEnabled)) { - LOG.info("Setting sasl.jaas.config {} and sasl and callback callback properties {}", jaasConfig, clientCallback); - properties.put(SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS, clientCallback); - properties.put(SaslConfigs.SASL_JAAS_CONFIG, jaasConfig); + LOG.info("Is IAM enabled : {}", iamEnabled); + if (iamEnabled) { + LOG.info("Setting SASL client callback {} and JAAS config to {}", saslClientCallback, saslJaasConfig); + properties.put(SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS, saslClientCallback); + properties.put(SaslConfigs.SASL_JAAS_CONFIG, saslJaasConfig); } LOG.info("Checking keystore file {}", keystoreFile); diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 203c4443..8f3f7fe8 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -45,5 +45,5 @@ kafka: propertiesFile : "${KAFKA_PROPERTIES_FILE:kafka.properties}" keystoreFile: "${KAFKA_KEYSTORE_FILE:kafka.keystore.jks}" iamEnabled: "${KAFKA_IAM_ENABLED:false}" - jaasConfig: "${KAFKA_JAAS_CONFIG}" - clientCallback: "software.amazon.msk.auth.iam.IAMClientCallbackHandler" + saslJaasConfig: "${KAFKA_SASL_JAAS_CONFIG}" + saslClientCallback: "${KAFKA_SASL_CLIENT_CALLBACK}"