diff --git a/CHANGELOG.md b/CHANGELOG.md index e7d6392..e988389 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 11.6.1 + - Expose the SASL client callback class setting to the Logstash configuration [#177](https://github.com/logstash-plugins/logstash-integration-kafka/pull/177) + - Adds a mechanism to load AWS IAM authentication as SASL client libraries at startup [#178](https://github.com/logstash-plugins/logstash-integration-kafka/pull/178) + ## 11.6.0 - Support additional `oauth` and `sasl` configuration options for configuring kafka client [#189](https://github.com/logstash-plugins/logstash-integration-kafka/pull/189) diff --git a/docs/input-kafka.asciidoc b/docs/input-kafka.asciidoc index ffe67f8..a312eb1 100644 --- a/docs/input-kafka.asciidoc +++ b/docs/input-kafka.asciidoc @@ -65,6 +65,23 @@ For more information see https://kafka.apache.org/{kafka_client_doc}/documentati Kafka consumer configuration: https://kafka.apache.org/{kafka_client_doc}/documentation.html#consumerconfigs +[id="plugins-{type}s-{plugin}-aws_msk_iam_auth"] +==== AWS MSK IAM authentication +If you use AWS MSK, the AWS MSK IAM access control enables you to handle both authentication and authorization for your MSK cluster with AWS IAM. +For more information on this AWS MSK feature see the https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control.html[AWS documentation]. + +To use this Kafka input with AWS MSK IAM authentication, download the uber jar which contains the client library for +this specific cloud vendor and all the transitive dependencies from this https://github.com/elastic/logstash-kafka-iams-packages/releases[repository]. +Configure the following setting: +``` +security_protocol => "SASL_SSL" +sasl_mechanism => "AWS_MSK_IAM" +sasl_iam_jar_paths => ["/path/to/aws_iam_uber.jar"] +sasl_jaas_config => "software.amazon.msk.auth.iam.IAMLoginModule required;" +sasl_client_callback_handler_class => "software.amazon.msk.auth.iam.IAMClientCallbackHandler" +``` +For more IAM authentication configurations, see the https://github.com/aws/aws-msk-iam-auth[AWS MSK IAM authentication library documentation]. + ==== Metadata fields The following metadata from Kafka broker are added under the `[@metadata]` field: @@ -133,6 +150,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai | <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No | <> |<>|No | <> |<>|No | <> |<>|No @@ -584,6 +602,14 @@ The URL for the OAuth 2.0 issuer token endpoint. (optional) The override name of the scope claim. +[id="plugins-{type}s-{plugin}-sasl_iam_jar_paths"] +===== `sasl_iam_jar_paths` +* Value type is <> +* There is no default value for this setting. + +Contains the list of paths to jar libraries that contains cloud providers MSK IAM's clients. +There is one jar per provider and can be retrieved as described in <<"plugins-{type}s-{plugin}-aws_msk_iam_auth">>. + [id="plugins-{type}s-{plugin}-sasl_login_callback_handler_class"] ===== `sasl_login_callback_handler_class` * Value type is <> @@ -654,6 +680,7 @@ This can be defined either in Kafka's JAAS config or in Kafka's config. http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. This may be any mechanism for which a security provider is available. +For AWS MSK IAM authentication use `AWS_MSK_IAM`. GSSAPI is the default mechanism. [id="plugins-{type}s-{plugin}-schema_registry_key"] diff --git a/docs/output-kafka.asciidoc b/docs/output-kafka.asciidoc index 64f3dbe..ac7c543 100644 --- a/docs/output-kafka.asciidoc +++ b/docs/output-kafka.asciidoc @@ -66,6 +66,23 @@ https://kafka.apache.org/{kafka_client_doc}/documentation.html#producerconfigs NOTE: This plugin does not support using a proxy when communicating to the Kafka broker. +[id="plugins-{type}s-{plugin}-aws_msk_iam_auth"] +==== AWS MSK IAM authentication +If you use AWS MSK, the AWS MSK IAM access control enables you to handle both authentication and authorization for your MSK cluster with AWS IAM. +For more information on this AWS MSK feature see the https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control.html[AWS documentation]. + +To use this Kafka input with AWS MSK IAM authentication, download the uber jar which contains the client library for +this specific cloud vendor and all the transitive dependencies from this https://github.com/elastic/logstash-kafka-iams-packages/releases[repository]. +Configure the following setting: +``` +security_protocol => "SASL_SSL" +sasl_mechanism => "AWS_MSK_IAM" +sasl_iam_jar_paths => ["/path/to/aws_iam_uber.jar"] +sasl_jaas_config => "software.amazon.msk.auth.iam.IAMLoginModule required;" +sasl_client_callback_handler_class => "software.amazon.msk.auth.iam.IAMClientCallbackHandler" +``` +For more IAM authentication configurations, see the https://github.com/aws/aws-msk-iam-auth[AWS MSK IAM authentication library documentation]. + [id="plugins-{type}s-{plugin}-options"] ==== Kafka Output Configuration Options @@ -104,6 +121,7 @@ See the https://kafka.apache.org/{kafka_client_doc}/documentation for more detai | <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No | <> |<>|No | <> |<>|No | <> |<>|No @@ -420,6 +438,14 @@ The URL for the OAuth 2.0 issuer token endpoint. (optional) The override name of the scope claim. +[id="plugins-{type}s-{plugin}-sasl_iam_jar_paths"] +===== `sasl_iam_jar_paths` +* Value type is <> +* There is no default value for this setting. + +Contains the list of paths to jar libraries that contains cloud providers MSK IAM's clients. +There is one jar per provider and can be retrieved as described in <<"plugins-{type}s-{plugin}-aws_msk_iam_auth">>. + [id="plugins-{type}s-{plugin}-sasl_login_callback_handler_class"] ===== `sasl_login_callback_handler_class` * Value type is <> @@ -490,6 +516,7 @@ This can be defined either in Kafka's JAAS config or in Kafka's config. http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. This may be any mechanism for which a security provider is available. +For AWS MSK IAM authentication use `AWS_MSK_IAM`. GSSAPI is the default mechanism. [id="plugins-{type}s-{plugin}-security_protocol"] diff --git a/lib/logstash/inputs/kafka.rb b/lib/logstash/inputs/kafka.rb index aa713c3..b8b1dbd 100644 --- a/lib/logstash/inputs/kafka.rb +++ b/lib/logstash/inputs/kafka.rb @@ -216,6 +216,8 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base config :sasl_oauthbearer_scope_claim_name, :validate => :string, :default => 'scope' # Kafka default # SASL login callback handler class config :sasl_login_callback_handler_class, :validate => :string + # Path to the jar containing client and all dependencies for SASL IAM authentication of specific cloud vendor + config :sasl_iam_jar_paths, :validate => :array # (optional) The duration, in milliseconds, for HTTPS connect timeout config :sasl_login_connect_timeout_ms, :validate => :number # (optional) The duration, in milliseconds, for HTTPS read timeout. @@ -224,7 +226,7 @@ class LogStash::Inputs::Kafka < LogStash::Inputs::Base config :sasl_login_retry_backoff_ms, :validate => :number, :default => 100 # Kafka default # (optional) The maximum duration, in milliseconds, for HTTPS call attempts. config :sasl_login_retry_backoff_max_ms, :validate => :number, :default => 10000 # Kafka default - # http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. + # http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. # This may be any mechanism for which a security provider is available. # GSSAPI is the default mechanism. config :sasl_mechanism, :validate => :string, :default => "GSSAPI" diff --git a/lib/logstash/outputs/kafka.rb b/lib/logstash/outputs/kafka.rb index 8ba0148..5557543 100644 --- a/lib/logstash/outputs/kafka.rb +++ b/lib/logstash/outputs/kafka.rb @@ -149,6 +149,8 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base config :security_protocol, :validate => ["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"], :default => "PLAINTEXT" # SASL client callback handler class config :sasl_client_callback_handler_class, :validate => :string + # Path to the jar containing client and all dependencies for SASL IAM authentication of specific cloud vendor + config :sasl_iam_jar_paths, :validate => :array # The URL for the OAuth 2.0 issuer token endpoint. config :sasl_oauthbearer_token_endpoint_url, :validate => :string # (optional) The override name of the scope claim. diff --git a/lib/logstash/plugin_mixins/kafka/common.rb b/lib/logstash/plugin_mixins/kafka/common.rb index 6564e80..238e27a 100644 --- a/lib/logstash/plugin_mixins/kafka/common.rb +++ b/lib/logstash/plugin_mixins/kafka/common.rb @@ -49,6 +49,7 @@ def set_sasl_config(props) props.put("sasl.login.read.timeout.ms", sasl_login_read_timeout_ms.to_s) unless sasl_login_read_timeout_ms.nil? props.put("sasl.login.retry.backoff.ms", sasl_login_retry_backoff_ms.to_s) unless sasl_login_retry_backoff_ms.nil? props.put("sasl.login.retry.backoff.max.ms", sasl_login_retry_backoff_max_ms.to_s) unless sasl_login_retry_backoff_max_ms.nil? + sasl_iam_jar_paths&.each {|jar_path| require jar_path } end def reassign_dns_lookup diff --git a/logstash-integration-kafka.gemspec b/logstash-integration-kafka.gemspec index 9c3bf38..0bb1518 100644 --- a/logstash-integration-kafka.gemspec +++ b/logstash-integration-kafka.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-integration-kafka' - s.version = '11.6.0' + s.version = '11.6.1' s.licenses = ['Apache-2.0'] s.summary = "Integration with Kafka - input and output plugins" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+