From ecefd668e6ed259fae0029a3370e244a2d847bc1 Mon Sep 17 00:00:00 2001 From: Andrea Singh Date: Fri, 29 Nov 2024 07:58:22 -0800 Subject: [PATCH 01/18] Adding support for AWS IAM authentication for MSK Signed-off-by: Andrea Singh --- lib/fluent/plugin/out_rdkafka2.rb | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/lib/fluent/plugin/out_rdkafka2.rb b/lib/fluent/plugin/out_rdkafka2.rb index b306cfb..2038d90 100644 --- a/lib/fluent/plugin/out_rdkafka2.rb +++ b/lib/fluent/plugin/out_rdkafka2.rb @@ -101,6 +101,8 @@ class Fluent::Rdkafka2Output < Output config_param :unrecoverable_error_codes, :array, :default => ["topic_authorization_failed", "msg_size_too_large"], :desc => 'Handle some of the error codes should be unrecoverable if specified' + config_param :aws_msk_region, :string, :default => nil, :desc => 'AWS region for MSK' + config_section :buffer do config_set_default :chunk_keys, ["topic"] end @@ -205,10 +207,17 @@ def add(level, message = nil) end end } + # HERE ----------------- Rdkafka::Config.logger = log config = build_config @rdkafka = Rdkafka::Config.new(config) + + if config[:"security.protocol"] == "sasl_ssl" && config[:"sasl.mechanisms"] == "OAUTHBEARER" + Rdkafka::Config.oauthbearer_token_refresh_callback = method(:refresh_token) + end + # HERE ----------------- + if @default_topic.nil? if @use_default_for_unknown_topic || @use_default_for_unknown_partition_error raise Fluent::ConfigError, "default_topic must be set when use_default_for_unknown_topic or use_default_for_unknown_partition_error is true" @@ -289,6 +298,7 @@ def build_config config[:"sasl.password"] = @password if @password config[:"enable.idempotence"] = @idempotent if @idempotent + # sasl.mechnisms and security.protocol are set as rdkafka_options @rdkafka_options.each { |k, v| config[k.to_sym] = v } @@ -296,6 +306,26 @@ def build_config config end + def refresh_token(_config, _client_name) + print "refreshing token\n" + client = get_producer + signer = AwsMskIamSaslSigner::MSKTokenProvider.new(region: @aws_msk_region) + token = signer.generate_auth_token + + if token + client.oauthbearer_set_token( + token: token.token, + lifetime_ms: token.expiration_time_ms, + principal_name: "kafka-cluster" + ) + else + client.oauthbearer_set_token_failure( + "Failed to generate token." + ) + end + end + + # HERE ----------------- def start if @share_producer @shared_producer = @rdkafka.producer @@ -306,6 +336,7 @@ def start super end + # HERE ----------------- def multi_workers_ready? true From 242acd2fddee0ab73a42cf534d31b9ab694a0004 Mon Sep 17 00:00:00 2001 From: Andrea Singh Date: Fri, 29 Nov 2024 08:34:30 -0800 Subject: [PATCH 02/18] Add required gems for MSK IAM support Signed-off-by: Andrea Singh --- Gemfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index db01f5a..25b1013 100644 --- a/Gemfile +++ b/Gemfile @@ -3,4 +3,6 @@ source 'https://rubygems.org' # Specify your gem's dependencies in fluent-plugin-kafka.gemspec gemspec -gem 'rdkafka', ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE'] if ENV['USE_RDKAFKA'] +gem "json", "2.7.3" # locked to 2.7.3 for compatibility with AWS SDK code +gem "aws-msk-iam-sasl-signer" +gem 'rdkafka' From e3ebd74fbf9fd0bb7d72e55f2ebbfefccf78942e Mon Sep 17 00:00:00 2001 From: Andrea Singh Date: Fri, 29 Nov 2024 10:19:38 -0800 Subject: [PATCH 03/18] Move dependencies to gemspec file Signed-off-by: Andrea Singh --- Gemfile | 4 +--- fluent-plugin-kafka.gemspec | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 25b1013..db01f5a 100644 --- a/Gemfile +++ b/Gemfile @@ -3,6 +3,4 @@ source 'https://rubygems.org' # Specify your gem's dependencies in fluent-plugin-kafka.gemspec gemspec -gem "json", "2.7.3" # locked to 2.7.3 for compatibility with AWS SDK code -gem "aws-msk-iam-sasl-signer" -gem 'rdkafka' +gem 'rdkafka', ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE'] if ENV['USE_RDKAFKA'] diff --git a/fluent-plugin-kafka.gemspec b/fluent-plugin-kafka.gemspec index 3620ddb..ea118bb 100644 --- a/fluent-plugin-kafka.gemspec +++ b/fluent-plugin-kafka.gemspec @@ -19,6 +19,9 @@ Gem::Specification.new do |gem| gem.add_dependency "fluentd", [">= 0.10.58", "< 2"] gem.add_dependency 'ltsv' gem.add_dependency 'ruby-kafka', '>= 1.5.0', '< 2' + gem.add_dependency 'rdkafka' + gem.add_dependency 'aws-msk-iam-sasl-signer' + gem.add_dependency 'json', '2.7.3' gem.add_development_dependency "rake", ">= 0.9.2" gem.add_development_dependency "test-unit", ">= 3.0.8" gem.add_development_dependency "test-unit-rr", "~> 1.0" From 6237d2177f5418a07e4df5a51f37337078147f9c Mon Sep 17 00:00:00 2001 From: Andrea Singh Date: Mon, 2 Dec 2024 11:23:39 -0800 Subject: [PATCH 04/18] Fix issues revealed in test setup; Add fork info to README Signed-off-by: Andrea Singh --- Gemfile | 1 + README.md | 41 ++++++++++++++++++++++++++++++- fluent-plugin-kafka.gemspec | 1 - lib/fluent/plugin/out_rdkafka2.rb | 16 +++++++++--- 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/Gemfile b/Gemfile index db01f5a..0e3cf2b 100644 --- a/Gemfile +++ b/Gemfile @@ -3,4 +3,5 @@ source 'https://rubygems.org' # Specify your gem's dependencies in fluent-plugin-kafka.gemspec gemspec +gem 'json', '2.7.3' # override of 2.7.4 version gem 'rdkafka', ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE'] if ENV['USE_RDKAFKA'] diff --git a/README.md b/README.md index 814d40e..9147728 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,45 @@ If you want to use zookeeper related parameters, you also need to install zookee ## Usage +### :exclamation: In this fork: MSK IAM Authentication Support for `rdkafka2` Output Type + +This fork adds support for using MSK IAM authentication with the `rdkafka2` output type in Fluentd. Authentication and authorization with an MSK cluster are facilitated through a base64-encoded signed URL, which is generated by the [aws-msk-iam-sasl-signer-ruby](https://github.com/bruce-szalwinski-he/aws-msk-iam-sasl-signer-ruby) library. + +The `aws-msk-iam-sasl-signer-ruby` library provides an [example](https://github.com/bruce-szalwinski-he/aws-msk-iam-sasl-signer-ruby/tree/main/examples/rdkafka) for generating the OAuthBearer token using rdkafka, which is one of the core Kafka libraries supported by the Fluentd fluent-plugin-kafka plugin. This fork integrates that example into the `Fluent::Rdkafka2Output` class, enabling AWS IAM authentication. + +The key change is the inclusion of a refresh callback: +```ruby +Rdkafka::Config.oauthbearer_token_refresh_callback = method(:refresh_token) +``` +This callback triggers token generation when needed, ensuring continuous authentication with the MSK cluster. + +#### Configuration Example +To enable this feature, configure your Fluentd input as follows: + +``` + + @type rdkafka2 + # Kafka brokers to connect to (typically port 9098 or 9198 for IAM authentication) + brokers + # Topic to write events to + topic_key test-topic-1 + default_topic test-topic-1 + + # AWS Region (required) + aws_msk_region us-east-1 + + # Use a shared producer for the connection (required) + share_producer true + + # MSK IAM authentication settings (required) + rdkafka_options { + "security.protocol": "sasl_ssl", + "sasl.mechanisms": "OAUTHBEARER" + } + +``` +With this configuration, Fluentd will handle the token refresh and manage the connection to your MSK cluster using AWS IAM authentication. + ### Common parameters #### SSL authentication @@ -563,7 +602,7 @@ You need to install rdkafka gem. `rdkafka2` supports `discard_kafka_delivery_failed_regex` parameter: -- `discard_kafka_delivery_failed_regex` - default: nil - discard the record where the Kafka::DeliveryFailed occurred and the emitted message matches the given regex pattern, such as `/unknown_topic/`. +- `discard_kafka_delivery_failed_regex` - default: nil - discard the record where the Kafka::DeliveryFailed occurred and the emitted message matches the given regex pattern, such as `/unknown_topic/`. If you use v0.12, use `rdkafka` instead. diff --git a/fluent-plugin-kafka.gemspec b/fluent-plugin-kafka.gemspec index ea118bb..ad57ea9 100644 --- a/fluent-plugin-kafka.gemspec +++ b/fluent-plugin-kafka.gemspec @@ -21,7 +21,6 @@ Gem::Specification.new do |gem| gem.add_dependency 'ruby-kafka', '>= 1.5.0', '< 2' gem.add_dependency 'rdkafka' gem.add_dependency 'aws-msk-iam-sasl-signer' - gem.add_dependency 'json', '2.7.3' gem.add_development_dependency "rake", ">= 0.9.2" gem.add_development_dependency "test-unit", ">= 3.0.8" gem.add_development_dependency "test-unit-rr", "~> 1.0" diff --git a/lib/fluent/plugin/out_rdkafka2.rb b/lib/fluent/plugin/out_rdkafka2.rb index 2038d90..c76e3f5 100644 --- a/lib/fluent/plugin/out_rdkafka2.rb +++ b/lib/fluent/plugin/out_rdkafka2.rb @@ -4,6 +4,7 @@ require 'fluent/plugin/kafka_plugin_util' require 'rdkafka' +require 'aws_msk_iam_sasl_signer' begin rdkafka_version = Gem::Version::create(Rdkafka::VERSION) @@ -307,8 +308,14 @@ def build_config end def refresh_token(_config, _client_name) - print "refreshing token\n" + log.info("+--- Refreshing token") client = get_producer + # This will happen once upon initialization and is expected to fail, as the producer isnt set yet + # We will set the token manually after creation and after that this refresh method will work + unless client + log.info("Could not get shared client handle, unable to set/refresh token (this is expected one time on startup)") + return + end signer = AwsMskIamSaslSigner::MSKTokenProvider.new(region: @aws_msk_region) token = signer.generate_auth_token @@ -325,10 +332,14 @@ def refresh_token(_config, _client_name) end end - # HERE ----------------- def start if @share_producer @shared_producer = @rdkafka.producer + log.info("Created shared producer") + if @aws_msk_region + refresh_token(nil, nil) + log.info("Set initial token for shared producer") + end else @producers = {} @producers_mutex = Mutex.new @@ -336,7 +347,6 @@ def start super end - # HERE ----------------- def multi_workers_ready? true From 7c5fbc5bf3661ff02edb84581bd42980bb1dde9c Mon Sep 17 00:00:00 2001 From: Andrea Singh Date: Sat, 7 Dec 2024 14:44:29 -0800 Subject: [PATCH 05/18] Adding gems conditionally Signed-off-by: Andrea Singh --- Gemfile | 11 +++++++++-- fluent-plugin-kafka.gemspec | 2 -- lib/fluent/plugin/out_rdkafka2.rb | 4 +--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Gemfile b/Gemfile index 0e3cf2b..45898ca 100644 --- a/Gemfile +++ b/Gemfile @@ -3,5 +3,12 @@ source 'https://rubygems.org' # Specify your gem's dependencies in fluent-plugin-kafka.gemspec gemspec -gem 'json', '2.7.3' # override of 2.7.4 version -gem 'rdkafka', ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE'] if ENV['USE_RDKAFKA'] +if ENV['USE_RDKAFKA'] + gem 'rdkafka', ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE'] + min_version = Gem::Version.new('0.16.0') + min_range_version = Gem::Version.new(ENV['RDKAFKA_VERSION_MIN_RANGE'].split(' ').last) + if min_range_version >= min_version + gem 'aws-msk-iam-sasl-signer' + gem 'json', '2.7.3' # override of 2.7.4 version + end +end diff --git a/fluent-plugin-kafka.gemspec b/fluent-plugin-kafka.gemspec index ad57ea9..3620ddb 100644 --- a/fluent-plugin-kafka.gemspec +++ b/fluent-plugin-kafka.gemspec @@ -19,8 +19,6 @@ Gem::Specification.new do |gem| gem.add_dependency "fluentd", [">= 0.10.58", "< 2"] gem.add_dependency 'ltsv' gem.add_dependency 'ruby-kafka', '>= 1.5.0', '< 2' - gem.add_dependency 'rdkafka' - gem.add_dependency 'aws-msk-iam-sasl-signer' gem.add_development_dependency "rake", ">= 0.9.2" gem.add_development_dependency "test-unit", ">= 3.0.8" gem.add_development_dependency "test-unit-rr", "~> 1.0" diff --git a/lib/fluent/plugin/out_rdkafka2.rb b/lib/fluent/plugin/out_rdkafka2.rb index c76e3f5..9a09ce0 100644 --- a/lib/fluent/plugin/out_rdkafka2.rb +++ b/lib/fluent/plugin/out_rdkafka2.rb @@ -4,7 +4,6 @@ require 'fluent/plugin/kafka_plugin_util' require 'rdkafka' -require 'aws_msk_iam_sasl_signer' begin rdkafka_version = Gem::Version::create(Rdkafka::VERSION) @@ -16,6 +15,7 @@ require_relative 'rdkafka_patch/0_14_0' elsif rdkafka_version >= Gem::Version.create('0.16.0') require_relative 'rdkafka_patch/0_16_0' + require 'aws_msk_iam_sasl_signer' end rescue LoadError, NameError raise "unable to patch rdkafka." @@ -208,7 +208,6 @@ def add(level, message = nil) end end } - # HERE ----------------- Rdkafka::Config.logger = log config = build_config @rdkafka = Rdkafka::Config.new(config) @@ -217,7 +216,6 @@ def add(level, message = nil) if config[:"security.protocol"] == "sasl_ssl" && config[:"sasl.mechanisms"] == "OAUTHBEARER" Rdkafka::Config.oauthbearer_token_refresh_callback = method(:refresh_token) end - # HERE ----------------- if @default_topic.nil? if @use_default_for_unknown_topic || @use_default_for_unknown_partition_error From d59740c341cd995ca43c0cbfcc903234fd6165b9 Mon Sep 17 00:00:00 2001 From: Andrea Singh Date: Sat, 7 Dec 2024 19:16:10 -0800 Subject: [PATCH 06/18] Moving required gems into gemspec Signed-off-by: Andrea Singh --- Gemfile | 10 ---------- fluent-plugin-kafka.gemspec | 6 ++++++ lib/fluent/plugin/out_rdkafka2.rb | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/Gemfile b/Gemfile index 45898ca..d5c272e 100644 --- a/Gemfile +++ b/Gemfile @@ -2,13 +2,3 @@ source 'https://rubygems.org' # Specify your gem's dependencies in fluent-plugin-kafka.gemspec gemspec - -if ENV['USE_RDKAFKA'] - gem 'rdkafka', ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE'] - min_version = Gem::Version.new('0.16.0') - min_range_version = Gem::Version.new(ENV['RDKAFKA_VERSION_MIN_RANGE'].split(' ').last) - if min_range_version >= min_version - gem 'aws-msk-iam-sasl-signer' - gem 'json', '2.7.3' # override of 2.7.4 version - end -end diff --git a/fluent-plugin-kafka.gemspec b/fluent-plugin-kafka.gemspec index 3620ddb..d6d3371 100644 --- a/fluent-plugin-kafka.gemspec +++ b/fluent-plugin-kafka.gemspec @@ -19,6 +19,12 @@ Gem::Specification.new do |gem| gem.add_dependency "fluentd", [">= 0.10.58", "< 2"] gem.add_dependency 'ltsv' gem.add_dependency 'ruby-kafka', '>= 1.5.0', '< 2' + + if ENV['USE_RDKAFKA'] + gem.add_dependency 'rdkafka', [ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE']] + gem.add_dependency 'aws-msk-iam-sasl-signer', '~> 0.1.1' + end + gem.add_development_dependency "rake", ">= 0.9.2" gem.add_development_dependency "test-unit", ">= 3.0.8" gem.add_development_dependency "test-unit-rr", "~> 1.0" diff --git a/lib/fluent/plugin/out_rdkafka2.rb b/lib/fluent/plugin/out_rdkafka2.rb index 9a09ce0..f3bfbdd 100644 --- a/lib/fluent/plugin/out_rdkafka2.rb +++ b/lib/fluent/plugin/out_rdkafka2.rb @@ -15,8 +15,8 @@ require_relative 'rdkafka_patch/0_14_0' elsif rdkafka_version >= Gem::Version.create('0.16.0') require_relative 'rdkafka_patch/0_16_0' - require 'aws_msk_iam_sasl_signer' end + require 'aws_msk_iam_sasl_signer' if rdkafka_version >= Gem::Version.create('0.16.0') rescue LoadError, NameError raise "unable to patch rdkafka." end From d8fb747a701baac1f771795c67e07a5b0b965a4e Mon Sep 17 00:00:00 2001 From: Andrea Singh Date: Sat, 7 Dec 2024 19:31:31 -0800 Subject: [PATCH 07/18] Make aws signer lib dependent on min ruby version Signed-off-by: Andrea Singh --- fluent-plugin-kafka.gemspec | 4 +++- lib/fluent/plugin/out_rdkafka2.rb | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/fluent-plugin-kafka.gemspec b/fluent-plugin-kafka.gemspec index d6d3371..c7700a4 100644 --- a/fluent-plugin-kafka.gemspec +++ b/fluent-plugin-kafka.gemspec @@ -22,7 +22,9 @@ Gem::Specification.new do |gem| if ENV['USE_RDKAFKA'] gem.add_dependency 'rdkafka', [ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE']] - gem.add_dependency 'aws-msk-iam-sasl-signer', '~> 0.1.1' + if Gem::Version.new('3.0' >= Gem::Version.new(RUBY_VERSION) + gem.add_dependency 'aws-msk-iam-sasl-signer', '~> 0.1.1' + end end gem.add_development_dependency "rake", ">= 0.9.2" diff --git a/lib/fluent/plugin/out_rdkafka2.rb b/lib/fluent/plugin/out_rdkafka2.rb index f3bfbdd..6301e19 100644 --- a/lib/fluent/plugin/out_rdkafka2.rb +++ b/lib/fluent/plugin/out_rdkafka2.rb @@ -16,11 +16,14 @@ elsif rdkafka_version >= Gem::Version.create('0.16.0') require_relative 'rdkafka_patch/0_16_0' end - require 'aws_msk_iam_sasl_signer' if rdkafka_version >= Gem::Version.create('0.16.0') rescue LoadError, NameError raise "unable to patch rdkafka." end +if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.0') + require 'aws-msk-iam-sasl-signer' +end + module Fluent::Plugin class Fluent::Rdkafka2Output < Output Fluent::Plugin.register_output('rdkafka2', self) From 7e572531750790936b0b74a9ea6ee51bfee5c0b4 Mon Sep 17 00:00:00 2001 From: Andrea Singh Date: Sun, 8 Dec 2024 14:11:43 -0800 Subject: [PATCH 08/18] Fix typo in fluent-plugin-kafka.gemspec Signed-off-by: Andrea Singh --- fluent-plugin-kafka.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluent-plugin-kafka.gemspec b/fluent-plugin-kafka.gemspec index c7700a4..db3381f 100644 --- a/fluent-plugin-kafka.gemspec +++ b/fluent-plugin-kafka.gemspec @@ -22,7 +22,7 @@ Gem::Specification.new do |gem| if ENV['USE_RDKAFKA'] gem.add_dependency 'rdkafka', [ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE']] - if Gem::Version.new('3.0' >= Gem::Version.new(RUBY_VERSION) + if Gem::Version.new('3.0') >= Gem::Version.new(RUBY_VERSION) gem.add_dependency 'aws-msk-iam-sasl-signer', '~> 0.1.1' end end From 68d6e1155d0b5764ae8f436e07ec784d32225369 Mon Sep 17 00:00:00 2001 From: Andrea Singh Date: Sun, 8 Dec 2024 18:08:12 -0800 Subject: [PATCH 09/18] Fix conditional in gemspec Signed-off-by: Andrea Singh --- fluent-plugin-kafka.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluent-plugin-kafka.gemspec b/fluent-plugin-kafka.gemspec index db3381f..ceace4f 100644 --- a/fluent-plugin-kafka.gemspec +++ b/fluent-plugin-kafka.gemspec @@ -22,7 +22,7 @@ Gem::Specification.new do |gem| if ENV['USE_RDKAFKA'] gem.add_dependency 'rdkafka', [ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE']] - if Gem::Version.new('3.0') >= Gem::Version.new(RUBY_VERSION) + if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.0') gem.add_dependency 'aws-msk-iam-sasl-signer', '~> 0.1.1' end end From 66ab6b5dadb817b93a816ce969628900de46cbf6 Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Mon, 27 Jan 2025 11:12:22 +0900 Subject: [PATCH 10/18] github: track issue with GitHub Project It helps to track new issues by aggregating ones from Project view. Signed-off-by: Kentaro Hayashi --- .github/workflows/add-to-project.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/add-to-project.yml diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml new file mode 100644 index 0000000..b09bbee --- /dev/null +++ b/.github/workflows/add-to-project.yml @@ -0,0 +1,17 @@ +name: Add bugs to fluent project + +on: + issues: + types: + - opened + +jobs: + add-to-project: + name: Add issue to project + runs-on: ubuntu-latest + steps: + - uses: actions/add-to-project@v1.0.2 + with: + project-url: https://github.com/orgs/fluent/projects/4 + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + labeled: waiting-for-triage From 6b533258fba7f1e7ac0c0f346e1c8cb60d183c7c Mon Sep 17 00:00:00 2001 From: Shizuo Fujita Date: Tue, 4 Mar 2025 13:45:39 +0900 Subject: [PATCH 11/18] ci: use bundler installed by ruby/setup-ruby action Signed-off-by: Shizuo Fujita --- .github/workflows/linux.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 443851d..4c660c9 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -20,18 +20,14 @@ jobs: - { min: '>= 0.12.0', max: '< 0.14.0' } - { min: '>= 0.14.0', max: '< 0.16.0' } - { min: '>= 0.16.0', max: '>= 0.16.0' } - bundler_version: - - '2.5.16' # rdkafka 0.15.2 is the last version which supports Ruby 2.7 include: - ruby: '2.7' os: ubuntu-latest rdkafka_versions: { min: '>= 0.6.0', max: '< 0.12.0' } - bundler_version: '2.4.22' - ruby: '2.7' os: ubuntu-latest rdkafka_versions: { min: '>= 0.12.0', max: '= 0.15.2' } - bundler_version: '2.4.22' name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} with rdkafka gem version (min ${{ matrix.rdkafka_versions.min }} max ${{ matrix.rdkafka_versions.max }}) steps: - uses: actions/checkout@v4 @@ -54,7 +50,6 @@ jobs: RDKAFKA_VERSION_MAX_RANGE: ${{ matrix.rdkafka_versions.max }} run: | sudo ./ci/prepare-kafka-server.sh - gem install bundler -v ${{ matrix.bundler_version }} gem install rake - bundle _${{ matrix.bundler_version }}_ install --jobs 4 --retry 3 - bundle _${{ matrix.bundler_version }}_ exec rake test + bundle install --jobs 4 --retry 3 + bundle exec rake test From 1ad7d36003d56fdeb8f9defb279db48e3eee785e Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Thu, 6 Feb 2025 17:04:34 +0900 Subject: [PATCH 12/18] github: add Ruby 3.4 for CI Signed-off-by: Kentaro Hayashi --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4c660c9..9b04528 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - ruby: [ '3.3', '3.2', '3.1', '3.0' ] + ruby: [ '3.4', '3.3', '3.2', '3.1', '3.0' ] os: - ubuntu-latest rdkafka_versions: From 206e42f552bc8e51f8e44c1ee557b4cf3108d068 Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Tue, 4 Mar 2025 15:12:55 +0900 Subject: [PATCH 13/18] Add bigdecimal gem dependency for Ruby 3.4 /opt/hostedtoolcache/Ruby/3.4.2/x64/lib/ruby/gems/3.4.0/gems/ruby-kafka-1.5.0/lib/kafka/protocol/record_batch.rb:1: warning: bigdecimal was loaded from the standard library, but is not part of the default gems starting from Ruby 3.4.0. You can add bigdecimal to your Gemfile or gemspec to silence this warning. Signed-off-by: Kentaro Hayashi --- fluent-plugin-kafka.gemspec | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fluent-plugin-kafka.gemspec b/fluent-plugin-kafka.gemspec index ceace4f..ccc6f03 100644 --- a/fluent-plugin-kafka.gemspec +++ b/fluent-plugin-kafka.gemspec @@ -20,6 +20,10 @@ Gem::Specification.new do |gem| gem.add_dependency 'ltsv' gem.add_dependency 'ruby-kafka', '>= 1.5.0', '< 2' + # gems that aren't default gems as of Ruby 3.4 + gem.add_dependency("bigdecimal", ["~> 3.1"]) + + if ENV['USE_RDKAFKA'] gem.add_dependency 'rdkafka', [ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE']] if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.0') From 632faa21e0d21a4a481ca4227db991eca74fc657 Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Tue, 25 Mar 2025 10:08:29 +0900 Subject: [PATCH 14/18] v0.19.4 (#530) Signed-off-by: Kentaro Hayashi --- ChangeLog | 3 +++ fluent-plugin-kafka.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index ebae296..ef21ec9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +Release 0.19.4 - 2025/03/24 + * Support Ruby 3.4. (#526) + Release 0.19.3 - 2024/08/02 * out_rdkafka2: Add `unrecoverable_error_codes` parameter to handle specific error code as unrecoverable errors. `topic_authorization_failed` diff --git a/fluent-plugin-kafka.gemspec b/fluent-plugin-kafka.gemspec index ccc6f03..701b9e6 100644 --- a/fluent-plugin-kafka.gemspec +++ b/fluent-plugin-kafka.gemspec @@ -13,7 +13,7 @@ Gem::Specification.new do |gem| gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) gem.name = "fluent-plugin-kafka" gem.require_paths = ["lib"] - gem.version = '0.19.3' + gem.version = '0.19.4' gem.required_ruby_version = ">= 2.1.0" gem.add_dependency "fluentd", [">= 0.10.58", "< 2"] From 1d313a6535af3e4b9ccc15b540e2a6b7b4650b8a Mon Sep 17 00:00:00 2001 From: kuckjwi Date: Thu, 10 Jul 2025 15:20:40 +0900 Subject: [PATCH 15/18] feat: Add aws msk iam option (#531) Signed-off-by: kuckjwi --- lib/fluent/plugin/in_kafka_group.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/fluent/plugin/in_kafka_group.rb b/lib/fluent/plugin/in_kafka_group.rb index 70dbdbe..851ab9c 100644 --- a/lib/fluent/plugin/in_kafka_group.rb +++ b/lib/fluent/plugin/in_kafka_group.rb @@ -74,6 +74,7 @@ class Fluent::KafkaGroupInput < Fluent::Input include Fluent::KafkaPluginUtil::SSLSettings include Fluent::KafkaPluginUtil::SaslSettings + include Fluent::KafkaPluginUtil::AwsIamSettings class ForShutdown < StandardError end @@ -185,6 +186,7 @@ def start super logger = @get_kafka_client_log ? log : nil + use_aws_msk_iam = @sasl_aws_msk_iam_access_key_id != nil && @sasl_aws_msk_iam_secret_key_id != nil if @scram_mechanism != nil && @username != nil && @password != nil @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert, ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), @@ -197,6 +199,10 @@ def start ssl_client_cert_key_password: @ssl_client_cert_key_password, ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname) + elsif use_aws_msk_iam + @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, + sasl_aws_msk_iam_secret_key_id: @sasl_aws_msk_iam_secret_key_id, sasl_aws_msk_iam_access_key_id: @sasl_aws_msk_iam_access_key_id, sasl_aws_msk_iam_aws_region: @sasl_aws_msk_iam_aws_region, + ssl_ca_certs_from_system: @ssl_ca_certs_from_system) else @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert, ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), From 57557c306d5b1a8f9e8d4eb91eb21851b01b6217 Mon Sep 17 00:00:00 2001 From: Shizuo Fujita Date: Fri, 11 Jul 2025 16:09:04 +0900 Subject: [PATCH 16/18] v0.19.5 Signed-off-by: Shizuo Fujita --- ChangeLog | 4 ++++ fluent-plugin-kafka.gemspec | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index ef21ec9..02e9859 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +Release 0.19.5 - 2025/07/11 + * in_kafka_group: Add `sasl_aws_msk_iam_access_key_id`, `sasl_aws_msk_iam_secret_access_key` + and `sasl_aws_msk_iam_aws_region` options (#531) + Release 0.19.4 - 2025/03/24 * Support Ruby 3.4. (#526) diff --git a/fluent-plugin-kafka.gemspec b/fluent-plugin-kafka.gemspec index 701b9e6..9e8e9ac 100644 --- a/fluent-plugin-kafka.gemspec +++ b/fluent-plugin-kafka.gemspec @@ -13,7 +13,7 @@ Gem::Specification.new do |gem| gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) gem.name = "fluent-plugin-kafka" gem.require_paths = ["lib"] - gem.version = '0.19.4' + gem.version = '0.19.5' gem.required_ruby_version = ">= 2.1.0" gem.add_dependency "fluentd", [">= 0.10.58", "< 2"] From acac7110a2a19cc0a3b14d3c905f0ff0f4886a15 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Aug 2025 10:10:34 +0900 Subject: [PATCH 17/18] build(deps): bump actions/checkout from 4 to 5 (#533) Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9b04528..eed4432 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -30,7 +30,7 @@ jobs: rdkafka_versions: { min: '>= 0.12.0', max: '= 0.15.2' } name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} with rdkafka gem version (min ${{ matrix.rdkafka_versions.min }} max ${{ matrix.rdkafka_versions.max }}) steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby }} From e23d33f542c722e921937799e453680c42e42940 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 16:02:03 +0900 Subject: [PATCH 18/18] build(deps): bump actions/stale from 9 to 10 (#535) Bumps [actions/stale](https://github.com/actions/stale) from 9 to 10. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v9...v10) --- updated-dependencies: - dependency-name: actions/stale dependency-version: '10' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/stale-actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale-actions.yml b/.github/workflows/stale-actions.yml index 2cadd07..845867c 100644 --- a/.github/workflows/stale-actions.yml +++ b/.github/workflows/stale-actions.yml @@ -7,7 +7,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v9 + - uses: actions/stale@v10 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 90