Merge branch 'master' into aws_msk_with_iam_support

Author: madebydna

.github/workflows/add-to-project.yml

@@ -0,0 +1,17 @@
+name: Add bugs to fluent project
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+        with:
+          project-url: https://github.com/orgs/fluent/projects/4
+          github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
+          labeled: waiting-for-triage

.github/workflows/linux.yml

@@ -12,29 +12,25 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.3', '3.2', '3.1', '3.0' ]
+        ruby: [ '3.4', '3.3', '3.2', '3.1', '3.0' ]
         os:
           - ubuntu-latest
         rdkafka_versions:
           - { min: '>= 0.6.0', max: '< 0.12.0' }
           - { min: '>= 0.12.0', max: '< 0.14.0' }
           - { min: '>= 0.14.0', max: '< 0.16.0' }
           - { min: '>= 0.16.0', max: '>= 0.16.0' }
-        bundler_version:
-          - '2.5.16'
         # rdkafka 0.15.2 is the last version which supports Ruby 2.7
         include:
           - ruby: '2.7'
             os: ubuntu-latest
             rdkafka_versions: { min: '>= 0.6.0', max: '< 0.12.0' }
-            bundler_version: '2.4.22'
           - ruby: '2.7'
             os: ubuntu-latest
             rdkafka_versions: { min: '>= 0.12.0', max: '= 0.15.2' }
-            bundler_version: '2.4.22'
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} with rdkafka gem version (min ${{ matrix.rdkafka_versions.min }} max ${{ matrix.rdkafka_versions.max }})
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
@@ -54,7 +50,6 @@ jobs:
         RDKAFKA_VERSION_MAX_RANGE: ${{ matrix.rdkafka_versions.max }}
       run: |
         sudo ./ci/prepare-kafka-server.sh
-        gem install bundler -v ${{ matrix.bundler_version }}
         gem install rake
-        bundle _${{ matrix.bundler_version }}_ install --jobs 4 --retry 3
-        bundle _${{ matrix.bundler_version }}_ exec rake test
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

.github/workflows/stale-actions.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/stale@v9
+    - uses: actions/stale@v10
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 90

ChangeLog

@@ -1,3 +1,10 @@
+Release 0.19.5 - 2025/07/11
+	* in_kafka_group: Add `sasl_aws_msk_iam_access_key_id`, `sasl_aws_msk_iam_secret_access_key`
+			and `sasl_aws_msk_iam_aws_region` options (#531)
+
+Release 0.19.4 - 2025/03/24
+	* Support Ruby 3.4. (#526)
+
 Release 0.19.3 - 2024/08/02
 	* out_rdkafka2: Add `unrecoverable_error_codes` parameter to handle
 			specific error code as unrecoverable errors. `topic_authorization_failed`

fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.19.3'
+  gem.version       = '0.19.5'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
@@ -27,6 +27,9 @@ Gem::Specification.new do |gem|
     end
   end
 
+  # gems that aren't default gems as of Ruby 3.4
+  gem.add_dependency("bigdecimal", ["~> 3.1"])
+
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
   gem.add_development_dependency "test-unit-rr", "~> 1.0"

lib/fluent/plugin/in_kafka_group.rb

@@ -74,6 +74,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
 
   include Fluent::KafkaPluginUtil::SSLSettings
   include Fluent::KafkaPluginUtil::SaslSettings
+  include Fluent::KafkaPluginUtil::AwsIamSettings
 
   class ForShutdown < StandardError
   end
@@ -185,6 +186,7 @@ def start
     super
 
     logger = @get_kafka_client_log ? log : nil
+    use_aws_msk_iam = @sasl_aws_msk_iam_access_key_id != nil && @sasl_aws_msk_iam_secret_key_id != nil
     if @scram_mechanism != nil && @username != nil && @password != nil
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
@@ -197,6 +199,10 @@ def start
                          ssl_client_cert_key_password: @ssl_client_cert_key_password,
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password,
                          sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
+    elsif use_aws_msk_iam
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout,
+                         sasl_aws_msk_iam_secret_key_id: @sasl_aws_msk_iam_secret_key_id, sasl_aws_msk_iam_access_key_id: @sasl_aws_msk_iam_access_key_id, sasl_aws_msk_iam_aws_region: @sasl_aws_msk_iam_aws_region,
+                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system)
     else
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),