Bump to KCL v2.7.2

.dockerignore

@@ -0,0 +1,39 @@
+# Git files
+.git
+.gitignore
+
+# Development
+.bundle
+vendor/bundle
+vendor/jar-dependencies
+.yardoc
+/_yardoc/
+/doc/
+/tmp/
+spec/reports/
+.envrc
+Dockerfile
+Makefile
+
+# Builds
+*.gem
+*.bundle
+*.so
+*.o
+*.a
+pkg/
+
+# IDE
+.vscode
+.idea
+*.swp
+*.swo
+*~
+
+# CI
+.travis.yml
+
+# Documentation
+docs/
+CHANGELOG.md
+CONTRIBUTORS

.envrc

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+source_up_if_exists
+
+# Development environment for logstash-input-kinesis plugin
+
+# Use JRuby for this plugin (required for Java interop)
+# Check if jruby is available via mise/asdf/rbenv/rvm
+if command -v rbenv &> /dev/null; then
+  # Use rbenv for version management
+  eval "$(rbenv init -)"
+  rbenv local jruby-10.0.2.0
+elif command -v mise &> /dev/null; then
+  # Use mise for version management
+  use jruby 9.4.5.0
+elif command -v asdf &> /dev/null; then
+  # Use asdf for version management  
+  use asdf
+fi
+
+# Logstash development settings
+export LOGSTASH_PATH="${LOGSTASH_PATH:-/usr/share/logstash}"
+export LOGSTASH_SOURCE="${LOGSTASH_SOURCE:-1}"
+
+# AWS credentials for Kinesis testing (configure as needed)
+# Uncomment and set these if you need to test against real AWS services
+# export AWS_ACCESS_KEY_ID="your-access-key-id"
+# export AWS_SECRET_ACCESS_KEY="your-secret-access-key"
+# export AWS_REGION="us-east-1"
+
+# AWS profile (if using ~/.aws/credentials)
+# export AWS_PROFILE="default"
+
+# Add vendor jars to classpath when running
+export JARS_HOME="$(pwd)/vendor/jar-dependencies"
+
+# Ruby optimization flags
+export JRUBY_OPTS="-J-Xmx2g"
+
+export JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java))))
+
+# Path additions
+PATH_add bin
+PATH_add vendor/jruby/*/bin
+
+echo "logstash-input-kinesis development environment loaded"
+echo "Ruby: $(ruby -v 2>/dev/null || echo 'not found')"
+echo "Bundler: $(bundle -v 2>/dev/null || echo 'not installed - run: gem install bundler')"
+echo ""
+make help

.ruby-version

@@ -0,0 +1 @@
+jruby-10.0.2.0

CHANGELOG.md

@@ -1,3 +1,12 @@
+## 3.0.0
+  - **Breaking**: Upgraded to AWS Kinesis Client Library (KCL) v2.7.2 from v1.15.0
+    - Requires AWS SDK v2 for Kinesis operations
+    - Enables custom endpoint configuration for LocalStack and testing environments
+  - Added support for `kinesis_endpoint`, `dynamodb_endpoint`, and `cloudwatch_endpoint` configuration options
+  - Fixed integration test script to properly detect message processing
+  - Improved error handling in integration tests to distinguish plugin errors from LocalStack compatibility issues
+  - Integration tests now pass successfully with LocalStack
+
 ## 2.3.0
   - Updated Kinesis client to `1.15.0`, now requires Logstash `>=8.9.0` [#102](https://github.com/logstash-plugins/logstash-input-kinesis/pull/102)
 

Dockerfile

@@ -0,0 +1,31 @@
+FROM docker.elastic.co/logstash/logstash:9.0.1 AS builder-kinesis
+
+USER root
+
+# Install build dependencies
+RUN microdnf install -y make && microdnf clean all
+
+# Set up environment to use Logstash's bundled tools
+ENV JAVA_HOME=/usr/share/logstash/jdk
+ENV PATH=/usr/share/logstash/vendor/jruby/bin:$JAVA_HOME/bin:$PATH
+ENV LOGSTASH_SOURCE=1
+ENV LOGSTASH_PATH=/usr/share/logstash
+
+# Set working directory
+WORKDIR /build
+
+# Copy the gem source code
+COPY . .
+
+# Build gem using Logstash's JRuby
+# Always download JARs in Docker to ensure consistent builds regardless of local state
+RUN gem install bundler && \
+    bundle install --with development && \
+    bundle exec rake install_jars && \
+    gem build logstash-input-kinesis.gemspec
+
+# Output the built gem to a mounted volume or final location
+
+FROM docker.elastic.co/logstash/logstash:9.0.1
+COPY --from=builder-kinesis /build/logstash-input-kinesis-*.gem /tmp/
+RUN /opt/logstash/bin/logstash-plugin install /tmp/logstash-input-kinesis-*.gem

Makefile

@@ -0,0 +1,55 @@
+.PHONY: help test clean dist-clean setup all integration gem docker
+
+help: ## Show this help message
+	@echo 'Usage: make [target]'
+	@echo ''
+	@echo 'Available targets:'
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2}'
+
+vendor/bundle/.timestamp: Gemfile
+	@echo "Installing Ruby dependencies..."
+	bundle install --with development
+	@mkdir -p vendor/bundle
+	@touch vendor/bundle/.timestamp
+
+install: vendor/bundle/.timestamp ## Install Ruby dependencies (including development gems)
+
+vendor/jar-dependencies/.timestamp: vendor/bundle/.timestamp logstash-input-kinesis.gemspec
+	@echo "Downloading Java JAR dependencies..."
+	bundle exec rake install_jars
+	@touch vendor/jar-dependencies/.timestamp
+
+install-jars: vendor/jar-dependencies/.timestamp ## Download Java JAR dependencies
+
+test: install-jars ## Run unit tests
+	@echo "Running tests..."
+	bundle exec rspec
+
+gem: install-jars ## Build gem package
+	@echo "Building gem package..."
+	gem build logstash-input-kinesis.gemspec
+
+docker: ## Build Docker image with the plugin installed
+	@echo "Building Docker image..."
+	docker build -t logstash-input-kinesis .
+
+integration: ## Run integration tests with docker-compose, localstack, and http mock
+	@./integration-test/run-test.sh
+
+clean: ## Clean vendor directories and installed dependencies
+	@echo "Cleaning vendor directories..."
+	@rm -rf vendor/bundle vendor/jar-dependencies
+	@rm -f Gemfile.lock vendor/bundle/.timestamp vendor/jar-dependencies/.timestamp
+
+dist-clean: clean ## Clean all generated files including vendor directory
+	@echo "Performing distribution clean..."
+	@rm -rf vendor
+
+setup: install-jars ## Full setup for local development (install + install-jars)
+	@echo "Setup complete! Ready for development."
+	@echo ""
+	@echo "Next steps:"
+	@echo "  - Run 'make test' to run tests"
+	@echo "  - See 'make help' for more commands"
+
+all: setup ## Alias for setup

README.md

@@ -96,4 +96,22 @@ To download all jars:
 `bundler exec rake install_jars`
 
 To run all specs: 
-`bundler exec rspec`
+`bundler exec rspec`
+
+### Integration Tests
+
+Integration tests are available that validate the plugin build, installation, and initialization process:
+
+```sh
+make integration
+```
+
+This will:
+- Build the gem package
+- Install the plugin in a Logstash container
+- Verify the plugin registers and initializes correctly
+- Test with LocalStack infrastructure
+
+**Note**: The plugin now uses Kinesis Client Library v2.7.2, which supports custom endpoint configuration for LocalStack integration. The test validates critical integration points (build, install, register, initialize) which catches most common issues.
+
+For full end-to-end testing with LocalStack or real AWS credentials with actual Kinesis streams, see [integration-test/README.md](integration-test/README.md).

Rakefile

@@ -12,9 +12,38 @@ task default: "spec"
 require 'jars/installer'
 desc 'Install the JAR dependencies to vendor/'
 task :install_jars do
+  # Monkey-patch jar-dependencies to strip ANSI escape codes and module info from file paths
+  # This is needed because Java's module system adds colored output and module info that breaks path parsing
+  # Example problematic output: "/path/to/file.jar[36m -- module foo.bar[0;1;33m (auto)[m"
+  module Jars
+    class Installer
+      class Dependency
+        alias_method :original_initialize, :initialize
+        def initialize(line)
+          # Strip ANSI escape codes before parsing
+          line = line.gsub(/\e\[[0-9;]*m/, '')
+          # Strip Java module system info (e.g., " -- module name", " -- module name (auto)", " -- module name [auto]")
+          line = line.gsub(/ -- module \S+( [\[\(][^\]\)]+[\]\)])?/, '')
+          original_initialize(line)
+        end
+      end
+    end
+  end
+
   # We actually want jar-dependencies will download the jars and place it in
   # vendor/jar-dependencies/runtime-jars
-  Jars::Installer.new.vendor_jars!(false, 'vendor/jar-dependencies/runtime-jars')
+  # Use positional arguments for compatibility with jar-dependencies 0.4.x and 0.5.x
+  # 0.4.x API: vendor_jars!(write_require_file, vendor_dir)
+  # 0.5.x API: vendor_jars!(vendor_dir, write_require_file:)
+  installer = Jars::Installer.new
+  method = installer.method(:vendor_jars!)
+  if method.parameters.any? { |type, name| type == :key || type == :keyreq }
+    # 0.5.x API with keyword arguments
+    installer.vendor_jars!('vendor/jar-dependencies/runtime-jars', write_require_file: false)
+  else
+    # 0.4.x API with positional arguments
+    installer.vendor_jars!(false, 'vendor/jar-dependencies/runtime-jars')
+  end
 end
 
 task build: :install_jars

VERSION

@@ -1 +1 @@
-2.3.0
+3.0.0

docker-compose.yml

@@ -0,0 +1,62 @@
+services:
+  localstack:
+    image: localstack/localstack:latest
+    container_name: logstash-kinesis-localstack
+    environment:
+      - SERVICES=kinesis,dynamodb,cloudwatch
+      - DEBUG=1
+      - AWS_ACCESS_KEY_ID=test
+      - AWS_SECRET_ACCESS_KEY=test
+      - AWS_DEFAULT_REGION=us-east-1
+    ports:
+      - "4566:4566"
+    volumes:
+      - ./integration-test/localstack:/etc/localstack/init/ready.d
+      - localstack-data:/var/lib/localstack
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:4566/_localstack/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  http-mock:
+    image: mockserver/mockserver:latest
+    container_name: logstash-kinesis-http-mock
+    environment:
+      - MOCKSERVER_PROPERTY_FILE=/config/mockserver.properties
+      - MOCKSERVER_INITIALIZATION_JSON_PATH=/config/expectations.json
+    ports:
+      - "1080:1080"
+    volumes:
+      - ./integration-test/http-mock:/config
+
+  logstash:
+    build:
+      context: .
+    container_name: logstash-kinesis-test
+    environment:
+      - XPACK_MONITORING_ENABLED=false
+      - AWS_ACCESS_KEY_ID=test
+      - AWS_SECRET_ACCESS_KEY=test
+      - AWS_REGION=us-east-1
+      - AWS_CBOR_DISABLE=true
+      # Override AWS SDK endpoints to use LocalStack
+      - JAVA_OPTS=-Dcom.amazonaws.sdk.disableCertChecking=true
+    ports:
+      - "9600:9600"
+    volumes:
+      - .:/plugin-src:ro
+      - ./integration-test/logstash/pipeline:/usr/share/logstash/pipeline:ro
+      - ./integration-test/logstash/config:/usr/share/logstash/config
+    command: >
+      bash -c "
+        logstash -f /usr/share/logstash/pipeline/kinesis-test.conf --config.reload.automatic
+      "
+    depends_on:
+      localstack:
+        condition: service_healthy
+      http-mock:
+        condition: service_started
+
+volumes:
+  localstack-data: