Apply Elasticsearch GeoIP changes and sync with Elasticsearch 8.16. (#170)

* Apply Elasticsearch GeoIP changes and sync with Elasticsearch 8.16.

* Use ES repo if defined, upgrade dependencies and replace shadow plugin.

* Target ES 8.16 branch with 8.latest (8.15.3 now) to make green build with #170 PR.

Author: mashhurs

.buildkite/pull-request-pipeline.yml

@@ -22,6 +22,9 @@ steps:
       ELASTIC_STACK_VERSION: "8.x"
       INTEGRATION: true
       SECURE_INTEGRATION: true
+      # temporary definition to cover PR-170
+      # TODO: remove once 8.16 released
+      ELASTICSEARCH_TREEISH: 8.16
 
   - label: ":hammer: CI setup and integration tests with ES 8.x-SNAPSHOT run on :docker:"
     command:

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.1.16
+  - Reflects the Elasticsearch GeoIP changes into the plugin and syncs with Elasticsearch 8.16 branch [#170](https://github.com/elastic/logstash-filter-elastic_integration/pull/170)
+
 ## 0.1.15
   - Fixes the connection failure where SSL verification mode is disabled over SSL connection [#165](https://github.com/elastic/logstash-filter-elastic_integration/pull/165)
 

VERSION

@@ -1 +1 @@
-0.1.15
+0.1.16

build.gradle

@@ -21,7 +21,7 @@ plugins {
     id "idea"
     id "com.adarshr.test-logger" version "3.2.0"
     id "de.undercouch.download" version "5.3.1"
-    id "com.github.johnrengelman.shadow" version "8.1.1"
+    id "com.gradleup.shadow" version "8.3.2"
 }
 
 // ===========================================================================
@@ -95,10 +95,10 @@ dependencies {
 
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.1.0'
     testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.1.0'
-    testImplementation 'org.mockito:mockito-junit-jupiter:5.1.1'
+    testImplementation 'org.mockito:mockito-junit-jupiter:5.14.1'
     testImplementation 'org.hamcrest:hamcrest-library:2.2'
     testImplementation "com.github.seregamorph:hamcrest-more-matchers:0.1"
-    testImplementation "com.github.tomakehurst:wiremock-jre8:2.35.0"
+    testImplementation 'com.github.tomakehurst:wiremock-jre8:2.35.1'
     testRuntimeOnly requiredLogstashCoreJar("log4j-core")
 
     geolite2('org.elasticsearch:geolite2-databases:20191119') {
@@ -161,7 +161,7 @@ def _requiredLogstashJar(pathPrefix, jarSpec, flavorSpec = null) {
 }
 
 // https://docs.github.com/en/repositories/working-with-files/using-files/downloading-source-code-archives#source-code-archive-urls
-String githubArchivePath(org, repo, treeish="main", archiveFormat="zip") {
+String githubArchivePath(repo, treeish="main", archiveFormat="zip") {
     def pathFragment = {
         switch(treeish) {
             case ~'^refs/(?:heads|tags)/'       : return "${treeish}" // unambiguous ref
@@ -171,7 +171,7 @@ String githubArchivePath(org, repo, treeish="main", archiveFormat="zip") {
         }
     }()
 
-    "https://github.com/${org}/${repo}/archive/${pathFragment}.${archiveFormat}"
+    "https://github.com/${repo}/archive/${pathFragment}.${archiveFormat}"
 }
 
 task downloadElasticsearchSourceZip(type: Download) {
@@ -186,7 +186,10 @@ task downloadElasticsearchSourceZip(type: Download) {
         """)
     }
 
-    src githubArchivePath("elastic", "elasticsearch", esTreeish, "zip")
+    def esRepo = envOrPropertyValue("ELASTICSEARCH_REPO") ?: "elastic/elasticsearch"
+    inputs.property("ELASTICSEARCH_REPO", esRepo)
+
+    src githubArchivePath(esRepo, esTreeish, "zip")
     dest "${buildDir}/elasticsearch-source.zip"
     onlyIfModified true
     useETag "all"
@@ -383,6 +386,8 @@ test {
     useJUnitPlatform()
     testLogging {
         exceptionFormat "full"
+        outputs.upToDateWhen { false }
+        showStandardStreams = true
     }
 }
 

gradle.properties

@@ -1,2 +1,2 @@
 LOGSTASH_PATH=../../ls
-ELASTICSEARCH_TREEISH=8.15
+ELASTICSEARCH_TREEISH=8.16

lib/logstash/filters/elastic_integration/geoip_database_provider_bridge.rb

@@ -20,8 +20,8 @@ module LogStash::Filters::ElasticIntegration::GeoipDatabaseProviderBridge
              "in a Logstash with Geoip Database Management enabled."
 
   def initialize_geoip_database_provider!
-    java_import('co.elastic.logstash.filters.elasticintegration.geoip.GeoIpDatabaseProvider')
-    @geoip_database_provider ||= GeoIpDatabaseProvider::Builder.new.tap do |builder|
+    java_import('co.elastic.logstash.filters.elasticintegration.geoip.IpDatabaseProvider')
+    @geoip_database_provider ||= IpDatabaseProvider::Builder.new.tap do |builder|
       if geoip_database_directory
         logger.debug("discovering geoip databases from #{geoip_database_directory}")
         builder.discoverDatabases(java.io.File.new(geoip_database_directory))
@@ -53,8 +53,8 @@ def load_geoip_database_manager!
     :UNAVAILABLE
   end
 
-  java_import('co.elastic.logstash.filters.elasticintegration.geoip.ManagedGeoipDatabaseHolder')
-  class ObservingDatabaseHolder < ManagedGeoipDatabaseHolder
+  java_import('co.elastic.logstash.filters.elasticintegration.geoip.ManagedIpDatabaseHolder')
+  class ObservingDatabaseHolder < ManagedIpDatabaseHolder
     def initialize(simple_database_type, eula_manager:, logger: nil)
       super("GeoLite2-#{simple_database_type}")
 

src/main/java/co/elastic/logstash/filters/elasticintegration/EventProcessorBuilder.java

@@ -316,7 +316,7 @@ public synchronized EventProcessor build(final PluginContext pluginContext) {
     private static ScriptService initScriptService(final Settings settings, final ThreadPool threadPool) throws IOException {
         Map<String, ScriptEngine> engines = new HashMap<>();
         engines.put(PainlessScriptEngine.NAME, getPainlessScriptEngine(settings));
-        engines.put(MustacheScriptEngine.NAME, new MustacheScriptEngine());
+        engines.put(MustacheScriptEngine.NAME, new MustacheScriptEngine(settings));
         return new ScriptService(settings, engines, ScriptModule.CORE_CONTEXTS, threadPool::absoluteTimeInMillis);
     }
 

src/main/java/co/elastic/logstash/filters/elasticintegration/geoip/ConstantGeoipDatabaseHolder.java

@@ -0,0 +1,40 @@
+package co.elastic.logstash.filters.elasticintegration.geoip;
+
+import org.elasticsearch.ingest.geoip.IpDatabase;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Objects;
+
+public class ConstantIpDatabaseHolder implements IpDatabaseHolder, Closeable {
+    private final IpDatabaseAdapter ipDatabase;
+
+    public ConstantIpDatabaseHolder(final IpDatabaseAdapter ipDatabase) {
+        this.ipDatabase = Objects.requireNonNull(ipDatabase);
+    }
+
+    @Override
+    public boolean isValid() {
+        return true;
+    }
+
+    @Override
+    public IpDatabase getDatabase() {
+        return this.ipDatabase;
+    }
+
+    @Override
+    public String getTypeIdentifier() {
+        return this.ipDatabase.getDatabaseType();
+    }
+
+    @Override
+    public String info() {
+        return String.format("ConstantIpDatabaseHolder{type=%s}", getTypeIdentifier());
+    }
+
+    @Override
+    public void close() throws IOException {
+        this.ipDatabase.closeReader();
+    }
+}