diff --git a/docs/changelog/137479.yaml b/docs/changelog/137479.yaml new file mode 100644 index 0000000000000..cb319d7c53572 --- /dev/null +++ b/docs/changelog/137479.yaml @@ -0,0 +1,6 @@ +pr: 137479 +summary: "Fix illegal_access_exception: class com.maxmind.db.Decoder from `ip_location`\ + \ processor" +area: Ingest Node +type: bug +issues: [] diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index 032ae3813d35b..dec96ac2ab79b 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -99,4 +99,6 @@ artifacts { tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("ingest_geoip/40_geoip_databases/Test adding, getting, and removing geoip databases", "get databases behavior began returning more results in 8.16") + task.skipTest("ingest_geoip/60_ip_location_databases/Test adding, getting, and removing ip location databases", + "get databases behavior began returning more results in 9.2.1") }) diff --git a/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java b/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java index 201de45dc4efd..f51eb8105f762 100644 --- a/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java +++ b/modules/ingest-geoip/qa/multi-project/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientMultiProjectYamlTestSuiteIT.java @@ -21,12 +21,20 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; import org.junit.rules.TestRule; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; + import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.assertDatabasesLoaded; +import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.getRootPath; import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.putGeoipPipeline; +import static org.hamcrest.Matchers.is; @FixForMultiProject(description = "Potentially remove this test after https://elasticco.atlassian.net/browse/ES-12094 is implemented") public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjectsClientYamlSuiteTestCase { @@ -35,7 +43,10 @@ public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjec private static GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture); + public static TemporaryFolder configDir = new TemporaryFolder(); + private static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .withConfigDir(() -> getRootPath(configDir)) .module("reindex") .module("ingest-geoip") .systemProperty("ingest.geoip.downloader.enabled.default", "true") @@ -51,7 +62,7 @@ public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjec .build(); @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster); + public static TestRule ruleChain = RuleChain.outerRule(fixture).around(configDir).around(cluster); @Override protected String getTestRestCluster() { @@ -67,6 +78,19 @@ public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + @BeforeClass + public static void copyExtraDatabase() throws Exception { + Path configPath = getRootPath(configDir); + assertThat(Files.exists(configPath), is(true)); + Path ingestGeoipDatabaseDir = configPath.resolve("ingest-geoip"); + Files.createDirectory(ingestGeoipDatabaseDir); + final var clazz = IngestGeoIpClientYamlTestSuiteIT.class; // long line prevention + Files.copy( + Objects.requireNonNull(clazz.getResourceAsStream("/ipinfo/asn_sample.mmdb")), + ingestGeoipDatabaseDir.resolve("asn.mmdb") + ); + } + @Before public void waitForDatabases() throws Exception { putGeoipPipeline("pipeline-with-geoip"); diff --git a/modules/ingest-geoip/src/main/java/module-info.java b/modules/ingest-geoip/src/main/java/module-info.java index afa01bc75fe10..4e274c1904793 100644 --- a/modules/ingest-geoip/src/main/java/module-info.java +++ b/modules/ingest-geoip/src/main/java/module-info.java @@ -19,5 +19,5 @@ exports org.elasticsearch.ingest.geoip.direct to org.elasticsearch.server; exports org.elasticsearch.ingest.geoip.stats to org.elasticsearch.server; - exports org.elasticsearch.ingest.geoip to org.elasticsearch.logstashbridge; + exports org.elasticsearch.ingest.geoip to org.elasticsearch.logstashbridge, com.maxmind.db; } diff --git a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java index 48718ce02f3d5..69bc58559bede 100644 --- a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java +++ b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java @@ -19,22 +19,29 @@ import org.elasticsearch.client.Request; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; import org.junit.rules.TestRule; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Objects; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { @@ -43,7 +50,10 @@ public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase private static GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture); + public static TemporaryFolder configDir = new TemporaryFolder(); + private static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .withConfigDir(() -> getRootPath(configDir)) .module("reindex") .module("ingest-geoip") .systemProperty("ingest.geoip.downloader.enabled.default", "true") @@ -56,7 +66,7 @@ public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase .build(); @ClassRule - public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster); + public static TestRule ruleChain = RuleChain.outerRule(fixture).around(configDir).around(cluster); @Override protected String getTestRestCluster() { @@ -72,6 +82,19 @@ public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + @BeforeClass + public static void copyExtraDatabase() throws Exception { + Path configPath = getRootPath(configDir); + assertThat(Files.exists(configPath), is(true)); + Path ingestGeoipDatabaseDir = configPath.resolve("ingest-geoip"); + Files.createDirectory(ingestGeoipDatabaseDir); + final var clazz = IngestGeoIpClientYamlTestSuiteIT.class; // long line prevention + Files.copy( + Objects.requireNonNull(clazz.getResourceAsStream("/ipinfo/asn_sample.mmdb")), + ingestGeoipDatabaseDir.resolve("asn.mmdb") + ); + } + @Before public void waitForDatabases() throws Exception { putGeoipPipeline("pipeline-with-geoip"); @@ -123,6 +146,8 @@ static void assertDatabasesLoaded() throws Exception { Map nodes = (Map) response.get("nodes"); assertThat(nodes.size(), equalTo(1)); Map node = (Map) nodes.values().iterator().next(); + + // confirm the downloaded databases are all correct List databases = ((List) node.get("databases")); assertThat(databases, notNullValue()); List databaseNames = databases.stream().map(o -> (String) ((Map) o).get("name")).toList(); @@ -130,6 +155,14 @@ static void assertDatabasesLoaded() throws Exception { databaseNames, containsInAnyOrder("GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "GeoLite2-ASN.mmdb", "MyCustomGeoLite2-City.mmdb") ); + + // ensure that the extra config database has been set up, too: + assertThat(node.get("config_databases"), equalTo(List.of("asn.mmdb"))); }); } + + @SuppressForbidden(reason = "fixtures use java.io.File based APIs") + public static Path getRootPath(TemporaryFolder folder) { + return folder.getRoot().toPath(); + } } diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/ipinfo/asn_sample.mmdb b/modules/ingest-geoip/src/yamlRestTest/resources/ipinfo/asn_sample.mmdb new file mode 100644 index 0000000000000..6f93d59731f1b Binary files /dev/null and b/modules/ingest-geoip/src/yamlRestTest/resources/ipinfo/asn_sample.mmdb differ diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml index 007c82db4c923..d31001aed8e0e 100644 --- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml @@ -69,7 +69,7 @@ teardown: - do: ingest.get_geoip_database: {} - - length: { databases: 6 } + - length: { databases: 7 } - do: ingest.get_geoip_database: @@ -95,7 +95,7 @@ teardown: - do: ingest.get_geoip_database: {} - - length: { databases: 5 } + - length: { databases: 6 } - do: ingest.get_geoip_database: diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml index 0947984769529..c6e140a914616 100644 --- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml @@ -37,3 +37,40 @@ - match: { _source.ip_location.country_name: "Sweden" } - match: { _source.ip_location.region_name: "Östergötland County" } - match: { _source.ip_location.continent_name: "Europe" } + +--- +"Test ip_location processor with a config directory database": + - do: + ingest.put_pipeline: + id: "my_pipeline_2" + body: > + { + "description": "_description", + "processors": [ + { + "ip_location" : { + "field" : "field1", + "database_file" : "asn.mmdb" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: "1" + pipeline: "my_pipeline_2" + body: {field1: "5.104.168.0"} + + - do: + get: + index: test + id: "1" + - match: { _source.field1: "5.104.168.0" } + - length: { _source.ip_location: 4 } + - match: { _source.ip_location.organization_name: "Telehouse EAD" } + - match: { _source.ip_location.asn: 57344 } + - match: { _source.ip_location.ip: "5.104.168.0" } + - match: { _source.ip_location.network: "5.104.168.0/23" } diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml index 47f09392df60e..3d043532bc900 100644 --- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml @@ -84,7 +84,7 @@ teardown: - do: ingest.get_ip_location_database: {} - - length: { databases: 7 } + - length: { databases: 8 } - do: ingest.get_ip_location_database: @@ -110,7 +110,7 @@ teardown: - do: ingest.get_ip_location_database: {} - - length: { databases: 6 } + - length: { databases: 7 } - do: ingest.get_ip_location_database: