Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/changelog/137479.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 137479
summary: "Fix illegal_access_exception: class com.maxmind.db.Decoder from `ip_location`\
\ processor"
area: Ingest Node
type: bug
issues: []
2 changes: 2 additions & 0 deletions modules/ingest-geoip/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -99,4 +99,6 @@ artifacts {
tasks.named("yamlRestCompatTestTransform").configure({ task ->
task.skipTest("ingest_geoip/40_geoip_databases/Test adding, getting, and removing geoip databases",
"get databases behavior began returning more results in 8.16")
task.skipTest("ingest_geoip/60_ip_location_databases/Test adding, getting, and removing ip location databases",
"get databases behavior began returning more results in 9.2.1")
})
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,20 @@
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.rules.RuleChain;
import org.junit.rules.TemporaryFolder;
import org.junit.rules.TestRule;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Objects;

import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.assertDatabasesLoaded;
import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.getRootPath;
import static org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT.putGeoipPipeline;
import static org.hamcrest.Matchers.is;

@FixForMultiProject(description = "Potentially remove this test after https://elasticco.atlassian.net/browse/ES-12094 is implemented")
public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjectsClientYamlSuiteTestCase {
Expand All @@ -35,7 +43,10 @@ public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjec

private static GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture);

public static TemporaryFolder configDir = new TemporaryFolder();

private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
.withConfigDir(() -> getRootPath(configDir))
.module("reindex")
.module("ingest-geoip")
.systemProperty("ingest.geoip.downloader.enabled.default", "true")
Expand All @@ -51,7 +62,7 @@ public class IngestGeoIpClientMultiProjectYamlTestSuiteIT extends MultipleProjec
.build();

@ClassRule
public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
public static TestRule ruleChain = RuleChain.outerRule(fixture).around(configDir).around(cluster);

@Override
protected String getTestRestCluster() {
Expand All @@ -67,6 +78,19 @@ public static Iterable<Object[]> parameters() throws Exception {
return ESClientYamlSuiteTestCase.createParameters();
}

@BeforeClass
public static void copyExtraDatabase() throws Exception {
Path configPath = getRootPath(configDir);
assertThat(Files.exists(configPath), is(true));
Path ingestGeoipDatabaseDir = configPath.resolve("ingest-geoip");
Files.createDirectory(ingestGeoipDatabaseDir);
final var clazz = IngestGeoIpClientYamlTestSuiteIT.class; // long line prevention
Files.copy(
Objects.requireNonNull(clazz.getResourceAsStream("/ipinfo/asn_sample.mmdb")),
ingestGeoipDatabaseDir.resolve("asn.mmdb")
);
}

@Before
public void waitForDatabases() throws Exception {
putGeoipPipeline("pipeline-with-geoip");
Expand Down
2 changes: 1 addition & 1 deletion modules/ingest-geoip/src/main/java/module-info.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,5 @@
exports org.elasticsearch.ingest.geoip.direct to org.elasticsearch.server;
exports org.elasticsearch.ingest.geoip.stats to org.elasticsearch.server;

exports org.elasticsearch.ingest.geoip to org.elasticsearch.logstashbridge;
exports org.elasticsearch.ingest.geoip to org.elasticsearch.logstashbridge, com.maxmind.db;
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,29 @@
import org.elasticsearch.client.Request;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.core.Booleans;
import org.elasticsearch.core.SuppressForbidden;
import org.elasticsearch.test.cluster.ElasticsearchCluster;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.json.JsonXContent;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.rules.RuleChain;
import org.junit.rules.TemporaryFolder;
import org.junit.rules.TestRule;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.Objects;

import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;

public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
Expand All @@ -43,7 +50,10 @@ public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase

private static GeoIpHttpFixture fixture = new GeoIpHttpFixture(useFixture);

public static TemporaryFolder configDir = new TemporaryFolder();

private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
.withConfigDir(() -> getRootPath(configDir))
.module("reindex")
.module("ingest-geoip")
.systemProperty("ingest.geoip.downloader.enabled.default", "true")
Expand All @@ -56,7 +66,7 @@ public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
.build();

@ClassRule
public static TestRule ruleChain = RuleChain.outerRule(fixture).around(cluster);
public static TestRule ruleChain = RuleChain.outerRule(fixture).around(configDir).around(cluster);

@Override
protected String getTestRestCluster() {
Expand All @@ -72,6 +82,19 @@ public static Iterable<Object[]> parameters() throws Exception {
return ESClientYamlSuiteTestCase.createParameters();
}

@BeforeClass
public static void copyExtraDatabase() throws Exception {
Path configPath = getRootPath(configDir);
assertThat(Files.exists(configPath), is(true));
Path ingestGeoipDatabaseDir = configPath.resolve("ingest-geoip");
Files.createDirectory(ingestGeoipDatabaseDir);
final var clazz = IngestGeoIpClientYamlTestSuiteIT.class; // long line prevention
Files.copy(
Objects.requireNonNull(clazz.getResourceAsStream("/ipinfo/asn_sample.mmdb")),
ingestGeoipDatabaseDir.resolve("asn.mmdb")
);
}

@Before
public void waitForDatabases() throws Exception {
putGeoipPipeline("pipeline-with-geoip");
Expand Down Expand Up @@ -123,13 +146,23 @@ static void assertDatabasesLoaded() throws Exception {
Map<?, ?> nodes = (Map<?, ?>) response.get("nodes");
assertThat(nodes.size(), equalTo(1));
Map<?, ?> node = (Map<?, ?>) nodes.values().iterator().next();

// confirm the downloaded databases are all correct
List<?> databases = ((List<?>) node.get("databases"));
assertThat(databases, notNullValue());
List<String> databaseNames = databases.stream().map(o -> (String) ((Map<?, ?>) o).get("name")).toList();
assertThat(
databaseNames,
containsInAnyOrder("GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "GeoLite2-ASN.mmdb", "MyCustomGeoLite2-City.mmdb")
);

// ensure that the extra config database has been set up, too:
assertThat(node.get("config_databases"), equalTo(List.of("asn.mmdb")));
});
}

@SuppressForbidden(reason = "fixtures use java.io.File based APIs")
public static Path getRootPath(TemporaryFolder folder) {
return folder.getRoot().toPath();
}
}
Binary file not shown.
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ teardown:

- do:
ingest.get_geoip_database: {}
- length: { databases: 6 }
- length: { databases: 7 }

- do:
ingest.get_geoip_database:
Expand All @@ -95,7 +95,7 @@ teardown:

- do:
ingest.get_geoip_database: {}
- length: { databases: 5 }
- length: { databases: 6 }

- do:
ingest.get_geoip_database:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,40 @@
- match: { _source.ip_location.country_name: "Sweden" }
- match: { _source.ip_location.region_name: "Östergötland County" }
- match: { _source.ip_location.continent_name: "Europe" }

---
"Test ip_location processor with a config directory database":
- do:
ingest.put_pipeline:
id: "my_pipeline_2"
body: >
{
"description": "_description",
"processors": [
{
"ip_location" : {
"field" : "field1",
"database_file" : "asn.mmdb"
}
}
]
}
- match: { acknowledged: true }

- do:
index:
index: test
id: "1"
pipeline: "my_pipeline_2"
body: {field1: "5.104.168.0"}

- do:
get:
index: test
id: "1"
- match: { _source.field1: "5.104.168.0" }
- length: { _source.ip_location: 4 }
- match: { _source.ip_location.organization_name: "Telehouse EAD" }
- match: { _source.ip_location.asn: 57344 }
- match: { _source.ip_location.ip: "5.104.168.0" }
- match: { _source.ip_location.network: "5.104.168.0/23" }
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ teardown:

- do:
ingest.get_ip_location_database: {}
- length: { databases: 7 }
- length: { databases: 8 }

- do:
ingest.get_ip_location_database:
Expand All @@ -110,7 +110,7 @@ teardown:

- do:
ingest.get_ip_location_database: {}
- length: { databases: 6 }
- length: { databases: 7 }

- do:
ingest.get_ip_location_database:
Expand Down