Skip to content

Commit 6ff3a26

Browse files
joegallojoegallo
authored
Add support for the 'Enterprise' database to the geoip processor (#107377)
1 parent 1d0e3cf commit 6ff3a26

File tree

9 files changed

+311
-4
lines changed

9 files changed

+311
-4
lines changed

‎docs/changelog/107377.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 107377
2+
summary: Add support for the 'Enterprise' database to the geoip processor
3+
area: Ingest Node
4+
type: enhancement
5+
issues: []

‎docs/reference/ingest/processors/geoip.asciidoc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,10 @@ in `properties`.
5959
* If the GeoIP2 Anonymous IP database is used, then the following fields may be added under the `target_field`: `ip`,
6060
`hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`. The fields actually added
6161
depend on what has been found and which properties were configured in `properties`.
62+
* If the GeoIP2 Enterprise database is used, then the following fields may be added under the `target_field`: `ip`,
63+
`country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `location`, `asn`,
64+
`organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`.
65+
The fields actually added depend on what has been found and which properties were configured in `properties`.
6266

6367

6468
Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field:

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,12 +74,44 @@ enum Database {
7474
Property.PUBLIC_PROXY,
7575
Property.RESIDENTIAL_PROXY
7676
)
77+
),
78+
Enterprise(
79+
Set.of(
80+
Property.IP,
81+
Property.COUNTRY_ISO_CODE,
82+
Property.COUNTRY_NAME,
83+
Property.CONTINENT_NAME,
84+
Property.REGION_ISO_CODE,
85+
Property.REGION_NAME,
86+
Property.CITY_NAME,
87+
Property.TIMEZONE,
88+
Property.LOCATION,
89+
Property.ASN,
90+
Property.ORGANIZATION_NAME,
91+
Property.NETWORK,
92+
Property.HOSTING_PROVIDER,
93+
Property.TOR_EXIT_NODE,
94+
Property.ANONYMOUS_VPN,
95+
Property.ANONYMOUS,
96+
Property.PUBLIC_PROXY,
97+
Property.RESIDENTIAL_PROXY
98+
),
99+
Set.of(
100+
Property.COUNTRY_ISO_CODE,
101+
Property.COUNTRY_NAME,
102+
Property.CONTINENT_NAME,
103+
Property.REGION_ISO_CODE,
104+
Property.REGION_NAME,
105+
Property.CITY_NAME,
106+
Property.LOCATION
107+
)
77108
);
78109

79110
private static final String CITY_DB_SUFFIX = "-City";
80111
private static final String COUNTRY_DB_SUFFIX = "-Country";
81112
private static final String ASN_DB_SUFFIX = "-ASN";
82113
private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP";
114+
private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise";
83115

84116
/**
85117
* Parses the passed-in databaseType (presumably from the passed-in databaseFile) and return the Database instance that is
@@ -101,6 +133,8 @@ public static Database getDatabase(final String databaseType, final String datab
101133
database = Database.Asn;
102134
} else if (databaseType.endsWith(Database.ANONYMOUS_IP_DB_SUFFIX)) {
103135
database = Database.AnonymousIp;
136+
} else if (databaseType.endsWith(Database.ENTERPRISE_DB_SUFFIX)) {
137+
database = Database.Enterprise;
104138
}
105139
}
106140

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import com.maxmind.geoip2.model.AsnResponse;
1717
import com.maxmind.geoip2.model.CityResponse;
1818
import com.maxmind.geoip2.model.CountryResponse;
19+
import com.maxmind.geoip2.model.EnterpriseResponse;
1920

2021
import org.apache.logging.log4j.LogManager;
2122
import org.apache.logging.log4j.Logger;
@@ -176,6 +177,12 @@ public AnonymousIpResponse getAnonymousIp(InetAddress ipAddress) {
176177
return getResponse(ipAddress, DatabaseReader::tryAnonymousIp);
177178
}
178179

180+
@Nullable
181+
@Override
182+
public EnterpriseResponse getEnterprise(InetAddress ipAddress) {
183+
return getResponse(ipAddress, DatabaseReader::tryEnterprise);
184+
}
185+
179186
boolean preLookup() {
180187
return currentUsages.updateAndGet(current -> current < 0 ? current : current + 1) > 0;
181188
}

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import com.maxmind.geoip2.model.AsnResponse;
1313
import com.maxmind.geoip2.model.CityResponse;
1414
import com.maxmind.geoip2.model.CountryResponse;
15+
import com.maxmind.geoip2.model.EnterpriseResponse;
1516

1617
import org.elasticsearch.core.Nullable;
1718

@@ -57,6 +58,9 @@ public interface GeoIpDatabase {
5758
@Nullable
5859
AnonymousIpResponse getAnonymousIp(InetAddress ipAddress);
5960

61+
@Nullable
62+
EnterpriseResponse getEnterprise(InetAddress ipAddress);
63+
6064
/**
6165
* Releases the current database object. Called after processing a single document. Databases should be closed or returned to a
6266
* resource pool. No further interactions should be expected.

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import com.maxmind.geoip2.model.AsnResponse;
1414
import com.maxmind.geoip2.model.CityResponse;
1515
import com.maxmind.geoip2.model.CountryResponse;
16+
import com.maxmind.geoip2.model.EnterpriseResponse;
1617
import com.maxmind.geoip2.record.City;
1718
import com.maxmind.geoip2.record.Continent;
1819
import com.maxmind.geoip2.record.Country;
@@ -174,6 +175,7 @@ private Map<String, Object> getGeoData(GeoIpDatabase geoIpDatabase, String ip) t
174175
case Country -> retrieveCountryGeoData(geoIpDatabase, ipAddress);
175176
case Asn -> retrieveAsnGeoData(geoIpDatabase, ipAddress);
176177
case AnonymousIp -> retrieveAnonymousIpGeoData(geoIpDatabase, ipAddress);
178+
case Enterprise -> retrieveEnterpriseGeoData(geoIpDatabase, ipAddress);
177179
};
178180
}
179181

@@ -382,6 +384,127 @@ private Map<String, Object> retrieveAnonymousIpGeoData(GeoIpDatabase geoIpDataba
382384
return geoData;
383385
}
384386

387+
private Map<String, Object> retrieveEnterpriseGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) {
388+
EnterpriseResponse response = geoIpDatabase.getEnterprise(ipAddress);
389+
if (response == null) {
390+
return Map.of();
391+
}
392+
393+
Country country = response.getCountry();
394+
City city = response.getCity();
395+
Location location = response.getLocation();
396+
Continent continent = response.getContinent();
397+
Subdivision subdivision = response.getMostSpecificSubdivision();
398+
399+
Long asn = response.getTraits().getAutonomousSystemNumber();
400+
String organization_name = response.getTraits().getAutonomousSystemOrganization();
401+
Network network = response.getTraits().getNetwork();
402+
403+
boolean isHostingProvider = response.getTraits().isHostingProvider();
404+
boolean isTorExitNode = response.getTraits().isTorExitNode();
405+
boolean isAnonymousVpn = response.getTraits().isAnonymousVpn();
406+
boolean isAnonymous = response.getTraits().isAnonymous();
407+
boolean isPublicProxy = response.getTraits().isPublicProxy();
408+
boolean isResidentialProxy = response.getTraits().isResidentialProxy();
409+
410+
Map<String, Object> geoData = new HashMap<>();
411+
for (Property property : this.properties) {
412+
switch (property) {
413+
case IP -> geoData.put("ip", NetworkAddress.format(ipAddress));
414+
case COUNTRY_ISO_CODE -> {
415+
String countryIsoCode = country.getIsoCode();
416+
if (countryIsoCode != null) {
417+
geoData.put("country_iso_code", countryIsoCode);
418+
}
419+
}
420+
case COUNTRY_NAME -> {
421+
String countryName = country.getName();
422+
if (countryName != null) {
423+
geoData.put("country_name", countryName);
424+
}
425+
}
426+
case CONTINENT_NAME -> {
427+
String continentName = continent.getName();
428+
if (continentName != null) {
429+
geoData.put("continent_name", continentName);
430+
}
431+
}
432+
case REGION_ISO_CODE -> {
433+
// ISO 3166-2 code for country subdivisions.
434+
// See iso.org/iso-3166-country-codes.html
435+
String countryIso = country.getIsoCode();
436+
String subdivisionIso = subdivision.getIsoCode();
437+
if (countryIso != null && subdivisionIso != null) {
438+
String regionIsoCode = countryIso + "-" + subdivisionIso;
439+
geoData.put("region_iso_code", regionIsoCode);
440+
}
441+
}
442+
case REGION_NAME -> {
443+
String subdivisionName = subdivision.getName();
444+
if (subdivisionName != null) {
445+
geoData.put("region_name", subdivisionName);
446+
}
447+
}
448+
case CITY_NAME -> {
449+
String cityName = city.getName();
450+
if (cityName != null) {
451+
geoData.put("city_name", cityName);
452+
}
453+
}
454+
case TIMEZONE -> {
455+
String locationTimeZone = location.getTimeZone();
456+
if (locationTimeZone != null) {
457+
geoData.put("timezone", locationTimeZone);
458+
}
459+
}
460+
case LOCATION -> {
461+
Double latitude = location.getLatitude();
462+
Double longitude = location.getLongitude();
463+
if (latitude != null && longitude != null) {
464+
Map<String, Object> locationObject = new HashMap<>();
465+
locationObject.put("lat", latitude);
466+
locationObject.put("lon", longitude);
467+
geoData.put("location", locationObject);
468+
}
469+
}
470+
case ASN -> {
471+
if (asn != null) {
472+
geoData.put("asn", asn);
473+
}
474+
}
475+
case ORGANIZATION_NAME -> {
476+
if (organization_name != null) {
477+
geoData.put("organization_name", organization_name);
478+
}
479+
}
480+
case NETWORK -> {
481+
if (network != null) {
482+
geoData.put("network", network.toString());
483+
}
484+
}
485+
case HOSTING_PROVIDER -> {
486+
geoData.put("is_hosting_provider", isHostingProvider);
487+
}
488+
case TOR_EXIT_NODE -> {
489+
geoData.put("is_tor_exit_node", isTorExitNode);
490+
}
491+
case ANONYMOUS_VPN -> {
492+
geoData.put("is_anonymous_vpn", isAnonymousVpn);
493+
}
494+
case ANONYMOUS -> {
495+
geoData.put("is_anonymous", isAnonymous);
496+
}
497+
case PUBLIC_PROXY -> {
498+
geoData.put("is_public_proxy", isPublicProxy);
499+
}
500+
case RESIDENTIAL_PROXY -> {
501+
geoData.put("is_residential_proxy", isResidentialProxy);
502+
}
503+
}
504+
}
505+
return geoData;
506+
}
507+
385508
/**
386509
* Retrieves and verifies a {@link GeoIpDatabase} instance for each execution of the {@link GeoIpProcessor}. Guards against missing
387510
* custom databases, and ensures that database instances are of the proper type before use.

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,51 @@ public void testAnonymmousIp() throws Exception {
336336
assertThat(geoData.get("residential_proxy"), equalTo(true));
337337
}
338338

339+
public void testEnterprise() throws Exception {
340+
String ip = "2.125.160.216";
341+
GeoIpProcessor processor = new GeoIpProcessor(
342+
randomAlphaOfLength(10),
343+
null,
344+
"source_field",
345+
loader("/GeoIP2-Enterprise-Test.mmdb"),
346+
() -> true,
347+
"target_field",
348+
ALL_PROPERTIES,
349+
false,
350+
false,
351+
"filename"
352+
);
353+
354+
Map<String, Object> document = new HashMap<>();
355+
document.put("source_field", ip);
356+
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
357+
processor.execute(ingestDocument);
358+
359+
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
360+
@SuppressWarnings("unchecked")
361+
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
362+
assertThat(geoData.size(), equalTo(16));
363+
assertThat(geoData.get("ip"), equalTo(ip));
364+
assertThat(geoData.get("country_iso_code"), equalTo("GB"));
365+
assertThat(geoData.get("country_name"), equalTo("United Kingdom"));
366+
assertThat(geoData.get("continent_name"), equalTo("Europe"));
367+
assertThat(geoData.get("region_iso_code"), equalTo("GB-WBK"));
368+
assertThat(geoData.get("region_name"), equalTo("West Berkshire"));
369+
assertThat(geoData.get("city_name"), equalTo("Boxford"));
370+
assertThat(geoData.get("timezone"), equalTo("Europe/London"));
371+
Map<String, Object> location = new HashMap<>();
372+
location.put("lat", 51.75);
373+
location.put("lon", -1.25);
374+
assertThat(geoData.get("location"), equalTo(location));
375+
assertThat(geoData.get("network"), equalTo("2.125.160.216/29"));
376+
assertThat(geoData.get("is_hosting_provider"), equalTo(false));
377+
assertThat(geoData.get("is_tor_exit_node"), equalTo(false));
378+
assertThat(geoData.get("is_anonymous_vpn"), equalTo(false));
379+
assertThat(geoData.get("is_anonymous"), equalTo(false));
380+
assertThat(geoData.get("is_public_proxy"), equalTo(false));
381+
assertThat(geoData.get("is_residential_proxy"), equalTo(false));
382+
}
383+
339384
public void testAddressIsNotInTheDatabase() throws Exception {
340385
GeoIpProcessor processor = new GeoIpProcessor(
341386
randomAlphaOfLength(10),

0 commit comments

Comments
 (0)