Skip to content

Commit 50c02f4

Browse files
joegallojoegallo
authored
Support IPinfo databases in the ip_location processor (#114735)
1 parent 1b321dd commit 50c02f4

File tree

8 files changed

+383
-97
lines changed

8 files changed

+383
-97
lines changed

‎modules/ingest-geoip/src/main/java/module-info.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,6 @@
1818

1919
exports org.elasticsearch.ingest.geoip.direct to org.elasticsearch.server;
2020
exports org.elasticsearch.ingest.geoip.stats to org.elasticsearch.server;
21+
22+
exports org.elasticsearch.ingest.geoip to com.maxmind.db;
2123
}

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,14 +73,14 @@ void updateDatabase(Path file, boolean update) {
7373
String databaseFileName = file.getFileName().toString();
7474
try {
7575
if (update) {
76-
logger.info("database file changed [{}], reload database...", file);
76+
logger.info("database file changed [{}], reloading database...", file);
7777
DatabaseReaderLazyLoader loader = new DatabaseReaderLazyLoader(cache, file, null);
7878
DatabaseReaderLazyLoader existing = configDatabases.put(databaseFileName, loader);
7979
if (existing != null) {
8080
existing.shutdown();
8181
}
8282
} else {
83-
logger.info("database file removed [{}], close database...", file);
83+
logger.info("database file removed [{}], closing database...", file);
8484
DatabaseReaderLazyLoader existing = configDatabases.remove(databaseFileName);
8585
assert existing != null;
8686
existing.shutdown();

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -196,13 +196,19 @@ public IpDatabase get() throws IOException {
196196
}
197197

198198
if (Assertions.ENABLED) {
199-
// Only check whether the suffix has changed and not the entire database type.
200-
// To sanity check whether a city db isn't overwriting with a country or asn db.
201-
// For example overwriting a geoip lite city db with geoip city db is a valid change, but the db type is slightly different,
202-
// by checking just the suffix this assertion doesn't fail.
203-
String expectedSuffix = databaseType.substring(databaseType.lastIndexOf('-'));
204-
assert loader.getDatabaseType().endsWith(expectedSuffix)
205-
: "database type [" + loader.getDatabaseType() + "] doesn't match with expected suffix [" + expectedSuffix + "]";
199+
// Note that the expected suffix might be null for providers that aren't amenable to using dashes as separator for
200+
// determining the database type.
201+
int last = databaseType.lastIndexOf('-');
202+
final String expectedSuffix = last == -1 ? null : databaseType.substring(last);
203+
204+
// If the entire database type matches, then that's a match. Otherwise, if there's a suffix to compare on, then
205+
// check whether the suffix has changed (not the entire database type).
206+
// This is to sanity check, for example, that a city db isn't overwritten with a country or asn db.
207+
// But there are permissible overwrites that make sense, for example overwriting a geolite city db with a geoip city db
208+
// is a valid change, but the db type is slightly different -- by checking just the suffix this assertion won't fail.
209+
final String loaderType = loader.getDatabaseType();
210+
assert loaderType.equals(databaseType) || expectedSuffix == null || loaderType.endsWith(expectedSuffix)
211+
: "database type [" + loaderType + "] doesn't match with expected suffix [" + expectedSuffix + "]";
206212
}
207213
return loader;
208214
}

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java‎

Lines changed: 23 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,16 @@
1313
import org.elasticsearch.core.Nullable;
1414

1515
import java.util.List;
16+
import java.util.Locale;
1617
import java.util.Set;
1718
import java.util.function.Function;
1819

20+
import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.IPINFO_PREFIX;
21+
import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.getIpinfoDatabase;
22+
import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.getIpinfoLookup;
23+
import static org.elasticsearch.ingest.geoip.MaxmindIpDataLookups.getMaxmindDatabase;
24+
import static org.elasticsearch.ingest.geoip.MaxmindIpDataLookups.getMaxmindLookup;
25+
1926
final class IpDataLookupFactories {
2027

2128
private IpDataLookupFactories() {
@@ -26,78 +33,44 @@ interface IpDataLookupFactory {
2633
IpDataLookup create(List<String> properties);
2734
}
2835

29-
private static final String CITY_DB_SUFFIX = "-City";
30-
private static final String COUNTRY_DB_SUFFIX = "-Country";
31-
private static final String ASN_DB_SUFFIX = "-ASN";
32-
private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP";
33-
private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type";
34-
private static final String DOMAIN_DB_SUFFIX = "-Domain";
35-
private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise";
36-
private static final String ISP_DB_SUFFIX = "-ISP";
37-
38-
@Nullable
39-
private static Database getMaxmindDatabase(final String databaseType) {
40-
if (databaseType.endsWith(CITY_DB_SUFFIX)) {
41-
return Database.City;
42-
} else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) {
43-
return Database.Country;
44-
} else if (databaseType.endsWith(ASN_DB_SUFFIX)) {
45-
return Database.Asn;
46-
} else if (databaseType.endsWith(ANONYMOUS_IP_DB_SUFFIX)) {
47-
return Database.AnonymousIp;
48-
} else if (databaseType.endsWith(CONNECTION_TYPE_DB_SUFFIX)) {
49-
return Database.ConnectionType;
50-
} else if (databaseType.endsWith(DOMAIN_DB_SUFFIX)) {
51-
return Database.Domain;
52-
} else if (databaseType.endsWith(ENTERPRISE_DB_SUFFIX)) {
53-
return Database.Enterprise;
54-
} else if (databaseType.endsWith(ISP_DB_SUFFIX)) {
55-
return Database.Isp;
56-
} else {
57-
return null; // no match was found
58-
}
59-
}
60-
6136
/**
6237
* Parses the passed-in databaseType and return the Database instance that is
6338
* associated with that databaseType.
6439
*
6540
* @param databaseType the database type String from the metadata of the database file
66-
* @return the Database instance that is associated with the databaseType
41+
* @return the Database instance that is associated with the databaseType (or null)
6742
*/
6843
@Nullable
6944
static Database getDatabase(final String databaseType) {
7045
Database database = null;
7146

7247
if (Strings.hasText(databaseType)) {
73-
database = getMaxmindDatabase(databaseType);
48+
final String databaseTypeLowerCase = databaseType.toLowerCase(Locale.ROOT);
49+
if (databaseTypeLowerCase.startsWith(IPINFO_PREFIX)) {
50+
database = getIpinfoDatabase(databaseTypeLowerCase); // all lower case!
51+
} else {
52+
// for historical reasons, fall back to assuming maxmind-like type parsing
53+
database = getMaxmindDatabase(databaseType);
54+
}
7455
}
7556

7657
return database;
7758
}
7859

79-
@Nullable
80-
static Function<Set<Database.Property>, IpDataLookup> getMaxmindLookup(final Database database) {
81-
return switch (database) {
82-
case City -> MaxmindIpDataLookups.City::new;
83-
case Country -> MaxmindIpDataLookups.Country::new;
84-
case Asn -> MaxmindIpDataLookups.Asn::new;
85-
case AnonymousIp -> MaxmindIpDataLookups.AnonymousIp::new;
86-
case ConnectionType -> MaxmindIpDataLookups.ConnectionType::new;
87-
case Domain -> MaxmindIpDataLookups.Domain::new;
88-
case Enterprise -> MaxmindIpDataLookups.Enterprise::new;
89-
case Isp -> MaxmindIpDataLookups.Isp::new;
90-
default -> null;
91-
};
92-
}
93-
9460
static IpDataLookupFactory get(final String databaseType, final String databaseFile) {
9561
final Database database = getDatabase(databaseType);
9662
if (database == null) {
9763
throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]");
9864
}
9965

100-
final Function<Set<Database.Property>, IpDataLookup> factoryMethod = getMaxmindLookup(database);
66+
final Function<Set<Database.Property>, IpDataLookup> factoryMethod;
67+
final String databaseTypeLowerCase = databaseType.toLowerCase(Locale.ROOT);
68+
if (databaseTypeLowerCase.startsWith(IPINFO_PREFIX)) {
69+
factoryMethod = getIpinfoLookup(database);
70+
} else {
71+
// for historical reasons, fall back to assuming maxmind-like types
72+
factoryMethod = getMaxmindLookup(database);
73+
}
10174

10275
if (factoryMethod == null) {
10376
throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]");

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,14 @@
2323

2424
import java.io.IOException;
2525
import java.net.InetAddress;
26+
import java.util.Arrays;
2627
import java.util.HashMap;
28+
import java.util.List;
2729
import java.util.Locale;
2830
import java.util.Map;
2931
import java.util.Set;
32+
import java.util.function.Function;
33+
import java.util.stream.Collectors;
3034

3135
/**
3236
* A collection of {@link IpDataLookup} implementations for IPinfo databases
@@ -43,6 +47,81 @@ private IpinfoIpDataLookups() {
4347
// prefix dispatch and checks case-insensitive, so that works out nicely
4448
static final String IPINFO_PREFIX = "ipinfo";
4549

50+
private static final Set<String> IPINFO_TYPE_STOP_WORDS = Set.of(
51+
"ipinfo",
52+
"extended",
53+
"free",
54+
"generic",
55+
"ip",
56+
"sample",
57+
"standard",
58+
"mmdb"
59+
);
60+
61+
/**
62+
* Cleans up the database_type String from an ipinfo database by splitting on punctuation, removing stop words, and then joining
63+
* with an underscore.
64+
* <p>
65+
* e.g. "ipinfo free_foo_sample.mmdb" -> "foo"
66+
*
67+
* @param type the database_type from an ipinfo database
68+
* @return a cleaned up database_type string
69+
*/
70+
// n.b. this is just based on observation of the types from a survey of such databases -- it's like browser user agent sniffing,
71+
// there aren't necessarily any amazing guarantees about this behavior
72+
static String ipinfoTypeCleanup(String type) {
73+
List<String> parts = Arrays.asList(type.split("[ _.]"));
74+
return parts.stream().filter((s) -> IPINFO_TYPE_STOP_WORDS.contains(s) == false).collect(Collectors.joining("_"));
75+
}
76+
77+
@Nullable
78+
static Database getIpinfoDatabase(final String databaseType) {
79+
// for ipinfo the database selection is more along the lines of user-agent sniffing than
80+
// string-based dispatch. the specific database_type strings could change in the future,
81+
// hence the somewhat loose nature of this checking.
82+
83+
final String cleanedType = ipinfoTypeCleanup(databaseType);
84+
85+
// early detection on any of the 'extended' types
86+
if (databaseType.contains("extended")) {
87+
// which are not currently supported
88+
logger.trace("returning null for unsupported database_type [{}]", databaseType);
89+
return null;
90+
}
91+
92+
// early detection on 'country_asn' so the 'country' and 'asn' checks don't get faked out
93+
if (cleanedType.contains("country_asn")) {
94+
// but it's not currently supported
95+
logger.trace("returning null for unsupported database_type [{}]", databaseType);
96+
return null;
97+
}
98+
99+
if (cleanedType.contains("asn")) {
100+
return Database.AsnV2;
101+
} else if (cleanedType.contains("country")) {
102+
return Database.CountryV2;
103+
} else if (cleanedType.contains("location")) { // note: catches 'location' and 'geolocation' ;)
104+
return Database.CityV2;
105+
} else if (cleanedType.contains("privacy")) {
106+
return Database.PrivacyDetection;
107+
} else {
108+
// no match was found
109+
logger.trace("returning null for unsupported database_type [{}]", databaseType);
110+
return null;
111+
}
112+
}
113+
114+
@Nullable
115+
static Function<Set<Database.Property>, IpDataLookup> getIpinfoLookup(final Database database) {
116+
return switch (database) {
117+
case Database.AsnV2 -> IpinfoIpDataLookups.Asn::new;
118+
case Database.CountryV2 -> IpinfoIpDataLookups.Country::new;
119+
case Database.CityV2 -> IpinfoIpDataLookups.Geolocation::new;
120+
case Database.PrivacyDetection -> IpinfoIpDataLookups.PrivacyDetection::new;
121+
default -> null;
122+
};
123+
}
124+
46125
/**
47126
* Lax-ly parses a string that (ideally) looks like 'AS123' into a Long like 123L (or null, if such parsing isn't possible).
48127
* @param asn a potentially empty (or null) ASN string that is expected to contain 'AS' and then a parsable long

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@
2626
import com.maxmind.geoip2.record.Postal;
2727
import com.maxmind.geoip2.record.Subdivision;
2828

29+
import org.apache.logging.log4j.LogManager;
30+
import org.apache.logging.log4j.Logger;
2931
import org.elasticsearch.common.network.InetAddresses;
3032
import org.elasticsearch.common.network.NetworkAddress;
3133
import org.elasticsearch.core.Nullable;
@@ -37,6 +39,7 @@
3739
import java.util.Locale;
3840
import java.util.Map;
3941
import java.util.Set;
42+
import java.util.function.Function;
4043

4144
/**
4245
* A collection of {@link IpDataLookup} implementations for MaxMind databases
@@ -47,11 +50,63 @@ private MaxmindIpDataLookups() {
4750
// utility class
4851
}
4952

53+
private static final Logger logger = LogManager.getLogger(MaxmindIpDataLookups.class);
54+
5055
// the actual prefixes from the metadata are cased like the literal strings, but
5156
// prefix dispatch and checks case-insensitive, so the actual constants are lowercase
5257
static final String GEOIP2_PREFIX = "GeoIP2".toLowerCase(Locale.ROOT);
5358
static final String GEOLITE2_PREFIX = "GeoLite2".toLowerCase(Locale.ROOT);
5459

60+
// note: the secondary dispatch on suffix happens to be case sensitive
61+
private static final String CITY_DB_SUFFIX = "-City";
62+
private static final String COUNTRY_DB_SUFFIX = "-Country";
63+
private static final String ASN_DB_SUFFIX = "-ASN";
64+
private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP";
65+
private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type";
66+
private static final String DOMAIN_DB_SUFFIX = "-Domain";
67+
private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise";
68+
private static final String ISP_DB_SUFFIX = "-ISP";
69+
70+
@Nullable
71+
static Database getMaxmindDatabase(final String databaseType) {
72+
if (databaseType.endsWith(CITY_DB_SUFFIX)) {
73+
return Database.City;
74+
} else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) {
75+
return Database.Country;
76+
} else if (databaseType.endsWith(ASN_DB_SUFFIX)) {
77+
return Database.Asn;
78+
} else if (databaseType.endsWith(ANONYMOUS_IP_DB_SUFFIX)) {
79+
return Database.AnonymousIp;
80+
} else if (databaseType.endsWith(CONNECTION_TYPE_DB_SUFFIX)) {
81+
return Database.ConnectionType;
82+
} else if (databaseType.endsWith(DOMAIN_DB_SUFFIX)) {
83+
return Database.Domain;
84+
} else if (databaseType.endsWith(ENTERPRISE_DB_SUFFIX)) {
85+
return Database.Enterprise;
86+
} else if (databaseType.endsWith(ISP_DB_SUFFIX)) {
87+
return Database.Isp;
88+
} else {
89+
// no match was found
90+
logger.trace("returning null for unsupported database_type [{}]", databaseType);
91+
return null;
92+
}
93+
}
94+
95+
@Nullable
96+
static Function<Set<Database.Property>, IpDataLookup> getMaxmindLookup(final Database database) {
97+
return switch (database) {
98+
case City -> MaxmindIpDataLookups.City::new;
99+
case Country -> MaxmindIpDataLookups.Country::new;
100+
case Asn -> MaxmindIpDataLookups.Asn::new;
101+
case AnonymousIp -> MaxmindIpDataLookups.AnonymousIp::new;
102+
case ConnectionType -> MaxmindIpDataLookups.ConnectionType::new;
103+
case Domain -> MaxmindIpDataLookups.Domain::new;
104+
case Enterprise -> MaxmindIpDataLookups.Enterprise::new;
105+
case Isp -> MaxmindIpDataLookups.Isp::new;
106+
default -> null;
107+
};
108+
}
109+
55110
static class AnonymousIp extends AbstractBase<AnonymousIpResponse> {
56111
AnonymousIp(final Set<Database.Property> properties) {
57112
super(

0 commit comments

Comments
 (0)