Skip to content

Commit b769059

Browse files
joegallojoegallo
authored
IPinfo ASN and Country (#114192)
Adding the building blocks to support IPinfo ASN and Country data
1 parent d8cc7d3 commit b769059

File tree

9 files changed

+495
-4
lines changed

9 files changed

+495
-4
lines changed

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@
2222
* <p>
2323
* A database has a set of properties that are valid to use with it (see {@link Database#properties()}),
2424
* as well as a list of default properties to use if no properties are specified (see {@link Database#defaultProperties()}).
25+
* <p>
26+
* Some database providers have similar concepts but might have slightly different properties associated with those types.
27+
* This can be accommodated, for example, by having a Foo value and a separate FooV2 value where the 'V' should be read as
28+
* 'variant' or 'variation'. A V-less Database type is inherently the first variant/variation (i.e. V1).
2529
*/
2630
enum Database {
2731

@@ -137,6 +141,18 @@ enum Database {
137141
Property.MOBILE_COUNTRY_CODE,
138142
Property.MOBILE_NETWORK_CODE
139143
)
144+
),
145+
AsnV2(
146+
Set.of(
147+
Property.IP,
148+
Property.ASN,
149+
Property.ORGANIZATION_NAME,
150+
Property.NETWORK,
151+
Property.DOMAIN,
152+
Property.COUNTRY_ISO_CODE,
153+
Property.TYPE
154+
),
155+
Set.of(Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK)
140156
);
141157

142158
private final Set<Property> properties;
@@ -211,7 +227,8 @@ enum Property {
211227
MOBILE_COUNTRY_CODE,
212228
MOBILE_NETWORK_CODE,
213229
CONNECTION_TYPE,
214-
USER_TYPE;
230+
USER_TYPE,
231+
TYPE;
215232

216233
/**
217234
* Parses a string representation of a property into an actual Property instance. Not all properties that exist are

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ static Database getDatabase(final String databaseType) {
7676
return database;
7777
}
7878

79+
@Nullable
7980
static Function<Set<Database.Property>, IpDataLookup> getMaxmindLookup(final Database database) {
8081
return switch (database) {
8182
case City -> MaxmindIpDataLookups.City::new;
@@ -86,6 +87,7 @@ static Function<Set<Database.Property>, IpDataLookup> getMaxmindLookup(final Dat
8687
case Domain -> MaxmindIpDataLookups.Domain::new;
8788
case Enterprise -> MaxmindIpDataLookups.Enterprise::new;
8889
case Isp -> MaxmindIpDataLookups.Isp::new;
90+
default -> null;
8991
};
9092
}
9193

@@ -97,7 +99,6 @@ static IpDataLookupFactory get(final String databaseType, final String databaseF
9799

98100
final Function<Set<Database.Property>, IpDataLookup> factoryMethod = getMaxmindLookup(database);
99101

100-
// note: this can't presently be null, but keep this check -- it will be useful in the near future
101102
if (factoryMethod == null) {
102103
throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]");
103104
}

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.ingest.geoip;
11+
12+
import com.maxmind.db.DatabaseRecord;
13+
import com.maxmind.db.MaxMindDbConstructor;
14+
import com.maxmind.db.MaxMindDbParameter;
15+
import com.maxmind.db.Reader;
16+
17+
import org.apache.logging.log4j.LogManager;
18+
import org.apache.logging.log4j.Logger;
19+
import org.elasticsearch.common.Strings;
20+
import org.elasticsearch.common.network.InetAddresses;
21+
import org.elasticsearch.common.network.NetworkAddress;
22+
import org.elasticsearch.core.Nullable;
23+
24+
import java.io.IOException;
25+
import java.net.InetAddress;
26+
import java.util.HashMap;
27+
import java.util.Locale;
28+
import java.util.Map;
29+
import java.util.Set;
30+
31+
/**
32+
* A collection of {@link IpDataLookup} implementations for IPinfo databases
33+
*/
34+
final class IpinfoIpDataLookups {
35+
36+
private IpinfoIpDataLookups() {
37+
// utility class
38+
}
39+
40+
private static final Logger logger = LogManager.getLogger(IpinfoIpDataLookups.class);
41+
42+
/**
43+
* Lax-ly parses a string that (ideally) looks like 'AS123' into a Long like 123L (or null, if such parsing isn't possible).
44+
* @param asn a potentially empty (or null) ASN string that is expected to contain 'AS' and then a parsable long
45+
* @return the parsed asn
46+
*/
47+
static Long parseAsn(final String asn) {
48+
if (asn == null || Strings.hasText(asn) == false) {
49+
return null;
50+
} else {
51+
String stripped = asn.toUpperCase(Locale.ROOT).replaceAll("AS", "").trim();
52+
try {
53+
return Long.parseLong(stripped);
54+
} catch (NumberFormatException e) {
55+
logger.trace("Unable to parse non-compliant ASN string [{}]", asn);
56+
return null;
57+
}
58+
}
59+
}
60+
61+
public record AsnResult(
62+
Long asn,
63+
@Nullable String country, // not present in the free asn database
64+
String domain,
65+
String name,
66+
@Nullable String type // not present in the free asn database
67+
) {
68+
@SuppressWarnings("checkstyle:RedundantModifier")
69+
@MaxMindDbConstructor
70+
public AsnResult(
71+
@MaxMindDbParameter(name = "asn") String asn,
72+
@Nullable @MaxMindDbParameter(name = "country") String country,
73+
@MaxMindDbParameter(name = "domain") String domain,
74+
@MaxMindDbParameter(name = "name") String name,
75+
@Nullable @MaxMindDbParameter(name = "type") String type
76+
) {
77+
this(parseAsn(asn), country, domain, name, type);
78+
}
79+
}
80+
81+
public record CountryResult(
82+
@MaxMindDbParameter(name = "continent") String continent,
83+
@MaxMindDbParameter(name = "continent_name") String continentName,
84+
@MaxMindDbParameter(name = "country") String country,
85+
@MaxMindDbParameter(name = "country_name") String countryName
86+
) {
87+
@MaxMindDbConstructor
88+
public CountryResult {}
89+
}
90+
91+
static class Asn extends AbstractBase<AsnResult> {
92+
Asn(Set<Database.Property> properties) {
93+
super(properties, AsnResult.class);
94+
}
95+
96+
@Override
97+
protected Map<String, Object> transform(final Result<AsnResult> result) {
98+
AsnResult response = result.result;
99+
Long asn = response.asn;
100+
String organizationName = response.name;
101+
String network = result.network;
102+
103+
Map<String, Object> data = new HashMap<>();
104+
for (Database.Property property : this.properties) {
105+
switch (property) {
106+
case IP -> data.put("ip", result.ip);
107+
case ASN -> {
108+
if (asn != null) {
109+
data.put("asn", asn);
110+
}
111+
}
112+
case ORGANIZATION_NAME -> {
113+
if (organizationName != null) {
114+
data.put("organization_name", organizationName);
115+
}
116+
}
117+
case NETWORK -> {
118+
if (network != null) {
119+
data.put("network", network);
120+
}
121+
}
122+
case COUNTRY_ISO_CODE -> {
123+
if (response.country != null) {
124+
data.put("country_iso_code", response.country);
125+
}
126+
}
127+
case DOMAIN -> {
128+
if (response.domain != null) {
129+
data.put("domain", response.domain);
130+
}
131+
}
132+
case TYPE -> {
133+
if (response.type != null) {
134+
data.put("type", response.type);
135+
}
136+
}
137+
}
138+
}
139+
return data;
140+
}
141+
}
142+
143+
static class Country extends AbstractBase<CountryResult> {
144+
Country(Set<Database.Property> properties) {
145+
super(properties, CountryResult.class);
146+
}
147+
148+
@Override
149+
protected Map<String, Object> transform(final Result<CountryResult> result) {
150+
CountryResult response = result.result;
151+
152+
Map<String, Object> data = new HashMap<>();
153+
for (Database.Property property : this.properties) {
154+
switch (property) {
155+
case IP -> data.put("ip", result.ip);
156+
case COUNTRY_ISO_CODE -> {
157+
String countryIsoCode = response.country;
158+
if (countryIsoCode != null) {
159+
data.put("country_iso_code", countryIsoCode);
160+
}
161+
}
162+
case COUNTRY_NAME -> {
163+
String countryName = response.countryName;
164+
if (countryName != null) {
165+
data.put("country_name", countryName);
166+
}
167+
}
168+
case CONTINENT_CODE -> {
169+
String continentCode = response.continent;
170+
if (continentCode != null) {
171+
data.put("continent_code", continentCode);
172+
}
173+
}
174+
case CONTINENT_NAME -> {
175+
String continentName = response.continentName;
176+
if (continentName != null) {
177+
data.put("continent_name", continentName);
178+
}
179+
}
180+
}
181+
}
182+
return data;
183+
}
184+
}
185+
186+
/**
187+
* Just a little record holder -- there's the data that we receive via the binding to our record objects from the Reader via the
188+
* getRecord call, but then we also need to capture the passed-in ip address that came from the caller as well as the network for
189+
* the returned DatabaseRecord from the Reader.
190+
*/
191+
public record Result<T>(T result, String ip, String network) {}
192+
193+
/**
194+
* The {@link IpinfoIpDataLookups.AbstractBase} is an abstract base implementation of {@link IpDataLookup} that
195+
* provides common functionality for getting a {@link IpinfoIpDataLookups.Result} that wraps a record from a {@link IpDatabase}.
196+
*
197+
* @param <RESPONSE> the record type that will be wrapped and returned
198+
*/
199+
private abstract static class AbstractBase<RESPONSE> implements IpDataLookup {
200+
201+
protected final Set<Database.Property> properties;
202+
protected final Class<RESPONSE> clazz;
203+
204+
AbstractBase(final Set<Database.Property> properties, final Class<RESPONSE> clazz) {
205+
this.properties = Set.copyOf(properties);
206+
this.clazz = clazz;
207+
}
208+
209+
@Override
210+
public Set<Database.Property> getProperties() {
211+
return this.properties;
212+
}
213+
214+
@Override
215+
public final Map<String, Object> getData(final IpDatabase ipDatabase, final String ipAddress) {
216+
final Result<RESPONSE> response = ipDatabase.getResponse(ipAddress, this::lookup);
217+
return (response == null || response.result == null) ? Map.of() : transform(response);
218+
}
219+
220+
@Nullable
221+
private Result<RESPONSE> lookup(final Reader reader, final String ipAddress) throws IOException {
222+
final InetAddress ip = InetAddresses.forString(ipAddress);
223+
final DatabaseRecord<RESPONSE> record = reader.getRecord(ip, clazz);
224+
final RESPONSE data = record.getData();
225+
return (data == null) ? null : new Result<>(data, NetworkAddress.format(ip), record.getNetwork().toString());
226+
}
227+
228+
/**
229+
* Extract the configured properties from the retrieved response
230+
* @param response the non-null response that was retrieved
231+
* @return a mapping of properties for the ip from the response
232+
*/
233+
protected abstract Map<String, Object> transform(Result<RESPONSE> response);
234+
}
235+
}

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import java.util.HashMap;
2525
import java.util.List;
2626
import java.util.Map;
27+
import java.util.Set;
2728
import java.util.concurrent.atomic.AtomicBoolean;
2829

2930
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
@@ -64,8 +65,16 @@ public void testDatabasePropertyInvariants() {
6465
assertThat(Sets.difference(Database.Asn.properties(), Database.Isp.properties()), is(empty()));
6566
assertThat(Sets.difference(Database.Asn.defaultProperties(), Database.Isp.defaultProperties()), is(empty()));
6667

67-
// the enterprise database is like everything joined together
68-
for (Database type : Database.values()) {
68+
// the enterprise database is like these other databases joined together
69+
for (Database type : Set.of(
70+
Database.City,
71+
Database.Country,
72+
Database.Asn,
73+
Database.AnonymousIp,
74+
Database.ConnectionType,
75+
Database.Domain,
76+
Database.Isp
77+
)) {
6978
assertThat(Sets.difference(type.properties(), Database.Enterprise.properties()), is(empty()));
7079
}
7180
// but in terms of the default fields, it's like a drop-in replacement for the city database

0 commit comments

Comments
 (0)