Skip to content

Commit 7bbebbd

Browse files
masseykemasseyke
authored
Supporting more maxmind fields in the geoip processor (#114268)
1 parent 3595956 commit 7bbebbd

File tree

5 files changed

+95
-19
lines changed

5 files changed

+95
-19
lines changed

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ enum Database {
3232
City(
3333
Set.of(
3434
Property.IP,
35+
Property.COUNTRY_IN_EUROPEAN_UNION,
3536
Property.COUNTRY_ISO_CODE,
3637
Property.CONTINENT_CODE,
3738
Property.COUNTRY_NAME,
@@ -41,7 +42,8 @@ enum Database {
4142
Property.CITY_NAME,
4243
Property.TIMEZONE,
4344
Property.LOCATION,
44-
Property.POSTAL_CODE
45+
Property.POSTAL_CODE,
46+
Property.ACCURACY_RADIUS
4547
),
4648
Set.of(
4749
Property.COUNTRY_ISO_CODE,
@@ -54,7 +56,14 @@ enum Database {
5456
)
5557
),
5658
Country(
57-
Set.of(Property.IP, Property.CONTINENT_CODE, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE),
59+
Set.of(
60+
Property.IP,
61+
Property.CONTINENT_CODE,
62+
Property.CONTINENT_NAME,
63+
Property.COUNTRY_NAME,
64+
Property.COUNTRY_IN_EUROPEAN_UNION,
65+
Property.COUNTRY_ISO_CODE
66+
),
5867
Set.of(Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE)
5968
),
6069
Asn(
@@ -85,12 +94,15 @@ enum Database {
8594
Enterprise(
8695
Set.of(
8796
Property.IP,
97+
Property.COUNTRY_CONFIDENCE,
98+
Property.COUNTRY_IN_EUROPEAN_UNION,
8899
Property.COUNTRY_ISO_CODE,
89100
Property.COUNTRY_NAME,
90101
Property.CONTINENT_CODE,
91102
Property.CONTINENT_NAME,
92103
Property.REGION_ISO_CODE,
93104
Property.REGION_NAME,
105+
Property.CITY_CONFIDENCE,
94106
Property.CITY_NAME,
95107
Property.TIMEZONE,
96108
Property.LOCATION,
@@ -110,7 +122,9 @@ enum Database {
110122
Property.MOBILE_NETWORK_CODE,
111123
Property.USER_TYPE,
112124
Property.CONNECTION_TYPE,
113-
Property.POSTAL_CODE
125+
Property.POSTAL_CODE,
126+
Property.POSTAL_CONFIDENCE,
127+
Property.ACCURACY_RADIUS
114128
),
115129
Set.of(
116130
Property.COUNTRY_ISO_CODE,
@@ -205,12 +219,15 @@ public Set<Property> parseProperties(@Nullable final List<String> propertyNames)
205219
enum Property {
206220

207221
IP,
222+
COUNTRY_CONFIDENCE,
223+
COUNTRY_IN_EUROPEAN_UNION,
208224
COUNTRY_ISO_CODE,
209225
COUNTRY_NAME,
210226
CONTINENT_CODE,
211227
CONTINENT_NAME,
212228
REGION_ISO_CODE,
213229
REGION_NAME,
230+
CITY_CONFIDENCE,
214231
CITY_NAME,
215232
TIMEZONE,
216233
LOCATION,
@@ -231,7 +248,9 @@ enum Property {
231248
CONNECTION_TYPE,
232249
USER_TYPE,
233250
TYPE,
234-
POSTAL_CODE;
251+
POSTAL_CODE,
252+
POSTAL_CONFIDENCE,
253+
ACCURACY_RADIUS;
235254

236255
/**
237256
* Parses a string representation of a property into an actual Property instance. Not all properties that exist are

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,12 @@ protected Map<String, Object> transform(final CityResponse response) {
146146
for (Database.Property property : this.properties) {
147147
switch (property) {
148148
case IP -> data.put("ip", response.getTraits().getIpAddress());
149+
case COUNTRY_IN_EUROPEAN_UNION -> {
150+
if (country.getIsoCode() != null) {
151+
// isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country
152+
data.put("country_in_european_union", country.isInEuropeanUnion());
153+
}
154+
}
149155
case COUNTRY_ISO_CODE -> {
150156
String countryIsoCode = country.getIsoCode();
151157
if (countryIsoCode != null) {
@@ -208,6 +214,12 @@ protected Map<String, Object> transform(final CityResponse response) {
208214
data.put("location", locationObject);
209215
}
210216
}
217+
case ACCURACY_RADIUS -> {
218+
Integer accuracyRadius = location.getAccuracyRadius();
219+
if (accuracyRadius != null) {
220+
data.put("accuracy_radius", accuracyRadius);
221+
}
222+
}
211223
case POSTAL_CODE -> {
212224
if (postal != null && postal.getCode() != null) {
213225
data.put("postal_code", postal.getCode());
@@ -261,6 +273,12 @@ protected Map<String, Object> transform(final CountryResponse response) {
261273
for (Database.Property property : this.properties) {
262274
switch (property) {
263275
case IP -> data.put("ip", response.getTraits().getIpAddress());
276+
case COUNTRY_IN_EUROPEAN_UNION -> {
277+
if (country.getIsoCode() != null) {
278+
// isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country
279+
data.put("country_in_european_union", country.isInEuropeanUnion());
280+
}
281+
}
264282
case COUNTRY_ISO_CODE -> {
265283
String countryIsoCode = country.getIsoCode();
266284
if (countryIsoCode != null) {
@@ -359,6 +377,18 @@ protected Map<String, Object> transform(final EnterpriseResponse response) {
359377
for (Database.Property property : this.properties) {
360378
switch (property) {
361379
case IP -> data.put("ip", response.getTraits().getIpAddress());
380+
case COUNTRY_CONFIDENCE -> {
381+
Integer countryConfidence = country.getConfidence();
382+
if (countryConfidence != null) {
383+
data.put("country_confidence", countryConfidence);
384+
}
385+
}
386+
case COUNTRY_IN_EUROPEAN_UNION -> {
387+
if (country.getIsoCode() != null) {
388+
// isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country
389+
data.put("country_in_european_union", country.isInEuropeanUnion());
390+
}
391+
}
362392
case COUNTRY_ISO_CODE -> {
363393
String countryIsoCode = country.getIsoCode();
364394
if (countryIsoCode != null) {
@@ -399,6 +429,12 @@ protected Map<String, Object> transform(final EnterpriseResponse response) {
399429
data.put("region_name", subdivisionName);
400430
}
401431
}
432+
case CITY_CONFIDENCE -> {
433+
Integer cityConfidence = city.getConfidence();
434+
if (cityConfidence != null) {
435+
data.put("city_confidence", cityConfidence);
436+
}
437+
}
402438
case CITY_NAME -> {
403439
String cityName = city.getName();
404440
if (cityName != null) {
@@ -421,11 +457,23 @@ protected Map<String, Object> transform(final EnterpriseResponse response) {
421457
data.put("location", locationObject);
422458
}
423459
}
460+
case ACCURACY_RADIUS -> {
461+
Integer accuracyRadius = location.getAccuracyRadius();
462+
if (accuracyRadius != null) {
463+
data.put("accuracy_radius", accuracyRadius);
464+
}
465+
}
424466
case POSTAL_CODE -> {
425467
if (postal != null && postal.getCode() != null) {
426468
data.put("postal_code", postal.getCode());
427469
}
428470
}
471+
case POSTAL_CONFIDENCE -> {
472+
Integer postalConfidence = postal.getConfidence();
473+
if (postalConfidence != null) {
474+
data.put("postal_confidence", postalConfidence);
475+
}
476+
}
429477
case ASN -> {
430478
if (asn != null) {
431479
data.put("asn", asn);

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ public void testBuildWithCountryDbAndAsnFields() {
195195
equalTo(
196196
"[properties] illegal property value ["
197197
+ asnProperty
198-
+ "]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME]"
198+
+ "]. valid values are [IP, COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME]"
199199
)
200200
);
201201
}
@@ -273,9 +273,9 @@ public void testBuildIllegalFieldOption() {
273273
assertThat(
274274
e.getMessage(),
275275
equalTo(
276-
"[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, "
276+
"[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, "
277277
+ "COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, "
278-
+ "LOCATION, POSTAL_CODE]"
278+
+ "LOCATION, POSTAL_CODE, ACCURACY_RADIUS]"
279279
)
280280
);
281281

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,9 @@ public void testCity() throws Exception {
106106
@SuppressWarnings("unchecked")
107107
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
108108
assertThat(geoData, notNullValue());
109-
assertThat(geoData.size(), equalTo(7));
109+
assertThat(geoData.size(), equalTo(9));
110110
assertThat(geoData.get("ip"), equalTo(ip));
111+
assertThat(geoData.get("country_in_european_union"), equalTo(false));
111112
assertThat(geoData.get("country_iso_code"), equalTo("US"));
112113
assertThat(geoData.get("country_name"), equalTo("United States"));
113114
assertThat(geoData.get("continent_code"), equalTo("NA"));
@@ -222,8 +223,9 @@ public void testCity_withIpV6() throws Exception {
222223
@SuppressWarnings("unchecked")
223224
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
224225
assertThat(geoData, notNullValue());
225-
assertThat(geoData.size(), equalTo(11));
226+
assertThat(geoData.size(), equalTo(13));
226227
assertThat(geoData.get("ip"), equalTo(ip));
228+
assertThat(geoData.get("country_in_european_union"), equalTo(false));
227229
assertThat(geoData.get("country_iso_code"), equalTo("US"));
228230
assertThat(geoData.get("country_name"), equalTo("United States"));
229231
assertThat(geoData.get("continent_code"), equalTo("NA"));
@@ -233,6 +235,7 @@ public void testCity_withIpV6() throws Exception {
233235
assertThat(geoData.get("city_name"), equalTo("Homestead"));
234236
assertThat(geoData.get("timezone"), equalTo("America/New_York"));
235237
assertThat(geoData.get("location"), equalTo(Map.of("lat", 25.4573d, "lon", -80.4572d)));
238+
assertThat(geoData.get("accuracy_radius"), equalTo(50));
236239
assertThat(geoData.get("postal_code"), equalTo("33035"));
237240
}
238241

@@ -288,8 +291,9 @@ public void testCountry() throws Exception {
288291
@SuppressWarnings("unchecked")
289292
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
290293
assertThat(geoData, notNullValue());
291-
assertThat(geoData.size(), equalTo(5));
294+
assertThat(geoData.size(), equalTo(6));
292295
assertThat(geoData.get("ip"), equalTo(ip));
296+
assertThat(geoData.get("country_in_european_union"), equalTo(true));
293297
assertThat(geoData.get("country_iso_code"), equalTo("NL"));
294298
assertThat(geoData.get("country_name"), equalTo("Netherlands"));
295299
assertThat(geoData.get("continent_code"), equalTo("EU"));
@@ -471,18 +475,23 @@ public void testEnterprise() throws Exception {
471475
@SuppressWarnings("unchecked")
472476
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
473477
assertThat(geoData, notNullValue());
474-
assertThat(geoData.size(), equalTo(25));
478+
assertThat(geoData.size(), equalTo(30));
475479
assertThat(geoData.get("ip"), equalTo(ip));
480+
assertThat(geoData.get("country_confidence"), equalTo(99));
481+
assertThat(geoData.get("country_in_european_union"), equalTo(false));
476482
assertThat(geoData.get("country_iso_code"), equalTo("US"));
477483
assertThat(geoData.get("country_name"), equalTo("United States"));
478484
assertThat(geoData.get("continent_code"), equalTo("NA"));
479485
assertThat(geoData.get("continent_name"), equalTo("North America"));
480486
assertThat(geoData.get("region_iso_code"), equalTo("US-NY"));
481487
assertThat(geoData.get("region_name"), equalTo("New York"));
488+
assertThat(geoData.get("city_confidence"), equalTo(11));
482489
assertThat(geoData.get("city_name"), equalTo("Chatham"));
483490
assertThat(geoData.get("timezone"), equalTo("America/New_York"));
484491
assertThat(geoData.get("location"), equalTo(Map.of("lat", 42.3478, "lon", -73.5549)));
492+
assertThat(geoData.get("accuracy_radius"), equalTo(27));
485493
assertThat(geoData.get("postal_code"), equalTo("12037"));
494+
assertThat(geoData.get("city_confidence"), equalTo(11));
486495
assertThat(geoData.get("asn"), equalTo(14671L));
487496
assertThat(geoData.get("organization_name"), equalTo("FairPoint Communications"));
488497
assertThat(geoData.get("network"), equalTo("74.209.16.0/20"));

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,10 @@ public class MaxMindSupportTests extends ESTestCase {
7878
"city.name",
7979
"continent.code",
8080
"continent.name",
81+
"country.inEuropeanUnion",
8182
"country.isoCode",
8283
"country.name",
84+
"location.accuracyRadius",
8385
"location.latitude",
8486
"location.longitude",
8587
"location.timeZone",
@@ -95,14 +97,12 @@ public class MaxMindSupportTests extends ESTestCase {
9597
"continent.names",
9698
"country.confidence",
9799
"country.geoNameId",
98-
"country.inEuropeanUnion",
99100
"country.names",
100101
"leastSpecificSubdivision.confidence",
101102
"leastSpecificSubdivision.geoNameId",
102103
"leastSpecificSubdivision.isoCode",
103104
"leastSpecificSubdivision.name",
104105
"leastSpecificSubdivision.names",
105-
"location.accuracyRadius",
106106
"location.averageIncome",
107107
"location.metroCode",
108108
"location.populationDensity",
@@ -159,6 +159,7 @@ public class MaxMindSupportTests extends ESTestCase {
159159

160160
private static final Set<String> COUNTRY_SUPPORTED_FIELDS = Set.of(
161161
"continent.name",
162+
"country.inEuropeanUnion",
162163
"country.isoCode",
163164
"continent.code",
164165
"country.name"
@@ -168,7 +169,6 @@ public class MaxMindSupportTests extends ESTestCase {
168169
"continent.names",
169170
"country.confidence",
170171
"country.geoNameId",
171-
"country.inEuropeanUnion",
172172
"country.names",
173173
"maxMind",
174174
"registeredCountry.confidence",
@@ -213,17 +213,22 @@ public class MaxMindSupportTests extends ESTestCase {
213213
private static final Set<String> DOMAIN_UNSUPPORTED_FIELDS = Set.of("ipAddress", "network");
214214

215215
private static final Set<String> ENTERPRISE_SUPPORTED_FIELDS = Set.of(
216+
"city.confidence",
216217
"city.name",
217218
"continent.code",
218219
"continent.name",
220+
"country.confidence",
221+
"country.inEuropeanUnion",
219222
"country.isoCode",
220223
"country.name",
224+
"location.accuracyRadius",
221225
"location.latitude",
222226
"location.longitude",
223227
"location.timeZone",
224228
"mostSpecificSubdivision.isoCode",
225229
"mostSpecificSubdivision.name",
226230
"postal.code",
231+
"postal.confidence",
227232
"traits.anonymous",
228233
"traits.anonymousVpn",
229234
"traits.autonomousSystemNumber",
@@ -242,29 +247,24 @@ public class MaxMindSupportTests extends ESTestCase {
242247
"traits.userType"
243248
);
244249
private static final Set<String> ENTERPRISE_UNSUPPORTED_FIELDS = Set.of(
245-
"city.confidence",
246250
"city.geoNameId",
247251
"city.names",
248252
"continent.geoNameId",
249253
"continent.names",
250-
"country.confidence",
251254
"country.geoNameId",
252-
"country.inEuropeanUnion",
253255
"country.names",
254256
"leastSpecificSubdivision.confidence",
255257
"leastSpecificSubdivision.geoNameId",
256258
"leastSpecificSubdivision.isoCode",
257259
"leastSpecificSubdivision.name",
258260
"leastSpecificSubdivision.names",
259-
"location.accuracyRadius",
260261
"location.averageIncome",
261262
"location.metroCode",
262263
"location.populationDensity",
263264
"maxMind",
264265
"mostSpecificSubdivision.confidence",
265266
"mostSpecificSubdivision.geoNameId",
266267
"mostSpecificSubdivision.names",
267-
"postal.confidence",
268268
"registeredCountry.confidence",
269269
"registeredCountry.geoNameId",
270270
"registeredCountry.inEuropeanUnion",

0 commit comments

Comments
 (0)