Skip to content

Commit 3adce49

Browse files
masseykejoegallo
masseyke
and
joegallo
authored
Adding support for registered country fields for maxmind geoip databases (#114521) (#114543)
Co-authored-by: Joe Gallo <[email protected]>
1 parent 695fce4 commit 3adce49

File tree

6 files changed

+108
-22
lines changed

6 files changed

+108
-22
lines changed

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,10 @@ enum Database {
4343
Property.TIMEZONE,
4444
Property.LOCATION,
4545
Property.POSTAL_CODE,
46-
Property.ACCURACY_RADIUS
46+
Property.ACCURACY_RADIUS,
47+
Property.REGISTERED_COUNTRY_IN_EUROPEAN_UNION,
48+
Property.REGISTERED_COUNTRY_ISO_CODE,
49+
Property.REGISTERED_COUNTRY_NAME
4750
),
4851
Set.of(
4952
Property.COUNTRY_ISO_CODE,
@@ -62,7 +65,10 @@ enum Database {
6265
Property.CONTINENT_NAME,
6366
Property.COUNTRY_NAME,
6467
Property.COUNTRY_IN_EUROPEAN_UNION,
65-
Property.COUNTRY_ISO_CODE
68+
Property.COUNTRY_ISO_CODE,
69+
Property.REGISTERED_COUNTRY_IN_EUROPEAN_UNION,
70+
Property.REGISTERED_COUNTRY_ISO_CODE,
71+
Property.REGISTERED_COUNTRY_NAME
6672
),
6773
Set.of(Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE)
6874
),
@@ -124,7 +130,10 @@ enum Database {
124130
Property.CONNECTION_TYPE,
125131
Property.POSTAL_CODE,
126132
Property.POSTAL_CONFIDENCE,
127-
Property.ACCURACY_RADIUS
133+
Property.ACCURACY_RADIUS,
134+
Property.REGISTERED_COUNTRY_IN_EUROPEAN_UNION,
135+
Property.REGISTERED_COUNTRY_ISO_CODE,
136+
Property.REGISTERED_COUNTRY_NAME
128137
),
129138
Set.of(
130139
Property.COUNTRY_ISO_CODE,
@@ -182,6 +191,10 @@ enum Database {
182191
),
183192
Set.of(Property.COUNTRY_ISO_CODE, Property.REGION_NAME, Property.CITY_NAME, Property.LOCATION)
184193
),
194+
CountryV2(
195+
Set.of(Property.IP, Property.CONTINENT_CODE, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE),
196+
Set.of(Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE)
197+
),
185198
PrivacyDetection(
186199
Set.of(Property.IP, Property.HOSTING, Property.PROXY, Property.RELAY, Property.TOR, Property.VPN, Property.SERVICE),
187200
Set.of(Property.HOSTING, Property.PROXY, Property.RELAY, Property.TOR, Property.VPN, Property.SERVICE)
@@ -272,7 +285,10 @@ enum Property {
272285
PROXY,
273286
RELAY,
274287
VPN,
275-
SERVICE;
288+
SERVICE,
289+
REGISTERED_COUNTRY_IN_EUROPEAN_UNION,
290+
REGISTERED_COUNTRY_ISO_CODE,
291+
REGISTERED_COUNTRY_NAME;
276292

277293
/**
278294
* Parses a string representation of a property into an actual Property instance. Not all properties that exist are

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ static class City extends AbstractBase<CityResponse> {
142142
@Override
143143
protected Map<String, Object> transform(final CityResponse response) {
144144
com.maxmind.geoip2.record.Country country = response.getCountry();
145+
com.maxmind.geoip2.record.Country registeredCountry = response.getRegisteredCountry();
145146
com.maxmind.geoip2.record.City city = response.getCity();
146147
Location location = response.getLocation();
147148
Continent continent = response.getContinent();
@@ -231,6 +232,22 @@ protected Map<String, Object> transform(final CityResponse response) {
231232
data.put("postal_code", postal.getCode());
232233
}
233234
}
235+
case REGISTERED_COUNTRY_IN_EUROPEAN_UNION -> {
236+
if (registeredCountry.getIsoCode() != null) {
237+
// isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country
238+
data.put("registered_country_in_european_union", registeredCountry.isInEuropeanUnion());
239+
}
240+
}
241+
case REGISTERED_COUNTRY_ISO_CODE -> {
242+
if (registeredCountry.getIsoCode() != null) {
243+
data.put("registered_country_iso_code", registeredCountry.getIsoCode());
244+
}
245+
}
246+
case REGISTERED_COUNTRY_NAME -> {
247+
if (registeredCountry.getName() != null) {
248+
data.put("registered_country_name", registeredCountry.getName());
249+
}
250+
}
234251
}
235252
}
236253
return data;
@@ -273,6 +290,7 @@ static class Country extends AbstractBase<CountryResponse> {
273290
@Override
274291
protected Map<String, Object> transform(final CountryResponse response) {
275292
com.maxmind.geoip2.record.Country country = response.getCountry();
293+
com.maxmind.geoip2.record.Country registeredCountry = response.getRegisteredCountry();
276294
Continent continent = response.getContinent();
277295

278296
Map<String, Object> data = new HashMap<>();
@@ -309,6 +327,22 @@ protected Map<String, Object> transform(final CountryResponse response) {
309327
data.put("continent_name", continentName);
310328
}
311329
}
330+
case REGISTERED_COUNTRY_IN_EUROPEAN_UNION -> {
331+
if (registeredCountry.getIsoCode() != null) {
332+
// isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country
333+
data.put("registered_country_in_european_union", registeredCountry.isInEuropeanUnion());
334+
}
335+
}
336+
case REGISTERED_COUNTRY_ISO_CODE -> {
337+
if (registeredCountry.getIsoCode() != null) {
338+
data.put("registered_country_iso_code", registeredCountry.getIsoCode());
339+
}
340+
}
341+
case REGISTERED_COUNTRY_NAME -> {
342+
if (registeredCountry.getName() != null) {
343+
data.put("registered_country_name", registeredCountry.getName());
344+
}
345+
}
312346
}
313347
}
314348
return data;
@@ -351,6 +385,7 @@ static class Enterprise extends AbstractBase<EnterpriseResponse> {
351385
@Override
352386
protected Map<String, Object> transform(final EnterpriseResponse response) {
353387
com.maxmind.geoip2.record.Country country = response.getCountry();
388+
com.maxmind.geoip2.record.Country registeredCountry = response.getRegisteredCountry();
354389
com.maxmind.geoip2.record.City city = response.getCity();
355390
Location location = response.getLocation();
356391
Continent continent = response.getContinent();
@@ -548,6 +583,22 @@ protected Map<String, Object> transform(final EnterpriseResponse response) {
548583
data.put("connection_type", connectionType.toString());
549584
}
550585
}
586+
case REGISTERED_COUNTRY_IN_EUROPEAN_UNION -> {
587+
if (registeredCountry.getIsoCode() != null) {
588+
// isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country
589+
data.put("registered_country_in_european_union", registeredCountry.isInEuropeanUnion());
590+
}
591+
}
592+
case REGISTERED_COUNTRY_ISO_CODE -> {
593+
if (registeredCountry.getIsoCode() != null) {
594+
data.put("registered_country_iso_code", registeredCountry.getIsoCode());
595+
}
596+
}
597+
case REGISTERED_COUNTRY_NAME -> {
598+
if (registeredCountry.getName() != null) {
599+
data.put("registered_country_name", registeredCountry.getName());
600+
}
601+
}
551602
}
552603
}
553604
return data;

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,8 @@ public void testBuildWithCountryDbAndAsnFields() {
196196
equalTo(
197197
"[properties] illegal property value ["
198198
+ asnProperty
199-
+ "]. valid values are [IP, COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME]"
199+
+ "]. valid values are [IP, COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, "
200+
+ "CONTINENT_NAME, REGISTERED_COUNTRY_IN_EUROPEAN_UNION, REGISTERED_COUNTRY_ISO_CODE, REGISTERED_COUNTRY_NAME]"
200201
)
201202
);
202203
}
@@ -276,7 +277,8 @@ public void testBuildIllegalFieldOption() {
276277
equalTo(
277278
"[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, "
278279
+ "COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, "
279-
+ "LOCATION, POSTAL_CODE, ACCURACY_RADIUS]"
280+
+ "LOCATION, POSTAL_CODE, ACCURACY_RADIUS, REGISTERED_COUNTRY_IN_EUROPEAN_UNION, REGISTERED_COUNTRY_ISO_CODE, "
281+
+ "REGISTERED_COUNTRY_NAME]"
280282
)
281283
);
282284

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ public void testCity() throws Exception {
108108
@SuppressWarnings("unchecked")
109109
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
110110
assertThat(geoData, notNullValue());
111-
assertThat(geoData.size(), equalTo(9));
111+
assertThat(geoData.size(), equalTo(12));
112112
assertThat(geoData.get("ip"), equalTo(ip));
113113
assertThat(geoData.get("country_in_european_union"), equalTo(false));
114114
assertThat(geoData.get("country_iso_code"), equalTo("US"));
@@ -117,6 +117,9 @@ public void testCity() throws Exception {
117117
assertThat(geoData.get("continent_name"), equalTo("North America"));
118118
assertThat(geoData.get("timezone"), equalTo("America/Chicago"));
119119
assertThat(geoData.get("location"), equalTo(Map.of("lat", 37.751d, "lon", -97.822d)));
120+
assertThat(geoData.get("registered_country_in_european_union"), equalTo(false));
121+
assertThat(geoData.get("registered_country_iso_code"), equalTo("US"));
122+
assertThat(geoData.get("registered_country_name"), equalTo("United States"));
120123
}
121124

122125
public void testNullValueWithIgnoreMissing() throws Exception {
@@ -230,7 +233,7 @@ public void testCity_withIpV6() throws Exception {
230233
@SuppressWarnings("unchecked")
231234
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
232235
assertThat(geoData, notNullValue());
233-
assertThat(geoData.size(), equalTo(13));
236+
assertThat(geoData.size(), equalTo(16));
234237
assertThat(geoData.get("ip"), equalTo(ip));
235238
assertThat(geoData.get("country_in_european_union"), equalTo(false));
236239
assertThat(geoData.get("country_iso_code"), equalTo("US"));
@@ -244,6 +247,9 @@ public void testCity_withIpV6() throws Exception {
244247
assertThat(geoData.get("location"), equalTo(Map.of("lat", 25.4573d, "lon", -80.4572d)));
245248
assertThat(geoData.get("accuracy_radius"), equalTo(50));
246249
assertThat(geoData.get("postal_code"), equalTo("33035"));
250+
assertThat(geoData.get("registered_country_in_european_union"), equalTo(false));
251+
assertThat(geoData.get("registered_country_iso_code"), equalTo("US"));
252+
assertThat(geoData.get("registered_country_name"), equalTo("United States"));
247253
}
248254

249255
public void testCityWithMissingLocation() throws Exception {
@@ -300,13 +306,16 @@ public void testCountry() throws Exception {
300306
@SuppressWarnings("unchecked")
301307
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
302308
assertThat(geoData, notNullValue());
303-
assertThat(geoData.size(), equalTo(6));
309+
assertThat(geoData.size(), equalTo(9));
304310
assertThat(geoData.get("ip"), equalTo(ip));
305311
assertThat(geoData.get("country_in_european_union"), equalTo(true));
306312
assertThat(geoData.get("country_iso_code"), equalTo("NL"));
307313
assertThat(geoData.get("country_name"), equalTo("Netherlands"));
308314
assertThat(geoData.get("continent_code"), equalTo("EU"));
309315
assertThat(geoData.get("continent_name"), equalTo("Europe"));
316+
assertThat(geoData.get("registered_country_in_european_union"), equalTo(true));
317+
assertThat(geoData.get("registered_country_iso_code"), equalTo("NL"));
318+
assertThat(geoData.get("registered_country_name"), equalTo("Netherlands"));
310319
}
311320

312321
public void testCountryWithMissingLocation() throws Exception {
@@ -490,7 +499,7 @@ public void testEnterprise() throws Exception {
490499
@SuppressWarnings("unchecked")
491500
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
492501
assertThat(geoData, notNullValue());
493-
assertThat(geoData.size(), equalTo(30));
502+
assertThat(geoData.size(), equalTo(33));
494503
assertThat(geoData.get("ip"), equalTo(ip));
495504
assertThat(geoData.get("country_confidence"), equalTo(99));
496505
assertThat(geoData.get("country_in_european_union"), equalTo(false));
@@ -521,6 +530,9 @@ public void testEnterprise() throws Exception {
521530
assertThat(geoData.get("isp_organization_name"), equalTo("Fairpoint Communications"));
522531
assertThat(geoData.get("user_type"), equalTo("residential"));
523532
assertThat(geoData.get("connection_type"), equalTo("Cable/DSL"));
533+
assertThat(geoData.get("registered_country_in_european_union"), equalTo(false));
534+
assertThat(geoData.get("registered_country_iso_code"), equalTo("US"));
535+
assertThat(geoData.get("registered_country_name"), equalTo("United States"));
524536
}
525537

526538
public void testIsp() throws Exception {

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,10 @@ public void testDatabasePropertyInvariants() {
7979
// the second City variant database is like a version of the ordinary City database but lacking many fields
8080
assertThat(Sets.difference(Database.CityV2.properties(), Database.City.properties()), is(empty()));
8181
assertThat(Sets.difference(Database.CityV2.defaultProperties(), Database.City.defaultProperties()), is(empty()));
82+
83+
// the second Country variant database is like a version of the ordinary Country database but lacking come fields
84+
assertThat(Sets.difference(Database.CountryV2.properties(), Database.CountryV2.properties()), is(empty()));
85+
assertThat(Database.CountryV2.defaultProperties(), equalTo(Database.Country.defaultProperties()));
8286
}
8387

8488
public void testParseAsn() {
@@ -219,7 +223,7 @@ public void testCountry() throws IOException {
219223

220224
// this is the 'free' Country database (sample)
221225
try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_country_sample.mmdb")) {
222-
IpDataLookup lookup = new IpinfoIpDataLookups.Country(Database.Country.properties());
226+
IpDataLookup lookup = new IpinfoIpDataLookups.Country(Database.CountryV2.properties());
223227
Map<String, Object> data = lookup.getData(loader, "4.221.143.168");
224228
assertThat(
225229
data,

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,10 @@ public class MaxMindSupportTests extends ESTestCase {
8787
"location.timeZone",
8888
"mostSpecificSubdivision.isoCode",
8989
"mostSpecificSubdivision.name",
90-
"postal.code"
90+
"postal.code",
91+
"registeredCountry.inEuropeanUnion",
92+
"registeredCountry.isoCode",
93+
"registeredCountry.name"
9194
);
9295
private static final Set<String> CITY_UNSUPPORTED_FIELDS = Set.of(
9396
"city.confidence",
@@ -113,9 +116,6 @@ public class MaxMindSupportTests extends ESTestCase {
113116
"postal.confidence",
114117
"registeredCountry.confidence",
115118
"registeredCountry.geoNameId",
116-
"registeredCountry.inEuropeanUnion",
117-
"registeredCountry.isoCode",
118-
"registeredCountry.name",
119119
"registeredCountry.names",
120120
"representedCountry.confidence",
121121
"representedCountry.geoNameId",
@@ -162,7 +162,10 @@ public class MaxMindSupportTests extends ESTestCase {
162162
"country.inEuropeanUnion",
163163
"country.isoCode",
164164
"continent.code",
165-
"country.name"
165+
"country.name",
166+
"registeredCountry.inEuropeanUnion",
167+
"registeredCountry.isoCode",
168+
"registeredCountry.name"
166169
);
167170
private static final Set<String> COUNTRY_UNSUPPORTED_FIELDS = Set.of(
168171
"continent.geoNameId",
@@ -173,9 +176,6 @@ public class MaxMindSupportTests extends ESTestCase {
173176
"maxMind",
174177
"registeredCountry.confidence",
175178
"registeredCountry.geoNameId",
176-
"registeredCountry.inEuropeanUnion",
177-
"registeredCountry.isoCode",
178-
"registeredCountry.name",
179179
"registeredCountry.names",
180180
"representedCountry.confidence",
181181
"representedCountry.geoNameId",
@@ -229,6 +229,9 @@ public class MaxMindSupportTests extends ESTestCase {
229229
"mostSpecificSubdivision.name",
230230
"postal.code",
231231
"postal.confidence",
232+
"registeredCountry.inEuropeanUnion",
233+
"registeredCountry.isoCode",
234+
"registeredCountry.name",
232235
"traits.anonymous",
233236
"traits.anonymousVpn",
234237
"traits.autonomousSystemNumber",
@@ -267,9 +270,6 @@ public class MaxMindSupportTests extends ESTestCase {
267270
"mostSpecificSubdivision.names",
268271
"registeredCountry.confidence",
269272
"registeredCountry.geoNameId",
270-
"registeredCountry.inEuropeanUnion",
271-
"registeredCountry.isoCode",
272-
"registeredCountry.name",
273273
"registeredCountry.names",
274274
"representedCountry.confidence",
275275
"representedCountry.geoNameId",
@@ -364,6 +364,7 @@ public class MaxMindSupportTests extends ESTestCase {
364364
private static final Set<Database> KNOWN_UNSUPPORTED_DATABASE_VARIANTS = Set.of(
365365
Database.AsnV2,
366366
Database.CityV2,
367+
Database.CountryV2,
367368
Database.PrivacyDetection
368369
);
369370

0 commit comments

Comments
 (0)