Skip to content

Commit 57f083e

Browse files
joegallojoegallo
authored
IPinfo privacy detection support (#114456) (#114462)
1 parent c7473ad commit 57f083e

File tree

5 files changed

+194
-3
lines changed

5 files changed

+194
-3
lines changed

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,11 @@ enum Database {
181181
Property.POSTAL_CODE
182182
),
183183
Set.of(Property.COUNTRY_ISO_CODE, Property.REGION_NAME, Property.CITY_NAME, Property.LOCATION)
184-
),;
184+
),
185+
PrivacyDetection(
186+
Set.of(Property.IP, Property.HOSTING, Property.PROXY, Property.RELAY, Property.TOR, Property.VPN, Property.SERVICE),
187+
Set.of(Property.HOSTING, Property.PROXY, Property.RELAY, Property.TOR, Property.VPN, Property.SERVICE)
188+
);
185189

186190
private final Set<Property> properties;
187191
private final Set<Property> defaultProperties;
@@ -262,7 +266,13 @@ enum Property {
262266
TYPE,
263267
POSTAL_CODE,
264268
POSTAL_CONFIDENCE,
265-
ACCURACY_RADIUS;
269+
ACCURACY_RADIUS,
270+
HOSTING,
271+
TOR,
272+
PROXY,
273+
RELAY,
274+
VPN,
275+
SERVICE;
266276

267277
/**
268278
* Parses a string representation of a property into an actual Property instance. Not all properties that exist are

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,31 @@ static Long parseAsn(final String asn) {
5858
}
5959
}
6060

61+
/**
62+
* Lax-ly parses a string that contains a boolean into a Boolean (or null, if such parsing isn't possible).
63+
* @param bool a potentially empty (or null) string that is expected to contain a parsable boolean
64+
* @return the parsed boolean
65+
*/
66+
static Boolean parseBoolean(final String bool) {
67+
if (bool == null) {
68+
return null;
69+
} else {
70+
String trimmed = bool.toLowerCase(Locale.ROOT).trim();
71+
if ("true".equals(trimmed)) {
72+
return true;
73+
} else if ("false".equals(trimmed)) {
74+
// "false" can represent false -- this an expected future enhancement in how the database represents booleans
75+
return false;
76+
} else if (trimmed.isEmpty()) {
77+
// empty string can represent false -- this is how the database currently represents 'false' values
78+
return false;
79+
} else {
80+
logger.trace("Unable to parse non-compliant boolean string [{}]", bool);
81+
return null;
82+
}
83+
}
84+
}
85+
6186
/**
6287
* Lax-ly parses a string that contains a double into a Double (or null, if such parsing isn't possible).
6388
* @param latlon a potentially empty (or null) string that is expected to contain a parsable double
@@ -132,6 +157,22 @@ public GeolocationResult(
132157
}
133158
}
134159

160+
public record PrivacyDetectionResult(Boolean hosting, Boolean proxy, Boolean relay, String service, Boolean tor, Boolean vpn) {
161+
@SuppressWarnings("checkstyle:RedundantModifier")
162+
@MaxMindDbConstructor
163+
public PrivacyDetectionResult(
164+
@MaxMindDbParameter(name = "hosting") String hosting,
165+
// @MaxMindDbParameter(name = "network") String network, // for now we're not exposing this
166+
@MaxMindDbParameter(name = "proxy") String proxy,
167+
@MaxMindDbParameter(name = "relay") String relay,
168+
@MaxMindDbParameter(name = "service") String service, // n.b. this remains a string, the rest are parsed as booleans
169+
@MaxMindDbParameter(name = "tor") String tor,
170+
@MaxMindDbParameter(name = "vpn") String vpn
171+
) {
172+
this(parseBoolean(hosting), parseBoolean(proxy), parseBoolean(relay), service, parseBoolean(tor), parseBoolean(vpn));
173+
}
174+
}
175+
135176
static class Asn extends AbstractBase<AsnResult> {
136177
Asn(Set<Database.Property> properties) {
137178
super(properties, AsnResult.class);
@@ -286,6 +327,55 @@ protected Map<String, Object> transform(final Result<GeolocationResult> result)
286327
}
287328
}
288329

330+
static class PrivacyDetection extends AbstractBase<PrivacyDetectionResult> {
331+
PrivacyDetection(Set<Database.Property> properties) {
332+
super(properties, PrivacyDetectionResult.class);
333+
}
334+
335+
@Override
336+
protected Map<String, Object> transform(final Result<PrivacyDetectionResult> result) {
337+
PrivacyDetectionResult response = result.result;
338+
339+
Map<String, Object> data = new HashMap<>();
340+
for (Database.Property property : this.properties) {
341+
switch (property) {
342+
case IP -> data.put("ip", result.ip);
343+
case HOSTING -> {
344+
if (response.hosting != null) {
345+
data.put("hosting", response.hosting);
346+
}
347+
}
348+
case TOR -> {
349+
if (response.tor != null) {
350+
data.put("tor", response.tor);
351+
}
352+
}
353+
case PROXY -> {
354+
if (response.proxy != null) {
355+
data.put("proxy", response.proxy);
356+
}
357+
}
358+
case RELAY -> {
359+
if (response.relay != null) {
360+
data.put("relay", response.relay);
361+
}
362+
}
363+
case VPN -> {
364+
if (response.vpn != null) {
365+
data.put("vpn", response.vpn);
366+
}
367+
}
368+
case SERVICE -> {
369+
if (Strings.hasText(response.service)) {
370+
data.put("service", response.service);
371+
}
372+
}
373+
}
374+
}
375+
return data;
376+
}
377+
}
378+
289379
/**
290380
* Just a little record holder -- there's the data that we receive via the binding to our record objects from the Reader via the
291381
* getRecord call, but then we also need to capture the passed-in ip address that came from the caller as well as the network for

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@
3838
import static java.util.Map.entry;
3939
import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase;
4040
import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseAsn;
41+
import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseBoolean;
4142
import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseLocationDouble;
43+
import static org.hamcrest.Matchers.anyOf;
4244
import static org.hamcrest.Matchers.empty;
4345
import static org.hamcrest.Matchers.equalTo;
4446
import static org.hamcrest.Matchers.is;
@@ -93,6 +95,21 @@ public void testParseAsn() {
9395
assertThat(parseAsn("anythingelse"), nullValue());
9496
}
9597

98+
public void testParseBoolean() {
99+
// expected cases: "true" is true and "" is false
100+
assertThat(parseBoolean("true"), equalTo(true));
101+
assertThat(parseBoolean(""), equalTo(false));
102+
assertThat(parseBoolean("false"), equalTo(false)); // future proofing
103+
// defensive case: null becomes null, this is not expected fwiw
104+
assertThat(parseBoolean(null), nullValue());
105+
// defensive cases: we strip whitespace and ignore case
106+
assertThat(parseBoolean(" "), equalTo(false));
107+
assertThat(parseBoolean(" TrUe "), equalTo(true));
108+
assertThat(parseBoolean(" FaLSE "), equalTo(false));
109+
// bottom case: a non-parsable string is null
110+
assertThat(parseBoolean(randomAlphaOfLength(8)), nullValue());
111+
}
112+
96113
public void testParseLocationDouble() {
97114
// expected case: "123.45" is 123.45
98115
assertThat(parseLocationDouble("123.45"), equalTo(123.45));
@@ -287,6 +304,76 @@ public void testGeolocationInvariants() {
287304
}
288305
}
289306

307+
public void testPrivacyDetection() throws IOException {
308+
assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
309+
Path configDir = tmpDir;
310+
copyDatabase("ipinfo/privacy_detection_sample.mmdb", configDir.resolve("privacy_detection_sample.mmdb"));
311+
312+
GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload
313+
ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache);
314+
configDatabases.initialize(resourceWatcherService);
315+
316+
// testing the first row in the sample database
317+
try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("privacy_detection_sample.mmdb")) {
318+
IpDataLookup lookup = new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties());
319+
Map<String, Object> data = lookup.getData(loader, "1.53.59.33");
320+
assertThat(
321+
data,
322+
equalTo(
323+
Map.ofEntries(
324+
entry("ip", "1.53.59.33"),
325+
entry("hosting", false),
326+
entry("proxy", false),
327+
entry("relay", false),
328+
entry("tor", false),
329+
entry("vpn", true)
330+
)
331+
)
332+
);
333+
}
334+
335+
// testing a row with a non-empty service in the sample database
336+
try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("privacy_detection_sample.mmdb")) {
337+
IpDataLookup lookup = new IpinfoIpDataLookups.PrivacyDetection(Database.PrivacyDetection.properties());
338+
Map<String, Object> data = lookup.getData(loader, "216.131.74.65");
339+
assertThat(
340+
data,
341+
equalTo(
342+
Map.ofEntries(
343+
entry("ip", "216.131.74.65"),
344+
entry("hosting", true),
345+
entry("proxy", false),
346+
entry("service", "FastVPN"),
347+
entry("relay", false),
348+
entry("tor", false),
349+
entry("vpn", true)
350+
)
351+
)
352+
);
353+
}
354+
}
355+
356+
public void testPrivacyDetectionInvariants() {
357+
assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS);
358+
Path configDir = tmpDir;
359+
copyDatabase("ipinfo/privacy_detection_sample.mmdb", configDir.resolve("privacy_detection_sample.mmdb"));
360+
361+
{
362+
final Set<String> expectedColumns = Set.of("network", "service", "hosting", "proxy", "relay", "tor", "vpn");
363+
364+
Path databasePath = configDir.resolve("privacy_detection_sample.mmdb");
365+
assertDatabaseInvariants(databasePath, (ip, row) -> {
366+
assertThat(row.keySet(), equalTo(expectedColumns));
367+
368+
for (String booleanColumn : Set.of("hosting", "proxy", "relay", "tor", "vpn")) {
369+
String bool = (String) row.get(booleanColumn);
370+
assertThat(bool, anyOf(equalTo("true"), equalTo(""), equalTo("false")));
371+
assertThat(parseBoolean(bool), notNullValue());
372+
}
373+
});
374+
}
375+
}
376+
290377
private static void assertDatabaseInvariants(final Path databasePath, final BiConsumer<InetAddress, Map<String, Object>> rowConsumer) {
291378
try (Reader reader = new Reader(pathToFile(databasePath))) {
292379
Networks<?> networks = reader.networks(Map.class);

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -361,7 +361,11 @@ public class MaxMindSupportTests extends ESTestCase {
361361

362362
private static final Set<Class<? extends AbstractResponse>> KNOWN_UNSUPPORTED_RESPONSE_CLASSES = Set.of(IpRiskResponse.class);
363363

364-
private static final Set<Database> KNOWN_UNSUPPORTED_DATABASE_VARIANTS = Set.of(Database.AsnV2, Database.CityV2);
364+
private static final Set<Database> KNOWN_UNSUPPORTED_DATABASE_VARIANTS = Set.of(
365+
Database.AsnV2,
366+
Database.CityV2,
367+
Database.PrivacyDetection
368+
);
365369

366370
public void testMaxMindSupport() {
367371
for (Database databaseType : Database.values()) {

‎modules/ingest-geoip/src/test/resources/ipinfo/privacy_detection_sample.mmdb‎

25.7 KB
Binary file not shown.

0 commit comments

Comments
 (0)