Skip to content

Commit e00aef1

Browse files
bhapasbhapas
committed
Implement MACAddress and IP types
1 parent 4e4bd11 commit e00aef1

File tree

2 files changed

+219
-78
lines changed

2 files changed

+219
-78
lines changed

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CefParser.java‎

Lines changed: 155 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,12 @@
88
*/
99
package org.elasticsearch.ingest.common;
1010

11+
import org.elasticsearch.common.network.NetworkAddress;
1112
import org.elasticsearch.common.time.DateFormatters;
1213
import org.elasticsearch.common.util.set.Sets;
1314

15+
import java.net.InetAddress;
16+
import java.net.UnknownHostException;
1417
import java.time.Instant;
1518
import java.time.LocalDate;
1619
import java.time.ZoneId;
@@ -62,6 +65,20 @@ final class CefParser {
6265
private static final Pattern EXTENSION_NEXT_KEY_VALUE_PATTERN = Pattern.compile(
6366
"(" + EXTENSION_KEY_PATTERN + ")=(" + EXTENSION_VALUE_PATTERN + ")(?:\\s+|$)"
6467
);
68+
69+
// Comprehensive regex pattern to match various MAC address formats
70+
public static final String MAC_ADDRESS_REGEX = "^(" +
71+
// Combined colon and hyphen separated 6-group patterns
72+
"(([0-9A-Fa-f]{2}[:|-]){5}[0-9A-Fa-f]{2})|" +
73+
// Dot-separated 6-group pattern
74+
"([0-9A-Fa-f]{4}\\.){2}[0-9A-Fa-f]{4}|" +
75+
// Combined colon and hyphen separated 8-group patterns
76+
"([0-9A-Fa-f]{2}[:|-]){7}[0-9A-Fa-f]{2}|" +
77+
// Dot-separated EUI-64
78+
"([0-9A-Fa-f]{4}\\.){3}[0-9A-Fa-f]{4}" + ")$";
79+
private static final int EUI48_HEX_LENGTH = 48 / 4;
80+
private static final int EUI64_HEX_LENGTH = 64 / 4;
81+
private static final int EUI64_HEX_WITH_SEPARATOR_MAX_LENGTH = EUI64_HEX_LENGTH + EUI64_HEX_LENGTH / 2 - 1;
6582
private static final Map<String, String> EXTENSION_VALUE_SANITIZER_REVERSE_MAPPING = Map.ofEntries(
6683
entry("\\\\", "\\"),
6784
entry("\\=", "="),
@@ -146,77 +163,77 @@ final class CefParser {
146163
Sets.union(FIELD_MAPPINGS.keySet(), Set.copyOf(FIELD_MAPPINGS.values()))
147164
);
148165

149-
private static final Map<String, Class<?>> FIELDS = Map.<String, Class<?>>ofEntries(
150-
entry("@timestamp", ZonedDateTime.class),
151-
entry("destination.bytes", Long.class),
152-
entry("destination.domain", String.class),
153-
entry("destination.geo.location.lat", Double.class),
154-
entry("destination.geo.location.lon", Double.class),
155-
entry("destination.ip", String.class),
156-
entry("destination.mac", String.class),
157-
entry("destination.port", Long.class),
158-
entry("destination.process.name", String.class),
159-
entry("destination.process.pid", Long.class),
160-
entry("destination.registered_domain", String.class),
161-
entry("destination.user.group.name", String.class),
162-
entry("destination.user.id", String.class),
163-
entry("destination.user.name", String.class),
164-
entry("device.event_class_id", String.class),
165-
entry("device.product", String.class),
166-
entry("device.vendor", String.class),
167-
entry("device.version", String.class),
168-
entry("event.action", String.class),
169-
entry("event.code", String.class),
170-
entry("event.end", ZonedDateTime.class),
171-
entry("event.id", String.class),
172-
entry("event.ingested", ZonedDateTime.class),
173-
entry("event.outcome", String.class),
174-
entry("event.reason", String.class),
175-
entry("event.start", ZonedDateTime.class),
176-
entry("event.timezone", String.class),
177-
entry("file.created", ZonedDateTime.class),
178-
entry("file.extension", String.class),
179-
entry("file.group", String.class),
180-
entry("file.hash", String.class),
181-
entry("file.inode", String.class),
182-
entry("file.mtime", ZonedDateTime.class),
183-
entry("file.name", String.class),
184-
entry("file.path", String.class),
185-
entry("file.size", Long.class),
186-
entry("host.nat.ip", String.class),
187-
entry("http.request.method", String.class),
188-
entry("http.request.referrer", String.class),
189-
entry("log.syslog.facility.code", Long.class),
190-
entry("message", String.class),
191-
entry("network.direction", String.class),
192-
entry("network.protocol", String.class),
193-
entry("network.transport", String.class),
194-
entry("observer.egress.interface.name", String.class),
195-
entry("observer.hostname", String.class),
196-
entry("observer.ingress.interface.name", String.class),
197-
entry("observer.ip", String.class),
198-
entry("observer.mac", String.class),
199-
entry("observer.name", String.class),
200-
entry("observer.registered_domain", String.class),
201-
entry("observer.version", String.class),
202-
entry("observer.vendor", String.class),
203-
entry("observer.product", String.class),
204-
entry("process.name", String.class),
205-
entry("process.pid", Long.class),
206-
entry("source.bytes", Long.class),
207-
entry("source.domain", String.class),
208-
entry("source.geo.location.lat", Double.class),
209-
entry("source.geo.location.lon", Double.class),
210-
entry("source.ip", String.class),
211-
entry("source.mac", String.class),
212-
entry("source.port", Long.class),
213-
entry("source.process.name", String.class),
214-
entry("source.process.pid", Long.class),
215-
entry("source.registered_domain", String.class),
216-
entry("source.service.name", String.class),
217-
entry("source.user.name", String.class),
218-
entry("url.original", String.class),
219-
entry("user_agent.original", String.class)
166+
private static final Map<String, DataType> FIELDS = Map.<String, DataType>ofEntries(
167+
entry("@timestamp", DataType.TimestampType),
168+
entry("destination.bytes", DataType.LongType),
169+
entry("destination.domain", DataType.StringType),
170+
entry("destination.geo.location.lat", DataType.DoubleType),
171+
entry("destination.geo.location.lon", DataType.DoubleType),
172+
entry("destination.ip", DataType.IPType),
173+
entry("destination.mac", DataType.MACAddressType),
174+
entry("destination.port", DataType.LongType),
175+
entry("destination.process.name", DataType.StringType),
176+
entry("destination.process.pid", DataType.LongType),
177+
entry("destination.registered_domain", DataType.StringType),
178+
entry("destination.user.group.name", DataType.StringType),
179+
entry("destination.user.id", DataType.StringType),
180+
entry("destination.user.name", DataType.StringType),
181+
entry("device.event_class_id", DataType.StringType),
182+
entry("device.product", DataType.StringType),
183+
entry("device.vendor", DataType.StringType),
184+
entry("device.version", DataType.StringType),
185+
entry("event.action", DataType.StringType),
186+
entry("event.code", DataType.StringType),
187+
entry("event.end", DataType.TimestampType),
188+
entry("event.id", DataType.StringType),
189+
entry("event.ingested", DataType.TimestampType),
190+
entry("event.outcome", DataType.StringType),
191+
entry("event.reason", DataType.StringType),
192+
entry("event.start", DataType.TimestampType),
193+
entry("event.timezone", DataType.StringType),
194+
entry("file.created", DataType.TimestampType),
195+
entry("file.extension", DataType.StringType),
196+
entry("file.group", DataType.StringType),
197+
entry("file.hash", DataType.StringType),
198+
entry("file.inode", DataType.StringType),
199+
entry("file.mtime", DataType.TimestampType),
200+
entry("file.name", DataType.StringType),
201+
entry("file.path", DataType.StringType),
202+
entry("file.size", DataType.LongType),
203+
entry("host.nat.ip", DataType.IPType),
204+
entry("http.request.method", DataType.StringType),
205+
entry("http.request.referrer", DataType.StringType),
206+
entry("log.syslog.facility.code", DataType.LongType),
207+
entry("message", DataType.StringType),
208+
entry("network.direction", DataType.StringType),
209+
entry("network.protocol", DataType.StringType),
210+
entry("network.transport", DataType.StringType),
211+
entry("observer.egress.interface.name", DataType.StringType),
212+
entry("observer.hostname", DataType.StringType),
213+
entry("observer.ingress.interface.name", DataType.StringType),
214+
entry("observer.ip", DataType.IPType),
215+
entry("observer.mac", DataType.MACAddressType),
216+
entry("observer.name", DataType.StringType),
217+
entry("observer.registered_domain", DataType.StringType),
218+
entry("observer.version", DataType.StringType),
219+
entry("observer.vendor", DataType.StringType),
220+
entry("observer.product", DataType.StringType),
221+
entry("process.name", DataType.StringType),
222+
entry("process.pid", DataType.LongType),
223+
entry("source.bytes", DataType.LongType),
224+
entry("source.domain", DataType.StringType),
225+
entry("source.geo.location.lat", DataType.DoubleType),
226+
entry("source.geo.location.lon", DataType.DoubleType),
227+
entry("source.ip", DataType.IPType),
228+
entry("source.mac", DataType.MACAddressType),
229+
entry("source.port", DataType.LongType),
230+
entry("source.process.name", DataType.StringType),
231+
entry("source.process.pid", DataType.LongType),
232+
entry("source.registered_domain", DataType.StringType),
233+
entry("source.service.name", DataType.StringType),
234+
entry("source.user.name", DataType.StringType),
235+
entry("url.original", DataType.StringType),
236+
entry("user_agent.original", DataType.StringType)
220237
);
221238

222239
private static final Set<String> ERROR_MESSAGE_INCOMPLETE_CEF_HEADER = Set.of("incomplete CEF header");
@@ -256,6 +273,18 @@ final class CefParser {
256273
MONTH_OF_YEAR
257274
);
258275

276+
private enum DataType {
277+
IntegerType,
278+
LongType,
279+
FloatType,
280+
DoubleType,
281+
StringType,
282+
BooleanType,
283+
IPType,
284+
MACAddressType,
285+
TimestampType;
286+
}
287+
259288
CEFEvent process(String cefString) {
260289
List<String> headers = new ArrayList<>();
261290
Matcher matcher = HEADER_NEXT_FIELD_PATTERN.matcher(cefString);
@@ -323,7 +352,7 @@ private void processExtensions(String cefString, int extensionStart, CEFEvent ev
323352
.stream()
324353
.filter(entry -> FIELD_MAPPINGS.containsKey(entry.getKey()))
325354
.collect(Collectors.toMap(entry -> FIELD_MAPPINGS.get(entry.getKey()), entry -> {
326-
Class<?> fieldType = FIELDS.get(FIELD_MAPPINGS.get(entry.getKey()));
355+
DataType fieldType = FIELDS.get(FIELD_MAPPINGS.get(entry.getKey()));
327356
return convertValueToType(entry.getValue(), fieldType);
328357
}));
329358
// Add ECS translations to the root of the document
@@ -361,17 +390,21 @@ private static Map<String, String> parseExtensions(String extensionString) {
361390
return extensions;
362391
}
363392

364-
private Object convertValueToType(String value, Class<?> type) {
365-
if (type == String.class) {
393+
private Object convertValueToType(String value, DataType type) {
394+
if (type == DataType.StringType) {
366395
return value;
367-
} else if (type == Long.class) {
396+
} else if (type == DataType.LongType) {
368397
return Long.parseLong(value);
369-
} else if (type == Double.class) {
398+
} else if (type == DataType.DoubleType) {
370399
return Double.parseDouble(value);
371-
} else if (type == Integer.class) {
400+
} else if (type == DataType.IntegerType) {
372401
return Integer.parseInt(value);
373-
} else if (type == ZonedDateTime.class) {
402+
} else if (type == DataType.TimestampType) {
374403
return toTimestamp(value);
404+
} else if (type == DataType.MACAddressType) {
405+
return toMACAddress(value);
406+
} else if (type == DataType.IPType) {
407+
return toIP(value);
375408
} else {
376409
throw new IllegalArgumentException("Unsupported type: " + type);
377410
}
@@ -416,6 +449,50 @@ ZonedDateTime toTimestamp(String value) {
416449
throw new IllegalArgumentException("Value is not a valid timestamp: " + value);
417450
}
418451

452+
String toMACAddress(String v) throws IllegalArgumentException {
453+
// Insert separators if necessary
454+
String macWithSeparators = insertMACSeparators(v);
455+
456+
// Validate MAC address format
457+
// Compiled pattern for efficient matching
458+
Pattern macAddressPattern = Pattern.compile(MAC_ADDRESS_REGEX);
459+
Matcher matcher = macAddressPattern.matcher(macWithSeparators);
460+
if (matcher.matches() == false) {
461+
throw new IllegalArgumentException("Invalid MAC address format");
462+
}
463+
// Convert to lowercase and return
464+
return macWithSeparators;
465+
}
466+
467+
String toIP(String v) {
468+
InetAddress address;
469+
try {
470+
address = InetAddress.getByName(v);
471+
} catch (UnknownHostException e) {
472+
throw new IllegalArgumentException("Invalid IP address format");
473+
}
474+
return NetworkAddress.format(address);
475+
}
476+
477+
static String insertMACSeparators(String v) {
478+
// Check that the length is correct for a MAC address without separators.
479+
// And check that there isn't already a separator in the string.
480+
if ((v.length() != EUI48_HEX_LENGTH && v.length() != EUI64_HEX_LENGTH)
481+
|| v.charAt(2) == ':'
482+
|| v.charAt(2) == '-'
483+
|| v.charAt(4) == '.') {
484+
return v;
485+
}
486+
StringBuilder sb = new StringBuilder(EUI64_HEX_WITH_SEPARATOR_MAX_LENGTH);
487+
for (int i = 0; i < v.length(); i++) {
488+
sb.append(v.charAt(i));
489+
if (i < v.length() - 1 && i % 2 != 0) {
490+
sb.append(':');
491+
}
492+
}
493+
return sb.toString();
494+
}
495+
419496
private static void removeEmptyValue(Map<String, String> map) {
420497
map.entrySet().removeIf(entry -> Objects.isNull(entry.getValue()) || entry.getValue().isEmpty());
421498
}

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CefProcessorTests.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import java.time.ZonedDateTime;
1919
import java.util.HashMap;
2020
import java.util.HashSet;
21+
import java.util.List;
2122
import java.util.Map;
2223
import java.util.Set;
2324

@@ -720,4 +721,67 @@ public void testToTimestampWithInvalidDate() {
720721
String invalidDate = "invalid date";
721722
assertThrows(IllegalArgumentException.class, () -> parser.toTimestamp(invalidDate));
722723
}
724+
725+
public void testToMacAddressWithSeparators() {
726+
CefParser parser = new CefParser(ZoneId.of("UTC"), false);
727+
List<String> macAddresses = List.of(
728+
// EUI-48 (with separators).
729+
"00:0D:60:AF:1B:61",
730+
"00-0D-60-AF-1B-61",
731+
"000D.60AF.1B61",
732+
733+
// EUI-64 (with separators).
734+
"00:0D:60:FF:FE:AF:1B:61",
735+
"00-0D-60-FF-FE-AF-1B-61",
736+
"000D.60FF.FEAF.1B61"
737+
);
738+
macAddresses.forEach(macAddress -> {
739+
String result = parser.toMACAddress(macAddress);
740+
assertEquals(macAddress, result);
741+
});
742+
}
743+
744+
public void testEUI48ToMacAddressWithOutSeparators() {
745+
CefParser parser = new CefParser(ZoneId.of("UTC"), false);
746+
String macAddress = "000D60AF1B61";
747+
String result = parser.toMACAddress(macAddress);
748+
assertEquals("00:0D:60:AF:1B:61", result);
749+
}
750+
751+
public void testEUI64ToMacAddressWithOutSeparators() {
752+
CefParser parser = new CefParser(ZoneId.of("UTC"), false);
753+
String macAddress = "000D60FFFEAF1B61";
754+
String result = parser.toMACAddress(macAddress);
755+
assertEquals("00:0D:60:FF:FE:AF:1B:61", result);
756+
}
757+
758+
public void toIP_validIPv4Address() {
759+
CefParser parser = new CefParser(ZoneId.of("UTC"), true);
760+
String result = parser.toIP("192.168.1.1");
761+
assertEquals("192.168.1.1", result);
762+
}
763+
764+
public void toIP_validIPv6Address() {
765+
CefParser parser = new CefParser(ZoneId.of("UTC"), true);
766+
String result = parser.toIP("2001:0db8:85a3:0000:0000:8a2e:0370:7334");
767+
assertEquals("2001:db8:85a3::8a2e:370:7334", result);
768+
}
769+
770+
public void toIP_invalidIPAddress() {
771+
CefParser parser = new CefParser(ZoneId.of("UTC"), true);
772+
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> { parser.toIP("invalid_ip"); });
773+
assertEquals("Invalid IP address format", exception.getMessage());
774+
}
775+
776+
public void toIP_emptyString() {
777+
CefParser parser = new CefParser(ZoneId.of("UTC"), true);
778+
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> { parser.toIP(""); });
779+
assertEquals("Invalid IP address format", exception.getMessage());
780+
}
781+
782+
public void toIP_nullString() {
783+
CefParser parser = new CefParser(ZoneId.of("UTC"), true);
784+
IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> { parser.toIP(null); });
785+
assertEquals("Invalid IP address format", exception.getMessage());
786+
}
723787
}

0 commit comments

Comments
 (0)