diff --git a/.editorconfig b/.editorconfig index 15f919078a15d..b48ae900ad7f7 100644 --- a/.editorconfig +++ b/.editorconfig @@ -231,3 +231,6 @@ indent_size = 3 [*.{csv,sql}-spec] trim_trailing_whitespace = false + +[*.cef.txt] +insert_final_newline = false diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CefParser.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CefParser.java new file mode 100644 index 0000000000000..d1065b5e58bbe --- /dev/null +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CefParser.java @@ -0,0 +1,619 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.ingest.common; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.time.DateFormatters; +import org.elasticsearch.core.Nullable; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.WeekFields; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.regex.MatchResult; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static java.time.temporal.ChronoField.DAY_OF_MONTH; +import static java.time.temporal.ChronoField.HOUR_OF_DAY; +import static java.time.temporal.ChronoField.MINUTE_OF_DAY; +import static java.time.temporal.ChronoField.MONTH_OF_YEAR; +import static java.time.temporal.ChronoField.NANO_OF_SECOND; +import static java.time.temporal.ChronoField.SECOND_OF_DAY; +import static java.util.Map.entry; +import static org.elasticsearch.ingest.common.CefParser.DataType.DoubleType; +import static org.elasticsearch.ingest.common.CefParser.DataType.IPType; +import static org.elasticsearch.ingest.common.CefParser.DataType.IntegerType; +import static org.elasticsearch.ingest.common.CefParser.DataType.LongType; +import static org.elasticsearch.ingest.common.CefParser.DataType.MACAddressType; +import static org.elasticsearch.ingest.common.CefParser.DataType.StringType; +import static org.elasticsearch.ingest.common.CefParser.DataType.TimestampType; + +final class CefParser { + private final boolean removeEmptyValues; + private final ZoneId timezone; + + CefParser(ZoneId timezone, boolean removeEmptyValues) { + this.removeEmptyValues = removeEmptyValues; + this.timezone = timezone; + } + + // New patterns for extension parsing + private static final String EXTENSION_KEY_PATTERN = "(?:[\\w-]+(?:\\.[^\\.=\\s\\|\\\\\\[\\]]+)*(?:\\[[0-9]+\\])?(?==))"; + private static final String EXTENSION_VALUE_PATTERN = "(?:[^\\s\\\\]|\\\\[^|]|\\s(?!" + EXTENSION_KEY_PATTERN + "=))*"; + private static final Pattern EXTENSION_NEXT_KEY_VALUE_PATTERN = Pattern.compile( + "(" + EXTENSION_KEY_PATTERN + ")=(" + EXTENSION_VALUE_PATTERN + ")(?:\\s+|$)" + ); + + // Comprehensive regex pattern to match various MAC address formats + private static final Pattern MAC_ADDRESS_PATTERN = Pattern.compile( + String.join( + "|", + // Combined colon and hyphen separated 6-group patterns + "(?:[0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}", + // Dot-separated 6-group pattern + "(?:[0-9A-Fa-f]{4}\\.){2}[0-9A-Fa-f]{4}", + // Combined colon and hyphen separated 8-group patterns + "(?:[0-9A-Fa-f]{2}[:-]){7}[0-9A-Fa-f]{2}", + // Dot-separated EUI-64 + "(?:[0-9A-Fa-f]{4}\\.){3}[0-9A-Fa-f]{4}" + ) + ); + private static final int EUI48_HEX_LENGTH = 48 / 4; + private static final int EUI64_HEX_LENGTH = 64 / 4; + private static final int EUI64_HEX_WITH_SEPARATOR_MAX_LENGTH = EUI64_HEX_LENGTH + EUI64_HEX_LENGTH / 2 - 1; + private static final Map EXTENSION_VALUE_SANITIZER_REVERSE_MAPPING = Map.ofEntries( + entry("\\=", "="), + entry("\\n", "\n"), + entry("\\t", "\t"), + entry("\\r", "\r"), + entry("\\\\", "\\") + ); + + private static final Map EXTENSION_MAPPINGS = Map.ofEntries( + entry("agt", new ExtensionMapping("agentAddress", IPType, "agent.ip")), + entry("agentDnsDomain", new ExtensionMapping("agentDnsDomain", StringType, "agent.name")), + entry("ahost", new ExtensionMapping("agentHostName", StringType, "agent.name")), + entry("aid", new ExtensionMapping("agentId", StringType, "agent.id")), + entry("amac", new ExtensionMapping("agentMacAddress", MACAddressType, "agent.mac")), + entry("agentNtDomain", new ExtensionMapping("agentNtDomain", StringType, null)), + entry("art", new ExtensionMapping("agentReceiptTime", TimestampType, "event.created")), + entry("atz", new ExtensionMapping("agentTimeZone", StringType, null)), + entry("agentTranslatedAddress", new ExtensionMapping("agentTranslatedAddress", IPType, null)), + entry("agentTranslatedZoneExternalID", new ExtensionMapping("agentTranslatedZoneExternalID", StringType, null)), + entry("agentTranslatedZoneURI", new ExtensionMapping("agentTranslatedZoneURI", StringType, null)), + entry("at", new ExtensionMapping("agentType", StringType, "agent.type")), + entry("av", new ExtensionMapping("agentVersion", StringType, "agent.version")), + entry("agentZoneExternalID", new ExtensionMapping("agentZoneExternalID", StringType, null)), + entry("agentZoneURI", new ExtensionMapping("agentZoneURI", StringType, null)), + entry("app", new ExtensionMapping("applicationProtocol", StringType, "network.protocol")), + entry("cnt", new ExtensionMapping("baseEventCount", IntegerType, null)), + entry("in", new ExtensionMapping("bytesIn", LongType, "source.bytes")), // LongType from Spec 1.x + entry("out", new ExtensionMapping("bytesOut", LongType, "destination.bytes")), // LongType from Spec 1.x + entry("customerExternalID", new ExtensionMapping("customerExternalID", StringType, "organization.id")), + entry("customerURI", new ExtensionMapping("customerURI", StringType, "organization.name")), + entry("dst", new ExtensionMapping("destinationAddress", IPType, "destination.ip")), + entry("destinationDnsDomain", new ExtensionMapping("destinationDnsDomain", StringType, "destination.registered_domain")), + entry("dlat", new ExtensionMapping("destinationGeoLatitude", DoubleType, "destination.geo.location.lat")), + entry("dlong", new ExtensionMapping("destinationGeoLongitude", DoubleType, "destination.geo.location.lon")), + entry("dhost", new ExtensionMapping("destinationHostName", StringType, "destination.domain")), + entry("dmac", new ExtensionMapping("destinationMacAddress", MACAddressType, "destination.mac")), + entry("dntdom", new ExtensionMapping("destinationNtDomain", StringType, "destination.registered_domain")), + entry("dpt", new ExtensionMapping("destinationPort", IntegerType, "destination.port")), + entry("dpid", new ExtensionMapping("destinationProcessId", IntegerType, "destination.process.pid")), + entry("dproc", new ExtensionMapping("destinationProcessName", StringType, "destination.process.name")), + entry("destinationServiceName", new ExtensionMapping("destinationServiceName", StringType, "destination.service.name")), + entry("destinationTranslatedAddress", new ExtensionMapping("destinationTranslatedAddress", IPType, "destination.nat.ip")), + entry("destinationTranslatedPort", new ExtensionMapping("destinationTranslatedPort", IntegerType, "destination.nat.port")), + entry("destinationTranslatedZoneExternalID", new ExtensionMapping("destinationTranslatedZoneExternalID", StringType, null)), + entry("destinationTranslatedZoneURI", new ExtensionMapping("destinationTranslatedZoneURI", StringType, null)), + entry("duid", new ExtensionMapping("destinationUserId", StringType, "destination.user.id")), + entry("duser", new ExtensionMapping("destinationUserName", StringType, "destination.user.name")), + entry("dpriv", new ExtensionMapping("destinationUserPrivileges", StringType, "destination.user.group.name")), + entry("destinationZoneExternalID", new ExtensionMapping("destinationZoneExternalID", StringType, null)), + entry("destinationZoneURI", new ExtensionMapping("destinationZoneURI", StringType, null)), + entry("act", new ExtensionMapping("deviceAction", StringType, "event.action")), + entry("dvc", new ExtensionMapping("deviceAddress", IPType, "observer.ip")), + entry("cfp1Label", new ExtensionMapping("deviceCustomFloatingPoint1Label", StringType, null)), + entry("cfp3Label", new ExtensionMapping("deviceCustomFloatingPoint3Label", StringType, null)), + entry("cfp4Label", new ExtensionMapping("deviceCustomFloatingPoint4Label", StringType, null)), + entry("deviceCustomDate1", new ExtensionMapping("deviceCustomDate1", TimestampType, null)), + entry("deviceCustomDate1Label", new ExtensionMapping("deviceCustomDate1Label", StringType, null)), + entry("deviceCustomDate2", new ExtensionMapping("deviceCustomDate2", TimestampType, null)), + entry("deviceCustomDate2Label", new ExtensionMapping("deviceCustomDate2Label", StringType, null)), + entry("cfp1", new ExtensionMapping("deviceCustomFloatingPoint1", DoubleType, null)), + entry("cfp2", new ExtensionMapping("deviceCustomFloatingPoint2", DoubleType, null)), + entry("cfp2Label", new ExtensionMapping("deviceCustomFloatingPoint2Label", StringType, null)), + entry("cfp3", new ExtensionMapping("deviceCustomFloatingPoint3", DoubleType, null)), + entry("cfp4", new ExtensionMapping("deviceCustomFloatingPoint4", DoubleType, null)), + entry("c6a1", new ExtensionMapping("deviceCustomIPv6Address1", IPType, null)), + entry("c6a1Label", new ExtensionMapping("deviceCustomIPv6Address1Label", StringType, null)), + entry("c6a2", new ExtensionMapping("deviceCustomIPv6Address2", IPType, null)), + entry("c6a2Label", new ExtensionMapping("deviceCustomIPv6Address2Label", StringType, null)), + entry("c6a3", new ExtensionMapping("deviceCustomIPv6Address3", IPType, null)), + entry("c6a3Label", new ExtensionMapping("deviceCustomIPv6Address3Label", StringType, null)), + entry("c6a4", new ExtensionMapping("deviceCustomIPv6Address4", IPType, null)), + entry("c6a4Label", new ExtensionMapping("deviceCustomIPv6Address4Label", StringType, null)), + entry("cn1", new ExtensionMapping("deviceCustomNumber1", LongType, null)), + entry("cn1Label", new ExtensionMapping("deviceCustomNumber1Label", StringType, null)), + entry("cn2", new ExtensionMapping("deviceCustomNumber2", LongType, null)), + entry("cn2Label", new ExtensionMapping("deviceCustomNumber2Label", StringType, null)), + entry("cn3", new ExtensionMapping("deviceCustomNumber3", LongType, null)), + entry("cn3Label", new ExtensionMapping("deviceCustomNumber3Label", StringType, null)), + entry("cs1", new ExtensionMapping("deviceCustomString1", StringType, null)), + entry("cs1Label", new ExtensionMapping("deviceCustomString1Label", StringType, null)), + entry("cs2", new ExtensionMapping("deviceCustomString2", StringType, null)), + entry("cs2Label", new ExtensionMapping("deviceCustomString2Label", StringType, null)), + entry("cs3", new ExtensionMapping("deviceCustomString3", StringType, null)), + entry("cs3Label", new ExtensionMapping("deviceCustomString3Label", StringType, null)), + entry("cs4", new ExtensionMapping("deviceCustomString4", StringType, null)), + entry("cs4Label", new ExtensionMapping("deviceCustomString4Label", StringType, null)), + entry("cs5", new ExtensionMapping("deviceCustomString5", StringType, null)), + entry("cs5Label", new ExtensionMapping("deviceCustomString5Label", StringType, null)), + entry("cs6", new ExtensionMapping("deviceCustomString6", StringType, null)), + entry("cs6Label", new ExtensionMapping("deviceCustomString6Label", StringType, null)), + entry("deviceDirection", new ExtensionMapping("deviceDirection", StringType, "network.direction")), + entry("deviceDnsDomain", new ExtensionMapping("deviceDnsDomain", StringType, "observer.registered_domain")), + entry("cat", new ExtensionMapping("deviceEventCategory", StringType, null)), + entry("deviceExternalId", new ExtensionMapping("deviceExternalId", StringType, "observer.name")), + entry("deviceFacility", new ExtensionMapping("deviceFacility", StringType, null)), + entry("dvchost", new ExtensionMapping("deviceHostName", StringType, "observer.hostname")), + entry("deviceInboundInterface", new ExtensionMapping("deviceInboundInterface", StringType, "observer.ingress.interface.name")), + entry("dvcmac", new ExtensionMapping("deviceMacAddress", MACAddressType, "observer.mac")), + entry("deviceNtDomain", new ExtensionMapping("deviceNtDomain", StringType, null)), + entry("deviceOutboundInterface", new ExtensionMapping("deviceOutboundInterface", StringType, "observer.egress.interface.name")), + entry("devicePayloadId", new ExtensionMapping("devicePayloadId", StringType, "event.id")), + entry("dvcpid", new ExtensionMapping("deviceProcessId", IntegerType, "process.pid")), + entry("deviceProcessName", new ExtensionMapping("deviceProcessName", StringType, "process.name")), + entry("rt", new ExtensionMapping("deviceReceiptTime", TimestampType, "@timestamp")), + entry("dtz", new ExtensionMapping("deviceTimeZone", StringType, "event.timezone")), + entry("deviceTranslatedAddress", new ExtensionMapping("deviceTranslatedAddress", IPType, "host.nat.ip")), + entry("deviceTranslatedZoneExternalID", new ExtensionMapping("deviceTranslatedZoneExternalID", StringType, null)), + entry("deviceTranslatedZoneURI", new ExtensionMapping("deviceTranslatedZoneURI", StringType, null)), + entry("deviceZoneExternalID", new ExtensionMapping("deviceZoneExternalID", StringType, null)), + entry("deviceZoneURI", new ExtensionMapping("deviceZoneURI", StringType, null)), + entry("end", new ExtensionMapping("endTime", TimestampType, "event.end")), + entry("eventId", new ExtensionMapping("eventId", StringType, "event.id")), + entry("outcome", new ExtensionMapping("eventOutcome", StringType, "event.outcome")), + entry("externalId", new ExtensionMapping("externalId", StringType, null)), + entry("fileCreateTime", new ExtensionMapping("fileCreateTime", TimestampType, "file.created")), + entry("fileHash", new ExtensionMapping("fileHash", StringType, "file.hash")), + entry("fileId", new ExtensionMapping("fileId", StringType, "file.inode")), + entry("fileModificationTime", new ExtensionMapping("fileModificationTime", TimestampType, "file.mtime")), + entry("flexNumber1", new ExtensionMapping("deviceFlexNumber1", LongType, null)), + entry("flexNumber1Label", new ExtensionMapping("deviceFlexNumber1Label", StringType, null)), + entry("flexNumber2", new ExtensionMapping("deviceFlexNumber2", LongType, null)), + entry("flexNumber2Label", new ExtensionMapping("deviceFlexNumber2Label", StringType, null)), + entry("fname", new ExtensionMapping("filename", StringType, "file.name")), + entry("filePath", new ExtensionMapping("filePath", StringType, "file.path")), + entry("filePermission", new ExtensionMapping("filePermission", StringType, "file.group")), + entry("fsize", new ExtensionMapping("fileSize", LongType, "file.size")), + entry("fileType", new ExtensionMapping("fileType", StringType, "file.type")), + entry("flexDate1", new ExtensionMapping("flexDate1", TimestampType, null)), + entry("flexDate1Label", new ExtensionMapping("flexDate1Label", StringType, null)), + entry("flexString1", new ExtensionMapping("flexString1", StringType, null)), + entry("flexString2", new ExtensionMapping("flexString2", StringType, null)), + entry("flexString1Label", new ExtensionMapping("flexString1Label", StringType, null)), + entry("flexString2Label", new ExtensionMapping("flexString2Label", StringType, null)), + entry("msg", new ExtensionMapping("message", StringType, "message")), + entry("oldFileCreateTime", new ExtensionMapping("oldFileCreateTime", TimestampType, null)), + entry("oldFileHash", new ExtensionMapping("oldFileHash", StringType, null)), + entry("oldFileId", new ExtensionMapping("oldFileId", StringType, null)), + entry("oldFileModificationTime", new ExtensionMapping("oldFileModificationTime", TimestampType, null)), + entry("oldFileName", new ExtensionMapping("oldFileName", StringType, null)), + entry("oldFilePath", new ExtensionMapping("oldFilePath", StringType, null)), + entry("oldFilePermission", new ExtensionMapping("oldFilePermission", StringType, null)), + entry("oldFileSize", new ExtensionMapping("oldFileSize", LongType, null)), + entry("oldFileType", new ExtensionMapping("oldFileType", StringType, null)), + entry("rawEvent", new ExtensionMapping("rawEvent", StringType, "event.original")), + entry("reason", new ExtensionMapping("reason", StringType, "event.reason")), + entry("requestClientApplication", new ExtensionMapping("requestClientApplication", StringType, "user_agent.original")), + entry("requestContext", new ExtensionMapping("requestContext", StringType, "http.request.referrer")), + entry("requestCookies", new ExtensionMapping("requestCookies", StringType, null)), + entry("requestMethod", new ExtensionMapping("requestMethod", StringType, "http.request.method")), + entry("request", new ExtensionMapping("requestUrl", StringType, "url.original")), + entry("src", new ExtensionMapping("sourceAddress", IPType, "source.ip")), + entry("sourceDnsDomain", new ExtensionMapping("sourceDnsDomain", StringType, "source.domain")), + entry("slat", new ExtensionMapping("sourceGeoLatitude", DoubleType, "source.geo.location.lat")), + entry("slong", new ExtensionMapping("sourceGeoLongitude", DoubleType, "source.geo.location.lon")), + entry("shost", new ExtensionMapping("sourceHostName", StringType, "source.domain")), + entry("smac", new ExtensionMapping("sourceMacAddress", MACAddressType, "source.mac")), + entry("sntdom", new ExtensionMapping("sourceNtDomain", StringType, "source.registered_domain")), + entry("spt", new ExtensionMapping("sourcePort", IntegerType, "source.port")), + entry("spid", new ExtensionMapping("sourceProcessId", IntegerType, "source.process.pid")), + entry("sproc", new ExtensionMapping("sourceProcessName", StringType, "source.process.name")), + entry("sourceServiceName", new ExtensionMapping("sourceServiceName", StringType, "source.service.name")), + entry("sourceTranslatedAddress", new ExtensionMapping("sourceTranslatedAddress", IPType, "source.nat.ip")), + entry("sourceTranslatedPort", new ExtensionMapping("sourceTranslatedPort", IntegerType, "source.nat.port")), + entry("sourceTranslatedZoneExternalID", new ExtensionMapping("sourceTranslatedZoneExternalID", StringType, null)), + entry("sourceTranslatedZoneURI", new ExtensionMapping("sourceTranslatedZoneURI", StringType, null)), + entry("suid", new ExtensionMapping("sourceUserId", StringType, "source.user.id")), + entry("suser", new ExtensionMapping("sourceUserName", StringType, "source.user.name")), + entry("spriv", new ExtensionMapping("sourceUserPrivileges", StringType, "source.user.group.name")), + entry("sourceZoneExternalID", new ExtensionMapping("sourceZoneExternalID", StringType, null)), + entry("sourceZoneURI", new ExtensionMapping("sourceZoneURI", StringType, null)), + entry("start", new ExtensionMapping("startTime", TimestampType, "event.start")), + entry("proto", new ExtensionMapping("transportProtocol", StringType, "network.transport")), + entry("type", new ExtensionMapping("type", StringType, "event.kind")), + entry("catdt", new ExtensionMapping("categoryDeviceType", StringType, null)), + entry("mrt", new ExtensionMapping("managerReceiptTime", TimestampType, "event.ingested")), + // CEF Spec version 1.2 + entry("agentTranslatedZoneKey", new ExtensionMapping("agentTranslatedZoneKey", LongType, null)), + entry("agentZoneKey", new ExtensionMapping("agentZoneKey", LongType, null)), + entry("customerKey", new ExtensionMapping("customerKey", LongType, null)), + entry("destinationTranslatedZoneKey", new ExtensionMapping("destinationTranslatedZoneKey", LongType, null)), + entry("dZoneKey", new ExtensionMapping("destinationZoneKey", LongType, null)), + entry("deviceTranslatedZoneKey", new ExtensionMapping("deviceTranslatedZoneKey", LongType, null)), + entry("deviceZoneKey", new ExtensionMapping("deviceZoneKey", LongType, null)), + entry("sTranslatedZoneKey", new ExtensionMapping("sourceTranslatedZoneKey", LongType, null)), + entry("sZoneKey", new ExtensionMapping("sourceZoneKey", LongType, null)), + entry("parserVersion", new ExtensionMapping("parserVersion", StringType, null)), + entry("parserIdentifier", new ExtensionMapping("parserIdentifier", StringType, null)) + ); + + private static final String INCOMPLETE_CEF_HEADER = "Incomplete CEF header"; + private static final String INVALID_CEF_FORMAT = "Invalid CEF format"; + private static final String UNESCAPED_EQUALS_SIGN = "CEF extensions contain unescaped equals sign"; + + /** + * List of allowed timestamp formats for CEF spec v27, see: Appendix A: Date Formats + * documentation + */ + private static final List TIME_FORMATS = Stream.of( + "MMM dd HH:mm:ss.SSS zzz", + "MMM dd HH:mm:ss.SSS", + "MMM dd HH:mm:ss zzz", + "MMM dd HH:mm:ss", + "MMM dd yyyy HH:mm:ss.SSS zzz", + "MMM dd yyyy HH:mm:ss.SSS", + "MMM dd yyyy HH:mm:ss zzz", + "MMM dd yyyy HH:mm:ss" + ).map(p -> DateTimeFormatter.ofPattern(p, Locale.ROOT)).toList(); + + private static final List CHRONO_FIELDS = List.of( + NANO_OF_SECOND, + SECOND_OF_DAY, + MINUTE_OF_DAY, + HOUR_OF_DAY, + DAY_OF_MONTH, + MONTH_OF_YEAR + ); + + CefEvent process(String cefString) { + List headers = parseHeaders(cefString); + // the last 'header' is the not-yet-parsed extension string, remove and then parse it + Map parsedExtensions = parseExtensions(headers.removeLast()); + CefEvent event = new CefEvent(); + processHeaders(headers, event); + processExtensions(parsedExtensions, event); + return event; + } + + // visible for testing + static List parseHeaders(String cefString) { + List headers = new ArrayList<>(); + int extensionStart = -1; + StringBuilder buffer = new StringBuilder(); + for (int i = 0; i < cefString.length(); i++) { + char curr = cefString.charAt(i); + char next = i < cefString.length() - 1 ? cefString.charAt(i + 1) : '\0'; + if (curr == '\\' && next == '\\') { // an escaped backslash + buffer.append('\\'); // emit a backslash + i++; // and skip the next character + } else if (curr == '\\' && next == '|') { // an escaped pipe + buffer.append('|'); // emit a pipe + i++; // and skip the next character + } else if (curr == '|') { // a pipe, it's the end of a header + headers.add(buffer.toString()); // emit the header + buffer = new StringBuilder(); // and reset the buffer + if (headers.size() == 7) { + extensionStart = i + 1; // the extensions begin after this pipe + break; // we've processed all the headers, so exit the loop + } + } else { // any other character + buffer.append(curr); // is just added to the header + } + } + + if (headers.isEmpty() || headers.getFirst().startsWith("CEF:") == false) { + throw new IllegalArgumentException(INVALID_CEF_FORMAT); + } + + if (headers.size() != 7) { + throw new IllegalArgumentException(INCOMPLETE_CEF_HEADER); + } + + // for simplicity of the interface, pack the unparsed extension string itself into the returned list of headers + String extensionString = cefString.substring(extensionStart); + headers.add(extensionString); + + return headers; + } + + private static void processHeaders(List headers, CefEvent event) { + for (int i = 0; i < headers.size(); i++) { + final String value = headers.get(i); + switch (i) { + case 0 -> event.addCefMapping("version", value.substring(4)); + case 1 -> { + event.addCefMapping("device.vendor", value); + event.addRootMapping("observer.vendor", value); + } + case 2 -> { + event.addCefMapping("device.product", value); + event.addRootMapping("observer.product", value); + } + case 3 -> { + event.addCefMapping("device.version", value); + event.addRootMapping("observer.version", value); + } + case 4 -> { + event.addCefMapping("device.event_class_id", value); + event.addRootMapping("event.code", value); + } + case 5 -> event.addCefMapping("name", value); + case 6 -> event.addCefMapping("severity", value); + default -> throw new IllegalArgumentException(INVALID_CEF_FORMAT); + } + } + } + + // visible for testing + static Map parseExtensions(String extensionString) { + Map extensions = new HashMap<>(); + Matcher matcher = EXTENSION_NEXT_KEY_VALUE_PATTERN.matcher(extensionString); + int lastEnd = 0; + + List allMatches = new ArrayList<>(); + while (matcher.find()) { + allMatches.add(matcher.toMatchResult()); + } + for (int i = 0; i < allMatches.size(); i++) { + MatchResult match = allMatches.get(i); + String key = match.group(1); + String value = match.group(2); + if (hasUnescapedEquals(value)) { + throw new IllegalArgumentException(UNESCAPED_EQUALS_SIGN); + } + // desanitize the value + value = unescapeExtensionValue(value); + // Only trim the last value + // Trimming after desanitization to unescape any trailing newline ,tab characters + if (i == allMatches.size() - 1) { + value = value.trim(); + } + extensions.put(key, value); + lastEnd = match.end(); + } + // If there's any remaining unparsed content, throw an exception + if (lastEnd < extensionString.length()) { + throw new IllegalArgumentException("Invalid extensions in the CEF event: " + extensionString.substring(lastEnd)); + } + return extensions; + } + + private void processExtensions(Map parsedExtensions, CefEvent event) { + // Cleanup empty values in extensions + if (removeEmptyValues) { + removeEmptyValues(parsedExtensions); + } + // Translate extensions to possible ECS fields + for (Map.Entry entry : parsedExtensions.entrySet()) { + ExtensionMapping mapping = EXTENSION_MAPPINGS.get(entry.getKey()); + if (mapping != null) { + String ecsKey = mapping.ecsKey(); + if (ecsKey != null) { + // Add the ECS translation to the root of document + event.addRootMapping(ecsKey, convertValueToType(entry.getValue(), mapping.dataType())); + } else { + // Add the extension to the CEF mappings if it doesn't have an ECS translation + event.addCefMapping("extensions." + mapping.key(), convertValueToType(entry.getValue(), mapping.dataType())); + } + } else { + // Add the extension if the key is not in the mapping + event.addCefMapping("extensions." + entry.getKey(), entry.getValue()); + } + } + } + + private static boolean hasUnescapedEquals(String value) { + if (value == null || value.isEmpty()) { + return false; // Empty or null strings have no unescaped equals signs + } + + // If there are no equals signs at all, return false + if (value.indexOf('=') < 0) { + return false; + } + + boolean escaped = true; + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + + if (escaped == false) { + escaped = true; // Reset escape flag after processing an escaped character + } else if (c == '\\') { + escaped = false; // Set escape flag when a backslash is encountered + } else if (c == '=') { + return true; // Found an unescaped equals sign, so return immediately + } + } + // If we get here without finding an unescaped equals sign, return false + return false; + } + + private Object convertValueToType(String value, DataType type) { + return switch (type) { + case StringType -> value; + case IntegerType -> Integer.parseInt(value); + case LongType -> Long.parseLong(value); + case DoubleType -> Double.parseDouble(value); + case TimestampType -> toTimestamp(value); + case MACAddressType -> toMACAddress(value); + case IPType -> toIP(value); + }; + } + + // visible for testing + ZonedDateTime toTimestamp(String value) { + // First, try parsing as milliseconds + try { + long milliseconds = Long.parseLong(value); + return Instant.ofEpochMilli(milliseconds).atZone(timezone); + } catch (NumberFormatException ignored) { + // Not a millisecond timestamp, continue to format parsing + } + // Try parsing with different layouts + for (DateTimeFormatter formatter : TIME_FORMATS) { + try { + TemporalAccessor accessor = formatter.parse(value); + // if there is no year nor year-of-era, we fall back to the current one and + // fill the rest of the date up with the parsed date + if (accessor.isSupported(ChronoField.YEAR) == false + && accessor.isSupported(ChronoField.YEAR_OF_ERA) == false + && accessor.isSupported(WeekFields.ISO.weekBasedYear()) == false + && accessor.isSupported(WeekFields.of(Locale.ROOT).weekBasedYear()) == false + && accessor.isSupported(ChronoField.INSTANT_SECONDS) == false) { + int year = LocalDate.now(ZoneOffset.UTC).getYear(); + ZonedDateTime newTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year); + for (ChronoField field : CHRONO_FIELDS) { + if (accessor.isSupported(field)) { + newTime = newTime.with(field, accessor.get(field)); + } + } + accessor = newTime.withZoneSameLocal(timezone); + } + return DateFormatters.from(accessor, Locale.ROOT, timezone).withZoneSameInstant(timezone); + } catch (DateTimeParseException ignored) { + // Try next layout + } + } + // If no layout matches, throw an exception + throw new IllegalArgumentException("Value is not a valid timestamp: " + value); + } + + // visible for testing + String toMACAddress(String v) throws IllegalArgumentException { + // Insert separators if necessary + String macWithSeparators = insertMACSeparators(v); + // Validate MAC address format + Matcher matcher = MAC_ADDRESS_PATTERN.matcher(macWithSeparators); + if (matcher.matches() == false) { + throw new IllegalArgumentException("Invalid MAC address format"); + } + return macWithSeparators; + } + + // visible for testing + String toIP(String v) { + try { + return NetworkAddress.format(InetAddresses.forString(v)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid IP address format", e); + } + } + + private static String insertMACSeparators(String v) { + // Check that the length is correct for a MAC address without separators. + // And check that there isn't already a separator in the string. + if ((v.length() != EUI48_HEX_LENGTH && v.length() != EUI64_HEX_LENGTH) + || v.charAt(2) == ':' + || v.charAt(2) == '-' + || v.charAt(4) == '.') { + return v; + } + StringBuilder sb = new StringBuilder(EUI64_HEX_WITH_SEPARATOR_MAX_LENGTH); + for (int i = 0; i < v.length(); i++) { + sb.append(v.charAt(i)); + if (i < v.length() - 1 && i % 2 != 0) { + sb.append(':'); + } + } + return sb.toString(); + } + + private static void removeEmptyValues(Map map) { + map.values().removeIf(Strings::isEmpty); + } + + private static String unescapeExtensionValue(String value) { + String unescaped = value; + // Protect escaped backslashes + unescaped = unescaped.replace("\\\\", "\u0000"); // Use null char as placeholder + for (Map.Entry entry : EXTENSION_VALUE_SANITIZER_REVERSE_MAPPING.entrySet()) { + unescaped = unescaped.replace(entry.getKey(), entry.getValue()); + } + // Restore single backslashes + unescaped = unescaped.replace("\u0000", "\\"); + return unescaped; + } + + static class CefEvent implements AutoCloseable { + private Map rootMappings = new HashMap<>(); + private Map cefMappings = new HashMap<>(); + + void addRootMapping(String key, Object value) { + this.rootMappings.put(key, value); + } + + void addCefMapping(String key, Object value) { + this.cefMappings.put(key, value); + } + + Map getRootMappings() { + return Objects.requireNonNull(rootMappings); + } + + Map getCefMappings() { + return Objects.requireNonNull(cefMappings); + } + + /** + * Nulls out the maps of the event so that future calls to methods of this class will fail with a + * {@link NullPointerException}. + */ + @Override + public void close() { + this.rootMappings = null; + this.cefMappings = null; + } + } + + enum DataType { + IntegerType, + LongType, + DoubleType, + StringType, + IPType, + MACAddressType, + TimestampType + } + + private record ExtensionMapping(String key, DataType dataType, @Nullable String ecsKey) { + ExtensionMapping { + Objects.requireNonNull(key); + Objects.requireNonNull(dataType); + } + } +} diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CefProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CefProcessor.java new file mode 100644 index 0000000000000..c49b0e11f1339 --- /dev/null +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CefProcessor.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.ingest.common; + +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.ingest.AbstractProcessor; +import org.elasticsearch.ingest.ConfigurationUtils; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; +import org.elasticsearch.ingest.common.CefParser.CefEvent; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateScript; + +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.Map; + +public final class CefProcessor extends AbstractProcessor { + + public static final String TYPE = "cef"; + + // visible for testing + final String field; + final String targetField; + final boolean ignoreMissing; + final boolean ignoreEmptyValues; + private final TemplateScript.Factory timezone; + + CefProcessor( + String tag, + String description, + String field, + String targetField, + boolean ignoreMissing, + boolean ignoreEmptyValues, + @Nullable TemplateScript.Factory timezone + ) { + super(tag, description); + this.field = field; + this.targetField = targetField; + this.ignoreMissing = ignoreMissing; + this.ignoreEmptyValues = ignoreEmptyValues; + this.timezone = timezone; + } + + @Override + public IngestDocument execute(IngestDocument document) { + String line = document.getFieldValue(field, String.class, ignoreMissing); + if (line == null && ignoreMissing) { + return document; + } else if (line == null) { + throw new IllegalArgumentException("field [" + field + "] is null, cannot process it."); + } + ZoneId timezone = getTimezone(document); + try (CefEvent event = new CefParser(timezone, ignoreEmptyValues).process(line)) { + event.getRootMappings().forEach(document::setFieldValue); + event.getCefMappings().forEach((k, v) -> document.setFieldValue(targetField + "." + k, v)); + } + return document; + } + + @Override + public String getType() { + return TYPE; + } + + ZoneId getTimezone(IngestDocument document) { + String value = timezone == null ? null : document.renderTemplate(timezone); + if (value == null) { + return ZoneOffset.UTC; + } else { + return ZoneId.of(value); + } + } + + public static final class Factory implements Processor.Factory { + + private final ScriptService scriptService; + + public Factory(ScriptService scriptService) { + this.scriptService = scriptService; + } + + @Override + public CefProcessor create( + Map registry, + String tag, + String description, + Map config, + ProjectId projectId + ) { + String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field"); + String targetField = ConfigurationUtils.readStringProperty(TYPE, tag, config, "target_field", "cef"); + boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, tag, config, "ignore_missing", false); + boolean ignoreEmptyValues = ConfigurationUtils.readBooleanProperty(TYPE, tag, config, "ignore_empty_values", true); + String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "timezone"); + TemplateScript.Factory compiledTimezoneTemplate = null; + if (timezoneString != null) { + compiledTimezoneTemplate = ConfigurationUtils.compileTemplate(TYPE, tag, "timezone", timezoneString, scriptService); + } + + return new CefProcessor(tag, description, field, targetField, ignoreMissing, ignoreEmptyValues, compiledTimezoneTemplate); + } + } +} diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index 31f7034c2fd88..457894267ab9c 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -43,6 +43,7 @@ public Map getProcessors(Processor.Parameters paramet return Map.ofEntries( entry(AppendProcessor.TYPE, new AppendProcessor.Factory(parameters.scriptService)), entry(BytesProcessor.TYPE, new BytesProcessor.Factory()), + entry(CefProcessor.TYPE, new CefProcessor.Factory(parameters.scriptService)), entry(CommunityIdProcessor.TYPE, new CommunityIdProcessor.Factory()), entry(ConvertProcessor.TYPE, new ConvertProcessor.Factory()), entry(CsvProcessor.TYPE, new CsvProcessor.Factory()), diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CefProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CefProcessorTests.java new file mode 100644 index 0000000000000..df783e69027d1 --- /dev/null +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CefProcessorTests.java @@ -0,0 +1,980 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.common; + +import org.elasticsearch.core.PathUtils; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.test.ESTestCase; +import org.junit.runners.model.TestClass; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static java.util.Map.entry; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; + +public class CefProcessorTests extends ESTestCase { + private static String readCefMessageFile(String fileName) throws IOException, URISyntaxException { + URL resource = TestClass.class.getResource("/" + fileName); + return Files.readString(PathUtils.get((Objects.requireNonNull(resource).toURI()))); + } + + private IngestDocument document; + + public void testParse() throws IOException, URISyntaxException { + String message; + List headers; + Map extensions; + { + message = readCefMessageFile("basic_message.cef.txt"); + headers = CefParser.parseHeaders(message); + extensions = CefParser.parseExtensions(headers.removeLast()); + assertThat(headers, equalTo(List.of("CEF:0", "vendor", "product", "version", "class", "name", "severity"))); + assertThat(extensions, aMapWithSize(0)); + } + { + message = readCefMessageFile("message_with_extension.cef.txt"); + headers = CefParser.parseHeaders(message); + extensions = CefParser.parseExtensions(headers.removeLast()); + assertThat(headers, equalTo(List.of("CEF:1", "vendor", "product", "version", "class", "name", "severity"))); + assertThat(extensions, equalTo(Map.of("someExtension", "someValue"))); + } + { + message = readCefMessageFile("message_with_escaped_pipe.cef.txt"); + headers = CefParser.parseHeaders(message); + extensions = CefParser.parseExtensions(headers.removeLast()); + assertThat(headers, equalTo(List.of("CEF:1", "vendor", "product|pipe", "version space", "class\\slash", "name", "severity"))); + assertMapsEqual(extensions, Map.ofEntries(entry("ext1", "some value "), entry("ext2", "pipe|value"))); + } + } + + public void testExecute() throws IOException, URISyntaxException { + Map source = new HashMap<>(); + String message = readCefMessageFile("message_execute.cef.txt"); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "0"), + entry( + "device", + Map.of("vendor", "Elastic", "product", "Vaporware", "version", "1.0.0-alpha", "event_class_id", "18") + ), + entry("name", "Web request"), + entry("severity", "low") + ) + ), + entry("observer", Map.of("product", "Vaporware", "vendor", "Elastic", "version", "1.0.0-alpha")), + entry("event", Map.of("id", "3457", "code", "18")), + entry( + "source", + Map.ofEntries( + entry("ip", "89.160.20.156"), + entry("port", 33876), + entry("geo", Map.of("location", Map.of("lon", -77.511, "lat", 38.915))), + entry("service", Map.of("name", "httpd")) + ) + ), + entry("destination", Map.of("ip", "192.168.10.1", "port", 443)), + entry("http", Map.of("request", Map.of("method", "POST", "referrer", "https://www.google.com"))), + entry("network", Map.of("transport", "TCP")), + entry("url", Map.of("original", "https://www.example.com/cart")), + entry("message", message) + ) + ); + } + + public void testInvalidCefFormat() { + Map invalidSource = new HashMap<>(); + invalidSource.put("message", "Invalid CEF message"); + IngestDocument invalidIngestDocument = new IngestDocument("index", "id", 1L, null, null, invalidSource); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + expectThrows(IllegalArgumentException.class, () -> processor.execute(invalidIngestDocument)); + } + + public void testStandardMessage() throws IOException, URISyntaxException { + String message = readCefMessageFile("standard_message.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "26"), + entry( + "device", + Map.of("vendor", "security", "product", "threatmanager", "version", "1.0", "event_class_id", "100") + ), + entry("name", "trojan successfully stopped"), + entry("severity", "10") + ) + ), + entry("observer", Map.of("product", "threatmanager", "vendor", "security", "version", "1.0")), + entry("source", Map.of("ip", "10.0.0.192", "port", 1232, "bytes", 4294L)), + entry("destination", Map.of("ip", "12.121.122.82", "bytes", 4294L)), + entry("event", Map.of("id", "1", "code", "100")), + entry("message", message) + ) + ); + } + + public void testHeaderOnly() throws IOException, URISyntaxException { + String message = readCefMessageFile("header_only.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "26"), + entry( + "device", + Map.of("vendor", "security", "product", "threatmanager", "version", "1.0", "event_class_id", "100") + ), + entry("name", "trojan successfully stopped"), + entry("severity", "10") + ) + ), + entry("event", Map.of("code", "100")), + entry("observer", Map.of("product", "threatmanager", "vendor", "security", "version", "1.0")), + entry("message", message) + ) + ); + } + + public void testEmptyDeviceFields() throws IOException, URISyntaxException { + String message = readCefMessageFile("empty_device_fields.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "0"), + entry("device", Map.of("vendor", "", "product", "", "version", "1.0", "event_class_id", "100")), + entry("name", "trojan successfully stopped"), + entry("severity", "10") + ) + ), + entry("event", Map.of("code", "100")), + entry("observer", Map.of("product", "", "vendor", "", "version", "1.0")), + entry("source", Map.of("ip", "10.0.0.192", "port", 1232)), + entry("destination", Map.of("ip", "12.121.122.82")), + entry("message", message) + ) + ); + } + + public void testEscapedPipeInHeader() throws IOException, URISyntaxException { + String message = readCefMessageFile("escaped_pipe_in_header.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "26"), + entry( + "device", + Map.of("vendor", "security", "product", "threat|->manager", "version", "1.0", "event_class_id", "100") + ), + entry("name", "trojan successfully stopped"), + entry("severity", "10") + ) + ), + entry("event", Map.of("code", "100")), + entry("observer", Map.of("product", "threat|->manager", "vendor", "security", "version", "1.0")), + entry("source", Map.of("ip", "10.0.0.192", "port", 1232)), + entry("destination", Map.of("ip", "12.121.122.82")), + entry("message", message) + ) + ); + } + + public void testEqualsSignInHeader() throws IOException, URISyntaxException { + String message = readCefMessageFile("equals_in_header.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "26"), + entry( + "device", + Map.of("vendor", "security", "product", "threat=manager", "version", "1.0", "event_class_id", "100") + ), + entry("name", "trojan successfully stopped"), + entry("severity", "10") + ) + ), + entry("event", Map.of("code", "100")), + entry("observer", Map.of("product", "threat=manager", "vendor", "security", "version", "1.0")), + entry("source", Map.of("ip", "10.0.0.192", "port", 1232)), + entry("destination", Map.of("ip", "12.121.122.82")), + entry("message", message) + ) + ); + } + + public void testEmptyExtensionValue() throws IOException, URISyntaxException { + String message = readCefMessageFile("empty_extension.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "26"), + entry( + "device", + Map.of("vendor", "security", "product", "threatmanager", "version", "1.0", "event_class_id", "100") + ), + entry("name", "trojan successfully stopped"), + entry("severity", "10") + ) + ), + entry("event", Map.of("code", "100")), + entry("observer", Map.of("product", "threatmanager", "vendor", "security", "version", "1.0")), + entry("source", Map.of("ip", "10.0.0.192", "port", 1232)), + entry("message", message) + ) + ); + } + + public void testLeadingWhitespace() throws IOException, URISyntaxException { + String message = readCefMessageFile("leading_whitespace.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "0"), + entry( + "device", + Map.of("vendor", "security", "product", "threatmanager", "version", "1.0", "event_class_id", "100") + ), + entry("name", "trojan successfully stopped"), + entry("severity", "10") + ) + ), + entry("event", Map.of("code", "100")), + entry("observer", Map.of("product", "threatmanager", "vendor", "security", "version", "1.0")), + entry("source", Map.of("ip", "10.0.0.192", "port", 1232)), + entry("destination", Map.of("ip", "12.121.122.82")), + entry("message", message) + ) + ); + } + + public void testEscapedPipeInExtension() throws IOException, URISyntaxException { + String message = readCefMessageFile("escaped_pipe_in_extension.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + expectThrows(IllegalArgumentException.class, () -> processor.execute(document)); + } + + public void testPipeInMessage() throws IOException, URISyntaxException { + String message = readCefMessageFile("pipe_in_message.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "0"), + entry( + "device", + Map.of("vendor", "security", "product", "threatmanager", "version", "1.0", "event_class_id", "100") + ), + entry("name", "trojan successfully stopped"), + entry("severity", "10"), + entry("extensions", Map.of("moo", "this|has a pipe")) + ) + ), + entry("event", Map.of("code", "100")), + entry("observer", Map.of("product", "threatmanager", "vendor", "security", "version", "1.0")), + entry("message", message) + ) + ); + } + + public void testEqualsInMessage() throws IOException, URISyntaxException { + String message = readCefMessageFile("equals_in_message.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + Exception e = expectThrows(IllegalArgumentException.class, () -> processor.execute(document)); + assertThat(e.getMessage(), equalTo("CEF extensions contain unescaped equals sign")); + } + + public void testEscapesInExtension() throws IOException, URISyntaxException { + String message = readCefMessageFile("escapes_in_extension.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "0"), + entry( + "device", + Map.of("vendor", "security", "product", "threatmanager", "version", "1.0", "event_class_id", "100") + ), + entry("name", "trojan successfully stopped"), + entry("severity", "10"), + entry("extensions", Map.of("x", "c\\d=z")) + ) + ), + entry("event", Map.of("code", "100")), + entry("observer", Map.of("product", "threatmanager", "vendor", "security", "version", "1.0")), + entry("message", "a+b=c") + ) + ); + } + + public void testMalformedExtensionEscape() throws IOException, URISyntaxException { + String message = readCefMessageFile("malformed_extension_escape.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + Exception e = expectThrows(IllegalArgumentException.class, () -> processor.execute(document)); + assertThat(e.getMessage(), equalTo("CEF extensions contain unescaped equals sign")); + } + + public void testMultipleMalformedExtensionValues() throws IOException, URISyntaxException { + String message = readCefMessageFile("multiple_malformed_extension_values.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + Exception e = expectThrows(IllegalArgumentException.class, () -> processor.execute(document)); + assertThat(e.getMessage(), equalTo("CEF extensions contain unescaped equals sign")); + } + + public void testPaddedMessage() throws IOException, URISyntaxException { + String message = readCefMessageFile("padded_message.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "0"), + entry( + "device", + Map.of("vendor", "security", "product", "threatmanager", "version", "1.0", "event_class_id", "100") + ), + entry("name", "message is padded"), + entry("severity", "10") + ) + ), + entry("event", Map.of("code", "100")), + entry("observer", Map.of("product", "threatmanager", "vendor", "security", "version", "1.0")), + entry("source", Map.of("ip", "10.0.0.192", "port", 1232)), + entry("message", "Trailing space in non-final extensions is preserved ") + ) + ); + } + + public void testCrlfMessage() throws IOException, URISyntaxException { + String message = readCefMessageFile("crlf_message.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "0"), + entry( + "device", + Map.of("vendor", "security", "product", "threatmanager", "version", "1.0", "event_class_id", "100") + ), + entry("name", "message is padded"), + entry("severity", "10") + ) + ), + entry("event", Map.of("code", "100")), + entry("observer", Map.of("product", "threatmanager", "vendor", "security", "version", "1.0")), + entry("source", Map.of("port", 1232)), + entry("message", "Trailing space in final extensions is not preserved") + ) + ); + } + + public void testTabMessage() throws IOException, URISyntaxException { + String message = readCefMessageFile("tab_message.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "0"), + entry( + "device", + Map.of("vendor", "security", "product", "threatmanager", "version", "1.0", "event_class_id", "100") + ), + entry("name", "message is padded"), + entry("severity", "10") + ) + ), + entry("event", Map.of("code", "100")), + entry("observer", Map.of("product", "threatmanager", "vendor", "security", "version", "1.0")), + entry("source", Map.of("port", 1232, "ip", "127.0.0.1")), + entry("message", "Tabs\tand\rcontrol\ncharacters are preserved\t") + ) + ); + } + + public void testTabNoSepMessage() throws IOException, URISyntaxException { + String message = readCefMessageFile("tab_no_sep_message.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + Exception e = expectThrows(IllegalArgumentException.class, () -> processor.execute(document)); + assertThat(e.getMessage(), equalTo("CEF extensions contain unescaped equals sign")); + } + + public void testEscapedMessage() throws IOException, URISyntaxException { + String message = readCefMessageFile("escaped_message.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "0"), + entry( + "device", + Map.ofEntries( + entry("vendor", "security\\compliance"), + entry("product", "threat|->manager"), + entry("version", "1.0"), + entry("event_class_id", "100") + ) + ), + entry("name", "message contains escapes"), + entry("severity", "10") + ) + ), + entry("event", Map.of("code", "100")), + entry("observer", Map.of("product", "threat|->manager", "vendor", "security\\compliance", "version", "1.0")), + entry("source", Map.of("port", 1232)), + entry("message", "Newlines in messages\nare allowed.\r\nAnd so are carriage feeds\\newlines\\=."), + entry("destination", Map.of("port", 4432)) + ) + ); + } + + public void testTruncatedHeader() throws IOException, URISyntaxException { + String message = readCefMessageFile("truncated_header.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + Exception e = expectThrows(IllegalArgumentException.class, () -> processor.execute(document)); + assertThat(e.getMessage(), equalTo("Incomplete CEF header")); + } + + public void testIgnoreEmptyValuesInExtension() throws IOException, URISyntaxException { + String message = readCefMessageFile("ignore_empty_values_in_extension.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "26"), + entry( + "device", + Map.of("vendor", "security", "product", "threat=manager", "version", "1.0", "event_class_id", "100") + ), + entry("name", "trojan successfully stopped"), + entry("severity", "10") + ) + ), + entry("event", Map.of("code", "100")), + entry("observer", Map.of("product", "threat=manager", "vendor", "security", "version", "1.0")), + entry("destination", Map.of("ip", "12.121.122.82")), + entry("message", message) + ) + ); + } + + public void testHyphenInExtensionKey() throws IOException, URISyntaxException { + String message = readCefMessageFile("hyphen_in_extension_key.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "26"), + entry( + "device", + Map.of("vendor", "security", "product", "threatmanager", "version", "1.0", "event_class_id", "100") + ), + entry("name", "trojan successfully stopped"), + entry("severity", "10"), + entry("extensions", Map.of("Some-Key", "123456")) + ) + ), + entry("event", Map.of("code", "100")), + entry("observer", Map.of("product", "threatmanager", "vendor", "security", "version", "1.0")), + entry("message", message) + ) + ); + } + + public void testAllFieldsInExtension() throws IOException, URISyntaxException { + String message = readCefMessageFile("all_fields_in_extension.cef.txt"); + Map source = new HashMap<>(); + source.put("message", message); + document = new IngestDocument("index", "id", 1L, null, null, source); + CefProcessor processor = new CefProcessor("tag", "description", "message", "cef", false, true, null); + processor.execute(document); + assertMapsEqual( + document.getSource(), + Map.ofEntries( + entry( + "cef", + Map.ofEntries( + entry("version", "0"), + entry( + "device", + Map.of("vendor", "security", "product", "threatmanager", "version", "1.0", "event_class_id", "100") + ), + entry("name", "trojan successfully stopped"), + entry("severity", "10"), + entry( + "extensions", + Map.ofEntries( + entry("agentTranslatedZoneKey", 54854L), + entry("agentZoneKey", 54855L), + entry("customerKey", 54866L), + entry("destinationTranslatedZoneKey", 54867L), + entry("destinationZoneKey", 54877L), + entry("deviceTranslatedZoneKey", 54898L), + entry("deviceZoneKey", 54899L), + entry("sourceTranslatedZoneKey", 54998L), + entry("sourceZoneKey", 546986L), + entry("parserVersion", "1.x.2"), + entry("parserIdentifier", "ABC123"), + entry("deviceNtDomain", "example.org"), + entry("agentZoneExternalID", "zoneExtId"), + entry("agentTimeZone", "UTC"), + entry("deviceCustomIPv6Address1Label", "c6a1Label"), + entry("deviceCustomString1", "customString1"), + entry("deviceCustomIPv6Address2Label", "c6a2Label"), + entry("deviceCustomNumber3", 345L), + entry("deviceCustomFloatingPoint1", 1.23), + entry("deviceCustomNumber2", 234L), + entry("deviceCustomFloatingPoint2", 2.34), + entry("deviceCustomFloatingPoint3", 3.45), + entry("deviceCustomFloatingPoint4", 4.56), + entry("flexDate1", ZonedDateTime.parse("2021-06-01T11:43:20Z")), + entry("destinationTranslatedZoneExternalID", "destExtId"), + entry("deviceCustomNumber1", 123L), + entry("deviceEventCategory", "category"), + entry("deviceCustomString6Label", "cs6Label"), + entry("deviceCustomNumber2Label", "cn2Label"), + entry("flexString1Label", "flexString1Label"), + entry("deviceCustomString5Label", "cs5Label"), + entry("agentZoneURI", "zoneUri"), + entry("deviceCustomString2Label", "cs2Label"), + entry("deviceCustomDate2Label", "customDate2Label"), + entry("deviceCustomNumber1Label", "cn1Label"), + entry("oldFileType", "oldType"), + entry("destinationZoneExternalID", "destZoneExtId"), + entry("categoryDeviceType", "catDeviceType"), + entry("deviceZoneURI", "zoneUri"), + entry("sourceTranslatedZoneExternalID", "sourceExtId"), + entry("agentTranslatedAddress", "10.0.0.1"), + entry("requestCookies", "cookies"), + entry("deviceCustomIPv6Address3", "2001:db8::3"), + entry("oldFilePath", "/old/path"), + entry("deviceCustomIPv6Address2", "2001:db8::2"), + entry("deviceCustomIPv6Address1", "2001:db8::1"), + entry("oldFileId", "oldId"), + entry("deviceTranslatedZoneExternalID", "transExtId"), + entry("deviceCustomFloatingPoint2Label", "cfp2Label"), + entry("deviceTranslatedZoneURI", "transUri"), + entry("deviceCustomIPv6Address4Label", "c6a4Label"), + entry("agentTranslatedZoneURI", "uri"), + entry("oldFilePermission", "rw-r--r--"), + entry("deviceCustomIPv6Address4", "2001:db8::4"), + entry("sourceZoneURI", "sourceZoneUri"), + entry("deviceCustomFloatingPoint3Label", "cfp3Label"), + entry("agentTranslatedZoneExternalID", "ext123"), + entry("destinationZoneURI", "destZoneUri"), + entry("flexDate1Label", "flexDate1Label"), + entry("agentNtDomain", "example.org"), + entry("deviceCustomDate2", ZonedDateTime.parse("2021-06-01T11:45Z")), + entry("deviceCustomDate1", ZonedDateTime.parse("2021-06-01T11:43:20Z")), + entry("deviceCustomString3Label", "cs3Label"), + entry("deviceCustomDate1Label", "customDate1Label"), + entry("destinationTranslatedZoneURI", "destUri"), + entry("oldFileModificationTime", ZonedDateTime.parse("2021-06-01T11:45Z")), + entry("deviceCustomFloatingPoint1Label", "cfp1Label"), + entry("deviceCustomIPv6Address3Label", "c6a3Label"), + entry("deviceCustomFloatingPoint4Label", "cfp4Label"), + entry("oldFileSize", 2048L), + entry("externalId", "extId"), + entry("baseEventCount", 1234), + entry("flexString2", "flexString2"), + entry("deviceCustomNumber3Label", "cn3Label"), + entry("flexString1", "flexString1"), + entry("deviceFacility", "16"), + entry("deviceCustomString4Label", "cs4Label"), + entry("flexString2Label", "flexString2Label"), + entry("deviceCustomString3", "customString3"), + entry("deviceCustomString2", "customString2"), + entry("deviceCustomString1Label", "cs1Label"), + entry("deviceCustomString5", "customString5"), + entry("deviceCustomString4", "customString4"), + entry("deviceZoneExternalID", "zoneExtId"), + entry("deviceCustomString6", "customString6"), + entry("oldFileName", "oldFile"), + entry("sourceZoneExternalID", "sourceZoneExtId"), + entry("oldFileHash", "oldHash"), + entry("sourceTranslatedZoneURI", "sourceUri"), + entry("oldFileCreateTime", ZonedDateTime.parse("2021-06-01T11:43:20Z")) + ) + ) + ) + ), + entry("host", Map.of("nat", Map.of("ip", "10.0.0.3"))), + entry( + "observer", + Map.ofEntries( + entry("ingress", Map.of("interface", Map.of("name", "eth0"))), + entry("registered_domain", "example.com"), + entry("product", "threatmanager"), + entry("hostname", "host1"), + entry("vendor", "security"), + entry("ip", "192.168.0.3"), + entry("name", "extId"), + entry("version", "1.0"), + entry("mac", "00:0a:95:9d:68:16"), + entry("egress", Map.of("interface", Map.of("name", "eth1"))) + ) + ), + entry( + "agent", + Map.ofEntries( + entry("ip", "192.168.0.1"), + entry("name", "example.com"), + entry("id", "agentId"), + entry("type", "agentType"), + entry("version", "1.0"), + entry("mac", "00:0a:95:9d:68:16") + ) + ), + entry("process", Map.of("name", "procName", "pid", 5678)), + entry( + "destination", + Map.ofEntries( + entry("nat", Map.of("port", 8080, "ip", "10.0.0.2")), + entry("geo", Map.of("location", Map.of("lon", -122.4194, "lat", 37.7749))), + entry("registered_domain", "destNtDomain"), + entry("process", Map.of("name", "destProc", "pid", 1234)), + entry("port", 80), + entry("bytes", 91011L), + entry("service", Map.of("name", "destService")), + entry("domain", "destHost"), + entry("ip", "192.168.0.2"), + entry("user", Map.of("name", "destUser", "id", "destUserId", "group", Map.of("name", "admin"))), + entry("mac", "00:0a:95:9d:68:16") + ) + ), + entry( + "source", + Map.ofEntries( + entry("geo", Map.of("location", Map.of("lon", -122.4194, "lat", 37.7749))), + entry("nat", Map.of("port", 8081, "ip", "10.0.0.4")), + entry("registered_domain", "sourceNtDomain"), + entry("process", Map.of("name", "sourceProc", "pid", 1234)), + entry("port", 443), + entry("service", Map.of("name", "sourceService")), + entry("bytes", 5678L), + entry("ip", "192.168.0.4"), + entry("domain", "sourceDomain"), + entry("user", Map.of("name", "sourceUser", "id", "sourceUserId", "group", Map.of("name", "sourcePriv"))), + entry("mac", "00:0a:95:9d:68:16") + ) + ), + entry("message", "message"), + entry("url", Map.of("original", "url")), + entry("network", Map.of("protocol", "HTTP", "transport", "TCP", "direction", "inbound")), + entry( + "file", + Map.ofEntries( + entry("inode", "5678"), + entry("path", "/path/to/file"), + entry("size", 1024L), + entry("created", ZonedDateTime.parse("2021-06-01T11:43:20Z")), + entry("name", "file.txt"), + entry("mtime", ZonedDateTime.parse("2021-06-01T11:45Z")), + entry("type", "txt"), + entry("hash", "abcd1234"), + entry("group", "rw-r--r--") + ) + ), + entry("@timestamp", ZonedDateTime.parse("2021-06-01T11:43:20Z")), + entry("organization", Map.of("name", "custUri", "id", "custExtId")), + entry( + "event", + Map.ofEntries( + entry("action", "blocked"), + entry("timezone", "UTC"), + entry("end", ZonedDateTime.parse("2021-06-01T11:45Z")), + entry("id", "evt123"), + entry("outcome", "success"), + entry("start", ZonedDateTime.parse("2021-06-01T11:43:20Z")), + entry("reason", "reason"), + entry("ingested", ZonedDateTime.parse("2021-06-01T11:43:20Z")), + entry("kind", "1"), + entry("original", "rawEvent"), + entry("created", ZonedDateTime.parse("2021-06-01T11:43:20Z")), + entry("code", "100") + ) + ), + entry("user_agent", Map.of("original", "Mozilla")), + entry("http", Map.of("request", Map.of("referrer", "referrer", "method", "GET"))) + ) + ); + } + + // Date parsing tests + public void testToTimestampWithUnixTimestamp() { + CefParser parser = new CefParser(ZoneId.of("UTC"), false); + String unixTimestamp = "1633072800000"; // Example Unix timestamp in milliseconds + ZonedDateTime expected = ZonedDateTime.ofInstant(Instant.ofEpochMilli(Long.parseLong(unixTimestamp)), ZoneId.of("UTC")); + ZonedDateTime result = parser.toTimestamp(unixTimestamp); + assertEquals(expected, result); + } + + public void testToTimestampWithFormattedDate() { + CefParser parser = new CefParser(ZoneId.of("Europe/Stockholm"), false); + String formattedDate = "Oct 01 2021 12:00:00 UTC"; // Example formatted date + ZonedDateTime expected = ZonedDateTime.parse("2021-10-01T14:00+02:00[Europe/Stockholm]"); + ZonedDateTime result = parser.toTimestamp(formattedDate); + assertEquals(expected, result); + } + + public void testToTimestampWithFormattedDateWithoutYear() { + CefParser parser = new CefParser(ZoneId.of("UTC"), false); + String formattedDate = "Oct 01 12:00:00 UTC"; // Example formatted date without year + int currentYear = ZonedDateTime.now(ZoneId.of("UTC")).getYear(); + ZonedDateTime expected = ZonedDateTime.parse(currentYear + "-10-01T12:00:00Z[UTC]"); + ZonedDateTime result = parser.toTimestamp(formattedDate); + assertEquals(expected, result); + } + + public void testToTimestampWithFormattedDateWithoutTimezone() { + CefParser parser = new CefParser(ZoneId.of("UTC"), false); + String formattedDate = "Sep 07 2018 14:50:39"; // Example formatted date without year + ZonedDateTime expected = ZonedDateTime.parse("2018-09-07T14:50:39Z[UTC]"); + ZonedDateTime result = parser.toTimestamp(formattedDate); + assertEquals(expected, result); + } + + public void testToTimestampWithInvalidDate() { + CefParser parser = new CefParser(ZoneId.of("UTC"), false); + String invalidDate = "invalid date"; + expectThrows(IllegalArgumentException.class, () -> parser.toTimestamp(invalidDate)); + } + + public void testToMacAddressWithSeparators() { + CefParser parser = new CefParser(ZoneId.of("UTC"), false); + List macAddresses = List.of( + // EUI-48 (with separators). + "00:0D:60:AF:1B:61", + "00-0D-60-AF-1B-61", + "000D.60AF.1B61", + + // EUI-64 (with separators). + "00:0D:60:FF:FE:AF:1B:61", + "00-0D-60-FF-FE-AF-1B-61", + "000D.60FF.FEAF.1B61" + ); + macAddresses.forEach(macAddress -> { + String result = parser.toMACAddress(macAddress); + assertEquals(macAddress, result); + }); + } + + public void testInvalidMacAddresses() { + CefParser parser = new CefParser(ZoneId.of("UTC"), false); + for (String invalid : List.of("00|0D|60|AF|1B|61", "00:0D:60:AF:1B:61 foo", "0000:0D:60:AF:1B:61")) { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parser.toMACAddress(invalid)); + assertThat(e.getMessage(), equalTo("Invalid MAC address format")); + } + } + + public void testEUI48ToMacAddressWithOutSeparators() { + CefParser parser = new CefParser(ZoneId.of("UTC"), false); + String macAddress = "000D60AF1B61"; + String result = parser.toMACAddress(macAddress); + assertEquals("00:0D:60:AF:1B:61", result); + } + + public void testEUI64ToMacAddressWithOutSeparators() { + CefParser parser = new CefParser(ZoneId.of("UTC"), false); + String macAddress = "000D60FFFEAF1B61"; + String result = parser.toMACAddress(macAddress); + assertEquals("00:0D:60:FF:FE:AF:1B:61", result); + } + + public void testtoIPValidIPv4Address() { + CefParser parser = new CefParser(ZoneId.of("UTC"), true); + String result = parser.toIP("192.168.1.1"); + assertEquals("192.168.1.1", result); + } + + public void testToIPValidIPv6Address() { + CefParser parser = new CefParser(ZoneId.of("UTC"), true); + String result = parser.toIP("2001:0db8:85a3:0000:0000:8a2e:0370:7334"); + assertEquals("2001:db8:85a3::8a2e:370:7334", result); + } + + public void testToIPInvalidIPAddress() { + CefParser parser = new CefParser(ZoneId.of("UTC"), true); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> parser.toIP("invalid_ip")); + assertEquals("Invalid IP address format", exception.getMessage()); + } + + private static void assertMapsEqual(Map actual, Map expected) { + innerAssertMapsEqual(actual, expected, ""); + } + + private static void innerAssertMapsEqual(final Map actual, final Map expected, final String path) { + // as a trivial check, make sure the key sets match + assertThat( + "The set of keys in the result are not the same as the set of expected keys", + actual.keySet(), + containsInAnyOrder(expected.keySet().toArray(new Object[0])) + ); + // then for each expected key, compare values + for (Map.Entry entry : expected.entrySet()) { + Object key = entry.getKey(); + String newPath = path.isEmpty() ? String.valueOf(key) : path + "." + key; + Object expectedValue = entry.getValue(); + Object actualValue = actual.get(key); + if (expectedValue instanceof Map expectedMap && actualValue instanceof Map actualMap) { + innerAssertMapsEqual(expectedMap, actualMap, newPath); + } else { + assertThat("Unexpected value for path [" + newPath + "]", actualValue, equalTo(expectedValue)); + } + } + // as a last check, make sure they're actually equal -- the above checks are intended to be friendly (and accurate), but this + // last check makes sure nothing ever sneaks through + assertThat(actual, equalTo(expected)); + } +} diff --git a/modules/ingest-common/src/test/resources/all_fields_in_extension.cef.txt b/modules/ingest-common/src/test/resources/all_fields_in_extension.cef.txt new file mode 100644 index 0000000000000..6d96aa9467a4e --- /dev/null +++ b/modules/ingest-common/src/test/resources/all_fields_in_extension.cef.txt @@ -0,0 +1 @@ +CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|agt=192.168.0.1 agentDnsDomain=example.com ahost=agentHost aid=agentId amac=00:0a:95:9d:68:16 agentNtDomain=example.org art=1622547800000 atz=UTC agentTranslatedAddress=10.0.0.1 agentTranslatedZoneExternalID=ext123 agentTranslatedZoneURI=uri at=agentType av=1.0 agentZoneExternalID=zoneExtId agentZoneURI=zoneUri app=HTTP cnt=1234 in=5678 out=91011 customerExternalID=custExtId customerURI=custUri dst=192.168.0.2 dlat=37.7749 dlong=-122.4194 dhost=destHost dmac=00:0a:95:9d:68:16 dntdom=destNtDomain dpt=80 dpid=1234 dproc=destProc destinationServiceName=destService destinationTranslatedAddress=10.0.0.2 destinationTranslatedPort=8080 destinationTranslatedZoneExternalID=destExtId destinationTranslatedZoneURI=destUri duid=destUserId duser=destUser dpriv=admin destinationZoneExternalID=destZoneExtId destinationZoneURI=destZoneUri act=blocked dvc=192.168.0.3 cfp1Label=cfp1Label cfp3Label=cfp3Label cfp4Label=cfp4Label deviceCustomDate1=1622547800000 deviceCustomDate1Label=customDate1Label deviceCustomDate2=1622547900000 deviceCustomDate2Label=customDate2Label cfp1=1.23 cfp2=2.34 cfp2Label=cfp2Label cfp3=3.45 cfp4=4.56 c6a1=2001:db8::1 c6a1Label=c6a1Label c6a2=2001:db8::2 c6a2Label=c6a2Label c6a3=2001:db8::3 c6a3Label=c6a3Label c6a4=2001:db8::4 c6a4Label=c6a4Label cn1=123 cn1Label=cn1Label cn2=234 cn2Label=cn2Label cn3=345 cn3Label=cn3Label cs1=customString1 cs1Label=cs1Label cs2=customString2 cs2Label=cs2Label cs3=customString3 cs3Label=cs3Label cs4=customString4 cs4Label=cs4Label cs5=customString5 cs5Label=cs5Label cs6=customString6 cs6Label=cs6Label deviceDirection=inbound deviceDnsDomain=example.com cat=category deviceExternalId=extId deviceFacility=16 dvchost=host1 deviceInboundInterface=eth0 dvcmac=00:0a:95:9d:68:16 deviceNtDomain=example.org deviceOutboundInterface=eth1 devicePayloadId=payloadId dvcpid=5678 deviceProcessName=procName rt=1622547800000 dtz=UTC deviceTranslatedAddress=10.0.0.3 deviceTranslatedZoneExternalID=transExtId deviceTranslatedZoneURI=transUri deviceZoneExternalID=zoneExtId deviceZoneURI=zoneUri end=1622547900000 eventId=evt123 outcome=success externalId=extId fileCreateTime=1622547800000 fileHash=abcd1234 fileId=5678 fileModificationTime=1622547900000 fname=file.txt filePath=/path/to/file filePermission=rw-r--r-- fsize=1024 fileType=txt flexDate1=1622547800000 flexDate1Label=flexDate1Label flexString1=flexString1 flexString2=flexString2 flexString1Label=flexString1Label flexString2Label=flexString2Label msg=message oldFileCreateTime=1622547800000 oldFileHash=oldHash oldFileId=oldId oldFileModificationTime=1622547900000 oldFileName=oldFile oldFilePath=/old/path oldFilePermission=rw-r--r-- oldFileSize=2048 oldFileType=oldType rawEvent=rawEvent reason=reason requestClientApplication=Mozilla requestContext=referrer requestCookies=cookies requestMethod=GET request=url src=192.168.0.4 sourceDnsDomain=sourceDomain slat=37.7749 slong=-122.4194 shost=sourceHost smac=00:0a:95:9d:68:16 sntdom=sourceNtDomain spt=443 spid=1234 sproc=sourceProc sourceServiceName=sourceService sourceTranslatedAddress=10.0.0.4 sourceTranslatedPort=8081 sourceTranslatedZoneExternalID=sourceExtId sourceTranslatedZoneURI=sourceUri suid=sourceUserId suser=sourceUser spriv=sourcePriv sourceZoneExternalID=sourceZoneExtId sourceZoneURI=sourceZoneUri start=1622547800000 proto=TCP type=1 catdt=catDeviceType mrt=1622547800000 agentTranslatedZoneKey=54854 agentZoneKey=54855 customerKey=54866 destinationTranslatedZoneKey=54867 dZoneKey=54877 deviceTranslatedZoneKey=54898 deviceZoneKey=54899 sTranslatedZoneKey=54998 sZoneKey=546986 parserVersion=1.x.2 parserIdentifier=ABC123 \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/basic_message.cef.txt b/modules/ingest-common/src/test/resources/basic_message.cef.txt new file mode 100644 index 0000000000000..5b496c025a5d3 --- /dev/null +++ b/modules/ingest-common/src/test/resources/basic_message.cef.txt @@ -0,0 +1 @@ +CEF:0|vendor|product|version|class|name|severity| \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/crlf_message.cef.txt b/modules/ingest-common/src/test/resources/crlf_message.cef.txt new file mode 100644 index 0000000000000..1befd4d46f599 --- /dev/null +++ b/modules/ingest-common/src/test/resources/crlf_message.cef.txt @@ -0,0 +1 @@ +CEF:0|security|threatmanager|1.0|100|message is padded|10|spt=1232 msg=Trailing space in final extensions is not preserved\t \r\n \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/empty_device_fields.cef.txt b/modules/ingest-common/src/test/resources/empty_device_fields.cef.txt new file mode 100644 index 0000000000000..889083195813d --- /dev/null +++ b/modules/ingest-common/src/test/resources/empty_device_fields.cef.txt @@ -0,0 +1 @@ +CEF:0|||1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232 \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/empty_extension.cef.txt b/modules/ingest-common/src/test/resources/empty_extension.cef.txt new file mode 100644 index 0000000000000..db3bfe5be57bc --- /dev/null +++ b/modules/ingest-common/src/test/resources/empty_extension.cef.txt @@ -0,0 +1 @@ +CEF:26|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst= spt=1232 \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/equals_in_header.cef.txt b/modules/ingest-common/src/test/resources/equals_in_header.cef.txt new file mode 100644 index 0000000000000..b397c96dd020f --- /dev/null +++ b/modules/ingest-common/src/test/resources/equals_in_header.cef.txt @@ -0,0 +1 @@ +CEF:26|security|threat=manager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232 \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/equals_in_message.cef.txt b/modules/ingest-common/src/test/resources/equals_in_message.cef.txt new file mode 100644 index 0000000000000..cd50573208409 --- /dev/null +++ b/modules/ingest-common/src/test/resources/equals_in_message.cef.txt @@ -0,0 +1 @@ +CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this =has = equals\= dst=12.121.122.82 spt=1232 \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/escaped_message.cef.txt b/modules/ingest-common/src/test/resources/escaped_message.cef.txt new file mode 100644 index 0000000000000..04d7cace28b6b --- /dev/null +++ b/modules/ingest-common/src/test/resources/escaped_message.cef.txt @@ -0,0 +1 @@ +CEF:0|security\\compliance|threat\|->manager|1.0|100|message contains escapes|10|spt=1232 msg=Newlines in messages\nare allowed.\r\nAnd so are carriage feeds\\newlines\\\=. dpt=4432 \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/escaped_pipe_in_extension.cef.txt b/modules/ingest-common/src/test/resources/escaped_pipe_in_extension.cef.txt new file mode 100644 index 0000000000000..018fa233f10d5 --- /dev/null +++ b/modules/ingest-common/src/test/resources/escaped_pipe_in_extension.cef.txt @@ -0,0 +1 @@ +CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this\|has an escaped pipe \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/escaped_pipe_in_header.cef.txt b/modules/ingest-common/src/test/resources/escaped_pipe_in_header.cef.txt new file mode 100644 index 0000000000000..6f4488a5cedfc --- /dev/null +++ b/modules/ingest-common/src/test/resources/escaped_pipe_in_header.cef.txt @@ -0,0 +1 @@ +CEF:26|security|threat\|->manager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232 \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/escapes_in_extension.cef.txt b/modules/ingest-common/src/test/resources/escapes_in_extension.cef.txt new file mode 100644 index 0000000000000..92db6e0509db0 --- /dev/null +++ b/modules/ingest-common/src/test/resources/escapes_in_extension.cef.txt @@ -0,0 +1 @@ +CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|msg=a+b\=c x=c\\d\=z \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/header_only.cef.txt b/modules/ingest-common/src/test/resources/header_only.cef.txt new file mode 100644 index 0000000000000..a83347bf12a35 --- /dev/null +++ b/modules/ingest-common/src/test/resources/header_only.cef.txt @@ -0,0 +1 @@ +CEF:26|security|threatmanager|1.0|100|trojan successfully stopped|10| \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/hyphen_in_extension_key.cef.txt b/modules/ingest-common/src/test/resources/hyphen_in_extension_key.cef.txt new file mode 100644 index 0000000000000..a0211b5287cae --- /dev/null +++ b/modules/ingest-common/src/test/resources/hyphen_in_extension_key.cef.txt @@ -0,0 +1 @@ +CEF:26|security|threatmanager|1.0|100|trojan successfully stopped|10|Some-Key=123456 \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/ignore_empty_values_in_extension.cef.txt b/modules/ingest-common/src/test/resources/ignore_empty_values_in_extension.cef.txt new file mode 100644 index 0000000000000..b77e452469769 --- /dev/null +++ b/modules/ingest-common/src/test/resources/ignore_empty_values_in_extension.cef.txt @@ -0,0 +1 @@ +CEF:26|security|threat=manager|1.0|100|trojan successfully stopped|10|src= dst=12.121.122.82 spt= \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/leading_whitespace.cef.txt b/modules/ingest-common/src/test/resources/leading_whitespace.cef.txt new file mode 100644 index 0000000000000..12a1981558212 --- /dev/null +++ b/modules/ingest-common/src/test/resources/leading_whitespace.cef.txt @@ -0,0 +1 @@ +CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232 \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/malformed_extension_escape.cef.txt b/modules/ingest-common/src/test/resources/malformed_extension_escape.cef.txt new file mode 100644 index 0000000000000..0522f8e83e99b --- /dev/null +++ b/modules/ingest-common/src/test/resources/malformed_extension_escape.cef.txt @@ -0,0 +1 @@ +CEF:0|FooBar|Web Gateway|1.2.3.45.67|200|Success|2|rt=Sep 07 2018 14:50:39 cat=Access Log dst=1.1.1.1 dhost=foo.example.com suser=redacted src=2.2.2.2 requestMethod=POST request='https://foo.example.com/bar/bingo/1' requestClientApplication='Foo-Bar/2018.1.7; =Email:user@example.com; Guid:test=' cs1= cs1Label=Foo Bar \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/message_execute.cef.txt b/modules/ingest-common/src/test/resources/message_execute.cef.txt new file mode 100644 index 0000000000000..fcb56e6e5cf23 --- /dev/null +++ b/modules/ingest-common/src/test/resources/message_execute.cef.txt @@ -0,0 +1 @@ +CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|eventId=3457 requestMethod=POST slat=38.915 slong=-77.511 proto=TCP sourceServiceName=httpd requestContext=https://www.google.com src=89.160.20.156 spt=33876 dst=192.168.10.1 dpt=443 request=https://www.example.com/cart \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/message_with_escaped_pipe.cef.txt b/modules/ingest-common/src/test/resources/message_with_escaped_pipe.cef.txt new file mode 100644 index 0000000000000..9df0ecb6e48c5 --- /dev/null +++ b/modules/ingest-common/src/test/resources/message_with_escaped_pipe.cef.txt @@ -0,0 +1 @@ +CEF:1|vendor|product\|pipe|version space|class\\slash|name|severity|ext1=some value ext2=pipe|value \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/message_with_extension.cef.txt b/modules/ingest-common/src/test/resources/message_with_extension.cef.txt new file mode 100644 index 0000000000000..17c09edb0c346 --- /dev/null +++ b/modules/ingest-common/src/test/resources/message_with_extension.cef.txt @@ -0,0 +1 @@ +CEF:1|vendor|product|version|class|name|severity|someExtension=someValue \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/multiple_malformed_extension_values.cef.txt b/modules/ingest-common/src/test/resources/multiple_malformed_extension_values.cef.txt new file mode 100644 index 0000000000000..cc873d0515d84 --- /dev/null +++ b/modules/ingest-common/src/test/resources/multiple_malformed_extension_values.cef.txt @@ -0,0 +1 @@ +CEF:0|vendor|product|version|event_id|name|Very-High| msg=Hello World error=Failed because id==old_id user=root angle=106.7<=180 \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/padded_message.cef.txt b/modules/ingest-common/src/test/resources/padded_message.cef.txt new file mode 100644 index 0000000000000..2eaa1b5ef698c --- /dev/null +++ b/modules/ingest-common/src/test/resources/padded_message.cef.txt @@ -0,0 +1 @@ +CEF:0|security|threatmanager|1.0|100|message is padded|10|spt=1232 msg=Trailing space in non-final extensions is preserved src=10.0.0.192 \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/pipe_in_message.cef.txt b/modules/ingest-common/src/test/resources/pipe_in_message.cef.txt new file mode 100644 index 0000000000000..464afd34ec819 --- /dev/null +++ b/modules/ingest-common/src/test/resources/pipe_in_message.cef.txt @@ -0,0 +1 @@ +CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this|has a pipe \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/standard_message.cef.txt b/modules/ingest-common/src/test/resources/standard_message.cef.txt new file mode 100644 index 0000000000000..62e0929501918 --- /dev/null +++ b/modules/ingest-common/src/test/resources/standard_message.cef.txt @@ -0,0 +1 @@ +CEF:26|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232 eventId=1 in=4294 out=4294 \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/tab_message.cef.txt b/modules/ingest-common/src/test/resources/tab_message.cef.txt new file mode 100644 index 0000000000000..04c1874935ecb --- /dev/null +++ b/modules/ingest-common/src/test/resources/tab_message.cef.txt @@ -0,0 +1 @@ +CEF:0|security|threatmanager|1.0|100|message is padded|10|spt=1232 msg=Tabs\tand\rcontrol\ncharacters are preserved\t src=127.0.0.1 \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/tab_no_sep_message.cef.txt b/modules/ingest-common/src/test/resources/tab_no_sep_message.cef.txt new file mode 100644 index 0000000000000..35c6cc1eff201 --- /dev/null +++ b/modules/ingest-common/src/test/resources/tab_no_sep_message.cef.txt @@ -0,0 +1 @@ +CEF:0|security|threatmanager|1.0|100|message has tabs|10|spt=1232 msg=Tab is not a separator\tsrc=127.0.0.1 \ No newline at end of file diff --git a/modules/ingest-common/src/test/resources/truncated_header.cef.txt b/modules/ingest-common/src/test/resources/truncated_header.cef.txt new file mode 100644 index 0000000000000..df455ce74b522 --- /dev/null +++ b/modules/ingest-common/src/test/resources/truncated_header.cef.txt @@ -0,0 +1 @@ +CEF:0|SentinelOne|Mgmt|activityID=1111111111111111111 activityType=3505 siteId=None siteName=None accountId=1222222222222222222 accountName=foo-bar mdr notificationScope=ACCOUNT \ No newline at end of file diff --git a/server/src/main/java/org/elasticsearch/common/Strings.java b/server/src/main/java/org/elasticsearch/common/Strings.java index a9cedb7cf7f50..c36c852dbc5d0 100644 --- a/server/src/main/java/org/elasticsearch/common/Strings.java +++ b/server/src/main/java/org/elasticsearch/common/Strings.java @@ -172,6 +172,24 @@ public static String trimLeadingCharacter(String str, char leadingCharacter) { return str.substring(i); } + /** + * Trim all occurrences of the supplied trailing character from the given String. + * + * @param str the String to check + * @param trailingCharacter the trailing character to be trimmed + * @return the trimmed String + */ + public static String trimTrailingCharacter(String str, char trailingCharacter) { + if (hasLength(str) == false) { + return str; + } + int i = str.length(); + while (i > 0 && str.charAt(i - 1) == trailingCharacter) { + i--; + } + return str.substring(0, i); + } + /** * Test whether the given string matches the given substring * at the given index. diff --git a/server/src/test/java/org/elasticsearch/common/StringsTests.java b/server/src/test/java/org/elasticsearch/common/StringsTests.java index a543ae73997b3..c9a8521b4f628 100644 --- a/server/src/test/java/org/elasticsearch/common/StringsTests.java +++ b/server/src/test/java/org/elasticsearch/common/StringsTests.java @@ -35,6 +35,7 @@ import static org.elasticsearch.common.Strings.toLowercaseAscii; import static org.elasticsearch.common.Strings.tokenizeByCommaToSet; import static org.elasticsearch.common.Strings.trimLeadingCharacter; +import static org.elasticsearch.common.Strings.trimTrailingCharacter; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.emptyArray; @@ -114,8 +115,19 @@ public void testCleanTruncate() { } public void testTrimLeadingCharacter() { + assertThat(trimLeadingCharacter(null, 'g'), equalTo(null)); + assertThat(trimLeadingCharacter("", 'g'), equalTo("")); assertThat(trimLeadingCharacter("abcdef", 'g'), equalTo("abcdef")); assertThat(trimLeadingCharacter("aaabcdef", 'a'), equalTo("bcdef")); + assertThat(trimLeadingCharacter("aaa", 'a'), equalTo("")); + } + + public void testTrimTrailingCharacter() { + assertThat(trimTrailingCharacter(null, 'g'), equalTo(null)); + assertThat(trimTrailingCharacter("", 'g'), equalTo("")); + assertThat(trimTrailingCharacter("abcdef", 'g'), equalTo("abcdef")); + assertThat(trimTrailingCharacter("abcdefggg", 'g'), equalTo("abcdef")); + assertThat(trimTrailingCharacter("aaa", 'a'), equalTo("")); } public void testToStringToXContent() {