diff --git a/docs/changelog/133745.yaml b/docs/changelog/133745.yaml new file mode 100644 index 0000000000000..a49628c68162a --- /dev/null +++ b/docs/changelog/133745.yaml @@ -0,0 +1,5 @@ +pr: 133745 +summary: Supporting more timestamp formats in `_text_structure/find_structure` +area: Machine Learning +type: feature +issues: [] diff --git a/libs/grok/src/main/resources/patterns/ecs-v1/grok-patterns b/libs/grok/src/main/resources/patterns/ecs-v1/grok-patterns index 6f58f3ff4750d..7982c014e165f 100644 --- a/libs/grok/src/main/resources/patterns/ecs-v1/grok-patterns +++ b/libs/grok/src/main/resources/patterns/ecs-v1/grok-patterns @@ -71,6 +71,8 @@ ISO8601_SECOND %{SECOND} TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? DATE %{DATE_US}|%{DATE_EU} DATESTAMP %{DATE}[- ]%{TIME} +DATE_YMD %{YEAR}[./-]%{MONTHNUM2}[./-]%{MONTHDAY} +TIMESTAMP_YMD %{DATE_YMD}[ ]%{TIME} TZ (?:[APMCE][SD]T|UTC) DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} diff --git a/libs/grok/src/main/resources/patterns/legacy/grok-patterns b/libs/grok/src/main/resources/patterns/legacy/grok-patterns index dc4a23dfee362..875f3faa1abf7 100644 --- a/libs/grok/src/main/resources/patterns/legacy/grok-patterns +++ b/libs/grok/src/main/resources/patterns/legacy/grok-patterns @@ -72,6 +72,8 @@ ISO8601_HOUR (?:2[0123]|[01][0-9]) TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{ISO8601_HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? DATE %{DATE_US}|%{DATE_EU} DATESTAMP %{DATE}[- ]%{TIME} +DATE_YMD %{YEAR}[./-]%{MONTHNUM2}[./-]%{MONTHDAY} +TIMESTAMP_YMD %{DATE_YMD}[ ]%{TIME} TZ (?:[APMCE][SD]T|UTC) DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 66ca8173bd73b..74d753af69325 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -53,6 +53,7 @@ tasks.named("thirdPartyAudit").configure { tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("ingest/30_date_processor/Test week based date parsing", "week-date behaviour has changed") + task.skipTest("ingest/120_grok/Test Grok Patterns Retrieval", "only counting the number of patterns, which may change") }) configurations { diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/120_grok.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/120_grok.yml index 1785e78435a50..188f3fb3545ba 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/120_grok.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/120_grok.yml @@ -152,7 +152,7 @@ teardown: "Test Grok Patterns Retrieval": - do: ingest.processor_grok: {} - - length: { patterns: 318 } + - length: { patterns: 320 } - match: { patterns.PATH: "(?:%{UNIXPATH}|%{WINPATH})" } diff --git a/x-pack/plugin/text-structure/build.gradle b/x-pack/plugin/text-structure/build.gradle index 4ebe380e7c724..57fb970fdffc9 100644 --- a/x-pack/plugin/text-structure/build.gradle +++ b/x-pack/plugin/text-structure/build.gradle @@ -1,4 +1,7 @@ apply plugin: 'elasticsearch.internal-es-plugin' +apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' + esplugin { name = 'x-pack-text-structure' description = 'Elasticsearch Expanded Pack Plugin - Text Structure' @@ -9,12 +12,23 @@ base { archivesName = 'x-pack-text-structure' } +restResources { + restApi { + include '_common', 'cluster', 'text_structure' + } +} + dependencies { compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation project(path: ':test:test-clusters') api project(':libs:grok') api "com.ibm.icu:icu4j:${versions.icu4j}" api "net.sf.supercsv:super-csv:${versions.supercsv}" } addQaCheckDependencies(project) + +tasks.named('javaRestTest') { + usesDefaultDistribution("to be triaged") +} diff --git a/x-pack/plugin/text-structure/src/javaRestTest/java/org/elasticsearch/xpack/textstructure/rest/TextStructureTimestampFormatsIT.java b/x-pack/plugin/text-structure/src/javaRestTest/java/org/elasticsearch/xpack/textstructure/rest/TextStructureTimestampFormatsIT.java new file mode 100644 index 0000000000000..76ba35109005c --- /dev/null +++ b/x-pack/plugin/text-structure/src/javaRestTest/java/org/elasticsearch/xpack/textstructure/rest/TextStructureTimestampFormatsIT.java @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.textstructure.rest; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.ClassRule; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; + +public class TextStructureTimestampFormatsIT extends ESRestTestCase { + + public static final String[] ISO_08601_JAVA_FORMATS = new String[] { "yyyy-MM-dd HH:mm:ss" }; + public static final String ISO_08601_TIMESTAMP_GROK_PATTERN = "%{TIMESTAMP_ISO8601:timestamp}"; + + public static final String[] TIMESTAMP_YMD_JAVA_FORMATS = new String[] { + "yyyy/MM/dd HH:mm:ss", + "yyyy.MM.dd HH:mm:ss", + "yyyy-MM-dd HH:mm:ss" }; + public static final String TIMESTAMP_YMD_TIMESTAMP_GROK_PATTERN = "%{TIMESTAMP_YMD:timestamp}"; + + public static final String[] MONTH_EXPLICIT_NAME_JAVA_FORMATS = new String[] { "MMM d, yyyy" }; + + private final String ecsCompatibility; + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .module("x-pack-text-structure") + .setting("xpack.security.enabled", "false") + .build(); + + public TextStructureTimestampFormatsIT(@Name("ecs_compatibility") String ecsCompatibility) { + this.ecsCompatibility = ecsCompatibility; + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @ParametersFactory + public static Iterable parameters() { + return Arrays.asList(new Object[] { "v1" }, new Object[] { "disabled" }); + } + + public void testTimestampYearYmdSlashFormat() throws IOException { + // use a multi-line sample to ensure we are detecting ndjson format + Map responseMap = executeAndVerifyRequest(""" + "2025/07/10 10:30:35" + "2025/07/10 10:31:42" + "2025/07/10 10:32:15" + """, ecsCompatibility); + verifyTimestampDetected(responseMap, "date"); + verifyTimestampFormat(responseMap, TIMESTAMP_YMD_TIMESTAMP_GROK_PATTERN, TIMESTAMP_YMD_JAVA_FORMATS); + } + + public void testTimestampYearYmdSlashFormat_WithDotAndMillis() throws IOException { + // use a multi-line sample to ensure we are detecting ndjson format + Map responseMap = executeAndVerifyRequest(""" + "2025/07/10 10:30:35.123" + "2025/07/10 10:31:42.123" + "2025/07/10 10:32:15.123" + """, ecsCompatibility); + verifyTimestampDetected(responseMap, "date"); + verifyTimestampFormat( + responseMap, + TIMESTAMP_YMD_TIMESTAMP_GROK_PATTERN, + "yyyy/MM/dd HH:mm:ss.SSS", + "yyyy.MM.dd HH:mm:ss.SSS", + "yyyy-MM-dd HH:mm:ss.SSS" + ); + } + + public void testTimestampYearYmdSlashFormat_WithSlashAndNanos() throws IOException { + // use a multi-line sample to ensure we are detecting ndjson format + Map responseMap = executeAndVerifyRequest(""" + "2025/07/10 10:30:35,123456789" + "2025/07/10 10:31:42,123456789" + "2025/07/10 10:32:15,123456789" + """, ecsCompatibility); + verifyTimestampDetected(responseMap, "date_nanos"); + verifyTimestampFormat( + responseMap, + TIMESTAMP_YMD_TIMESTAMP_GROK_PATTERN, + "yyyy/MM/dd HH:mm:ss,SSSSSSSSS", + "yyyy.MM.dd HH:mm:ss,SSSSSSSSS", + "yyyy-MM-dd HH:mm:ss,SSSSSSSSS" + ); + } + + public void testTimestampYearYmdDotFormat() throws IOException { + // use a multi-line sample to ensure we are detecting ndjson format + Map responseMap = executeAndVerifyRequest(""" + "2025.07.10 10:30:35" + "2025.07.10 10:31:42" + "2025.07.10 10:32:15" + """, ecsCompatibility); + verifyTimestampDetected(responseMap, "date"); + verifyTimestampFormat(responseMap, TIMESTAMP_YMD_TIMESTAMP_GROK_PATTERN, TIMESTAMP_YMD_JAVA_FORMATS); + } + + public void testIso08601TimestampFormat() throws IOException { + // use a multi-line sample to ensure we are detecting ndjson format + Map responseMap = executeAndVerifyRequest(""" + "2025-07-10 10:30:35" + "2025-07-10 10:31:42" + "2025-07-10 10:32:15" + """, ecsCompatibility); + verifyTimestampDetected(responseMap, "date"); + // ISO_8601 should have higher priority than TIMESTAMP_YMD + verifyTimestampFormat(responseMap, ISO_08601_TIMESTAMP_GROK_PATTERN, ISO_08601_JAVA_FORMATS); + } + + public void testMonthExplicitNameFormat() throws IOException { + // use a multi-line sample to ensure we are detecting ndjson format + Map responseMap = executeAndVerifyRequest(""" + "Aug 9, 2025" + "Aug 10, 2025" + "Aug 11, 2025" + """, ecsCompatibility); + verifyTimestampDetected(responseMap, "date"); + verifyTimestampFormat(responseMap, "CUSTOM_TIMESTAMP", MONTH_EXPLICIT_NAME_JAVA_FORMATS); + } + + private static Map executeAndVerifyRequest(String sample, String ecsCompatibility) throws IOException { + Request request = new Request("POST", "/_text_structure/find_structure"); + request.addParameter("ecs_compatibility", ecsCompatibility); + request.setEntity(new StringEntity(sample, ContentType.APPLICATION_JSON)); + Response response = client().performRequest(request); + assertOK(response); + return entityAsMap(response); + } + + private static void verifyTimestampDetected(Map responseMap, String expectedType) { + @SuppressWarnings("unchecked") + Map mappings = (Map) responseMap.get("mappings"); + assertThat(mappings, hasKey("properties")); + @SuppressWarnings("unchecked") + Map properties = (Map) mappings.get("properties"); + assertThat(properties, hasKey("@timestamp")); + @SuppressWarnings("unchecked") + Map timestamp = (Map) properties.get("@timestamp"); + assertThat(timestamp.get("type"), equalTo(expectedType)); + } + + private static void verifyTimestampFormat(Map responseMap, String expectedGrokPattern, String... expectedJavaFormats) { + assertThat(responseMap, hasKey("java_timestamp_formats")); + @SuppressWarnings("unchecked") + List javaTimestampFormats = (List) responseMap.get("java_timestamp_formats"); + assertThat(javaTimestampFormats, containsInAnyOrder(expectedJavaFormats)); + String grokPattern = (String) responseMap.get("grok_pattern"); + assertThat(grokPattern, containsString(expectedGrokPattern)); + } +} diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/structurefinder/TimestampFormatFinder.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/structurefinder/TimestampFormatFinder.java index 228c7c522adaa..7b85239edea8e 100644 --- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/structurefinder/TimestampFormatFinder.java +++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/structurefinder/TimestampFormatFinder.java @@ -287,6 +287,28 @@ public final class TimestampFormatFinder { Arrays.asList(" 11 1111 11 11 11 111", " 1 1111 11 11 11 111"), 0, 0 + ), + new CandidateTimestampFormat( + example -> Arrays.asList( + CandidateTimestampFormat.adjustFractionalSecondsFromEndOfExample(example, "yyyy/MM/dd HH:mm:ss"), + CandidateTimestampFormat.adjustFractionalSecondsFromEndOfExample(example, "yyyy.MM.dd HH:mm:ss"), + CandidateTimestampFormat.adjustFractionalSecondsFromEndOfExample(example, "yyyy-MM-dd HH:mm:ss") + ), + "\\b\\d{4}[./-]\\d{2}[./-]\\d{2} \\d{2}:\\d{2}:\\d{2}(?:[.,]\\d+)?\\b", + "\\b%{TIMESTAMP_YMD}\\b", + "TIMESTAMP_YMD", + List.of("1111 11 11 11 11 11"), + 0, + 10 + ), + new CandidateTimestampFormat( + example -> Collections.singletonList("MMM d, yyyy"), + "\\b[A-Z][a-z]{2} \\d{1,2}, \\d{4}\\b", + "\\b%{MONTH} %{MONTHDAY}, %{YEAR}\\b", + CUSTOM_TIMESTAMP_GROK_NAME, + Arrays.asList(" 11 1111", " 1 1111"), + 5, + 0 ) ); diff --git a/x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/EnhancedTimestampDetectionTests.java b/x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/EnhancedTimestampDetectionTests.java new file mode 100644 index 0000000000000..381631dca78bb --- /dev/null +++ b/x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/EnhancedTimestampDetectionTests.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.textstructure.structurefinder; + +import org.elasticsearch.xpack.core.textstructure.structurefinder.TextStructure; + +import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +/** + * Tests for enhanced timestamp detection covering the formats mentioned in GitHub issue 404 + */ +public class EnhancedTimestampDetectionTests extends TextStructureTestCase { + + private ScheduledExecutorService scheduler; + private TextStructureFinderManager manager; + + @Override + public void setUp() throws Exception { + super.setUp(); + scheduler = new org.elasticsearch.threadpool.Scheduler.SafeScheduledThreadPoolExecutor(1); + manager = new TextStructureFinderManager(scheduler); + } + + @Override + public void tearDown() throws Exception { + if (scheduler != null) { + scheduler.shutdown(); + } + super.tearDown(); + } + + public void testTimestampYMDSlash() throws Exception { + verifyTimestampYmdFormat(""" + 2025/07/10 10:30:35 INFO An informational message + 2025/07/10 10:31:42 WARN A warning message + 2025/07/10 10:32:15 ERROR An error message + """); + } + + public void testTimestampYMDDot() throws Exception { + verifyTimestampYmdFormat(""" + 2025.07.10 10:30:35 INFO An informational message + 2025.07.10 10:31:42 WARN A warning message + 2025.07.10 10:32:15 ERROR An error message + """); + } + + public void testTimestampYMDSlashMillisWithDot() throws Exception { + String sample = """ + 2025/07/10 10:30:35.123 INFO An informational message + 2025/07/10 10:31:42.123 WARN A warning message + 2025/07/10 10:32:15.123 ERROR An error message + """; + TextStructure structure = findStructure(sample); + assertThat(structure.getGrokPattern(), containsString("%{TIMESTAMP_YMD:timestamp}")); + assertNotNull("Should detect timestamp field", structure.getTimestampField()); + assertThat( + structure.getJavaTimestampFormats(), + containsInAnyOrder("yyyy/MM/dd HH:mm:ss.SSS", "yyyy.MM.dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss.SSS") + ); + verifyMappings(structure); + } + + public void testTimestampYMDDash() throws Exception { + // ISO_8601 should have higher priority than TIMESTAMP_YMD + verifyTimestampISO8601Format(""" + 2025-07-10 10:30:35 INFO An informational message + 2025-07-10 10:31:42 WARN A warning message + 2025-07-10 10:32:15 ERROR An error message + """); + } + + private void verifyTimestampYmdFormat(String sample) throws Exception { + TextStructure structure = findStructure(sample); + assertThat(structure.getGrokPattern(), containsString("%{TIMESTAMP_YMD:timestamp}")); + assertNotNull("Should detect timestamp field", structure.getTimestampField()); + assertThat( + structure.getJavaTimestampFormats(), + containsInAnyOrder("yyyy/MM/dd HH:mm:ss", "yyyy.MM.dd HH:mm:ss", "yyyy-MM-dd HH:mm:ss") + ); + verifyMappings(structure); + } + + private void verifyTimestampISO8601Format(String sample) throws Exception { + TextStructure structure = findStructure(sample); + assertThat(structure.getGrokPattern(), containsString("%{TIMESTAMP_ISO8601:timestamp}")); + assertNotNull("Should detect timestamp field", structure.getTimestampField()); + assertThat(structure.getJavaTimestampFormats(), containsInAnyOrder("yyyy-MM-dd HH:mm:ss")); + verifyMappings(structure); + } + + private static void verifyMappings(TextStructure structure) { + Map mappings = structure.getMappings(); + assertNotNull("Should have mappings", mappings); + Object propertiesValue = mappings.get("properties"); + assertThat(propertiesValue, instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map properties = (Map) propertiesValue; + + // Verify @timestamp field is properly configured + @SuppressWarnings("unchecked") + Map timestampMapping = (Map) properties.get("@timestamp"); + assertThat("@timestamp should be date type", timestampMapping.get("type"), equalTo("date")); + @SuppressWarnings("unchecked") + Map logLevelMapping = (Map) properties.get("log.level"); + assertThat("log.level should be keyword type", logLevelMapping.get("type"), equalTo("keyword")); + @SuppressWarnings("unchecked") + Map messageMapping = (Map) properties.get("message"); + assertThat("message should be text type", messageMapping.get("type"), equalTo("text")); + } + + /** + * Helper method to find text structure using the manager + */ + private TextStructure findStructure(String sample) throws Exception { + // Convert the string sample to an InputStream as required by the method signature + byte[] sampleBytes = sample.getBytes(java.nio.charset.StandardCharsets.UTF_8); + java.io.ByteArrayInputStream inputStream = new java.io.ByteArrayInputStream(sampleBytes); + + // Create TextStructureOverrides with ECS compatibility enabled + TextStructureOverrides overrides = new TextStructureOverrides.Builder().setEcsCompatibility("v1").build(); + + TextStructureFinder finder = manager.findTextStructure( + explanation, + TextStructureFinderManager.DEFAULT_IDEAL_SAMPLE_LINE_COUNT, + TextStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, + inputStream, + overrides, + org.elasticsearch.core.TimeValue.timeValueSeconds(30) + ); + + return finder.getStructure(); + } +} diff --git a/x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/TimestampFormatFinderTests.java b/x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/TimestampFormatFinderTests.java index 048a4c2e6e512..9d4f76c117be7 100644 --- a/x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/TimestampFormatFinderTests.java +++ b/x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/TimestampFormatFinderTests.java @@ -1335,6 +1335,50 @@ public void testFindFormatGivenOnlySystemDate() { } + public void testFindFormatGivenYyyyMmDdWithSlashes() { + Consumer testFindFormatGivenYyyyMmDdWithSlashesAndEcsCompatibility = (ecsCompatibility) -> { + validateTimestampMatch( + "2018/05/15 17:14:56", + "TIMESTAMP_YMD", + "\\b\\d{4}[./-]\\d{2}[./-]\\d{2} \\d{2}:\\d{2}:\\d{2}(?:[.,]\\d+)?\\b", + List.of("yyyy/MM/dd HH:mm:ss", "yyyy.MM.dd HH:mm:ss", "yyyy-MM-dd HH:mm:ss"), + 1526400896000L, + ecsCompatibility + ); + }; + ecsCompatibilityModes.forEach(testFindFormatGivenYyyyMmDdWithSlashesAndEcsCompatibility); + } + + public void testFindFormatGivenYyyyMmDdWithDashes() { + Consumer testFindFormatGivenYyyyMmDdWithDashesAndEcsCompatibility = (ecsCompatibility) -> { + validateTimestampMatch( + "2018-05-15 17:14:56", + "TIMESTAMP_ISO8601", // TIMESTAMP_ISO8601 should have precedence + "\\b\\d{4}-\\d{2}-\\d{2}[T ]\\d{2}:\\d{2}", + List.of("yyyy-MM-dd HH:mm:ss"), + 1526400896000L, + ecsCompatibility + ); + }; + ecsCompatibilityModes.forEach(testFindFormatGivenYyyyMmDdWithDashesAndEcsCompatibility); + } + + public void testFindFormatGivenMmmDCommaYyyy() { + + Consumer testFindFormatGivenMmmDCommaYyyyAndEcsCompatibility = (ecsCompatibility) -> { + validateTimestampMatch( + "May 15, 2018", + "CUSTOM_TIMESTAMP", + "\\b[A-Z][a-z]{2} \\d{1,2}, \\d{4}\\b", + "MMM d, yyyy", + 1526338800000L, + ecsCompatibility + ); + }; + + ecsCompatibilityModes.forEach(testFindFormatGivenMmmDCommaYyyyAndEcsCompatibility); + } + public void testCustomOverrideMatchingBuiltInFormat() { String overrideFormat = "yyyy-MM-dd HH:mm:ss,SSS"; @@ -1857,12 +1901,10 @@ private void validateFindInFullMessage( ); timestampFormatFinder.addSample(message); timestampFormatFinder.selectBestMatch(); - if ("CATALINA7_DATESTAMP".equals(expectedGrokPatternName)) { - if (ecsCompatibility) { - assertEquals(expectedGrokPatternName, timestampFormatFinder.getGrokPatternName()); - } else { - assertEquals("CATALINA_DATESTAMP", timestampFormatFinder.getGrokPatternName()); - } + if ("CATALINA7_DATESTAMP".equals(expectedGrokPatternName) && ecsCompatibility == false) { + assertEquals("CATALINA_DATESTAMP", timestampFormatFinder.getGrokPatternName()); + } else { + assertEquals(expectedGrokPatternName, timestampFormatFinder.getGrokPatternName()); } assertEquals(expectedSimplePattern.pattern(), timestampFormatFinder.getSimplePattern().pattern()); assertEquals(expectedJavaTimestampFormats, timestampFormatFinder.getJavaTimestampFormats());