Extend ISO8601 parser to specify forbidden fields, allowing it to be used on more formats (#108606)

Use the new ISO8601 parser for `strict_year`, `strict_year_month`, `strict_date_time`, `strict_date_time_no_millis`, `strict_date_hour_minute_second`, `strict_date_hour_minute_second_millis`, and `strict_date_hour_minute_second_fraction` date formats.

Author: thecoop

docs/changelog/108606.yaml

@@ -0,0 +1,14 @@
+pr: 108606
+summary: "Extend ISO8601 datetime parser to specify forbidden fields, allowing it to be used\
+  \ on more formats"
+area: Infra/Core
+type: enhancement
+issues: []
+highlight:
+  title: New custom parser for more ISO-8601 date formats
+  body: |-
+    Following on from #106486, this extends the custom ISO-8601 datetime parser to cover the `strict_year`,
+    `strict_year_month`, `strict_date_time`, `strict_date_time_no_millis`, `strict_date_hour_minute_second`,
+    `strict_date_hour_minute_second_millis`, and `strict_date_hour_minute_second_fraction` date formats.
+    As before, the parser will use the existing java.time parser if there are parsing issues, and the
+    `es.datetime.java_time_parsers=true` JVM property will force the use of the old parsers regardless.

server/src/main/java/org/elasticsearch/common/time/DateFormatters.java

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.time;
+
+enum DecimalSeparator {
+    DOT,
+    COMMA,
+    BOTH
+}

server/src/main/java/org/elasticsearch/common/time/DecimalSeparator.java

@@ -24,8 +24,14 @@ class Iso8601DateTimeParser implements DateTimeParser {
     // and we already account for . or , in decimals
     private final Locale locale;
 
-    Iso8601DateTimeParser(Set<ChronoField> mandatoryFields, boolean optionalTime) {
-        parser = new Iso8601Parser(mandatoryFields, optionalTime, Map.of());
+    Iso8601DateTimeParser(
+        Set<ChronoField> mandatoryFields,
+        boolean optionalTime,
+        ChronoField maxAllowedField,
+        DecimalSeparator decimalSeparator,
+        TimezonePresence timezonePresence
+    ) {
+        parser = new Iso8601Parser(mandatoryFields, optionalTime, maxAllowedField, decimalSeparator, timezonePresence, Map.of());
         timezone = null;
         locale = null;
     }
@@ -57,7 +63,18 @@ public DateTimeParser withLocale(Locale locale) {
     }
 
     Iso8601DateTimeParser withDefaults(Map<ChronoField, Integer> defaults) {
-        return new Iso8601DateTimeParser(new Iso8601Parser(parser.mandatoryFields(), parser.optionalTime(), defaults), timezone, locale);
+        return new Iso8601DateTimeParser(
+            new Iso8601Parser(
+                parser.mandatoryFields(),
+                parser.optionalTime(),
+                parser.maxAllowedField(),
+                parser.decimalSeparator(),
+                parser.timezonePresence(),
+                defaults
+            ),
+            timezone,
+            locale
+        );
     }
 
     @Override

server/src/main/java/org/elasticsearch/common/time/Iso8601DateTimeParser.java

@@ -13,16 +13,18 @@
 import java.time.DateTimeException;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 import java.time.temporal.ChronoField;
 import java.util.EnumMap;
 import java.util.EnumSet;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 /**
  * Parses datetimes in ISO8601 format (and subsequences thereof).
  * <p>
- * This is faster than the generic parsing in {@link java.time.format.DateTimeFormatter}, as this is hard-coded and specific to ISO-8601.
+ * This is faster than the generic parsing in {@link DateTimeFormatter}, as this is hard-coded and specific to ISO-8601.
  * Various public libraries provide their own variant of this mechanism. We use our own for a few reasons:
  * <ul>
  *     <li>
@@ -37,13 +39,14 @@
  */
 class Iso8601Parser {
 
-    private static final Set<ChronoField> VALID_MANDATORY_FIELDS = EnumSet.of(
+    private static final Set<ChronoField> VALID_SPECIFIED_FIELDS = EnumSet.of(
         ChronoField.YEAR,
         ChronoField.MONTH_OF_YEAR,
         ChronoField.DAY_OF_MONTH,
         ChronoField.HOUR_OF_DAY,
         ChronoField.MINUTE_OF_HOUR,
-        ChronoField.SECOND_OF_MINUTE
+        ChronoField.SECOND_OF_MINUTE,
+        ChronoField.NANO_OF_SECOND
     );
 
     private static final Set<ChronoField> VALID_DEFAULT_FIELDS = EnumSet.of(
@@ -57,31 +60,51 @@ class Iso8601Parser {
 
     private final Set<ChronoField> mandatoryFields;
     private final boolean optionalTime;
+    @Nullable
+    private final ChronoField maxAllowedField;
+    private final DecimalSeparator decimalSeparator;
+    private final TimezonePresence timezonePresence;
     private final Map<ChronoField, Integer> defaults;
 
     /**
      * Constructs a new {@code Iso8601Parser} object
      *
-     * @param mandatoryFields
-     *          The set of fields that must be present for a valid parse. These should be specified in field order
-     *          (eg if {@link ChronoField#DAY_OF_MONTH} is specified, {@link ChronoField#MONTH_OF_YEAR} should also be specified).
-     *          {@link ChronoField#YEAR} is always mandatory.
-     * @param optionalTime
-     *          {@code false} if the presence of time fields follows {@code mandatoryFields},
-     *          {@code true} if a time component is always optional, despite the presence of time fields in {@code mandatoryFields}.
-     *          This makes it possible to specify 'time is optional, but if it is present, it must have these fields'
-     *          by settings {@code optionalTime = true} and putting time fields such as {@link ChronoField#HOUR_OF_DAY}
-     *          and {@link ChronoField#MINUTE_OF_HOUR} in {@code mandatoryFields}.
-     * @param defaults
-     *          Map of default field values, if they are not present in the parsed string.
+     * @param mandatoryFields  The set of fields that must be present for a valid parse. These should be specified in field order
+     *                         (eg if {@link ChronoField#DAY_OF_MONTH} is specified,
+     *                         {@link ChronoField#MONTH_OF_YEAR} should also be specified).
+     *                         {@link ChronoField#YEAR} is always mandatory.
+     * @param optionalTime     {@code false} if the presence of time fields follows {@code mandatoryFields},
+     *                         {@code true} if a time component is always optional,
+     *                         despite the presence of time fields in {@code mandatoryFields}.
+     *                         This makes it possible to specify 'time is optional, but if it is present, it must have these fields'
+     *                         by settings {@code optionalTime = true} and putting time fields such as {@link ChronoField#HOUR_OF_DAY}
+     *                         and {@link ChronoField#MINUTE_OF_HOUR} in {@code mandatoryFields}.
+     * @param maxAllowedField  The most-specific field allowed in the parsed string,
+     *                         or {@code null} if everything up to nanoseconds is allowed.
+     * @param decimalSeparator The decimal separator that is allowed.
+     * @param timezonePresence Specifies if the timezone is optional, mandatory, or forbidden.
+     * @param defaults         Map of default field values, if they are not present in the parsed string.
      */
-    Iso8601Parser(Set<ChronoField> mandatoryFields, boolean optionalTime, Map<ChronoField, Integer> defaults) {
-        checkChronoFields(mandatoryFields, VALID_MANDATORY_FIELDS);
+    Iso8601Parser(
+        Set<ChronoField> mandatoryFields,
+        boolean optionalTime,
+        @Nullable ChronoField maxAllowedField,
+        DecimalSeparator decimalSeparator,
+        TimezonePresence timezonePresence,
+        Map<ChronoField, Integer> defaults
+    ) {
+        checkChronoFields(mandatoryFields, VALID_SPECIFIED_FIELDS);
+        if (maxAllowedField != null && VALID_SPECIFIED_FIELDS.contains(maxAllowedField) == false) {
+            throw new IllegalArgumentException("Invalid chrono field specified " + maxAllowedField);
+        }
         checkChronoFields(defaults.keySet(), VALID_DEFAULT_FIELDS);
 
         this.mandatoryFields = EnumSet.of(ChronoField.YEAR); // year is always mandatory
         this.mandatoryFields.addAll(mandatoryFields);
         this.optionalTime = optionalTime;
+        this.maxAllowedField = maxAllowedField;
+        this.decimalSeparator = Objects.requireNonNull(decimalSeparator);
+        this.timezonePresence = Objects.requireNonNull(timezonePresence);
         this.defaults = defaults.isEmpty() ? Map.of() : new EnumMap<>(defaults);
     }
 
@@ -103,6 +126,18 @@ Set<ChronoField> mandatoryFields() {
         return mandatoryFields;
     }
 
+    ChronoField maxAllowedField() {
+        return maxAllowedField;
+    }
+
+    DecimalSeparator decimalSeparator() {
+        return decimalSeparator;
+    }
+
+    TimezonePresence timezonePresence() {
+        return timezonePresence;
+    }
+
     private boolean isOptional(ChronoField field) {
         return mandatoryFields.contains(field) == false;
     }
@@ -186,7 +221,7 @@ private ParseResult parse(CharSequence str, @Nullable ZoneId defaultTimezone) {
                 : ParseResult.error(4);
         }
 
-        if (str.charAt(4) != '-') return ParseResult.error(4);
+        if (str.charAt(4) != '-' || maxAllowedField == ChronoField.YEAR) return ParseResult.error(4);
 
         // MONTHS
         Integer months = parseInt(str, 5, 7);
@@ -208,7 +243,7 @@ private ParseResult parse(CharSequence str, @Nullable ZoneId defaultTimezone) {
                 : ParseResult.error(7);
         }
 
-        if (str.charAt(7) != '-') return ParseResult.error(7);
+        if (str.charAt(7) != '-' || maxAllowedField == ChronoField.MONTH_OF_YEAR) return ParseResult.error(7);
 
         // DAYS
         Integer days = parseInt(str, 8, 10);
@@ -230,7 +265,7 @@ private ParseResult parse(CharSequence str, @Nullable ZoneId defaultTimezone) {
                 : ParseResult.error(10);
         }
 
-        if (str.charAt(10) != 'T') return ParseResult.error(10);
+        if (str.charAt(10) != 'T' || maxAllowedField == ChronoField.DAY_OF_MONTH) return ParseResult.error(10);
         if (len == 11) {
             return isOptional(ChronoField.HOUR_OF_DAY)
                 ? new ParseResult(
@@ -252,7 +287,7 @@ private ParseResult parse(CharSequence str, @Nullable ZoneId defaultTimezone) {
         Integer hours = parseInt(str, 11, 13);
         if (hours == null || hours > 23) return ParseResult.error(11);
         if (len == 13) {
-            return isOptional(ChronoField.MINUTE_OF_HOUR)
+            return isOptional(ChronoField.MINUTE_OF_HOUR) && timezonePresence != TimezonePresence.MANDATORY
                 ? new ParseResult(
                     withZoneOffset(
                         years,
@@ -285,13 +320,13 @@ private ParseResult parse(CharSequence str, @Nullable ZoneId defaultTimezone) {
                 : ParseResult.error(13);
         }
 
-        if (str.charAt(13) != ':') return ParseResult.error(13);
+        if (str.charAt(13) != ':' || maxAllowedField == ChronoField.HOUR_OF_DAY) return ParseResult.error(13);
 
         // MINUTES + timezone
         Integer minutes = parseInt(str, 14, 16);
         if (minutes == null || minutes > 59) return ParseResult.error(14);
         if (len == 16) {
-            return isOptional(ChronoField.SECOND_OF_MINUTE)
+            return isOptional(ChronoField.SECOND_OF_MINUTE) && timezonePresence != TimezonePresence.MANDATORY
                 ? new ParseResult(
                     withZoneOffset(
                         years,
@@ -324,15 +359,17 @@ private ParseResult parse(CharSequence str, @Nullable ZoneId defaultTimezone) {
                 : ParseResult.error(16);
         }
 
-        if (str.charAt(16) != ':') return ParseResult.error(16);
+        if (str.charAt(16) != ':' || maxAllowedField == ChronoField.MINUTE_OF_HOUR) return ParseResult.error(16);
 
         // SECONDS + timezone
         Integer seconds = parseInt(str, 17, 19);
         if (seconds == null || seconds > 59) return ParseResult.error(17);
         if (len == 19) {
-            return new ParseResult(
-                withZoneOffset(years, months, days, hours, minutes, seconds, defaultZero(ChronoField.NANO_OF_SECOND), defaultTimezone)
-            );
+            return isOptional(ChronoField.NANO_OF_SECOND) && timezonePresence != TimezonePresence.MANDATORY
+                ? new ParseResult(
+                    withZoneOffset(years, months, days, hours, minutes, seconds, defaultZero(ChronoField.NANO_OF_SECOND), defaultTimezone)
+                )
+                : ParseResult.error(19);
         }
         if (isZoneId(str, 19)) {
             ZoneId timezone = parseZoneId(str, 19);
@@ -343,11 +380,9 @@ private ParseResult parse(CharSequence str, @Nullable ZoneId defaultTimezone) {
                 : ParseResult.error(19);
         }
 
-        char decSeparator = str.charAt(19);
-        if (decSeparator != '.' && decSeparator != ',') return ParseResult.error(19);
+        if (checkDecimalSeparator(str.charAt(19)) == false || maxAllowedField == ChronoField.SECOND_OF_MINUTE) return ParseResult.error(19);
 
         // NANOS + timezone
-        // nanos are always optional
         // the last number could be millis or nanos, or any combination in the middle
         // so we keep parsing numbers until we get to not a number
         int nanos = 0;
@@ -364,7 +399,9 @@ private ParseResult parse(CharSequence str, @Nullable ZoneId defaultTimezone) {
         nanos *= NANO_MULTIPLICANDS[29 - pos];
 
         if (len == pos) {
-            return new ParseResult(withZoneOffset(years, months, days, hours, minutes, seconds, nanos, defaultTimezone));
+            return timezonePresence != TimezonePresence.MANDATORY
+                ? new ParseResult(withZoneOffset(years, months, days, hours, minutes, seconds, nanos, defaultTimezone))
+                : ParseResult.error(pos);
         }
         if (isZoneId(str, pos)) {
             ZoneId timezone = parseZoneId(str, pos);
@@ -377,6 +414,16 @@ private ParseResult parse(CharSequence str, @Nullable ZoneId defaultTimezone) {
         return ParseResult.error(pos);
     }
 
+    private boolean checkDecimalSeparator(char separator) {
+        boolean isDot = separator == '.';
+        boolean isComma = separator == ',';
+        return switch (decimalSeparator) {
+            case DOT -> isDot;
+            case COMMA -> isComma;
+            case BOTH -> isDot || isComma;
+        };
+    }
+
     private static boolean isZoneId(CharSequence str, int pos) {
         // all region zoneIds must start with [A-Za-z] (see ZoneId#of)
         // this also covers Z and UT/UTC/GMT zone variants
@@ -385,10 +432,14 @@ private static boolean isZoneId(CharSequence str, int pos) {
     }
 
     /**
-     * This parses the zone offset, which is of the format accepted by {@link java.time.ZoneId#of(String)}.
+     * This parses the zone offset, which is of the format accepted by {@link ZoneId#of(String)}.
      * It has fast paths for numerical offsets, but falls back on {@code ZoneId.of} for non-trivial zone ids.
      */
     private ZoneId parseZoneId(CharSequence str, int pos) {
+        if (timezonePresence == TimezonePresence.FORBIDDEN) {
+            return null;
+        }
+
         int len = str.length();
         char first = str.charAt(pos);
 

server/src/main/java/org/elasticsearch/common/time/Iso8601Parser.java

@@ -18,7 +18,6 @@
 import java.time.temporal.TemporalAccessor;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -149,19 +148,24 @@ static DateFormatter combined(String input, List<DateFormatter> formatters) {
         assert formatters.isEmpty() == false;
 
         DateTimePrinter printer = null;
-        List<DateTimeParser> parsers = new ArrayList<>(formatters.size());
-        List<DateTimeParser> roundUpParsers = new ArrayList<>(formatters.size());
+        List<DateTimeParser[]> parsers = new ArrayList<>(formatters.size());
+        List<DateTimeParser[]> roundUpParsers = new ArrayList<>(formatters.size());
 
         for (DateFormatter formatter : formatters) {
             JavaDateFormatter javaDateFormatter = (JavaDateFormatter) formatter;
             if (printer == null) {
                 printer = javaDateFormatter.printer;
             }
-            Collections.addAll(parsers, javaDateFormatter.parsers);
-            Collections.addAll(roundUpParsers, javaDateFormatter.roundupParsers);
+            parsers.add(javaDateFormatter.parsers);
+            roundUpParsers.add(javaDateFormatter.roundupParsers);
         }
 
-        return new JavaDateFormatter(input, printer, roundUpParsers.toArray(DateTimeParser[]::new), parsers.toArray(DateTimeParser[]::new));
+        return new JavaDateFormatter(
+            input,
+            printer,
+            roundUpParsers.stream().flatMap(Arrays::stream).toArray(DateTimeParser[]::new),
+            parsers.stream().flatMap(Arrays::stream).toArray(DateTimeParser[]::new)
+        );
     }
 
     private JavaDateFormatter(String format, DateTimePrinter printer, DateTimeParser[] roundupParsers, DateTimeParser[] parsers) {

server/src/main/java/org/elasticsearch/common/time/JavaDateFormatter.java

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.time;
+
+enum TimezonePresence {
+    OPTIONAL,
+    MANDATORY,
+    FORBIDDEN
+}