diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc index 47ca4859b3f20..ab845280eedb3 100644 --- a/docs/reference/mapping/types/date.asciidoc +++ b/docs/reference/mapping/types/date.asciidoc @@ -133,8 +133,7 @@ The following parameters are accepted by `date` fields: `locale`:: The locale to use when parsing dates since months do not have the same names - and/or abbreviations in all languages. The default is the - https://docs.oracle.com/javase/8/docs/api/java/util/Locale.html#ROOT[`ROOT` locale]. + and/or abbreviations in all languages. The default is ENGLISH. <>:: diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java index 5455daf0a79ec..227557590731e 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.aggregations.bucket.histogram; -import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.aggregations.bucket.AggregationMultiBucketAggregationTestCase; import org.elasticsearch.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder.RoundingInfo; @@ -28,7 +27,6 @@ import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.test.InternalAggregationTestCase; -import org.elasticsearch.test.TransportVersionUtils; import java.io.IOException; import java.time.Instant; @@ -459,33 +457,6 @@ public void testCreateWithReplacementBuckets() { assertThat(copy.getInterval(), equalTo(orig.getInterval())); } - public void testSerializationPre830() throws IOException { - // we need to test without sub-aggregations, otherwise we need to also update the interval within the inner aggs - InternalAutoDateHistogram instance = createTestInstance( - randomAlphaOfLengthBetween(3, 7), - createTestMetadata(), - InternalAggregations.EMPTY - ); - TransportVersion version = TransportVersionUtils.randomVersionBetween( - random(), - TransportVersions.MINIMUM_COMPATIBLE, - TransportVersionUtils.getPreviousVersion(TransportVersions.V_8_3_0) - ); - InternalAutoDateHistogram deserialized = copyInstance(instance, version); - assertEquals(1, deserialized.getBucketInnerInterval()); - - InternalAutoDateHistogram modified = new InternalAutoDateHistogram( - deserialized.getName(), - deserialized.getBuckets(), - deserialized.getTargetBuckets(), - deserialized.getBucketInfo(), - deserialized.getFormatter(), - deserialized.getMetadata(), - instance.getBucketInnerInterval() - ); - assertEqualInstances(instance, modified); - } - public void testReadFromPre830() throws IOException { byte[] bytes = Base64.getDecoder() .decode( diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/date_agg_per_day_of_week.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/date_agg_per_day_of_week.yml index dc349171e3a27..5157d563540b1 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/date_agg_per_day_of_week.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/date_agg_per_day_of_week.yml @@ -1,8 +1,8 @@ --- setup: - requires: - cluster_features: ["gte_v7.7.0"] - reason: "Start of the week Monday was enabled in a backport to 7.7 PR#50916" + cluster_features: ["gte_v8.16.0"] + reason: "Start of the week Sunday was changed in 8.16 as part of the locale changes" - do: indices.create: @@ -25,7 +25,7 @@ setup: --- # The inserted document has a field date=2009-11-15T14:12:12 which is Sunday. -# When aggregating per day of the week this should be considered as last day of the week (7) +# When aggregating per day of the week this should be considered as first day of the week (1) # and this value should be used in 'key_as_string' "Date aggregartion per day of week": - do: @@ -44,4 +44,4 @@ setup: - match: {hits.total: 1} - length: { aggregations.test.buckets: 1 } - - match: { aggregations.test.buckets.0.key_as_string: "7" } + - match: { aggregations.test.buckets.0.key_as_string: "1" } diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml index d9298a832e650..82371c973407c 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml @@ -30,7 +30,7 @@ setup: cluster_features: "gte_v8.15.0" reason: fixed in 8.15.0 - do: - catch: /Cannot format stat \[sum\] with format \[DocValueFormat.DateTime\(format\[date_hour_minute_second_millis\] locale\[\], Z, MILLISECONDS\)\]/ + catch: /Cannot format stat \[sum\] with format \[DocValueFormat.DateTime\(format\[date_hour_minute_second_millis\] locale\[(en)?\], Z, MILLISECONDS\)\]/ search: index: test_date body: diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index a4a4d60b9d40c..72485bb873d57 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -234,6 +234,7 @@ static TransportVersion def(int id) { public static final TransportVersion RRF_QUERY_REWRITE = def(8_758_00_0); public static final TransportVersion SEARCH_FAILURE_STATS = def(8_759_00_0); public static final TransportVersion INGEST_GEO_DATABASE_PROVIDERS = def(8_760_00_0); + public static final TransportVersion DATE_TIME_DOC_VALUES_LOCALES = def(8_761_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index 651d9e76e84a2..481901f7c03ce 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -165,10 +165,11 @@ public void doValidate(MappingLookup lookup) { Map configuredSettings = XContentHelper.convertToMap(BytesReference.bytes(builder), false, XContentType.JSON).v2(); configuredSettings = (Map) configuredSettings.values().iterator().next(); - // Only type, meta and format attributes are allowed: + // Only type, meta, format, and locale attributes are allowed: configuredSettings.remove("type"); configuredSettings.remove("meta"); configuredSettings.remove("format"); + configuredSettings.remove("locale"); // ignoring malformed values is disallowed (see previous check), // however if `index.mapping.ignore_malformed` has been set to true then diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 7d414839bff90..012f08a1db01d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -83,11 +83,16 @@ public final class DateFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "date"; public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos"; - public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis"); + public static final Locale DEFAULT_LOCALE = Locale.ENGLISH; + // although the locale doesn't affect the results, tests still check formatter equality, which does include locale + public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis") + .withLocale(DEFAULT_LOCALE); public static final DateFormatter DEFAULT_DATE_TIME_NANOS_FORMATTER = DateFormatter.forPattern( "strict_date_optional_time_nanos||epoch_millis" - ); - private static final DateMathParser EPOCH_MILLIS_PARSER = DateFormatter.forPattern("epoch_millis").toDateMathParser(); + ).withLocale(DEFAULT_LOCALE); + private static final DateMathParser EPOCH_MILLIS_PARSER = DateFormatter.forPattern("epoch_millis") + .withLocale(DEFAULT_LOCALE) + .toDateMathParser(); public enum Resolution { MILLISECONDS(CONTENT_TYPE, NumericType.DATE, DateMillisDocValuesField::new) { @@ -236,7 +241,7 @@ public static final class Builder extends FieldMapper.Builder { private final Parameter locale = new Parameter<>( "locale", false, - () -> Locale.ROOT, + () -> DEFAULT_LOCALE, (n, c, o) -> LocaleUtils.parse(o.toString()), m -> toType(m).locale, (xContentBuilder, n, v) -> xContentBuilder.field(n, v.toString()), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java index e519fec09ce78..341944c3d687a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java @@ -69,7 +69,7 @@ private static class Builder extends AbstractScriptFieldType.Builder o == null ? null : LocaleUtils.parse(o.toString()), RuntimeField.initializerNotSupported(), (b, n, v) -> { - if (v != null && false == v.equals(Locale.ROOT)) { + if (v != null && false == v.equals(DateFieldMapper.DEFAULT_LOCALE)) { b.field(n, v.toString()); } }, @@ -97,7 +97,7 @@ protected AbstractScriptFieldType createFieldType( OnScriptError onScriptError ) { String pattern = format.getValue() == null ? DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.pattern() : format.getValue(); - Locale locale = this.locale.getValue() == null ? Locale.ROOT : this.locale.getValue(); + Locale locale = this.locale.getValue() == null ? DateFieldMapper.DEFAULT_LOCALE : this.locale.getValue(); DateFormatter dateTimeFormatter = DateFormatter.forPattern(pattern, supportedVersion).withLocale(locale); return new DateScriptFieldType(name, factory, dateTimeFormatter, script, meta, onScriptError); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index af0dc0c0ad7fe..6ca30304201b2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -59,6 +59,7 @@ public class RangeFieldMapper extends FieldMapper { public static class Defaults { public static final DateFormatter DATE_FORMATTER = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; + public static final Locale LOCALE = DateFieldMapper.DEFAULT_LOCALE; } // this is private since it has a different default @@ -83,7 +84,7 @@ public static class Builder extends FieldMapper.Builder { private final Parameter locale = new Parameter<>( "locale", false, - () -> Locale.ROOT, + () -> Defaults.LOCALE, (n, c, o) -> LocaleUtils.parse(o.toString()), m -> toType(m).locale, (xContentBuilder, n, v) -> xContentBuilder.field(n, v.toString()), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java index f086526eec78e..b60feb4d5746e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java @@ -179,7 +179,7 @@ public static boolean parseMultiField( public static DateFormatter parseDateTimeFormatter(Object node) { if (node instanceof String) { - return DateFormatter.forPattern((String) node); + return DateFormatter.forPattern((String) node).withLocale(DateFieldMapper.DEFAULT_LOCALE); } throw new IllegalArgumentException("Invalid format: [" + node.toString() + "]: expected string value"); } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index be9ad0ed0a9cd..f1d4f678c5fb9 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateMathParser; +import org.elasticsearch.common.util.LocaleUtils; import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; @@ -236,9 +237,12 @@ private DateTime(DateFormatter formatter, ZoneId timeZone, DateFieldMapper.Resol public DateTime(StreamInput in) throws IOException { String formatterPattern = in.readString(); + Locale locale = in.getTransportVersion().onOrAfter(TransportVersions.DATE_TIME_DOC_VALUES_LOCALES) + ? LocaleUtils.parse(in.readString()) + : DateFieldMapper.DEFAULT_LOCALE; String zoneId = in.readString(); this.timeZone = ZoneId.of(zoneId); - this.formatter = DateFormatter.forPattern(formatterPattern).withZone(this.timeZone); + this.formatter = DateFormatter.forPattern(formatterPattern).withZone(this.timeZone).withLocale(locale); this.parser = formatter.toDateMathParser(); this.resolution = DateFieldMapper.Resolution.ofOrdinal(in.readVInt()); if (in.getTransportVersion().between(TransportVersions.V_7_7_0, TransportVersions.V_8_0_0)) { @@ -259,6 +263,9 @@ public String getWriteableName() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(formatter.pattern()); + if (out.getTransportVersion().onOrAfter(TransportVersions.DATE_TIME_DOC_VALUES_LOCALES)) { + out.writeString(formatter.locale().toString()); + } out.writeString(timeZone.getId()); out.writeVInt(resolution.ordinal()); if (out.getTransportVersion().between(TransportVersions.V_7_7_0, TransportVersions.V_8_0_0)) {