[8.15] Use CLDR locale provider on JDK 23 (#112544)

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -167,7 +167,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
             vmParameters = [
               '-ea',
               '-Djava.security.manager=allow',
-              '-Djava.locale.providers=SPI,COMPAT',
+              '-Djava.locale.providers=SPI,CLDR',
               '-Djava.library.path=' + testLibraryPath,
               '-Djna.library.path=' + testLibraryPath,
               // TODO: only open these for mockito when it is modularized

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -92,7 +92,7 @@ public void execute(Task t) {
                     mkdirs(test.getWorkingDir().toPath().resolve("temp").toFile());
 
                     // TODO remove once jvm.options are added to test system properties
-                    test.systemProperty("java.locale.providers", "SPI,COMPAT");
+                    test.systemProperty("java.locale.providers", "SPI,CLDR");
                 }
             });
             test.getJvmArgumentProviders().add(nonInputProperties);

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.core.UpdateForV9;
 
 import java.io.File;
 import java.nio.file.Path;
@@ -64,11 +65,7 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 "-Dlog4j.shutdownHookEnabled=false",
                 "-Dlog4j2.disable.jmx=true",
                 "-Dlog4j2.formatMsgNoLookups=true",
-                /*
-                 * Due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise time/date
-                 * parsing will break in an incompatible way for some date patterns and locales.
-                 */
-                "-Djava.locale.providers=SPI,COMPAT",
+                "-Djava.locale.providers=" + getLocaleProviders(),
                 /*
                  * Temporarily suppress illegal reflective access in searchable snapshots shared cache preallocation; this is temporary
                  * while we explore alternatives. See org.elasticsearch.xpack.searchablesnapshots.preallocate.Preallocate.
@@ -87,6 +84,16 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
         ).filter(e -> e.isEmpty() == false).collect(Collectors.toList());
     }
 
+    @UpdateForV9    // only use CLDR in v9+
+    private static String getLocaleProviders() {
+        /*
+         * Specify SPI to load IsoCalendarDataProvider (see #48209), specifying the first day of week as Monday.
+         * When on pre-23, use COMPAT instead to maintain existing date formats as much as we can.
+         * When on JDK 23+, use the default CLDR locale database, as COMPAT was removed in JDK 23.
+         */
+        return Runtime.version().feature() >= 23 ? "SPI,CLDR" : "SPI,COMPAT";
+    }
+
     /*
      * The virtual file /proc/self/cgroup should list the current cgroup
      * membership. For each hierarchy, you can follow the cgroup path from

docs/painless/painless-contexts/painless-field-context.asciidoc

@@ -64,14 +64,14 @@ actors that appear in each play:
 ----
 GET seats/_search
 {
-  "size": 2, 
+  "size": 2,
   "query": {
     "match_all": {}
   },
   "script_fields": {
     "day-of-week": {
       "script": {
-        "source": "doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)"
+        "source": "doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ENGLISH)"
       }
     },
     "number-of-actors": {
@@ -132,4 +132,4 @@ GET seats/_search
   }
 }
 ----
-// TESTRESPONSE[s/"took" : 68/"took" : "$body.took"/]
+// TESTRESPONSE[s/"took" : 68/"took" : "$body.took"/]

docs/painless/painless-guide/painless-execute-script.asciidoc

@@ -749,7 +749,7 @@ POST /_scripts/painless/_execute
 {
   "script": {
     "source": """
-      emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+      emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
     """
   },
   "context": "keyword_field",

docs/reference/aggregations/bucket/composite-aggregation.asciidoc

@@ -156,7 +156,7 @@ GET /_search
       "type": "keyword",
       "script": """
         emit(doc['timestamp'].value.dayOfWeekEnum
-          .getDisplayName(TextStyle.FULL, Locale.ROOT))
+          .getDisplayName(TextStyle.FULL, Locale.ENGLISH))
       """
     }
   },

docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc

@@ -582,7 +582,7 @@ For example, the offset of `+19d` will result in buckets with names like `2022-0
 
 Increasing the offset to `+20d`, each document will appear in a bucket for the previous month,
 with all bucket keys ending with the same day of the month, as normal.
-However, further increasing to `+28d`, 
+However, further increasing to `+28d`,
 what used to be a February bucket has now become `"2022-03-01"`.
 
 [source,console,id=datehistogram-aggregation-offset-example-28d]
@@ -819,7 +819,7 @@ POST /sales/_search?size=0
   "runtime_mappings": {
     "date.day_of_week": {
       "type": "keyword",
-      "script": "emit(doc['date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+      "script": "emit(doc['date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
     }
   },
   "aggs": {

docs/reference/indices/index-templates.asciidoc

@@ -102,7 +102,7 @@ PUT _component_template/runtime_component_template
         "day_of_week": {
           "type": "keyword",
           "script": {
-            "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+            "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
           }
         }
       }

docs/reference/mapping/runtime.asciidoc

@@ -135,7 +135,7 @@ PUT my-index-000001/
       "day_of_week": {
         "type": "keyword",
         "script": {
-          "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+          "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
         }
       }
     },
@@ -291,7 +291,7 @@ GET my-index-000001/_search
     "day_of_week": {
       "type": "keyword",
       "script": {
-        "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+        "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
       }
     }
   },
@@ -667,7 +667,7 @@ PUT my-index-000001/
       "day_of_week": {
         "type": "keyword",
         "script": {
-          "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+          "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
         }
       }
     },

docs/reference/ml/ml-shared.asciidoc

@@ -913,7 +913,7 @@ For example:
   "day_of_week": {
     "type": "keyword",
     "script": {
-      "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+      "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
     }
   }
 }

docs/reference/search/search-your-data/search-api.asciidoc

@@ -173,7 +173,7 @@ GET /my-index-000001/_search
       "script": {
         "source":
         """emit(doc['@timestamp'].value.dayOfWeekEnum
-        .getDisplayName(TextStyle.FULL, Locale.ROOT))"""
+        .getDisplayName(TextStyle.FULL, Locale.ENGLISH))"""
       }
     }
   },

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/65_runtime_doc_values.yml

@@ -12,7 +12,7 @@ setup:
                 script:
                   source: |
                     for (date in field('date')) {
-                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                     }
               total_value_double:
                 type: double
@@ -55,7 +55,7 @@ setup:
                   source: |
                     if (doc.containsKey('date')) {
                       for (date in doc['date']) {
-                        emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                        emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                       }
                     }
               doc_total_value_double:
@@ -737,7 +737,7 @@ setup:
               script:
                 source: |
                   for (date in field('date')) {
-                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                   }
           sort: [ { rank: asc } ]
           script_fields:
@@ -758,7 +758,7 @@ setup:
               script:
                 source: |
                   for (date in field('date')) {
-                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                   }
           sort: [ { rank: asc } ]
           script_fields:
@@ -924,7 +924,7 @@ setup:
                 source: |
                   if (doc.containsKey('date')) {
                     for (date in doc['date']) {
-                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                     }
                   }
           sort: [ { rank: asc } ]
@@ -947,7 +947,7 @@ setup:
                 source: |
                   if (doc.containsKey('date')) {
                     for (date in doc['date']) {
-                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                     }
                   }
           sort: [ { rank: asc } ]
@@ -1133,7 +1133,7 @@ setup:
               script:
                 source: |
                   for (date in field('date')) {
-                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                   }
           sort: [ { rank: asc } ]
           script_fields:
@@ -1156,7 +1156,7 @@ setup:
               script:
                 source: |
                   for (date in field('date')) {
-                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                   }
           sort: [ { rank: asc } ]
           script_fields:
@@ -1337,7 +1337,7 @@ setup:
                 source: |
                   if (doc.containsKey('date')) {
                     for (date in doc['date']) {
-                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                     }
                   }
           sort: [ { rank: asc } ]
@@ -1362,7 +1362,7 @@ setup:
                 source: |
                   if (doc.containsKey('date')) {
                     for (date in doc['date']) {
-                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                     }
                   }
           sort: [ { rank: asc } ]

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/10_keyword.yml

@@ -12,7 +12,7 @@ setup:
               day_of_week:
                 type: keyword
                 script: |
-                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
               # Test fetching from _source
               day_of_week_from_source:
                 type: keyword
@@ -75,7 +75,7 @@ setup:
   - match: {sensor.mappings.runtime.day_of_week.type: keyword }
   - match:
       sensor.mappings.runtime.day_of_week.script.source: |
-        emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+        emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
   - match: {sensor.mappings.runtime.day_of_week.script.lang: painless }
 
 # --- TODO get field mappings needs to be adapted
@@ -90,7 +90,7 @@ setup:
 #        type: keyword
 #        script:
 #          source: |
-#            emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+#            emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
 #          lang: painless
 #        meta: {}
 #

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml

@@ -21,7 +21,7 @@ setup:
               day_of_week:
                 type: keyword
                 script: |
-                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
               # Test fetching from _source
               day_of_week_from_source:
                 type: keyword
@@ -74,7 +74,7 @@ setup:
   - match: {sensor.mappings.properties.day_of_week.type: keyword }
   - match:
       sensor.mappings.properties.day_of_week.script.source: |
-        emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+        emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
   - match: {sensor.mappings.properties.day_of_week.script.lang: painless }
 
 ---

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml

@@ -34,7 +34,7 @@ setup:
             day_of_week:
               type: keyword
               script:
-                source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+                source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
 
   - match: {indices:                             ["test-1"]}
   - length: {fields.timestamp:                            1}
@@ -78,7 +78,7 @@ setup:
             day_of_week:
               type: keyword
               script:
-                source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+                source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
 
   - match: {indices:                    ["test-1", "test-2"]}
   - length: {fields.day_of_week:                          1}

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/80_multiple_indices.yml

@@ -12,7 +12,7 @@ setup:
               day_of_week:
                 type: keyword
                 script: |
-                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
               tomorrow:
                 type: date
                 script:

muted-tests.yml

@@ -1,25 +1,10 @@
 tests:
-- class: "org.elasticsearch.xpack.textstructure.structurefinder.TimestampFormatFinderTests"
-  issue: "https://github.com/elastic/elasticsearch/issues/108855"
-  method: "testGuessIsDayFirstFromLocale"
-- class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/108857"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
 - class: "org.elasticsearch.upgrades.SearchStatesIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108991"
   method: "testCanMatch"
 - class: "org.elasticsearch.upgrades.MlTrainedModelsUpgradeIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108993"
   method: "testTrainedModelInference"
-- class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/109188"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
-- class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/109189"
-  method: "test {p0=esql/70_locale/Date format with Italian locale}"
-- class: "org.elasticsearch.xpack.test.rest.XPackRestIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/109200"
-  method: "test {p0=esql/70_locale/Date format with Italian locale}"
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/esql/esql-async-query-api/line_17}
   issue: https://github.com/elastic/elasticsearch/issues/109260

qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml

@@ -13,7 +13,7 @@ setup:
                 day_of_week:
                   type: keyword
                   script:
-                    source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+                    source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
   - do:
       bulk:
         refresh: true

qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/20_runtime_mappings.yml

@@ -9,7 +9,7 @@ setup:
                 day_of_week:
                   type: keyword
                   script:
-                    source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+                    source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
   - do:
       bulk:
         refresh: true