[7.17] Use CLDR locale provider on JDK 23 (#110222) (#112544)

thecoop · web-flow · commit 9a80a37d7834 · 2024-09-06T16:24:37.000+01:00
Backports #110222 to 7.17. JDK 23 removes the COMPAT locale provider, leaving CLDR as the only option. This commit configures Elasticsearch to use the CLDR provider when on JDK 23, but still use the existing COMPAT provider when on JDK 22 and below. This causes some differences in locale behaviour; this also adapts various tests to still work whether run on COMPAT or CLDR.
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
@@ -120,7 +120,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
           defaults(JUnit) {
             vmParameters = [
               '-ea',
-              '-Djava.locale.providers=SPI,COMPAT',
+              '-Djava.locale.providers=SPI,CLDR',
               "--illegal-access=deny",
               // TODO: only open these for mockito when it is modularized
               '--add-opens=java.base/java.security.cert=ALL-UNNAMED',
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java
@@ -87,7 +87,11 @@ public void execute(Task t) {
                     if (BuildParams.getRuntimeJavaVersion() == JavaVersion.VERSION_1_8) {
                         test.systemProperty("java.locale.providers", "SPI,JRE");
                     } else {
-                        test.systemProperty("java.locale.providers", "SPI,COMPAT");
+                        if (BuildParams.getRuntimeJavaVersion().compareTo(JavaVersion.VERSION_22) <= 0) {
+                            test.systemProperty("java.locale.providers", "SPI,COMPAT");
+                        } else {
+                            test.systemProperty("java.locale.providers", "SPI,CLDR");
+                        }
                         test.jvmArgs(
                             "--illegal-access=deny",
                             // TODO: only open these for mockito when it is modularized
diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/SystemJvmOptions.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/SystemJvmOptions.java
@@ -91,11 +91,15 @@ private static String javaLocaleProviders() {
          *  parsing will break in an incompatible way for some date patterns and locales.
          *  //TODO COMPAT will be deprecated in jdk14 https://bugs.openjdk.java.net/browse/JDK-8232906
          * See also: documentation in <code>server/org.elasticsearch.common.time.IsoCalendarDataProvider</code>
+         *
+         * COMPAT is removed in JDK 23, so we have to use CLDR for 23
          */
         if (JavaVersion.majorVersion(JavaVersion.CURRENT) == 8) {
             return "-Djava.locale.providers=SPI,JRE";
-        } else {
+        } else if (JavaVersion.majorVersion(JavaVersion.CURRENT) <= 22) {
             return "-Djava.locale.providers=SPI,COMPAT";
+        } else {
+            return "-Djava.locale.providers=SPI,CLDR";
         }
     }
 
diff --git a/docs/painless/painless-guide/painless-execute-script.asciidoc b/docs/painless/painless-guide/painless-execute-script.asciidoc
@@ -730,7 +730,7 @@ POST /_scripts/painless/_execute
 {
   "script": {
     "source": """
-      emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+      emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
     """
   },
   "context": "keyword_field",
@@ -823,4 +823,4 @@ The response includes the calculated value from the script valuation:
     8624909
   ]
 }
-----
+----
diff --git a/docs/reference/aggregations/bucket/composite-aggregation.asciidoc b/docs/reference/aggregations/bucket/composite-aggregation.asciidoc
@@ -156,7 +156,7 @@ GET /_search
       "type": "keyword",
       "script": """
         emit(doc['timestamp'].value.dayOfWeekEnum
-          .getDisplayName(TextStyle.FULL, Locale.ROOT))
+          .getDisplayName(TextStyle.FULL, Locale.ENGLISH))
       """
     }
   },
diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
@@ -749,7 +749,7 @@ POST /sales/_search?size=0
   "runtime_mappings": {
     "date.day_of_week": {
       "type": "keyword",
-      "script": "emit(doc['date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+      "script": "emit(doc['date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
     }
   },
   "aggs": {
diff --git a/docs/reference/indices/index-templates.asciidoc b/docs/reference/indices/index-templates.asciidoc
@@ -99,7 +99,7 @@ PUT _component_template/runtime_component_template
         "day_of_week": {
           "type": "keyword",
           "script": {
-            "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+            "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
           }
         }
       }
diff --git a/docs/reference/mapping/runtime.asciidoc b/docs/reference/mapping/runtime.asciidoc
@@ -135,7 +135,7 @@ PUT my-index-000001/
       "day_of_week": {
         "type": "keyword",
         "script": {
-          "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+          "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
         }
       }
     },
@@ -277,7 +277,7 @@ GET my-index-000001/_search
     "day_of_week": {
       "type": "keyword",
       "script": {
-        "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+        "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
       }
     }
   },
@@ -661,7 +661,7 @@ PUT my-index-000001/
       "day_of_week": {
         "type": "keyword",
         "script": {
-          "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+          "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
         }
       }
     },
diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc
@@ -877,7 +877,7 @@ For example:
   "day_of_week": {
     "type": "keyword",
     "script": {
-      "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+      "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
     }
   }
 }
diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/10_keyword.yml
@@ -12,14 +12,14 @@ setup:
               day_of_week:
                 type: keyword
                 script: |
-                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
               # Test fetching from _source
               day_of_week_from_source:
                 type: keyword
                 script: |
                   Instant instant = Instant.ofEpochMilli(params._source.timestamp);
                   ZonedDateTime dt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC"));
-                  emit(dt.dayOfWeek.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                  emit(dt.dayOfWeek.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
               # Test fetching many values
               day_of_week_letters:
                 type: keyword
@@ -75,7 +75,7 @@ setup:
   - match: {sensor.mappings.runtime.day_of_week.type: keyword }
   - match:
       sensor.mappings.runtime.day_of_week.script.source: |
-        emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+        emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
   - match: {sensor.mappings.runtime.day_of_week.script.lang: painless }
 
 # --- TODO get field mappings needs to be adapted
@@ -90,7 +90,7 @@ setup:
 #        type: keyword
 #        script:
 #          source: |
-#            emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+#            emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
 #          lang: painless
 #        meta: {}
 #
@@ -148,7 +148,7 @@ setup:
             day_of_week:
               type: keyword
               script: |
-                emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.SHORT, Locale.ROOT));
+                emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.SHORT, Locale.ENGLISH));
   - do:
       search:
         index: sensor
diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml
@@ -21,14 +21,14 @@ setup:
               day_of_week:
                 type: keyword
                 script: |
-                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
               # Test fetching from _source
               day_of_week_from_source:
                 type: keyword
                 script: |
                   Instant instant = Instant.ofEpochMilli(params._source.timestamp);
                   ZonedDateTime dt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC"));
-                  emit(dt.dayOfWeek.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                  emit(dt.dayOfWeek.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
               # Test fetching many values
               day_of_week_letters:
                 type: keyword
@@ -74,7 +74,7 @@ setup:
   - match: {sensor.mappings.properties.day_of_week.type: keyword }
   - match:
       sensor.mappings.properties.day_of_week.script.source: |
-        emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+        emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
   - match: {sensor.mappings.properties.day_of_week.script.lang: painless }
 
 ---
diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml
@@ -34,7 +34,7 @@ setup:
             day_of_week:
               type: keyword
               script:
-                source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+                source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
 
   - match: {indices:                             ["test-1"]}
   - length: {fields.timestamp:                            1}
@@ -78,7 +78,7 @@ setup:
             day_of_week:
               type: keyword
               script:
-                source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+                source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
 
   - match: {indices:                    ["test-1", "test-2"]}
   - length: {fields.day_of_week:                          1}
diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/80_multiple_indices.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/80_multiple_indices.yml
@@ -12,7 +12,7 @@ setup:
               day_of_week:
                 type: keyword
                 script: |
-                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
               tomorrow:
                 type: date
                 script:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml
@@ -16,26 +16,26 @@
                       date_field:
                           type: date
                           format: "8E, d MMM uuuu HH:mm:ss Z"
-                          locale: "de"
+                          locale: "fr"
   - do:
       bulk:
           refresh: true
           body:
               - '{"index": {"_index": "test_index", "_id": "1"}}'
-              - '{"date_field": "Mi, 06 Dez 2000 02:55:00 -0800"}'
+              - '{"date_field": "mer., 6 déc. 2000 02:55:00 -0800"}'
               - '{"index": {"_index": "test_index", "_id": "2"}}'
-              - '{"date_field": "Do, 07 Dez 2000 02:55:00 -0800"}'
+              - '{"date_field": "jeu., 7 déc. 2000 02:55:00 -0800"}'
 
   - do:
       search:
           rest_total_hits_as_int: true
           index: test_index
-          body: {"query" : {"range" : {"date_field" : {"gte": "Di, 05 Dez 2000 02:55:00 -0800", "lte": "Do, 07 Dez 2000 00:00:00 -0800"}}}}
+          body: {"query" : {"range" : {"date_field" : {"gte": "mar., 5 déc. 2000 02:55:00 -0800", "lte": "jeu., 7 déc. 2000 00:00:00 -0800"}}}}
   - match: { hits.total: 1 }
 
   - do:
       search:
           rest_total_hits_as_int: true
           index: test_index
-          body: {"query" : {"range" : {"date_field" : {"gte": "Di, 05 Dez 2000 02:55:00 -0800", "lte": "Fr, 08 Dez 2000 00:00:00 -0800"}}}}
+          body: {"query" : {"range" : {"date_field" : {"gte": "mar., 5 déc. 2000 02:55:00 -0800", "lte": "ven., 8 déc. 2000 00:00:00 -0800"}}}}
   - match: { hits.total: 2 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -1871,14 +1871,9 @@ public void testRangeQueryWithTimeZone() throws Exception {
      * Test range with a custom locale, e.g. "de" in this case. Documents here mention the day of week
      * as "Mi" for "Mittwoch (Wednesday" and "Do" for "Donnerstag (Thursday)" and the month in the query
      * as "Dez" for "Dezember (December)".
-     * Note: this test currently needs the JVM arg `-Djava.locale.providers=SPI,COMPAT` to be set.
-     * When running with gradle this is done implicitly through the BuildPlugin, but when running from
-     * an IDE this might need to be set manually in the run configuration. See also CONTRIBUTING.md section
-     * on "Configuring IDEs And Running Tests".
      */
     public void testRangeQueryWithLocaleMapping() throws Exception {
         assumeTrue("need java 9 for testing ", JavaVersion.current().compareTo(JavaVersion.parse("9")) >= 0);
-        assert ("SPI,COMPAT".equals(System.getProperty("java.locale.providers"))) : "`-Djava.locale.providers=SPI,COMPAT` needs to be set";
 
         assertAcked(
             prepareCreate("test").addMapping(
@@ -1888,7 +1883,7 @@ public void testRangeQueryWithLocaleMapping() throws Exception {
                     .startObject("date_field")
                     .field("type", "date")
                     .field("format", "E, d MMM yyyy HH:mm:ss Z")
-                    .field("locale", "de")
+                    .field("locale", "fr")
                     .endObject()
                     .endObject()
                     .endObject()
@@ -1897,17 +1892,21 @@ public void testRangeQueryWithLocaleMapping() throws Exception {
 
         indexRandom(
             true,
-            client().prepareIndex("test", "type1", "1").setSource("date_field", "Mi, 06 Dez 2000 02:55:00 -0800"),
-            client().prepareIndex("test", "type1", "2").setSource("date_field", "Do, 07 Dez 2000 02:55:00 -0800")
+            client().prepareIndex("test", "type1", "1").setSource("date_field", "mer., 6 déc. 2000 02:55:00 -0800"),
+            client().prepareIndex("test", "type1", "2").setSource("date_field", "jeu., 7 déc. 2000 02:55:00 -0800")
         );
 
         SearchResponse searchResponse = client().prepareSearch("test")
-            .setQuery(QueryBuilders.rangeQuery("date_field").gte("Di, 05 Dez 2000 02:55:00 -0800").lte("Do, 07 Dez 2000 00:00:00 -0800"))
+            .setQuery(
+                QueryBuilders.rangeQuery("date_field").gte("mar., 5 déc. 2000 02:55:00 -0800").lte("jeu., 7 déc. 2000 00:00:00 -0800")
+            )
             .get();
         assertHitCount(searchResponse, 1L);
 
         searchResponse = client().prepareSearch("test")
-            .setQuery(QueryBuilders.rangeQuery("date_field").gte("Di, 05 Dez 2000 02:55:00 -0800").lte("Fr, 08 Dez 2000 00:00:00 -0800"))
+            .setQuery(
+                QueryBuilders.rangeQuery("date_field").gte("mar., 5 déc. 2000 02:55:00 -0800").lte("ven., 8 déc. 2000 00:00:00 -0800")
+            )
             .get();
         assertHitCount(searchResponse, 2L);
     }

Original file line number	Diff line number	Diff line change
`@@ -730,7 +730,7 @@ POST /_scripts/painless/_execute`
`730`	`730`	`{`
`731`	`731`	`"script": {`
`732`	`732`	`"source": """`
`733`		`- emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));`
	`733`	`+ emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));`
`734`	`734`	`"""`
`735`	`735`	`},`
`736`	`736`	`"context": "keyword_field",`
`@@ -823,4 +823,4 @@ The response includes the calculated value from the script valuation:`
`823`	`823`	`8624909`
`824`	`824`	`]`
`825`	`825`	`}`
`826`		`-----`
	`826`	`+----`
Original file line number	Diff line number	Diff line change
`@@ -156,7 +156,7 @@ GET /_search`
`156`	`156`	`"type": "keyword",`
`157`	`157`	`"script": """`
`158`	`158`	`emit(doc['timestamp'].value.dayOfWeekEnum`
`159`		`- .getDisplayName(TextStyle.FULL, Locale.ROOT))`
	`159`	`+ .getDisplayName(TextStyle.FULL, Locale.ENGLISH))`
`160`	`160`	`"""`
`161`	`161`	`}`
`162`	`162`	`},`
Original file line number	Diff line number	Diff line change
`@@ -749,7 +749,7 @@ POST /sales/_search?size=0`
`749`	`749`	`"runtime_mappings": {`
`750`	`750`	`"date.day_of_week": {`
`751`	`751`	`"type": "keyword",`
`752`		`- "script": "emit(doc['date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"`
	`752`	`+ "script": "emit(doc['date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"`
`753`	`753`	`}`
`754`	`754`	`},`
`755`	`755`	`"aggs": {`
Original file line number	Diff line number	Diff line change
`@@ -99,7 +99,7 @@ PUT _component_template/runtime_component_template`
`99`	`99`	`"day_of_week": {`
`100`	`100`	`"type": "keyword",`
`101`	`101`	`"script": {`
`102`		`- "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"`
	`102`	`+ "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"`
`103`	`103`	`}`
`104`	`104`	`}`
`105`	`105`	`}`
Original file line number	Diff line number	Diff line change
`@@ -135,7 +135,7 @@ PUT my-index-000001/`
`135`	`135`	`"day_of_week": {`
`136`	`136`	`"type": "keyword",`
`137`	`137`	`"script": {`
`138`		`- "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"`
	`138`	`+ "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"`
`139`	`139`	`}`
`140`	`140`	`}`
`141`	`141`	`},`
`@@ -277,7 +277,7 @@ GET my-index-000001/_search`
`277`	`277`	`"day_of_week": {`
`278`	`278`	`"type": "keyword",`
`279`	`279`	`"script": {`
`280`		`- "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"`
	`280`	`+ "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"`
`281`	`281`	`}`
`282`	`282`	`}`
`283`	`283`	`},`
`@@ -661,7 +661,7 @@ PUT my-index-000001/`
`661`	`661`	`"day_of_week": {`
`662`	`662`	`"type": "keyword",`
`663`	`663`	`"script": {`
`664`		`- "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"`
	`664`	`+ "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"`
`665`	`665`	`}`
`666`	`666`	`}`
`667`	`667`	`},`
Original file line number	Diff line number	Diff line change
`@@ -877,7 +877,7 @@ For example:`
`877`	`877`	`"day_of_week": {`
`878`	`878`	`"type": "keyword",`
`879`	`879`	`"script": {`
`880`		`- "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"`
	`880`	`+ "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"`
`881`	`881`	`}`
`882`	`882`	`}`
`883`	`883`	`}`