Return unique deprecation for old indices with incompatible date formats (#124597) (#134371)

(cherry picked from commit e5ad498)

Co-authored-by: Mark Vieira <[email protected]>

Author: mosche

docs/changelog/124597.yaml

@@ -0,0 +1,5 @@
+pr: 124597
+summary: Return unique deprecation for old indices with incompatible date formats
+area: Infra/Core
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/common/time/DateUtils.java

@@ -519,8 +519,10 @@ public static ZonedDateTime nowWithMillisResolution(Clock clock) {
     private static final boolean USES_COMPAT = System.getProperty("java.locale.providers", "").contains("COMPAT");
     // check for all textual fields, and localized zone offset
     // the weird thing with Z is to ONLY match 4 in a row, with no Z before or after (but those groups can also be empty)
+    private static final Predicate<String> LEGACY_DATE_FORMAT_MATCHER = Pattern.compile("[BEGOavz]|LLL|MMM|QQQ|qqq|ccc|eee|(?<!Z)Z{4}(?!Z)")
+        .asPredicate();
     private static final Predicate<String> CONTAINS_CHANGING_TEXT_SPECIFIERS = USES_COMPAT
-        ? Pattern.compile("[BEGOavz]|LLL|MMM|QQQ|qqq|ccc|eee|(?<!Z)Z{4}(?!Z)").asPredicate()
+        ? LEGACY_DATE_FORMAT_MATCHER
         : Predicates.never();
     // week dates are changing on CLDR, as the rules are changing for start-of-week and min-days-in-week
     private static final Predicate<String> CONTAINS_WEEK_DATE_SPECIFIERS = USES_COMPAT
@@ -547,4 +549,8 @@ static void checkTextualDateFormats(String format) {
             );
         }
     }
+
+    public static boolean containsCompatOnlyDateFormat(String format) {
+        return LEGACY_DATE_FORMAT_MATCHER.test(format);
+    }
 }

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.time.LegacyFormatNames;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexModule;
@@ -99,6 +100,21 @@ private DeprecationIssue oldIndicesCheck(
         IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion();
         // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks
         if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false, false) && isNotDataStreamIndex(indexMetadata, clusterState)) {
+            List<String> cldrIncompatibleFieldMappings = new ArrayList<>();
+            fieldLevelMappingIssue(
+                indexMetadata,
+                (mappingMetadata, sourceAsMap) -> cldrIncompatibleFieldMappings.addAll(
+                    findInPropertiesRecursively(
+                        mappingMetadata.type(),
+                        sourceAsMap,
+                        this::isDateFieldWithCompatFormatPattern,
+                        this::cldrIncompatibleFormatPattern,
+                        "",
+                        ""
+                    )
+                )
+            );
+
             var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds);
             if (transforms.isEmpty() == false) {
                 return new DeprecationIssue(
@@ -116,6 +132,17 @@ private DeprecationIssue oldIndicesCheck(
                     false,
                     Map.of("reindex_required", true, "transform_ids", transforms)
                 );
+            } else if (cldrIncompatibleFieldMappings.isEmpty() == false) {
+                return new DeprecationIssue(
+                    DeprecationIssue.Level.CRITICAL,
+                    "Field mappings with incompatible date format patterns in old index",
+                    "https://www.elastic.co/blog/locale-changes-elasticsearch-8-16-jdk-23",
+                    "The index was created before 8.0 and contains mappings that must be reindexed due to locale changes in 8.16+. "
+                        + "Manual reindexing is required. "
+                        + String.join(", ", cldrIncompatibleFieldMappings),
+                    false,
+                    null
+                );
             } else {
                 return new DeprecationIssue(
                     DeprecationIssue.Level.CRITICAL,
@@ -396,6 +423,24 @@ private DeprecationIssue deprecatedCamelCasePattern(
         return null;
     }
 
+    private boolean isDateFieldWithCompatFormatPattern(Map<?, ?> property) {
+        if ("date".equals(property.get("type")) && property.containsKey("format")) {
+            String[] patterns = DateFormatter.splitCombinedPatterns((String) property.get("format"));
+            for (String pattern : patterns) {
+                if (DateUtils.containsCompatOnlyDateFormat(pattern)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    private String cldrIncompatibleFormatPattern(String type, Map.Entry<?, ?> entry) {
+        Map<?, ?> value = (Map<?, ?>) entry.getValue();
+        final String formatFieldValue = (String) value.get("format");
+        return "Field [" + entry.getKey() + "] with format pattern [" + formatFieldValue + "].";
+    }
+
     private boolean isDateFieldWithCamelCasePattern(Map<?, ?> property) {
         if ("date".equals(property.get("type")) && property.containsKey("format")) {
             String[] patterns = DateFormatter.splitCombinedPatterns((String) property.get("format"));

x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java

@@ -203,6 +203,44 @@ public void testMultipleOldIndicesCheckWithTransforms() {
         assertEquals(expected, issuesByIndex);
     }
 
+    public void testOldIndicesWithIncompatibleDateFormatsCheck() {
+        IndexMetadata indexMetadata = IndexMetadata.builder("test")
+            .settings(settings(OLD_VERSION))
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .state(indexMetdataState)
+            .putMapping("""
+                {
+                  "properties": {
+                    "date": {
+                      "type": "date",
+                      "format": "qqqq yyyy"
+                    }
+                  }
+                }""")
+            .build();
+        ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE)
+            .metadata(Metadata.builder().put(indexMetadata, true))
+            .blocks(clusterBlocksForIndices(indexMetadata))
+            .build();
+        DeprecationIssue expected = new DeprecationIssue(
+            DeprecationIssue.Level.CRITICAL,
+            "Field mappings with incompatible date format patterns in old index",
+            "https://www.elastic.co/blog/locale-changes-elasticsearch-8-16-jdk-23",
+            "The index was created before 8.0 and contains mappings that must be reindexed due to locale changes in 8.16+. "
+                + "Manual reindexing is required. Field [date] with format pattern [qqqq yyyy].",
+            false,
+            null
+        );
+        Map<String, List<DeprecationIssue>> issuesByIndex = checker.check(
+            clusterState,
+            new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS),
+            emptyPrecomputedData
+        );
+        List<DeprecationIssue> issues = issuesByIndex.get("test");
+        assertEquals(singletonList(expected), issues);
+    }
+
     private IndexMetadata indexMetadata(String indexName, IndexVersion indexVersion) {
         return IndexMetadata.builder(indexName)
             .settings(settings(indexVersion))