Update upgrade assistant API to account for percolator fields (elastic#137548)

jdconrad · web-flow · commit a678f85cf4f0 · 2025-11-10T15:07:04.000-08:00
This change modifies the deprecation API to check for 
any indices prior to 8.19 that have percolator fields in 
the mappings. Any that do are then returned as 
required to be reindexed or deleted. Read-only is 
excluded as this would not fix the underlying 
transport version issue.

Note this change only concerns user created 
indices. System indices will need to be addressed 
separately as part of the migration API, but should 
be able to share code.

ES-13234
diff --git a/docs/reference/rest-api/common-options.asciidoc b/docs/reference/rest-api/common-options.asciidoc
@@ -243,6 +243,7 @@ Returns:
       "index.creation_date": "1474389951325",
       "index.uuid": "n6gzFZTgS664GUfx0Xrpjw",
       "index.version.created": ...,
+      "index.transport_version.created": ...,
       "index.routing.allocation.include._tier_preference" : "data_content",
       "index.provided_name" : "my-index-000001"
     }
@@ -252,6 +253,7 @@ Returns:
 // TESTRESPONSE[s/1474389951325/$body.my-index-000001.settings.index\\\\.creation_date/]
 // TESTRESPONSE[s/n6gzFZTgS664GUfx0Xrpjw/$body.my-index-000001.settings.index\\\\.uuid/]
 // TESTRESPONSE[s/"index.version.created": \.\.\./"index.version.created": $body.my-index-000001.settings.index\\\\.version\\\\.created/]
+// TESTRESPONSE[s/"index.transport_version.created": \.\.\./"index.transport_version.created": $body.my-index-000001.settings.index\\\\.transport_version\\\\.created/]
 
 When the `flat_settings` flag is `false`, settings are returned in a more
 human readable structured format:
@@ -275,7 +277,10 @@ Returns:
         "creation_date": "1474389951325",
         "uuid": "n6gzFZTgS664GUfx0Xrpjw",
         "version": {
-          "created": ...
+          "created": .v.
+        },
+        "transport_version": {
+          "created": .tv.
         },
         "routing": {
           "allocation": {
@@ -292,7 +297,8 @@ Returns:
 --------------------------------------------------
 // TESTRESPONSE[s/1474389951325/$body.my-index-000001.settings.index.creation_date/]
 // TESTRESPONSE[s/n6gzFZTgS664GUfx0Xrpjw/$body.my-index-000001.settings.index.uuid/]
-// TESTRESPONSE[s/"created": \.\.\./"created": $body.my-index-000001.settings.index.version.created/]
+// TESTRESPONSE[s/"created": \.v\./"created": $body.my-index-000001.settings.index.version.created/]
+// TESTRESPONSE[s/"created": \.tv\./"created": $body.my-index-000001.settings.index.transport_version.created/]
 
 By default `flat_settings` is set to `false`.
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
@@ -369,6 +369,16 @@ public static APIBlock readFrom(StreamInput input) throws IOException {
         Property.PrivateIndex
     );
 
+    public static final String SETTING_TRANSPORT_VERSION_CREATED = "index.transport_version.created";
+
+    public static final Setting<TransportVersion> SETTING_INDEX_TRANSPORT_VERSION_CREATED = Setting.versionIdSetting(
+        SETTING_TRANSPORT_VERSION_CREATED,
+        TransportVersion.fromId(0),
+        TransportVersion::fromId,
+        Property.IndexScope,
+        Property.PrivateIndex
+    );
+
     public static final String SETTING_VERSION_CREATED_STRING = "index.version.created_string";
     public static final String SETTING_CREATION_DATE = "index.creation_date";
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
@@ -1127,6 +1127,9 @@ static Settings aggregateIndexSettings(
             IndexVersion createdVersion = IndexVersion.min(IndexVersion.current(), nodes.getMaxDataNodeCompatibleIndexVersion());
             indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, createdVersion);
         }
+        if (indexSettingsBuilder.get(IndexMetadata.SETTING_TRANSPORT_VERSION_CREATED) == null) {
+            indexSettingsBuilder.put(IndexMetadata.SETTING_TRANSPORT_VERSION_CREATED, TransportVersion.current());
+        }
         if (INDEX_NUMBER_OF_SHARDS_SETTING.exists(indexSettingsBuilder) == false) {
             indexSettingsBuilder.put(SETTING_NUMBER_OF_SHARDS, INDEX_NUMBER_OF_SHARDS_SETTING.get(settings));
         }
diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -58,6 +58,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING,
         IndexMetadata.SETTING_INDEX_VERSION_CREATED,
         IndexMetadata.SETTING_INDEX_VERSION_COMPATIBILITY,
+        IndexMetadata.SETTING_INDEX_TRANSPORT_VERSION_CREATED,
         IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_SETTING,
         IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING,
         IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java
@@ -7,13 +7,19 @@
 
 package org.elasticsearch.xpack.core.deprecation;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiFunction;
+import java.util.function.Function;
 import java.util.function.Predicate;
 
 public class DeprecatedIndexPredicate {
@@ -49,7 +55,7 @@ public static Predicate<Index> getReindexRequiredPredicate(Metadata metadata, bo
      *                              if false, only those without a block are returned
      * @param includeSystem if true, all indices including system will be returned,
      *                              if false, only non-system indices are returned
-     * @return a predicate that returns true for indices that need to be reindexed
+     * @return returns true for indices that need to be reindexed
      */
     public static boolean reindexRequired(IndexMetadata indexMetadata, boolean filterToBlockedStatus, boolean includeSystem) {
         return creationVersionBeforeMinimumWritableVersion(indexMetadata)
@@ -58,6 +64,59 @@ && isNotSearchableSnapshot(indexMetadata)
             && matchBlockedStatus(indexMetadata, filterToBlockedStatus);
     }
 
+    /**
+     * This method checks if this index is on the current transport version for the current minor release version.
+     *
+     * @param indexMetadata the index metadata
+     * @param filterToBlockedStatus if true, only indices that are write blocked will be returned,
+     *                              if false, only those without a block are returned
+     * @param includeSystem if true, all indices including system will be returned,
+     *                              if false, only non-system indices are returned
+     * @return returns true for indices that need to be reindexed
+     */
+    public static boolean reindexRequiredForTransportVersion(
+        IndexMetadata indexMetadata,
+        boolean filterToBlockedStatus,
+        boolean includeSystem
+    ) {
+        return transportVersionBeforeCurrentMinorRelease(indexMetadata)
+            && (includeSystem || isNotSystem(indexMetadata))
+            && isNotSearchableSnapshot(indexMetadata)
+            && matchBlockedStatus(indexMetadata, filterToBlockedStatus);
+    }
+
+    /**
+     * This method checks if this index requires reindexing based on if it has percolator fields older than the current transport version
+     * for the current minor release.
+     *
+     * @param indexMetadata the index metadata
+     * @param filterToBlockedStatus if true, only indices that are write blocked will be returned,
+     *                              if false, only those without a block are returned
+     * @param includeSystem if true, all indices including system will be returned,
+     *                              if false, only non-system indices are returned
+     * @return returns a message as a string for each incompatible percolator field found
+     */
+    public static List<String> reindexRequiredForPecolatorFields(
+        IndexMetadata indexMetadata,
+        boolean filterToBlockedStatus,
+        boolean includeSystem
+    ) {
+        List<String> percolatorIncompatibleFieldMappings = new ArrayList<>();
+        if (reindexRequiredForTransportVersion(indexMetadata, filterToBlockedStatus, includeSystem) && indexMetadata.mapping() != null) {
+            percolatorIncompatibleFieldMappings.addAll(
+                findInPropertiesRecursively(
+                    indexMetadata.mapping().type(),
+                    indexMetadata.mapping().sourceAsMap(),
+                    property -> "percolator".equals(property.get("type")),
+                    (type, entry) -> "Field [" + entry.getKey() + "] is of type [" + indexMetadata.mapping().type() + "]",
+                    "",
+                    ""
+                )
+            );
+        }
+        return percolatorIncompatibleFieldMappings;
+    }
+
     private static boolean isNotSystem(IndexMetadata indexMetadata) {
         return indexMetadata.isSystem() == false;
     }
@@ -73,4 +132,76 @@ private static boolean creationVersionBeforeMinimumWritableVersion(IndexMetadata
     private static boolean matchBlockedStatus(IndexMetadata indexMetadata, boolean filterToBlockedStatus) {
         return MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.get(indexMetadata.getSettings()) == filterToBlockedStatus;
     }
+
+    private static boolean transportVersionBeforeCurrentMinorRelease(IndexMetadata indexMetadata) {
+        // We divide each transport version by 1000 to get the base id.
+        return IndexMetadata.SETTING_INDEX_TRANSPORT_VERSION_CREATED.get(indexMetadata.getSettings()).id() / 1000 < TransportVersion
+            .current()
+            .id() / 1000;
+    }
+
+    /**
+     * iterates through the "properties" field of mappings and returns any predicates that match in the
+     * form of issue-strings.
+     *
+     * @param type the document type
+     * @param parentMap the mapping to read properties from
+     * @param predicate the predicate to check against for issues, issue is returned if predicate evaluates to true
+     * @param fieldFormatter a function that takes a type and mapping field entry and returns a formatted field representation
+     * @return a list of issues found in fields
+     */
+    @SuppressWarnings("unchecked")
+    public static List<String> findInPropertiesRecursively(
+        String type,
+        Map<String, Object> parentMap,
+        Function<Map<?, ?>, Boolean> predicate,
+        BiFunction<String, Map.Entry<?, ?>, String> fieldFormatter,
+        String fieldBeginMarker,
+        String fieldEndMarker
+    ) {
+        List<String> issues = new ArrayList<>();
+        Map<?, ?> properties = (Map<?, ?>) parentMap.get("properties");
+        if (properties == null) {
+            return issues;
+        }
+        for (Map.Entry<?, ?> entry : properties.entrySet()) {
+            Map<String, Object> valueMap = (Map<String, Object>) entry.getValue();
+            if (predicate.apply(valueMap)) {
+                issues.add(fieldBeginMarker + fieldFormatter.apply(type, entry) + fieldEndMarker);
+            }
+
+            Map<?, ?> values = (Map<?, ?>) valueMap.get("fields");
+            if (values != null) {
+                for (Map.Entry<?, ?> multifieldEntry : values.entrySet()) {
+                    Map<String, Object> multifieldValueMap = (Map<String, Object>) multifieldEntry.getValue();
+                    if (predicate.apply(multifieldValueMap)) {
+                        issues.add(
+                            fieldBeginMarker
+                                + fieldFormatter.apply(type, entry)
+                                + ", multifield: "
+                                + multifieldEntry.getKey()
+                                + fieldEndMarker
+                        );
+                    }
+                    if (multifieldValueMap.containsKey("properties")) {
+                        issues.addAll(
+                            findInPropertiesRecursively(
+                                type,
+                                multifieldValueMap,
+                                predicate,
+                                fieldFormatter,
+                                fieldBeginMarker,
+                                fieldEndMarker
+                            )
+                        );
+                    }
+                }
+            }
+            if (valueMap.containsKey("properties")) {
+                issues.addAll(findInPropertiesRecursively(type, valueMap, predicate, fieldFormatter, fieldBeginMarker, fieldEndMarker));
+            }
+        }
+
+        return issues;
+    }
 }
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java
@@ -88,9 +88,24 @@ public Map<String, List<DeprecationIssue>> check(ClusterState clusterState) {
 
     static DeprecationIssue oldIndicesCheck(DataStream dataStream, ClusterState clusterState) {
         List<Index> backingIndices = dataStream.getIndices();
-
+        Set<String> percolatorIndicesNeedingUpgrade = getReindexRequiredIndicesWithPercolatorFields(backingIndices, clusterState, false);
+        if (percolatorIndicesNeedingUpgrade.isEmpty() == false) {
+            return new DeprecationIssue(
+                DeprecationIssue.Level.CRITICAL,
+                "Field mappings with incompatible percolator type",
+                "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries",
+                "The data stream was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields.",
+                false,
+                ofEntries(
+                    entry("reindex_required", true),
+                    entry("excluded_actions", List.of("readOnly")),
+                    entry("total_backing_indices", backingIndices.size()),
+                    entry("indices_requiring_upgrade_count", percolatorIndicesNeedingUpgrade.size()),
+                    entry("indices_requiring_upgrade", percolatorIndicesNeedingUpgrade)
+                )
+            );
+        }
         Set<String> indicesNeedingUpgrade = getReindexRequiredIndices(backingIndices, clusterState, false);
-
         if (indicesNeedingUpgrade.isEmpty() == false) {
             return new DeprecationIssue(
                 DeprecationIssue.Level.CRITICAL,
@@ -112,6 +127,23 @@ static DeprecationIssue oldIndicesCheck(DataStream dataStream, ClusterState clus
 
     static DeprecationIssue ignoredOldIndicesCheck(DataStream dataStream, ClusterState clusterState) {
         List<Index> backingIndices = dataStream.getIndices();
+        Set<String> percolatorIgnoredIndices = getReindexRequiredIndicesWithPercolatorFields(backingIndices, clusterState, true);
+        if (percolatorIgnoredIndices.isEmpty() == false) {
+            return new DeprecationIssue(
+                DeprecationIssue.Level.CRITICAL,
+                "Field mappings with incompatible percolator type",
+                "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries",
+                "The data stream was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields.",
+                false,
+                ofEntries(
+                    entry("reindex_required", true),
+                    entry("excluded_actions", List.of("readOnly")),
+                    entry("total_backing_indices", backingIndices.size()),
+                    entry("ignored_indices_requiring_upgrade_count", percolatorIgnoredIndices.size()),
+                    entry("ignored_indices_requiring_upgrade", percolatorIgnoredIndices)
+                )
+            );
+        }
         Set<String> ignoredIndices = getReindexRequiredIndices(backingIndices, clusterState, true);
         if (ignoredIndices.isEmpty() == false) {
             return new DeprecationIssue(
@@ -144,6 +176,23 @@ private static Set<String> getReindexRequiredIndices(
             .collect(Collectors.toUnmodifiableSet());
     }
 
+    private static Set<String> getReindexRequiredIndicesWithPercolatorFields(
+        List<Index> backingIndices,
+        ClusterState clusterState,
+        boolean filterToBlockedStatus
+    ) {
+        return backingIndices.stream()
+            .filter(
+                index -> DeprecatedIndexPredicate.reindexRequiredForPecolatorFields(
+                    clusterState.metadata().index(index),
+                    filterToBlockedStatus,
+                    false
+                ).isEmpty() == false
+            )
+            .map(Index::getName)
+            .collect(Collectors.toUnmodifiableSet());
+    }
+
     @Override
     public String getName() {
         return NAME;
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java

Original file line number	Diff line number	Diff line change
`@@ -1127,6 +1127,9 @@ static Settings aggregateIndexSettings(`
`1127`	`1127`	`IndexVersion createdVersion = IndexVersion.min(IndexVersion.current(), nodes.getMaxDataNodeCompatibleIndexVersion());`
`1128`	`1128`	`indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, createdVersion);`
`1129`	`1129`	`}`
	`1130`	`+ if (indexSettingsBuilder.get(IndexMetadata.SETTING_TRANSPORT_VERSION_CREATED) == null) {`
	`1131`	`+ indexSettingsBuilder.put(IndexMetadata.SETTING_TRANSPORT_VERSION_CREATED, TransportVersion.current());`
	`1132`	`+ }`
`1130`	`1133`	`if (INDEX_NUMBER_OF_SHARDS_SETTING.exists(indexSettingsBuilder) == false) {`
`1131`	`1134`	`indexSettingsBuilder.put(SETTING_NUMBER_OF_SHARDS, INDEX_NUMBER_OF_SHARDS_SETTING.get(settings));`
`1132`	`1135`	`}`