Optimize usage calculation in ILM policies retrieval API

docs/changelog/106953.yaml

@@ -0,0 +1,6 @@
+pr: 106953
+summary: Optimize usage calculation in ILM policies retrieval API
+area: ILM+SLM
+type: enhancement
+issues:
+ - 105773

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -61,6 +61,7 @@
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -1050,29 +1051,24 @@ static Set<String> dataStreamsExclusivelyUsingTemplates(final ClusterState state
             .reduce(Sets::union)
             .orElse(Set.of());
 
+        // Filter and sort all composable templates in advance, to speed up template retrieval later on.
+        var templates = state.metadata()
+            .templatesV2()
+            .entrySet()
+            .stream()
+            .filter(
+                entry -> templateNames.contains(entry.getKey()) == false
+                    && isGlobalAndHasIndexHiddenSetting(metadata, entry.getValue(), entry.getKey()) == false
+            )
+            .sorted(Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder()))
+            .toList();
+
         return metadata.dataStreams()
             .values()
             .stream()
             // Limit to checking data streams that match any of the templates' index patterns
             .filter(ds -> namePatterns.stream().anyMatch(pattern -> Regex.simpleMatch(pattern, ds.getName())))
-            .filter(ds -> {
-                // Retrieve the templates that match the data stream name ordered by priority
-                List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(metadata, ds.getName(), ds.isHidden());
-                if (candidates.isEmpty()) {
-                    throw new IllegalStateException("Data stream " + ds.getName() + " did not match any composable index templates.");
-                }
-
-                // Limit data streams that can ONLY use any of the specified templates, we do this by filtering
-                // the matching templates that are others than the ones requested and could be a valid template to use.
-                return candidates.stream()
-                    .filter(
-                        template -> templateNames.contains(template.v1()) == false
-                            && isGlobalAndHasIndexHiddenSetting(metadata, template.v2(), template.v1()) == false
-                    )
-                    .map(Tuple::v1)
-                    .toList()
-                    .isEmpty();
-            })
+            .filter(ds -> findV2Template(state.metadata(), templates, ds.getName(), ds.isHidden(), true) == null)
             .map(DataStream::getName)
             .collect(Collectors.toSet());
     }
@@ -1268,7 +1264,27 @@ public static List<IndexTemplateMetadata> findV1Templates(Metadata metadata, Str
      */
     @Nullable
     public static String findV2Template(Metadata metadata, String indexName, boolean isHidden) {
-        final List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(metadata, indexName, isHidden);
+        return findV2Template(metadata, metadata.templatesV2().entrySet(), indexName, isHidden, false);
+    }
+
+    /**
+     * Return the name (id) of the highest matching index template for the given index name. In
+     * the event that no templates are matched, {@code null} is returned.
+     */
+    @Nullable
+    public static String findV2Template(
+        Metadata metadata,
+        Collection<Map.Entry<String, ComposableIndexTemplate>> templateEntries,
+        String indexName,
+        boolean isHidden,
+        boolean exitOnFirstMatch
+    ) {
+        final List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(
+            templateEntries,
+            indexName,
+            isHidden,
+            exitOnFirstMatch
+        );
         if (candidates.isEmpty()) {
             return null;
         }
@@ -1296,27 +1312,45 @@ public static String findV2Template(Metadata metadata, String indexName, boolean
     /**
      * Return an ordered list of the name (id) and composable index templates that would apply to an index. The first
      * one is the winner template that is applied to this index. In the event that no templates are matched,
-     * an empty list is returned.
+     * an empty list is returned. If <code>exitOnFirstMatch</code> is true, we return immediately after finding a match.
      */
-    static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(Metadata metadata, String indexName, boolean isHidden) {
+    static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(
+        Collection<Map.Entry<String, ComposableIndexTemplate>> templateEntries,
+        String indexName,
+        boolean isHidden,
+        boolean exitOnFirstMatch
+    ) {
         final String resolvedIndexName = IndexNameExpressionResolver.DateMathExpressionResolver.resolveExpression(indexName);
-        final Predicate<String> patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, resolvedIndexName);
         final List<Tuple<String, ComposableIndexTemplate>> candidates = new ArrayList<>();
-        for (Map.Entry<String, ComposableIndexTemplate> entry : metadata.templatesV2().entrySet()) {
+        outerLoop:
+        for (Map.Entry<String, ComposableIndexTemplate> entry : templateEntries) {
             final String name = entry.getKey();
             final ComposableIndexTemplate template = entry.getValue();
             if (isHidden == false) {
-                final boolean matched = template.indexPatterns().stream().anyMatch(patternMatchPredicate);
-                if (matched) {
-                    candidates.add(Tuple.tuple(name, template));
+                for (String indexPattern : template.indexPatterns()) {
+                    if (Regex.simpleMatch(indexPattern, resolvedIndexName)) {
+                        candidates.add(Tuple.tuple(name, template));
+                        if (exitOnFirstMatch) {
+                            return candidates;
+                        }
+                    }
                 }
             } else {
-                final boolean isNotMatchAllTemplate = template.indexPatterns().stream().noneMatch(Regex::isMatchAllPattern);
-                if (isNotMatchAllTemplate) {
-                    if (template.indexPatterns().stream().anyMatch(patternMatchPredicate)) {
+                // For hidden indices, we don't want to return any templates that match "all".
+                boolean match = false;
+                for (String indexPattern : template.indexPatterns()) {
+                    if (Regex.isMatchAllPattern(indexPattern)) {
+                        continue outerLoop;
+                    }
+                    // If we've already found a match, we still want to keep looping to make sure this template doesn't have a "*" pattern.
+                    if (match == false && Regex.simpleMatch(indexPattern, resolvedIndexName)) {
                         candidates.add(Tuple.tuple(name, template));
+                        match = true;
                     }
                 }
+                if (match && exitOnFirstMatch) {
+                    return candidates;
+                }
             }
         }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ilm;
+
+import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.ItemUsage;
+import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A class that can be used to calculate the usages of ILM policies. This class computes some information on initialization, which will
+ * use a bit more memory but speeds up the usage calculation significantly.
+ */
+public class LifecyclePolicyUsageCalculator {
+
+    private final ClusterState state;
+    /** Whether {@link #calculateUsage} will be called multiple times or not. */
+    private final boolean willIterate;
+    /** A map from policy name to list of data streams that use that policy. */
+    private final Map<String, List<String>> policyToDataStream;
+    /** A map from composable template name to the policy name it uses (or null) */
+    private final Map<String, String> templateToPolicy;
+
+    public LifecyclePolicyUsageCalculator(
+        final IndexNameExpressionResolver indexNameExpressionResolver,
+        final ClusterState state,
+        List<String> names
+    ) {
+        this.state = state;
+        this.willIterate = names.size() > 1 || Regex.isSimpleMatchPattern(names.get(0));
+
+        var allDataStreams = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN);
+        // Sort all templates by descending priority. That way, findV2Template can exit on the first found template.
+        var indexTemplates = new ArrayList<>(state.metadata().templatesV2().entrySet());
+        CollectionUtil.timSort(indexTemplates, Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder()));
+
+        // Build the maps that will be used for the usage calculation later on.
+        IndexLifecycleMetadata metadata = state.metadata().custom(IndexLifecycleMetadata.TYPE);
+        policyToDataStream = new HashMap<>(Regex.isSimpleMatchPattern(names.get(0)) ? metadata.getPolicyMetadatas().size() : names.size());
+        templateToPolicy = new HashMap<>(indexTemplates.size());
+        for (String dataStream : allDataStreams) {
+            String indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), indexTemplates, dataStream, false, true);
+            if (indexTemplate == null) {
+                continue;
+            }
+            Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate);
+            var policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings);
+            if (names.stream().noneMatch(name -> Regex.simpleMatch(name, policyName))) {
+                // If a template's policy doesn't match any of the supplied names, we can skip it later on.
+                templateToPolicy.put(indexTemplate, null);
+                continue;
+            }
+            templateToPolicy.put(indexTemplate, policyName);
+            policyToDataStream.computeIfAbsent(policyName, k -> new ArrayList<>()).add(dataStream);
+        }
+    }
+
+    /**
+     * Calculate the indices, data streams, and composable templates that use the given policy.
+     */
+    public ItemUsage calculateUsage(String policyName) {
+        List<String> indices = new ArrayList<>();
+        for (IndexMetadata indexMetadata : state.metadata().indices().values()) {
+            if (policyName.equals(indexMetadata.getLifecyclePolicyName())) {
+                indices.add(indexMetadata.getIndex().getName());
+            }
+        }
+
+        List<String> composableTemplates = new ArrayList<>();
+        for (Map.Entry<String, ComposableIndexTemplate> entry : state.metadata().templatesV2().entrySet()) {
+            var foundPolicy = templateToPolicy.get(entry.getKey());
+            // Extra `containsKey` check to account for templates not using any policy.
+            if (foundPolicy == null && templateToPolicy.containsKey(entry.getKey()) == false) {
+                Settings settings = MetadataIndexTemplateService.resolveSettings(entry.getValue(), state.metadata().componentTemplates());
+                foundPolicy = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings);
+                // If this method will only be called once, we don't need to keep building the map.
+                if (willIterate) {
+                    templateToPolicy.put(entry.getKey(), foundPolicy);
+                }
+            }
+            if (policyName.equals(foundPolicy)) {
+                composableTemplates.add(entry.getKey());
+            }
+        }
+
+        return new ItemUsage(indices, policyToDataStream.getOrDefault(policyName, List.of()), composableTemplates);
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java

@@ -8,24 +8,16 @@
 package org.elasticsearch.xpack.core.ilm;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.ItemUsage;
-import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.NotXContentException;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.template.resources.TemplateResources;
 
-import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 /**
  * A utility class used for index lifecycle policies
@@ -91,44 +83,4 @@ private static void validate(String source) {
             throw new ElasticsearchParseException("invalid policy", e);
         }
     }
-
-    /**
-     * Given a cluster state and ILM policy, calculate the {@link ItemUsage} of
-     * the policy (what indices, data streams, and templates use the policy)
-     */
-    public static ItemUsage calculateUsage(
-        final IndexNameExpressionResolver indexNameExpressionResolver,
-        final ClusterState state,
-        final String policyName
-    ) {
-        final List<String> indices = state.metadata()
-            .indices()
-            .values()
-            .stream()
-            .filter(indexMetadata -> policyName.equals(indexMetadata.getLifecyclePolicyName()))
-            .map(indexMetadata -> indexMetadata.getIndex().getName())
-            .collect(Collectors.toList());
-
-        final List<String> allDataStreams = indexNameExpressionResolver.dataStreamNames(
-            state,
-            IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN
-        );
-
-        final List<String> dataStreams = allDataStreams.stream().filter(dsName -> {
-            String indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dsName, false);
-            if (indexTemplate != null) {
-                Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate);
-                return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings));
-            } else {
-                return false;
-            }
-        }).collect(Collectors.toList());
-
-        final List<String> composableTemplates = state.metadata().templatesV2().keySet().stream().filter(templateName -> {
-            Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), templateName);
-            return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings));
-        }).collect(Collectors.toList());
-
-        return new ItemUsage(indices, dataStreams, composableTemplates);
-    }
 }