Optimize usage calculation in ILM policies retrieval API

docs/changelog/106953.yaml

@@ -0,0 +1,6 @@
+pr: 106953
+summary: Optimize usage calculation in ILM policies retrieval API
+area: ILM+SLM
+type: enhancement
+issues:
+ - 105773

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -62,6 +62,7 @@
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -1060,33 +1061,28 @@ static Set<String> dataStreamsExclusivelyUsingTemplates(final ClusterState state
             .map(templateName -> metadata.templatesV2().get(templateName))
             .filter(Objects::nonNull)
             .map(ComposableIndexTemplate::indexPatterns)
-            .map(Set::copyOf)
-            .reduce(Sets::union)
-            .orElse(Set.of());
+            .flatMap(List::stream)
+            .collect(Collectors.toSet());
+
+        // Determine all the composable templates that are not one of the provided templates.
+        var otherTemplates = state.metadata()
+            .templatesV2()
+            .entrySet()
+            .stream()
+            .filter(
+                entry -> templateNames.contains(entry.getKey()) == false
+                    && isGlobalAndHasIndexHiddenSetting(metadata, entry.getValue(), entry.getKey()) == false
+            )
+            // Sort here so we can `exitOnFirstMatch` in `findV2Template`.
+            .sorted(Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder()))
+            .toList();
 
         return metadata.dataStreams()
             .values()
             .stream()
             // Limit to checking data streams that match any of the templates' index patterns
             .filter(ds -> namePatterns.stream().anyMatch(pattern -> Regex.simpleMatch(pattern, ds.getName())))
-            .filter(ds -> {
-                // Retrieve the templates that match the data stream name ordered by priority
-                List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(metadata, ds.getName(), ds.isHidden());
-                if (candidates.isEmpty()) {
-                    throw new IllegalStateException("Data stream " + ds.getName() + " did not match any composable index templates.");
-                }
-
-                // Limit data streams that can ONLY use any of the specified templates, we do this by filtering
-                // the matching templates that are others than the ones requested and could be a valid template to use.
-                return candidates.stream()
-                    .filter(
-                        template -> templateNames.contains(template.v1()) == false
-                            && isGlobalAndHasIndexHiddenSetting(metadata, template.v2(), template.v1()) == false
-                    )
-                    .map(Tuple::v1)
-                    .toList()
-                    .isEmpty();
-            })
+            .filter(ds -> findV2TemplateFromSortedList(state.metadata(), otherTemplates, ds.getName(), ds.isHidden()) == null)
             .map(DataStream::getName)
             .collect(Collectors.toSet());
     }
@@ -1319,7 +1315,41 @@ public static List<IndexTemplateMetadata> findV1Templates(Metadata metadata, Str
      */
     @Nullable
     public static String findV2Template(Metadata metadata, String indexName, boolean isHidden) {
-        final List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(metadata, indexName, isHidden);
+        return findV2Template(metadata, metadata.templatesV2().entrySet(), indexName, isHidden, false);
+    }
+
+    /**
+     * Return the name (id) of the highest matching index template, out of the provided templates (that <i>need</i> to be sorted descending
+     * on priority beforehand), or the given index name. In the event that no templates are matched, {@code null} is returned.
+     */
+    @Nullable
+    public static String findV2TemplateFromSortedList(
+        Metadata metadata,
+        Collection<Map.Entry<String, ComposableIndexTemplate>> templates,
+        String indexName,
+        boolean isHidden
+    ) {
+        return findV2Template(metadata, templates, indexName, isHidden, true);
+    }
+
+    /**
+     * Return the name (id) of the highest matching index template, out of the provided templates, for the given index name. In
+     * the event that no templates are matched, {@code null} is returned.
+     */
+    @Nullable
+    private static String findV2Template(
+        Metadata metadata,
+        Collection<Map.Entry<String, ComposableIndexTemplate>> templates,
+        String indexName,
+        boolean isHidden,
+        boolean exitOnFirstMatch
+    ) {
+        final List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(
+            templates,
+            indexName,
+            isHidden,
+            exitOnFirstMatch
+        );
         if (candidates.isEmpty()) {
             return null;
         }
@@ -1348,29 +1378,35 @@ public static String findV2Template(Metadata metadata, String indexName, boolean
      * Return an ordered list of the name (id) and composable index templates that would apply to an index. The first
      * one is the winner template that is applied to this index. In the event that no templates are matched,
      * an empty list is returned.
+     * <p>
+     * If <code>exitOnFirstMatch</code> is true, we return immediately after finding a match. That means that the <code>templates</code>
+     * parameter needs to be sorted based on priority (descending) for this method to return a sensible result -- otherwise this method
+     * would just return the first template that matches the name, in an unspecified order.
      */
-    static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(Metadata metadata, String indexName, boolean isHidden) {
+    private static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(
+        Collection<Map.Entry<String, ComposableIndexTemplate>> templates,
+        String indexName,
+        boolean isHidden,
+        boolean exitOnFirstMatch
+    ) {
         final String resolvedIndexName = IndexNameExpressionResolver.DateMathExpressionResolver.resolveExpression(indexName);
         final Predicate<String> patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, resolvedIndexName);
         final List<Tuple<String, ComposableIndexTemplate>> candidates = new ArrayList<>();
-        for (Map.Entry<String, ComposableIndexTemplate> entry : metadata.templatesV2().entrySet()) {
+        for (Map.Entry<String, ComposableIndexTemplate> entry : templates) {
             final String name = entry.getKey();
             final ComposableIndexTemplate template = entry.getValue();
             /*
              * We do not ordinarily return match-all templates for hidden indices. But all backing indices for data streams are hidden,
              * and we do want to return even match-all templates for those. Not doing so can result in a situation where a data stream is
              * built with a template that none of its indices match.
              */
-            if (isHidden == false || template.getDataStreamTemplate() != null) {
-                if (anyMatch(template.indexPatterns(), patternMatchPredicate)) {
-                    candidates.add(Tuple.tuple(name, template));
-                }
-            } else {
-                final boolean isNotMatchAllTemplate = noneMatch(template.indexPatterns(), Regex::isMatchAllPattern);
-                if (isNotMatchAllTemplate) {
-                    if (anyMatch(template.indexPatterns(), patternMatchPredicate)) {
-                        candidates.add(Tuple.tuple(name, template));
-                    }
+            if (isHidden && template.getDataStreamTemplate() == null && anyMatch(template.indexPatterns(), Regex::isMatchAllPattern)) {
+                continue;
+            }
+            if (anyMatch(template.indexPatterns(), patternMatchPredicate)) {
+                candidates.add(Tuple.tuple(name, template));
+                if (exitOnFirstMatch) {
+                    return candidates;
                 }
             }
         }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java

@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ilm;
+
+import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.ItemUsage;
+import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.Maps;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A class that can be used to calculate the usages of ILM policies across the cluster. The class makes a tradeoff by using some more memory
+ * (but not a lot) to significantly improve the processing time.
+ */
+public class LifecyclePolicyUsageCalculator {
+
+    /** A map from policy name to list of composable templates that use that policy. */
+    private final Map<String, List<String>> policyToTemplates;
+    /** A map from policy name to list of data streams that use that policy. */
+    private final Map<String, List<String>> policyToDataStreams;
+    /** A map from policy name to list of indices that use that policy. */
+    private final Map<String, List<String>> policyToIndices;
+
+    public LifecyclePolicyUsageCalculator(
+        final IndexNameExpressionResolver indexNameExpressionResolver,
+        final ClusterState state,
+        List<String> requestedPolicyNames
+    ) {
+        final List<String> allDataStreams = indexNameExpressionResolver.dataStreamNames(
+            state,
+            IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR
+        );
+        // Sort all templates by descending priority. That way, findV2Template can exit on the first found template.
+        final var indexTemplates = new ArrayList<>(state.metadata().templatesV2().entrySet());
+        CollectionUtil.timSort(indexTemplates, Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder()));
+
+        final IndexLifecycleMetadata metadata = state.metadata().custom(IndexLifecycleMetadata.TYPE);
+        // We're making a bet here that if the `name` contains a wildcard, there's a large chance it'll simply match all policies.
+        final var expectedSize = Regex.isSimpleMatchPattern(requestedPolicyNames.get(0))
+            ? metadata.getPolicyMetadatas().size()
+            : requestedPolicyNames.size();
+
+        // We keep a map from composable template name to policy name to avoid having to resolve the template settings to determine
+        // the template's policy twice.
+        final Map<String, String> templateToPolicy = Maps.newHashMapWithExpectedSize(indexTemplates.size());
+
+        // Build the maps that will be used for the usage calculation later on.
+        policyToTemplates = Maps.newHashMapWithExpectedSize(expectedSize);
+        for (Map.Entry<String, ComposableIndexTemplate> entry : state.metadata().templatesV2().entrySet()) {
+            Settings settings = MetadataIndexTemplateService.resolveSettings(entry.getValue(), state.metadata().componentTemplates());
+            final var policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings);
+            // We only store the template if its policy matched any of the requested names.
+            if (doesPolicyMatchAnyName(policyName, requestedPolicyNames) == false) {
+                continue;
+            }
+            policyToTemplates.computeIfAbsent(policyName, k -> new ArrayList<>()).add(entry.getKey());
+            templateToPolicy.put(entry.getKey(), policyName);
+        }
+
+        policyToDataStreams = Maps.newHashMapWithExpectedSize(expectedSize);
+        for (String dataStream : allDataStreams) {
+            String indexTemplate = MetadataIndexTemplateService.findV2TemplateFromSortedList(
+                state.metadata(),
+                indexTemplates,
+                dataStream,
+                false
+            );
+            if (indexTemplate == null) {
+                // Every data stream should ordinarily have an index template, so this branch should not fire under normal circumstances.
+                continue;
+            }
+            final var policyName = templateToPolicy.get(indexTemplate);
+            // If there was no entry, either the template didn't specify an ILM policy or the policy didn't match any of the requested names
+            if (policyName == null) {
+                continue;
+            }
+            policyToDataStreams.computeIfAbsent(policyName, k -> new ArrayList<>()).add(dataStream);
+        }
+
+        policyToIndices = Maps.newHashMapWithExpectedSize(expectedSize);
+        for (IndexMetadata indexMetadata : state.metadata().indices().values()) {
+            final var policyName = indexMetadata.getLifecyclePolicyName();
+            // We only store the index if its policy matched any of the specified names.
+            if (doesPolicyMatchAnyName(policyName, requestedPolicyNames) == false) {
+                continue;
+            }
+            policyToIndices.computeIfAbsent(policyName, k -> new ArrayList<>()).add(indexMetadata.getIndex().getName());
+        }
+    }
+
+    /**
+     * Retrieves the pre-calculated indices, data streams, and composable templates that use the given policy.
+     */
+    public ItemUsage retrieveCalculatedUsage(String policyName) {
+        return new ItemUsage(
+            policyToIndices.getOrDefault(policyName, List.of()),
+            policyToDataStreams.getOrDefault(policyName, List.of()),
+            policyToTemplates.getOrDefault(policyName, List.of())
+        );
+    }
+
+    private boolean doesPolicyMatchAnyName(String policyName, List<String> names) {
+        for (var name : names) {
+            if (Regex.simpleMatch(name, policyName)) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java

@@ -8,14 +8,8 @@
 package org.elasticsearch.xpack.core.ilm;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.ItemUsage;
-import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.NotXContentException;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
@@ -24,7 +18,6 @@
 import org.elasticsearch.xpack.core.template.resources.TemplateResources;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.Map;
 
 /**
@@ -104,44 +97,4 @@ private static void validate(String source) {
             throw new ElasticsearchParseException("invalid policy", e);
         }
     }
-
-    /**
-     * Given a cluster state and ILM policy, calculate the {@link ItemUsage} of
-     * the policy (what indices, data streams, and templates use the policy)
-     */
-    public static ItemUsage calculateUsage(
-        final IndexNameExpressionResolver indexNameExpressionResolver,
-        final ClusterState state,
-        final String policyName
-    ) {
-        final List<String> indices = state.metadata()
-            .indices()
-            .values()
-            .stream()
-            .filter(indexMetadata -> policyName.equals(indexMetadata.getLifecyclePolicyName()))
-            .map(indexMetadata -> indexMetadata.getIndex().getName())
-            .toList();
-
-        final List<String> allDataStreams = indexNameExpressionResolver.dataStreamNames(
-            state,
-            IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR
-        );
-
-        final List<String> dataStreams = allDataStreams.stream().filter(dsName -> {
-            String indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dsName, false);
-            if (indexTemplate != null) {
-                Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate);
-                return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings));
-            } else {
-                return false;
-            }
-        }).toList();
-
-        final List<String> composableTemplates = state.metadata().templatesV2().keySet().stream().filter(templateName -> {
-            Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), templateName);
-            return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings));
-        }).toList();
-
-        return new ItemUsage(indices, dataStreams, composableTemplates);
-    }
 }