Optimize usage calculation in ILM policies retrieval API (elastic#106953)

Optimize calculating the usage of ILM policies in the `GET _ilm/policy` and `GET _ilm/policy/<policy_id>` endpoints by xtracting a separate class that pre-computes some parts on initialization (i.e. only once per request) and then uses those pre-computed parts when calculating the usage for an individual policy. By precomputing all the usages, the class makes a tradeoff by using a little bit more memory to significantly improve the overall processing time.

Author: nielsbauman

docs/changelog/106953.yaml

@@ -0,0 +1,6 @@
+pr: 106953
+summary: Optimize usage calculation in ILM policies retrieval API
+area: ILM+SLM
+type: enhancement
+issues:
+ - 105773

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -62,6 +62,7 @@
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -1087,9 +1088,8 @@ static Set<String> dataStreamsExclusivelyUsingTemplates(final ProjectMetadata pr
             .map(templateName -> projectMetadata.templatesV2().get(templateName))
             .filter(Objects::nonNull)
             .map(ComposableIndexTemplate::indexPatterns)
-            .map(Set::copyOf)
-            .reduce(Sets::union)
-            .orElse(Set.of());
+            .flatMap(List::stream)
+            .collect(Collectors.toSet());
 
         return projectMetadata.dataStreams()
             .values()
@@ -1099,9 +1099,10 @@ static Set<String> dataStreamsExclusivelyUsingTemplates(final ProjectMetadata pr
             .filter(ds -> {
                 // Retrieve the templates that match the data stream name ordered by priority
                 List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(
-                    projectMetadata,
+                    projectMetadata.templatesV2().entrySet(),
                     ds.getName(),
-                    ds.isHidden()
+                    ds.isHidden(),
+                    false
                 );
                 if (candidates.isEmpty()) {
                     throw new IllegalStateException("Data stream " + ds.getName() + " did not match any composable index templates.");
@@ -1110,13 +1111,10 @@ static Set<String> dataStreamsExclusivelyUsingTemplates(final ProjectMetadata pr
                 // Limit data streams that can ONLY use any of the specified templates, we do this by filtering
                 // the matching templates that are others than the ones requested and could be a valid template to use.
                 return candidates.stream()
-                    .filter(
+                    .noneMatch(
                         template -> templateNames.contains(template.v1()) == false
                             && isGlobalAndHasIndexHiddenSetting(projectMetadata, template.v2(), template.v1()) == false
-                    )
-                    .map(Tuple::v1)
-                    .toList()
-                    .isEmpty();
+                    );
             })
             .map(DataStream::getName)
             .collect(Collectors.toSet());
@@ -1357,7 +1355,41 @@ public static List<IndexTemplateMetadata> findV1Templates(
      */
     @Nullable
     public static String findV2Template(ProjectMetadata projectMetadata, String indexName, boolean isHidden) {
-        final List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(projectMetadata, indexName, isHidden);
+        return findV2Template(projectMetadata, projectMetadata.templatesV2().entrySet(), indexName, isHidden, false);
+    }
+
+    /**
+     * Return the name (id) of the highest matching index template out of the provided templates (that <i>need</i> to be sorted descending
+     * on priority beforehand), or the given index name. In the event that no templates are matched, {@code null} is returned.
+     */
+    @Nullable
+    public static String findV2TemplateFromSortedList(
+        ProjectMetadata projectMetadata,
+        Collection<Map.Entry<String, ComposableIndexTemplate>> templates,
+        String indexName,
+        boolean isHidden
+    ) {
+        return findV2Template(projectMetadata, templates, indexName, isHidden, true);
+    }
+
+    /**
+     * Return the name (id) of the highest matching index template, out of the provided templates, for the given index name. In
+     * the event that no templates are matched, {@code null} is returned.
+     */
+    @Nullable
+    private static String findV2Template(
+        ProjectMetadata projectMetadata,
+        Collection<Map.Entry<String, ComposableIndexTemplate>> templates,
+        String indexName,
+        boolean isHidden,
+        boolean exitOnFirstMatch
+    ) {
+        final List<Tuple<String, ComposableIndexTemplate>> candidates = findV2CandidateTemplates(
+            templates,
+            indexName,
+            isHidden,
+            exitOnFirstMatch
+        );
         if (candidates.isEmpty()) {
             return null;
         }
@@ -1386,33 +1418,39 @@ public static String findV2Template(ProjectMetadata projectMetadata, String inde
      * Return an ordered list of the name (id) and composable index templates that would apply to an index. The first
      * one is the winner template that is applied to this index. In the event that no templates are matched,
      * an empty list is returned.
+     * @param templates a list of template entries (name, template) - needs to be sorted when {@code exitOnFirstMatch} is {@code true}
+     * @param indexName the index (or data stream) name that should be used for matching the index patterns on the templates
+     * @param isHidden whether {@code indexName} belongs to a hidden index - this option is redundant for data streams, as backing indices
+     *                 of data streams will always be returned, regardless of whether the data stream is hidden or not
+     * @param exitOnFirstMatch if true, we return immediately after finding a match. That means that the <code>templates</code>
+     *                         parameter needs to be sorted based on priority (descending) for this method to return a sensible result,
+     *                         otherwise this method would just return the first template that matches the name, in an unspecified order
      */
-    static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(
-        ProjectMetadata projectMetadata,
+    private static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(
+        Collection<Map.Entry<String, ComposableIndexTemplate>> templates,
         String indexName,
-        boolean isHidden
+        boolean isHidden,
+        boolean exitOnFirstMatch
     ) {
+        assert exitOnFirstMatch == false || areTemplatesSorted(templates) : "Expected templates to be sorted";
         final String resolvedIndexName = IndexNameExpressionResolver.DateMathExpressionResolver.resolveExpression(indexName);
         final Predicate<String> patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, resolvedIndexName);
         final List<Tuple<String, ComposableIndexTemplate>> candidates = new ArrayList<>();
-        for (Map.Entry<String, ComposableIndexTemplate> entry : projectMetadata.templatesV2().entrySet()) {
+        for (Map.Entry<String, ComposableIndexTemplate> entry : templates) {
             final String name = entry.getKey();
             final ComposableIndexTemplate template = entry.getValue();
             /*
              * We do not ordinarily return match-all templates for hidden indices. But all backing indices for data streams are hidden,
              * and we do want to return even match-all templates for those. Not doing so can result in a situation where a data stream is
              * built with a template that none of its indices match.
              */
-            if (isHidden == false || template.getDataStreamTemplate() != null) {
-                if (anyMatch(template.indexPatterns(), patternMatchPredicate)) {
-                    candidates.add(Tuple.tuple(name, template));
-                }
-            } else {
-                final boolean isNotMatchAllTemplate = noneMatch(template.indexPatterns(), Regex::isMatchAllPattern);
-                if (isNotMatchAllTemplate) {
-                    if (anyMatch(template.indexPatterns(), patternMatchPredicate)) {
-                        candidates.add(Tuple.tuple(name, template));
-                    }
+            if (anyMatch(template.indexPatterns(), Regex::isMatchAllPattern) && isHidden && template.getDataStreamTemplate() == null) {
+                continue;
+            }
+            if (anyMatch(template.indexPatterns(), patternMatchPredicate)) {
+                candidates.add(Tuple.tuple(name, template));
+                if (exitOnFirstMatch) {
+                    return candidates;
                 }
             }
         }
@@ -1421,6 +1459,17 @@ static List<Tuple<String, ComposableIndexTemplate>> findV2CandidateTemplates(
         return candidates;
     }
 
+    private static boolean areTemplatesSorted(Collection<Map.Entry<String, ComposableIndexTemplate>> templates) {
+        ComposableIndexTemplate previousTemplate = null;
+        for (Map.Entry<String, ComposableIndexTemplate> template : templates) {
+            if (previousTemplate != null && template.getValue().priorityOrZero() > previousTemplate.priorityOrZero()) {
+                return false;
+            }
+            previousTemplate = template.getValue();
+        }
+        return true;
+    }
+
     // Checks if a global template specifies the `index.hidden` setting. This check is important because a global
     // template shouldn't specify the `index.hidden` setting, we leave it up to the caller to handle this situation.
     private static boolean isGlobalAndHasIndexHiddenSetting(

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java

@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ilm;
+
+import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.ItemUsage;
+import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.Maps;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A class that can be used to calculate the usages of ILM policies across the cluster. By precomputing all the usages,
+ * the class makes a tradeoff by using a little bit more memory to significantly improve the overall processing time.
+ */
+public class LifecyclePolicyUsageCalculator {
+
+    /** A map from policy name to list of composable templates that use that policy. */
+    private final Map<String, List<String>> policyToTemplates;
+    /** A map from policy name to list of data streams that use that policy. */
+    private final Map<String, List<String>> policyToDataStreams;
+    /** A map from policy name to list of indices that use that policy. */
+    private final Map<String, List<String>> policyToIndices;
+
+    public LifecyclePolicyUsageCalculator(
+        final IndexNameExpressionResolver indexNameExpressionResolver,
+        ProjectMetadata project,
+        List<String> requestedPolicyNames
+    ) {
+        final IndexLifecycleMetadata ilmMetadata = project.custom(IndexLifecycleMetadata.TYPE);
+        // We're making a bet here that if the `name` contains a wildcard, there's a large chance it'll simply match all policies.
+        final var expectedSize = Regex.isSimpleMatchPattern(requestedPolicyNames.get(0))
+            ? ilmMetadata.getPolicyMetadatas().size()
+            : requestedPolicyNames.size();
+
+        // We keep a map from composable template name to policy name to avoid having to resolve the template settings to determine
+        // the template's policy twice.
+        final Map<String, String> templateToPolicy = new HashMap<>();
+
+        // Build the map of which policy is used by which index templates.
+        policyToTemplates = Maps.newHashMapWithExpectedSize(expectedSize);
+        for (Map.Entry<String, ComposableIndexTemplate> entry : project.templatesV2().entrySet()) {
+            Settings settings = MetadataIndexTemplateService.resolveSettings(entry.getValue(), project.componentTemplates());
+            final var policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings);
+            // We only store the template if its policy matched any of the requested names.
+            if (doesPolicyMatchAnyName(policyName, requestedPolicyNames) == false) {
+                continue;
+            }
+            policyToTemplates.computeIfAbsent(policyName, k -> new ArrayList<>()).add(entry.getKey());
+            templateToPolicy.put(entry.getKey(), policyName);
+        }
+
+        // Sort all templates by descending priority. That way, findV2Template can exit on the first-matched template.
+        final var indexTemplates = new ArrayList<>(project.templatesV2().entrySet());
+        CollectionUtil.timSort(indexTemplates, Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder()));
+
+        // Build the map of which policy is used by which data streams.
+        policyToDataStreams = Maps.newHashMapWithExpectedSize(expectedSize);
+        final List<String> allDataStreams = indexNameExpressionResolver.dataStreamNames(
+            project,
+            IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR
+        );
+        for (String dataStream : allDataStreams) {
+            // Find the index template with the highest priority that matches this data stream's name.
+            String indexTemplate = MetadataIndexTemplateService.findV2TemplateFromSortedList(project, indexTemplates, dataStream, false);
+            if (indexTemplate == null) {
+                assert false : "Data stream [" + dataStream + "] has no matching template";
+                continue;
+            }
+            final var policyName = templateToPolicy.get(indexTemplate);
+            // If there was no entry, either the template didn't specify an ILM policy or the policy didn't match any of the requested names
+            if (policyName == null) {
+                continue;
+            }
+            policyToDataStreams.computeIfAbsent(policyName, k -> new ArrayList<>()).add(dataStream);
+        }
+
+        // Build the map of which policy is used by which indices.
+        policyToIndices = Maps.newHashMapWithExpectedSize(expectedSize);
+        for (IndexMetadata indexMetadata : project.indices().values()) {
+            final var policyName = indexMetadata.getLifecyclePolicyName();
+            // We only store the index if its policy matched any of the specified names.
+            if (doesPolicyMatchAnyName(policyName, requestedPolicyNames) == false) {
+                continue;
+            }
+            policyToIndices.computeIfAbsent(policyName, k -> new ArrayList<>()).add(indexMetadata.getIndex().getName());
+        }
+    }
+
+    /**
+     * Retrieves the pre-calculated indices, data streams, and composable templates that use the given policy.
+     */
+    public ItemUsage retrieveCalculatedUsage(String policyName) {
+        return new ItemUsage(
+            policyToIndices.getOrDefault(policyName, List.of()),
+            policyToDataStreams.getOrDefault(policyName, List.of()),
+            policyToTemplates.getOrDefault(policyName, List.of())
+        );
+    }
+
+    private boolean doesPolicyMatchAnyName(String policyName, List<String> names) {
+        for (var name : names) {
+            if (Regex.simpleMatch(name, policyName)) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java

@@ -8,14 +8,8 @@
 package org.elasticsearch.xpack.core.ilm;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.ItemUsage;
-import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
-import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.NotXContentException;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
@@ -24,7 +18,6 @@
 import org.elasticsearch.xpack.core.template.resources.TemplateResources;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.Map;
 
 /**
@@ -104,43 +97,4 @@ private static void validate(String source) {
             throw new ElasticsearchParseException("invalid policy", e);
         }
     }
-
-    /**
-     * Given a cluster state and ILM policy, calculate the {@link ItemUsage} of
-     * the policy (what indices, data streams, and templates use the policy)
-     */
-    public static ItemUsage calculateUsage(
-        final IndexNameExpressionResolver indexNameExpressionResolver,
-        final ProjectMetadata project,
-        final String policyName
-    ) {
-        final List<String> indices = project.indices()
-            .values()
-            .stream()
-            .filter(indexMetadata -> policyName.equals(indexMetadata.getLifecyclePolicyName()))
-            .map(indexMetadata -> indexMetadata.getIndex().getName())
-            .toList();
-
-        final List<String> allDataStreams = indexNameExpressionResolver.dataStreamNames(
-            project,
-            IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR
-        );
-
-        final List<String> dataStreams = allDataStreams.stream().filter(dsName -> {
-            String indexTemplate = MetadataIndexTemplateService.findV2Template(project, dsName, false);
-            if (indexTemplate != null) {
-                Settings settings = MetadataIndexTemplateService.resolveSettings(project, indexTemplate);
-                return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings));
-            } else {
-                return false;
-            }
-        }).toList();
-
-        final List<String> composableTemplates = project.templatesV2().keySet().stream().filter(templateName -> {
-            Settings settings = MetadataIndexTemplateService.resolveSettings(project, templateName);
-            return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings));
-        }).toList();
-
-        return new ItemUsage(indices, dataStreams, composableTemplates);
-    }
 }