From 5bf393c7ffa5fbbe030f3faa3145784a94197e62 Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Mon, 1 Apr 2024 11:55:23 +0200 Subject: [PATCH 01/16] Optimize retrieving ILM policies --- .../xpack/core/ilm/LifecyclePolicyUtils.java | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java index 4fb94dce1dcd0..23c124b53d810 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.NotXContentException; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -23,8 +24,10 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.template.resources.TemplateResources; +import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.stream.Collectors; /** @@ -109,26 +112,36 @@ public static ItemUsage calculateUsage( .map(indexMetadata -> indexMetadata.getIndex().getName()) .collect(Collectors.toList()); + // First find all the index templates that use this policy, and sort them descending on priority. + final var composableTemplates = state.metadata().templatesV2().entrySet().stream().filter(entry -> { + Settings settings = MetadataIndexTemplateService.resolveSettings(entry.getValue(), state.metadata().componentTemplates()); + return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings)); + }).sorted(Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder())).toList(); + + // These index templates are returned as a type of usage themselves. + final var composableTemplateNames = composableTemplates.stream().map(Map.Entry::getKey).toList(); + final List allDataStreams = indexNameExpressionResolver.dataStreamNames( state, IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN ); + // We filter all the data streams by finding the first index template (highest priority) whose index pattern covers the data stream. final List dataStreams = allDataStreams.stream().filter(dsName -> { - String indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dsName, false); - if (indexTemplate != null) { - Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate); - return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings)); - } else { - return false; + final Predicate patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, dsName); + for (var entry : composableTemplates) { + final boolean matched = entry.getValue().indexPatterns().stream().anyMatch(patternMatchPredicate); + if (matched) { + Settings settings = MetadataIndexTemplateService.resolveSettings( + entry.getValue(), + state.metadata().componentTemplates() + ); + return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings)); + } } + return false; }).collect(Collectors.toList()); - final List composableTemplates = state.metadata().templatesV2().keySet().stream().filter(templateName -> { - Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), templateName); - return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings)); - }).collect(Collectors.toList()); - - return new ItemUsage(indices, dataStreams, composableTemplates); + return new ItemUsage(indices, dataStreams, composableTemplateNames); } } From d160dc2255586953299c256b744c319fa8a6a349 Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Mon, 1 Apr 2024 13:35:44 +0200 Subject: [PATCH 02/16] Update docs/changelog/106953.yaml --- docs/changelog/106953.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/106953.yaml diff --git a/docs/changelog/106953.yaml b/docs/changelog/106953.yaml new file mode 100644 index 0000000000000..6993b1960a401 --- /dev/null +++ b/docs/changelog/106953.yaml @@ -0,0 +1,6 @@ +pr: 106953 +summary: Optimize usage calculation in ILM policies retrieval API +area: ILM+SLM +type: enhancement +issues: + - 105773 From 092cb0cfcc8fd472d507c195e3af8c18bfc7e137 Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Tue, 2 Apr 2024 12:41:58 +0200 Subject: [PATCH 03/16] Remove redundant computation --- .../elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java index 23c124b53d810..a571ae4358a7a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java @@ -132,11 +132,7 @@ public static ItemUsage calculateUsage( for (var entry : composableTemplates) { final boolean matched = entry.getValue().indexPatterns().stream().anyMatch(patternMatchPredicate); if (matched) { - Settings settings = MetadataIndexTemplateService.resolveSettings( - entry.getValue(), - state.metadata().componentTemplates() - ); - return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings)); + return true; } } return false; From d30376e9da6b3952ef0a029c8d0ecb2e02787266 Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Sat, 4 May 2024 17:43:31 +0200 Subject: [PATCH 04/16] Change approach --- .../MetadataIndexTemplateService.java | 92 ++++--- .../ilm/LifecyclePolicyUsageCalculator.java | 105 ++++++++ .../xpack/core/ilm/LifecyclePolicyUtils.java | 57 ----- .../LifecyclePolicyUsageCalculatorTests.java | 237 ++++++++++++++++++ .../core/ilm/LifecyclePolicyUtilsTests.java | 187 -------------- .../action/TransportGetLifecycleAction.java | 7 +- 6 files changed, 409 insertions(+), 276 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculatorTests.java delete mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 631845dc33288..8c39e5607c731 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -61,6 +61,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -1050,29 +1051,24 @@ static Set dataStreamsExclusivelyUsingTemplates(final ClusterState state .reduce(Sets::union) .orElse(Set.of()); + // Filter and sort all composable templates in advance, to speed up template retrieval later on. + var templates = state.metadata() + .templatesV2() + .entrySet() + .stream() + .filter( + entry -> templateNames.contains(entry.getKey()) == false + && isGlobalAndHasIndexHiddenSetting(metadata, entry.getValue(), entry.getKey()) == false + ) + .sorted(Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder())) + .toList(); + return metadata.dataStreams() .values() .stream() // Limit to checking data streams that match any of the templates' index patterns .filter(ds -> namePatterns.stream().anyMatch(pattern -> Regex.simpleMatch(pattern, ds.getName()))) - .filter(ds -> { - // Retrieve the templates that match the data stream name ordered by priority - List> candidates = findV2CandidateTemplates(metadata, ds.getName(), ds.isHidden()); - if (candidates.isEmpty()) { - throw new IllegalStateException("Data stream " + ds.getName() + " did not match any composable index templates."); - } - - // Limit data streams that can ONLY use any of the specified templates, we do this by filtering - // the matching templates that are others than the ones requested and could be a valid template to use. - return candidates.stream() - .filter( - template -> templateNames.contains(template.v1()) == false - && isGlobalAndHasIndexHiddenSetting(metadata, template.v2(), template.v1()) == false - ) - .map(Tuple::v1) - .toList() - .isEmpty(); - }) + .filter(ds -> findV2Template(state.metadata(), templates, ds.getName(), ds.isHidden(), true) == null) .map(DataStream::getName) .collect(Collectors.toSet()); } @@ -1268,7 +1264,27 @@ public static List findV1Templates(Metadata metadata, Str */ @Nullable public static String findV2Template(Metadata metadata, String indexName, boolean isHidden) { - final List> candidates = findV2CandidateTemplates(metadata, indexName, isHidden); + return findV2Template(metadata, metadata.templatesV2().entrySet(), indexName, isHidden, false); + } + + /** + * Return the name (id) of the highest matching index template for the given index name. In + * the event that no templates are matched, {@code null} is returned. + */ + @Nullable + public static String findV2Template( + Metadata metadata, + Collection> templateEntries, + String indexName, + boolean isHidden, + boolean exitOnFirstMatch + ) { + final List> candidates = findV2CandidateTemplates( + templateEntries, + indexName, + isHidden, + exitOnFirstMatch + ); if (candidates.isEmpty()) { return null; } @@ -1296,27 +1312,45 @@ public static String findV2Template(Metadata metadata, String indexName, boolean /** * Return an ordered list of the name (id) and composable index templates that would apply to an index. The first * one is the winner template that is applied to this index. In the event that no templates are matched, - * an empty list is returned. + * an empty list is returned. If exitOnFirstMatch is true, we return immediately after finding a match. */ - static List> findV2CandidateTemplates(Metadata metadata, String indexName, boolean isHidden) { + static List> findV2CandidateTemplates( + Collection> templateEntries, + String indexName, + boolean isHidden, + boolean exitOnFirstMatch + ) { final String resolvedIndexName = IndexNameExpressionResolver.DateMathExpressionResolver.resolveExpression(indexName); - final Predicate patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, resolvedIndexName); final List> candidates = new ArrayList<>(); - for (Map.Entry entry : metadata.templatesV2().entrySet()) { + outerLoop: + for (Map.Entry entry : templateEntries) { final String name = entry.getKey(); final ComposableIndexTemplate template = entry.getValue(); if (isHidden == false) { - final boolean matched = template.indexPatterns().stream().anyMatch(patternMatchPredicate); - if (matched) { - candidates.add(Tuple.tuple(name, template)); + for (String indexPattern : template.indexPatterns()) { + if (Regex.simpleMatch(indexPattern, resolvedIndexName)) { + candidates.add(Tuple.tuple(name, template)); + if (exitOnFirstMatch) { + return candidates; + } + } } } else { - final boolean isNotMatchAllTemplate = template.indexPatterns().stream().noneMatch(Regex::isMatchAllPattern); - if (isNotMatchAllTemplate) { - if (template.indexPatterns().stream().anyMatch(patternMatchPredicate)) { + // For hidden indices, we don't want to return any templates that match "all". + boolean match = false; + for (String indexPattern : template.indexPatterns()) { + if (Regex.isMatchAllPattern(indexPattern)) { + continue outerLoop; + } + // If we've already found a match, we still want to keep looping to make sure this template doesn't have a "*" pattern. + if (match == false && Regex.simpleMatch(indexPattern, resolvedIndexName)) { candidates.add(Tuple.tuple(name, template)); + match = true; } } + if (match && exitOnFirstMatch) { + return candidates; + } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java new file mode 100644 index 0000000000000..28c84f1344aad --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ilm; + +import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.ItemUsage; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Settings; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * A class that can be used to calculate the usages of ILM policies. This class computes some information on initialization, which will + * use a bit more memory but speeds up the usage calculation significantly. + */ +public class LifecyclePolicyUsageCalculator { + + private final ClusterState state; + /** Whether {@link #calculateUsage} will be called multiple times or not. */ + private final boolean willIterate; + /** A map from policy name to list of data streams that use that policy. */ + private final Map> policyToDataStream; + /** A map from composable template name to the policy name it uses (or null) */ + private final Map templateToPolicy; + + public LifecyclePolicyUsageCalculator( + final IndexNameExpressionResolver indexNameExpressionResolver, + final ClusterState state, + List names + ) { + this.state = state; + this.willIterate = names.size() > 1 || Regex.isSimpleMatchPattern(names.get(0)); + + var allDataStreams = indexNameExpressionResolver.dataStreamNames(state, IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN); + // Sort all templates by descending priority. That way, findV2Template can exit on the first found template. + var indexTemplates = new ArrayList<>(state.metadata().templatesV2().entrySet()); + CollectionUtil.timSort(indexTemplates, Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder())); + + // Build the maps that will be used for the usage calculation later on. + IndexLifecycleMetadata metadata = state.metadata().custom(IndexLifecycleMetadata.TYPE); + policyToDataStream = new HashMap<>(Regex.isSimpleMatchPattern(names.get(0)) ? metadata.getPolicyMetadatas().size() : names.size()); + templateToPolicy = new HashMap<>(indexTemplates.size()); + for (String dataStream : allDataStreams) { + String indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), indexTemplates, dataStream, false, true); + if (indexTemplate == null) { + continue; + } + Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate); + var policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings); + if (names.stream().noneMatch(name -> Regex.simpleMatch(name, policyName))) { + // If a template's policy doesn't match any of the supplied names, we can skip it later on. + templateToPolicy.put(indexTemplate, null); + continue; + } + templateToPolicy.put(indexTemplate, policyName); + policyToDataStream.computeIfAbsent(policyName, k -> new ArrayList<>()).add(dataStream); + } + } + + /** + * Calculate the indices, data streams, and composable templates that use the given policy. + */ + public ItemUsage calculateUsage(String policyName) { + List indices = new ArrayList<>(); + for (IndexMetadata indexMetadata : state.metadata().indices().values()) { + if (policyName.equals(indexMetadata.getLifecyclePolicyName())) { + indices.add(indexMetadata.getIndex().getName()); + } + } + + List composableTemplates = new ArrayList<>(); + for (Map.Entry entry : state.metadata().templatesV2().entrySet()) { + var foundPolicy = templateToPolicy.get(entry.getKey()); + // Extra `containsKey` check to account for templates not using any policy. + if (foundPolicy == null && templateToPolicy.containsKey(entry.getKey()) == false) { + Settings settings = MetadataIndexTemplateService.resolveSettings(entry.getValue(), state.metadata().componentTemplates()); + foundPolicy = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings); + // If this method will only be called once, we don't need to keep building the map. + if (willIterate) { + templateToPolicy.put(entry.getKey(), foundPolicy); + } + } + if (policyName.equals(foundPolicy)) { + composableTemplates.add(entry.getKey()); + } + } + + return new ItemUsage(indices, policyToDataStream.getOrDefault(policyName, List.of()), composableTemplates); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java index a571ae4358a7a..e595334d002b1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java @@ -8,15 +8,8 @@ package org.elasticsearch.xpack.core.ilm; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.ItemUsage; -import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.NotXContentException; -import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParser; @@ -24,11 +17,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.template.resources.TemplateResources; -import java.util.Comparator; -import java.util.List; import java.util.Map; -import java.util.function.Predicate; -import java.util.stream.Collectors; /** * A utility class used for index lifecycle policies @@ -94,50 +83,4 @@ private static void validate(String source) { throw new ElasticsearchParseException("invalid policy", e); } } - - /** - * Given a cluster state and ILM policy, calculate the {@link ItemUsage} of - * the policy (what indices, data streams, and templates use the policy) - */ - public static ItemUsage calculateUsage( - final IndexNameExpressionResolver indexNameExpressionResolver, - final ClusterState state, - final String policyName - ) { - final List indices = state.metadata() - .indices() - .values() - .stream() - .filter(indexMetadata -> policyName.equals(indexMetadata.getLifecyclePolicyName())) - .map(indexMetadata -> indexMetadata.getIndex().getName()) - .collect(Collectors.toList()); - - // First find all the index templates that use this policy, and sort them descending on priority. - final var composableTemplates = state.metadata().templatesV2().entrySet().stream().filter(entry -> { - Settings settings = MetadataIndexTemplateService.resolveSettings(entry.getValue(), state.metadata().componentTemplates()); - return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings)); - }).sorted(Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder())).toList(); - - // These index templates are returned as a type of usage themselves. - final var composableTemplateNames = composableTemplates.stream().map(Map.Entry::getKey).toList(); - - final List allDataStreams = indexNameExpressionResolver.dataStreamNames( - state, - IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN - ); - - // We filter all the data streams by finding the first index template (highest priority) whose index pattern covers the data stream. - final List dataStreams = allDataStreams.stream().filter(dsName -> { - final Predicate patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, dsName); - for (var entry : composableTemplates) { - final boolean matched = entry.getValue().indexPatterns().stream().anyMatch(patternMatchPredicate); - if (matched) { - return true; - } - } - return false; - }).collect(Collectors.toList()); - - return new ItemUsage(indices, dataStreams, composableTemplateNames); - } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculatorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculatorTests.java new file mode 100644 index 0000000000000..88b43b5e78c31 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculatorTests.java @@ -0,0 +1,237 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ilm; + +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.ItemUsage; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.EmptySystemIndices; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class LifecyclePolicyUsageCalculatorTests extends ESTestCase { + + private IndexNameExpressionResolver iner; + + @Before + public void init() { + iner = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY), EmptySystemIndices.INSTANCE); + } + + public void testGetUsageNonExistentPolicy() { + // Test where policy does not exist + ClusterState state = ClusterState.builder(new ClusterName("mycluster")).build(); + assertThat( + new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).calculateUsage("mypolicy"), + equalTo(new ItemUsage(Collections.emptyList(), Collections.emptyList(), Collections.emptyList())) + ); + } + + public void testGetUsageUnusedPolicy() { + // Test where policy is not used by anything + ClusterState state = ClusterState.builder(new ClusterName("mycluster")) + .metadata( + Metadata.builder() + .putCustom( + IndexLifecycleMetadata.TYPE, + new IndexLifecycleMetadata( + Collections.singletonMap("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), + OperationMode.RUNNING + ) + ) + .build() + ) + .build(); + assertThat( + new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).calculateUsage("mypolicy"), + equalTo(new ItemUsage(Collections.emptyList(), Collections.emptyList(), Collections.emptyList())) + ); + } + + public void testGetUsagePolicyUsedByIndex() { + + // Test where policy exists and is used by an index + ClusterState state = ClusterState.builder(new ClusterName("mycluster")) + .metadata( + Metadata.builder() + .putCustom( + IndexLifecycleMetadata.TYPE, + new IndexLifecycleMetadata( + Collections.singletonMap("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), + OperationMode.RUNNING + ) + ) + .put( + IndexMetadata.builder("myindex") + .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) + ) + .build() + ) + .build(); + assertThat( + new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).calculateUsage("mypolicy"), + equalTo(new ItemUsage(Collections.singleton("myindex"), Collections.emptyList(), Collections.emptyList())) + ); + } + + public void testGetUsagePolicyUsedByIndexAndTemplate() { + + // Test where policy exists and is used by an index, and template + ClusterState state = ClusterState.builder(new ClusterName("mycluster")) + .metadata( + Metadata.builder() + .putCustom( + IndexLifecycleMetadata.TYPE, + new IndexLifecycleMetadata( + Collections.singletonMap("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), + OperationMode.RUNNING + ) + ) + .put( + IndexMetadata.builder("myindex") + .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) + ) + .putCustom( + ComposableIndexTemplateMetadata.TYPE, + new ComposableIndexTemplateMetadata( + Collections.singletonMap( + "mytemplate", + ComposableIndexTemplate.builder() + .indexPatterns(Collections.singletonList("myds")) + .template( + new Template( + Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), + null, + null + ) + ) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ) + ) + ) + .build() + ) + .build(); + assertThat( + new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).calculateUsage("mypolicy"), + equalTo(new ItemUsage(Collections.singleton("myindex"), Collections.emptyList(), Collections.singleton("mytemplate"))) + ); + } + + public void testGetUsagePolicyUsedByIndexAndTemplateAndDataStream() { + // Test where policy exists and is used by an index, data stream, and template + Metadata.Builder mBuilder = Metadata.builder() + .putCustom( + IndexLifecycleMetadata.TYPE, + new IndexLifecycleMetadata( + Collections.singletonMap("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), + OperationMode.RUNNING + ) + ) + .put( + IndexMetadata.builder("myindex") + .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) + ) + .put( + IndexMetadata.builder("another") + .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) + ) + .put( + IndexMetadata.builder("other") + .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "otherpolicy")) + ) + + .putCustom( + ComposableIndexTemplateMetadata.TYPE, + new ComposableIndexTemplateMetadata( + Collections.singletonMap( + "mytemplate", + ComposableIndexTemplate.builder() + .indexPatterns(Collections.singletonList("myds")) + .template( + new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null) + ) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ) + ) + ); + // Need to get the real Index instance of myindex: + mBuilder.put(DataStreamTestHelper.newInstance("myds", Collections.singletonList(mBuilder.get("myindex").getIndex()))); + + ClusterState state = ClusterState.builder(new ClusterName("mycluster")).metadata(mBuilder.build()).build(); + assertThat( + new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).calculateUsage("mypolicy"), + equalTo(new ItemUsage(Arrays.asList("myindex", "another"), Collections.singleton("myds"), Collections.singleton("mytemplate"))) + ); + } + + public void testGetUsagePolicyNotUsedByDataStreamDueToOverride() { + // Test when a data stream does not use the policy anymore because of a higher template + Metadata.Builder mBuilder = Metadata.builder() + .put( + IndexMetadata.builder("myindex") + .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) + ) + .putCustom( + IndexLifecycleMetadata.TYPE, + new IndexLifecycleMetadata( + Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), + OperationMode.RUNNING + ) + ) + .putCustom( + ComposableIndexTemplateMetadata.TYPE, + new ComposableIndexTemplateMetadata( + Map.of( + "mytemplate", + ComposableIndexTemplate.builder() + .indexPatterns(Collections.singletonList("myds*")) + .template( + new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null) + ) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build(), + "myhighertemplate", + ComposableIndexTemplate.builder() + .indexPatterns(Collections.singletonList("myds")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .priority(1_000L) + .build() + ) + ) + ); + + mBuilder.put(DataStreamTestHelper.newInstance("myds", Collections.singletonList(mBuilder.get("myindex").getIndex()))); + + // Test where policy exists and is used by an index, datastream, and template + ClusterState state = ClusterState.builder(new ClusterName("mycluster")).metadata(mBuilder.build()).build(); + assertThat( + new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).calculateUsage("mypolicy"), + equalTo(new ItemUsage(List.of("myindex"), List.of(), List.of("mytemplate"))) + ); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java deleted file mode 100644 index 3efe2dc04ea19..0000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core.ilm; - -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata; -import org.elasticsearch.cluster.metadata.DataStreamTestHelper; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.ItemUsage; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.indices.EmptySystemIndices; -import org.elasticsearch.test.ESTestCase; - -import java.util.Arrays; -import java.util.Collections; - -import static org.hamcrest.Matchers.equalTo; - -public class LifecyclePolicyUtilsTests extends ESTestCase { - public void testCalculateUsage() { - final IndexNameExpressionResolver iner = new IndexNameExpressionResolver( - new ThreadContext(Settings.EMPTY), - EmptySystemIndices.INSTANCE - ); - - { - // Test where policy does not exist - ClusterState state = ClusterState.builder(new ClusterName("mycluster")).build(); - assertThat( - LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"), - equalTo(new ItemUsage(Collections.emptyList(), Collections.emptyList(), Collections.emptyList())) - ); - } - - { - // Test where policy is not used by anything - ClusterState state = ClusterState.builder(new ClusterName("mycluster")) - .metadata( - Metadata.builder() - .putCustom( - IndexLifecycleMetadata.TYPE, - new IndexLifecycleMetadata( - Collections.singletonMap("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), - OperationMode.RUNNING - ) - ) - .build() - ) - .build(); - assertThat( - LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"), - equalTo(new ItemUsage(Collections.emptyList(), Collections.emptyList(), Collections.emptyList())) - ); - } - - { - // Test where policy exists and is used by an index - ClusterState state = ClusterState.builder(new ClusterName("mycluster")) - .metadata( - Metadata.builder() - .putCustom( - IndexLifecycleMetadata.TYPE, - new IndexLifecycleMetadata( - Collections.singletonMap("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), - OperationMode.RUNNING - ) - ) - .put( - IndexMetadata.builder("myindex") - .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) - ) - .build() - ) - .build(); - assertThat( - LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"), - equalTo(new ItemUsage(Collections.singleton("myindex"), Collections.emptyList(), Collections.emptyList())) - ); - } - - { - // Test where policy exists and is used by an index, and template - ClusterState state = ClusterState.builder(new ClusterName("mycluster")) - .metadata( - Metadata.builder() - .putCustom( - IndexLifecycleMetadata.TYPE, - new IndexLifecycleMetadata( - Collections.singletonMap("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), - OperationMode.RUNNING - ) - ) - .put( - IndexMetadata.builder("myindex") - .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) - ) - .putCustom( - ComposableIndexTemplateMetadata.TYPE, - new ComposableIndexTemplateMetadata( - Collections.singletonMap( - "mytemplate", - ComposableIndexTemplate.builder() - .indexPatterns(Collections.singletonList("myds")) - .template( - new Template( - Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), - null, - null - ) - ) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) - .build() - ) - ) - ) - .build() - ) - .build(); - assertThat( - LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"), - equalTo(new ItemUsage(Collections.singleton("myindex"), Collections.emptyList(), Collections.singleton("mytemplate"))) - ); - } - - { - Metadata.Builder mBuilder = Metadata.builder() - .putCustom( - IndexLifecycleMetadata.TYPE, - new IndexLifecycleMetadata( - Collections.singletonMap("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), - OperationMode.RUNNING - ) - ) - .put( - IndexMetadata.builder("myindex") - .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) - ) - .put( - IndexMetadata.builder("another") - .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) - ) - .put( - IndexMetadata.builder("other") - .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "otherpolicy")) - ) - - .putCustom( - ComposableIndexTemplateMetadata.TYPE, - new ComposableIndexTemplateMetadata( - Collections.singletonMap( - "mytemplate", - ComposableIndexTemplate.builder() - .indexPatterns(Collections.singletonList("myds")) - .template( - new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null) - ) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) - .build() - ) - ) - ); - // Need to get the real Index instance of myindex: - mBuilder.put(DataStreamTestHelper.newInstance("myds", Collections.singletonList(mBuilder.get("myindex").getIndex()))); - - // Test where policy exists and is used by an index, datastream, and template - ClusterState state = ClusterState.builder(new ClusterName("mycluster")).metadata(mBuilder.build()).build(); - assertThat( - LifecyclePolicyUtils.calculateUsage(iner, state, "mypolicy"), - equalTo( - new ItemUsage(Arrays.asList("myindex", "another"), Collections.singleton("myds"), Collections.singleton("mytemplate")) - ) - ); - } - } -} diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java index f58d101a330c8..6bd550a602d7a 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java @@ -24,7 +24,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata; -import org.elasticsearch.xpack.core.ilm.LifecyclePolicyUtils; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicyUsageCalculator; import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction; import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction.LifecyclePolicyResponseItem; import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction.Request; @@ -91,6 +91,7 @@ protected void masterOperation(Task task, Request request, ClusterState state, A ); } + var lifecyclePolicyUsageCalculator = new LifecyclePolicyUsageCalculator(indexNameExpressionResolver, state, names); Map policyResponseItemMap = new LinkedHashMap<>(); for (String name : names) { if (Regex.isSimpleMatchPattern(name)) { @@ -106,7 +107,7 @@ protected void masterOperation(Task task, Request request, ClusterState state, A policyMetadata.getPolicy(), policyMetadata.getVersion(), policyMetadata.getModifiedDateString(), - LifecyclePolicyUtils.calculateUsage(indexNameExpressionResolver, state, policyMetadata.getName()) + lifecyclePolicyUsageCalculator.calculateUsage(policyMetadata.getName()) ) ); } @@ -123,7 +124,7 @@ protected void masterOperation(Task task, Request request, ClusterState state, A policyMetadata.getPolicy(), policyMetadata.getVersion(), policyMetadata.getModifiedDateString(), - LifecyclePolicyUtils.calculateUsage(indexNameExpressionResolver, state, policyMetadata.getName()) + lifecyclePolicyUsageCalculator.calculateUsage(policyMetadata.getName()) ) ); } From 293ff9d7ecd187523221204c6f3c50920d51d011 Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Tue, 21 May 2024 16:17:14 +0200 Subject: [PATCH 05/16] Refactor confusing loop --- .../MetadataIndexTemplateService.java | 33 ++++++------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 0081941fe296f..683326feec10b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1322,35 +1322,22 @@ static List> findV2CandidateTemplates( ) { final String resolvedIndexName = IndexNameExpressionResolver.DateMathExpressionResolver.resolveExpression(indexName); final List> candidates = new ArrayList<>(); - outerLoop: for (Map.Entry entry : templateEntries) { final String name = entry.getKey(); final ComposableIndexTemplate template = entry.getValue(); - if (isHidden == false) { - for (String indexPattern : template.indexPatterns()) { - if (Regex.simpleMatch(indexPattern, resolvedIndexName)) { - candidates.add(Tuple.tuple(name, template)); - if (exitOnFirstMatch) { - return candidates; - } - } + if (isHidden) { + final boolean hasMatchAllTemplate = template.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern); + if (hasMatchAllTemplate) { + continue; } - } else { - // For hidden indices, we don't want to return any templates that match "all". - boolean match = false; - for (String indexPattern : template.indexPatterns()) { - if (Regex.isMatchAllPattern(indexPattern)) { - continue outerLoop; - } - // If we've already found a match, we still want to keep looping to make sure this template doesn't have a "*" pattern. - if (match == false && Regex.simpleMatch(indexPattern, resolvedIndexName)) { - candidates.add(Tuple.tuple(name, template)); - match = true; + } + for (String indexPattern : template.indexPatterns()) { + if (Regex.simpleMatch(indexPattern, resolvedIndexName)) { + candidates.add(Tuple.tuple(name, template)); + if (exitOnFirstMatch) { + return candidates; } } - if (match && exitOnFirstMatch) { - return candidates; - } } } From 02fb7a25d26a7b98747b1ba30eff72c279a8a731 Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Tue, 21 May 2024 16:37:42 +0200 Subject: [PATCH 06/16] PR feedback --- .../metadata/MetadataIndexTemplateService.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 683326feec10b..9c82606cbe81d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1051,8 +1051,8 @@ static Set dataStreamsExclusivelyUsingTemplates(final ClusterState state .reduce(Sets::union) .orElse(Set.of()); - // Filter and sort all composable templates in advance, to speed up template retrieval later on. - var templates = state.metadata() + // Determine all the composable templates that are not one of the provided templates. + var otherTemplates = state.metadata() .templatesV2() .entrySet() .stream() @@ -1060,6 +1060,7 @@ static Set dataStreamsExclusivelyUsingTemplates(final ClusterState state entry -> templateNames.contains(entry.getKey()) == false && isGlobalAndHasIndexHiddenSetting(metadata, entry.getValue(), entry.getKey()) == false ) + // Sort here so we can `exitOnFirstMatch` in `findV2Template`. .sorted(Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder())) .toList(); @@ -1068,7 +1069,7 @@ && isGlobalAndHasIndexHiddenSetting(metadata, entry.getValue(), entry.getKey()) .stream() // Limit to checking data streams that match any of the templates' index patterns .filter(ds -> namePatterns.stream().anyMatch(pattern -> Regex.simpleMatch(pattern, ds.getName()))) - .filter(ds -> findV2Template(state.metadata(), templates, ds.getName(), ds.isHidden(), true) == null) + .filter(ds -> findV2Template(state.metadata(), otherTemplates, ds.getName(), ds.isHidden(), true) == null) .map(DataStream::getName) .collect(Collectors.toSet()); } @@ -1268,19 +1269,19 @@ public static String findV2Template(Metadata metadata, String indexName, boolean } /** - * Return the name (id) of the highest matching index template for the given index name. In + * Return the name (id) of the highest matching index template, out of the provided templates, for the given index name. In * the event that no templates are matched, {@code null} is returned. */ @Nullable public static String findV2Template( Metadata metadata, - Collection> templateEntries, + Collection> templates, String indexName, boolean isHidden, boolean exitOnFirstMatch ) { final List> candidates = findV2CandidateTemplates( - templateEntries, + templates, indexName, isHidden, exitOnFirstMatch @@ -1315,14 +1316,14 @@ public static String findV2Template( * an empty list is returned. If exitOnFirstMatch is true, we return immediately after finding a match. */ static List> findV2CandidateTemplates( - Collection> templateEntries, + Collection> templates, String indexName, boolean isHidden, boolean exitOnFirstMatch ) { final String resolvedIndexName = IndexNameExpressionResolver.DateMathExpressionResolver.resolveExpression(indexName); final List> candidates = new ArrayList<>(); - for (Map.Entry entry : templateEntries) { + for (Map.Entry entry : templates) { final String name = entry.getKey(); final ComposableIndexTemplate template = entry.getValue(); if (isHidden) { From e193a30ab5d5b7c3660920bf417188bc4db62ca7 Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Tue, 21 May 2024 16:58:15 +0200 Subject: [PATCH 07/16] Add missing break --- .../cluster/metadata/MetadataIndexTemplateService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 9c82606cbe81d..07d8eb6383b07 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1338,6 +1338,7 @@ static List> findV2CandidateTemplates( if (exitOnFirstMatch) { return candidates; } + break; } } } From 4cd9bfbd5e77e924b0f9d0a56bf5d82a2430052b Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Sat, 15 Feb 2025 22:11:19 +1000 Subject: [PATCH 08/16] Small refactorings --- .../ilm/LifecyclePolicyUsageCalculator.java | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java index 499d051fdaed4..27de2d493f5a0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java @@ -17,10 +17,10 @@ import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.Maps; import java.util.ArrayList; import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -30,6 +30,9 @@ */ public class LifecyclePolicyUsageCalculator { + /** A 'not found' sentinel value for use in getOrDefault calls in order to avoid containsKey-and-then-get. */ + private static final String NOT_FOUND = ""; + private final ClusterState state; /** Whether {@link #calculateUsage} will be called multiple times or not. */ private final boolean willIterate; @@ -46,26 +49,29 @@ public LifecyclePolicyUsageCalculator( this.state = state; this.willIterate = names.size() > 1 || Regex.isSimpleMatchPattern(names.get(0)); - var allDataStreams = indexNameExpressionResolver.dataStreamNames( + final List allDataStreams = indexNameExpressionResolver.dataStreamNames( state, IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR ); // Sort all templates by descending priority. That way, findV2Template can exit on the first found template. - var indexTemplates = new ArrayList<>(state.metadata().templatesV2().entrySet()); + final var indexTemplates = new ArrayList<>(state.metadata().templatesV2().entrySet()); CollectionUtil.timSort(indexTemplates, Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder())); // Build the maps that will be used for the usage calculation later on. - IndexLifecycleMetadata metadata = state.metadata().custom(IndexLifecycleMetadata.TYPE); - policyToDataStream = new HashMap<>(Regex.isSimpleMatchPattern(names.get(0)) ? metadata.getPolicyMetadatas().size() : names.size()); - templateToPolicy = new HashMap<>(indexTemplates.size()); + final IndexLifecycleMetadata metadata = state.metadata().custom(IndexLifecycleMetadata.TYPE); + policyToDataStream = Maps.newHashMapWithExpectedSize( + Regex.isSimpleMatchPattern(names.get(0)) ? metadata.getPolicyMetadatas().size() : names.size() + ); + templateToPolicy = Maps.newHashMapWithExpectedSize(indexTemplates.size()); for (String dataStream : allDataStreams) { String indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), indexTemplates, dataStream, false, true); if (indexTemplate == null) { + // Every data stream should ordinarily have an index template, so this branch should not fire under normal circumstances. continue; } - Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate); - var policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings); - if (names.stream().noneMatch(name -> Regex.simpleMatch(name, policyName))) { + final Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate); + final var policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings); + if (doesPolicyMatchAnyName(policyName, names) == false) { // If a template's policy doesn't match any of the supplied names, we can skip it later on. templateToPolicy.put(indexTemplate, null); continue; @@ -79,18 +85,18 @@ public LifecyclePolicyUsageCalculator( * Calculate the indices, data streams, and composable templates that use the given policy. */ public ItemUsage calculateUsage(String policyName) { - List indices = new ArrayList<>(); + final List indices = new ArrayList<>(); for (IndexMetadata indexMetadata : state.metadata().indices().values()) { if (policyName.equals(indexMetadata.getLifecyclePolicyName())) { indices.add(indexMetadata.getIndex().getName()); } } - List composableTemplates = new ArrayList<>(); + final List composableTemplates = new ArrayList<>(); for (Map.Entry entry : state.metadata().templatesV2().entrySet()) { - var foundPolicy = templateToPolicy.get(entry.getKey()); - // Extra `containsKey` check to account for templates not using any policy. - if (foundPolicy == null && templateToPolicy.containsKey(entry.getKey()) == false) { + var foundPolicy = templateToPolicy.getOrDefault(entry.getKey(), NOT_FOUND); + // Intentionally use strict equals here as we're using the sentinel value. + if (foundPolicy == NOT_FOUND) { Settings settings = MetadataIndexTemplateService.resolveSettings(entry.getValue(), state.metadata().componentTemplates()); foundPolicy = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings); // If this method will only be called once, we don't need to keep building the map. @@ -105,4 +111,13 @@ public ItemUsage calculateUsage(String policyName) { return new ItemUsage(indices, policyToDataStream.getOrDefault(policyName, List.of()), composableTemplates); } + + private boolean doesPolicyMatchAnyName(String policyName, List names) { + for (var name : names) { + if (Regex.simpleMatch(name, policyName)) { + return true; + } + } + return false; + } } From fe9ae0ddb0075bb67935459352fbb3a5462b99da Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Sat, 15 Feb 2025 23:11:13 +1000 Subject: [PATCH 09/16] Refactor approach again to cache everything --- .../MetadataIndexTemplateService.java | 12 +- .../ilm/LifecyclePolicyUsageCalculator.java | 104 +++++++++--------- 2 files changed, 55 insertions(+), 61 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 2fd46e1b4b04e..409d2b7f1e5ce 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1061,9 +1061,8 @@ static Set dataStreamsExclusivelyUsingTemplates(final ClusterState state .map(templateName -> metadata.templatesV2().get(templateName)) .filter(Objects::nonNull) .map(ComposableIndexTemplate::indexPatterns) - .map(Set::copyOf) - .reduce(Sets::union) - .orElse(Set.of()); + .flatMap(List::stream) + .collect(Collectors.toSet()); // Determine all the composable templates that are not one of the provided templates. var otherTemplates = state.metadata() @@ -1383,11 +1382,8 @@ static List> findV2CandidateTemplates( * and we do want to return even match-all templates for those. Not doing so can result in a situation where a data stream is * built with a template that none of its indices match. */ - if (isHidden && template.getDataStreamTemplate() == null) { - final boolean hasMatchAllTemplate = anyMatch(template.indexPatterns(), Regex::isMatchAllPattern); - if (hasMatchAllTemplate) { - continue; - } + if (isHidden && template.getDataStreamTemplate() == null && anyMatch(template.indexPatterns(), Regex::isMatchAllPattern)) { + continue; } if (anyMatch(template.indexPatterns(), patternMatchPredicate)) { candidates.add(Tuple.tuple(name, template)); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java index 27de2d493f5a0..71c3fe453c440 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java @@ -25,30 +25,23 @@ import java.util.Map; /** - * A class that can be used to calculate the usages of ILM policies. This class computes some information on initialization, which will - * use a bit more memory but speeds up the usage calculation significantly. + * A class that can be used to calculate the usages of ILM policies across the cluster. The class makes a tradeoff by using some more memory + * (but not a lot) to significantly improve the processing time. */ public class LifecyclePolicyUsageCalculator { - /** A 'not found' sentinel value for use in getOrDefault calls in order to avoid containsKey-and-then-get. */ - private static final String NOT_FOUND = ""; - - private final ClusterState state; - /** Whether {@link #calculateUsage} will be called multiple times or not. */ - private final boolean willIterate; + /** A map from policy name to list of composable templates that use that policy. */ + private final Map> policyToTemplates; /** A map from policy name to list of data streams that use that policy. */ - private final Map> policyToDataStream; - /** A map from composable template name to the policy name it uses (or null) */ - private final Map templateToPolicy; + private final Map> policyToDataStreams; + /** A map from policy name to list of indices that use that policy. */ + private final Map> policyToIndices; public LifecyclePolicyUsageCalculator( final IndexNameExpressionResolver indexNameExpressionResolver, final ClusterState state, - List names + List requestedPolicyNames ) { - this.state = state; - this.willIterate = names.size() > 1 || Regex.isSimpleMatchPattern(names.get(0)); - final List allDataStreams = indexNameExpressionResolver.dataStreamNames( state, IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR @@ -57,59 +50,64 @@ public LifecyclePolicyUsageCalculator( final var indexTemplates = new ArrayList<>(state.metadata().templatesV2().entrySet()); CollectionUtil.timSort(indexTemplates, Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder())); - // Build the maps that will be used for the usage calculation later on. final IndexLifecycleMetadata metadata = state.metadata().custom(IndexLifecycleMetadata.TYPE); - policyToDataStream = Maps.newHashMapWithExpectedSize( - Regex.isSimpleMatchPattern(names.get(0)) ? metadata.getPolicyMetadatas().size() : names.size() - ); - templateToPolicy = Maps.newHashMapWithExpectedSize(indexTemplates.size()); + // We're making a bet here that if the `name` contains a wildcard, there's a large chance it'll simply match all policies. + final var expectedSize = Regex.isSimpleMatchPattern(requestedPolicyNames.get(0)) + ? metadata.getPolicyMetadatas().size() + : requestedPolicyNames.size(); + + // We keep a map from composable template name to policy name to avoid having to resolve the template settings to determine + // the template's policy twice. + final Map templateToPolicy = Maps.newHashMapWithExpectedSize(indexTemplates.size()); + + // Build the maps that will be used for the usage calculation later on. + policyToTemplates = Maps.newHashMapWithExpectedSize(expectedSize); + for (Map.Entry entry : state.metadata().templatesV2().entrySet()) { + Settings settings = MetadataIndexTemplateService.resolveSettings(entry.getValue(), state.metadata().componentTemplates()); + final var policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings); + // We only store the template if its policy matched any of the requested names. + if (doesPolicyMatchAnyName(policyName, requestedPolicyNames) == false) { + continue; + } + policyToTemplates.computeIfAbsent(policyName, k -> new ArrayList<>()).add(entry.getKey()); + templateToPolicy.put(entry.getKey(), policyName); + } + + policyToDataStreams = Maps.newHashMapWithExpectedSize(expectedSize); for (String dataStream : allDataStreams) { String indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), indexTemplates, dataStream, false, true); if (indexTemplate == null) { // Every data stream should ordinarily have an index template, so this branch should not fire under normal circumstances. continue; } - final Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate); - final var policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings); - if (doesPolicyMatchAnyName(policyName, names) == false) { - // If a template's policy doesn't match any of the supplied names, we can skip it later on. - templateToPolicy.put(indexTemplate, null); + final var policyName = templateToPolicy.get(indexTemplate); + // If there was no entry, either the template didn't specify an ILM policy or the policy didn't match any of the requested names + if (policyName == null) { continue; } - templateToPolicy.put(indexTemplate, policyName); - policyToDataStream.computeIfAbsent(policyName, k -> new ArrayList<>()).add(dataStream); + policyToDataStreams.computeIfAbsent(policyName, k -> new ArrayList<>()).add(dataStream); } - } - /** - * Calculate the indices, data streams, and composable templates that use the given policy. - */ - public ItemUsage calculateUsage(String policyName) { - final List indices = new ArrayList<>(); + policyToIndices = Maps.newHashMapWithExpectedSize(expectedSize); for (IndexMetadata indexMetadata : state.metadata().indices().values()) { - if (policyName.equals(indexMetadata.getLifecyclePolicyName())) { - indices.add(indexMetadata.getIndex().getName()); - } - } - - final List composableTemplates = new ArrayList<>(); - for (Map.Entry entry : state.metadata().templatesV2().entrySet()) { - var foundPolicy = templateToPolicy.getOrDefault(entry.getKey(), NOT_FOUND); - // Intentionally use strict equals here as we're using the sentinel value. - if (foundPolicy == NOT_FOUND) { - Settings settings = MetadataIndexTemplateService.resolveSettings(entry.getValue(), state.metadata().componentTemplates()); - foundPolicy = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings); - // If this method will only be called once, we don't need to keep building the map. - if (willIterate) { - templateToPolicy.put(entry.getKey(), foundPolicy); - } - } - if (policyName.equals(foundPolicy)) { - composableTemplates.add(entry.getKey()); + final var policyName = indexMetadata.getLifecyclePolicyName(); + // We only store the index if its policy matched any of the specified names. + if (doesPolicyMatchAnyName(policyName, requestedPolicyNames) == false) { + continue; } + policyToIndices.computeIfAbsent(policyName, k -> new ArrayList<>()).add(indexMetadata.getIndex().getName()); } + } - return new ItemUsage(indices, policyToDataStream.getOrDefault(policyName, List.of()), composableTemplates); + /** + * Retrieves the pre-calculated indices, data streams, and composable templates that use the given policy. + */ + public ItemUsage calculateUsage(String policyName) { + return new ItemUsage( + policyToIndices.getOrDefault(policyName, List.of()), + policyToDataStreams.getOrDefault(policyName, List.of()), + policyToTemplates.getOrDefault(policyName, List.of()) + ); } private boolean doesPolicyMatchAnyName(String policyName, List names) { From 3641d1a7bf4c97fa30fe9d8f14444ca1a16081d6 Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Tue, 18 Feb 2025 13:05:17 +0100 Subject: [PATCH 10/16] More feedback --- .../MetadataIndexTemplateService.java | 26 ++++++++++++++++--- .../ilm/LifecyclePolicyUsageCalculator.java | 7 ++++- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 409d2b7f1e5ce..5ba35c2058fa0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1082,7 +1082,7 @@ && isGlobalAndHasIndexHiddenSetting(metadata, entry.getValue(), entry.getKey()) .stream() // Limit to checking data streams that match any of the templates' index patterns .filter(ds -> namePatterns.stream().anyMatch(pattern -> Regex.simpleMatch(pattern, ds.getName()))) - .filter(ds -> findV2Template(state.metadata(), otherTemplates, ds.getName(), ds.isHidden(), true) == null) + .filter(ds -> findV2TemplateFromSortedList(state.metadata(), otherTemplates, ds.getName(), ds.isHidden()) == null) .map(DataStream::getName) .collect(Collectors.toSet()); } @@ -1318,12 +1318,26 @@ public static String findV2Template(Metadata metadata, String indexName, boolean return findV2Template(metadata, metadata.templatesV2().entrySet(), indexName, isHidden, false); } + /** + * Return the name (id) of the highest matching index template, out of the provided templates (that need to be sorted descending + * on priority beforehand), or the given index name. In the event that no templates are matched, {@code null} is returned. + */ + @Nullable + public static String findV2TemplateFromSortedList( + Metadata metadata, + Collection> templates, + String indexName, + boolean isHidden + ) { + return findV2Template(metadata, templates, indexName, isHidden, true); + } + /** * Return the name (id) of the highest matching index template, out of the provided templates, for the given index name. In * the event that no templates are matched, {@code null} is returned. */ @Nullable - public static String findV2Template( + private static String findV2Template( Metadata metadata, Collection> templates, String indexName, @@ -1363,9 +1377,13 @@ public static String findV2Template( /** * Return an ordered list of the name (id) and composable index templates that would apply to an index. The first * one is the winner template that is applied to this index. In the event that no templates are matched, - * an empty list is returned. If exitOnFirstMatch is true, we return immediately after finding a match. + * an empty list is returned. + *

+ * If exitOnFirstMatch is true, we return immediately after finding a match. That means that the templates + * parameter needs to be sorted based on priority (descending) for this method to return a sensible result -- otherwise this method + * would just return the first template that matches the name, in an unspecified order. */ - static List> findV2CandidateTemplates( + private static List> findV2CandidateTemplates( Collection> templates, String indexName, boolean isHidden, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java index 71c3fe453c440..2fcb27cb2b725 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java @@ -75,7 +75,12 @@ public LifecyclePolicyUsageCalculator( policyToDataStreams = Maps.newHashMapWithExpectedSize(expectedSize); for (String dataStream : allDataStreams) { - String indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), indexTemplates, dataStream, false, true); + String indexTemplate = MetadataIndexTemplateService.findV2TemplateFromSortedList( + state.metadata(), + indexTemplates, + dataStream, + false + ); if (indexTemplate == null) { // Every data stream should ordinarily have an index template, so this branch should not fire under normal circumstances. continue; From 5a101fe61b252a72df602677ed8ac225b4ddfb90 Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Tue, 18 Feb 2025 13:09:35 +0100 Subject: [PATCH 11/16] Rename method --- .../core/ilm/LifecyclePolicyUsageCalculator.java | 2 +- .../ilm/LifecyclePolicyUsageCalculatorTests.java | 12 ++++++------ .../ilm/action/TransportGetLifecycleAction.java | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java index 2fcb27cb2b725..706f7203d064a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java @@ -107,7 +107,7 @@ public LifecyclePolicyUsageCalculator( /** * Retrieves the pre-calculated indices, data streams, and composable templates that use the given policy. */ - public ItemUsage calculateUsage(String policyName) { + public ItemUsage retrieveCalculatedUsage(String policyName) { return new ItemUsage( policyToIndices.getOrDefault(policyName, List.of()), policyToDataStreams.getOrDefault(policyName, List.of()), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculatorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculatorTests.java index 5c7087e7b7816..67633b351ede1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculatorTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculatorTests.java @@ -42,7 +42,7 @@ public void testGetUsageNonExistentPolicy() { // Test where policy does not exist ClusterState state = ClusterState.builder(new ClusterName("mycluster")).build(); assertThat( - new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).calculateUsage("mypolicy"), + new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"), equalTo(new ItemUsage(List.of(), List.of(), List.of())) ); } @@ -63,7 +63,7 @@ public void testGetUsageUnusedPolicy() { ) .build(); assertThat( - new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).calculateUsage("mypolicy"), + new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"), equalTo(new ItemUsage(List.of(), List.of(), List.of())) ); } @@ -89,7 +89,7 @@ public void testGetUsagePolicyUsedByIndex() { ) .build(); assertThat( - new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).calculateUsage("mypolicy"), + new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"), equalTo(new ItemUsage(List.of("myindex"), List.of(), List.of())) ); } @@ -134,7 +134,7 @@ public void testGetUsagePolicyUsedByIndexAndTemplate() { ) .build(); assertThat( - new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).calculateUsage("mypolicy"), + new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"), equalTo(new ItemUsage(List.of("myindex"), List.of(), List.of("mytemplate"))) ); } @@ -182,7 +182,7 @@ public void testGetUsagePolicyUsedByIndexAndTemplateAndDataStream() { ClusterState state = ClusterState.builder(new ClusterName("mycluster")).metadata(mBuilder.build()).build(); assertThat( - new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).calculateUsage("mypolicy"), + new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"), equalTo(new ItemUsage(List.of("myindex", "another"), List.of("myds"), List.of("mytemplate"))) ); } @@ -228,7 +228,7 @@ public void testGetUsagePolicyNotUsedByDataStreamDueToOverride() { // Test where policy exists and is used by an index, datastream, and template ClusterState state = ClusterState.builder(new ClusterName("mycluster")).metadata(mBuilder.build()).build(); assertThat( - new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).calculateUsage("mypolicy"), + new LifecyclePolicyUsageCalculator(iner, state, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"), equalTo(new ItemUsage(List.of("myindex"), List.of(), List.of("mytemplate"))) ); } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java index 268b91aec82fd..c755dc08bd307 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java @@ -108,7 +108,7 @@ protected void masterOperation(Task task, Request request, ClusterState state, A policyMetadata.getPolicy(), policyMetadata.getVersion(), policyMetadata.getModifiedDateString(), - lifecyclePolicyUsageCalculator.calculateUsage(policyMetadata.getName()) + lifecyclePolicyUsageCalculator.retrieveCalculatedUsage(policyMetadata.getName()) ) ); } @@ -125,7 +125,7 @@ protected void masterOperation(Task task, Request request, ClusterState state, A policyMetadata.getPolicy(), policyMetadata.getVersion(), policyMetadata.getModifiedDateString(), - lifecyclePolicyUsageCalculator.calculateUsage(policyMetadata.getName()) + lifecyclePolicyUsageCalculator.retrieveCalculatedUsage(policyMetadata.getName()) ) ); } From d346665eef470e0f2948f7a789a06d89c42518ab Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Tue, 25 Mar 2025 11:12:07 +0100 Subject: [PATCH 12/16] Revert optimization in `dataStreamsExclusivelyUsingTemplates` --- .../MetadataIndexTemplateService.java | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 6188e7768f326..b052b6ada1de8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1091,24 +1091,31 @@ static Set dataStreamsExclusivelyUsingTemplates(final ProjectMetadata pr .flatMap(List::stream) .collect(Collectors.toSet()); - // Determine all the composable templates that are not one of the provided templates. - var otherTemplates = projectMetadata.templatesV2() - .entrySet() - .stream() - .filter( - entry -> templateNames.contains(entry.getKey()) == false - && isGlobalAndHasIndexHiddenSetting(projectMetadata, entry.getValue(), entry.getKey()) == false - ) - // Sort here so we can `exitOnFirstMatch` in `findV2Template`. - .sorted(Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder())) - .toList(); - return projectMetadata.dataStreams() .values() .stream() // Limit to checking data streams that match any of the templates' index patterns .filter(ds -> namePatterns.stream().anyMatch(pattern -> Regex.simpleMatch(pattern, ds.getName()))) - .filter(ds -> findV2TemplateFromSortedList(projectMetadata, otherTemplates, ds.getName(), ds.isHidden()) == null) + .filter(ds -> { + // Retrieve the templates that match the data stream name ordered by priority + List> candidates = findV2CandidateTemplates( + projectMetadata.templatesV2().entrySet(), + ds.getName(), + ds.isHidden(), + false + ); + if (candidates.isEmpty()) { + throw new IllegalStateException("Data stream " + ds.getName() + " did not match any composable index templates."); + } + + // Limit data streams that can ONLY use any of the specified templates, we do this by filtering + // the matching templates that are others than the ones requested and could be a valid template to use. + return candidates.stream() + .noneMatch( + template -> templateNames.contains(template.v1()) == false + && isGlobalAndHasIndexHiddenSetting(projectMetadata, template.v2(), template.v1()) == false + ); + }) .map(DataStream::getName) .collect(Collectors.toSet()); } From 5e7c0117e5f794659fd586051c7fe9761c2d77e0 Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Tue, 25 Mar 2025 14:45:25 +0100 Subject: [PATCH 13/16] Small additional fixes --- .../MetadataIndexTemplateService.java | 2 +- .../ilm/LifecyclePolicyUsageCalculator.java | 34 +++++++++++-------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index b052b6ada1de8..9293cabbfd9a6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1359,7 +1359,7 @@ public static String findV2Template(ProjectMetadata projectMetadata, String inde } /** - * Return the name (id) of the highest matching index template, out of the provided templates (that need to be sorted descending + * Return the name (id) of the highest matching index template out of the provided templates (that need to be sorted descending * on priority beforehand), or the given index name. In the event that no templates are matched, {@code null} is returned. */ @Nullable diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java index 1d42fe3c2710b..e8bdec60ade0b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java @@ -21,12 +21,13 @@ import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; /** - * A class that can be used to calculate the usages of ILM policies across the cluster. The class makes a tradeoff by using some more memory - * (but not a lot) to significantly improve the processing time. + * A class that can be used to calculate the usages of ILM policies across the cluster. By precomputing all the usages, + * the class makes a tradeoff by using a little bit more memory to significantly improve the overall processing time. */ public class LifecyclePolicyUsageCalculator { @@ -42,25 +43,17 @@ public LifecyclePolicyUsageCalculator( ProjectMetadata project, List requestedPolicyNames ) { - final List allDataStreams = indexNameExpressionResolver.dataStreamNames( - project, - IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR - ); - // Sort all templates by descending priority. That way, findV2Template can exit on the first found template. - final var indexTemplates = new ArrayList<>(project.templatesV2().entrySet()); - CollectionUtil.timSort(indexTemplates, Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder())); - - final IndexLifecycleMetadata metadata = project.custom(IndexLifecycleMetadata.TYPE); + final IndexLifecycleMetadata ilmMetadata = project.custom(IndexLifecycleMetadata.TYPE); // We're making a bet here that if the `name` contains a wildcard, there's a large chance it'll simply match all policies. final var expectedSize = Regex.isSimpleMatchPattern(requestedPolicyNames.get(0)) - ? metadata.getPolicyMetadatas().size() + ? ilmMetadata.getPolicyMetadatas().size() : requestedPolicyNames.size(); // We keep a map from composable template name to policy name to avoid having to resolve the template settings to determine // the template's policy twice. - final Map templateToPolicy = Maps.newHashMapWithExpectedSize(indexTemplates.size()); + final Map templateToPolicy = new HashMap<>(); - // Build the maps that will be used for the usage calculation later on. + // Build the map of which policy is used by which index templates. policyToTemplates = Maps.newHashMapWithExpectedSize(expectedSize); for (Map.Entry entry : project.templatesV2().entrySet()) { Settings settings = MetadataIndexTemplateService.resolveSettings(entry.getValue(), project.componentTemplates()); @@ -73,11 +66,21 @@ public LifecyclePolicyUsageCalculator( templateToPolicy.put(entry.getKey(), policyName); } + // Sort all templates by descending priority. That way, findV2Template can exit on the first-matched template. + final var indexTemplates = new ArrayList<>(project.templatesV2().entrySet()); + CollectionUtil.timSort(indexTemplates, Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder())); + + // Build the map of which policy is used by which data streams. policyToDataStreams = Maps.newHashMapWithExpectedSize(expectedSize); + final List allDataStreams = indexNameExpressionResolver.dataStreamNames( + project, + IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR + ); for (String dataStream : allDataStreams) { + // Find the index template with the highest priority that matches this data stream's name. String indexTemplate = MetadataIndexTemplateService.findV2TemplateFromSortedList(project, indexTemplates, dataStream, false); if (indexTemplate == null) { - // Every data stream should ordinarily have an index template, so this branch should not fire under normal circumstances. + assert false : "Data stream [" + dataStream + "] has no matching template"; continue; } final var policyName = templateToPolicy.get(indexTemplate); @@ -88,6 +91,7 @@ public LifecyclePolicyUsageCalculator( policyToDataStreams.computeIfAbsent(policyName, k -> new ArrayList<>()).add(dataStream); } + // Build the map of which policy is used by which indices. policyToIndices = Maps.newHashMapWithExpectedSize(expectedSize); for (IndexMetadata indexMetadata : project.indices().values()) { final var policyName = indexMetadata.getLifecyclePolicyName(); From 8b894d62a7305132b3480b3b3bae54097455621e Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Mon, 31 Mar 2025 13:57:56 +0100 Subject: [PATCH 14/16] Assert templates are sorted --- .../metadata/MetadataIndexTemplateService.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 9293cabbfd9a6..1daeac604e17d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1429,6 +1429,7 @@ private static List> findV2CandidateTempl boolean isHidden, boolean exitOnFirstMatch ) { + assert exitOnFirstMatch == false || areTemplatesSorted(templates) : "Expected templates to be sorted"; final String resolvedIndexName = IndexNameExpressionResolver.DateMathExpressionResolver.resolveExpression(indexName); final Predicate patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, resolvedIndexName); final List> candidates = new ArrayList<>(); @@ -1455,6 +1456,17 @@ private static List> findV2CandidateTempl return candidates; } + private static boolean areTemplatesSorted(Collection> templates) { + ComposableIndexTemplate previousTemplate = null; + for (Map.Entry template : templates) { + if (previousTemplate != null && template.getValue().priorityOrZero() > previousTemplate.priorityOrZero()) { + return false; + } + previousTemplate = template.getValue(); + } + return true; + } + // Checks if a global template specifies the `index.hidden` setting. This check is important because a global // template shouldn't specify the `index.hidden` setting, we leave it up to the caller to handle this situation. private static boolean isGlobalAndHasIndexHiddenSetting( From 27b84f32b613c96ca5e8899030e4dbc84ee5da18 Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Mon, 31 Mar 2025 14:22:49 +0100 Subject: [PATCH 15/16] Switch order of conditions in if-statement --- .../cluster/metadata/MetadataIndexTemplateService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 1daeac604e17d..cce92a6b20e13 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1441,7 +1441,7 @@ private static List> findV2CandidateTempl * and we do want to return even match-all templates for those. Not doing so can result in a situation where a data stream is * built with a template that none of its indices match. */ - if (isHidden && template.getDataStreamTemplate() == null && anyMatch(template.indexPatterns(), Regex::isMatchAllPattern)) { + if (anyMatch(template.indexPatterns(), Regex::isMatchAllPattern) && isHidden && template.getDataStreamTemplate() == null) { continue; } if (anyMatch(template.indexPatterns(), patternMatchPredicate)) { From 7c416c42a59eda84ac88391a229d37b1f9649dda Mon Sep 17 00:00:00 2001 From: Niels Bauman Date: Mon, 31 Mar 2025 15:04:16 +0100 Subject: [PATCH 16/16] Add comment --- .../metadata/MetadataIndexTemplateService.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index cce92a6b20e13..492d057c80af3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1418,10 +1418,13 @@ private static String findV2Template( * Return an ordered list of the name (id) and composable index templates that would apply to an index. The first * one is the winner template that is applied to this index. In the event that no templates are matched, * an empty list is returned. - *

- * If exitOnFirstMatch is true, we return immediately after finding a match. That means that the templates - * parameter needs to be sorted based on priority (descending) for this method to return a sensible result -- otherwise this method - * would just return the first template that matches the name, in an unspecified order. + * @param templates a list of template entries (name, template) - needs to be sorted when {@code exitOnFirstMatch} is {@code true} + * @param indexName the index (or data stream) name that should be used for matching the index patterns on the templates + * @param isHidden whether {@code indexName} belongs to a hidden index - this option is redundant for data streams, as backing indices + * of data streams will always be returned, regardless of whether the data stream is hidden or not + * @param exitOnFirstMatch if true, we return immediately after finding a match. That means that the templates + * parameter needs to be sorted based on priority (descending) for this method to return a sensible result, + * otherwise this method would just return the first template that matches the name, in an unspecified order */ private static List> findV2CandidateTemplates( Collection> templates,