diff --git a/docs/changelog/106953.yaml b/docs/changelog/106953.yaml new file mode 100644 index 0000000000000..6993b1960a401 --- /dev/null +++ b/docs/changelog/106953.yaml @@ -0,0 +1,6 @@ +pr: 106953 +summary: Optimize usage calculation in ILM policies retrieval API +area: ILM+SLM +type: enhancement +issues: + - 105773 diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 4e92f87f990c1..492d057c80af3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -62,6 +62,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -1087,9 +1088,8 @@ static Set dataStreamsExclusivelyUsingTemplates(final ProjectMetadata pr .map(templateName -> projectMetadata.templatesV2().get(templateName)) .filter(Objects::nonNull) .map(ComposableIndexTemplate::indexPatterns) - .map(Set::copyOf) - .reduce(Sets::union) - .orElse(Set.of()); + .flatMap(List::stream) + .collect(Collectors.toSet()); return projectMetadata.dataStreams() .values() @@ -1099,9 +1099,10 @@ static Set dataStreamsExclusivelyUsingTemplates(final ProjectMetadata pr .filter(ds -> { // Retrieve the templates that match the data stream name ordered by priority List> candidates = findV2CandidateTemplates( - projectMetadata, + projectMetadata.templatesV2().entrySet(), ds.getName(), - ds.isHidden() + ds.isHidden(), + false ); if (candidates.isEmpty()) { throw new IllegalStateException("Data stream " + ds.getName() + " did not match any composable index templates."); @@ -1110,13 +1111,10 @@ static Set dataStreamsExclusivelyUsingTemplates(final ProjectMetadata pr // Limit data streams that can ONLY use any of the specified templates, we do this by filtering // the matching templates that are others than the ones requested and could be a valid template to use. return candidates.stream() - .filter( + .noneMatch( template -> templateNames.contains(template.v1()) == false && isGlobalAndHasIndexHiddenSetting(projectMetadata, template.v2(), template.v1()) == false - ) - .map(Tuple::v1) - .toList() - .isEmpty(); + ); }) .map(DataStream::getName) .collect(Collectors.toSet()); @@ -1357,7 +1355,41 @@ public static List findV1Templates( */ @Nullable public static String findV2Template(ProjectMetadata projectMetadata, String indexName, boolean isHidden) { - final List> candidates = findV2CandidateTemplates(projectMetadata, indexName, isHidden); + return findV2Template(projectMetadata, projectMetadata.templatesV2().entrySet(), indexName, isHidden, false); + } + + /** + * Return the name (id) of the highest matching index template out of the provided templates (that need to be sorted descending + * on priority beforehand), or the given index name. In the event that no templates are matched, {@code null} is returned. + */ + @Nullable + public static String findV2TemplateFromSortedList( + ProjectMetadata projectMetadata, + Collection> templates, + String indexName, + boolean isHidden + ) { + return findV2Template(projectMetadata, templates, indexName, isHidden, true); + } + + /** + * Return the name (id) of the highest matching index template, out of the provided templates, for the given index name. In + * the event that no templates are matched, {@code null} is returned. + */ + @Nullable + private static String findV2Template( + ProjectMetadata projectMetadata, + Collection> templates, + String indexName, + boolean isHidden, + boolean exitOnFirstMatch + ) { + final List> candidates = findV2CandidateTemplates( + templates, + indexName, + isHidden, + exitOnFirstMatch + ); if (candidates.isEmpty()) { return null; } @@ -1386,16 +1418,25 @@ public static String findV2Template(ProjectMetadata projectMetadata, String inde * Return an ordered list of the name (id) and composable index templates that would apply to an index. The first * one is the winner template that is applied to this index. In the event that no templates are matched, * an empty list is returned. + * @param templates a list of template entries (name, template) - needs to be sorted when {@code exitOnFirstMatch} is {@code true} + * @param indexName the index (or data stream) name that should be used for matching the index patterns on the templates + * @param isHidden whether {@code indexName} belongs to a hidden index - this option is redundant for data streams, as backing indices + * of data streams will always be returned, regardless of whether the data stream is hidden or not + * @param exitOnFirstMatch if true, we return immediately after finding a match. That means that the templates + * parameter needs to be sorted based on priority (descending) for this method to return a sensible result, + * otherwise this method would just return the first template that matches the name, in an unspecified order */ - static List> findV2CandidateTemplates( - ProjectMetadata projectMetadata, + private static List> findV2CandidateTemplates( + Collection> templates, String indexName, - boolean isHidden + boolean isHidden, + boolean exitOnFirstMatch ) { + assert exitOnFirstMatch == false || areTemplatesSorted(templates) : "Expected templates to be sorted"; final String resolvedIndexName = IndexNameExpressionResolver.DateMathExpressionResolver.resolveExpression(indexName); final Predicate patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, resolvedIndexName); final List> candidates = new ArrayList<>(); - for (Map.Entry entry : projectMetadata.templatesV2().entrySet()) { + for (Map.Entry entry : templates) { final String name = entry.getKey(); final ComposableIndexTemplate template = entry.getValue(); /* @@ -1403,16 +1444,13 @@ static List> findV2CandidateTemplates( * and we do want to return even match-all templates for those. Not doing so can result in a situation where a data stream is * built with a template that none of its indices match. */ - if (isHidden == false || template.getDataStreamTemplate() != null) { - if (anyMatch(template.indexPatterns(), patternMatchPredicate)) { - candidates.add(Tuple.tuple(name, template)); - } - } else { - final boolean isNotMatchAllTemplate = noneMatch(template.indexPatterns(), Regex::isMatchAllPattern); - if (isNotMatchAllTemplate) { - if (anyMatch(template.indexPatterns(), patternMatchPredicate)) { - candidates.add(Tuple.tuple(name, template)); - } + if (anyMatch(template.indexPatterns(), Regex::isMatchAllPattern) && isHidden && template.getDataStreamTemplate() == null) { + continue; + } + if (anyMatch(template.indexPatterns(), patternMatchPredicate)) { + candidates.add(Tuple.tuple(name, template)); + if (exitOnFirstMatch) { + return candidates; } } } @@ -1421,6 +1459,17 @@ static List> findV2CandidateTemplates( return candidates; } + private static boolean areTemplatesSorted(Collection> templates) { + ComposableIndexTemplate previousTemplate = null; + for (Map.Entry template : templates) { + if (previousTemplate != null && template.getValue().priorityOrZero() > previousTemplate.priorityOrZero()) { + return false; + } + previousTemplate = template.getValue(); + } + return true; + } + // Checks if a global template specifies the `index.hidden` setting. This check is important because a global // template shouldn't specify the `index.hidden` setting, we leave it up to the caller to handle this situation. private static boolean isGlobalAndHasIndexHiddenSetting( diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java new file mode 100644 index 0000000000000..e8bdec60ade0b --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculator.java @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ilm; + +import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.ItemUsage; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; +import org.elasticsearch.cluster.metadata.ProjectMetadata; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.Maps; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * A class that can be used to calculate the usages of ILM policies across the cluster. By precomputing all the usages, + * the class makes a tradeoff by using a little bit more memory to significantly improve the overall processing time. + */ +public class LifecyclePolicyUsageCalculator { + + /** A map from policy name to list of composable templates that use that policy. */ + private final Map> policyToTemplates; + /** A map from policy name to list of data streams that use that policy. */ + private final Map> policyToDataStreams; + /** A map from policy name to list of indices that use that policy. */ + private final Map> policyToIndices; + + public LifecyclePolicyUsageCalculator( + final IndexNameExpressionResolver indexNameExpressionResolver, + ProjectMetadata project, + List requestedPolicyNames + ) { + final IndexLifecycleMetadata ilmMetadata = project.custom(IndexLifecycleMetadata.TYPE); + // We're making a bet here that if the `name` contains a wildcard, there's a large chance it'll simply match all policies. + final var expectedSize = Regex.isSimpleMatchPattern(requestedPolicyNames.get(0)) + ? ilmMetadata.getPolicyMetadatas().size() + : requestedPolicyNames.size(); + + // We keep a map from composable template name to policy name to avoid having to resolve the template settings to determine + // the template's policy twice. + final Map templateToPolicy = new HashMap<>(); + + // Build the map of which policy is used by which index templates. + policyToTemplates = Maps.newHashMapWithExpectedSize(expectedSize); + for (Map.Entry entry : project.templatesV2().entrySet()) { + Settings settings = MetadataIndexTemplateService.resolveSettings(entry.getValue(), project.componentTemplates()); + final var policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings); + // We only store the template if its policy matched any of the requested names. + if (doesPolicyMatchAnyName(policyName, requestedPolicyNames) == false) { + continue; + } + policyToTemplates.computeIfAbsent(policyName, k -> new ArrayList<>()).add(entry.getKey()); + templateToPolicy.put(entry.getKey(), policyName); + } + + // Sort all templates by descending priority. That way, findV2Template can exit on the first-matched template. + final var indexTemplates = new ArrayList<>(project.templatesV2().entrySet()); + CollectionUtil.timSort(indexTemplates, Comparator.comparing(entry -> entry.getValue().priorityOrZero(), Comparator.reverseOrder())); + + // Build the map of which policy is used by which data streams. + policyToDataStreams = Maps.newHashMapWithExpectedSize(expectedSize); + final List allDataStreams = indexNameExpressionResolver.dataStreamNames( + project, + IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR + ); + for (String dataStream : allDataStreams) { + // Find the index template with the highest priority that matches this data stream's name. + String indexTemplate = MetadataIndexTemplateService.findV2TemplateFromSortedList(project, indexTemplates, dataStream, false); + if (indexTemplate == null) { + assert false : "Data stream [" + dataStream + "] has no matching template"; + continue; + } + final var policyName = templateToPolicy.get(indexTemplate); + // If there was no entry, either the template didn't specify an ILM policy or the policy didn't match any of the requested names + if (policyName == null) { + continue; + } + policyToDataStreams.computeIfAbsent(policyName, k -> new ArrayList<>()).add(dataStream); + } + + // Build the map of which policy is used by which indices. + policyToIndices = Maps.newHashMapWithExpectedSize(expectedSize); + for (IndexMetadata indexMetadata : project.indices().values()) { + final var policyName = indexMetadata.getLifecyclePolicyName(); + // We only store the index if its policy matched any of the specified names. + if (doesPolicyMatchAnyName(policyName, requestedPolicyNames) == false) { + continue; + } + policyToIndices.computeIfAbsent(policyName, k -> new ArrayList<>()).add(indexMetadata.getIndex().getName()); + } + } + + /** + * Retrieves the pre-calculated indices, data streams, and composable templates that use the given policy. + */ + public ItemUsage retrieveCalculatedUsage(String policyName) { + return new ItemUsage( + policyToIndices.getOrDefault(policyName, List.of()), + policyToDataStreams.getOrDefault(policyName, List.of()), + policyToTemplates.getOrDefault(policyName, List.of()) + ); + } + + private boolean doesPolicyMatchAnyName(String policyName, List names) { + for (var name : names) { + if (Regex.simpleMatch(name, policyName)) { + return true; + } + } + return false; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java index 73a9bce0b1e93..13dda4b4aa135 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java @@ -8,14 +8,8 @@ package org.elasticsearch.xpack.core.ilm; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.ItemUsage; -import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; -import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.NotXContentException; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParser; @@ -24,7 +18,6 @@ import org.elasticsearch.xpack.core.template.resources.TemplateResources; import java.io.IOException; -import java.util.List; import java.util.Map; /** @@ -104,43 +97,4 @@ private static void validate(String source) { throw new ElasticsearchParseException("invalid policy", e); } } - - /** - * Given a cluster state and ILM policy, calculate the {@link ItemUsage} of - * the policy (what indices, data streams, and templates use the policy) - */ - public static ItemUsage calculateUsage( - final IndexNameExpressionResolver indexNameExpressionResolver, - final ProjectMetadata project, - final String policyName - ) { - final List indices = project.indices() - .values() - .stream() - .filter(indexMetadata -> policyName.equals(indexMetadata.getLifecyclePolicyName())) - .map(indexMetadata -> indexMetadata.getIndex().getName()) - .toList(); - - final List allDataStreams = indexNameExpressionResolver.dataStreamNames( - project, - IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR - ); - - final List dataStreams = allDataStreams.stream().filter(dsName -> { - String indexTemplate = MetadataIndexTemplateService.findV2Template(project, dsName, false); - if (indexTemplate != null) { - Settings settings = MetadataIndexTemplateService.resolveSettings(project, indexTemplate); - return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings)); - } else { - return false; - } - }).toList(); - - final List composableTemplates = project.templatesV2().keySet().stream().filter(templateName -> { - Settings settings = MetadataIndexTemplateService.resolveSettings(project, templateName); - return policyName.equals(LifecycleSettings.LIFECYCLE_NAME_SETTING.get(settings)); - }).toList(); - - return new ItemUsage(indices, dataStreams, composableTemplates); - } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculatorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculatorTests.java new file mode 100644 index 0000000000000..8afd9b8be6ad4 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUsageCalculatorTests.java @@ -0,0 +1,221 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ilm; + +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.ItemUsage; +import org.elasticsearch.cluster.metadata.ProjectMetadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.cluster.project.TestProjectResolvers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.EmptySystemIndices; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class LifecyclePolicyUsageCalculatorTests extends ESTestCase { + + private IndexNameExpressionResolver iner; + + @Before + public void init() { + iner = new IndexNameExpressionResolver( + new ThreadContext(Settings.EMPTY), + EmptySystemIndices.INSTANCE, + TestProjectResolvers.alwaysThrow() + ); + } + + public void testGetUsageNonExistentPolicy() { + // Test where policy does not exist + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + assertThat( + new LifecyclePolicyUsageCalculator(iner, project, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"), + equalTo(new ItemUsage(List.of(), List.of(), List.of())) + ); + } + + public void testGetUsageUnusedPolicy() { + // Test where policy is not used by anything + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .putCustom( + IndexLifecycleMetadata.TYPE, + new IndexLifecycleMetadata( + Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), + OperationMode.RUNNING + ) + ) + .build(); + + assertThat( + new LifecyclePolicyUsageCalculator(iner, project, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"), + equalTo(new ItemUsage(List.of(), List.of(), List.of())) + ); + } + + public void testGetUsagePolicyUsedByIndex() { + // Test where policy exists and is used by an index + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .putCustom( + IndexLifecycleMetadata.TYPE, + new IndexLifecycleMetadata( + Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), + OperationMode.RUNNING + ) + ) + .put( + IndexMetadata.builder("myindex") + .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) + ) + .build(); + + assertThat( + new LifecyclePolicyUsageCalculator(iner, project, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"), + equalTo(new ItemUsage(List.of("myindex"), List.of(), List.of())) + ); + } + + public void testGetUsagePolicyUsedByIndexAndTemplate() { + // Test where policy exists and is used by an index, and template + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .putCustom( + IndexLifecycleMetadata.TYPE, + new IndexLifecycleMetadata( + Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), + OperationMode.RUNNING + ) + ) + .put( + IndexMetadata.builder("myindex") + .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) + ) + .putCustom( + ComposableIndexTemplateMetadata.TYPE, + new ComposableIndexTemplateMetadata( + Map.of( + "mytemplate", + ComposableIndexTemplate.builder() + .indexPatterns(List.of("myds")) + .template( + new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null) + ) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ) + ) + ) + .build(); + + assertThat( + new LifecyclePolicyUsageCalculator(iner, project, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"), + equalTo(new ItemUsage(List.of("myindex"), List.of(), List.of("mytemplate"))) + ); + } + + public void testGetUsagePolicyUsedByIndexAndTemplateAndDataStream() { + // Test where policy exists and is used by an index, data stream, and template + IndexMetadata index = IndexMetadata.builder("myindex") + .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) + .build(); + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .putCustom( + IndexLifecycleMetadata.TYPE, + new IndexLifecycleMetadata( + Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), + OperationMode.RUNNING + ) + ) + .put(index, false) + .put(DataStreamTestHelper.newInstance("myds", List.of(index.getIndex()))) + .put( + IndexMetadata.builder("another") + .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) + ) + .put( + IndexMetadata.builder("other") + .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "otherpolicy")) + ) + + .putCustom( + ComposableIndexTemplateMetadata.TYPE, + new ComposableIndexTemplateMetadata( + Map.of( + "mytemplate", + ComposableIndexTemplate.builder() + .indexPatterns(List.of("myds")) + .template( + new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null) + ) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ) + ) + ) + .build(); + + assertThat( + new LifecyclePolicyUsageCalculator(iner, project, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"), + equalTo(new ItemUsage(List.of("myindex", "another"), List.of("myds"), List.of("mytemplate"))) + ); + } + + public void testGetUsagePolicyNotUsedByDataStreamDueToOverride() { + // Test when a data stream does not use the policy anymore because of a higher template + IndexMetadata index = IndexMetadata.builder("myindex") + .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) + .build(); + final var project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put(index, false) + .put(DataStreamTestHelper.newInstance("myds", List.of(index.getIndex()))) + .putCustom( + IndexLifecycleMetadata.TYPE, + new IndexLifecycleMetadata( + Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), + OperationMode.RUNNING + ) + ) + .putCustom( + ComposableIndexTemplateMetadata.TYPE, + new ComposableIndexTemplateMetadata( + Map.of( + "mytemplate", + ComposableIndexTemplate.builder() + .indexPatterns(List.of("myds*")) + .template( + new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null) + ) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build(), + "myhighertemplate", + ComposableIndexTemplate.builder() + .indexPatterns(List.of("myds")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .priority(1_000L) + .build() + ) + ) + ) + .build(); + + // Test where policy exists and is used by an index, datastream, and template + assertThat( + new LifecyclePolicyUsageCalculator(iner, project, List.of("mypolicy")).retrieveCalculatedUsage("mypolicy"), + equalTo(new ItemUsage(List.of("myindex"), List.of(), List.of("mytemplate"))) + ); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java deleted file mode 100644 index 2dcb8d6c2a10e..0000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtilsTests.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core.ilm; - -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata; -import org.elasticsearch.cluster.metadata.DataStreamTestHelper; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.ItemUsage; -import org.elasticsearch.cluster.metadata.ProjectMetadata; -import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.indices.TestIndexNameExpressionResolver; -import org.elasticsearch.test.ESTestCase; - -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; - -public class LifecyclePolicyUtilsTests extends ESTestCase { - public void testCalculateUsage() { - final IndexNameExpressionResolver iner = TestIndexNameExpressionResolver.newInstance(); - - { - // Test where policy does not exist - var project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); - assertThat( - LifecyclePolicyUtils.calculateUsage(iner, project, "mypolicy"), - equalTo(new ItemUsage(List.of(), List.of(), List.of())) - ); - } - - { - // Test where policy is not used by anything - var project = ProjectMetadata.builder(randomProjectIdOrDefault()) - .putCustom( - IndexLifecycleMetadata.TYPE, - new IndexLifecycleMetadata( - Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), - OperationMode.RUNNING - ) - ) - .build(); - assertThat( - LifecyclePolicyUtils.calculateUsage(iner, project, "mypolicy"), - equalTo(new ItemUsage(List.of(), List.of(), List.of())) - ); - } - - { - // Test where policy exists and is used by an index - var project = ProjectMetadata.builder(randomProjectIdOrDefault()) - .putCustom( - IndexLifecycleMetadata.TYPE, - new IndexLifecycleMetadata( - Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), - OperationMode.RUNNING - ) - ) - .put( - IndexMetadata.builder("myindex") - .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) - ) - .build(); - assertThat( - LifecyclePolicyUtils.calculateUsage(iner, project, "mypolicy"), - equalTo(new ItemUsage(List.of("myindex"), List.of(), List.of())) - ); - } - - { - // Test where policy exists and is used by an index, and template - var project = ProjectMetadata.builder(randomProjectIdOrDefault()) - .putCustom( - IndexLifecycleMetadata.TYPE, - new IndexLifecycleMetadata( - Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), - OperationMode.RUNNING - ) - ) - .put( - IndexMetadata.builder("myindex") - .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) - ) - .putCustom( - ComposableIndexTemplateMetadata.TYPE, - new ComposableIndexTemplateMetadata( - Map.of( - "mytemplate", - ComposableIndexTemplate.builder() - .indexPatterns(List.of("myds")) - .template( - new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null) - ) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) - .build() - ) - ) - ) - .build(); - assertThat( - LifecyclePolicyUtils.calculateUsage(iner, project, "mypolicy"), - equalTo(new ItemUsage(List.of("myindex"), List.of(), List.of("mytemplate"))) - ); - } - - { - var projectBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()) - .putCustom( - IndexLifecycleMetadata.TYPE, - new IndexLifecycleMetadata( - Map.of("mypolicy", LifecyclePolicyMetadataTests.createRandomPolicyMetadata("mypolicy")), - OperationMode.RUNNING - ) - ) - .put( - IndexMetadata.builder("myindex") - .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) - ) - .put( - IndexMetadata.builder("another") - .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy")) - ) - .put( - IndexMetadata.builder("other") - .settings(indexSettings(IndexVersion.current(), 1, 0).put(LifecycleSettings.LIFECYCLE_NAME, "otherpolicy")) - ) - - .putCustom( - ComposableIndexTemplateMetadata.TYPE, - new ComposableIndexTemplateMetadata( - Map.of( - "mytemplate", - ComposableIndexTemplate.builder() - .indexPatterns(List.of("myds")) - .template( - new Template(Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, "mypolicy").build(), null, null) - ) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) - .build() - ) - ) - ); - // Need to get the real Index instance of myindex: - projectBuilder.put(DataStreamTestHelper.newInstance("myds", List.of(projectBuilder.get("myindex").getIndex()))); - - // Test where policy exists and is used by an index, datastream, and template - assertThat( - LifecyclePolicyUtils.calculateUsage(iner, projectBuilder.build(), "mypolicy"), - equalTo(new ItemUsage(List.of("myindex", "another"), List.of("myds"), List.of("mytemplate"))) - ); - } - } -} diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java index 63ea9f35a4a47..7ec6aa5695415 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetLifecycleAction.java @@ -25,7 +25,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata; -import org.elasticsearch.xpack.core.ilm.LifecyclePolicyUtils; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicyUsageCalculator; import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction; import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction.LifecyclePolicyResponseItem; import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction.Request; @@ -97,6 +97,7 @@ protected void masterOperation(Task task, Request request, ClusterState state, A ); } + var lifecyclePolicyUsageCalculator = new LifecyclePolicyUsageCalculator(indexNameExpressionResolver, project, names); Map policyResponseItemMap = new LinkedHashMap<>(); for (String name : names) { if (Regex.isSimpleMatchPattern(name)) { @@ -112,7 +113,7 @@ protected void masterOperation(Task task, Request request, ClusterState state, A policyMetadata.getPolicy(), policyMetadata.getVersion(), policyMetadata.getModifiedDateString(), - LifecyclePolicyUtils.calculateUsage(indexNameExpressionResolver, project, policyMetadata.getName()) + lifecyclePolicyUsageCalculator.retrieveCalculatedUsage(policyMetadata.getName()) ) ); } @@ -129,7 +130,7 @@ protected void masterOperation(Task task, Request request, ClusterState state, A policyMetadata.getPolicy(), policyMetadata.getVersion(), policyMetadata.getModifiedDateString(), - LifecyclePolicyUtils.calculateUsage(indexNameExpressionResolver, project, policyMetadata.getName()) + lifecyclePolicyUsageCalculator.retrieveCalculatedUsage(policyMetadata.getName()) ) ); }