Skip data stream reference check in some cases (#135457)

During component and composable template validation we check if any data
streams become unreferenced if the composite template would be added to
the project metadata. We currently do this regardless of what changed in
the component or composable template. This check is quite expensive if
the cluster has many index templates or many data streams, mostly
because the regex matching on index patterns in `findV2Template` is
expensive.

We know that only a few properties of the composite template can affect
data stream references. Specifically, we know that the index patterns,
the `hidden` index setting, the priority, and the presence of the
`data_stream` field in the template are used to determine the template
that a data stream is based on.

By checking if only one of those properties changed, we can avoid
running the expensive checks in a great deal of template updates. For
example, mapping updates, `_meta` updates, any index settings other than
`hidden` are all irrelevant to this check, and we can thus skip it for
those updates. Especially component template updates will benefit from
this change, as they aren't even able to influence the index patterns,
data stream template, or priority of the composite template.

The main downside of this change is that we have an implicit dependency
from this new check on the implementation of `findV2Template`. We try
to mitigate that risk by doing the expensive/original check in an
assertion.

Author: nielsbauman

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.Assertions;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
@@ -818,7 +819,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo
         var templateToValidate = indexTemplate.toBuilder().template(Template.builder(finalTemplate).settings(finalSettings)).build();
 
         validate(name, templateToValidate, additionalSettings);
-        validateDataStreamsStillReferenced(projectMetadata, name, templateToValidate);
+        maybeValidateDataStreamsStillReferenced(projectMetadata, name, templateToValidate);
         validateLifecycle(componentTemplates, name, templateToValidate, globalRetentionSettings.get(false));
         validateDataStreamOptions(componentTemplates, name, templateToValidate, globalRetentionSettings.get(true));
 
@@ -944,6 +945,43 @@ static void validateDataStreamOptions(
         }
     }
 
+    /**
+     * Maybe runs {@link #validateDataStreamsStillReferenced} if it looks like the new composite template could change data stream coverage.
+     */
+    private static void maybeValidateDataStreamsStillReferenced(
+        ProjectMetadata project,
+        String templateName,
+        ComposableIndexTemplate newTemplate
+    ) {
+        final ComposableIndexTemplate existingTemplate = project.templatesV2().get(templateName);
+        final Settings existingSettings = Optional.ofNullable(existingTemplate)
+            .map(ComposableIndexTemplate::template)
+            .map(Template::settings)
+            .orElse(Settings.EMPTY);
+        final Settings newSettings = Optional.ofNullable(newTemplate)
+            .map(ComposableIndexTemplate::template)
+            .map(Template::settings)
+            .orElse(Settings.EMPTY);
+        // We check whether anything relevant has changed that could affect data stream coverage and return early if not.
+        // These checks are based on the implementation of findV2Template and the data stream template check in this method.
+        // If we're adding a new template, we do the full check in case this template's priority changes coverage.
+        if (existingTemplate != null
+            && Objects.equals(existingTemplate.indexPatterns(), newTemplate.indexPatterns())
+            && Objects.equals(existingSettings.get(IndexMetadata.SETTING_INDEX_HIDDEN), newSettings.get(IndexMetadata.SETTING_INDEX_HIDDEN))
+            && Objects.equals(existingTemplate.getDataStreamTemplate() != null, newTemplate.getDataStreamTemplate() != null)
+            && Objects.equals(existingTemplate.priorityOrZero(), newTemplate.priorityOrZero())) {
+            if (Assertions.ENABLED) {
+                try {
+                    validateDataStreamsStillReferenced(project, templateName, newTemplate);
+                } catch (IllegalArgumentException e) {
+                    assert false : "Data stream reference validation took a shortcut but the full check failed: " + e.getMessage();
+                }
+            }
+            return;
+        }
+        validateDataStreamsStillReferenced(project, templateName, newTemplate);
+    }
+
     /**
      * Validate that by changing or adding {@code newTemplate}, there are
      * no unreferenced data streams. Note that this scenario is still possible
@@ -955,18 +993,16 @@ private static void validateDataStreamsStillReferenced(
         String templateName,
         ComposableIndexTemplate newTemplate
     ) {
-        final Set<String> dataStreams = project.dataStreams()
-            .entrySet()
-            .stream()
-            .filter(entry -> entry.getValue().isSystem() == false)
-            .map(Map.Entry::getKey)
-            .collect(Collectors.toSet());
-
         Function<Map<String, ComposableIndexTemplate>, Set<String>> findUnreferencedDataStreams = composableTemplates -> {
             final Set<String> unreferenced = new HashSet<>();
             // For each data stream that we have, see whether it's covered by a different
             // template (which is great), or whether it's now uncovered by any template
-            for (String dataStream : dataStreams) {
+            for (var dataStreamEntry : project.dataStreams().entrySet()) {
+                // Exclude system data streams
+                if (dataStreamEntry.getValue().isSystem()) {
+                    continue;
+                }
+                final String dataStream = dataStreamEntry.getKey();
                 final String matchingTemplate = findV2Template(project, composableTemplates.entrySet(), dataStream, false, false);
                 if (matchingTemplate == null) {
                     unreferenced.add(dataStream);