Merge branch 'main' into reshard-search-filter

Author: bcully

docs/changelog/134835.yaml

@@ -0,0 +1,5 @@
+pr: 134835
+summary: Add new `/_security/stats` endpoint
+area: Authorization
+type: enhancement
+issues: []

docs/changelog/135051.yaml

@@ -0,0 +1,5 @@
+pr: 135051
+summary: Ban Limit + `MvExpand` before remote Enrich
+area: ES|QL
+type: bug
+issues: []

docs/internal/Versioning.md

@@ -175,19 +175,19 @@ a no-op.
 ## Index version
 
 Index version is a single incrementing version number for the index data format,
-metadata, and associated mappings. It is declared the same way as the
-transport version - with the pattern `M_NNN_S_PP`, for the major version, version id,
-subsidiary version id, and patch number respectively.
+metadata, and associated mappings. It is declared with the pattern `M_NNN_S_PP`,
+for the major version, version id, subsidiary version id, and patch number
+respectively.
 
 Index version is stored in index metadata when an index is created,
 and it is used to determine the storage format and what functionality that index supports.
 The index version does not change once an index is created.
 
-In the same way as transport versions, when a change is needed to the index
-data format or metadata, or new mapping types are added, create a new version constant
-below the last one, incrementing the `NNN` version component.
+When a change is needed to the index data format or metadata, or new mapping
+types are added, create a new version constant below the last one, incrementing
+the `NNN` version component.
 
-Unlike transport version, version constants cannot be collapsed together,
+Index version constants cannot be collapsed together,
 as an index keeps its creation version id once it is created.
 Fortunately, new index versions are only created once a month or so,
 so we don’t have a large list of index versions that need managing.

muted-tests.yml

@@ -531,9 +531,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.expression.function.scalar.score.DecayTests
   method: "testEvaluateBlockWithNulls {TestCase=<integer>, <integer>, <integer>, <_source> #2}"
   issue: https://github.com/elastic/elasticsearch/issues/134679
-- class: org.elasticsearch.search.vectors.CachingEnableFilterQueryTests
-  method: testTermQuery
-  issue: https://github.com/elastic/elasticsearch/issues/134723
 - class: org.elasticsearch.discovery.DiscoveryDisruptionIT
   method: testElectMasterWithLatestVersion
   issue: https://github.com/elastic/elasticsearch/issues/134748
@@ -597,9 +594,6 @@ tests:
 - class: org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceComputerTests
   method: testDesiredBalanceShouldConvergeInABigCluster
   issue: https://github.com/elastic/elasticsearch/issues/135123
-- class: org.elasticsearch.search.vectors.CachingEnableFilterQueryTests
-  method: testBooleanQuery
-  issue: https://github.com/elastic/elasticsearch/issues/135124
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Test stop transform with force and wait_for_checkpoint true}
   issue: https://github.com/elastic/elasticsearch/issues/135135
@@ -633,6 +627,12 @@ tests:
 - class: org.elasticsearch.upgrades.QueryableBuiltInRolesUpgradeIT
   method: testBuiltInRolesSyncedOnClusterUpgrade
   issue: https://github.com/elastic/elasticsearch/issues/135194
+- class: org.elasticsearch.compute.operator.topn.TopNOperatorTests
+  method: testSimpleWithCranky
+  issue: https://github.com/elastic/elasticsearch/issues/135224
+- class: org.elasticsearch.compute.aggregation.SampleDoubleAggregatorFunctionTests
+  method: testSimpleWithCranky
+  issue: https://github.com/elastic/elasticsearch/issues/135239
 
 # Examples:
 #

rest-api-spec/src/main/resources/rest-api-spec/api/security.get_stats.json

@@ -0,0 +1,29 @@
+{
+  "security.get_stats": {
+    "documentation": {
+      "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-security-get-stats",
+      "description": "Get security statistics for all nodes"
+    },
+    "stability": "stable",
+    "visibility": "public",
+    "headers": {
+      "accept": [
+        "application/json"
+      ],
+      "content_type": [
+        "application/json"
+      ]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_security/stats",
+          "methods": [
+            "GET"
+          ]
+        }
+      ]
+    },
+    "params": {}
+  }
+}

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -47,7 +47,6 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
-import org.elasticsearch.index.shard.IndexLongFieldRange;
 import org.elasticsearch.indices.IndexTemplateMissingException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidIndexTemplateException;
@@ -173,9 +172,12 @@ private abstract static class TemplateClusterStateUpdateTask implements ClusterS
         }
 
         public final ClusterState execute(ClusterState currentState) throws Exception {
-            ProjectMetadata metadata = currentState.metadata().getProject(projectId);
-            ProjectMetadata newMetadata = execute(metadata);
-            return metadata == newMetadata ? currentState : ClusterState.builder(currentState).putProjectMetadata(newMetadata).build();
+            ProjectMetadata currentProject = currentState.metadata().getProject(projectId);
+            ProjectMetadata newProject = execute(currentProject);
+            if (currentProject == newProject) {
+                return currentState;
+            }
+            return ClusterState.builder(currentState).metadata(currentState.metadata().withUpdatedProject(newProject)).build();
         }
 
         public abstract ProjectMetadata execute(ProjectMetadata currentProject) throws Exception;
@@ -384,19 +386,17 @@ public ProjectMetadata addComponentTemplate(
         }
 
         validateTemplate(finalSettings, wrappedMappings, indicesService);
-        validate(name, finalComponentTemplate);
+        validate(name, finalComponentTemplate.template(), List.of(), null);
 
+        ProjectMetadata projectWithComponentTemplateAdded = ProjectMetadata.builder(project).put(name, finalComponentTemplate).build();
         // Validate all composable index templates that use this component template
-        if (templatesUsingComponent.size() > 0) {
-            ProjectMetadata tempProjectWithComponentTemplateAdded = ProjectMetadata.builder(project)
-                .put(name, finalComponentTemplate)
-                .build();
+        if (templatesUsingComponent.isEmpty() == false) {
             Exception validationFailure = null;
             for (Map.Entry<String, ComposableIndexTemplate> entry : templatesUsingComponent.entrySet()) {
                 final String composableTemplateName = entry.getKey();
                 final ComposableIndexTemplate composableTemplate = entry.getValue();
                 try {
-                    validateIndexTemplateV2(tempProjectWithComponentTemplateAdded, composableTemplateName, composableTemplate);
+                    validateIndexTemplateV2(projectWithComponentTemplateAdded, composableTemplateName, composableTemplate);
                 } catch (Exception e) {
                     if (validationFailure == null) {
                         validationFailure = new IllegalArgumentException(
@@ -426,7 +426,7 @@ public ProjectMetadata addComponentTemplate(
         }
 
         logger.info("{} component template [{}]", existing == null ? "adding" : "updating", name);
-        return ProjectMetadata.builder(project).put(name, finalComponentTemplate).build();
+        return projectWithComponentTemplateAdded;
     }
 
     /**
@@ -784,8 +784,9 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo
         var finalTemplate = indexTemplate.template();
         final var now = instantSource.instant();
 
-        final var combinedMappings = collectMappings(indexTemplate, projectMetadata.componentTemplates(), "tmp_idx");
-        final var combinedSettings = resolveSettings(indexTemplate, projectMetadata.componentTemplates());
+        final var componentTemplates = projectMetadata.componentTemplates();
+        final var combinedMappings = collectMappings(indexTemplate, componentTemplates, "tmp_idx");
+        final var combinedSettings = resolveSettings(indexTemplate, componentTemplates);
         var additionalSettingsBuilder = Settings.builder();
         ImmutableOpenMap.Builder<String, Map<String, String>> customMetadataBuilder = ImmutableOpenMap.builder();
         for (var provider : indexSettingProviders) {
@@ -820,11 +821,11 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo
 
         validate(name, templateToValidate, additionalSettings);
         validateDataStreamsStillReferenced(projectMetadata, name, templateToValidate);
-        validateLifecycle(projectMetadata, name, templateToValidate, globalRetentionSettings.get(false));
-        validateDataStreamOptions(projectMetadata, name, templateToValidate, globalRetentionSettings.get(true));
+        validateLifecycle(componentTemplates, name, templateToValidate, globalRetentionSettings.get(false));
+        validateDataStreamOptions(componentTemplates, name, templateToValidate, globalRetentionSettings.get(true));
 
         if (templateToValidate.isDeprecated() == false) {
-            validateUseOfDeprecatedComponentTemplates(name, templateToValidate, projectMetadata.componentTemplates());
+            validateUseOfDeprecatedComponentTemplates(name, templateToValidate, componentTemplates);
             validateUseOfDeprecatedIngestPipelines(name, projectMetadata.custom(IngestMetadata.TYPE), combinedSettings);
             // TODO come up with a plan how to validate usage of deprecated ILM policies
             // we don't have access to the core/main plugin here so we can't use the IndexLifecycleMetadata type
@@ -900,12 +901,12 @@ private void emitWarningIfPipelineIsDeprecated(String name, Map<String, Pipeline
 
     // Visible for testing
     static void validateLifecycle(
-        ProjectMetadata project,
+        Map<String, ComponentTemplate> componentTemplates,
         String indexTemplateName,
         ComposableIndexTemplate template,
         @Nullable DataStreamGlobalRetention globalRetention
     ) {
-        DataStreamLifecycle.Builder builder = resolveLifecycle(template, project.componentTemplates());
+        DataStreamLifecycle.Builder builder = resolveLifecycle(template, componentTemplates);
         if (builder != null) {
             if (template.getDataStreamTemplate() == null) {
                 throw new IllegalArgumentException(
@@ -925,12 +926,12 @@ static void validateLifecycle(
 
     // Visible for testing
     static void validateDataStreamOptions(
-        ProjectMetadata projectMetadata,
+        Map<String, ComponentTemplate> componentTemplates,
         String indexTemplateName,
         ComposableIndexTemplate template,
         DataStreamGlobalRetention globalRetention
     ) {
-        DataStreamOptions.Builder dataStreamOptionsBuilder = resolveDataStreamOptions(template, projectMetadata.componentTemplates());
+        DataStreamOptions.Builder dataStreamOptionsBuilder = resolveDataStreamOptions(template, componentTemplates);
         if (dataStreamOptionsBuilder != null) {
             if (template.getDataStreamTemplate() == null) {
                 throw new IllegalArgumentException(
@@ -971,18 +972,18 @@ private static void validateDataStreamsStillReferenced(
             .map(Map.Entry::getKey)
             .collect(Collectors.toSet());
 
-        Function<ProjectMetadata, Set<String>> findUnreferencedDataStreams = meta -> {
+        Function<Map<String, ComposableIndexTemplate>, Set<String>> findUnreferencedDataStreams = composableTemplates -> {
             final Set<String> unreferenced = new HashSet<>();
             // For each data stream that we have, see whether it's covered by a different
             // template (which is great), or whether it's now uncovered by any template
             for (String dataStream : dataStreams) {
-                final String matchingTemplate = findV2Template(meta, dataStream, false);
+                final String matchingTemplate = findV2Template(project, composableTemplates.entrySet(), dataStream, false, false);
                 if (matchingTemplate == null) {
                     unreferenced.add(dataStream);
                 } else {
                     // We found a template that still matches, great! Buuuuttt... check whether it
                     // is a data stream template, as it's only useful if it has a data stream definition
-                    if (meta.templatesV2().get(matchingTemplate).getDataStreamTemplate() == null) {
+                    if (composableTemplates.get(matchingTemplate).getDataStreamTemplate() == null) {
                         unreferenced.add(dataStream);
                     }
                 }
@@ -991,12 +992,13 @@ private static void validateDataStreamsStillReferenced(
         };
 
         // Find data streams that are currently unreferenced
-        final Set<String> currentlyUnreferenced = findUnreferencedDataStreams.apply(project);
+        final Set<String> currentlyUnreferenced = findUnreferencedDataStreams.apply(project.templatesV2());
 
-        // Generate a metadata as if the new template were actually in the cluster state
-        final ProjectMetadata updatedMetadata = ProjectMetadata.builder(project).put(templateName, newTemplate).build();
+        // Generate a map as if the new template were actually in the cluster state
+        final var updatedTemplatesMap = new HashMap<>(project.templatesV2());
+        updatedTemplatesMap.put(templateName, newTemplate);
         // Find the data streams that would be unreferenced now that the template is updated/added
-        final Set<String> newlyUnreferenced = findUnreferencedDataStreams.apply(updatedMetadata);
+        final Set<String> newlyUnreferenced = findUnreferencedDataStreams.apply(updatedTemplatesMap);
 
         // If we found any data streams that used to be covered, but will no longer be covered by
         // changing this template, then blow up with as much helpful information as we can muster
@@ -1226,7 +1228,7 @@ static Set<String> dataStreamsExclusivelyUsingTemplates(final ProjectMetadata pr
                 return candidates.stream()
                     .noneMatch(
                         template -> templateNames.contains(template.v1()) == false
-                            && isGlobalAndHasIndexHiddenSetting(projectMetadata, template.v2(), template.v1()) == false
+                            && isGlobalAndHasIndexHiddenSetting(template.v2(), projectMetadata.componentTemplates()) == false
                     );
             })
             .map(DataStream::getName)
@@ -1514,7 +1516,7 @@ private static String findV2Template(
         // a restored index cluster state that modified a component template used by this global template such that it has this setting)
         // we will fail and the user will have to update the index template and remove this setting or update the corresponding component
         // template that contributes to the index template resolved settings
-        if (isGlobalAndHasIndexHiddenSetting(projectMetadata, winner, winnerName)) {
+        if (isGlobalAndHasIndexHiddenSetting(winner, projectMetadata.componentTemplates())) {
             throw new IllegalStateException(
                 "global index template ["
                     + winnerName
@@ -1586,12 +1588,11 @@ private static boolean areTemplatesSorted(Collection<Map.Entry<String, Composabl
     // Checks if a global template specifies the `index.hidden` setting. This check is important because a global
     // template shouldn't specify the `index.hidden` setting, we leave it up to the caller to handle this situation.
     private static boolean isGlobalAndHasIndexHiddenSetting(
-        ProjectMetadata projectMetadata,
         ComposableIndexTemplate template,
-        String templateName
+        Map<String, ComponentTemplate> componentTemplates
     ) {
         return anyMatch(template.indexPatterns(), Regex::isMatchAllPattern)
-            && IndexMetadata.INDEX_HIDDEN_SETTING.exists(resolveSettings(projectMetadata, templateName));
+            && IndexMetadata.INDEX_HIDDEN_SETTING.exists(resolveSettings(template, componentTemplates));
     }
 
     /**
@@ -1962,10 +1963,8 @@ private static void validateCompositeTemplate(
         final NamedXContentRegistry xContentRegistry,
         final SystemIndices systemIndices
     ) throws Exception {
-        final ProjectMetadata projectMetadataWithTemplate = ProjectMetadata.builder(project).put(templateName, template).build();
-
         final String temporaryIndexName = "validate-template-" + UUIDs.randomBase64UUID().toLowerCase(Locale.ROOT);
-        Settings resolvedSettings = resolveSettings(projectMetadataWithTemplate, templateName);
+        Settings resolvedSettings = resolveSettings(template, project.componentTemplates());
 
         // use the provided values, otherwise just pick valid dummy values
         int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(resolvedSettings);
@@ -1985,23 +1984,17 @@ private static void validateCompositeTemplate(
             .build();
 
         // Validate index metadata (settings)
-        final ProjectMetadata projectMetadataWithIndex = ProjectMetadata.builder(projectMetadataWithTemplate)
-            .put(
-                IndexMetadata.builder(temporaryIndexName)
-                    // necessary to pass asserts in ClusterState constructor
-                    .eventIngestedRange(IndexLongFieldRange.UNKNOWN)
-                    .settings(finalResolvedSettings)
-                    .putCustom(customMetadata)
-            )
+        final IndexMetadata tmpIndexMetadata = IndexMetadata.builder(temporaryIndexName)
+            .settings(finalResolvedSettings)
+            .putCustom(customMetadata)
             .build();
-        final IndexMetadata tmpIndexMetadata = projectMetadataWithIndex.index(temporaryIndexName);
         indicesService.withTempIndexService(tmpIndexMetadata, tempIndexService -> {
             // Validate aliases
             MetadataCreateIndexService.resolveAndValidateAliases(
                 temporaryIndexName,
                 Collections.emptySet(),
-                MetadataIndexTemplateService.resolveAliases(projectMetadataWithIndex, templateName),
-                projectMetadataWithIndex,
+                MetadataIndexTemplateService.resolveAliases(project, template),
+                project,
                 // the context is only used for validation so it's fine to pass fake values for the
                 // shard id and the current timestamp
                 xContentRegistry,
@@ -2014,7 +2007,7 @@ private static void validateCompositeTemplate(
             String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName;
             // Parse mappings to ensure they are valid after being composed
 
-            List<CompressedXContent> mappings = collectMappings(projectMetadataWithIndex, templateName, indexName);
+            List<CompressedXContent> mappings = collectMappings(template, project.componentTemplates(), indexName);
             try {
                 MapperService mapperService = tempIndexService.mapperService();
                 mapperService.merge(MapperService.SINGLE_MAPPING_NAME, mappings, MapperService.MergeReason.INDEX_TEMPLATE);
@@ -2061,10 +2054,6 @@ public static void validateTemplate(Settings validateSettings, CompressedXConten
         });
     }
 
-    public void validate(String name, ComponentTemplate template) {
-        validate(name, template.template(), Collections.emptyList(), null);
-    }
-
     private void validate(String name, ComposableIndexTemplate template, @Nullable Settings systemProvided) {
         validate(name, template.template(), template.indexPatterns(), systemProvided);
     }