Merge branch 'main' into fix-semantic-query-rewrite-boost-issue

Author: elasticmachine

muted-tests.yml

@@ -64,8 +64,6 @@ tests:
 - class: org.elasticsearch.xpack.shutdown.NodeShutdownIT
   method: testAllocationPreventedForRemoval
   issue: https://github.com/elastic/elasticsearch/issues/116363
-- class: org.elasticsearch.xpack.security.authc.ldap.ActiveDirectoryGroupsResolverTests
-  issue: https://github.com/elastic/elasticsearch/issues/116182
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=snapshot/20_operator_privileges_disabled/Operator only settings can be set and restored by non-operator user when operator privileges is disabled}
   issue: https://github.com/elastic/elasticsearch/issues/116775
@@ -114,8 +112,6 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.ForecastIT
   method: testOverflowToDisk
   issue: https://github.com/elastic/elasticsearch/issues/117740
-- class: org.elasticsearch.xpack.security.authc.ldap.MultiGroupMappingIT
-  issue: https://github.com/elastic/elasticsearch/issues/119599
 - class: org.elasticsearch.multi_cluster.MultiClusterYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/119983
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
@@ -138,8 +134,6 @@ tests:
 - class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
   method: testMultipleInferencesTriggeringDownloadAndDeploy
   issue: https://github.com/elastic/elasticsearch/issues/120668
-- class: org.elasticsearch.xpack.security.authc.ldap.ADLdapUserSearchSessionFactoryTests
-  issue: https://github.com/elastic/elasticsearch/issues/119882
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/3rd_party_deployment/Test start deployment fails while model download in progress}
   issue: https://github.com/elastic/elasticsearch/issues/120810
@@ -166,8 +160,6 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/120950
 - class: org.elasticsearch.xpack.ml.integration.PyTorchModelIT
   issue: https://github.com/elastic/elasticsearch/issues/121165
-- class: org.elasticsearch.xpack.security.authc.ldap.ActiveDirectorySessionFactoryTests
-  issue: https://github.com/elastic/elasticsearch/issues/121285
 - class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/121407
 - class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
@@ -446,9 +438,6 @@ tests:
 - class: org.elasticsearch.xpack.profiling.action.GetStatusActionIT
   method: testWaitsUntilResourcesAreCreated
   issue: https://github.com/elastic/elasticsearch/issues/129486
-- class: org.elasticsearch.xpack.security.PermissionsIT
-  method: testWhenUserLimitedByOnlyAliasOfIndexCanWriteToIndexWhichWasRolledoverByILMPolicy
-  issue: https://github.com/elastic/elasticsearch/issues/129481
 - class: org.elasticsearch.upgrades.MlJobSnapshotUpgradeIT
   method: testSnapshotUpgrader
   issue: https://github.com/elastic/elasticsearch/issues/98560
@@ -549,6 +538,9 @@ tests:
 - class: org.elasticsearch.xpack.esql.ccq.MultiClustersIT
   method: testLookupJoinAliases
   issue: https://github.com/elastic/elasticsearch/issues/131166
+- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
+  method: test {p0=field_caps/40_time_series/Get simple time series field caps}
+  issue: https://github.com/elastic/elasticsearch/issues/131225
 
 # Examples:
 #

server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
@@ -85,7 +86,8 @@ public void testMappingValidationIndexExists() {
         assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(0L));
         searchResponse.decRef();
         ClusterStateResponse clusterStateResponse = admin().cluster().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet();
-        Map<String, Object> indexMapping = clusterStateResponse.getState().metadata().getProject().index(indexName).mapping().sourceAsMap();
+        final var project = clusterStateResponse.getState().metadata().getProject(ProjectId.DEFAULT);
+        Map<String, Object> indexMapping = project.index(indexName).mapping().sourceAsMap();
         Map<String, Object> fields = (Map<String, Object>) indexMapping.get("properties");
         assertThat(fields.size(), equalTo(1));
     }
@@ -142,7 +144,8 @@ public void testMappingValidationIndexExistsTemplateSubstitutions() throws IOExc
         assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(0L));
         searchResponse.decRef();
         ClusterStateResponse clusterStateResponse = admin().cluster().state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)).actionGet();
-        Map<String, Object> indexMapping = clusterStateResponse.getState().metadata().getProject().index(indexName).mapping().sourceAsMap();
+        final var project = clusterStateResponse.getState().metadata().getProject(ProjectId.DEFAULT);
+        Map<String, Object> indexMapping = project.index(indexName).mapping().sourceAsMap();
         Map<String, Object> fields = (Map<String, Object>) indexMapping.get("properties");
         assertThat(fields.size(), equalTo(1));
     }

server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java

@@ -26,15 +26,13 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Assertions;
-import org.elasticsearch.core.FixForMultiProject;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexingPressure;
@@ -193,34 +191,33 @@ private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor exec
         boolean hasIndexRequestsWithPipelines = false;
         ClusterState state = clusterService.state();
         ProjectId projectId = projectResolver.getProjectId();
-        final Metadata metadata;
+        final ProjectMetadata project;
         Map<String, ComponentTemplate> componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions();
         Map<String, ComposableIndexTemplate> indexTemplateSubstitutions = bulkRequest.getIndexTemplateSubstitutions();
         if (bulkRequest.isSimulated()
             && (componentTemplateSubstitutions.isEmpty() == false || indexTemplateSubstitutions.isEmpty() == false)) {
             /*
-             * If this is a simulated request, and there are template substitutions, then we want to create and use a new metadata that has
+             * If this is a simulated request, and there are template substitutions, then we want to create and use a new project that has
              * those templates. That is, we want to add the new templates (which will replace any that already existed with the same name),
              * and remove the indices and data streams that are referred to from the bulkRequest so that we get settings from the templates
              * rather than from the indices/data streams.
              */
-            Metadata originalMetadata = state.metadata();
-            @FixForMultiProject // properly ensure simulated actions work with MP
-            Metadata.Builder simulatedMetadataBuilder = Metadata.builder(originalMetadata);
+            ProjectMetadata originalProject = state.metadata().getProject(projectId);
+            ProjectMetadata.Builder simulatedMetadataBuilder = ProjectMetadata.builder(originalProject);
             if (componentTemplateSubstitutions.isEmpty() == false) {
                 Map<String, ComponentTemplate> updatedComponentTemplates = new HashMap<>();
-                updatedComponentTemplates.putAll(originalMetadata.getProject(projectId).componentTemplates());
+                updatedComponentTemplates.putAll(originalProject.componentTemplates());
                 updatedComponentTemplates.putAll(componentTemplateSubstitutions);
                 simulatedMetadataBuilder.componentTemplates(updatedComponentTemplates);
             }
             if (indexTemplateSubstitutions.isEmpty() == false) {
                 Map<String, ComposableIndexTemplate> updatedIndexTemplates = new HashMap<>();
-                updatedIndexTemplates.putAll(originalMetadata.getProject(projectId).templatesV2());
+                updatedIndexTemplates.putAll(originalProject.templatesV2());
                 updatedIndexTemplates.putAll(indexTemplateSubstitutions);
                 simulatedMetadataBuilder.indexTemplates(updatedIndexTemplates);
             }
             /*
-             * We now remove the index from the simulated metadata to force the templates to be used. Note that simulated requests are
+             * We now remove the index from the simulated project to force the templates to be used. Note that simulated requests are
              * always index requests -- no other type of request is supported.
              */
             for (DocWriteRequest<?> actionRequest : bulkRequest.requests) {
@@ -236,12 +233,11 @@ private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor exec
                     }
                 }
             }
-            metadata = simulatedMetadataBuilder.build();
+            project = simulatedMetadataBuilder.build();
         } else {
-            metadata = state.getMetadata();
+            project = state.metadata().getProject(projectId);
         }
 
-        ProjectMetadata project = metadata.getProject(projectId);
         Map<String, IngestService.Pipelines> resolvedPipelineCache = new HashMap<>();
         for (DocWriteRequest<?> actionRequest : bulkRequest.requests) {
             IndexRequest indexRequest = getIndexWriteRequest(actionRequest);

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -1499,12 +1499,6 @@ public Builder put(String name, ComponentTemplate componentTemplate) {
             return this;
         }
 
-        @Deprecated(forRemoval = true)
-        public Builder removeComponentTemplate(String name) {
-            getSingleProject().removeComponentTemplate(name);
-            return this;
-        }
-
         @Deprecated(forRemoval = true)
         public Builder componentTemplates(Map<String, ComponentTemplate> componentTemplates) {
             getSingleProject().componentTemplates(componentTemplates);
@@ -1523,12 +1517,6 @@ public Builder put(String name, ComposableIndexTemplate indexTemplate) {
             return this;
         }
 
-        @Deprecated(forRemoval = true)
-        public Builder removeIndexTemplate(String name) {
-            getSingleProject().removeIndexTemplate(name);
-            return this;
-        }
-
         @Deprecated(forRemoval = true)
         public Builder dataStreams(Map<String, DataStream> dataStreams, Map<String, DataStreamAlias> dataStreamAliases) {
             getSingleProject().dataStreams(dataStreams, dataStreamAliases);
@@ -1557,11 +1545,6 @@ public Builder removeDataStream(String name) {
             return this;
         }
 
-        @Deprecated(forRemoval = true)
-        public boolean removeDataStreamAlias(String aliasName, String dataStreamName, boolean mustExist) {
-            return getSingleProject().removeDataStreamAlias(aliasName, dataStreamName, mustExist);
-        }
-
         public Builder putCustom(String type, ClusterCustom custom) {
             customs.put(type, Objects.requireNonNull(custom, type));
             return this;

server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java

@@ -19,7 +19,8 @@
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
-import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
 import org.elasticsearch.cluster.project.TestProjectResolvers;
@@ -229,7 +230,7 @@ public void testIndexDataWithValidation() throws IOException {
         Map<String, IndexMetadata> indicesMap = new HashMap<>();
         Map<String, IndexTemplateMetadata> v1Templates = new HashMap<>();
         Map<String, ComposableIndexTemplate> v2Templates = new HashMap<>();
-        Metadata.Builder metadataBuilder = new Metadata.Builder();
+        ProjectMetadata.Builder projectBuilder = ProjectMetadata.builder(ProjectId.DEFAULT);
         Set<String> indicesWithInvalidMappings = new HashSet<>();
         for (int i = 0; i < bulkItemCount; i++) {
             Map<String, ?> source = Map.of(randomAlphaOfLength(10), randomAlphaOfLength(5));
@@ -275,10 +276,10 @@ public void testIndexDataWithValidation() throws IOException {
                 default -> throw new AssertionError("Illegal branch");
             }
         }
-        metadataBuilder.indices(indicesMap);
-        metadataBuilder.templates(v1Templates);
-        metadataBuilder.indexTemplates(v2Templates);
-        ClusterServiceUtils.setState(clusterService, new ClusterState.Builder(clusterService.state()).metadata(metadataBuilder));
+        projectBuilder.indices(indicesMap);
+        projectBuilder.templates(v1Templates);
+        projectBuilder.indexTemplates(v2Templates);
+        ClusterServiceUtils.setState(clusterService, ClusterState.builder(clusterService.state()).putProjectMetadata(projectBuilder));
         AtomicBoolean onResponseCalled = new AtomicBoolean(false);
         ActionListener<BulkResponse> listener = new ActionListener<>() {
             @Override