Merge branch 'main' into allow-fc-cps

Author: piergm

docs/changelog/134359.yaml

@@ -0,0 +1,6 @@
+pr: 134359
+summary: Make `MutableSearchResponse` ref-counted to prevent use-after-close in async
+  search
+area: Search
+type: bug
+issues: []

docs/changelog/137407.yaml

@@ -0,0 +1,7 @@
+pr: 137407
+summary: Taking additional settings providers into account for data stream effective
+  settings
+area: Data streams
+type: bug
+issues:
+ - 137381

docs/changelog/137434.yaml

@@ -0,0 +1,5 @@
+pr: 137434
+summary: Require basic licence for the Elastic Inference Service
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/137560.yaml

@@ -0,0 +1,5 @@
+pr: 137560
+summary: More friendly exceptions for validation errors
+area: SQL
+type: bug
+issues: []

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
 import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
@@ -38,6 +39,7 @@ public class TransportGetDataStreamSettingsAction extends TransportLocalProjectM
     GetDataStreamSettingsAction.Response> {
     private final IndexNameExpressionResolver indexNameExpressionResolver;
     private final SettingsFilter settingsFilter;
+    private final MetadataDataStreamsService metadataDataStreamsService;
 
     @Inject
     public TransportGetDataStreamSettingsAction(
@@ -47,7 +49,8 @@ public TransportGetDataStreamSettingsAction(
         SettingsFilter settingsFilter,
         ActionFilters actionFilters,
         ProjectResolver projectResolver,
-        IndexNameExpressionResolver indexNameExpressionResolver
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        MetadataDataStreamsService metadataDataStreamsService
     ) {
         super(
             GetSettingsAction.NAME,
@@ -59,6 +62,7 @@ public TransportGetDataStreamSettingsAction(
         );
         this.indexNameExpressionResolver = indexNameExpressionResolver;
         this.settingsFilter = settingsFilter;
+        this.metadataDataStreamsService = metadataDataStreamsService;
     }
 
     @Override
@@ -83,7 +87,9 @@ protected void localClusterStateOperation(
         for (String dataStreamName : dataStreamNames) {
             DataStream dataStream = dataStreamMap.get(dataStreamName);
             Settings settings = settingsFilter.filter(dataStream.getSettings());
-            Settings effectiveSettings = settingsFilter.filter(dataStream.getEffectiveSettings(project.metadata()));
+            Settings effectiveSettings = settingsFilter.filter(
+                metadataDataStreamsService.getEffectiveSettings(project.metadata(), dataStream)
+            );
             responseList.add(new GetDataStreamSettingsAction.DataStreamSettingsResponse(dataStreamName, settings, effectiveSettings));
         }
         listener.onResponse(new GetDataStreamSettingsAction.Response(responseList));

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java

@@ -32,6 +32,7 @@
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.project.ProjectResolver;
@@ -80,6 +81,7 @@ public class TransportGetDataStreamsAction extends TransportLocalProjectMetadata
     private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings;
     private final IndexSettingProviders indexSettingProviders;
     private final Client client;
+    private final MetadataDataStreamsService metadataDataStreamsService;
 
     /**
      * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until
@@ -99,7 +101,8 @@ public TransportGetDataStreamsAction(
         DataStreamGlobalRetentionSettings globalRetentionSettings,
         DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
         IndexSettingProviders indexSettingProviders,
-        Client client
+        Client client,
+        MetadataDataStreamsService metadataDataStreamsService
     ) {
         super(
             GetDataStreamAction.NAME,
@@ -116,6 +119,7 @@ public TransportGetDataStreamsAction(
         this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings;
         this.indexSettingProviders = indexSettingProviders;
         this.client = new OriginSettingClient(client, "stack");
+        this.metadataDataStreamsService = metadataDataStreamsService;
 
         transportService.registerRequestHandler(
             actionName,
@@ -159,7 +163,8 @@ public void onResponse(DataStreamsStatsAction.Response response) {
                             globalRetentionSettings,
                             dataStreamFailureStoreSettings,
                             indexSettingProviders,
-                            maxTimestamps
+                            maxTimestamps,
+                            metadataDataStreamsService
                         )
                     );
                 }
@@ -180,7 +185,8 @@ public void onFailure(Exception e) {
                     globalRetentionSettings,
                     dataStreamFailureStoreSettings,
                     indexSettingProviders,
-                    null
+                    null,
+                    metadataDataStreamsService
                 )
             );
         }
@@ -230,7 +236,8 @@ static GetDataStreamAction.Response innerOperation(
         DataStreamGlobalRetentionSettings globalRetentionSettings,
         DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
         IndexSettingProviders indexSettingProviders,
-        @Nullable Map<String, Long> maxTimestamps
+        @Nullable Map<String, Long> maxTimestamps,
+        MetadataDataStreamsService metadataDataStreamsService
     ) {
         List<DataStream> dataStreams = getDataStreams(state.metadata(), indexNameExpressionResolver, request);
         List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = new ArrayList<>(dataStreams.size());
@@ -265,7 +272,12 @@ static GetDataStreamAction.Response innerOperation(
             } else {
                 indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false);
                 if (indexTemplate != null) {
-                    Settings settings = dataStream.getEffectiveSettings(state.metadata());
+                    final Settings settings;
+                    try {
+                        settings = metadataDataStreamsService.getEffectiveSettings(state.metadata(), dataStream);
+                    } catch (IOException e) {
+                        throw new RuntimeException("Failed to get effective settings for data stream: " + dataStream.getName(), e);
+                    }
                     ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
                     if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) {
                         try {
@@ -277,7 +289,7 @@ static GetDataStreamAction.Response innerOperation(
                                 dataStream.getEffectiveIndexTemplate(state.metadata())
                             );
                         } catch (IOException e) {
-                            throw new RuntimeException(e);
+                            throw new RuntimeException("Failed to determine indexMode for data stream: " + dataStream.getName(), e);
                         }
                     }
                     indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java

@@ -40,6 +40,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -231,7 +232,13 @@ private void updateSettingsOnIndices(
         List<String> appliedToDataStreamOnly = new ArrayList<>();
         List<String> appliedToDataStreamAndWriteIndexOnly = new ArrayList<>();
         List<String> appliedToDataStreamAndBackingIndices = new ArrayList<>();
-        Settings effectiveSettings = dataStream.getEffectiveSettings(projectResolver.getProjectMetadata(clusterService.state()));
+        ProjectMetadata projectMetadata = projectResolver.getProjectMetadata(clusterService.state());
+        final Settings effectiveSettings;
+        try {
+            effectiveSettings = metadataDataStreamsService.getEffectiveSettings(projectMetadata, dataStream);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
         for (String settingName : requestSettings.keySet()) {
             if (APPLY_TO_WRITE_INDEX.contains(settingName)) {
                 settingsToApplyToWriteIndex.put(settingName, effectiveSettings.get(settingName));

modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java

@@ -20,8 +20,10 @@
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.metadata.Template;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
@@ -31,10 +33,17 @@
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettingProviders;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.indices.TestIndexNameExpressionResolver;
+import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
 
+import java.io.IOException;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
@@ -55,6 +64,7 @@
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
 
 public class TransportGetDataStreamsActionTests extends ESTestCase {
 
@@ -66,6 +76,29 @@ public class TransportGetDataStreamsActionTests extends ESTestCase {
     private final DataStreamFailureStoreSettings emptyDataStreamFailureStoreSettings = DataStreamFailureStoreSettings.create(
         ClusterSettings.createBuiltInClusterSettings()
     );
+    private ThreadPool testThreadPool;
+    private MetadataDataStreamsService metadataDataStreamsService;
+
+    @Before
+    public void setup() throws IOException {
+        testThreadPool = new TestThreadPool(getTestName());
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool);
+        IndicesService indicesService = mock(IndicesService.class);
+        metadataDataStreamsService = new MetadataDataStreamsService(
+            clusterService,
+            indicesService,
+            DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()),
+            IndexSettingProviders.EMPTY
+        );
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        if (testThreadPool != null) {
+            testThreadPool.shutdown();
+        }
+    }
 
     public void testGetDataStream() {
         final String dataStreamName = "my-data-stream";
@@ -186,7 +219,8 @@ public void testGetTimeSeriesDataStream() {
             dataStreamGlobalRetentionSettings,
             emptyDataStreamFailureStoreSettings,
             new IndexSettingProviders(Set.of()),
-            null
+            null,
+            metadataDataStreamsService
         );
         assertThat(
             response.getDataStreams(),
@@ -219,7 +253,8 @@ public void testGetTimeSeriesDataStream() {
             dataStreamGlobalRetentionSettings,
             emptyDataStreamFailureStoreSettings,
             new IndexSettingProviders(Set.of()),
-            null
+            null,
+            metadataDataStreamsService
         );
         assertThat(
             response.getDataStreams(),
@@ -273,7 +308,8 @@ public void testGetTimeSeriesDataStreamWithOutOfOrderIndices() {
             dataStreamGlobalRetentionSettings,
             emptyDataStreamFailureStoreSettings,
             new IndexSettingProviders(Set.of()),
-            null
+            null,
+            metadataDataStreamsService
         );
         assertThat(
             response.getDataStreams(),
@@ -313,7 +349,8 @@ public void testGetTimeSeriesMixedDataStream() {
             dataStreamGlobalRetentionSettings,
             emptyDataStreamFailureStoreSettings,
             new IndexSettingProviders(Set.of()),
-            null
+            null,
+            metadataDataStreamsService
         );
 
         var name1 = getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli());
@@ -349,7 +386,8 @@ public void testPassingGlobalRetention() {
             dataStreamGlobalRetentionSettings,
             emptyDataStreamFailureStoreSettings,
             new IndexSettingProviders(Set.of()),
-            null
+            null,
+            metadataDataStreamsService
         );
         assertThat(response.getDataGlobalRetention(), nullValue());
         DataStreamGlobalRetention dataGlobalRetention = new DataStreamGlobalRetention(
@@ -376,7 +414,8 @@ public void testPassingGlobalRetention() {
             withGlobalRetentionSettings,
             emptyDataStreamFailureStoreSettings,
             new IndexSettingProviders(Set.of()),
-            null
+            null,
+            metadataDataStreamsService
         );
         assertThat(response.getDataGlobalRetention(), equalTo(dataGlobalRetention));
         // We used the default failures retention here which is greater than the max
@@ -404,7 +443,8 @@ public void testDataStreamIsFailureStoreEffectivelyEnabled_disabled() {
             dataStreamGlobalRetentionSettings,
             emptyDataStreamFailureStoreSettings,
             new IndexSettingProviders(Set.of()),
-            null
+            null,
+            metadataDataStreamsService
         );
         assertThat(response.getDataStreams(), hasSize(1));
         assertThat(response.getDataStreams().getFirst().isFailureStoreEffectivelyEnabled(), is(false));
@@ -431,7 +471,8 @@ public void testDataStreamIsFailureStoreEffectivelyEnabled_enabledExplicitly() {
             dataStreamGlobalRetentionSettings,
             emptyDataStreamFailureStoreSettings,
             new IndexSettingProviders(Set.of()),
-            null
+            null,
+            metadataDataStreamsService
         );
         assertThat(response.getDataStreams(), hasSize(1));
         assertThat(response.getDataStreams().getFirst().isFailureStoreEffectivelyEnabled(), is(true));
@@ -464,7 +505,8 @@ public void testDataStreamIsFailureStoreEffectivelyEnabled_enabledByClusterSetti
                 )
             ),
             new IndexSettingProviders(Set.of()),
-            null
+            null,
+            metadataDataStreamsService
         );
         assertThat(response.getDataStreams(), hasSize(1));
         assertThat(response.getDataStreams().getFirst().isFailureStoreEffectivelyEnabled(), is(true));
@@ -491,7 +533,8 @@ public void testProvidersAffectMode() {
             dataStreamGlobalRetentionSettings,
             emptyDataStreamFailureStoreSettings,
             IndexSettingProviders.of((additionalSettings) -> additionalSettings.put("index.mode", IndexMode.LOOKUP)),
-            null
+            null,
+            metadataDataStreamsService
         );
         assertThat(response.getDataStreams().getFirst().getIndexModeName(), equalTo("lookup"));
         assertThat(
@@ -531,7 +574,8 @@ public void testGetEffectiveSettingsTemplateOnlySettings() {
             dataStreamGlobalRetentionSettings,
             emptyDataStreamFailureStoreSettings,
             new IndexSettingProviders(Set.of()),
-            null
+            null,
+            metadataDataStreamsService
         );
         assertNotNull(response.getDataStreams());
         assertThat(response.getDataStreams().size(), equalTo(1));
@@ -563,7 +607,8 @@ public void testGetEffectiveSettingsComponentTemplateOnlySettings() {
             dataStreamGlobalRetentionSettings,
             emptyDataStreamFailureStoreSettings,
             new IndexSettingProviders(Set.of()),
-            null
+            null,
+            metadataDataStreamsService
         );
         assertNotNull(response.getDataStreams());
         assertThat(response.getDataStreams().size(), equalTo(1));
@@ -601,7 +646,8 @@ public void testGetEffectiveSettings() {
             dataStreamGlobalRetentionSettings,
             emptyDataStreamFailureStoreSettings,
             new IndexSettingProviders(Set.of()),
-            null
+            null,
+            metadataDataStreamsService
         );
         assertNotNull(response.getDataStreams());
         assertThat(response.getDataStreams().size(), equalTo(1));