Merge branch 'main' into uiam-cloud-api-key-authentication

Author: n1v0lg

build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/TestBuildInfoPluginFuncTest.groovy

@@ -39,17 +39,6 @@ class TestBuildInfoPluginFuncTest extends AbstractGradleFuncTest {
         }
         """
 
-        when:
-        def result = gradleRunner('generateTestBuildInfo').build()
-        def task = result.task(":generateTestBuildInfo")
-
-
-        then:
-        task.outcome == TaskOutcome.SUCCESS
-
-        def output = file("build/generated-build-info/plugin-test-build-info.json")
-        output.exists() == true
-
         def location = Map.of(
             "module", "com.example",
             "representative_class", "com/example/Example.class"
@@ -58,6 +47,15 @@ class TestBuildInfoPluginFuncTest extends AbstractGradleFuncTest {
             "component", "example-component",
             "locations", List.of(location)
         )
+
+        def output = file("build/generated-build-info/plugin-test-build-info.json")
+
+        when:
+        def result = gradleRunner('generateTestBuildInfo').build()
+
+        then:
+        result.task(":generateTestBuildInfo").outcome == TaskOutcome.SUCCESS
+        output.exists() == true
         new ObjectMapper().readValue(output, Map.class) == expectedOutput
     }
 
@@ -87,16 +85,7 @@ class TestBuildInfoPluginFuncTest extends AbstractGradleFuncTest {
         }
         """
 
-        when:
-        def result = gradleRunner('generateTestBuildInfo').build()
-        def task = result.task(":generateTestBuildInfo")
-
-
-        then:
-        task.outcome == TaskOutcome.SUCCESS
-
         def output = file("build/generated-build-info/plugin-test-build-info.json")
-        output.exists() == true
 
         def locationFromModuleInfo = Map.of(
             "module", "org.objectweb.asm",
@@ -115,7 +104,12 @@ class TestBuildInfoPluginFuncTest extends AbstractGradleFuncTest {
             "locations", List.of(locationFromModuleInfo, locationFromManifest, locationFromJarFileName)
         )
 
-        def value = new ObjectMapper().readValue(output, Map.class)
-        value == expectedOutput
+        when:
+        def result = gradleRunner('generateTestBuildInfo').build()
+
+        then:
+        result.task(":generateTestBuildInfo").outcome == TaskOutcome.SUCCESS
+        output.exists() == true
+        new ObjectMapper().readValue(output, Map.class) == expectedOutput
     }
 }

docs/changelog/127767.yaml

@@ -0,0 +1,5 @@
+pr: 127767
+summary: Integrate `OpenAi` Chat Completion in `SageMaker`
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/128139.yaml

@@ -0,0 +1,5 @@
+pr: 128139
+summary: Skip indexing points for `seq_no` in tsdb and logsdb
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/128163.yaml

@@ -0,0 +1,17 @@
+pr: 128163
+summary: Make `skip_unavailable` catch all errors
+area: ES|QL
+type: breaking
+issues: [ ]
+breaking:
+  title: Cluster setting "skip_unavailable" catches all runtime errors
+  area: ES|QL
+  details: "If `skip_unavailable` is set to `true`, the runtime errors from this cluster\
+    \ do not lead to a failure of the query. Instead, the cluster is set to `skipped`\
+    \ or `partial` status, and the query execution continues. This is a breaking change\
+    \ from previous versions, where `skip_unavailable` only applied to errors related\
+    \ to a cluster being unavailable."
+  impact: "The errors on remote clusters, e.g. missing indices, will not lead to a\
+    \ failure of the query. Instead, the cluster is set to `skipped` or `partial` status\
+    \ in the response metadata."
+  notable: false

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java

@@ -33,6 +33,7 @@
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
@@ -53,7 +54,7 @@ public class TransportUpdateDataStreamSettingsAction extends TransportMasterNode
     UpdateDataStreamSettingsAction.Request,
     UpdateDataStreamSettingsAction.Response> {
     private static final Logger logger = LogManager.getLogger(TransportUpdateDataStreamSettingsAction.class);
-    private static final Set<String> APPLY_TO_BACKING_INDICES = Set.of("index.lifecycle.name");
+    private static final Set<String> APPLY_TO_BACKING_INDICES = Set.of("index.lifecycle.name", IndexSettings.PREFER_ILM);
     private static final Set<String> APPLY_TO_DATA_STREAM_ONLY = Set.of("index.number_of_shards");
     private final MetadataDataStreamsService metadataDataStreamsService;
     private final MetadataUpdateSettingsService updateSettingsService;

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml

@@ -51,16 +51,20 @@ setup:
         body:
           index:
             number_of_shards: 2
-            lifecycle.name: my-new-policy
+            lifecycle:
+              name: my-new-policy
+              prefer_ilm: true
   - match: { data_streams.0.name: my-data-stream-1 }
   - match: { data_streams.0.applied_to_data_stream: true }
   - match: { data_streams.0.index_settings_results.applied_to_data_stream_only: [index.number_of_shards]}
-  - match: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: [index.lifecycle.name] }
+  - length: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: 2  }
   - match: { data_streams.0.settings.index.number_of_shards: "2" }
   - match: { data_streams.0.settings.index.lifecycle.name: "my-new-policy" }
+  - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: "true" }
   - match: { data_streams.0.effective_settings.index.number_of_shards: "2" }
   - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" }
   - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-new-policy" }
+  - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: "true" }
 
   - do:
       indices.rollover:
@@ -79,13 +83,15 @@ setup:
   - match: { data_streams.0.effective_settings.index.number_of_shards: "2" }
   - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" }
   - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-new-policy" }
+  - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: "true" }
 
   - do:
       indices.get_data_stream:
         name: my-data-stream-1
   - match: { data_streams.0.name: my-data-stream-1 }
   - match: { data_streams.0.settings.index.number_of_shards: "2" }
   - match: { data_streams.0.settings.index.lifecycle.name: "my-new-policy" }
+  - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: "true" }
   - match: { data_streams.0.effective_settings: null }
 
   - do:
@@ -101,6 +107,7 @@ setup:
   - match: { .$idx0name.settings.index.lifecycle.name: "my-new-policy" }
   - match: { .$idx1name.settings.index.number_of_shards: "2" }
   - match: { .$idx1name.settings.index.lifecycle.name: "my-new-policy" }
+  - match: { .$idx1name.settings.index.lifecycle.prefer_ilm: "true" }
 
 ---
 "Test multiple data streams":

modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java

@@ -11,6 +11,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
@@ -110,6 +111,7 @@ public static final class Repository {
     private final RepositoriesMetrics repositoriesMetrics;
 
     public AzureRepository(
+        final ProjectId projectId,
         final RepositoryMetadata metadata,
         final NamedXContentRegistry namedXContentRegistry,
         final AzureStorageService storageService,
@@ -119,6 +121,7 @@ public AzureRepository(
         final RepositoriesMetrics repositoriesMetrics
     ) {
         super(
+            projectId,
             metadata,
             namedXContentRegistry,
             clusterService,

modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java

@@ -62,10 +62,11 @@ public Map<String, Repository.Factory> getRepositories(
         RecoverySettings recoverySettings,
         RepositoriesMetrics repositoriesMetrics
     ) {
-        return Collections.singletonMap(AzureRepository.TYPE, metadata -> {
+        return Collections.singletonMap(AzureRepository.TYPE, (projectId, metadata) -> {
             AzureStorageService storageService = azureStoreService.get();
             assert storageService != null;
             return new AzureRepository(
+                projectId,
                 metadata,
                 namedXContentRegistry,
                 storageService,

modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.repositories.azure;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -23,6 +24,7 @@
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
 import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.READONLY_SETTING_KEY;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
@@ -35,7 +37,9 @@ private AzureRepository azureRepository(Settings settings) {
             .putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths())
             .put(settings)
             .build();
+        final ProjectId projectId = randomProjectIdOrDefault();
         final AzureRepository azureRepository = new AzureRepository(
+            projectId,
             new RepositoryMetadata("foo", "azure", internalSettings),
             NamedXContentRegistry.EMPTY,
             mock(AzureStorageService.class),
@@ -44,6 +48,7 @@ private AzureRepository azureRepository(Settings settings) {
             new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
             RepositoriesMetrics.NOOP
         );
+        assertThat(azureRepository.getProjectId(), equalTo(projectId));
         assertThat(azureRepository.getBlobStore(), is(nullValue()));
         return azureRepository;
     }

modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java

@@ -276,7 +276,8 @@ public Map<String, Repository.Factory> getRepositories(
         ) {
             return Collections.singletonMap(
                 GoogleCloudStorageRepository.TYPE,
-                metadata -> new GoogleCloudStorageRepository(
+                (projectId, metadata) -> new GoogleCloudStorageRepository(
+                    projectId,
                     metadata,
                     registry,
                     this.storageService,