Merge branch 'main' into fix/reenable_concurrency_top_hits_sorted

Author: javanna

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -26,8 +26,7 @@ public enum DockerBase {
         "-wolfi",
         "apk"
     ),
-
-    FIPS("docker.elastic.co/wolfi/chainguard-base-fips:sha256-feb7aeb1bbcb331afa089388f2fa1e81997fc24642ca4fa06b7e502ff599a4cf", "-fips", "apk"),
+    FIPS("docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7", "-fips", "apk"),
     // spotless:on
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the wolfi image.

distribution/docker/fips-docker-aarch64-export/build.gradle

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java

@@ -211,7 +211,8 @@ public Collection<?> createComponents(PluginServices services) {
                 errorStoreInitialisationService.get(),
                 services.allocationService(),
                 dataStreamLifecycleErrorsPublisher.get(),
-                services.dataStreamGlobalRetentionSettings()
+                services.dataStreamGlobalRetentionSettings(),
+                services.projectResolver()
             )
         );
         dataLifecycleInitialisationService.get().init();

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java

@@ -17,7 +17,6 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.metadata.ResettableValue;
 import org.elasticsearch.cluster.metadata.Template;
@@ -49,22 +48,6 @@
  */
 public class DataStreamLifecycleFixtures {
 
-    public static DataStream createDataStream(
-        Metadata.Builder builder,
-        String dataStreamName,
-        int backingIndicesCount,
-        Settings.Builder backingIndicesSettings,
-        @Nullable DataStreamLifecycle lifecycle,
-        Long now
-    ) {
-        var projectBuilder = builder.getProject(Metadata.DEFAULT_PROJECT_ID);
-        if (projectBuilder == null) {
-            projectBuilder = ProjectMetadata.builder(Metadata.DEFAULT_PROJECT_ID);
-            builder.put(projectBuilder);
-        }
-        return createDataStream(projectBuilder, dataStreamName, backingIndicesCount, 0, backingIndicesSettings, lifecycle, now);
-    }
-
     public static DataStream createDataStream(
         ProjectMetadata.Builder builder,
         String dataStreamName,
@@ -76,31 +59,6 @@ public static DataStream createDataStream(
         return createDataStream(builder, dataStreamName, backingIndicesCount, 0, backingIndicesSettings, lifecycle, now);
     }
 
-    public static DataStream createDataStream(
-        Metadata.Builder builder,
-        String dataStreamName,
-        int backingIndicesCount,
-        int failureIndicesCount,
-        Settings.Builder backingIndicesSettings,
-        @Nullable DataStreamLifecycle lifecycle,
-        Long now
-    ) {
-        var projectBuilder = builder.getProject(Metadata.DEFAULT_PROJECT_ID);
-        if (projectBuilder == null) {
-            projectBuilder = ProjectMetadata.builder(Metadata.DEFAULT_PROJECT_ID);
-            builder.put(projectBuilder);
-        }
-        return createDataStream(
-            projectBuilder,
-            dataStreamName,
-            backingIndicesCount,
-            failureIndicesCount,
-            backingIndicesSettings,
-            lifecycle,
-            now
-        );
-    }
-
     public static DataStream createDataStream(
         ProjectMetadata.Builder builder,
         String dataStreamName,

modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java

@@ -285,7 +285,7 @@ public void testMetrics() throws Exception {
             final BlobStore blobStore = blobStoreRepository.blobStore();
             final BlobStore delegateBlobStore = ((BlobStoreWrapper) blobStore).delegate();
             final S3BlobStore s3BlobStore = (S3BlobStore) delegateBlobStore;
-            final Map<S3BlobStore.StatsKey, S3BlobStore.IgnoreNoResponseMetricsCollector> statsCollectors = s3BlobStore
+            final Map<S3BlobStore.StatsKey, S3BlobStore.ElasticsearchS3MetricsCollector> statsCollectors = s3BlobStore
                 .getStatsCollectors().collectors;
 
             final var plugins = internalCluster().getInstance(PluginsService.class, nodeName)

modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java

@@ -143,16 +143,17 @@ public TimeValue getCompareAndExchangeAntiContentionDelay() {
         return service.compareAndExchangeAntiContentionDelay;
     }
 
-    // metrics collector that ignores null responses that we interpret as the request not reaching the S3 endpoint due to a network
-    // issue
-    class IgnoreNoResponseMetricsCollector extends RequestMetricCollector {
+    /**
+     * A {@link RequestMetricCollector} that processes the metrics related to each API invocation attempt according to Elasticsearch's needs
+     */
+    class ElasticsearchS3MetricsCollector extends RequestMetricCollector {
 
         final LongAdder requests = new LongAdder();
         final LongAdder operations = new LongAdder();
         private final Operation operation;
         private final Map<String, Object> attributes;
 
-        private IgnoreNoResponseMetricsCollector(Operation operation, OperationPurpose purpose) {
+        private ElasticsearchS3MetricsCollector(Operation operation, OperationPurpose purpose) {
             this.operation = operation;
             this.attributes = RepositoriesMetrics.createAttributesMap(repositoryMetadata, purpose, operation.getKey());
         }
@@ -582,7 +583,7 @@ public String toString() {
     }
 
     class StatsCollectors {
-        final Map<StatsKey, IgnoreNoResponseMetricsCollector> collectors = new ConcurrentHashMap<>();
+        final Map<StatsKey, ElasticsearchS3MetricsCollector> collectors = new ConcurrentHashMap<>();
 
         RequestMetricCollector getMetricCollector(Operation operation, OperationPurpose purpose) {
             return collectors.computeIfAbsent(new StatsKey(operation, purpose), k -> buildMetricCollector(k.operation(), k.purpose()));
@@ -605,8 +606,8 @@ Map<String, BlobStoreActionStats> statsMap(boolean isStateless) {
             }
         }
 
-        IgnoreNoResponseMetricsCollector buildMetricCollector(Operation operation, OperationPurpose purpose) {
-            return new IgnoreNoResponseMetricsCollector(operation, purpose);
+        ElasticsearchS3MetricsCollector buildMetricCollector(Operation operation, OperationPurpose purpose) {
+            return new ElasticsearchS3MetricsCollector(operation, purpose);
         }
     }
 

modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java

@@ -233,9 +233,6 @@ tests:
 - class: org.elasticsearch.xpack.ilm.TimeSeriesLifecycleActionsIT
   method: testHistoryIsWrittenWithFailure
   issue: https://github.com/elastic/elasticsearch/issues/123203
-- class: org.elasticsearch.action.admin.indices.diskusage.IndexDiskUsageAnalyzerTests
-  method: testCompletionField
-  issue: https://github.com/elastic/elasticsearch/issues/123269
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test151MachineDependentHeapWithSizeOverride
   issue: https://github.com/elastic/elasticsearch/issues/123437
@@ -377,9 +374,6 @@ tests:
 - class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceTests
   method: testIORateIsAdjustedForRunningMergeTasks
   issue: https://github.com/elastic/elasticsearch/issues/125842
-- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
-  method: test {p0=search.vectors/41_knn_search_bbq_hnsw/Test index configured rescore vector score consistency}
-  issue: https://github.com/elastic/elasticsearch/issues/125851
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Verify start transform creates destination index with appropriate mapping}
   issue: https://github.com/elastic/elasticsearch/issues/125854
@@ -389,9 +383,6 @@ tests:
 - class: org.elasticsearch.xpack.ilm.TimeSeriesDataStreamsIT
   method: testSearchableSnapshotAction
   issue: https://github.com/elastic/elasticsearch/issues/125867
-- class: org.elasticsearch.xpack.inference.registry.ModelRegistryMetadataTests
-  method: testUpgrade
-  issue: https://github.com/elastic/elasticsearch/issues/125554
 - class: org.elasticsearch.xpack.downsample.DataStreamLifecycleDownsampleDisruptionIT
   method: testDataStreamLifecycleDownsampleRollingRestart
   issue: https://github.com/elastic/elasticsearch/issues/123769
@@ -407,6 +398,9 @@ tests:
 - class: org.elasticsearch.indices.stats.IndexStatsIT
   method: testThrottleStats
   issue: https://github.com/elastic/elasticsearch/issues/125910
+- class: org.elasticsearch.xpack.esql.action.ManyShardsIT
+  method: testCancelUnnecessaryRequests
+  issue: https://github.com/elastic/elasticsearch/issues/125947
 
 # Examples:
 #

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java

@@ -568,29 +568,3 @@ setup:
   - match: { hits.hits.1._score: $default_rescore1 }
   - match: { hits.hits.2._score: $override_score2 }
   - match: { hits.hits.2._score: $default_rescore2 }
-
-  - do:
-      headers:
-        Content-Type: application/json
-      search:
-        rest_total_hits_as_int: true
-        index: bbq_rescore_zero_hnsw
-        body:
-          knn:
-            field: vector
-            query_vector: [0.128,  0.067, -0.08 ,  0.395, -0.11 , -0.259,  0.473, -0.393,
-                           0.292,  0.571, -0.491,  0.444, -0.288,  0.198, -0.343,  0.015,
-                           0.232,  0.088,  0.228,  0.151, -0.136,  0.236, -0.273, -0.259,
-                           -0.217,  0.359, -0.207,  0.352, -0.142,  0.192, -0.061, -0.17 ,
-                           -0.343,  0.189, -0.221,  0.32 , -0.301, -0.1  ,  0.005,  0.232,
-                           -0.344,  0.136,  0.252,  0.157, -0.13 , -0.244,  0.193, -0.034,
-                           -0.12 , -0.193, -0.102,  0.252, -0.185, -0.167, -0.575,  0.582,
-                           -0.426,  0.983,  0.212,  0.204,  0.03 , -0.276, -0.425, -0.158]
-            k: 3
-            num_candidates: 3
-
-  # Compare scores as hit IDs may change depending on how things are distributed
-  - match: { hits.total: 3 }
-  - match: { hits.hits.0._score: $raw_score0 }
-  - match: { hits.hits.1._score: $raw_score1 }
-  - match: { hits.hits.2._score: $raw_score2 }