Merge remote-tracking branch 'es/main' into es87_tsdb_doc_values_format_change_1

Author: martijnvg

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -26,8 +26,7 @@ public enum DockerBase {
         "-wolfi",
         "apk"
     ),
-
-    FIPS("docker.elastic.co/wolfi/chainguard-base-fips:sha256-feb7aeb1bbcb331afa089388f2fa1e81997fc24642ca4fa06b7e502ff599a4cf", "-fips", "apk"),
+    FIPS("docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7", "-fips", "apk"),
     // spotless:on
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the wolfi image.

distribution/docker/fips-docker-aarch64-export/build.gradle

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+

modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java

@@ -285,7 +285,7 @@ public void testMetrics() throws Exception {
             final BlobStore blobStore = blobStoreRepository.blobStore();
             final BlobStore delegateBlobStore = ((BlobStoreWrapper) blobStore).delegate();
             final S3BlobStore s3BlobStore = (S3BlobStore) delegateBlobStore;
-            final Map<S3BlobStore.StatsKey, S3BlobStore.IgnoreNoResponseMetricsCollector> statsCollectors = s3BlobStore
+            final Map<S3BlobStore.StatsKey, S3BlobStore.ElasticsearchS3MetricsCollector> statsCollectors = s3BlobStore
                 .getStatsCollectors().collectors;
 
             final var plugins = internalCluster().getInstance(PluginsService.class, nodeName)

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java

@@ -143,16 +143,17 @@ public TimeValue getCompareAndExchangeAntiContentionDelay() {
         return service.compareAndExchangeAntiContentionDelay;
     }
 
-    // metrics collector that ignores null responses that we interpret as the request not reaching the S3 endpoint due to a network
-    // issue
-    class IgnoreNoResponseMetricsCollector extends RequestMetricCollector {
+    /**
+     * A {@link RequestMetricCollector} that processes the metrics related to each API invocation attempt according to Elasticsearch's needs
+     */
+    class ElasticsearchS3MetricsCollector extends RequestMetricCollector {
 
         final LongAdder requests = new LongAdder();
         final LongAdder operations = new LongAdder();
         private final Operation operation;
         private final Map<String, Object> attributes;
 
-        private IgnoreNoResponseMetricsCollector(Operation operation, OperationPurpose purpose) {
+        private ElasticsearchS3MetricsCollector(Operation operation, OperationPurpose purpose) {
             this.operation = operation;
             this.attributes = RepositoriesMetrics.createAttributesMap(repositoryMetadata, purpose, operation.getKey());
         }
@@ -582,7 +583,7 @@ public String toString() {
     }
 
     class StatsCollectors {
-        final Map<StatsKey, IgnoreNoResponseMetricsCollector> collectors = new ConcurrentHashMap<>();
+        final Map<StatsKey, ElasticsearchS3MetricsCollector> collectors = new ConcurrentHashMap<>();
 
         RequestMetricCollector getMetricCollector(Operation operation, OperationPurpose purpose) {
             return collectors.computeIfAbsent(new StatsKey(operation, purpose), k -> buildMetricCollector(k.operation(), k.purpose()));
@@ -605,8 +606,8 @@ Map<String, BlobStoreActionStats> statsMap(boolean isStateless) {
             }
         }
 
-        IgnoreNoResponseMetricsCollector buildMetricCollector(Operation operation, OperationPurpose purpose) {
-            return new IgnoreNoResponseMetricsCollector(operation, purpose);
+        ElasticsearchS3MetricsCollector buildMetricCollector(Operation operation, OperationPurpose purpose) {
+            return new ElasticsearchS3MetricsCollector(operation, purpose);
         }
     }
 

muted-tests.yml

@@ -233,9 +233,6 @@ tests:
 - class: org.elasticsearch.xpack.ilm.TimeSeriesLifecycleActionsIT
   method: testHistoryIsWrittenWithFailure
   issue: https://github.com/elastic/elasticsearch/issues/123203
-- class: org.elasticsearch.action.admin.indices.diskusage.IndexDiskUsageAnalyzerTests
-  method: testCompletionField
-  issue: https://github.com/elastic/elasticsearch/issues/123269
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test151MachineDependentHeapWithSizeOverride
   issue: https://github.com/elastic/elasticsearch/issues/123437
@@ -377,9 +374,6 @@ tests:
 - class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceTests
   method: testIORateIsAdjustedForRunningMergeTasks
   issue: https://github.com/elastic/elasticsearch/issues/125842
-- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
-  method: test {p0=search.vectors/41_knn_search_bbq_hnsw/Test index configured rescore vector score consistency}
-  issue: https://github.com/elastic/elasticsearch/issues/125851
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Verify start transform creates destination index with appropriate mapping}
   issue: https://github.com/elastic/elasticsearch/issues/125854
@@ -389,9 +383,21 @@ tests:
 - class: org.elasticsearch.xpack.ilm.TimeSeriesDataStreamsIT
   method: testSearchableSnapshotAction
   issue: https://github.com/elastic/elasticsearch/issues/125867
-- class: org.elasticsearch.xpack.inference.registry.ModelRegistryMetadataTests
-  method: testUpgrade
-  issue: https://github.com/elastic/elasticsearch/issues/125554
+- class: org.elasticsearch.xpack.downsample.DataStreamLifecycleDownsampleDisruptionIT
+  method: testDataStreamLifecycleDownsampleRollingRestart
+  issue: https://github.com/elastic/elasticsearch/issues/123769
+- class: org.elasticsearch.multiproject.test.CoreWithMultipleProjectsClientYamlTestSuiteIT
+  method: test {yaml=search.vectors/41_knn_search_bbq_hnsw/Test index configured rescore vector score consistency}
+  issue: https://github.com/elastic/elasticsearch/issues/125902
+- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
+  method: test {yaml=cat.allocation/10_basic/Node forecasts}
+  issue: https://github.com/elastic/elasticsearch/issues/125848
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=ml/start_data_frame_analytics/Test start given dest index is not empty}
+  issue: https://github.com/elastic/elasticsearch/issues/125909
+- class: org.elasticsearch.indices.stats.IndexStatsIT
+  method: testThrottleStats
+  issue: https://github.com/elastic/elasticsearch/issues/125910
 
 # Examples:
 #

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml

@@ -568,29 +568,3 @@ setup:
   - match: { hits.hits.1._score: $default_rescore1 }
   - match: { hits.hits.2._score: $override_score2 }
   - match: { hits.hits.2._score: $default_rescore2 }
-
-  - do:
-      headers:
-        Content-Type: application/json
-      search:
-        rest_total_hits_as_int: true
-        index: bbq_rescore_zero_hnsw
-        body:
-          knn:
-            field: vector
-            query_vector: [0.128,  0.067, -0.08 ,  0.395, -0.11 , -0.259,  0.473, -0.393,
-                           0.292,  0.571, -0.491,  0.444, -0.288,  0.198, -0.343,  0.015,
-                           0.232,  0.088,  0.228,  0.151, -0.136,  0.236, -0.273, -0.259,
-                           -0.217,  0.359, -0.207,  0.352, -0.142,  0.192, -0.061, -0.17 ,
-                           -0.343,  0.189, -0.221,  0.32 , -0.301, -0.1  ,  0.005,  0.232,
-                           -0.344,  0.136,  0.252,  0.157, -0.13 , -0.244,  0.193, -0.034,
-                           -0.12 , -0.193, -0.102,  0.252, -0.185, -0.167, -0.575,  0.582,
-                           -0.426,  0.983,  0.212,  0.204,  0.03 , -0.276, -0.425, -0.158]
-            k: 3
-            num_candidates: 3
-
-  # Compare scores as hit IDs may change depending on how things are distributed
-  - match: { hits.total: 3 }
-  - match: { hits.hits.0._score: $raw_score0 }
-  - match: { hits.hits.1._score: $raw_score1 }
-  - match: { hits.hits.2._score: $raw_score2 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml

@@ -884,22 +884,3 @@ setup:
   - match: { hits.hits.1._score: $default_rescore1 }
   - match: { hits.hits.2._score: $override_score2 }
   - match: { hits.hits.2._score: $default_rescore2 }
-
-  - do:
-      headers:
-        Content-Type: application/json
-      search:
-        rest_total_hits_as_int: true
-        index: int8_rescore_zero_hnsw
-        body:
-          knn:
-            field: vector
-            query_vector: [-0.5, 90.0, -10, 14.8]
-            k: 3
-            num_candidates: 3
-
-  # Compare scores as hit IDs may change depending on how things are distributed
-  - match: { hits.total: 3 }
-  - match: { hits.hits.0._score: $raw_score0 }
-  - match: { hits.hits.1._score: $raw_score1 }
-  - match: { hits.hits.2._score: $raw_score2 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml

@@ -916,22 +916,3 @@ setup:
   - match: { hits.hits.1._score: $default_rescore1 }
   - match: { hits.hits.2._score: $override_score2 }
   - match: { hits.hits.2._score: $default_rescore2 }
-
-  - do:
-      headers:
-        Content-Type: application/json
-      search:
-        rest_total_hits_as_int: true
-        index: int4_rescore_zero_hnsw
-        body:
-          knn:
-            field: vector
-            query_vector: [-0.5, 90.0, -10, 14.8]
-            k: 3
-            num_candidates: 3
-
-  # Compare scores as hit IDs may change depending on how things are distributed
-  - match: { hits.total: 3 }
-  - match: { hits.hits.0._score: $raw_score0 }
-  - match: { hits.hits.1._score: $raw_score1 }
-  - match: { hits.hits.2._score: $raw_score2 }

server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java

@@ -79,6 +79,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
@@ -466,7 +467,7 @@ public void testNonThrottleStats() throws Exception {
         assertThat(stats.getPrimaries().getIndexing().getTotal().getThrottleTime().millis(), equalTo(0L));
     }
 
-    public void testThrottleStats() throws Exception {
+    public void testThrottleStats() {
         assertAcked(
             prepareCreate("test").setSettings(
                 settingsBuilder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1")
@@ -479,31 +480,38 @@ public void testThrottleStats() throws Exception {
             )
         );
         ensureGreen();
-        long termUpto = 0;
-        IndicesStatsResponse stats;
         // make sure we see throttling kicking in:
-        boolean done = false;
+        AtomicBoolean done = new AtomicBoolean();
+        AtomicLong termUpTo = new AtomicLong();
         long start = System.currentTimeMillis();
-        while (done == false) {
-            for (int i = 0; i < 100; i++) {
-                // Provoke slowish merging by making many unique terms:
-                StringBuilder sb = new StringBuilder();
-                for (int j = 0; j < 100; j++) {
-                    sb.append(' ');
-                    sb.append(termUpto++);
-                }
-                prepareIndex("test").setId("" + termUpto).setSource("field" + (i % 10), sb.toString()).get();
-                if (i % 2 == 0) {
+        for (int threadIdx = 0; threadIdx < 5; threadIdx++) {
+            int finalThreadIdx = threadIdx;
+            new Thread(() -> {
+                IndicesStatsResponse stats;
+                while (done.get() == false) {
+                    for (int i = 0; i < 100; i++) {
+                        // Provoke slowish merging by making many unique terms:
+                        StringBuilder sb = new StringBuilder();
+                        for (int j = 0; j < 100; j++) {
+                            sb.append(' ');
+                            sb.append(termUpTo.incrementAndGet());
+                        }
+                        prepareIndex("test").setId("" + termUpTo.get()).setSource("field" + (i % 10), sb.toString()).get();
+                        if (i % 2 == 0) {
+                            refresh();
+                        }
+                    }
                     refresh();
+                    if (finalThreadIdx == 0) {
+                        stats = indicesAdmin().prepareStats().get();
+                        done.set(stats.getPrimaries().getIndexing().getTotal().getThrottleTime().millis() > 0);
+                    }
+                    if (System.currentTimeMillis() - start > 300 * 1000) { // Wait 5 minutes for throttling to kick in
+                        done.set(true);
+                        fail("index throttling didn't kick in after 5 minutes of intense merging");
+                    }
                 }
-            }
-            refresh();
-            stats = indicesAdmin().prepareStats().get();
-            // nodesStats = clusterAdmin().prepareNodesStats().setIndices(true).get();
-            done = stats.getPrimaries().getIndexing().getTotal().getThrottleTime().millis() > 0;
-            if (System.currentTimeMillis() - start > 300 * 1000) { // Wait 5 minutes for throttling to kick in
-                fail("index throttling didn't kick in after 5 minutes of intense merging");
-            }
+            }).start();
         }
 
         // Optimize & flush and wait; else we sometimes get a "Delete Index failed - not acked"

server/src/main/java/org/elasticsearch/index/engine/Engine.java

@@ -2371,7 +2371,7 @@ public record FlushResult(boolean flushPerformed, long generation) {
     }
 
     /**
-     * Ensures the engine is in a state that it can be closed by a call to {@link IndexShard#resetEngine(Consumer<Engine>)}.
+     * Ensures the engine is in a state that it can be closed by a call to {@link IndexShard#resetEngine()}.
      *
      * In general, resetting the engine should be done with care, to consider any
      * in-progress operations and listeners (e.g., primary term and generation listeners).