Merge branch 'main' into migrate_block_loader_tests

Author: lkts

.buildkite/pipelines/periodic-packaging.yml

@@ -287,8 +287,8 @@ steps:
         env:
           BWC_VERSION: 8.15.5
 
-      - label: "{{matrix.image}} / 8.16.5 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.5
+      - label: "{{matrix.image}} / 8.16.6 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.6
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -301,7 +301,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.16.5
+          BWC_VERSION: 8.16.6
 
       - label: "{{matrix.image}} / 8.17.6 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.6

.buildkite/pipelines/periodic.yml

@@ -306,8 +306,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.16.5 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.5#bwcTest
+      - label: 8.16.6 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.6#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -316,7 +316,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.16.5
+          BWC_VERSION: 8.16.6
         retry:
           automatic:
             - exit_status: "-1"

.ci/bwcVersions

@@ -15,7 +15,7 @@ BWC_VERSION:
   - "8.13.4"
   - "8.14.3"
   - "8.15.5"
-  - "8.16.5"
+  - "8.16.6"
   - "8.17.6"
   - "8.18.1"
   - "8.19.0"

Could

@@ -39,7 +39,6 @@
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.profile.AsyncProfiler;
@@ -51,9 +50,8 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.Random;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
 
 @BenchmarkMode(Mode.SingleShotTime)
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
@@ -71,23 +69,10 @@ public class TSDBDocValuesMergeBenchmark {
         LogConfigurator.setNodeName("test");
     }
 
-    @Param("20431204")
-    private int nDocs;
-
-    @Param("1000")
-    private int deltaTime;
-
-    @Param("42")
-    private int seed;
-
     private static final String TIMESTAMP_FIELD = "@timestamp";
     private static final String HOSTNAME_FIELD = "host.name";
     private static final long BASE_TIMESTAMP = 1704067200000L;
 
-    private IndexWriter indexWriterWithoutOptimizedMerge;
-    private IndexWriter indexWriterWithOptimizedMerge;
-    private ExecutorService executorService;
-
     public static void main(String[] args) throws RunnerException {
         final Options options = new OptionsBuilder().include(TSDBDocValuesMergeBenchmark.class.getSimpleName())
             .addProfiler(AsyncProfiler.class)
@@ -96,78 +81,168 @@ public static void main(String[] args) throws RunnerException {
         new Runner(options).run();
     }
 
-    @Setup(Level.Trial)
-    public void setup() throws IOException {
-        executorService = Executors.newSingleThreadExecutor();
+    @State(Scope.Benchmark)
+    public static class StateDenseWithoutOptimizeMerge {
 
-        final Directory tempDirectoryWithoutDocValuesSkipper = FSDirectory.open(Files.createTempDirectory("temp1-"));
-        final Directory tempDirectoryWithDocValuesSkipper = FSDirectory.open(Files.createTempDirectory("temp2-"));
+        @Param("20431204")
+        private int nDocs;
+
+        @Param("1000")
+        private int deltaTime;
+
+        @Param("42")
+        private int seed;
+
+        private Directory directory;
+        private final Supplier<IndexWriterConfig> iwc = () -> createIndexWriterConfig(false);
+
+        @Setup(Level.Trial)
+        public void setup() throws IOException {
+            directory = FSDirectory.open(Files.createTempDirectory("temp2-"));
+            createIndex(directory, iwc.get(), false, nDocs, deltaTime, seed);
+        }
 
-        indexWriterWithoutOptimizedMerge = createIndex(tempDirectoryWithoutDocValuesSkipper, false);
-        indexWriterWithOptimizedMerge = createIndex(tempDirectoryWithDocValuesSkipper, true);
     }
 
-    private IndexWriter createIndex(final Directory directory, final boolean optimizedMergeEnabled) throws IOException {
-        final var iwc = createIndexWriterConfig(optimizedMergeEnabled);
-        long counter1 = 0;
-        long counter2 = 10_000_000;
-        long[] gauge1Values = new long[] { 2, 4, 6, 8, 10, 12, 14, 16 };
-        long[] gauge2Values = new long[] { -2, -4, -6, -8, -10, -12, -14, -16 };
-        int numHosts = 1000;
-        String[] tags = new String[] { "tag_1", "tag_2", "tag_3", "tag_4", "tag_5", "tag_6", "tag_7", "tag_8" };
+    @Benchmark
+    public void forceMergeDenseWithoutOptimizedMerge(StateDenseWithoutOptimizeMerge state) throws IOException {
+        forceMerge(state.directory, state.iwc.get());
+    }
 
-        final Random random = new Random(seed);
-        IndexWriter indexWriter = new IndexWriter(directory, iwc);
-        for (int i = 0; i < nDocs; i++) {
-            final Document doc = new Document();
-
-            final int batchIndex = i / numHosts;
-            final String hostName = "host-" + batchIndex;
-            // Slightly vary the timestamp in each document
-            final long timestamp = BASE_TIMESTAMP + ((i % numHosts) * deltaTime) + random.nextInt(0, deltaTime);
-
-            doc.add(new SortedDocValuesField(HOSTNAME_FIELD, new BytesRef(hostName)));
-            doc.add(new SortedNumericDocValuesField(TIMESTAMP_FIELD, timestamp));
-            doc.add(new SortedNumericDocValuesField("counter_1", counter1++));
-            doc.add(new SortedNumericDocValuesField("counter_2", counter2++));
-            doc.add(new SortedNumericDocValuesField("gauge_1", gauge1Values[i % gauge1Values.length]));
-            doc.add(new SortedNumericDocValuesField("gauge_2", gauge2Values[i % gauge1Values.length]));
-            int numTags = tags.length % (i + 1);
-            for (int j = 0; j < numTags; j++) {
-                doc.add(new SortedSetDocValuesField("tags", new BytesRef(tags[j])));
-            }
+    @State(Scope.Benchmark)
+    public static class StateDenseWithOptimizeMerge {
+
+        @Param("20431204")
+        private int nDocs;
+
+        @Param("1000")
+        private int deltaTime;
+
+        @Param("42")
+        private int seed;
+
+        private Directory directory;
+        private final Supplier<IndexWriterConfig> iwc = () -> createIndexWriterConfig(true);
+
+        @Setup(Level.Trial)
+        public void setup() throws IOException {
+            directory = FSDirectory.open(Files.createTempDirectory("temp1-"));
+            createIndex(directory, iwc.get(), false, nDocs, deltaTime, seed);
+        }
+
+    }
+
+    @Benchmark
+    public void forceMergeDenseWithOptimizedMerge(StateDenseWithOptimizeMerge state) throws IOException {
+        forceMerge(state.directory, state.iwc.get());
+    }
+
+    @State(Scope.Benchmark)
+    public static class StateSparseWithoutOptimizeMerge {
 
-            indexWriter.addDocument(doc);
+        @Param("20431204")
+        private int nDocs;
+
+        @Param("1000")
+        private int deltaTime;
+
+        @Param("42")
+        private int seed;
+
+        private Directory directory;
+        private final Supplier<IndexWriterConfig> iwc = () -> createIndexWriterConfig(false);
+
+        @Setup(Level.Trial)
+        public void setup() throws IOException {
+            directory = FSDirectory.open(Files.createTempDirectory("temp4-"));
+            createIndex(directory, iwc.get(), true, nDocs, deltaTime, seed);
         }
-        indexWriter.commit();
-        return indexWriter;
+
     }
 
     @Benchmark
-    public void forceMergeWithoutOptimizedMerge() throws IOException {
-        forceMerge(indexWriterWithoutOptimizedMerge);
+    public void forceMergeSparseWithoutOptimizedMerge(StateSparseWithoutOptimizeMerge state) throws IOException {
+        forceMerge(state.directory, state.iwc.get());
+    }
+
+    @State(Scope.Benchmark)
+    public static class StateSparseWithOptimizeMerge {
+
+        @Param("20431204")
+        private int nDocs;
+
+        @Param("1000")
+        private int deltaTime;
+
+        @Param("42")
+        private int seed;
+
+        private Directory directory;
+        private final Supplier<IndexWriterConfig> iwc = () -> createIndexWriterConfig(true);
+
+        @Setup(Level.Trial)
+        public void setup() throws IOException {
+            directory = FSDirectory.open(Files.createTempDirectory("temp3-"));
+            createIndex(directory, iwc.get(), true, nDocs, deltaTime, seed);
+        }
+
     }
 
     @Benchmark
-    public void forceMergeWithOptimizedMerge() throws IOException {
-        forceMerge(indexWriterWithOptimizedMerge);
+    public void forceMergeSparseWithOptimizedMerge(StateSparseWithOptimizeMerge state) throws IOException {
+        forceMerge(state.directory, state.iwc.get());
     }
 
-    private void forceMerge(final IndexWriter indexWriter) throws IOException {
-        indexWriter.forceMerge(1);
+    private void forceMerge(Directory directory, IndexWriterConfig config) throws IOException {
+        try (var indexWriter = new IndexWriter(directory, config)) {
+            indexWriter.forceMerge(1);
+        }
     }
 
-    @TearDown(Level.Trial)
-    public void tearDown() {
-        if (executorService != null) {
-            executorService.shutdown();
-            try {
-                if (executorService.awaitTermination(30, TimeUnit.SECONDS) == false) {
-                    executorService.shutdownNow();
+    static void createIndex(Directory directory, IndexWriterConfig iwc, boolean sparse, int nDocs, int deltaTime, int seed)
+        throws IOException {
+        long counter1 = 0;
+        long counter2 = 10_000_000;
+        long[] gauge1Values = new long[] { 2, 4, 6, 8, 10, 12, 14, 16 };
+        long[] gauge2Values = new long[] { -2, -4, -6, -8, -10, -12, -14, -16 };
+        int numHosts = 10000;
+        String[] tags = new String[] { "tag_1", "tag_2", "tag_3", "tag_4", "tag_5", "tag_6", "tag_7", "tag_8" };
+
+        final Random random = new Random(seed);
+        try (var indexWriter = new IndexWriter(directory, iwc)) {
+            for (int i = 0; i < nDocs; i++) {
+                final Document doc = new Document();
+
+                final int batchIndex = i % numHosts;
+                final String hostName = "host-" + batchIndex;
+                // Slightly vary the timestamp in each document
+                final long timestamp = BASE_TIMESTAMP + ((i % numHosts) * deltaTime) + random.nextInt(0, deltaTime);
+
+                doc.add(new SortedDocValuesField(HOSTNAME_FIELD, new BytesRef(hostName)));
+                doc.add(new SortedNumericDocValuesField(TIMESTAMP_FIELD, timestamp));
+                if (sparse == false || random.nextBoolean()) {
+                    doc.add(new SortedNumericDocValuesField("counter_1", counter1++));
+                }
+                if (sparse == false || random.nextBoolean()) {
+                    doc.add(new SortedNumericDocValuesField("counter_2", counter2++));
+                }
+                if (sparse == false || random.nextBoolean()) {
+                    doc.add(new SortedNumericDocValuesField("gauge_1", gauge1Values[i % gauge1Values.length]));
+                }
+                if (sparse == false || random.nextBoolean()) {
+                    doc.add(new SortedNumericDocValuesField("gauge_2", gauge2Values[i % gauge1Values.length]));
+                }
+                if (sparse == false || random.nextBoolean()) {
+                    int numTags = tags.length % (i + 1);
+                    for (int j = 0; j < numTags; j++) {
+                        doc.add(new SortedSetDocValuesField("tags", new BytesRef(tags[j])));
+                    }
+                }
+                indexWriter.addDocument(doc);
+
+                if (i % 10000 == 0) {
+                    indexWriter.commit();
                 }
-            } catch (InterruptedException e) {
-                executorService.shutdownNow();
-                Thread.currentThread().interrupt();
             }
         }
     }

benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java

@@ -0,0 +1,5 @@
+pr: 120363
+summary: Add thread pool utilization metric
+area: "Infra/Metrics"
+type: enhancement
+issues: []

docs/changelog/120363.yaml

@@ -0,0 +1,5 @@
+pr: 126629
+summary: Default new `semantic_text` fields to use BBQ when models are compatible
+area: Relevance
+type: enhancement
+issues: []

docs/changelog/126629.yaml

@@ -0,0 +1,6 @@
+pr: 126770
+summary: Remove empty results before merging
+area: Search
+type: bug
+issues:
+ - 126742

docs/changelog/126770.yaml

@@ -475,7 +475,7 @@ In addition to the [Settings valid for all realms](#ref-realm-settings), you can
 
 
 `ssl.supported_protocols`
-:   ([Static](docs-content://deploy-manage/deploy/self-managed/configure-elasticsearch.md#static-cluster-setting)) Supported protocols with versions. Valid protocols: `SSLv2Hello`, `SSLv3`, `TLSv1`, `TLSv1.1`, `TLSv1.2`, `TLSv1.3`. If the JVM’s SSL provider supports TLSv1.3, the default is `TLSv1.3,TLSv1.2,TLSv1.1`. Otherwise, the default is `TLSv1.2,TLSv1.1`.
+:   ([Static](docs-content://deploy-manage/deploy/self-managed/configure-elasticsearch.md#static-cluster-setting)) Supported protocols with versions. Valid protocols: `SSLv2Hello`, `SSLv3`, `TLSv1`, `TLSv1.1`, `TLSv1.2`, `TLSv1.3`. The default is `TLSv1.3,TLSv1.2`.
 
     {{es}} relies on your JDK’s implementation of SSL and TLS. View [Supported SSL/TLS versions by JDK version](docs-content://deploy-manage/security/supported-ssltls-versions-by-jdk-version.md) for more information.
 

docs/reference/elasticsearch/configuration-reference/security-settings.md

@@ -560,11 +560,11 @@ Refer to [*Semantic re-ranking*](docs-content://solutions/search/ranking/semanti
 
 ### Prerequisites [_prerequisites_15]
 
-To use `text_similarity_reranker`, you can rely on the preconfigured `.rerank-v1-elasticsearch` inference endpoint, which is based on [Elastic Rerank](https://www.elastic.co/guide/en/machine-learning/current/ml-nlp-rerank.html) and serves as the default if no `inference_id` is provided. This model is optimized for reranking based on text similarity. If you'd like to use a different model, you can set up a custom inference endpoint for the `rerank` task using the [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put). The endpoint should be configured with a machine learning model capable of computing text similarity. Refer to [the Elastic NLP model reference](docs-content://explore-analyze/machine-learning/nlp/ml-nlp-model-ref.md#ml-nlp-model-ref-text-similarity) for a list of third-party text similarity models supported by {{es}}.
+To use `text_similarity_reranker`, you can rely on the preconfigured `.rerank-v1-elasticsearch` inference endpoint, which uses the [Elastic Rerank model](docs-content://explore-analyze/machine-learning/nlp/ml-nlp-rerank.md) and serves as the default if no `inference_id` is provided. This model is optimized for reranking based on text similarity. If you'd like to use a different model, you can set up a custom inference endpoint for the `rerank` task using the [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put). The endpoint should be configured with a machine learning model capable of computing text similarity. Refer to [the Elastic NLP model reference](docs-content://explore-analyze/machine-learning/nlp/ml-nlp-model-ref.md#ml-nlp-model-ref-text-similarity) for a list of third-party text similarity models supported by {{es}}.
 
 You have the following options:
 
-* Use the built-in [Elastic Rerank](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) cross-encoder model via the inference API’s {{es}} service. For an example of creating an endpoint using the Elastic Rerank model, refer to [this guide](https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-elasticsearch.html#inference-example-elastic-reranker).
+* Use the built-in [Elastic Rerank](docs-content://explore-analyze/machine-learning/nlp/ml-nlp-rerank.md) cross-encoder model via the inference API’s {{es}} service. See [this example](https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-service-elasticsearch.html#inference-example-elastic-reranker) for creating an endpoint using the Elastic Rerank model.
 * Use the [Cohere Rerank inference endpoint](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) with the `rerank` task type.
 * Use the [Google Vertex AI inference endpoint](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) with the `rerank` task type.
 * Upload a model to {{es}} with [Eland](eland://reference/machine-learning.md#ml-nlp-pytorch) using the `text_similarity` NLP task type.