Merge branch 'main' into 2025/04/02/S3SearchableSnapshotsCredentialsReloadIT-fips

Author: DaveCTurner

build-conventions/build.gradle

@@ -77,7 +77,7 @@ dependencies {
     api buildLibs.apache.rat
     compileOnly buildLibs.checkstyle
     constraints {
-      api("org.eclipse.platform:org.eclipse.osgi:3.18.500") {
+      api("org.eclipse.platform:org.eclipse.osgi:3.18.300") {
         because("Use the same version as we do in spotless gradle plugin at runtime")
       }
     }

docs/changelog/125764.yaml

@@ -0,0 +1,8 @@
+pr: 125764
+summary: Fix `ReplaceMissingFieldsWithNull`
+area: ES|QL
+type: bug
+issues:
+ - 126036
+ - 121754
+ - 126030

docs/changelog/126038.yaml

@@ -0,0 +1,5 @@
+pr: 126038
+summary: Mark `rescore_vector` as generally available
+area: Vector Search
+type: enhancement
+issues: []

docs/reference/elasticsearch/mapping-reference/dense-vector.md

@@ -288,7 +288,7 @@ $$$dense-vector-index-options$$$
     :   (Optional, float) Only applicable to `int8_hnsw`, `int4_hnsw`, `int8_flat`, and `int4_flat` index types. The confidence interval to use when quantizing the vectors. Can be any value between and including `0.90` and `1.0` or exactly `0`. When the value is `0`, this indicates that dynamic quantiles should be calculated for optimized quantization. When between `0.90` and `1.0`, this value restricts the values used when calculating the quantization thresholds. For example, a value of `0.95` will only use the middle 95% of the values when calculating the quantization thresholds (e.g. the highest and lowest 2.5% of values will be ignored). Defaults to `1/(dims + 1)` for `int8` quantized vectors and `0` for `int4` for dynamic quantile calculation.
 
     `rescore_vector`
-    :   (Optional, object) Functionality in [preview]. An optional section that configures automatic vector rescoring on knn queries for the given field. Only applicable to quantized index types.
+    :   (Optional, object) An optional section that configures automatic vector rescoring on knn queries for the given field. Only applicable to quantized index types.
     :::::{dropdown} Properties of `rescore_vector`
     `oversample`
     :   (required, float) The amount to oversample the search results by. This value should be greater than `1.0` and less than `10.0` or exactly `0` to indicate no oversampling & rescoring should occur. The higher the value, the more vectors will be gathered and rescored with the raw values per shard.

docs/reference/elasticsearch/rest-apis/retrievers.md

@@ -198,7 +198,7 @@ A kNN retriever returns top documents from a [k-nearest neighbor search (kNN)](d
 
 
 `rescore_vector`
-:   (Optional, object) Functionality in [preview]. Apply oversampling and rescoring to quantized vectors.
+:   (Optional, object) Apply oversampling and rescoring to quantized vectors.
 
 ::::{note}
 Rescoring only makes sense for quantized vectors; when [quantization](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-quantization) is not used, the original vectors are used for scoring. Rescore option will be ignored for non-quantized `dense_vector` fields.

docs/reference/query-languages/query-dsl/query-dsl-knn-query.md

@@ -98,7 +98,7 @@ The filter is a pre-filter, meaning that it is applied **during** the approximat
 
 
 `rescore_vector`
-:   (Optional, object) Functionality in [preview]. Apply oversampling and rescoring to quantized vectors.
+:   (Optional, object) Apply oversampling and rescoring to quantized vectors.
 
 ::::{note}
 Rescoring only makes sense for quantized vectors; when [quantization](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-quantization) is not used, the original vectors are used for scoring. Rescore option will be ignored for non-quantized `dense_vector` fields.

gradle/build.versions.toml

@@ -43,7 +43,7 @@ snakeyaml = { group = "org.yaml", name = "snakeyaml", version = { strictly = "2.
 spock-core = { group = "org.spockframework", name="spock-core", version.ref="spock" }
 spock-junit4 = { group = "org.spockframework", name="spock-junit4", version.ref="spock" }
 spock-platform = { group = "org.spockframework", name="spock-bom", version.ref="spock" }
-spotless-plugin = "com.diffplug.spotless:spotless-plugin-gradle:7.0.2"
+spotless-plugin = "com.diffplug.spotless:spotless-plugin-gradle:6.25.0"
 wiremock = "com.github.tomakehurst:wiremock-jre8-standalone:2.23.2"
 xmlunit-core = "org.xmlunit:xmlunit-core:2.8.2"
 

gradle/verification-metadata.xml

@@ -308,9 +308,6 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test012SecurityCanBeDisabled
   issue: https://github.com/elastic/elasticsearch/issues/116636
-- class: org.elasticsearch.index.engine.ThreadPoolMergeSchedulerTests
-  method: testSchedulerCloseWaitsForRunningMerge
-  issue: https://github.com/elastic/elasticsearch/issues/125236
 - class: org.elasticsearch.index.shard.StoreRecoveryTests
   method: testAddIndices
   issue: https://github.com/elastic/elasticsearch/issues/124104

muted-tests.yml

@@ -508,18 +508,21 @@ public void testSchedulerCloseWaitsForRunningMerge() throws Exception {
                 )
             ) {
                 CountDownLatch mergeDoneLatch = new CountDownLatch(1);
+                CountDownLatch mergeRunningLatch = new CountDownLatch(1);
                 MergeSource mergeSource = mock(MergeSource.class);
                 OneMerge oneMerge = mock(OneMerge.class);
                 when(oneMerge.getStoreMergeInfo()).thenReturn(getNewMergeInfo(randomLongBetween(1L, 10L)));
                 when(oneMerge.getMergeProgress()).thenReturn(new MergePolicy.OneMergeProgress());
                 when(mergeSource.getNextMerge()).thenReturn(oneMerge, (OneMerge) null);
                 doAnswer(invocation -> {
+                    mergeRunningLatch.countDown();
                     OneMerge merge = (OneMerge) invocation.getArguments()[0];
                     assertFalse(merge.isAborted());
                     // wait to be signalled before completing the merge
                     mergeDoneLatch.await();
                     return null;
                 }).when(mergeSource).merge(any(OneMerge.class));
+                // submit the merge
                 threadPoolMergeScheduler.merge(mergeSource, randomFrom(MergeTrigger.values()));
                 Thread t = new Thread(() -> {
                     try {
@@ -531,6 +534,8 @@ public void testSchedulerCloseWaitsForRunningMerge() throws Exception {
                 t.start();
                 try {
                     assertTrue(t.isAlive());
+                    // wait for the merge to actually run
+                    mergeRunningLatch.await();
                     // ensure the merge scheduler is effectively "closed"
                     assertBusy(() -> {
                         MergeSource mergeSource2 = mock(MergeSource.class);