Merge remote-tracking branch 'upstream/main' into gcp-metrics-threadlocal

Author: mhl-b

build-conventions/build.gradle

@@ -77,7 +77,7 @@ dependencies {
     api buildLibs.apache.rat
     compileOnly buildLibs.checkstyle
     constraints {
-      api("org.eclipse.platform:org.eclipse.osgi:3.18.500") {
+      api("org.eclipse.platform:org.eclipse.osgi:3.18.300") {
         because("Use the same version as we do in spotless gradle plugin at runtime")
       }
     }

docs/changelog/126038.yaml

@@ -0,0 +1,5 @@
+pr: 126038
+summary: Mark `rescore_vector` as generally available
+area: Vector Search
+type: enhancement
+issues: []

docs/reference/elasticsearch/mapping-reference/dense-vector.md

@@ -288,7 +288,7 @@ $$$dense-vector-index-options$$$
     :   (Optional, float) Only applicable to `int8_hnsw`, `int4_hnsw`, `int8_flat`, and `int4_flat` index types. The confidence interval to use when quantizing the vectors. Can be any value between and including `0.90` and `1.0` or exactly `0`. When the value is `0`, this indicates that dynamic quantiles should be calculated for optimized quantization. When between `0.90` and `1.0`, this value restricts the values used when calculating the quantization thresholds. For example, a value of `0.95` will only use the middle 95% of the values when calculating the quantization thresholds (e.g. the highest and lowest 2.5% of values will be ignored). Defaults to `1/(dims + 1)` for `int8` quantized vectors and `0` for `int4` for dynamic quantile calculation.
 
     `rescore_vector`
-    :   (Optional, object) Functionality in [preview]. An optional section that configures automatic vector rescoring on knn queries for the given field. Only applicable to quantized index types.
+    :   (Optional, object) An optional section that configures automatic vector rescoring on knn queries for the given field. Only applicable to quantized index types.
     :::::{dropdown} Properties of `rescore_vector`
     `oversample`
     :   (required, float) The amount to oversample the search results by. This value should be greater than `1.0` and less than `10.0` or exactly `0` to indicate no oversampling & rescoring should occur. The higher the value, the more vectors will be gathered and rescored with the raw values per shard.

docs/reference/elasticsearch/rest-apis/retrievers.md

@@ -198,7 +198,7 @@ A kNN retriever returns top documents from a [k-nearest neighbor search (kNN)](d
 
 
 `rescore_vector`
-:   (Optional, object) Functionality in [preview]. Apply oversampling and rescoring to quantized vectors.
+:   (Optional, object) Apply oversampling and rescoring to quantized vectors.
 
 ::::{note}
 Rescoring only makes sense for quantized vectors; when [quantization](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-quantization) is not used, the original vectors are used for scoring. Rescore option will be ignored for non-quantized `dense_vector` fields.

docs/reference/query-languages/query-dsl/query-dsl-knn-query.md

@@ -98,7 +98,7 @@ The filter is a pre-filter, meaning that it is applied **during** the approximat
 
 
 `rescore_vector`
-:   (Optional, object) Functionality in [preview]. Apply oversampling and rescoring to quantized vectors.
+:   (Optional, object) Apply oversampling and rescoring to quantized vectors.
 
 ::::{note}
 Rescoring only makes sense for quantized vectors; when [quantization](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-quantization) is not used, the original vectors are used for scoring. Rescore option will be ignored for non-quantized `dense_vector` fields.

gradle/build.versions.toml

@@ -43,7 +43,7 @@ snakeyaml = { group = "org.yaml", name = "snakeyaml", version = { strictly = "2.
 spock-core = { group = "org.spockframework", name="spock-core", version.ref="spock" }
 spock-junit4 = { group = "org.spockframework", name="spock-junit4", version.ref="spock" }
 spock-platform = { group = "org.spockframework", name="spock-bom", version.ref="spock" }
-spotless-plugin = "com.diffplug.spotless:spotless-plugin-gradle:7.0.2"
+spotless-plugin = "com.diffplug.spotless:spotless-plugin-gradle:6.25.0"
 wiremock = "com.github.tomakehurst:wiremock-jre8-standalone:2.23.2"
 xmlunit-core = "org.xmlunit:xmlunit-core:2.8.2"
 

gradle/verification-metadata.xml

@@ -380,15 +380,9 @@ tests:
 - class: org.elasticsearch.search.CCSDuelIT
   method: testTerminateAfter
   issue: https://github.com/elastic/elasticsearch/issues/126085
-- class: org.elasticsearch.index.mapper.blockloader.KeywordFieldBlockLoaderTests
-  method: testBlockLoaderForFieldInObject {preference=Params[syntheticSource=false, preference=DOC_VALUES]}
-  issue: https://github.com/elastic/elasticsearch/issues/126088
 - class: org.elasticsearch.search.sort.GeoDistanceIT
   method: testDistanceSortingWithUnmappedField
   issue: https://github.com/elastic/elasticsearch/issues/126118
-- class: org.elasticsearch.index.mapper.blockloader.KeywordFieldBlockLoaderTests
-  method: testBlockLoader {preference=Params[syntheticSource=false, preference=DOC_VALUES]}
-  issue: https://github.com/elastic/elasticsearch/issues/126121
 - class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT
   method: testSearchWithRandomDisconnects
   issue: https://github.com/elastic/elasticsearch/issues/122707

muted-tests.yml

@@ -156,6 +156,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion INFERENCE_MODEL_REGISTRY_METADATA_8_19 = def(8_841_0_13);
     public static final TransportVersion INTRODUCE_LIFECYCLE_TEMPLATE_8_19 = def(8_841_0_14);
     public static final TransportVersion RERANK_COMMON_OPTIONS_ADDED_8_19 = def(8_841_0_15);
+    public static final TransportVersion REMOTE_EXCEPTION_8_19 = def(8_841_0_16);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0 = def(9_000_0_00);
     public static final TransportVersion REMOVE_SNAPSHOT_FAILURES_90 = def(9_000_0_01);
     public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED_90 = def(9_000_0_02);

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -46,7 +46,9 @@ protected Object expected(Map<String, Object> fieldMapping, Object value, TestCo
             .filter(Objects::nonNull);
 
         boolean hasDocValues = hasDocValues(fieldMapping, false);
-        boolean useDocValues = params.preference() == MappedFieldType.FieldExtractPreference.NONE || params.syntheticSource();
+        boolean useDocValues = params.preference() == MappedFieldType.FieldExtractPreference.NONE
+            || params.preference() == MappedFieldType.FieldExtractPreference.DOC_VALUES
+            || params.syntheticSource();
         if (hasDocValues && useDocValues) {
             // Sorted and no duplicates
             var resultList = convertValues.andThen(Stream::distinct)