Merge branch 'main' into ES-10037-data-stream-autosharding-extra-logging

Author: joegallo

build-conventions/build.gradle

@@ -77,7 +77,7 @@ dependencies {
     api buildLibs.apache.rat
     compileOnly buildLibs.checkstyle
     constraints {
-      api("org.eclipse.platform:org.eclipse.osgi:3.18.500") {
+      api("org.eclipse.platform:org.eclipse.osgi:3.18.300") {
         because("Use the same version as we do in spotless gradle plugin at runtime")
       }
     }

docs/changelog/126038.yaml

@@ -0,0 +1,5 @@
+pr: 126038
+summary: Mark `rescore_vector` as generally available
+area: Vector Search
+type: enhancement
+issues: []

docs/reference/elasticsearch/mapping-reference/dense-vector.md

@@ -288,7 +288,7 @@ $$$dense-vector-index-options$$$
     :   (Optional, float) Only applicable to `int8_hnsw`, `int4_hnsw`, `int8_flat`, and `int4_flat` index types. The confidence interval to use when quantizing the vectors. Can be any value between and including `0.90` and `1.0` or exactly `0`. When the value is `0`, this indicates that dynamic quantiles should be calculated for optimized quantization. When between `0.90` and `1.0`, this value restricts the values used when calculating the quantization thresholds. For example, a value of `0.95` will only use the middle 95% of the values when calculating the quantization thresholds (e.g. the highest and lowest 2.5% of values will be ignored). Defaults to `1/(dims + 1)` for `int8` quantized vectors and `0` for `int4` for dynamic quantile calculation.
 
     `rescore_vector`
-    :   (Optional, object) Functionality in [preview]. An optional section that configures automatic vector rescoring on knn queries for the given field. Only applicable to quantized index types.
+    :   (Optional, object) An optional section that configures automatic vector rescoring on knn queries for the given field. Only applicable to quantized index types.
     :::::{dropdown} Properties of `rescore_vector`
     `oversample`
     :   (required, float) The amount to oversample the search results by. This value should be greater than `1.0` and less than `10.0` or exactly `0` to indicate no oversampling & rescoring should occur. The higher the value, the more vectors will be gathered and rescored with the raw values per shard.

docs/reference/elasticsearch/rest-apis/retrievers.md

@@ -198,7 +198,7 @@ A kNN retriever returns top documents from a [k-nearest neighbor search (kNN)](d
 
 
 `rescore_vector`
-:   (Optional, object) Functionality in [preview]. Apply oversampling and rescoring to quantized vectors.
+:   (Optional, object) Apply oversampling and rescoring to quantized vectors.
 
 ::::{note}
 Rescoring only makes sense for quantized vectors; when [quantization](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-quantization) is not used, the original vectors are used for scoring. Rescore option will be ignored for non-quantized `dense_vector` fields.

docs/reference/query-languages/query-dsl/query-dsl-knn-query.md

@@ -98,7 +98,7 @@ The filter is a pre-filter, meaning that it is applied **during** the approximat
 
 
 `rescore_vector`
-:   (Optional, object) Functionality in [preview]. Apply oversampling and rescoring to quantized vectors.
+:   (Optional, object) Apply oversampling and rescoring to quantized vectors.
 
 ::::{note}
 Rescoring only makes sense for quantized vectors; when [quantization](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-quantization) is not used, the original vectors are used for scoring. Rescore option will be ignored for non-quantized `dense_vector` fields.

gradle/build.versions.toml

@@ -43,7 +43,7 @@ snakeyaml = { group = "org.yaml", name = "snakeyaml", version = { strictly = "2.
 spock-core = { group = "org.spockframework", name="spock-core", version.ref="spock" }
 spock-junit4 = { group = "org.spockframework", name="spock-junit4", version.ref="spock" }
 spock-platform = { group = "org.spockframework", name="spock-bom", version.ref="spock" }
-spotless-plugin = "com.diffplug.spotless:spotless-plugin-gradle:7.0.2"
+spotless-plugin = "com.diffplug.spotless:spotless-plugin-gradle:6.25.0"
 wiremock = "com.github.tomakehurst:wiremock-jre8-standalone:2.23.2"
 xmlunit-core = "org.xmlunit:xmlunit-core:2.8.2"
 

gradle/verification-metadata.xml

@@ -380,15 +380,9 @@ tests:
 - class: org.elasticsearch.search.CCSDuelIT
   method: testTerminateAfter
   issue: https://github.com/elastic/elasticsearch/issues/126085
-- class: org.elasticsearch.index.mapper.blockloader.KeywordFieldBlockLoaderTests
-  method: testBlockLoaderForFieldInObject {preference=Params[syntheticSource=false, preference=DOC_VALUES]}
-  issue: https://github.com/elastic/elasticsearch/issues/126088
 - class: org.elasticsearch.search.sort.GeoDistanceIT
   method: testDistanceSortingWithUnmappedField
   issue: https://github.com/elastic/elasticsearch/issues/126118
-- class: org.elasticsearch.index.mapper.blockloader.KeywordFieldBlockLoaderTests
-  method: testBlockLoader {preference=Params[syntheticSource=false, preference=DOC_VALUES]}
-  issue: https://github.com/elastic/elasticsearch/issues/126121
 - class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT
   method: testSearchWithRandomDisconnects
   issue: https://github.com/elastic/elasticsearch/issues/122707

muted-tests.yml

@@ -156,6 +156,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion INFERENCE_MODEL_REGISTRY_METADATA_8_19 = def(8_841_0_13);
     public static final TransportVersion INTRODUCE_LIFECYCLE_TEMPLATE_8_19 = def(8_841_0_14);
     public static final TransportVersion RERANK_COMMON_OPTIONS_ADDED_8_19 = def(8_841_0_15);
+    public static final TransportVersion REMOTE_EXCEPTION_8_19 = def(8_841_0_16);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0 = def(9_000_0_00);
     public static final TransportVersion REMOVE_SNAPSHOT_FAILURES_90 = def(9_000_0_01);
     public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED_90 = def(9_000_0_02);

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.common.Strings;
 
 import java.util.Map;
+import java.util.Set;
 
 /**
  * Interface for grouping index expressions, along with IndicesOptions by cluster alias.
@@ -29,23 +30,36 @@
 public interface IndicesExpressionGrouper {
 
     /**
+     * @param remoteClusterNames Set of configured remote cluster names.
      * @param indicesOptions IndicesOptions to clarify how the index expression should be parsed/applied
      * @param indexExpressionCsv Multiple index expressions as CSV string (with no spaces), e.g., "logs1,logs2,cluster-a:logs1".
      *                           A single index expression is also supported.
      * @return Map where the key is the cluster alias (for "local" cluster, it is RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)
      *         and the value for that cluster from the index expression is an OriginalIndices object.
      */
-    default Map<String, OriginalIndices> groupIndices(IndicesOptions indicesOptions, String indexExpressionCsv) {
-        return groupIndices(indicesOptions, Strings.splitStringByCommaToArray(indexExpressionCsv));
+    default Map<String, OriginalIndices> groupIndices(
+        Set<String> remoteClusterNames,
+        IndicesOptions indicesOptions,
+        String indexExpressionCsv
+    ) {
+        return groupIndices(remoteClusterNames, indicesOptions, Strings.splitStringByCommaToArray(indexExpressionCsv));
     }
 
     /**
      * Same behavior as the other groupIndices, except the incoming multiple index expressions must already be
      * parsed into a String array.
+     * @param remoteClusterNames Set of configured remote cluster names.
      * @param indicesOptions IndicesOptions to clarify how the index expressions should be parsed/applied
      * @param indexExpressions Multiple index expressions as string[].
      * @return Map where the key is the cluster alias (for "local" cluster, it is RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)
      *         and the value for that cluster from the index expression is an OriginalIndices object.
      */
-    Map<String, OriginalIndices> groupIndices(IndicesOptions indicesOptions, String[] indexExpressions);
+    Map<String, OriginalIndices> groupIndices(Set<String> remoteClusterNames, IndicesOptions indicesOptions, String[] indexExpressions);
+
+    /**
+     * Returns a set of currently configured remote clusters.
+     */
+    default Set<String> getConfiguredClusters() {
+        return Set.of();
+    }
 }