Merge branch 'main' into indexLike_final

Author: julian-elastic

docs/changelog/130705.yaml

@@ -0,0 +1,5 @@
+pr: 130705
+summary: Fix `BytesRef2BlockHash`
+area: ES|QL
+type: bug
+issues: []

docs/changelog/130834.yaml

@@ -0,0 +1,5 @@
+pr: 130834
+summary: Ensure vectors are always included in reindex actions
+area: Vector Search
+type: enhancement
+issues: []

docs/reference/elasticsearch/mapping-reference/sparse-vector.md

@@ -83,7 +83,7 @@ Parameters for `index_options` are:
     :   (Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned. This value must be between 0 and 1. Default: `0.4`.
 
     ::::{note}
-    The default values for `tokens_freq_ratio_threshold` and `tokens_weight_threshold` were chosen based on tests using ELSERv2 that provided the most optimal results.
+    The default values for `tokens_freq_ratio_threshold` and `tokens_weight_threshold` were chosen based on tests using ELSERv2 that provided the optimal results.
     ::::
 
 When token pruning is applied, non-significant tokens will be pruned from the query.

docs/reference/elasticsearch/rest-apis/retrievers.md

@@ -1200,8 +1200,8 @@ Note, however, that wildcard field patterns will only resolve to fields that eit
 
 ### Examples
 
-<!-- - [RRF with the multi-field query format](docs-content://solutions/search/retrievers-examples.md#retrievers-examples-rrf-multi-field-query-format) -->
-<!-- - [Linear retriever with the multi-field query format](docs-content://solutions/search/retrievers-examples.md#retrievers-examples-linear-multi-field-query-format) -->
+- [RRF with the multi-field query format](docs-content://solutions/search/retrievers-examples.md#retrievers-examples-rrf-multi-field-query-format)
+- [Linear retriever with the multi-field query format](docs-content://solutions/search/retrievers-examples.md#retrievers-examples-linear-multi-field-query-format)
 
 ## Common usage guidelines [retriever-common-parameters]
 

docs/reference/query-languages/query-dsl/query-dsl-sparse-vector-query.md

@@ -80,7 +80,7 @@ GET _search
     :   (Optional, boolean) If `true` we only input pruned tokens into scoring, and discard non-pruned tokens. It is strongly recommended to set this to `false` for the main query, but this can be set to `true` for a rescore query to get more relevant results. Default: `false`.
 
     ::::{note}
-    The default values for `tokens_freq_ratio_threshold` and `tokens_weight_threshold` were chosen based on tests using ELSERv2 that provided the most optimal results.
+    The default values for `tokens_freq_ratio_threshold` and `tokens_weight_threshold` were chosen based on tests using ELSERv2 that provided the optimal results.
     ::::
 
 When token pruning is applied, non-significant tokens will be pruned from the query.

docs/reference/query-languages/query-dsl/query-dsl-text-expansion-query.md

@@ -69,7 +69,7 @@ GET _search
     :   (Optional, boolean) [preview] If `true` we only input pruned tokens into scoring, and discard non-pruned tokens. It is strongly recommended to set this to `false` for the main query, but this can be set to `true` for a rescore query to get more relevant results. Default: `false`.
 
     ::::{note}
-    The default values for `tokens_freq_ratio_threshold` and `tokens_weight_threshold` were chosen based on tests using ELSER that provided the most optimal results.
+    The default values for `tokens_freq_ratio_threshold` and `tokens_weight_threshold` were chosen based on tests using ELSER that provided the optimal results.
     ::::
 
 

docs/reference/query-languages/query-dsl/query-dsl-weighted-tokens-query.md

@@ -66,7 +66,7 @@ POST _search
     :   (Optional, boolean) If `true` we only input pruned tokens into scoring, and discard non-pruned tokens. It is strongly recommended to set this to `false` for the main query, but this can be set to `true` for a rescore query to get more relevant results. Default: `false`.
 
     ::::{note}
-    The default values for `tokens_freq_ratio_threshold` and `tokens_weight_threshold` were chosen based on tests using ELSER that provided the most optimal results.
+    The default values for `tokens_freq_ratio_threshold` and `tokens_weight_threshold` were chosen based on tests using ELSER that provided the optimal results.
     ::::
 
 

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -14,7 +14,6 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.StoredField;
-import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
@@ -48,6 +47,7 @@
 import org.elasticsearch.index.mapper.BlockStoredFieldsReader;
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.SourceValueFetcher;
 import org.elasticsearch.index.mapper.StringFieldType;
@@ -254,7 +254,8 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
                 if (parent.isStored()) {
                     return storedFieldFetcher(parentField);
                 } else if (parent.hasDocValues()) {
-                    return docValuesFieldFetcher(parentField);
+                    var ifd = searchExecutionContext.getForField(parent, MappedFieldType.FielddataOperation.SEARCH);
+                    return docValuesFieldFetcher(ifd);
                 } else {
                     assert false : "parent field should either be stored or have doc values";
                 }
@@ -266,7 +267,8 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
                     if (fieldType.isStored()) {
                         return storedFieldFetcher(fieldType.name());
                     } else if (fieldType.hasDocValues()) {
-                        return docValuesFieldFetcher(fieldType.name());
+                        var ifd = searchExecutionContext.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH);
+                        return docValuesFieldFetcher(ifd);
                     } else {
                         assert false : "multi field should either be stored or have doc values";
                     }
@@ -291,15 +293,16 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
             };
         }
 
-        private static IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOException>> docValuesFieldFetcher(String name) {
+        private static IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOException>> docValuesFieldFetcher(
+            IndexFieldData<?> ifd
+        ) {
             return context -> {
-                var sortedDocValues = DocValues.getSortedSet(context.reader(), name);
+                var sortedBinaryDocValues = ifd.load(context).getBytesValues();
                 return docId -> {
-                    if (sortedDocValues.advanceExact(docId)) {
-                        var values = new ArrayList<>(sortedDocValues.docValueCount());
-                        for (int i = 0; i < sortedDocValues.docValueCount(); i++) {
-                            long ord = sortedDocValues.nextOrd();
-                            values.add(sortedDocValues.lookupOrd(ord).utf8ToString());
+                    if (sortedBinaryDocValues.advanceExact(docId)) {
+                        var values = new ArrayList<>(sortedBinaryDocValues.docValueCount());
+                        for (int i = 0; i < sortedBinaryDocValues.docValueCount(); i++) {
+                            values.add(sortedBinaryDocValues.nextValue().utf8ToString());
                         }
                         return values;
                     } else {

modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractAsyncBulkByScrollAction.java

@@ -44,6 +44,7 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -119,6 +120,7 @@ public abstract class AbstractAsyncBulkByScrollAction<
         BulkByScrollTask task,
         boolean needsSourceDocumentVersions,
         boolean needsSourceDocumentSeqNoAndPrimaryTerm,
+        boolean needsVectors,
         Logger logger,
         ParentTaskAssigningClient client,
         ThreadPool threadPool,
@@ -131,6 +133,7 @@ public abstract class AbstractAsyncBulkByScrollAction<
             task,
             needsSourceDocumentVersions,
             needsSourceDocumentSeqNoAndPrimaryTerm,
+            needsVectors,
             logger,
             client,
             client,
@@ -146,6 +149,7 @@ public abstract class AbstractAsyncBulkByScrollAction<
         BulkByScrollTask task,
         boolean needsSourceDocumentVersions,
         boolean needsSourceDocumentSeqNoAndPrimaryTerm,
+        boolean needsVectors,
         Logger logger,
         ParentTaskAssigningClient searchClient,
         ParentTaskAssigningClient bulkClient,
@@ -173,7 +177,7 @@ public abstract class AbstractAsyncBulkByScrollAction<
         bulkRetry = new Retry(BackoffPolicy.wrap(backoffPolicy, worker::countBulkRetry), threadPool);
         scrollSource = buildScrollableResultSource(
             backoffPolicy,
-            prepareSearchRequest(mainRequest, needsSourceDocumentVersions, needsSourceDocumentSeqNoAndPrimaryTerm)
+            prepareSearchRequest(mainRequest, needsSourceDocumentVersions, needsSourceDocumentSeqNoAndPrimaryTerm, needsVectors)
         );
         scriptApplier = Objects.requireNonNull(buildScriptApplier(), "script applier must not be null");
     }
@@ -186,7 +190,8 @@ public abstract class AbstractAsyncBulkByScrollAction<
     static <Request extends AbstractBulkByScrollRequest<Request>> SearchRequest prepareSearchRequest(
         Request mainRequest,
         boolean needsSourceDocumentVersions,
-        boolean needsSourceDocumentSeqNoAndPrimaryTerm
+        boolean needsSourceDocumentSeqNoAndPrimaryTerm,
+        boolean needsVectors
     ) {
         var preparedSearchRequest = new SearchRequest(mainRequest.getSearchRequest());
 
@@ -205,6 +210,16 @@ static <Request extends AbstractBulkByScrollRequest<Request>> SearchRequest prep
         sourceBuilder.version(needsSourceDocumentVersions);
         sourceBuilder.seqNoAndPrimaryTerm(needsSourceDocumentSeqNoAndPrimaryTerm);
 
+        if (needsVectors) {
+            // always include vectors in the response unless explicitly set
+            var fetchSource = sourceBuilder.fetchSource();
+            if (fetchSource == null) {
+                sourceBuilder.fetchSource(FetchSourceContext.FETCH_ALL_SOURCE);
+            } else if (fetchSource.excludeVectors() == null) {
+                sourceBuilder.excludeVectors(false);
+            }
+        }
+
         /*
          * Do not open scroll if max docs <= scroll size and not resuming on version conflicts
          */

modules/reindex/src/main/java/org/elasticsearch/reindex/AsyncDeleteByQueryAction.java

@@ -34,7 +34,7 @@ public AsyncDeleteByQueryAction(
         ScriptService scriptService,
         ActionListener<BulkByScrollResponse> listener
     ) {
-        super(task, false, true, logger, client, threadPool, request, listener, scriptService, null);
+        super(task, false, true, false, logger, client, threadPool, request, listener, scriptService, null);
     }
 
     @Override