Merge branch 'main' into active_shards_percent_issue

Author: DaveCTurner

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java

@@ -41,8 +41,9 @@
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.LongVector;
 import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.lucene.AlwaysReferencedIndexedByShardId;
+import org.elasticsearch.compute.lucene.IndexedByShardIdFromSingleton;
 import org.elasticsearch.compute.lucene.LuceneSourceOperator;
-import org.elasticsearch.compute.lucene.ShardRefCounted;
 import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperator;
 import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperatorStatus;
 import org.elasticsearch.compute.operator.topn.TopNOperator;
@@ -368,7 +369,7 @@ public void benchmark() {
             blockFactory,
             ByteSizeValue.ofMb(1).getBytes(),
             fields(name),
-            List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> {
+            new IndexedByShardIdFromSingleton<>(new ValuesSourceReaderOperator.ShardContext(reader, () -> {
                 throw new UnsupportedOperationException("can't load _source here");
             }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))),
             0
@@ -538,7 +539,7 @@ private void setupPages() {
                         pages.add(
                             new Page(
                                 new DocVector(
-                                    ShardRefCounted.ALWAYS_REFERENCED,
+                                    AlwaysReferencedIndexedByShardId.INSTANCE,
                                     blockFactory.newConstantIntBlockWith(0, end - begin).asVector(),
                                     blockFactory.newConstantIntBlockWith(ctx.ord, end - begin).asVector(),
                                     docs.build(),
@@ -575,8 +576,7 @@ record ItrAndOrd(PrimitiveIterator.OfInt itr, int ord) {}
                             pages.add(
                                 new Page(
                                     new DocVector(
-
-                                        ShardRefCounted.ALWAYS_REFERENCED,
+                                        AlwaysReferencedIndexedByShardId.INSTANCE,
                                         blockFactory.newConstantIntVector(0, size),
                                         leafs.build(),
                                         docs.build(),
@@ -594,7 +594,7 @@ record ItrAndOrd(PrimitiveIterator.OfInt itr, int ord) {}
                     pages.add(
                         new Page(
                             new DocVector(
-                                ShardRefCounted.ALWAYS_REFERENCED,
+                                AlwaysReferencedIndexedByShardId.INSTANCE,
                                 blockFactory.newConstantIntBlockWith(0, size).asVector(),
                                 leafs.build().asBlock().asVector(),
                                 docs.build(),
@@ -621,8 +621,7 @@ record ItrAndOrd(PrimitiveIterator.OfInt itr, int ord) {}
                         pages.add(
                             new Page(
                                 new DocVector(
-
-                                    ShardRefCounted.ALWAYS_REFERENCED,
+                                    AlwaysReferencedIndexedByShardId.INSTANCE,
                                     blockFactory.newConstantIntVector(0, 1),
                                     blockFactory.newConstantIntVector(next.ord, 1),
                                     blockFactory.newConstantIntVector(next.itr.nextInt(), 1),

benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java

@@ -15,7 +15,6 @@
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
@@ -29,6 +28,7 @@
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.SortedNumericLongValues;
 import org.elasticsearch.index.mapper.IndexType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MappingLookup;
@@ -179,14 +179,14 @@ private ScoreScript.Factory bareMetalScript() {
             return new ScoreScript.LeafFactory() {
                 @Override
                 public ScoreScript newInstance(DocReader docReader) throws IOException {
-                    SortedNumericDocValues values = ifd.load(((DocValuesDocReader) docReader).getLeafReaderContext()).getLongValues();
+                    SortedNumericLongValues values = ifd.load(((DocValuesDocReader) docReader).getLeafReaderContext()).getLongValues();
                     return new ScoreScript(params, null, docReader) {
                         private int docId;
 
                         @Override
                         public double execute(ExplanationHolder explanation) {
                             try {
-                                values.advance(docId);
+                                values.advanceExact(docId);
                                 if (values.docValueCount() != 1) {
                                     throw new IllegalArgumentException("script only works when there is exactly one value");
                                 }

docs/changelog/132757.yaml

@@ -0,0 +1,5 @@
+pr: 132757
+summary: Late materialization after TopN (Node level)
+area: ES|QL
+type: feature
+issues: []

docs/changelog/136265.yaml

@@ -0,0 +1,5 @@
+pr: 136265
+summary: Adding `match_only_text` subfield to `*.display_name` fields in `ecs@mappings` to be compliant with the latest additions in ECS
+area: Data streams
+type: feature
+issues: []

docs/reference/elasticsearch/rest-apis/retrievers/retrievers-examples.md

@@ -440,7 +440,7 @@ GET /retrievers_example/_search
             "query": "artificial intelligence"
         }
     }
-}     
+}
 ```
 
 This returns the following response based on the final rrf score for each result.
@@ -497,7 +497,7 @@ GET /retrievers_example/_search
             "fields": ["text", "text_semantic"]
         }
     }
-}     
+}
 ```
 
 ::::{note}
@@ -570,7 +570,7 @@ GET /retrievers_example/_search
             "normalizer": "minmax"
         }
     }
-}     
+}
 ```
 
 This returns the following response based on the normalized score for each result:
@@ -1503,6 +1503,7 @@ PUT _inference/rerank/my-rerank-model
 ```
 
 Let’s start by reranking the results of the `rrf` retriever in our previous example.
+We'll also apply a `chunk_rescorer` to ensure that we only consider the best scoring chunks when sending information to the reranker.
 
 ```console
 GET retrievers_example/_search
@@ -1541,7 +1542,15 @@ GET retrievers_example/_search
             },
             "field": "text",
             "inference_id": "my-rerank-model",
-            "inference_text": "What are the state of the art applications of AI in information retrieval?"
+            "inference_text": "What are the state of the art applications of AI in information retrieval?",
+            "chunk_rescorer": {
+                "size": 1,
+                "chunking_settings": {
+                    "strategy": "sentence",
+                    "max_chunk_size": 300,
+                    "sentence_overlap": 0
+                }
+            },
         }
     },
     "_source": false

docs/reference/elasticsearch/rest-apis/retrievers/text-similarity-reranker-retriever.md

@@ -86,6 +86,25 @@ score = ln(score), if score < 0
 
     Applies the specified [boolean query filter](/reference/query-languages/query-dsl/query-dsl-bool-query.md) to the child  `retriever`. If the child retriever already specifies any filters, then this top-level filter is applied in conjuction with the filter defined in the child retriever.
 
+`chunk_rescorer` {applies_to}`stack: beta 9.2`
+:   (Optional, `object`)
+
+    Chunks and scores documents based on configured chunking settings, and only sends the best scoring chunks to the reranking model as input. This helps improve relevance when reranking long documents that would otherwise be truncated by the reranking model's token limit.
+
+    Parameters for `chunk_rescorer`:
+
+    `size`
+    :   (Optional, `int`)
+
+    The number of chunks to pass to the reranker. Defaults to `1`.
+
+    `chunking_settings`
+    :   (Optional, `object`)
+
+    Settings for chunking text into smaller passages for scoring and reranking. Defaults to the optimal chunking settings for [Elastic Rerank](docs-content:///explore-analyze/machine-learning/nlp/ml-nlp-rerank.md). Refer to the [Inference API documentation](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put#operation-inference-put-body-application-json-chunking_settings) for valid values for `chunking_settings`. 
+    :::{warning} 
+    If you configure chunks larger than the reranker's token limit, the results may be truncated. This can degrade relevance significantly.
+    :::
 
 
 ## Example: Elastic Rerank [text-similarity-reranker-retriever-example-elastic-rerank]

modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregator.java

@@ -8,9 +8,7 @@
  */
 package org.elasticsearch.aggregations.bucket.histogram;
 
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.LongValues;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder.RoundingInfo;
@@ -19,6 +17,7 @@
 import org.elasticsearch.common.util.IntArray;
 import org.elasticsearch.common.util.LongArray;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.index.fielddata.SortedNumericLongValues;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationExecutionContext;
 import org.elasticsearch.search.aggregations.Aggregator;
@@ -125,17 +124,17 @@ public final DeferringBucketCollector buildDeferringCollector() {
         return deferringCollector;
     }
 
-    protected abstract LeafBucketCollector getLeafCollector(SortedNumericDocValues values, LeafBucketCollector sub) throws IOException;
+    protected abstract LeafBucketCollector getLeafCollector(SortedNumericLongValues values, LeafBucketCollector sub) throws IOException;
 
-    protected abstract LeafBucketCollector getLeafCollector(NumericDocValues values, LeafBucketCollector sub) throws IOException;
+    protected abstract LeafBucketCollector getLeafCollector(LongValues values, LeafBucketCollector sub) throws IOException;
 
     @Override
     public final LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, LeafBucketCollector sub) throws IOException {
         if (valuesSource == null) {
             return LeafBucketCollector.NO_OP_COLLECTOR;
         }
-        final SortedNumericDocValues values = valuesSource.longValues(aggCtx.getLeafReaderContext());
-        final NumericDocValues singleton = DocValues.unwrapSingleton(values);
+        final SortedNumericLongValues values = valuesSource.longValues(aggCtx.getLeafReaderContext());
+        final LongValues singleton = SortedNumericLongValues.unwrapSingleton(values);
         return singleton != null ? getLeafCollector(singleton, sub) : getLeafCollector(values, sub);
     }
 
@@ -239,7 +238,7 @@ private static class FromSingle extends AutoDateHistogramAggregator {
         }
 
         @Override
-        protected LeafBucketCollector getLeafCollector(SortedNumericDocValues values, LeafBucketCollector sub) {
+        protected LeafBucketCollector getLeafCollector(SortedNumericLongValues values, LeafBucketCollector sub) {
             return new LeafBucketCollectorBase(sub, values) {
                 @Override
                 public void collect(int doc, long owningBucketOrd) throws IOException {
@@ -265,7 +264,7 @@ public void collect(int doc, long owningBucketOrd) throws IOException {
         }
 
         @Override
-        protected LeafBucketCollector getLeafCollector(NumericDocValues values, LeafBucketCollector sub) {
+        protected LeafBucketCollector getLeafCollector(LongValues values, LeafBucketCollector sub) {
             return new LeafBucketCollectorBase(sub, values) {
                 @Override
                 public void collect(int doc, long owningBucketOrd) throws IOException {
@@ -461,7 +460,7 @@ private static class FromMany extends AutoDateHistogramAggregator {
         }
 
         @Override
-        protected LeafBucketCollector getLeafCollector(SortedNumericDocValues values, LeafBucketCollector sub) {
+        protected LeafBucketCollector getLeafCollector(SortedNumericLongValues values, LeafBucketCollector sub) {
             return new LeafBucketCollectorBase(sub, values) {
                 @Override
                 public void collect(int doc, long owningBucketOrd) throws IOException {
@@ -487,7 +486,7 @@ public void collect(int doc, long owningBucketOrd) throws IOException {
         }
 
         @Override
-        protected LeafBucketCollector getLeafCollector(NumericDocValues values, LeafBucketCollector sub) {
+        protected LeafBucketCollector getLeafCollector(LongValues values, LeafBucketCollector sub) {
             return new LeafBucketCollectorBase(sub, values) {
                 @Override
                 public void collect(int doc, long owningBucketOrd) throws IOException {

modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java

@@ -9,12 +9,12 @@
 
 package org.elasticsearch.aggregations.bucket.timeseries;
 
-import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.util.LongArray;
 import org.elasticsearch.common.util.ObjectArray;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+import org.elasticsearch.index.fielddata.SortedNumericLongValues;
 import org.elasticsearch.index.mapper.RoutingPathFields;
 import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
 import org.elasticsearch.search.aggregations.AggregationExecutionContext;
@@ -121,7 +121,7 @@ protected LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCt
         for (var entry : dimensionValueSources.entrySet()) {
             String fieldName = entry.getKey();
             if (entry.getValue() instanceof ValuesSource.Numeric numericVS) {
-                SortedNumericDocValues docValues = numericVS.longValues(aggCtx.getLeafReaderContext());
+                SortedNumericLongValues docValues = numericVS.longValues(aggCtx.getLeafReaderContext());
                 dimensionConsumers.put(entry.getKey(), (docId, tsidBuilder) -> {
                     if (docValues.advanceExact(docId)) {
                         assert docValues.docValueCount() == 1 : "Dimension field cannot be a multi-valued field";

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java

@@ -9,10 +9,8 @@
 
 package org.elasticsearch.index.mapper.extras;
 
-import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.LongValues;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.settings.Setting;
@@ -28,6 +26,7 @@
 import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.index.fielddata.SortedNumericLongValues;
 import org.elasticsearch.index.fielddata.SourceValueFetcherSortedDoubleIndexFieldData;
 import org.elasticsearch.index.fielddata.plain.LeafDoubleFieldData;
 import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
@@ -831,8 +830,8 @@ public void close() {
 
         @Override
         public SortedNumericDoubleValues getDoubleValues() {
-            final SortedNumericDocValues values = scaledFieldData.getLongValues();
-            final NumericDocValues singleValues = DocValues.unwrapSingleton(values);
+            final SortedNumericLongValues values = scaledFieldData.getLongValues();
+            final LongValues singleValues = SortedNumericLongValues.unwrapSingleton(values);
             if (singleValues != null) {
                 return FieldData.singleton(new NumericDoubleValues() {
                     @Override

muted-tests.yml

@@ -591,9 +591,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
   method: test {csv-spec:union_types.MultiIndexTsLongStatsDrop}
   issue: https://github.com/elastic/elasticsearch/issues/136113
-- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
-  method: test {csv-spec:lookup-join-expression.LookupMultiColTwoExprAndNoMatch}
-  issue: https://github.com/elastic/elasticsearch/issues/136121
 - class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
   method: test {csv-spec:bucket.BucketByWeekInString}
   issue: https://github.com/elastic/elasticsearch/issues/136136