Merge remote-tracking branch 'upstream/main' into enh/sort_before_ijoin

Author: bpintea

docs/changelog/137395.yaml

@@ -0,0 +1,6 @@
+pr: 137395
+summary: Fix attribute only in full text function not found
+area: ES|QL
+type: bug
+issues:
+ - 137396

docs/changelog/137476.yaml

@@ -0,0 +1,6 @@
+pr: 137476
+summary: Handle missing geotile buckets
+area: Transform
+type: bug
+issues:
+ - 126591

libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java

@@ -21,10 +21,12 @@
 
 package org.elasticsearch.tdigest;
 
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
 public abstract class BigCountTests extends TDigestTestCase {
 
     public void testBigMerge() {
-        try (TDigest digest = createDigest()) {
+        try (TDigest digest = createDigest(100)) {
             for (int i = 0; i < 5; i++) {
                 try (TDigest digestToMerge = getDigest()) {
                     digest.add(digestToMerge);
@@ -35,13 +37,25 @@ public void testBigMerge() {
         }
     }
 
+    /**
+     * Verify that, at a range of compression values, the size of the produced digest is not much larger than 10 times the compression
+     */
+    public void testCompression() {
+        for (int compression : new int[] { 100, 500, 1000, 10000 }) {
+            try (TDigest digest = createDigest(compression)) {
+                addData(digest);
+                assertThat("Compression = " + compression, digest.centroidCount(), lessThanOrEqualTo(compression * 10));
+            }
+        }
+    }
+
     private TDigest getDigest() {
-        TDigest digest = createDigest();
+        TDigest digest = createDigest(100);
         addData(digest);
         return digest;
     }
 
-    public TDigest createDigest() {
+    public TDigest createDigest(int compression) {
         throw new IllegalStateException("Should have over-ridden createDigest");
     }
 

libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java

@@ -23,7 +23,7 @@
 
 public class BigCountTestsMergingDigestTests extends BigCountTests {
     @Override
-    public TDigest createDigest() {
-        return TDigest.createMergingDigest(arrays(), 100);
+    public TDigest createDigest(int compression) {
+        return TDigest.createMergingDigest(arrays(), compression);
     }
 }

libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java

@@ -23,7 +23,7 @@
 
 public class BigCountTestsTreeDigestTests extends BigCountTests {
     @Override
-    public TDigest createDigest() {
-        return TDigest.createAvlTreeDigest(arrays(), 100);
+    public TDigest createDigest(int compression) {
+        return TDigest.createAvlTreeDigest(arrays(), compression);
     }
 }

server/src/main/java/org/elasticsearch/index/codec/vectors/es93/DirectIOCapableLucene99FlatVectorsFormat.java

@@ -266,22 +266,25 @@ public void nextDocsAndScores(int nextCount, Bits liveDocs, DocAndFloatFeatureBu
                     buffer.docs[size++] = indexIterator.index();
                 }
             }
-            int loopBound = size - (size % bulkSize);
+            final int firstBulkSize = Math.min(bulkSize, size);
+            for (int j = 0; j < firstBulkSize; j++) {
+                final long ord = buffer.docs[j];
+                inputSlice.prefetch(ord * byteSize, byteSize);
+            }
+            final int loopBound = size - (size % bulkSize);
             int i = 0;
             for (; i < loopBound; i += bulkSize) {
-                for (int j = 0; j < bulkSize; j++) {
-                    long ord = buffer.docs[i + j];
+                final int nextI = i + bulkSize;
+                final int nextBulkSize = Math.min(bulkSize, size - nextI);
+                for (int j = 0; j < nextBulkSize; j++) {
+                    final long ord = buffer.docs[nextI + j];
                     inputSlice.prefetch(ord * byteSize, byteSize);
                 }
                 System.arraycopy(buffer.docs, i, docBuffer, 0, bulkSize);
                 inner.bulkScore(docBuffer, scoreBuffer, bulkSize);
                 System.arraycopy(scoreBuffer, 0, buffer.features, i, bulkSize);
             }
-            int countLeft = size - i;
-            for (int j = i; j < size; j++) {
-                long ord = buffer.docs[j];
-                inputSlice.prefetch(ord * byteSize, byteSize);
-            }
+            final int countLeft = size - i;
             System.arraycopy(buffer.docs, i, docBuffer, 0, countLeft);
             inner.bulkScore(docBuffer, scoreBuffer, countLeft);
             System.arraycopy(scoreBuffer, 0, buffer.features, i, countLeft);

server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestState.java

@@ -42,7 +42,7 @@ public class TDigestState implements Releasable, Accountable {
     private final TDigest tdigest;
 
     // Supported tdigest types.
-    protected enum Type {
+    public enum Type {
         HYBRID,
         AVL_TREE,
         MERGING,

server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java

@@ -10,7 +10,6 @@
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.KeywordField;
-import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
@@ -223,18 +222,27 @@ public void testFielddataLookupTerminatesInLoop() {
     }
 
     public void testFielddataLookupSometimesLoop() throws IOException {
-        SearchExecutionContext searchExecutionContext = createSearchExecutionContext(
-            // simulate a runtime field cycle in the second doc: 1: doc['2'] 2: doc['3'] 3: doc['4'] 4: doc['4']
+        // create this field so we can use it to make sure we're escaping the loop on only the "first" document
+        var concreteField = new KeywordFieldMapper.KeywordFieldType("indexed_field", true, true, Collections.emptyMap());
+
+        // simulate a runtime field cycle in the second doc: 1: doc['2'] 2: doc['3'] 3: doc['4'] 4: doc['4']
+        var runtimeFields = List.of(
             runtimeField("1", leafLookup -> leafLookup.doc().get("2").get(0).toString()),
             runtimeField("2", leafLookup -> leafLookup.doc().get("3").get(0).toString()),
             runtimeField("3", leafLookup -> leafLookup.doc().get("4").get(0).toString()),
-            runtimeField("4", (leafLookup, docId) -> {
-                if (docId == 0) {
+            runtimeField("4", leafLookup -> {
+                if (leafLookup.doc().get("indexed_field").getFirst().equals("first")) {
                     return "escape!";
                 }
-                return leafLookup.doc().get("4").get(0).toString();
+                return leafLookup.doc().get("4").getFirst().toString();
             })
         );
+        SearchExecutionContext searchExecutionContext = createSearchExecutionContext(
+            "uuid",
+            null,
+            createMappingLookup(List.of(concreteField), runtimeFields),
+            Collections.emptyMap()
+        );
         List<String> values = collect("1", searchExecutionContext, new TermQuery(new Term("indexed_field", "first")));
         assertEquals(List.of("escape!"), values);
         IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("1", searchExecutionContext));
@@ -776,8 +784,8 @@ private static List<String> collect(String field, SearchExecutionContext searchE
     private static List<String> collect(String field, SearchExecutionContext searchExecutionContext, Query query) throws IOException {
         List<String> result = new ArrayList<>();
         try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
-            indexWriter.addDocument(List.of(new StringField("indexed_field", "first", Field.Store.NO)));
-            indexWriter.addDocument(List.of(new StringField("indexed_field", "second", Field.Store.NO)));
+            indexWriter.addDocument(List.of(new KeywordField("indexed_field", "first", Field.Store.YES)));
+            indexWriter.addDocument(List.of(new KeywordField("indexed_field", "second", Field.Store.YES)));
             try (DirectoryReader reader = indexWriter.getReader()) {
                 IndexSearcher searcher = newSearcher(reader);
                 MappedFieldType fieldType = searchExecutionContext.getFieldType(field);

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.xpack.analytics.cumulativecardinality.CumulativeCardinalityPipelineAggregationBuilder;
 import org.elasticsearch.xpack.analytics.cumulativecardinality.InternalSimpleLongValue;
 import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper;
+import org.elasticsearch.xpack.analytics.mapper.TDigestFieldMapper;
 import org.elasticsearch.xpack.analytics.movingPercentiles.MovingPercentilesPipelineAggregationBuilder;
 import org.elasticsearch.xpack.analytics.multiterms.InternalMultiTerms;
 import org.elasticsearch.xpack.analytics.multiterms.MultiTermsAggregationBuilder;
@@ -140,6 +141,14 @@ public List<Setting<?>> getSettings() {
 
     @Override
     public Map<String, Mapper.TypeParser> getMappers() {
+        if (TDigestFieldMapper.TDIGEST_FIELD_MAPPER.isEnabled()) {
+            return Map.of(
+                HistogramFieldMapper.CONTENT_TYPE,
+                HistogramFieldMapper.PARSER,
+                TDigestFieldMapper.CONTENT_TYPE,
+                TDigestFieldMapper.PARSER
+            );
+        }
         return Map.of(HistogramFieldMapper.CONTENT_TYPE, HistogramFieldMapper.PARSER);
     }
 

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/ExponentialHistogramParser.java

@@ -113,7 +113,7 @@ private record ParsedZeroBucket(long count, double threshold) {
 
     /**
      * Parses an XContent object into an exponential histogram.
-     * The parse is expected to point at the next token after {@link XContentParser.Token#START_OBJECT}.
+     * The parser is expected to point at the next token after {@link XContentParser.Token#START_OBJECT}.
      *
      * @param mappedFieldName the name of the field being parsed, used for error messages
      * @param parser the parser to use