Merge branch 'main' into ml-ccm-api-settings

Author: jonathan-buttner

docs/changelog/137395.yaml

@@ -0,0 +1,6 @@
+pr: 137395
+summary: Fix attribute only in full text function not found
+area: ES|QL
+type: bug
+issues:
+ - 137396

docs/changelog/137476.yaml

@@ -0,0 +1,6 @@
+pr: 137476
+summary: Handle missing geotile buckets
+area: Transform
+type: bug
+issues:
+ - 126591

libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java

@@ -21,10 +21,12 @@
 
 package org.elasticsearch.tdigest;
 
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
 public abstract class BigCountTests extends TDigestTestCase {
 
     public void testBigMerge() {
-        try (TDigest digest = createDigest()) {
+        try (TDigest digest = createDigest(100)) {
             for (int i = 0; i < 5; i++) {
                 try (TDigest digestToMerge = getDigest()) {
                     digest.add(digestToMerge);
@@ -35,13 +37,25 @@ public void testBigMerge() {
         }
     }
 
+    /**
+     * Verify that, at a range of compression values, the size of the produced digest is not much larger than 10 times the compression
+     */
+    public void testCompression() {
+        for (int compression : new int[] { 100, 500, 1000, 10000 }) {
+            try (TDigest digest = createDigest(compression)) {
+                addData(digest);
+                assertThat("Compression = " + compression, digest.centroidCount(), lessThanOrEqualTo(compression * 10));
+            }
+        }
+    }
+
     private TDigest getDigest() {
-        TDigest digest = createDigest();
+        TDigest digest = createDigest(100);
         addData(digest);
         return digest;
     }
 
-    public TDigest createDigest() {
+    public TDigest createDigest(int compression) {
         throw new IllegalStateException("Should have over-ridden createDigest");
     }
 

libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java

@@ -23,7 +23,7 @@
 
 public class BigCountTestsMergingDigestTests extends BigCountTests {
     @Override
-    public TDigest createDigest() {
-        return TDigest.createMergingDigest(arrays(), 100);
+    public TDigest createDigest(int compression) {
+        return TDigest.createMergingDigest(arrays(), compression);
     }
 }

libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java

@@ -23,7 +23,7 @@
 
 public class BigCountTestsTreeDigestTests extends BigCountTests {
     @Override
-    public TDigest createDigest() {
-        return TDigest.createAvlTreeDigest(arrays(), 100);
+    public TDigest createDigest(int compression) {
+        return TDigest.createAvlTreeDigest(arrays(), compression);
     }
 }

server/src/main/java/org/elasticsearch/index/codec/vectors/es93/DirectIOCapableLucene99FlatVectorsFormat.java

@@ -266,22 +266,25 @@ public void nextDocsAndScores(int nextCount, Bits liveDocs, DocAndFloatFeatureBu
                     buffer.docs[size++] = indexIterator.index();
                 }
             }
-            int loopBound = size - (size % bulkSize);
+            final int firstBulkSize = Math.min(bulkSize, size);
+            for (int j = 0; j < firstBulkSize; j++) {
+                final long ord = buffer.docs[j];
+                inputSlice.prefetch(ord * byteSize, byteSize);
+            }
+            final int loopBound = size - (size % bulkSize);
             int i = 0;
             for (; i < loopBound; i += bulkSize) {
-                for (int j = 0; j < bulkSize; j++) {
-                    long ord = buffer.docs[i + j];
+                final int nextI = i + bulkSize;
+                final int nextBulkSize = Math.min(bulkSize, size - nextI);
+                for (int j = 0; j < nextBulkSize; j++) {
+                    final long ord = buffer.docs[nextI + j];
                     inputSlice.prefetch(ord * byteSize, byteSize);
                 }
                 System.arraycopy(buffer.docs, i, docBuffer, 0, bulkSize);
                 inner.bulkScore(docBuffer, scoreBuffer, bulkSize);
                 System.arraycopy(scoreBuffer, 0, buffer.features, i, bulkSize);
             }
-            int countLeft = size - i;
-            for (int j = i; j < size; j++) {
-                long ord = buffer.docs[j];
-                inputSlice.prefetch(ord * byteSize, byteSize);
-            }
+            final int countLeft = size - i;
             System.arraycopy(buffer.docs, i, docBuffer, 0, countLeft);
             inner.bulkScore(docBuffer, scoreBuffer, countLeft);
             System.arraycopy(scoreBuffer, 0, buffer.features, i, countLeft);

server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java

@@ -18,16 +18,53 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.routing.IndexRouting;
 import org.elasticsearch.cluster.routing.RoutingHashBuilder;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
 
 /**
  * Responsible for loading the _id from stored fields or for TSDB synthesizing the _id from the routing, _tsid and @timestamp fields.
  */
 public sealed interface IdLoader permits IdLoader.TsIdLoader, IdLoader.StoredIdLoader {
 
+    /**
+     * @return returns an {@link IdLoader} instance to load the value of the _id field.
+     */
+    static IdLoader create(MapperService mapperService) {
+        var indexSettings = mapperService.getIndexSettings();
+        if (indexSettings.getMode() == IndexMode.TIME_SERIES) {
+            IndexRouting.ExtractFromSource.ForRoutingPath indexRouting = null;
+            List<String> routingPaths = null;
+            if (indexSettings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID)) {
+                indexRouting = (IndexRouting.ExtractFromSource.ForRoutingPath) indexSettings.getIndexRouting();
+                routingPaths = indexSettings.getIndexMetadata().getRoutingPaths();
+                for (String routingField : routingPaths) {
+                    if (routingField.contains("*")) {
+                        // In case the routing fields include path matches, find any matches and add them as distinct fields
+                        // to the routing path.
+                        Set<String> matchingRoutingPaths = new TreeSet<>(routingPaths);
+                        for (Mapper mapper : mapperService.mappingLookup().fieldMappers()) {
+                            if (mapper instanceof KeywordFieldMapper && indexRouting.matchesField(mapper.fullPath())) {
+                                matchingRoutingPaths.add(mapper.fullPath());
+                            }
+                        }
+                        routingPaths = new ArrayList<>(matchingRoutingPaths);
+                        break;
+                    }
+                }
+            }
+            return createTsIdLoader(indexRouting, routingPaths, indexSettings.useTimeSeriesSyntheticId());
+        } else {
+            return fromLeafStoredFieldLoader();
+        }
+    }
+
     /**
      * @return returns an {@link IdLoader} instance the loads the _id from stored field.
      */

server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java

@@ -23,26 +23,21 @@
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.cluster.routing.IndexRouting;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.mapper.IdLoader;
-import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.NestedLookup;
 import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
@@ -88,8 +83,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.function.LongSupplier;
@@ -950,30 +943,6 @@ public SourceLoader newSourceLoader(@Nullable SourceFilter filter) {
 
     @Override
     public IdLoader newIdLoader() {
-        if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) {
-            IndexRouting.ExtractFromSource.ForRoutingPath indexRouting = null;
-            List<String> routingPaths = null;
-            if (indexService.getIndexSettings().getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID)) {
-                indexRouting = (IndexRouting.ExtractFromSource.ForRoutingPath) indexService.getIndexSettings().getIndexRouting();
-                routingPaths = indexService.getMetadata().getRoutingPaths();
-                for (String routingField : routingPaths) {
-                    if (routingField.contains("*")) {
-                        // In case the routing fields include path matches, find any matches and add them as distinct fields
-                        // to the routing path.
-                        Set<String> matchingRoutingPaths = new TreeSet<>(routingPaths);
-                        for (Mapper mapper : indexService.mapperService().mappingLookup().fieldMappers()) {
-                            if (mapper instanceof KeywordFieldMapper && indexRouting.matchesField(mapper.fullPath())) {
-                                matchingRoutingPaths.add(mapper.fullPath());
-                            }
-                        }
-                        routingPaths = new ArrayList<>(matchingRoutingPaths);
-                        break;
-                    }
-                }
-            }
-            return IdLoader.createTsIdLoader(indexRouting, routingPaths, indexService.getIndexSettings().useTimeSeriesSyntheticId());
-        } else {
-            return IdLoader.fromLeafStoredFieldLoader();
-        }
+        return IdLoader.create(indexService.mapperService());
     }
 }

server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestState.java

@@ -42,7 +42,7 @@ public class TDigestState implements Releasable, Accountable {
     private final TDigest tdigest;
 
     // Supported tdigest types.
-    protected enum Type {
+    public enum Type {
         HYBRID,
         AVL_TREE,
         MERGING,

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.xpack.analytics.cumulativecardinality.CumulativeCardinalityPipelineAggregationBuilder;
 import org.elasticsearch.xpack.analytics.cumulativecardinality.InternalSimpleLongValue;
 import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper;
+import org.elasticsearch.xpack.analytics.mapper.TDigestFieldMapper;
 import org.elasticsearch.xpack.analytics.movingPercentiles.MovingPercentilesPipelineAggregationBuilder;
 import org.elasticsearch.xpack.analytics.multiterms.InternalMultiTerms;
 import org.elasticsearch.xpack.analytics.multiterms.MultiTermsAggregationBuilder;
@@ -140,6 +141,14 @@ public List<Setting<?>> getSettings() {
 
     @Override
     public Map<String, Mapper.TypeParser> getMappers() {
+        if (TDigestFieldMapper.TDIGEST_FIELD_MAPPER.isEnabled()) {
+            return Map.of(
+                HistogramFieldMapper.CONTENT_TYPE,
+                HistogramFieldMapper.PARSER,
+                TDigestFieldMapper.CONTENT_TYPE,
+                TDigestFieldMapper.PARSER
+            );
+        }
         return Map.of(HistogramFieldMapper.CONTENT_TYPE, HistogramFieldMapper.PARSER);
     }