Merge branch 'main' into esql-test-cleanup

Author: nik9000

libs/x-content/src/main/java/org/elasticsearch/xcontent/AbstractObjectParser.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.xcontent;
 
 import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.xcontent.ObjectParser.NamedObjectParser;
 import org.elasticsearch.xcontent.ObjectParser.ValueType;
 
@@ -230,11 +231,13 @@ public void declareDoubleOrNull(BiConsumer<Value, Double> consumer, double nullV
         );
     }
 
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) // https://github.com/elastic/elasticsearch/issues/130797
     public void declareLong(BiConsumer<Value, Long> consumer, ParseField field) {
         // Using a method reference here angers some compilers
         declareField(consumer, p -> p.longValue(), field, ValueType.LONG);
     }
 
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) // https://github.com/elastic/elasticsearch/issues/130797
     public void declareLongOrNull(BiConsumer<Value, Long> consumer, long nullValue, ParseField field) {
         // Using a method reference here angers some compilers
         declareField(
@@ -245,6 +248,7 @@ public void declareLongOrNull(BiConsumer<Value, Long> consumer, long nullValue,
         );
     }
 
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) // https://github.com/elastic/elasticsearch/issues/130797
     public void declareInt(BiConsumer<Value, Integer> consumer, ParseField field) {
         // Using a method reference here angers some compilers
         declareField(consumer, p -> p.intValue(), field, ValueType.INT);
@@ -253,6 +257,7 @@ public void declareInt(BiConsumer<Value, Integer> consumer, ParseField field) {
     /**
      * Declare an integer field that parses explicit {@code null}s in the json to a default value.
      */
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) // https://github.com/elastic/elasticsearch/issues/130797
     public void declareIntOrNull(BiConsumer<Value, Integer> consumer, int nullValue, ParseField field) {
         declareField(
             consumer,
@@ -320,10 +325,12 @@ public void declareFloatArray(BiConsumer<Value, List<Float>> consumer, ParseFiel
         declareFieldArray(consumer, (p, c) -> p.floatValue(), field, ValueType.FLOAT_ARRAY);
     }
 
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) // https://github.com/elastic/elasticsearch/issues/130797
     public void declareLongArray(BiConsumer<Value, List<Long>> consumer, ParseField field) {
         declareFieldArray(consumer, (p, c) -> p.longValue(), field, ValueType.LONG_ARRAY);
     }
 
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) // https://github.com/elastic/elasticsearch/issues/130797
     public void declareIntArray(BiConsumer<Value, List<Integer>> consumer, ParseField field) {
         declareFieldArray(consumer, (p, c) -> p.intValue(), field, ValueType.INT_ARRAY);
     }

muted-tests.yml

@@ -461,9 +461,6 @@ tests:
 - class: org.elasticsearch.upgrades.MlJobSnapshotUpgradeIT
   method: testSnapshotUpgrader
   issue: https://github.com/elastic/elasticsearch/issues/98560
-- class: org.elasticsearch.upgrades.QueryableBuiltInRolesUpgradeIT
-  method: testBuiltInRolesSyncedOnClusterUpgrade
-  issue: https://github.com/elastic/elasticsearch/issues/129534
 - class: org.elasticsearch.search.query.VectorIT
   method: testFilteredQueryStrategy
   issue: https://github.com/elastic/elasticsearch/issues/129517
@@ -546,12 +543,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
   method: testStopQueryLocal
   issue: https://github.com/elastic/elasticsearch/issues/121672
-- class: org.elasticsearch.xpack.esql.qa.multi_node.GenerativeIT
-  method: test
-  issue: https://github.com/elastic/elasticsearch/issues/130067
-- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
-  method: test
-  issue: https://github.com/elastic/elasticsearch/issues/130067
 - class: org.elasticsearch.xpack.esql.action.EsqlRemoteErrorWrapIT
   method: testThatRemoteErrorsAreWrapped
   issue: https://github.com/elastic/elasticsearch/issues/130794
@@ -576,9 +567,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
   method: testRowStatsProjectGroupByInt
   issue: https://github.com/elastic/elasticsearch/issues/131024
-- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT
-  method: test {p0=esql/60_enrich/Enrich in fork}
-  issue: https://github.com/elastic/elasticsearch/issues/131028
 
 # Examples:
 #

rest-api-spec/src/main/resources/rest-api-spec/api/search_mvt.json

@@ -73,6 +73,15 @@
         "description":"Determines the geometry type for features in the aggs layer.",
         "default":"grid"
       },
+      "grid_agg":{
+        "type":"enum",
+        "options":[
+          "geotile",
+          "geohex"
+        ],
+        "description":"Aggregation used to create a grid for `field`.",
+        "default":"geotile"
+      },
       "size":{
         "type":"int",
         "description":"Maximum number of features to return in the hits layer. Accepts 0-10000.",

server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java

@@ -452,4 +452,72 @@ public Builder add(ShardId shardId, long reservedBytes) {
             }
         }
     }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static class Builder {
+        private Map<String, DiskUsage> leastAvailableSpaceUsage = Map.of();
+        private Map<String, DiskUsage> mostAvailableSpaceUsage = Map.of();
+        private Map<String, Long> shardSizes = Map.of();
+        private Map<ShardId, Long> shardDataSetSizes = Map.of();
+        private Map<NodeAndShard, String> dataPath = Map.of();
+        private Map<NodeAndPath, ReservedSpace> reservedSpace = Map.of();
+        private Map<String, EstimatedHeapUsage> estimatedHeapUsages = Map.of();
+        private Map<String, NodeUsageStatsForThreadPools> nodeUsageStatsForThreadPools = Map.of();
+
+        public ClusterInfo build() {
+            return new ClusterInfo(
+                leastAvailableSpaceUsage,
+                mostAvailableSpaceUsage,
+                shardSizes,
+                shardDataSetSizes,
+                dataPath,
+                reservedSpace,
+                estimatedHeapUsages,
+                nodeUsageStatsForThreadPools
+            );
+        }
+
+        public Builder leastAvailableSpaceUsage(Map<String, DiskUsage> leastAvailableSpaceUsage) {
+            this.leastAvailableSpaceUsage = leastAvailableSpaceUsage;
+            return this;
+        }
+
+        public Builder mostAvailableSpaceUsage(Map<String, DiskUsage> mostAvailableSpaceUsage) {
+            this.mostAvailableSpaceUsage = mostAvailableSpaceUsage;
+            return this;
+        }
+
+        public Builder shardSizes(Map<String, Long> shardSizes) {
+            this.shardSizes = shardSizes;
+            return this;
+        }
+
+        public Builder shardDataSetSizes(Map<ShardId, Long> shardDataSetSizes) {
+            this.shardDataSetSizes = shardDataSetSizes;
+            return this;
+        }
+
+        public Builder dataPath(Map<NodeAndShard, String> dataPath) {
+            this.dataPath = dataPath;
+            return this;
+        }
+
+        public Builder reservedSpace(Map<NodeAndPath, ReservedSpace> reservedSpace) {
+            this.reservedSpace = reservedSpace;
+            return this;
+        }
+
+        public Builder estimatedHeapUsages(Map<String, EstimatedHeapUsage> estimatedHeapUsages) {
+            this.estimatedHeapUsages = estimatedHeapUsages;
+            return this;
+        }
+
+        public Builder nodeUsageStatsForThreadPools(Map<String, NodeUsageStatsForThreadPools> nodeUsageStatsForThreadPools) {
+            this.nodeUsageStatsForThreadPools = nodeUsageStatsForThreadPools;
+            return this;
+        }
+    }
 }

server/src/main/java/org/elasticsearch/common/lucene/Lucene.java

@@ -739,15 +739,26 @@ public static Version parseVersionLenient(String toParse, Version defaultValue)
      * If no SegmentReader can be extracted an {@link IllegalStateException} is thrown.
      */
     public static SegmentReader segmentReader(LeafReader reader) {
+        SegmentReader segmentReader = tryUnwrapSegmentReader(reader);
+        if (segmentReader == null) {
+            throw new IllegalStateException("Can not extract segment reader from given index reader [" + reader + "]");
+        }
+        return segmentReader;
+    }
+
+    /**
+     * Tries to extract a segment reader from the given index reader. Unlike {@link #segmentReader(LeafReader)} this method returns
+     * null if no SegmentReader can be unwrapped instead of throwing an exception.
+     */
+    public static SegmentReader tryUnwrapSegmentReader(LeafReader reader) {
         if (reader instanceof SegmentReader) {
             return (SegmentReader) reader;
         } else if (reader instanceof final FilterLeafReader fReader) {
-            return segmentReader(FilterLeafReader.unwrap(fReader));
+            return tryUnwrapSegmentReader(FilterLeafReader.unwrap(fReader));
         } else if (reader instanceof final FilterCodecReader fReader) {
-            return segmentReader(FilterCodecReader.unwrap(fReader));
+            return tryUnwrapSegmentReader(FilterCodecReader.unwrap(fReader));
         }
-        // hard fail - we can't get a SegmentReader
-        throw new IllegalStateException("Can not extract segment reader from given index reader [" + reader + "]");
+        return null;
     }
 
     @SuppressForbidden(reason = "Version#parseLeniently() used in a central place")

server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java

@@ -737,6 +737,7 @@ public void sanitizeHeaders() {
                 entry -> entry.getKey().equalsIgnoreCase("authorization")
                     || entry.getKey().equalsIgnoreCase("es-secondary-authorization")
                     || entry.getKey().equalsIgnoreCase("ES-Client-Authentication")
+                    || entry.getKey().equalsIgnoreCase("X-Client-Authentication")
             );
 
         final ThreadContextStruct newContext = new ThreadContextStruct(

server/src/main/java/org/elasticsearch/index/codec/TrackingPostingsInMemoryBytesCodec.java

@@ -0,0 +1,160 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.codec;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.FieldsConsumer;
+import org.apache.lucene.codecs.FieldsProducer;
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FilterLeafReader;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.internal.hppc.IntIntHashMap;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.util.FeatureFlag;
+
+import java.io.IOException;
+import java.util.function.IntConsumer;
+
+/**
+ * A codec that tracks the length of the min and max written terms. Used to improve memory usage estimates in serverless, since
+ * {@link org.apache.lucene.codecs.lucene90.blocktree.FieldReader} keeps an in-memory reference to the min and max term.
+ */
+public class TrackingPostingsInMemoryBytesCodec extends FilterCodec {
+    public static final FeatureFlag TRACK_POSTINGS_IN_MEMORY_BYTES = new FeatureFlag("track_postings_in_memory_bytes");
+
+    public static final String IN_MEMORY_POSTINGS_BYTES_KEY = "es.postings.in_memory_bytes";
+
+    public TrackingPostingsInMemoryBytesCodec(Codec delegate) {
+        super(delegate.getName(), delegate);
+    }
+
+    @Override
+    public PostingsFormat postingsFormat() {
+        PostingsFormat format = super.postingsFormat();
+
+        return new PostingsFormat(format.getName()) {
+            @Override
+            public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+                FieldsConsumer consumer = format.fieldsConsumer(state);
+                return new TrackingLengthFieldsConsumer(state, consumer);
+            }
+
+            @Override
+            public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
+                return format.fieldsProducer(state);
+            }
+        };
+    }
+
+    static final class TrackingLengthFieldsConsumer extends FieldsConsumer {
+        final SegmentWriteState state;
+        final FieldsConsumer in;
+        final IntIntHashMap termsBytesPerField;
+
+        TrackingLengthFieldsConsumer(SegmentWriteState state, FieldsConsumer in) {
+            this.state = state;
+            this.in = in;
+            this.termsBytesPerField = new IntIntHashMap(state.fieldInfos.size());
+        }
+
+        @Override
+        public void write(Fields fields, NormsProducer norms) throws IOException {
+            in.write(new TrackingLengthFields(fields, termsBytesPerField, state.fieldInfos), norms);
+            long totalBytes = 0;
+            for (int bytes : termsBytesPerField.values) {
+                totalBytes += bytes;
+            }
+            state.segmentInfo.putAttribute(IN_MEMORY_POSTINGS_BYTES_KEY, Long.toString(totalBytes));
+        }
+
+        @Override
+        public void close() throws IOException {
+            in.close();
+        }
+    }
+
+    static final class TrackingLengthFields extends FilterLeafReader.FilterFields {
+        final IntIntHashMap termsBytesPerField;
+        final FieldInfos fieldInfos;
+
+        TrackingLengthFields(Fields in, IntIntHashMap termsBytesPerField, FieldInfos fieldInfos) {
+            super(in);
+            this.termsBytesPerField = termsBytesPerField;
+            this.fieldInfos = fieldInfos;
+        }
+
+        @Override
+        public Terms terms(String field) throws IOException {
+            Terms terms = super.terms(field);
+            if (terms == null) {
+                return null;
+            }
+            int fieldNum = fieldInfos.fieldInfo(field).number;
+            return new TrackingLengthTerms(
+                terms,
+                bytes -> termsBytesPerField.put(fieldNum, Math.max(termsBytesPerField.getOrDefault(fieldNum, 0), bytes))
+            );
+        }
+    }
+
+    static final class TrackingLengthTerms extends FilterLeafReader.FilterTerms {
+        final IntConsumer onFinish;
+
+        TrackingLengthTerms(Terms in, IntConsumer onFinish) {
+            super(in);
+            this.onFinish = onFinish;
+        }
+
+        @Override
+        public TermsEnum iterator() throws IOException {
+            return new TrackingLengthTermsEnum(super.iterator(), onFinish);
+        }
+    }
+
+    static final class TrackingLengthTermsEnum extends FilterLeafReader.FilterTermsEnum {
+        int maxTermLength = 0;
+        int minTermLength = 0;
+        int termCount = 0;
+        final IntConsumer onFinish;
+
+        TrackingLengthTermsEnum(TermsEnum in, IntConsumer onFinish) {
+            super(in);
+            this.onFinish = onFinish;
+        }
+
+        @Override
+        public BytesRef next() throws IOException {
+            final BytesRef term = super.next();
+            if (term != null) {
+                if (termCount == 0) {
+                    minTermLength = term.length;
+                }
+                maxTermLength = term.length;
+                termCount++;
+            } else {
+                if (termCount == 1) {
+                    // If the minTerm and maxTerm are the same, only one instance is kept on the heap.
+                    assert minTermLength == maxTermLength;
+                    onFinish.accept(maxTermLength);
+                } else {
+                    onFinish.accept(maxTermLength + minTermLength);
+                }
+            }
+            return term;
+        }
+    }
+}