Merge branch 'main' into fork_grammar_update

ioanatia · web-flow · commit 81cc7ca8c90f · 2025-06-11T17:17:46.000+02:00
diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=d7042b3c11565c192041fc8c4703f541b888286404b4f267138c1d094d8ecdca
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.1-all.zip
+distributionSha256Sum=443c9c8ee2ac1ee0e11881a40f2376d79c66386264a44b24a9f8ca67e633375f
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion
@@ -1 +1 @@
-8.14.1
+8.14.2
diff --git a/docs/changelog/128913.yml b/docs/changelog/128913.yml
@@ -0,0 +1,5 @@
+pr: 128913
+summary: "[apm-data] Enable 'date_detection' for all apm data streams"
+area: Data streams
+type: enhancement
+issues: []
diff --git a/docs/changelog/129164.yaml b/docs/changelog/129164.yaml
@@ -0,0 +1,5 @@
+pr: 129164
+summary: Log partial failures
+area: ES|QL
+type: feature
+issues: []
diff --git a/docs/reference/query-languages/esql/_snippets/lists/search-functions.md b/docs/reference/query-languages/esql/_snippets/lists/search-functions.md
@@ -2,5 +2,4 @@
 * [preview] [`MATCH`](../../functions-operators/search-functions.md#esql-match)
 * [preview] [`MATCH_PHRASE`](../../functions-operators/search-functions.md#esql-match_phrase)
 * [preview] [`QSTR`](../../functions-operators/search-functions.md#esql-qstr)
-  % * [preview] [
-  `TERM`](../../functions-operators/search-functions.md#esql-term)
+% * [preview] [`TERM`](../../functions-operators/search-functions.md#esql-term)
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=d7042b3c11565c192041fc8c4703f541b888286404b4f267138c1d094d8ecdca
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.1-all.zip
+distributionSha256Sum=443c9c8ee2ac1ee0e11881a40f2376d79c66386264a44b24a9f8ca67e633375f
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
diff --git a/plugins/examples/gradle/wrapper/gradle-wrapper.properties b/plugins/examples/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=d7042b3c11565c192041fc8c4703f541b888286404b4f267138c1d094d8ecdca
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.1-all.zip
+distributionSha256Sum=443c9c8ee2ac1ee0e11881a40f2376d79c66386264a44b24a9f8ca67e633375f
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/DefaultIVFVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/DefaultIVFVectorsReader.java
@@ -11,7 +11,6 @@
 
 import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
 import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.search.KnnCollector;
@@ -20,10 +19,12 @@
 import org.apache.lucene.util.VectorUtil;
 import org.apache.lucene.util.hnsw.NeighborQueue;
 import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
+import org.elasticsearch.index.codec.vectors.reflect.OffHeapStats;
 import org.elasticsearch.simdvec.ES91OSQVectorsScorer;
 import org.elasticsearch.simdvec.ESVectorUtil;
 
 import java.io.IOException;
+import java.util.Map;
 import java.util.function.IntPredicate;
 
 import static org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.QUERY_BITS;
@@ -38,7 +39,7 @@
  * Default implementation of {@link IVFVectorsReader}. It scores the posting lists centroids using
  * brute force and then scores the top ones using the posting list.
  */
-public class DefaultIVFVectorsReader extends IVFVectorsReader {
+public class DefaultIVFVectorsReader extends IVFVectorsReader implements OffHeapStats {
     private static final float FOUR_BIT_SCALE = 1f / ((1 << 4) - 1);
 
     public DefaultIVFVectorsReader(SegmentReadState state, FlatVectorsReader rawVectorsReader) throws IOException {
@@ -163,57 +164,9 @@ static float int4QuantizedScore(
         }
     }
 
-    static class OffHeapCentroidFloatVectorValues extends FloatVectorValues {
-        private final int numCentroids;
-        private final IndexInput input;
-        private final int dimension;
-        private final float[] centroid;
-        private final long centroidByteSize;
-        private int ord = -1;
-
-        OffHeapCentroidFloatVectorValues(int numCentroids, IndexInput input, int dimension) {
-            this.numCentroids = numCentroids;
-            this.input = input;
-            this.dimension = dimension;
-            this.centroid = new float[dimension];
-            this.centroidByteSize = dimension + 3 * Float.BYTES + Short.BYTES;
-        }
-
-        @Override
-        public float[] vectorValue(int ord) throws IOException {
-            if (ord < 0 || ord >= numCentroids) {
-                throw new IllegalArgumentException("ord must be in [0, " + numCentroids + "]");
-            }
-            if (ord == this.ord) {
-                return centroid;
-            }
-            readQuantizedCentroid(ord);
-            return centroid;
-        }
-
-        private void readQuantizedCentroid(int centroidOrdinal) throws IOException {
-            if (centroidOrdinal == ord) {
-                return;
-            }
-            input.seek(numCentroids * centroidByteSize + (long) Float.BYTES * dimension * centroidOrdinal);
-            input.readFloats(centroid, 0, centroid.length);
-            ord = centroidOrdinal;
-        }
-
-        @Override
-        public int dimension() {
-            return dimension;
-        }
-
-        @Override
-        public int size() {
-            return numCentroids;
-        }
-
-        @Override
-        public FloatVectorValues copy() throws IOException {
-            return new OffHeapCentroidFloatVectorValues(numCentroids, input.clone(), dimension);
-        }
+    @Override
+    public Map<String, Long> getOffHeapByteSize(FieldInfo fieldInfo) {
+        return Map.of();
     }
 
     private static class MemorySegmentPostingsVisitor implements PostingVisitor {
diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/IVFVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/IVFVectorsFormatTests.java
@@ -13,13 +13,25 @@
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.FilterCodec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.KnnVectorsReader;
+import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.index.CodecReader;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.VectorEncoding;
 import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
 import org.apache.lucene.tests.util.TestUtil;
 import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.index.codec.vectors.reflect.OffHeapByteSizeUtils;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Locale;
 
@@ -94,4 +106,26 @@ public void testLimits() {
         expectThrows(IllegalArgumentException.class, () -> new IVFVectorsFormat(MIN_VECTORS_PER_CLUSTER - 1));
         expectThrows(IllegalArgumentException.class, () -> new IVFVectorsFormat(MAX_VECTORS_PER_CLUSTER + 1));
     }
+
+    public void testSimpleOffHeapSize() throws IOException {
+        float[] vector = randomVector(random().nextInt(12, 500));
+        try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
+            Document doc = new Document();
+            doc.add(new KnnFloatVectorField("f", vector, VectorSimilarityFunction.EUCLIDEAN));
+            w.addDocument(doc);
+            w.commit();
+            try (IndexReader reader = DirectoryReader.open(w)) {
+                LeafReader r = getOnlyLeafReader(reader);
+                if (r instanceof CodecReader codecReader) {
+                    KnnVectorsReader knnVectorsReader = codecReader.getVectorReader();
+                    if (knnVectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) {
+                        knnVectorsReader = fieldsReader.getFieldReader("f");
+                    }
+                    var fieldInfo = r.getFieldInfos().fieldInfo("f");
+                    var offHeap = OffHeapByteSizeUtils.getOffHeapByteSize(knnVectorsReader, fieldInfo);
+                    assertEquals(0, offHeap.size());
+                }
+            }
+        }
+    }
 }
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/apm@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/apm@mappings.yaml
@@ -4,7 +4,6 @@ _meta:
   managed: true
 template:
   mappings:
-    date_detection: false
     dynamic: true
     dynamic_templates:
     - numeric_labels:
diff --git a/x-pack/plugin/apm-data/src/main/resources/resources.yaml b/x-pack/plugin/apm-data/src/main/resources/resources.yaml
@@ -1,7 +1,7 @@
 # "version" holds the version of the templates and ingest pipelines installed
 # by xpack-plugin apm-data. This must be increased whenever an existing template or
 # pipeline is changed, in order for it to be updated on Elasticsearch upgrade.
-version: 13
+version: 14
 
 component-templates:
   # Data lifecycle.
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResponseListener.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.TimeValue;
@@ -21,6 +22,7 @@
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener;
+import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.xcontent.MediaType;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.esql.arrow.ArrowFormat;
@@ -30,6 +32,7 @@
 
 import java.io.IOException;
 import java.util.Locale;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 
@@ -121,6 +124,7 @@ private EsqlResponseListener(RestChannel channel, RestRequest restRequest, Strin
 
     @Override
     protected void processResponse(EsqlQueryResponse esqlQueryResponse) throws IOException {
+        logPartialFailures(channel.request().rawPath(), channel.request().params(), esqlQueryResponse.getExecutionInfo());
         channel.sendResponse(buildResponse(esqlQueryResponse));
     }
 
@@ -229,4 +233,24 @@ private void checkDelimiter() {
             throw new IllegalArgumentException(message);
         }
     }
+
+    /**
+     * Log all partial request failures to the {@code rest.suppressed} logger
+     * so an operator can categorize them after the fact.
+     */
+    static void logPartialFailures(String rawPath, Map<String, String> params, EsqlExecutionInfo executionInfo) {
+        if (executionInfo == null) {
+            return;
+        }
+        for (EsqlExecutionInfo.Cluster cluster : executionInfo.getClusters().values()) {
+            for (ShardSearchFailure failure : cluster.getFailures()) {
+                if (LOGGER.isWarnEnabled()) {
+                    String clusterMessage = cluster.getClusterAlias().equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)
+                        ? ""
+                        : ", cluster: " + cluster.getClusterAlias();
+                    LOGGER.warn("partial failure at path: {}, params: {}{}", rawPath, params, clusterMessage, failure);
+                }
+            }
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlResponseListenerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlResponseListenerTests.java
