Merge remote-tracking branch 'origin/main' into add-projects-delta-to-cluster-changed-event

Author: ywangd

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OSQScorerBenchmark.java

@@ -17,7 +17,7 @@
 import org.apache.lucene.util.VectorUtil;
 import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
 import org.elasticsearch.common.logging.LogConfigurator;
-import org.elasticsearch.simdvec.ES91OSQVectorsScorer;
+import org.elasticsearch.simdvec.internal.vectorization.ES91OSQVectorsScorer;
 import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;

docs/changelog/125479.yaml

@@ -0,0 +1,6 @@
+pr: 125479
+summary: ES|QL - Allow full text functions to be used in STATS
+area: ES|QL
+type: enhancement
+issues:
+ - 125481

docs/changelog/127796.yaml

@@ -0,0 +1,6 @@
+pr: 127796
+summary: Do not respect synthetic_source_keep=arrays if type parses arrays
+area: Mapping
+type: enhancement
+issues:
+ - 126155

docs/changelog/127921.yaml

@@ -0,0 +1,5 @@
+pr: 127921
+summary: "[9.x] Revert \"Enable madvise by default for all builds\""
+area: Vector Search
+type: bug
+issues: []

docs/changelog/127949.yaml

@@ -0,0 +1,5 @@
+pr: 127949
+summary: Ensure ordinal builder emit ordinal blocks
+area: ES|QL
+type: bug
+issues: []

docs/changelog/127975.yaml

@@ -0,0 +1,5 @@
+pr: 127975
+summary: Fix a bug in `significant_terms`
+area: Aggregations
+type: bug
+issues: []

docs/reference/elasticsearch/mapping-reference/geo-point.md

@@ -218,7 +218,9 @@ of official GA features.
 
 
 Synthetic source may sort `geo_point` fields (first by latitude and then
-longitude) and reduces them to their stored precision. For example:
+longitude) and reduces them to their stored precision. Additionally, unlike most
+types, arrays of `geo_point` fields will not preserve order if
+`synthetic_source_keep` is set to `arrays`. For example:
 
 $$$synthetic-source-geo-point-example$$$
 
@@ -236,15 +238,18 @@ PUT idx
   },
   "mappings": {
     "properties": {
-      "point": { "type": "geo_point" }
+      "point": {
+        "type": "geo_point",
+        "synthetic_source_keep": "arrays"
+      }
     }
   }
 }
 PUT idx/_doc/1
 {
   "point": [
-    {"lat":-90, "lon":-80},
-    {"lat":10, "lon":30}
+    {"lat":10, "lon":30},
+    {"lat":-90, "lon":-80}
   ]
 }
 ```

libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java

@@ -9,13 +9,11 @@
 
 package org.elasticsearch.simdvec;
 
-import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.BitUtil;
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.simdvec.internal.vectorization.ESVectorUtilSupport;
 import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
 
-import java.io.IOException;
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
@@ -43,10 +41,6 @@ public class ESVectorUtil {
 
     private static final ESVectorUtilSupport IMPL = ESVectorizationProvider.getInstance().getVectorUtilSupport();
 
-    public static ES91OSQVectorsScorer getES91OSQVectorsScorer(IndexInput input, int dimension) throws IOException {
-        return ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(input, dimension);
-    }
-
     public static long ipByteBinByte(byte[] q, byte[] d) {
         if (q.length != d.length * B_QUERY) {
             throw new IllegalArgumentException("vector dimensions incompatible: " + q.length + "!= " + B_QUERY + " x " + d.length);
@@ -217,40 +211,4 @@ public static void centerAndCalculateOSQStatsDp(float[] target, float[] centroid
         assert stats.length == 6;
         IMPL.centerAndCalculateOSQStatsDp(target, centroid, centered, stats);
     }
-
-    /**
-     * Calculates the difference between two vectors and stores the result in a third vector.
-     * @param v1 the first vector
-     * @param v2 the second vector
-     * @param result the result vector, must be the same length as the input vectors
-     */
-    public static void subtract(float[] v1, float[] v2, float[] result) {
-        if (v1.length != v2.length) {
-            throw new IllegalArgumentException("vector dimensions differ: " + v1.length + "!=" + v2.length);
-        }
-        if (result.length != v1.length) {
-            throw new IllegalArgumentException("vector dimensions differ: " + result.length + "!=" + v1.length);
-        }
-        for (int i = 0; i < v1.length; i++) {
-            result[i] = v1[i] - v2[i];
-        }
-    }
-
-    /**
-     * calculates the spill-over score for a vector and a centroid, given its residual with
-     * its actually nearest centroid
-     * @param v1 the vector
-     * @param centroid the centroid
-     * @param originalResidual the residual with the actually nearest centroid
-     * @return the spill-over score (soar)
-     */
-    public static float soarResidual(float[] v1, float[] centroid, float[] originalResidual) {
-        if (v1.length != centroid.length) {
-            throw new IllegalArgumentException("vector dimensions differ: " + v1.length + "!=" + centroid.length);
-        }
-        if (originalResidual.length != v1.length) {
-            throw new IllegalArgumentException("vector dimensions differ: " + originalResidual.length + "!=" + v1.length);
-        }
-        return IMPL.soarResidual(v1, centroid, originalResidual);
-    }
 }

libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java

@@ -138,18 +138,6 @@ public void centerAndCalculateOSQStatsDp(float[] target, float[] centroid, float
         stats[5] = centroidDot;
     }
 
-    @Override
-    public float soarResidual(float[] v1, float[] centroid, float[] originalResidual) {
-        assert v1.length == centroid.length;
-        assert v1.length == originalResidual.length;
-        float proj = 0;
-        for (int i = 0; i < v1.length; i++) {
-            float djk = v1[i] - centroid[i];
-            proj = fma(djk, originalResidual[i], proj);
-        }
-        return proj;
-    }
-
     public static int ipByteBitImpl(byte[] q, byte[] d) {
         return ipByteBitImpl(q, d, 0);
     }

libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorizationProvider.java

@@ -10,7 +10,6 @@
 package org.elasticsearch.simdvec.internal.vectorization;
 
 import org.apache.lucene.store.IndexInput;
-import org.elasticsearch.simdvec.ES91OSQVectorsScorer;
 
 import java.io.IOException;