Merge branch 'main' into rn-test2

Author: lcawl

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerFloat32OperationBenchmark.java

@@ -31,6 +31,7 @@
 import java.lang.foreign.Arena;
 import java.lang.foreign.MemorySegment;
 import java.lang.foreign.ValueLayout;
+import java.lang.invoke.MethodHandle;
 import java.nio.ByteOrder;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
@@ -72,13 +73,8 @@ private interface LuceneFunction {
         float run(float[] vec1, float[] vec2);
     }
 
-    @FunctionalInterface
-    private interface NativeFunction {
-        float run(MemorySegment vec1, MemorySegment vec2, int length);
-    }
-
     private LuceneFunction luceneImpl;
-    private NativeFunction nativeImpl;
+    private MethodHandle nativeImpl;
 
     @Setup(Level.Iteration)
     public void init() {
@@ -105,11 +101,11 @@ public void init() {
             case EUCLIDEAN -> VectorUtil::squareDistance;
             default -> throw new UnsupportedOperationException("Not used");
         };
-        nativeImpl = switch (function) {
-            case DOT_PRODUCT -> VectorScorerFloat32OperationBenchmark::dotProductFloat32;
-            case EUCLIDEAN -> VectorScorerFloat32OperationBenchmark::squareDistanceFloat32;
-            default -> throw new UnsupportedOperationException("Not used");
-        };
+        nativeImpl = vectorSimilarityFunctions.getHandle(switch (function) {
+            case DOT_PRODUCT -> VectorSimilarityFunctions.Function.DOT_PRODUCT;
+            case EUCLIDEAN -> VectorSimilarityFunctions.Function.SQUARE_DISTANCE;
+            default -> throw new IllegalArgumentException(function.toString());
+        }, VectorSimilarityFunctions.DataType.FLOAT32, VectorSimilarityFunctions.Operation.SINGLE);
     }
 
     @TearDown
@@ -131,29 +127,21 @@ public float luceneWithCopy() {
 
     @Benchmark
     public float nativeWithNativeSeg() {
-        return nativeImpl.run(nativeSegA, nativeSegB, size);
+        try {
+            return (float) nativeImpl.invokeExact(nativeSegA, nativeSegB, size);
+        } catch (Throwable t) {
+            throw rethrow(t);
+        }
     }
 
     @Benchmark
     public float nativeWithHeapSeg() {
-        return nativeImpl.run(heapSegA, heapSegB, size);
-    }
-
-    static final VectorSimilarityFunctions vectorSimilarityFunctions = NativeAccess.instance().getVectorSimilarityFunctions().orElseThrow();
-
-    static float dotProductFloat32(MemorySegment a, MemorySegment b, int length) {
         try {
-            return (float) vectorSimilarityFunctions.dotProductHandleFloat32().invokeExact(a, b, length);
-        } catch (Throwable e) {
-            throw rethrow(e);
+            return (float) nativeImpl.invokeExact(heapSegA, heapSegB, size);
+        } catch (Throwable t) {
+            throw rethrow(t);
         }
     }
 
-    static float squareDistanceFloat32(MemorySegment a, MemorySegment b, int length) {
-        try {
-            return (float) vectorSimilarityFunctions.squareDistanceHandleFloat32().invokeExact(a, b, length);
-        } catch (Throwable e) {
-            throw rethrow(e);
-        }
-    }
+    static final VectorSimilarityFunctions vectorSimilarityFunctions = NativeAccess.instance().getVectorSimilarityFunctions().orElseThrow();
 }

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerInt7uOperationBenchmark.java

@@ -30,6 +30,7 @@
 
 import java.lang.foreign.Arena;
 import java.lang.foreign.MemorySegment;
+import java.lang.invoke.MethodHandle;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.benchmark.vector.scorer.BenchmarkUtils.randomInt7BytesBetween;
@@ -67,13 +68,8 @@ private interface LuceneFunction {
         float run(byte[] vec1, byte[] vec2);
     }
 
-    @FunctionalInterface
-    private interface NativeFunction {
-        float run(MemorySegment vec1, MemorySegment vec2, int length);
-    }
-
     private LuceneFunction luceneImpl;
-    private NativeFunction nativeImpl;
+    private MethodHandle nativeImpl;
 
     @Setup(Level.Iteration)
     public void init() {
@@ -97,11 +93,11 @@ public void init() {
             case EUCLIDEAN -> VectorUtil::squareDistance;
             default -> throw new UnsupportedOperationException("Not used");
         };
-        nativeImpl = switch (function) {
-            case DOT_PRODUCT -> VectorScorerInt7uOperationBenchmark::dotProduct7u;
-            case EUCLIDEAN -> VectorScorerInt7uOperationBenchmark::squareDistance7u;
-            default -> throw new UnsupportedOperationException("Not used");
-        };
+        nativeImpl = vectorSimilarityFunctions.getHandle(switch (function) {
+            case DOT_PRODUCT -> VectorSimilarityFunctions.Function.DOT_PRODUCT;
+            case EUCLIDEAN -> VectorSimilarityFunctions.Function.SQUARE_DISTANCE;
+            default -> throw new IllegalArgumentException(function.toString());
+        }, VectorSimilarityFunctions.DataType.INT7, VectorSimilarityFunctions.Operation.SINGLE);
     }
 
     @TearDown
@@ -115,30 +111,22 @@ public float lucene() {
     }
 
     @Benchmark
-    public float nativeWithNativeSeg() {
-        return nativeImpl.run(nativeSegA, nativeSegB, size);
+    public int nativeWithNativeSeg() {
+        try {
+            return (int) nativeImpl.invokeExact(nativeSegA, nativeSegB, size);
+        } catch (Throwable t) {
+            throw rethrow(t);
+        }
     }
 
     @Benchmark
     public float nativeWithHeapSeg() {
-        return nativeImpl.run(heapSegA, heapSegB, size);
-    }
-
-    static final VectorSimilarityFunctions vectorSimilarityFunctions = NativeAccess.instance().getVectorSimilarityFunctions().orElseThrow();
-
-    static int dotProduct7u(MemorySegment a, MemorySegment b, int length) {
         try {
-            return (int) vectorSimilarityFunctions.dotProductHandle7u().invokeExact(a, b, length);
-        } catch (Throwable e) {
-            throw rethrow(e);
+            return (int) nativeImpl.invokeExact(heapSegA, heapSegB, size);
+        } catch (Throwable t) {
+            throw rethrow(t);
         }
     }
 
-    static int squareDistance7u(MemorySegment a, MemorySegment b, int length) {
-        try {
-            return (int) vectorSimilarityFunctions.squareDistanceHandle7u().invokeExact(a, b, length);
-        } catch (Throwable e) {
-            throw rethrow(e);
-        }
-    }
+    static final VectorSimilarityFunctions vectorSimilarityFunctions = NativeAccess.instance().getVectorSimilarityFunctions().orElseThrow();
 }

distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile

@@ -25,7 +25,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:c756ffe412df1a84c87d474ba90784e98484c1dc74500051770e67fbce456251 AS builder
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:11d66ae7ec7ca1d5a7289750bdd96eb3d8eb592e5b7377f1fc8e4fb556fa7383 AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -104,7 +104,7 @@ WORKDIR /usr/share/elasticsearch/config
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:c756ffe412df1a84c87d474ba90784e98484c1dc74500051770e67fbce456251
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:11d66ae7ec7ca1d5a7289750bdd96eb3d8eb592e5b7377f1fc8e4fb556fa7383
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile

@@ -25,7 +25,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:2eae982b1ace31929369639fe4eefd38f3b12e85a5d5843f541c0cfc14045e27 AS builder
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:b5a03b6a754fa2f9a29e44e316a6c4df1f606cd8a3cd8810ce3d6a3a3134428b AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -80,7 +80,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:2eae982b1ace31929369639fe4eefd38f3b12e85a5d5843f541c0cfc14045e27
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:b5a03b6a754fa2f9a29e44e316a6c4df1f606cd8a3cd8810ce3d6a3a3134428b
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

docs/changelog/140478.yaml

@@ -0,0 +1,5 @@
+pr: 140478
+summary: T digest field type docs
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/141179.yaml

@@ -0,0 +1,5 @@
+pr: 141179
+summary: Add `SparseFileTracker.getAbsentBytesWithin`
+area: Searchable Snapshots
+type: enhancement
+issues: []

docs/changelog/141185.yaml

@@ -0,0 +1,5 @@
+pr: 141185
+summary: Add `max_batch_size` setting to EIS dense and sparse service settings
+area: Inference
+type: enhancement
+issues: []

docs/changelog/141237.yaml

@@ -0,0 +1,5 @@
+pr: 141237
+summary: Introduce adaptive block hash for long/int
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/141323.yaml

@@ -0,0 +1,5 @@
+pr: 141323
+summary: Fix race condition in `CancellableRateLimitedFluxIterator`
+area: Snapshot/Restore
+type: bug
+issues: []

docs/reference/elasticsearch/configuration-reference/discovery-cluster-formation-settings.md

@@ -97,6 +97,8 @@ $$$fault-detection-settings$$$`cluster.fault_detection.follower_check.interval`
 `cluster.follower_lag.timeout`
 :   ([Static](docs-content://deploy-manage/stack-settings.md#static-cluster-setting)) Sets how long the master node waits to receive acknowledgements for cluster state updates from lagging nodes. The default value is `90s`. If a node does not successfully apply the cluster state update within this period of time, it is considered to have failed and is removed from the cluster. See [Publishing the cluster state](docs-content://deploy-manage/distributed-architecture/discovery-cluster-formation/cluster-state-overview.md#cluster-state-publishing).
 
+    {applies_to}`stack: ga 9.4` The value `-1` means that the master node will wait indefinitely for cluster state updates to be acknowledged.
+
 `cluster.max_voting_config_exclusions`
 :   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) Sets a limit on the number of voting configuration exclusions at any one time. The default value is `10`. See [*Add and remove nodes in your cluster*](docs-content://deploy-manage/maintenance/add-and-remove-elasticsearch-nodes.md).