Merge branch 'main' into esql-mv-rerank

Author: ioanatia

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerFloat32OperationBenchmark.java

@@ -31,6 +31,7 @@
 import java.lang.foreign.Arena;
 import java.lang.foreign.MemorySegment;
 import java.lang.foreign.ValueLayout;
+import java.lang.invoke.MethodHandle;
 import java.nio.ByteOrder;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
@@ -72,13 +73,8 @@ private interface LuceneFunction {
         float run(float[] vec1, float[] vec2);
     }
 
-    @FunctionalInterface
-    private interface NativeFunction {
-        float run(MemorySegment vec1, MemorySegment vec2, int length);
-    }
-
     private LuceneFunction luceneImpl;
-    private NativeFunction nativeImpl;
+    private MethodHandle nativeImpl;
 
     @Setup(Level.Iteration)
     public void init() {
@@ -105,11 +101,11 @@ public void init() {
             case EUCLIDEAN -> VectorUtil::squareDistance;
             default -> throw new UnsupportedOperationException("Not used");
         };
-        nativeImpl = switch (function) {
-            case DOT_PRODUCT -> VectorScorerFloat32OperationBenchmark::dotProductFloat32;
-            case EUCLIDEAN -> VectorScorerFloat32OperationBenchmark::squareDistanceFloat32;
-            default -> throw new UnsupportedOperationException("Not used");
-        };
+        nativeImpl = vectorSimilarityFunctions.getHandle(switch (function) {
+            case DOT_PRODUCT -> VectorSimilarityFunctions.Function.DOT_PRODUCT;
+            case EUCLIDEAN -> VectorSimilarityFunctions.Function.SQUARE_DISTANCE;
+            default -> throw new IllegalArgumentException(function.toString());
+        }, VectorSimilarityFunctions.DataType.FLOAT32, VectorSimilarityFunctions.Operation.SINGLE);
     }
 
     @TearDown
@@ -131,29 +127,21 @@ public float luceneWithCopy() {
 
     @Benchmark
     public float nativeWithNativeSeg() {
-        return nativeImpl.run(nativeSegA, nativeSegB, size);
+        try {
+            return (float) nativeImpl.invokeExact(nativeSegA, nativeSegB, size);
+        } catch (Throwable t) {
+            throw rethrow(t);
+        }
     }
 
     @Benchmark
     public float nativeWithHeapSeg() {
-        return nativeImpl.run(heapSegA, heapSegB, size);
-    }
-
-    static final VectorSimilarityFunctions vectorSimilarityFunctions = NativeAccess.instance().getVectorSimilarityFunctions().orElseThrow();
-
-    static float dotProductFloat32(MemorySegment a, MemorySegment b, int length) {
         try {
-            return (float) vectorSimilarityFunctions.dotProductHandleFloat32().invokeExact(a, b, length);
-        } catch (Throwable e) {
-            throw rethrow(e);
+            return (float) nativeImpl.invokeExact(heapSegA, heapSegB, size);
+        } catch (Throwable t) {
+            throw rethrow(t);
         }
     }
 
-    static float squareDistanceFloat32(MemorySegment a, MemorySegment b, int length) {
-        try {
-            return (float) vectorSimilarityFunctions.squareDistanceHandleFloat32().invokeExact(a, b, length);
-        } catch (Throwable e) {
-            throw rethrow(e);
-        }
-    }
+    static final VectorSimilarityFunctions vectorSimilarityFunctions = NativeAccess.instance().getVectorSimilarityFunctions().orElseThrow();
 }

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerInt7uOperationBenchmark.java

@@ -30,6 +30,7 @@
 
 import java.lang.foreign.Arena;
 import java.lang.foreign.MemorySegment;
+import java.lang.invoke.MethodHandle;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.benchmark.vector.scorer.BenchmarkUtils.randomInt7BytesBetween;
@@ -67,13 +68,8 @@ private interface LuceneFunction {
         float run(byte[] vec1, byte[] vec2);
     }
 
-    @FunctionalInterface
-    private interface NativeFunction {
-        float run(MemorySegment vec1, MemorySegment vec2, int length);
-    }
-
     private LuceneFunction luceneImpl;
-    private NativeFunction nativeImpl;
+    private MethodHandle nativeImpl;
 
     @Setup(Level.Iteration)
     public void init() {
@@ -97,11 +93,11 @@ public void init() {
             case EUCLIDEAN -> VectorUtil::squareDistance;
             default -> throw new UnsupportedOperationException("Not used");
         };
-        nativeImpl = switch (function) {
-            case DOT_PRODUCT -> VectorScorerInt7uOperationBenchmark::dotProduct7u;
-            case EUCLIDEAN -> VectorScorerInt7uOperationBenchmark::squareDistance7u;
-            default -> throw new UnsupportedOperationException("Not used");
-        };
+        nativeImpl = vectorSimilarityFunctions.getHandle(switch (function) {
+            case DOT_PRODUCT -> VectorSimilarityFunctions.Function.DOT_PRODUCT;
+            case EUCLIDEAN -> VectorSimilarityFunctions.Function.SQUARE_DISTANCE;
+            default -> throw new IllegalArgumentException(function.toString());
+        }, VectorSimilarityFunctions.DataType.INT7, VectorSimilarityFunctions.Operation.SINGLE);
     }
 
     @TearDown
@@ -115,30 +111,22 @@ public float lucene() {
     }
 
     @Benchmark
-    public float nativeWithNativeSeg() {
-        return nativeImpl.run(nativeSegA, nativeSegB, size);
+    public int nativeWithNativeSeg() {
+        try {
+            return (int) nativeImpl.invokeExact(nativeSegA, nativeSegB, size);
+        } catch (Throwable t) {
+            throw rethrow(t);
+        }
     }
 
     @Benchmark
     public float nativeWithHeapSeg() {
-        return nativeImpl.run(heapSegA, heapSegB, size);
-    }
-
-    static final VectorSimilarityFunctions vectorSimilarityFunctions = NativeAccess.instance().getVectorSimilarityFunctions().orElseThrow();
-
-    static int dotProduct7u(MemorySegment a, MemorySegment b, int length) {
         try {
-            return (int) vectorSimilarityFunctions.dotProductHandle7u().invokeExact(a, b, length);
-        } catch (Throwable e) {
-            throw rethrow(e);
+            return (int) nativeImpl.invokeExact(heapSegA, heapSegB, size);
+        } catch (Throwable t) {
+            throw rethrow(t);
         }
     }
 
-    static int squareDistance7u(MemorySegment a, MemorySegment b, int length) {
-        try {
-            return (int) vectorSimilarityFunctions.squareDistanceHandle7u().invokeExact(a, b, length);
-        } catch (Throwable e) {
-            throw rethrow(e);
-        }
-    }
+    static final VectorSimilarityFunctions vectorSimilarityFunctions = NativeAccess.instance().getVectorSimilarityFunctions().orElseThrow();
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java

@@ -114,8 +114,12 @@ private List<File> resolveProjectLogs(File projectDir) {
             Files.walkFileTree(projectDir.toPath(), new SimpleFileVisitor<>() {
                 @Override
                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                    if (Files.isSymbolicLink(file)) {
-                        Files.delete(file);
+                    try {
+                        if (Files.isSymbolicLink(file)) {
+                            Files.delete(file);
+                        }
+                    } catch (java.nio.file.NoSuchFileException e) {
+                        System.out.println("Symlink : " + file + " already deleted.");
                     }
                     return FileVisitResult.CONTINUE;
                 }

build-tools-internal/version.properties

@@ -16,6 +16,7 @@ ecsLogging        = 1.7.0
 jna               = 5.12.1
 netty             = 4.1.130.Final
 commons_lang3     = 3.9
+commons_math3     = 3.6.1
 google_oauth_client = 1.34.1
 awsv2sdk            = 2.31.78
 reactive_streams    = 1.0.4

build.gradle

@@ -342,6 +342,11 @@ allprojects {
     if (project.path == ':') {
       resolveJavaToolChain = true
     }
+    project.pluginManager.withPlugin('elasticsearch.mrjar') {
+      // ensure we implicitly resolve all jdks needed from the java toolchain resolution
+      println "Configuring resolveAllDependencies to depend on JavaCompile tasks for project ${project.path} because it applies the mrjar plugin"
+      dependsOn project.getTasks().withType(JavaCompile.class)
+    }
     // we run the packer script for all active branches. so we should be able to skip bwc here
     if(project.path.startsWith(":distribution:bwc:")) {
       enabled = false

distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile

@@ -25,7 +25,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:c756ffe412df1a84c87d474ba90784e98484c1dc74500051770e67fbce456251 AS builder
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:11d66ae7ec7ca1d5a7289750bdd96eb3d8eb592e5b7377f1fc8e4fb556fa7383 AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -104,7 +104,7 @@ WORKDIR /usr/share/elasticsearch/config
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:c756ffe412df1a84c87d474ba90784e98484c1dc74500051770e67fbce456251
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:11d66ae7ec7ca1d5a7289750bdd96eb3d8eb592e5b7377f1fc8e4fb556fa7383
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile

@@ -25,7 +25,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:2eae982b1ace31929369639fe4eefd38f3b12e85a5d5843f541c0cfc14045e27 AS builder
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:b5a03b6a754fa2f9a29e44e316a6c4df1f606cd8a3cd8810ce3d6a3a3134428b AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -80,7 +80,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:2eae982b1ace31929369639fe4eefd38f3b12e85a5d5843f541c0cfc14045e27
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:b5a03b6a754fa2f9a29e44e316a6c4df1f606cd8a3cd8810ce3d6a3a3134428b
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

docs/changelog/131828.yaml

@@ -0,0 +1,5 @@
+pr: 131828
+summary: ES|QL approximate analytical queries
+area: ES|QL
+type: feature
+issues: []

docs/changelog/139422.yaml

@@ -0,0 +1,5 @@
+pr: 139422
+summary: Use common retry logic for Azure
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/changelog/139944.yaml

@@ -0,0 +1,5 @@
+pr: 139944
+summary: Make quantization level of indexed vectors for 'bbq_disk' configurable
+area: Vector Search
+type: enhancement
+issues: []