Merge branch 'main' into kderusso/rerank-snippet-poc

Author: kderusso

benchmarks/build.gradle

@@ -13,6 +13,7 @@ import org.elasticsearch.gradle.OS
 apply plugin: org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin
 apply plugin: 'java-library'
 apply plugin: 'application'
+apply plugin: 'elasticsearch.mrjar'
 
 var os = org.gradle.internal.os.OperatingSystem.current()
 
@@ -46,6 +47,7 @@ dependencies {
   api(project(':x-pack:plugin:core'))
   api(project(':x-pack:plugin:esql'))
   api(project(':x-pack:plugin:esql:compute'))
+  implementation project(path: ':libs:native')
   implementation project(path: ':libs:simdvec')
   expression(project(path: ':modules:lang-expression', configuration: 'zip'))
   painless(project(path: ':modules:lang-painless', configuration: 'zip'))

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/JDKVectorInt7uBenchmark.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.benchmark.vector;
+
+import org.apache.lucene.util.VectorUtil;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.common.logging.NodeNamePatternConverter;
+import org.elasticsearch.nativeaccess.NativeAccess;
+import org.elasticsearch.nativeaccess.VectorSimilarityFunctions;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+public class JDKVectorInt7uBenchmark {
+
+    static {
+        NodeNamePatternConverter.setGlobalNodeName("foo");
+        LogConfigurator.loadLog4jPlugins();
+        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+    }
+
+    byte[] byteArrayA;
+    byte[] byteArrayB;
+    MemorySegment heapSegA, heapSegB;
+    MemorySegment nativeSegA, nativeSegB;
+
+    Arena arena;
+
+    @Param({ "1", "128", "207", "256", "300", "512", "702", "1024" })
+    public int size;
+
+    @Setup(Level.Iteration)
+    public void init() {
+        byteArrayA = new byte[size];
+        byteArrayB = new byte[size];
+        for (int i = 0; i < size; ++i) {
+            randomInt7BytesBetween(byteArrayA);
+            randomInt7BytesBetween(byteArrayB);
+        }
+        heapSegA = MemorySegment.ofArray(byteArrayA);
+        heapSegB = MemorySegment.ofArray(byteArrayB);
+
+        arena = Arena.ofConfined();
+        nativeSegA = arena.allocate((long) byteArrayA.length);
+        MemorySegment.copy(MemorySegment.ofArray(byteArrayA), 0L, nativeSegA, 0L, byteArrayA.length);
+        nativeSegB = arena.allocate((long) byteArrayB.length);
+        MemorySegment.copy(MemorySegment.ofArray(byteArrayB), 0L, nativeSegB, 0L, byteArrayB.length);
+    }
+
+    @TearDown
+    public void teardown() {
+        arena.close();
+    }
+
+    @Benchmark
+    @Fork(value = 3, jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public int dotProductLucene() {
+        return VectorUtil.dotProduct(byteArrayA, byteArrayB);
+    }
+
+    @Benchmark
+    @Fork(value = 3, jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public int dotProductNativeWithNativeSeg() {
+        return dotProduct7u(nativeSegA, nativeSegB, size);
+    }
+
+    @Benchmark
+    @Fork(value = 3, jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public int dotProductNativeWithHeapSeg() {
+        return dotProduct7u(heapSegA, heapSegB, size);
+    }
+
+    static final VectorSimilarityFunctions vectorSimilarityFunctions = vectorSimilarityFunctions();
+
+    static VectorSimilarityFunctions vectorSimilarityFunctions() {
+        return NativeAccess.instance().getVectorSimilarityFunctions().get();
+    }
+
+    int dotProduct7u(MemorySegment a, MemorySegment b, int length) {
+        try {
+            return (int) vectorSimilarityFunctions.dotProductHandle7u().invokeExact(a, b, length);
+        } catch (Throwable e) {
+            if (e instanceof Error err) {
+                throw err;
+            } else if (e instanceof RuntimeException re) {
+                throw re;
+            } else {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    // Unsigned int7 byte vectors have values in the range of 0 to 127 (inclusive).
+    static final byte MIN_INT7_VALUE = 0;
+    static final byte MAX_INT7_VALUE = 127;
+
+    static void randomInt7BytesBetween(byte[] bytes) {
+        var random = ThreadLocalRandom.current();
+        for (int i = 0, len = bytes.length; i < len;) {
+            bytes[i++] = (byte) random.nextInt(MIN_INT7_VALUE, MAX_INT7_VALUE + 1);
+        }
+    }
+}

benchmarks/src/test/java/org/elasticsearch/benchmark/vector/JDKVectorInt7uBenchmarkTests.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.vector;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.BeforeClass;
+import org.openjdk.jmh.annotations.Param;
+
+import java.util.Arrays;
+
+public class JDKVectorInt7uBenchmarkTests extends ESTestCase {
+
+    final double delta = 1e-3;
+    final int size;
+
+    public JDKVectorInt7uBenchmarkTests(int size) {
+        this.size = size;
+    }
+
+    @BeforeClass
+    public static void skipWindows() {
+        assumeFalse("doesn't work on windows yet", Constants.WINDOWS);
+    }
+
+    static boolean supportsHeapSegments() {
+        return Runtime.version().feature() >= 22;
+    }
+
+    public void testDotProduct() {
+        for (int i = 0; i < 100; i++) {
+            var bench = new JDKVectorInt7uBenchmark();
+            bench.size = size;
+            bench.init();
+            try {
+                float expected = dotProductScalar(bench.byteArrayA, bench.byteArrayB);
+                assertEquals(expected, bench.dotProductLucene(), delta);
+                assertEquals(expected, bench.dotProductNativeWithNativeSeg(), delta);
+                if (supportsHeapSegments()) {
+                    assertEquals(expected, bench.dotProductNativeWithHeapSeg(), delta);
+                }
+            } finally {
+                bench.teardown();
+            }
+        }
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parametersFactory() {
+        try {
+            var params = JDKVectorInt7uBenchmark.class.getField("size").getAnnotationsByType(Param.class)[0].value();
+            return () -> Arrays.stream(params).map(Integer::parseInt).map(i -> new Object[] { i }).iterator();
+        } catch (NoSuchFieldException e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    /** Computes the dot product of the given vectors a and b. */
+    static int dotProductScalar(byte[] a, byte[] b) {
+        int res = 0;
+        for (int i = 0; i < a.length; i++) {
+            res += a[i] * b[i];
+        }
+        return res;
+    }
+}

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java

@@ -19,6 +19,7 @@
 import java.util.Objects;
 
 import static org.elasticsearch.gradle.internal.release.GenerateReleaseNotesTask.getSortedBundlesWithUniqueChangelogs;
+import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
@@ -100,7 +101,10 @@ public void testTemplate(String templateFilename, String outputFilename, List<Ch
             writeResource(outputFile, actualOutput);
             assertFalse("UPDATE_EXPECTED_OUTPUT should be set back to false after updating output", UPDATE_EXPECTED_OUTPUT);
         } else {
-            assertThat(actualOutput, equalTo(expectedOutput));
+            String[] expectedLines = expectedOutput.replace("\r", "").split("\n");
+            String[] actualLines = actualOutput.split("\n");
+
+            assertThat(actualLines, arrayContaining(expectedLines));
         }
     }
 

docs/changelog/127223.yaml

@@ -0,0 +1,5 @@
+pr: 127223
+summary: Wrap ES KNN queries with PatienceKNN query
+area: Vector Search
+type: feature
+issues: []

docs/changelog/129945.yaml

@@ -0,0 +1,5 @@
+pr: 129945
+summary: "Enhancement: ILM sets `indexing_complete` to true from `ReadOnly` action"
+area: ILM+SLM
+type: enhancement
+issues: []

docs/changelog/130221.yaml

@@ -0,0 +1,5 @@
+pr: 130221
+summary: Fix incorrect accounting of semantic text indexing memory pressure
+area: Distributed
+type: bug
+issues: []

docs/changelog/130251.yaml

@@ -0,0 +1,5 @@
+pr: 130251
+summary: Speed up (filtered) KNN queries for flat vector fields
+area: Vector Search
+type: enhancement
+issues: []

docs/changelog/130308.yaml

@@ -0,0 +1,5 @@
+pr: 130308
+summary: Force niofs for fdt tmp file read access when flushing stored fields
+area: Logs
+type: bug
+issues: []

docs/changelog/130330.yaml

@@ -0,0 +1,6 @@
+pr: 130330
+summary: "TopNOperator, release Row on failure"
+area: ES|QL
+type: bug
+issues:
+ - 130215