Merge branch 'main' of github.com:elastic/elasticsearch into ml-eis-auth-v2

Author: jonathan-buttner

benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.index.codec.Elasticsearch92Lucene103Codec;
+import org.elasticsearch.index.codec.tsdb.BinaryDVCompressionMode;
 import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -257,7 +258,13 @@ private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeE
         );
         config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
         config.setMergePolicy(new LogByteSizeMergePolicy());
-        var docValuesFormat = new ES819TSDBDocValuesFormat(4096, 512, optimizedMergeEnabled);
+        var docValuesFormat = new ES819TSDBDocValuesFormat(
+            4096,
+            512,
+            optimizedMergeEnabled,
+            BinaryDVCompressionMode.COMPRESSED_ZSTD_LEVEL_1,
+            true
+        );
         config.setCodec(new Elasticsearch92Lucene103Codec() {
             @Override
             public DocValuesFormat getDocValuesFormatForField(String field) {

docs/changelog/137263.yaml

@@ -0,0 +1,5 @@
+pr: 137263
+summary: Add late chunking configuration for JinaAI embedding task settings
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/137862.yaml

@@ -0,0 +1,5 @@
+pr: 137862
+summary: "[Vector Search] Fix  wrong vector docvalue_fields"
+area: Vector Search
+type: bug
+issues: []

docs/changelog/138204.yaml

@@ -0,0 +1,5 @@
+pr: 138265
+summary: Fix `index.mapping.pattern_text.disable_templating` not registered issue
+area: Mapping
+type: bug
+issues: []

docs/changelog/138265.yaml

@@ -110,8 +110,34 @@ needed to backport to `8.19` you would run (in `main`):
     ./gradlew generateTransportVersion --name=my_tv --backport-branches=9.1,8.19
 
 In the above case CI will not know what transport version name to update, so you
-must run the generate task again as described. After merging the updated transport
-version it will need to be backported to all the applicable branches.
+must run the generate task again as described. After merging the updated
+transport version it will need to be backported to all the applicable branches.
+Backport branches that already included the original change can use
+auto-backport in CI or cherry-pick the updated transport version. Backport
+branches that did not include the original change need to cherry-pick both the
+original change and the updated transport version. For example, with the above
+change that is first backported to `9.1`, and subsequently added to `8.19`, the
+sequence of events would look like the following:
+
+1. Create a pull request with target branch `main` for a new change
+(`C0`) with a newly created transport version (`TV`) required by `C0`
+   1. Use the `github` labels `auto-backport` and `branch:9.1` to backport `C0`
+   and `TV` into the target branch `9.1`.
+2. Discover that `C0` is also required in target branch `8.19`.
+3. Create a pull request with target branch `main` for a change (`C1`) to only
+update `TV` to include `8.19`
+   1. Use the gradle task
+   `./gradlew generateTransportVersion --name=my_tv
+   --backport-branches=9.1,8.19`
+   to update `TV` to include `8.19`
+   2. Use the `github` labels `auto-backport`, `branch:9.1`, and `branch:8.19`
+   to backport `C1` and updated `TV` into the target branches `9.1` and `8.19`.
+   This creates a source branch for `8.19` (`SB819`) that does not contain `C0`.
+   3. Cherry-pick `C0` into the source branch `SB819`
+
+Note there are several ways to achieve the desired end state. This
+specific example takes advantage of the available CI tooling to simplify the
+process.
 
 ### Resolving merge conflicts
 

docs/internal/Versioning.md

@@ -19,7 +19,7 @@ configurations {
 }
 
 var zstdVersion = "1.5.5"
-var vecVersion = "1.0.14"
+var vecVersion = "1.0.13"
 
 repositories {
   exclusiveContent {

libs/native/libraries/build.gradle

@@ -30,8 +30,6 @@ public interface VectorSimilarityFunctions {
      */
     MethodHandle dotProductHandle7u();
 
-    MethodHandle dotProductHandle7uBulk();
-
     /**
      * Produces a method handle returning the square distance of byte (unsigned int7) vectors.
      *

libs/native/src/main/java/org/elasticsearch/nativeaccess/VectorSimilarityFunctions.java

@@ -32,7 +32,6 @@ public final class JdkVectorLibrary implements VectorLibrary {
     static final Logger logger = LogManager.getLogger(JdkVectorLibrary.class);
 
     static final MethodHandle dot7u$mh;
-    static final MethodHandle dot7uBulk$mh;
     static final MethodHandle sqr7u$mh;
     static final MethodHandle cosf32$mh;
     static final MethodHandle dotf32$mh;
@@ -54,11 +53,6 @@ public final class JdkVectorLibrary implements VectorLibrary {
                         FunctionDescriptor.of(JAVA_INT, ADDRESS, ADDRESS, JAVA_INT),
                         LinkerHelperUtil.critical()
                     );
-                    dot7uBulk$mh = downcallHandle(
-                        "dot7u_bulk_2",
-                        FunctionDescriptor.ofVoid(ADDRESS, ADDRESS, JAVA_INT, JAVA_INT, ADDRESS),
-                        LinkerHelperUtil.critical()
-                    );
                     sqr7u$mh = downcallHandle(
                         "sqr7u_2",
                         FunctionDescriptor.of(JAVA_INT, ADDRESS, ADDRESS, JAVA_INT),
@@ -85,11 +79,6 @@ public final class JdkVectorLibrary implements VectorLibrary {
                         FunctionDescriptor.of(JAVA_INT, ADDRESS, ADDRESS, JAVA_INT),
                         LinkerHelperUtil.critical()
                     );
-                    dot7uBulk$mh = downcallHandle(
-                        "dot7u_bulk",
-                        FunctionDescriptor.ofVoid(ADDRESS, ADDRESS, JAVA_INT, JAVA_INT, ADDRESS),
-                        LinkerHelperUtil.critical()
-                    );
                     sqr7u$mh = downcallHandle(
                         "sqr7u",
                         FunctionDescriptor.of(JAVA_INT, ADDRESS, ADDRESS, JAVA_INT),
@@ -119,7 +108,6 @@ public final class JdkVectorLibrary implements VectorLibrary {
                         enable them in your OS/Hypervisor/VM/container""");
                 }
                 dot7u$mh = null;
-                dot7uBulk$mh = null;
                 sqr7u$mh = null;
                 cosf32$mh = null;
                 dotf32$mh = null;
@@ -154,10 +142,6 @@ static int dotProduct7u(MemorySegment a, MemorySegment b, int length) {
             return dot7u(a, b, length);
         }
 
-        static void dotProduct7uBulk(MemorySegment a, MemorySegment b, int length, int count, MemorySegment result) {
-            dot7uBulk(a, b, length, count, result);
-        }
-
         /**
          * Computes the square distance of given unsigned int7 byte vectors.
          *
@@ -226,14 +210,6 @@ private static int dot7u(MemorySegment a, MemorySegment b, int length) {
             }
         }
 
-        private static void dot7uBulk(MemorySegment a, MemorySegment b, int length, int count, MemorySegment result) {
-            try {
-                JdkVectorLibrary.dot7uBulk$mh.invokeExact(a, b, length, count, result);
-            } catch (Throwable t) {
-                throw new AssertionError(t);
-            }
-        }
-
         private static int sqr7u(MemorySegment a, MemorySegment b, int length) {
             try {
                 return (int) JdkVectorLibrary.sqr7u$mh.invokeExact(a, b, length);
@@ -267,7 +243,6 @@ private static float sqrf32(MemorySegment a, MemorySegment b, int length) {
         }
 
         static final MethodHandle DOT_HANDLE_7U;
-        static final MethodHandle DOT_HANDLE_7U_BULK;
         static final MethodHandle SQR_HANDLE_7U;
         static final MethodHandle COS_HANDLE_FLOAT32;
         static final MethodHandle DOT_HANDLE_FLOAT32;
@@ -278,11 +253,6 @@ private static float sqrf32(MemorySegment a, MemorySegment b, int length) {
                 var lookup = MethodHandles.lookup();
                 var mt = MethodType.methodType(int.class, MemorySegment.class, MemorySegment.class, int.class);
                 DOT_HANDLE_7U = lookup.findStatic(JdkVectorSimilarityFunctions.class, "dotProduct7u", mt);
-                DOT_HANDLE_7U_BULK = lookup.findStatic(
-                    JdkVectorSimilarityFunctions.class,
-                    "dotProduct7uBulk",
-                    MethodType.methodType(void.class, MemorySegment.class, MemorySegment.class, int.class, int.class, MemorySegment.class)
-                );
                 SQR_HANDLE_7U = lookup.findStatic(JdkVectorSimilarityFunctions.class, "squareDistance7u", mt);
 
                 mt = MethodType.methodType(float.class, MemorySegment.class, MemorySegment.class, int.class);
@@ -299,11 +269,6 @@ public MethodHandle dotProductHandle7u() {
             return DOT_HANDLE_7U;
         }
 
-        @Override
-        public MethodHandle dotProductHandle7uBulk() {
-            return DOT_HANDLE_7U_BULK;
-        }
-
         @Override
         public MethodHandle squareDistanceHandle7u() {
             return SQR_HANDLE_7U;

libs/native/src/main/java/org/elasticsearch/nativeaccess/jdk/JdkVectorLibrary.java

@@ -22,23 +22,8 @@ public class Similarities {
         .orElseThrow(AssertionError::new);
 
     static final MethodHandle DOT_PRODUCT_7U = DISTANCE_FUNCS.dotProductHandle7u();
-    static final MethodHandle DOT_PRODUCT_7U_BULK = DISTANCE_FUNCS.dotProductHandle7uBulk();
     static final MethodHandle SQUARE_DISTANCE_7U = DISTANCE_FUNCS.squareDistanceHandle7u();
 
-    static void dotProduct7uBulk(MemorySegment a, MemorySegment b, int length, int count, MemorySegment scores) {
-        try {
-            DOT_PRODUCT_7U_BULK.invokeExact(a, b, length, count, scores);
-        } catch (Throwable e) {
-            if (e instanceof Error err) {
-                throw err;
-            } else if (e instanceof RuntimeException re) {
-                throw re;
-            } else {
-                throw new RuntimeException(e);
-            }
-        }
-    }
-
     static int dotProduct7u(MemorySegment a, MemorySegment b, int length) {
         try {
             return (int) DOT_PRODUCT_7U.invokeExact(a, b, length);