Merge branch 'main' into retry_shard_movements_during_query

Author: idegtiarenko

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.16.7", "8.17.5", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
+            BWC_VERSION: ["8.17.6", "8.18.0", "8.19.0", "9.0.1", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -303,8 +303,8 @@ steps:
         env:
           BWC_VERSION: 8.16.7
 
-      - label: "{{matrix.image}} / 8.17.5 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.5
+      - label: "{{matrix.image}} / 8.17.6 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.6
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -317,7 +317,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.17.5
+          BWC_VERSION: 8.17.6
 
       - label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0
@@ -351,8 +351,8 @@ steps:
         env:
           BWC_VERSION: 8.19.0
 
-      - label: "{{matrix.image}} / 9.0.0 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.0
+      - label: "{{matrix.image}} / 9.0.1 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.1
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -365,7 +365,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.0.0
+          BWC_VERSION: 9.0.1
 
       - label: "{{matrix.image}} / 9.1.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.0

.buildkite/pipelines/periodic.yml

@@ -325,8 +325,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.17.5 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.5#bwcTest
+      - label: 8.17.6 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.6#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -335,7 +335,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.17.5
+          BWC_VERSION: 8.17.6
         retry:
           automatic:
             - exit_status: "-1"
@@ -382,8 +382,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.0 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.0#bwcTest
+      - label: 9.0.1 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.1#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -392,7 +392,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.0
+          BWC_VERSION: 9.0.1
         retry:
           automatic:
             - exit_status: "-1"
@@ -486,7 +486,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.16.7", "8.17.5", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
+            BWC_VERSION: ["8.17.6", "8.18.0", "8.19.0", "9.0.1", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -533,7 +533,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.16.7", "8.17.5", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
+            BWC_VERSION: ["8.17.6", "8.18.0", "8.19.0", "9.0.1", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -16,8 +16,8 @@ BWC_VERSION:
   - "8.14.3"
   - "8.15.5"
   - "8.16.7"
-  - "8.17.5"
+  - "8.17.6"
   - "8.18.0"
   - "8.19.0"
-  - "9.0.0"
+  - "9.0.1"
   - "9.1.0"

.ci/snapshotBwcVersions

@@ -1,7 +1,6 @@
 BWC_VERSION:
-  - "8.16.7"
-  - "8.17.5"
+  - "8.17.6"
   - "8.18.0"
   - "8.19.0"
-  - "9.0.0"
+  - "9.0.1"
   - "9.1.0"

benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java

@@ -55,7 +55,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
-@BenchmarkMode(Mode.SampleTime)
+@BenchmarkMode(Mode.SingleShotTime)
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
 @State(Scope.Benchmark)
 @Fork(1)

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java

@@ -19,6 +19,7 @@
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
 import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 import org.apache.lucene.util.quantization.ScalarQuantizer;
 import org.elasticsearch.common.logging.LogConfigurator;
@@ -76,10 +77,10 @@ public class VectorScorerBenchmark {
     float vec2Offset;
     float scoreCorrectionConstant;
 
-    RandomVectorScorer luceneDotScorer;
-    RandomVectorScorer luceneSqrScorer;
-    RandomVectorScorer nativeDotScorer;
-    RandomVectorScorer nativeSqrScorer;
+    UpdateableRandomVectorScorer luceneDotScorer;
+    UpdateableRandomVectorScorer luceneSqrScorer;
+    UpdateableRandomVectorScorer nativeDotScorer;
+    UpdateableRandomVectorScorer nativeSqrScorer;
 
     RandomVectorScorer luceneDotScorerQuery;
     RandomVectorScorer nativeDotScorerQuery;
@@ -118,12 +119,16 @@ public void setup() throws IOException {
         in = dir.openInput("vector.data", IOContext.DEFAULT);
         var values = vectorValues(dims, 2, in, VectorSimilarityFunction.DOT_PRODUCT);
         scoreCorrectionConstant = values.getScalarQuantizer().getConstantMultiplier();
-        luceneDotScorer = luceneScoreSupplier(values, VectorSimilarityFunction.DOT_PRODUCT).scorer(0);
+        luceneDotScorer = luceneScoreSupplier(values, VectorSimilarityFunction.DOT_PRODUCT).scorer();
+        luceneDotScorer.setScoringOrdinal(0);
         values = vectorValues(dims, 2, in, VectorSimilarityFunction.EUCLIDEAN);
-        luceneSqrScorer = luceneScoreSupplier(values, VectorSimilarityFunction.EUCLIDEAN).scorer(0);
+        luceneSqrScorer = luceneScoreSupplier(values, VectorSimilarityFunction.EUCLIDEAN).scorer();
+        luceneSqrScorer.setScoringOrdinal(0);
 
-        nativeDotScorer = factory.getInt7SQVectorScorerSupplier(DOT_PRODUCT, in, values, scoreCorrectionConstant).get().scorer(0);
-        nativeSqrScorer = factory.getInt7SQVectorScorerSupplier(EUCLIDEAN, in, values, scoreCorrectionConstant).get().scorer(0);
+        nativeDotScorer = factory.getInt7SQVectorScorerSupplier(DOT_PRODUCT, in, values, scoreCorrectionConstant).get().scorer();
+        nativeDotScorer.setScoringOrdinal(0);
+        nativeSqrScorer = factory.getInt7SQVectorScorerSupplier(EUCLIDEAN, in, values, scoreCorrectionConstant).get().scorer();
+        nativeSqrScorer.setScoringOrdinal(0);
 
         // setup for getInt7SQVectorScorer / query vector scoring
         float[] queryVec = new float[dims];

branches.json

@@ -4,9 +4,6 @@
     {
       "branch": "main"
     },
-    {
-      "branch": "8.16"
-    },
     {
       "branch": "9.0"
     },

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/Awsv2ClassPatcher.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.dependencies.patches.awsv2sdk;
+
+import org.elasticsearch.gradle.internal.dependencies.patches.PatcherInfo;
+import org.elasticsearch.gradle.internal.dependencies.patches.Utils;
+import org.gradle.api.artifacts.transform.CacheableTransform;
+import org.gradle.api.artifacts.transform.InputArtifact;
+import org.gradle.api.artifacts.transform.TransformAction;
+import org.gradle.api.artifacts.transform.TransformOutputs;
+import org.gradle.api.artifacts.transform.TransformParameters;
+import org.gradle.api.file.FileSystemLocation;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.tasks.Classpath;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.File;
+import java.util.List;
+
+import static org.elasticsearch.gradle.internal.dependencies.patches.PatcherInfo.classPatcher;
+
+@CacheableTransform
+public abstract class Awsv2ClassPatcher implements TransformAction<TransformParameters.None> {
+
+    private static final String JAR_FILE_TO_PATCH = "aws-query-protocol";
+
+    private static final List<PatcherInfo> CLASS_PATCHERS = List.of(
+        // This patcher is needed because of this AWS bug: https://github.com/aws/aws-sdk-java-v2/issues/5968
+        // As soon as the bug is resolved and we upgrade our AWS SDK v2 libraries, we can remove this.
+        classPatcher(
+            "software/amazon/awssdk/protocols/query/internal/marshall/ListQueryMarshaller.class",
+            "213e84d9a745bdae4b844334d17aecdd6499b36df32aa73f82dc114b35043009",
+            StringFormatInPathResolverPatcher::new
+        )
+    );
+
+    @Classpath
+    @InputArtifact
+    public abstract Provider<FileSystemLocation> getInputArtifact();
+
+    @Override
+    public void transform(@NotNull TransformOutputs outputs) {
+        File inputFile = getInputArtifact().get().getAsFile();
+
+        if (inputFile.getName().startsWith(JAR_FILE_TO_PATCH)) {
+            System.out.println("Patching " + inputFile.getName());
+            File outputFile = outputs.file(inputFile.getName().replace(".jar", "-patched.jar"));
+            Utils.patchJar(inputFile, outputFile, CLASS_PATCHERS);
+        } else {
+            System.out.println("Skipping " + inputFile.getName());
+            outputs.file(getInputArtifact());
+        }
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/awsv2sdk/StringFormatInPathResolverPatcher.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.dependencies.patches.awsv2sdk;
+
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Type;
+
+import java.util.Locale;
+
+import static org.objectweb.asm.Opcodes.ASM9;
+import static org.objectweb.asm.Opcodes.GETSTATIC;
+import static org.objectweb.asm.Opcodes.INVOKESTATIC;
+
+class StringFormatInPathResolverPatcher extends ClassVisitor {
+
+    StringFormatInPathResolverPatcher(ClassWriter classWriter) {
+        super(ASM9, classWriter);
+    }
+
+    @Override
+    public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
+        return new ReplaceCallMethodVisitor(super.visitMethod(access, name, descriptor, signature, exceptions));
+    }
+
+    /**
+     * Replaces calls to String.format(format, args); with calls to String.format(Locale.ROOT, format, args);
+     */
+    private static class ReplaceCallMethodVisitor extends MethodVisitor {
+        private static final String CLASS_INTERNAL_NAME = Type.getInternalName(String.class);
+        private static final String METHOD_NAME = "format";
+        private static final String OLD_METHOD_DESCRIPTOR = Type.getMethodDescriptor(
+            Type.getType(String.class),
+            Type.getType(String.class),
+            Type.getType(Object[].class)
+        );
+        private static final String NEW_METHOD_DESCRIPTOR = Type.getMethodDescriptor(
+            Type.getType(String.class),
+            Type.getType(Locale.class),
+            Type.getType(String.class),
+            Type.getType(Object[].class)
+        );
+
+        private boolean foundFormatPattern = false;
+
+        ReplaceCallMethodVisitor(MethodVisitor methodVisitor) {
+            super(ASM9, methodVisitor);
+        }
+
+        @Override
+        public void visitLdcInsn(Object value) {
+            if (value instanceof String s && s.startsWith("%s")) {
+                if (foundFormatPattern) {
+                    throw new IllegalStateException(
+                        "A previous string format constant was not paired with a String.format() call. "
+                            + "Patching would generate an unbalances stack"
+                    );
+                }
+                // Push the extra arg on the stack
+                mv.visitFieldInsn(GETSTATIC, Type.getInternalName(Locale.class), "ROOT", Type.getDescriptor(Locale.class));
+                foundFormatPattern = true;
+            }
+            super.visitLdcInsn(value);
+        }
+
+        @Override
+        public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) {
+            if (opcode == INVOKESTATIC
+                && foundFormatPattern
+                && CLASS_INTERNAL_NAME.equals(owner)
+                && METHOD_NAME.equals(name)
+                && OLD_METHOD_DESCRIPTOR.equals(descriptor)) {
+                // Replace the call with String.format(Locale.ROOT, format, args)
+                mv.visitMethodInsn(INVOKESTATIC, CLASS_INTERNAL_NAME, METHOD_NAME, NEW_METHOD_DESCRIPTOR, false);
+                foundFormatPattern = false;
+            } else {
+                super.visitMethodInsn(opcode, owner, name, descriptor, isInterface);
+            }
+        }
+    }
+}