Merge branch 'main' into chunk-multi-value

Author: mridula-s109

.buildkite/pipelines/periodic-java-ea.yml

@@ -404,8 +404,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.1.11 / bwc
-        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.1.11#bwcTest
+      - label: 9.1.10 / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.1.10#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -414,7 +414,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.1.11
+          BWC_VERSION: 9.1.10
         retry:
           automatic:
             - exit_status: "-1"

.buildkite/pipelines/periodic-packaging.yml

@@ -368,8 +368,8 @@ steps:
         env:
           BWC_VERSION: 9.0.8
 
-      - label: "{{matrix.image}} / 9.1.11 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.11
+      - label: "{{matrix.image}} / 9.1.10 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.10
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -382,7 +382,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.1.11
+          BWC_VERSION: 9.1.10
 
       - label: "{{matrix.image}} / 9.2.6 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.2.6

.buildkite/pipelines/periodic.yml

@@ -457,8 +457,8 @@ steps:
             - exit_status: "1"
               limit: 1
 
-      - label: 9.1.11 / bwc
-        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v9.1.11#bwcTest
+      - label: 9.1.10 / bwc
+        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v9.1.10#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -467,7 +467,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.1.11
+          BWC_VERSION: 9.1.10
         retry:
           automatic:
             - exit_status: "-1"

.ci/bwcVersions

@@ -20,7 +20,7 @@ BWC_VERSION:
   - "8.18.8"
   - "8.19.12"
   - "9.0.8"
-  - "9.1.11"
+  - "9.1.10"
   - "9.2.6"
   - "9.3.0"
   - "9.4.0"

…mark/compute/operator/TopNBenchmark.java …nchmark/_nightly/esql/TopNBenchmark.javabenchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/TopNBenchmark.java benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/TopNBenchmark.java

@@ -7,11 +7,12 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.benchmark.compute.operator;
+package org.elasticsearch.benchmark._nightly.esql;
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
@@ -41,6 +42,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Random;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.IntStream;
 
@@ -57,7 +59,7 @@ public class TopNBenchmark {
         BigArrays.NON_RECYCLING_INSTANCE
     );
 
-    private static final int BLOCK_LENGTH = 8 * 1024;
+    private static final int BLOCK_LENGTH = 4 * 1024;
 
     private static final String LONGS = "longs";
     private static final String INTS = "ints";
@@ -68,6 +70,7 @@ public class TopNBenchmark {
     private static final String LONGS_AND_BYTES_REFS = LONGS + "_and_" + BYTES_REFS;
 
     static {
+        LogConfigurator.configureESLogging();
         // Smoke test all the expected values and force loading subclasses more like prod
         selfTest();
     }
@@ -76,7 +79,9 @@ static void selfTest() {
         try {
             for (String data : TopNBenchmark.class.getField("data").getAnnotationsByType(Param.class)[0].value()) {
                 for (String topCount : TopNBenchmark.class.getField("topCount").getAnnotationsByType(Param.class)[0].value()) {
-                    run(data, Integer.parseInt(topCount));
+                    for (String sortedInput : TopNBenchmark.class.getField("sortedInput").getAnnotationsByType(Param.class)[0].value()) {
+                        run(data, Integer.parseInt(topCount), Boolean.parseBoolean(sortedInput));
+                    }
                 }
             }
         } catch (NoSuchFieldException e) {
@@ -87,10 +92,19 @@ static void selfTest() {
     @Param({ LONGS, INTS, DOUBLES, BOOLEANS, BYTES_REFS, TWO_LONGS, LONGS_AND_BYTES_REFS })
     public String data;
 
-    @Param({ "10", "10000" })
+    @Param({ "true", "false" })
+    public boolean sortedInput;
+
+    /*
+        - 4096 is the page size,
+        - 10000 reflects using a LIMIT with smaller pages, which seems to be a more realistic
+          benchmark than having a LIMIT 10 and receiving pages from the data nodes that
+          contain 4096 documents
+     */
+    @Param({ "10", "1000", "4096", "10000" })
     public int topCount;
 
-    private static Operator operator(String data, int topCount) {
+    private static Operator operator(String data, int topCount, boolean sortedInput) {
         int count = switch (data) {
             case LONGS, INTS, DOUBLES, BOOLEANS, BYTES_REFS -> 1;
             case TWO_LONGS, LONGS_AND_BYTES_REFS -> 2;
@@ -125,8 +139,9 @@ private static Operator operator(String data, int topCount) {
             topCount,
             elementTypes,
             encoders,
-            IntStream.range(0, count).mapToObj(c -> new TopNOperator.SortOrder(c, false, false)).toList(),
-            16 * 1024
+            IntStream.range(0, count).mapToObj(c -> new TopNOperator.SortOrder(c, true, false)).toList(),
+            8 * 1024,
+            sortedInput ? TopNOperator.InputOrdering.SORTED : TopNOperator.InputOrdering.NOT_SORTED
         );
     }
 
@@ -144,41 +159,52 @@ private static Page page(String data) {
         };
     }
 
+    // This creates blocks with uniformly random distributed and sorted data
     private static Block block(String data) {
         return switch (data) {
             case LONGS -> {
                 var builder = blockFactory.newLongBlockBuilder(BLOCK_LENGTH);
-                for (int i = 0; i < BLOCK_LENGTH; i++) {
-                    builder.appendLong(i);
-                }
+
+                new Random().longs(BLOCK_LENGTH, 0, Long.MAX_VALUE).sorted().forEachOrdered(builder::appendLong);
+
                 yield builder.build();
             }
             case INTS -> {
                 var builder = blockFactory.newIntBlockBuilder(BLOCK_LENGTH);
-                for (int i = 0; i < BLOCK_LENGTH; i++) {
-                    builder.appendInt(i);
-                }
+
+                new Random().ints(BLOCK_LENGTH, 0, Integer.MAX_VALUE).sorted().forEachOrdered(builder::appendInt);
+
                 yield builder.build();
             }
             case DOUBLES -> {
                 var builder = blockFactory.newDoubleBlockBuilder(BLOCK_LENGTH);
-                for (int i = 0; i < BLOCK_LENGTH; i++) {
-                    builder.appendDouble(i);
-                }
+
+                new Random().doubles(BLOCK_LENGTH, 0, Double.MAX_VALUE).sorted().forEachOrdered(builder::appendDouble);
+
                 yield builder.build();
             }
             case BOOLEANS -> {
                 BooleanBlock.Builder builder = blockFactory.newBooleanBlockBuilder(BLOCK_LENGTH);
-                for (int i = 0; i < BLOCK_LENGTH; i++) {
-                    builder.appendBoolean(i % 2 == 1);
+
+                int falseCount = BLOCK_LENGTH / 2;
+                int trueCount = BLOCK_LENGTH - falseCount;
+
+                for (int i = 0; i < falseCount; i++) {
+                    builder.appendBoolean(false);
                 }
+                for (int i = 0; i < trueCount; i++) {
+                    builder.appendBoolean(true);
+                }
+
                 yield builder.build();
             }
             case BYTES_REFS -> {
                 BytesRefBlock.Builder builder = blockFactory.newBytesRefBlockBuilder(BLOCK_LENGTH);
-                for (int i = 0; i < BLOCK_LENGTH; i++) {
-                    builder.appendBytesRef(new BytesRef(Integer.toString(i)));
-                }
+                new Random().ints(BLOCK_LENGTH, 0, Integer.MAX_VALUE)
+                    .mapToObj(Integer::toString)
+                    .sorted()
+                    .forEachOrdered(s -> builder.appendBytesRef(new BytesRef(s)));
+
                 yield builder.build();
             }
             default -> throw new UnsupportedOperationException("unsupported data [" + data + "]");
@@ -188,11 +214,11 @@ private static Block block(String data) {
     @Benchmark
     @OperationsPerInvocation(1024 * BLOCK_LENGTH)
     public void run() {
-        run(data, topCount);
+        run(data, topCount, sortedInput);
     }
 
-    private static void run(String data, int topCount) {
-        try (Operator operator = operator(data, topCount)) {
+    private static void run(String data, int topCount, boolean sortedInput) {
+        try (Operator operator = operator(data, topCount, sortedInput)) {
             Page page = page(data);
             for (int i = 0; i < 1024; i++) {
                 operator.addInput(page.shallowCopy());

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerOSQBenchmark.java

@@ -198,7 +198,7 @@ public float[] score() throws IOException {
                     result.upperInterval(),
                     result.quantizedComponentSum(),
                     result.additionalCorrection(),
-                    VectorSimilarityFunction.EUCLIDEAN,
+                    similarityFunction,
                     centroidDp,
                     corrections[0],
                     corrections[1],

…compute/operator/TopNBenchmarkTests.java …rk/_nightly/esql/TopNBenchmarkTests.javabenchmarks/src/test/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmarkTests.java renamed to benchmarks/src/test/java/org/elasticsearch/benchmark/_nightly/esql/TopNBenchmarkTests.java benchmarks/src/test/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmarkTests.java renamed to benchmarks/src/test/java/org/elasticsearch/benchmark/_nightly/esql/TopNBenchmarkTests.java

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.benchmark.compute.operator;
+package org.elasticsearch.benchmark._nightly.esql;
 
 import org.elasticsearch.test.ESTestCase;
 

benchmarks/src/test/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerOSQBenchmarkTests.java

@@ -73,7 +73,7 @@ public void testSingleScalarVsVectorized() throws Exception {
 
                 float[] result = vectorized.score();
 
-                assertArrayEqualsPercent("single scoring, scalar VS vectorized", expected, result, deltaPercent);
+                assertArrayEqualsPercent("single scoring, scalar VS vectorized", expected, result, deltaPercent, DEFAULT_DELTA);
             } finally {
                 scalar.teardown();
                 vectorized.teardown();
@@ -93,6 +93,7 @@ public void testBulkScalarVsVectorized() throws Exception {
                 scalar.dims = dims;
                 scalar.bits = bits;
                 scalar.directoryType = directoryType;
+                scalar.similarityFunction = similarityFunction;
                 scalar.setup(new Random(seed));
 
                 float[] expected = scalar.bulkScore();
@@ -101,11 +102,12 @@ public void testBulkScalarVsVectorized() throws Exception {
                 vectorized.dims = dims;
                 vectorized.bits = bits;
                 vectorized.directoryType = directoryType;
+                vectorized.similarityFunction = similarityFunction;
                 vectorized.setup(new Random(seed));
 
                 float[] result = vectorized.bulkScore();
 
-                assertArrayEqualsPercent("bulk scoring, scalar VS vectorized", expected, result, deltaPercent);
+                assertArrayEqualsPercent("bulk scoring, scalar VS vectorized", expected, result, deltaPercent, DEFAULT_DELTA);
             } finally {
                 scalar.teardown();
                 vectorized.teardown();

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -44,8 +44,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:rolling-upgrade-multi-cluster");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:runtime-fields:core-with-mapped");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:runtime-fields:core-with-search");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:security-example-spi-extension");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:security-setup-password-tests");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:xpack-prefix-rest-compat");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ent-search:qa:rest");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:ccs-rolling-upgrade");
@@ -66,8 +64,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:license-enforcing");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:hdfs");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:rest");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:security:with-ssl");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:security:without-ssl");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:text-structure:qa:text-structure-with-security");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:transform:qa:multi-cluster-tests-with-security");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:transform:qa:multi-node-tests");

build-tools-internal/src/main/resources/checkstyle_suppressions.xml

@@ -14,6 +14,7 @@
   <suppress files="plugin[/\\]eql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]eql[/\\]parser[/\\]EqlBase(Base(Listener|Visitor)|Lexer|Listener|Parser|Visitor).java" checks="." />
   <suppress files="plugin[/\\]esql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]esql[/\\]parser[/\\](EsqlBase|PromqlBase)(Parser|Lexer).*.java" checks="." />
   <suppress files="x-pack[/\\]plugin[/\\]otel-data[/\\]build[/\\]generated[/\\]sources[/\\]" checks="." />
+  <suppress files="x-pack[/\\]plugin[/\\]prometheus[/\\]build[/\\]generated[/\\]sources[/\\]" checks="." />
 
   <!-- JNA requires the no-argument constructor on JNAKernel32Library.SizeT to be public-->
   <suppress files="server[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JNAKernel32Library.java" checks="RedundantModifier" />