Merge branch 'main' into metering/keep_index_uuid

Author: mosche

.backportrc.json

@@ -1,10 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.x", "9.0", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
     "^v9.1.0$" : "main",
-    "^v8.19.0$" : "8.x",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/periodic.template.yml

@@ -237,7 +237,7 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.buildkite/pipelines/periodic.yml

@@ -656,7 +656,7 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.ci/scripts/resolve-dra-manifest.sh

@@ -24,11 +24,6 @@ if [ "$LATEST_VERSION" != "$ES_VERSION" ]; then
   echo "Latest build for '$ARTIFACT' is version $LATEST_VERSION but expected version $ES_VERSION." 1>&2
   NEW_BRANCH=$(echo $ES_VERSION | sed -E "s/([0-9]+\.[0-9]+)\.[0-9]/\1/g")
 
-  # Temporary
-  if [[ "$ES_VERSION" == "8.16.0" ]]; then
-    NEW_BRANCH="8.x"
-  fi
-
   echo "Using branch $NEW_BRANCH instead of $BRANCH." 1>&2
   LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $NEW_BRANCH)
 fi

benchmarks/run.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the "Elastic License
+# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+# Public License v 1"; you may not use this file except in compliance with, at
+# your election, the "Elastic License 2.0", the "GNU Affero General Public
+# License v3.0 only", or the "Server Side Public License, v 1".
+#
+
+EXTRA=""
+POSITIONAL_ARGS=()
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --test)
+      # Get inaccurate results quickly by shortening all measurements
+      # to 50ms each and skip self tests.
+      EXTRA="-r 50ms -w 50ms -jvmArgsAppend -DskipSelfTest=true"
+      shift
+      ;;
+    *)
+      POSITIONAL_ARGS+=("$1")
+      shift
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+run() {
+  ../gradlew run --args "$2 -rf json $EXTRA"
+  mv jmh-result.json build/benchmarks/$1.json
+}
+
+cd "$(dirname "$0")"
+mkdir -p build/benchmarks
+run 'esql_agg' 'AggregatorBenchmark -pgrouping=none,longs -pfilter=none -pblockType=vector_longs,half_null_longs'
+run 'esql_block_keep_mask' 'BlockKeepMaskBenchmark -pdataTypeAndBlockKind=BytesRef/array,BytesRef/vector,long/array,long/vector'
+run 'esql_block_read' 'BlockReadBenchmark -paccessType=sequential'
+run 'esql_eval' 'EvalBenchmark'
+run 'esql_parse_ip' 'ParseIpBenchmark'
+run 'esql_topn' 'TopNBenchmark'
+run 'esql_values_agg' 'ValuesAggregatorBenchmark'
+run 'esql_values_source_reader' 'ValuesSourceReaderBenchmark'

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java

@@ -113,6 +113,12 @@ public class AggregatorBenchmark {
 
     static {
         // Smoke test all the expected values and force loading subclasses more like prod
+        if (false == "true".equals(System.getProperty("skipSelfTest"))) {
+            selfTest();
+        }
+    }
+
+    static void selfTest() {
         try {
             for (String grouping : AggregatorBenchmark.class.getField("grouping").getAnnotationsByType(Param.class)[0].value()) {
                 for (String op : AggregatorBenchmark.class.getField("op").getAnnotationsByType(Param.class)[0].value()) {

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java

@@ -37,6 +37,7 @@
 import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.Random;
+import java.util.stream.IntStream;
 
 public class BlockBenchmark {
     /**
@@ -112,15 +113,15 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
                             blocks[blockIndex] = blockFactory.newBooleanArrayBlock(
                                 values,
                                 totalPositions,
-                                null,
+                                IntStream.rangeClosed(0, totalPositions).toArray(),
                                 null,
                                 Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING
                             );
                         }
                         case "array-multivalue-null" -> {
                             int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
                             int positionCount = firstValueIndexes.length - 1;
-                            BitSet nulls = randomNulls(positionCount);
+                            BitSet nulls = nullsFromFirstValues(firstValueIndexes);
 
                             blocks[blockIndex] = blockFactory.newBooleanArrayBlock(
                                 values,
@@ -141,7 +142,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
                             blocks[blockIndex] = new BooleanBigArrayBlock(
                                 valuesBigArray,
                                 totalPositions,
-                                null,
+                                IntStream.rangeClosed(0, totalPositions).toArray(),
                                 null,
                                 Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
                                 blockFactory
@@ -150,7 +151,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
                         case "big-array-multivalue-null" -> {
                             int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
                             int positionCount = firstValueIndexes.length - 1;
-                            BitSet nulls = randomNulls(positionCount);
+                            BitSet nulls = nullsFromFirstValues(firstValueIndexes);
                             BitArray valuesBigArray = new BitArray(totalPositions, BigArrays.NON_RECYCLING_INSTANCE);
                             for (int i = 0; i < values.length; i++) {
                                 if (values[i]) {
@@ -211,15 +212,15 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
                             blocks[blockIndex] = blockFactory.newBytesRefArrayBlock(
                                 values,
                                 totalPositions,
-                                null,
+                                IntStream.rangeClosed(0, totalPositions).toArray(),
                                 null,
                                 Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING
                             );
                         }
                         case "array-multivalue-null" -> {
                             int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
                             int positionCount = firstValueIndexes.length - 1;
-                            BitSet nulls = randomNulls(positionCount);
+                            BitSet nulls = nullsFromFirstValues(firstValueIndexes);
 
                             blocks[blockIndex] = blockFactory.newBytesRefArrayBlock(
                                 values,
@@ -257,15 +258,15 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
                             blocks[blockIndex] = blockFactory.newDoubleArrayBlock(
                                 values,
                                 totalPositions,
-                                null,
+                                IntStream.rangeClosed(0, totalPositions).toArray(),
                                 null,
                                 Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING
                             );
                         }
                         case "array-multivalue-null" -> {
                             int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
                             int positionCount = firstValueIndexes.length - 1;
-                            BitSet nulls = randomNulls(positionCount);
+                            BitSet nulls = nullsFromFirstValues(firstValueIndexes);
 
                             blocks[blockIndex] = blockFactory.newDoubleArrayBlock(
                                 values,
@@ -284,7 +285,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
                             blocks[blockIndex] = new DoubleBigArrayBlock(
                                 valuesBigArray,
                                 totalPositions,
-                                null,
+                                IntStream.rangeClosed(0, totalPositions).toArray(),
                                 null,
                                 Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
                                 blockFactory
@@ -293,7 +294,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
                         case "big-array-multivalue-null" -> {
                             int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
                             int positionCount = firstValueIndexes.length - 1;
-                            BitSet nulls = randomNulls(positionCount);
+                            BitSet nulls = nullsFromFirstValues(firstValueIndexes);
                             DoubleArray valuesBigArray = blockFactory.bigArrays().newDoubleArray(totalPositions, false);
                             for (int i = 0; i < values.length; i++) {
                                 valuesBigArray.set(i, values[i]);
@@ -344,15 +345,15 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
                             blocks[blockIndex] = blockFactory.newIntArrayBlock(
                                 values,
                                 totalPositions,
-                                null,
+                                IntStream.rangeClosed(0, totalPositions).toArray(),
                                 null,
                                 Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING
                             );
                         }
                         case "array-multivalue-null" -> {
                             int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
                             int positionCount = firstValueIndexes.length - 1;
-                            BitSet nulls = randomNulls(positionCount);
+                            BitSet nulls = nullsFromFirstValues(firstValueIndexes);
 
                             blocks[blockIndex] = blockFactory.newIntArrayBlock(
                                 values,
@@ -371,7 +372,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
                             blocks[blockIndex] = new IntBigArrayBlock(
                                 valuesBigArray,
                                 totalPositions,
-                                null,
+                                IntStream.rangeClosed(0, totalPositions).toArray(),
                                 null,
                                 Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
                                 blockFactory
@@ -380,7 +381,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
                         case "big-array-multivalue-null" -> {
                             int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
                             int positionCount = firstValueIndexes.length - 1;
-                            BitSet nulls = randomNulls(positionCount);
+                            BitSet nulls = nullsFromFirstValues(firstValueIndexes);
                             IntArray valuesBigArray = blockFactory.bigArrays().newIntArray(totalPositions, false);
                             for (int i = 0; i < values.length; i++) {
                                 valuesBigArray.set(i, values[i]);
@@ -431,15 +432,15 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
                             blocks[blockIndex] = blockFactory.newLongArrayBlock(
                                 values,
                                 totalPositions,
-                                null,
+                                IntStream.rangeClosed(0, totalPositions).toArray(),
                                 null,
                                 Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING
                             );
                         }
                         case "array-multivalue-null" -> {
                             int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
                             int positionCount = firstValueIndexes.length - 1;
-                            BitSet nulls = randomNulls(positionCount);
+                            BitSet nulls = nullsFromFirstValues(firstValueIndexes);
 
                             blocks[blockIndex] = blockFactory.newLongArrayBlock(
                                 values,
@@ -458,7 +459,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
                             blocks[blockIndex] = new LongBigArrayBlock(
                                 valuesBigArray,
                                 totalPositions,
-                                null,
+                                IntStream.rangeClosed(0, totalPositions).toArray(),
                                 null,
                                 Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
                                 blockFactory
@@ -467,7 +468,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
                         case "big-array-multivalue-null" -> {
                             int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
                             int positionCount = firstValueIndexes.length - 1;
-                            BitSet nulls = randomNulls(positionCount);
+                            BitSet nulls = nullsFromFirstValues(firstValueIndexes);
                             LongArray valuesBigArray = blockFactory.bigArrays().newLongArray(totalPositions, false);
                             for (int i = 0; i < values.length; i++) {
                                 valuesBigArray.set(i, values[i]);
@@ -526,10 +527,10 @@ private static int[] randomFirstValueIndexes(int totalPositions) {
         return firstValueIndexes.stream().mapToInt(x -> x).toArray();
     }
 
-    private static BitSet randomNulls(int positionCount) {
-        BitSet nulls = new BitSet(positionCount);
-        for (int i = 0; i < positionCount; i++) {
-            if (random.nextDouble() < NULL_PERCENTAGE) {
+    private static BitSet nullsFromFirstValues(int[] firstValueIndexes) {
+        BitSet nulls = new BitSet(firstValueIndexes.length - 1);
+        for (int i = 0; i < firstValueIndexes.length - 1; i++) {
+            if (firstValueIndexes[i + 1] - firstValueIndexes[i] == 1 && random.nextDouble() < NULL_PERCENTAGE) {
                 nulls.set(i);
             }
         }

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockKeepMaskBenchmark.java

@@ -42,7 +42,13 @@
 @Fork(1)
 public class BlockKeepMaskBenchmark extends BlockBenchmark {
     static {
-        // Smoke test all the expected values and force loading subclasses more like prod
+        if (false == "true".equals(System.getProperty("skipSelfTest"))) {
+            // Smoke test all the expected values and force loading subclasses more like prod
+            selfTest();
+        }
+    }
+
+    static void selfTest() {
         int totalPositions = 10;
         for (String paramString : RELEVANT_TYPE_BLOCK_COMBINATIONS) {
             String[] params = paramString.split("/");

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockReadBenchmark.java

@@ -26,6 +26,13 @@
 @Fork(1)
 public class BlockReadBenchmark extends BlockBenchmark {
     static {
+        if (false == "true".equals(System.getProperty("skipSelfTest"))) {
+            // Smoke test all the expected values and force loading subclasses more like prod
+            selfTest();
+        }
+    }
+
+    static void selfTest() {
         // Smoke test all the expected values and force loading subclasses more like prod
         int totalPositions = 10;
         long[] actualCheckSums = new long[NUM_BLOCKS_PER_ITERATION];

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -94,8 +94,10 @@ public class EvalBenchmark {
 
     static {
         LogConfigurator.configureESLogging();
-        // Smoke test all the expected values and force loading subclasses more like prod
-        selfTest();
+        if (false == "true".equals(System.getProperty("skipSelfTest"))) {
+            // Smoke test all the expected values and force loading subclasses more like prod
+            selfTest();
+        }
     }
 
     static void selfTest() {