Merge branch 'main' into default_elser_on_eis_semantic

Author: mridula-s109

.buildkite/scripts/dra-workflow.trigger.sh

@@ -46,4 +46,18 @@ for BRANCH in "${BRANCHES[@]}"; do
         DRA_WORKFLOW: staging
         VERSION_QUALIFIER: ${VERSION_QUALIFIER:-}
 EOF
+
+  if [ "$BRANCH" = "7.17" ]; then
+    cat <<EOF
+  - trigger: elasticsearch-dra-workflow
+    label: Trigger DRA snapshot workflow for $BRANCH
+    async: true
+    build:
+      branch: "$BRANCH"
+      commit: "$LAST_GOOD_COMMIT"
+      env:
+        DRA_WORKFLOW: snapshot
+        VERSION_QUALIFIER: ${VERSION_QUALIFIER:-}
+EOF
+  fi
 done

.buildkite/scripts/generate-pr-performance-benchmark.sh

@@ -46,6 +46,7 @@ steps:
         CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
         ENV_ID: ${env_id_baseline}
         REVISION: ${merge_base}
+        BENCHMARK_TYPE: baseline
   - label: Trigger contender benchmark with ${GITHUB_PR_TRIGGERED_SHA:0:7}
     trigger: elasticsearch-performance-esbench-pr
     build:
@@ -56,6 +57,7 @@ steps:
         ENV_ID: ${env_id_contender}
         ES_REPO_URL: https://github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}.git
         REVISION: ${GITHUB_PR_TRIGGERED_SHA}
+        BENCHMARK_TYPE: contender
   - wait: ~
   - label: Update PR comment and Buildkite annotation
     command: |

.gitignore

@@ -46,6 +46,7 @@ build/
 **/.local*
 .vagrant/
 /logs/
+**/target/
 
 # osx stuff
 .DS_Store

.idea/inspectionProfiles/Project_Default.xml

@@ -119,7 +119,7 @@ public void setup() {
     }
 
     private LogicalPlan plan(EsqlParser parser, Analyzer analyzer, LogicalPlanOptimizer optimizer, String query) {
-        var parsed = parser.createStatement(query, new QueryParams(), telemetry, config);
+        var parsed = parser.createStatement(query, new QueryParams(), telemetry);
         var analyzed = analyzer.analyze(parsed);
         var optimized = optimizer.optimize(analyzed);
         return optimized;

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -41,8 +41,9 @@
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.LongVector;
 import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.lucene.AlwaysReferencedIndexedByShardId;
+import org.elasticsearch.compute.lucene.IndexedByShardIdFromSingleton;
 import org.elasticsearch.compute.lucene.LuceneSourceOperator;
-import org.elasticsearch.compute.lucene.ShardRefCounted;
 import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperator;
 import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperatorStatus;
 import org.elasticsearch.compute.operator.topn.TopNOperator;
@@ -368,7 +369,7 @@ public void benchmark() {
             blockFactory,
             ByteSizeValue.ofMb(1).getBytes(),
             fields(name),
-            List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> {
+            new IndexedByShardIdFromSingleton<>(new ValuesSourceReaderOperator.ShardContext(reader, () -> {
                 throw new UnsupportedOperationException("can't load _source here");
             }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))),
             0
@@ -538,7 +539,7 @@ private void setupPages() {
                         pages.add(
                             new Page(
                                 new DocVector(
-                                    ShardRefCounted.ALWAYS_REFERENCED,
+                                    AlwaysReferencedIndexedByShardId.INSTANCE,
                                     blockFactory.newConstantIntBlockWith(0, end - begin).asVector(),
                                     blockFactory.newConstantIntBlockWith(ctx.ord, end - begin).asVector(),
                                     docs.build(),
@@ -575,8 +576,7 @@ record ItrAndOrd(PrimitiveIterator.OfInt itr, int ord) {}
                             pages.add(
                                 new Page(
                                     new DocVector(
-
-                                        ShardRefCounted.ALWAYS_REFERENCED,
+                                        AlwaysReferencedIndexedByShardId.INSTANCE,
                                         blockFactory.newConstantIntVector(0, size),
                                         leafs.build(),
                                         docs.build(),
@@ -594,7 +594,7 @@ record ItrAndOrd(PrimitiveIterator.OfInt itr, int ord) {}
                     pages.add(
                         new Page(
                             new DocVector(
-                                ShardRefCounted.ALWAYS_REFERENCED,
+                                AlwaysReferencedIndexedByShardId.INSTANCE,
                                 blockFactory.newConstantIntBlockWith(0, size).asVector(),
                                 leafs.build().asBlock().asVector(),
                                 docs.build(),
@@ -621,8 +621,7 @@ record ItrAndOrd(PrimitiveIterator.OfInt itr, int ord) {}
                         pages.add(
                             new Page(
                                 new DocVector(
-
-                                    ShardRefCounted.ALWAYS_REFERENCED,
+                                    AlwaysReferencedIndexedByShardId.INSTANCE,
                                     blockFactory.newConstantIntVector(0, 1),
                                     blockFactory.newConstantIntVector(next.ord, 1),
                                     blockFactory.newConstantIntVector(next.itr.nextInt(), 1),

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.benchmark.bytes;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.benchmark.common.util.UTF8StringBytesBenchmark;
 import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput;
 import org.elasticsearch.common.recycler.Recycler;
 import org.openjdk.jmh.annotations.Benchmark;
@@ -65,10 +66,10 @@ public void initResults() throws IOException {
         // We use weights to generate certain sized UTF-8 characters and vInts. However, there is still some non-determinism which could
         // impact direct comparisons run-to-run
 
-        shortString = generateAsciiString(20);
-        longString = generateAsciiString(100);
-        nonAsciiString = generateUtf8String(200);
-        veryLongString = generateAsciiString(800);
+        shortString = UTF8StringBytesBenchmark.generateAsciiString(20);
+        longString = UTF8StringBytesBenchmark.generateAsciiString(100);
+        nonAsciiString = UTF8StringBytesBenchmark.generateUTF8String(200);
+        veryLongString = UTF8StringBytesBenchmark.generateAsciiString(800);
         // vint values for benchmarking
         vints = new int[1000];
         for (int i = 0; i < vints.length; i++) {
@@ -143,49 +144,6 @@ public void writeVInt() throws IOException {
         }
     }
 
-    public static String generateAsciiString(int n) {
-        ThreadLocalRandom random = ThreadLocalRandom.current();
-        StringBuilder sb = new StringBuilder(n);
-
-        for (int i = 0; i < n; i++) {
-            int ascii = random.nextInt(128);
-            sb.append((char) ascii);
-        }
-
-        return sb.toString();
-    }
-
-    public static String generateUtf8String(int n) {
-        ThreadLocalRandom random = ThreadLocalRandom.current();
-        StringBuilder sb = new StringBuilder(n);
-
-        for (int i = 0; i < n; i++) {
-            int codePoint;
-            int probability = random.nextInt(100);
-
-            if (probability < 85) {
-                // 1-byte UTF-8 (ASCII range)
-                // 0x0000 to 0x007F
-                codePoint = random.nextInt(0x0080);
-            } else if (probability < 95) {
-                // 2-byte UTF-8
-                // 0x0080 to 0x07FF
-                codePoint = random.nextInt(0x0080, 0x0800);
-            } else {
-                // 3-byte UTF-8
-                // 0x0800 to 0xFFFF
-                do {
-                    codePoint = random.nextInt(0x0800, 0x10000);
-                    // Skip surrogate pairs (0xD800-0xDFFF)
-                } while (codePoint >= 0xD800 && codePoint <= 0xDFFF);
-            }
-
-            sb.appendCodePoint(codePoint);
-        }
-
-        return sb.toString();
-    }
-
     private record BenchmarkRecycler(AtomicReference<BytesRef> bytesRef) implements Recycler<BytesRef> {
 
         @Override

benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/RecyclerBytesStreamOutputBenchmark.java

@@ -0,0 +1,148 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.common.util;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.UnicodeUtil;
+import org.elasticsearch.common.UUIDs;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+@Warmup(iterations = 3)
+@Measurement(iterations = 3)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Fork(value = 1)
+public class UTF8StringBytesBenchmark {
+
+    @State(Scope.Thread)
+    public static class StringState {
+        @Param({ "uuid", "short", "long", "nonAscii", "veryLong" })
+        String stringType;
+
+        String string;
+        BytesRef bytes;
+
+        @Setup
+        public void setup() {
+            string = switch (stringType) {
+                case "uuid" -> UUIDs.base64UUID();
+                case "short" -> generateAsciiString(20);
+                case "long" -> generateAsciiString(100);
+                case "nonAscii" -> generateUTF8String(200);
+                case "veryLong" -> generateAsciiString(1000);
+                default -> throw new IllegalArgumentException("Unknown stringType: " + stringType);
+            };
+            bytes = getBytes(string);
+        }
+    }
+
+    @Benchmark
+    public BytesRef getBytesJDK(StringState state) {
+        byte[] bytes = state.string.getBytes(StandardCharsets.UTF_8);
+        return new BytesRef(bytes, 0, bytes.length);
+    }
+
+    @Benchmark
+    public BytesRef getBytesUnicodeUtils(StringState state) {
+        String string = state.string;
+        int length = string.length();
+        int size = UnicodeUtil.calcUTF16toUTF8Length(string, 0, length);
+        byte[] out = new byte[size];
+        UnicodeUtil.UTF16toUTF8(string, 0, length, out, 0);
+        return new BytesRef(out, 0, out.length);
+    }
+
+    @Benchmark
+    public BytesRef getBytesByteBufferEncoder(StringState state) {
+        var byteBuff = StandardCharsets.UTF_8.encode(state.string);
+        assert byteBuff.hasArray();
+        return new BytesRef(byteBuff.array(), byteBuff.arrayOffset() + byteBuff.position(), byteBuff.remaining());
+    }
+
+    @Benchmark
+    public String getStringJDK(StringState state) {
+        BytesRef bytes = state.bytes;
+        return new String(bytes.bytes, bytes.offset, bytes.length, StandardCharsets.UTF_8);
+    }
+
+    @Benchmark
+    public String getStringByteBufferDecoder(StringState state) {
+        BytesRef bytes = state.bytes;
+        var byteBuff = ByteBuffer.wrap(bytes.bytes, bytes.offset, bytes.length);
+        return StandardCharsets.UTF_8.decode(byteBuff).toString();
+    }
+
+    private static BytesRef getBytes(String string) {
+        int before = ThreadLocalRandom.current().nextInt(0, 50);
+        int after = ThreadLocalRandom.current().nextInt(0, 50);
+        byte[] stringBytes = string.getBytes(StandardCharsets.UTF_8);
+        byte[] finalBytes = new byte[before + after + stringBytes.length];
+        System.arraycopy(stringBytes, 0, finalBytes, before, stringBytes.length);
+        return new BytesRef(finalBytes, before, stringBytes.length);
+    }
+
+    public static String generateAsciiString(int n) {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        StringBuilder sb = new StringBuilder(n);
+
+        for (int i = 0; i < n; i++) {
+            int ascii = random.nextInt(128);
+            sb.append((char) ascii);
+        }
+
+        return sb.toString();
+    }
+
+    public static String generateUTF8String(int n) {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        StringBuilder sb = new StringBuilder(n);
+
+        for (int i = 0; i < n; i++) {
+            int codePoint;
+            int probability = random.nextInt(100);
+
+            if (probability < 85) {
+                // 1-byte UTF-8 (ASCII range)
+                // 0x0000 to 0x007F
+                codePoint = random.nextInt(0x0080);
+            } else if (probability < 95) {
+                // 2-byte UTF-8
+                // 0x0080 to 0x07FF
+                codePoint = random.nextInt(0x0080, 0x0800);
+            } else {
+                // 3-byte UTF-8
+                // 0x0800 to 0xFFFF
+                do {
+                    codePoint = random.nextInt(0x0800, 0x10000);
+                    // Skip surrogate pairs (0xD800-0xDFFF)
+                } while (codePoint >= 0xD800 && codePoint <= 0xDFFF);
+            }
+
+            sb.appendCodePoint(codePoint);
+        }
+
+        return sb.toString();
+    }
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/common/util/UTF8StringBytesBenchmark.java

@@ -15,7 +15,6 @@
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
@@ -29,6 +28,7 @@
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.fielddata.SortedNumericLongValues;
 import org.elasticsearch.index.mapper.IndexType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MappingLookup;
@@ -179,14 +179,14 @@ private ScoreScript.Factory bareMetalScript() {
             return new ScoreScript.LeafFactory() {
                 @Override
                 public ScoreScript newInstance(DocReader docReader) throws IOException {
-                    SortedNumericDocValues values = ifd.load(((DocValuesDocReader) docReader).getLeafReaderContext()).getLongValues();
+                    SortedNumericLongValues values = ifd.load(((DocValuesDocReader) docReader).getLeafReaderContext()).getLongValues();
                     return new ScoreScript(params, null, docReader) {
                         private int docId;
 
                         @Override
                         public double execute(ExplanationHolder explanation) {
                             try {
-                                values.advance(docId);
+                                values.advanceExact(docId);
                                 if (values.docValueCount() != 1) {
                                     throw new IllegalArgumentException("script only works when there is exactly one value");
                                 }