Merge branch 'main' into 2025/10/07/refactor-MoveDecision-method-names

Author: DiannaHohensee

.buildkite/scripts/generate-pr-performance-benchmark.sh

@@ -46,6 +46,7 @@ steps:
         CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
         ENV_ID: ${env_id_baseline}
         REVISION: ${merge_base}
+        BENCHMARK_TYPE: baseline
   - label: Trigger contender benchmark with ${GITHUB_PR_TRIGGERED_SHA:0:7}
     trigger: elasticsearch-performance-esbench-pr
     build:
@@ -56,6 +57,7 @@ steps:
         ENV_ID: ${env_id_contender}
         ES_REPO_URL: https://github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}.git
         REVISION: ${GITHUB_PR_TRIGGERED_SHA}
+        BENCHMARK_TYPE: contender
   - wait: ~
   - label: Update PR comment and Buildkite annotation
     command: |

.gitignore

@@ -46,6 +46,7 @@ build/
 **/.local*
 .vagrant/
 /logs/
+**/target/
 
 # osx stuff
 .DS_Store

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java

@@ -51,6 +51,7 @@
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.BlockLoader;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
+import org.elasticsearch.index.mapper.IndexType;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
@@ -290,9 +291,8 @@ private static BlockLoader numericBlockLoader(WhereAndBaseName w, NumberFieldMap
         return new NumberFieldMapper.NumberFieldType(
             w.name,
             numberType,
-            true,
+            IndexType.points(true, docValues),
             stored,
-            docValues,
             true,
             null,
             Map.of(),

benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/RecyclerBytesStreamOutputBenchmark.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.benchmark.bytes;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.benchmark.common.util.UTF8StringBytesBenchmark;
 import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput;
 import org.elasticsearch.common.recycler.Recycler;
 import org.openjdk.jmh.annotations.Benchmark;
@@ -65,10 +66,10 @@ public void initResults() throws IOException {
         // We use weights to generate certain sized UTF-8 characters and vInts. However, there is still some non-determinism which could
         // impact direct comparisons run-to-run
 
-        shortString = generateAsciiString(20);
-        longString = generateAsciiString(100);
-        nonAsciiString = generateUtf8String(200);
-        veryLongString = generateAsciiString(800);
+        shortString = UTF8StringBytesBenchmark.generateAsciiString(20);
+        longString = UTF8StringBytesBenchmark.generateAsciiString(100);
+        nonAsciiString = UTF8StringBytesBenchmark.generateUTF8String(200);
+        veryLongString = UTF8StringBytesBenchmark.generateAsciiString(800);
         // vint values for benchmarking
         vints = new int[1000];
         for (int i = 0; i < vints.length; i++) {
@@ -143,49 +144,6 @@ public void writeVInt() throws IOException {
         }
     }
 
-    public static String generateAsciiString(int n) {
-        ThreadLocalRandom random = ThreadLocalRandom.current();
-        StringBuilder sb = new StringBuilder(n);
-
-        for (int i = 0; i < n; i++) {
-            int ascii = random.nextInt(128);
-            sb.append((char) ascii);
-        }
-
-        return sb.toString();
-    }
-
-    public static String generateUtf8String(int n) {
-        ThreadLocalRandom random = ThreadLocalRandom.current();
-        StringBuilder sb = new StringBuilder(n);
-
-        for (int i = 0; i < n; i++) {
-            int codePoint;
-            int probability = random.nextInt(100);
-
-            if (probability < 85) {
-                // 1-byte UTF-8 (ASCII range)
-                // 0x0000 to 0x007F
-                codePoint = random.nextInt(0x0080);
-            } else if (probability < 95) {
-                // 2-byte UTF-8
-                // 0x0080 to 0x07FF
-                codePoint = random.nextInt(0x0080, 0x0800);
-            } else {
-                // 3-byte UTF-8
-                // 0x0800 to 0xFFFF
-                do {
-                    codePoint = random.nextInt(0x0800, 0x10000);
-                    // Skip surrogate pairs (0xD800-0xDFFF)
-                } while (codePoint >= 0xD800 && codePoint <= 0xDFFF);
-            }
-
-            sb.appendCodePoint(codePoint);
-        }
-
-        return sb.toString();
-    }
-
     private record BenchmarkRecycler(AtomicReference<BytesRef> bytesRef) implements Recycler<BytesRef> {
 
         @Override

benchmarks/src/main/java/org/elasticsearch/benchmark/common/util/UTF8StringBytesBenchmark.java

@@ -0,0 +1,148 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.common.util;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.UnicodeUtil;
+import org.elasticsearch.common.UUIDs;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+@Warmup(iterations = 3)
+@Measurement(iterations = 3)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Fork(value = 1)
+public class UTF8StringBytesBenchmark {
+
+    @State(Scope.Thread)
+    public static class StringState {
+        @Param({ "uuid", "short", "long", "nonAscii", "veryLong" })
+        String stringType;
+
+        String string;
+        BytesRef bytes;
+
+        @Setup
+        public void setup() {
+            string = switch (stringType) {
+                case "uuid" -> UUIDs.base64UUID();
+                case "short" -> generateAsciiString(20);
+                case "long" -> generateAsciiString(100);
+                case "nonAscii" -> generateUTF8String(200);
+                case "veryLong" -> generateAsciiString(1000);
+                default -> throw new IllegalArgumentException("Unknown stringType: " + stringType);
+            };
+            bytes = getBytes(string);
+        }
+    }
+
+    @Benchmark
+    public BytesRef getBytesJDK(StringState state) {
+        byte[] bytes = state.string.getBytes(StandardCharsets.UTF_8);
+        return new BytesRef(bytes, 0, bytes.length);
+    }
+
+    @Benchmark
+    public BytesRef getBytesUnicodeUtils(StringState state) {
+        String string = state.string;
+        int length = string.length();
+        int size = UnicodeUtil.calcUTF16toUTF8Length(string, 0, length);
+        byte[] out = new byte[size];
+        UnicodeUtil.UTF16toUTF8(string, 0, length, out, 0);
+        return new BytesRef(out, 0, out.length);
+    }
+
+    @Benchmark
+    public BytesRef getBytesByteBufferEncoder(StringState state) {
+        var byteBuff = StandardCharsets.UTF_8.encode(state.string);
+        assert byteBuff.hasArray();
+        return new BytesRef(byteBuff.array(), byteBuff.arrayOffset() + byteBuff.position(), byteBuff.remaining());
+    }
+
+    @Benchmark
+    public String getStringJDK(StringState state) {
+        BytesRef bytes = state.bytes;
+        return new String(bytes.bytes, bytes.offset, bytes.length, StandardCharsets.UTF_8);
+    }
+
+    @Benchmark
+    public String getStringByteBufferDecoder(StringState state) {
+        BytesRef bytes = state.bytes;
+        var byteBuff = ByteBuffer.wrap(bytes.bytes, bytes.offset, bytes.length);
+        return StandardCharsets.UTF_8.decode(byteBuff).toString();
+    }
+
+    private static BytesRef getBytes(String string) {
+        int before = ThreadLocalRandom.current().nextInt(0, 50);
+        int after = ThreadLocalRandom.current().nextInt(0, 50);
+        byte[] stringBytes = string.getBytes(StandardCharsets.UTF_8);
+        byte[] finalBytes = new byte[before + after + stringBytes.length];
+        System.arraycopy(stringBytes, 0, finalBytes, before, stringBytes.length);
+        return new BytesRef(finalBytes, before, stringBytes.length);
+    }
+
+    public static String generateAsciiString(int n) {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        StringBuilder sb = new StringBuilder(n);
+
+        for (int i = 0; i < n; i++) {
+            int ascii = random.nextInt(128);
+            sb.append((char) ascii);
+        }
+
+        return sb.toString();
+    }
+
+    public static String generateUTF8String(int n) {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        StringBuilder sb = new StringBuilder(n);
+
+        for (int i = 0; i < n; i++) {
+            int codePoint;
+            int probability = random.nextInt(100);
+
+            if (probability < 85) {
+                // 1-byte UTF-8 (ASCII range)
+                // 0x0000 to 0x007F
+                codePoint = random.nextInt(0x0080);
+            } else if (probability < 95) {
+                // 2-byte UTF-8
+                // 0x0080 to 0x07FF
+                codePoint = random.nextInt(0x0080, 0x0800);
+            } else {
+                // 3-byte UTF-8
+                // 0x0800 to 0xFFFF
+                do {
+                    codePoint = random.nextInt(0x0800, 0x10000);
+                    // Skip surrogate pairs (0xD800-0xDFFF)
+                } while (codePoint >= 0xD800 && codePoint <= 0xDFFF);
+            }
+
+            sb.appendCodePoint(codePoint);
+        }
+
+        return sb.toString();
+    }
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.mapper.IndexType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType;
@@ -85,7 +86,23 @@ public class ScriptScoreBenchmark {
     private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class).toList());
 
     private final Map<String, MappedFieldType> fieldTypes = Map.ofEntries(
-        Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null, null, false))
+        Map.entry(
+            "n",
+            new NumberFieldType(
+                "n",
+                NumberType.LONG,
+                IndexType.docValuesOnly(),
+                false,
+                true,
+                null,
+                Map.of(),
+                null,
+                false,
+                null,
+                null,
+                false
+            )
+        )
     );
     private final IndexFieldDataCache fieldDataCache = new IndexFieldDataCache.None();
     private final CircuitBreakerService breakerService = new NoneCircuitBreakerService();

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy

@@ -151,7 +151,11 @@ class BuildPluginFuncTest extends AbstractGradleFuncTest {
             tasks.named('checkstyleMain').configure { enabled = false }
             tasks.named('loggerUsageCheck').configure { enabled = false }
             // tested elsewhere
-            tasks.named('thirdPartyAudit').configure { enabled = false }
+            tasks.named('thirdPartyAudit').configure {
+                getRuntimeJavaVersion().set(JavaVersion.VERSION_21)
+                getTargetCompatibility().set(JavaVersion.VERSION_21)
+                enabled = false
+            }
             """
         when:
         def result = gradleRunner("check").build()
@@ -167,6 +171,25 @@ class BuildPluginFuncTest extends AbstractGradleFuncTest {
         result.task(":loggerUsageCheck").outcome == TaskOutcome.SKIPPED
     }
 
+    def "can generate dependency infos file"() {
+        given:
+        repository.generateJar("junit", "junit", "4.12", 'org.acme.JunitMock')
+        repository.configureBuild(buildFile)
+        file("licenses/junit-4.12.jar.sha1").text = "2973d150c0dc1fefe998f834810d68f278ea58ec"
+        file("licenses/junit-LICENSE.txt").text = EXAMPLE_LICENSE
+        file("licenses/junit-NOTICE.txt").text = "mock notice"
+        buildFile << """
+        dependencies {
+            api "junit:junit:4.12"
+        }
+        """
+        when:
+        def result = gradleRunner("dependenciesInfo").build()
+        then:
+        result.task(":dependenciesInfo").outcome == TaskOutcome.SUCCESS
+        file("build/reports/dependencies/dependencies.csv").text == "junit:junit,4.12,https://repo1.maven.org/maven2/junit/junit/4.12,BSD-3-Clause,\n"
+    }
+
     def assertValidJar(File jar) {
         try (ZipFile zipFile = new ZipFile(jar)) {
             ZipEntry licenseEntry = zipFile.getEntry("META-INF/LICENSE.txt")

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -7,15 +7,13 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import java.lang.management.ManagementFactory;
-import java.time.LocalDateTime;
 
-import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.OS
-import static org.elasticsearch.gradle.internal.util.CiUtils.safeName
-
 import java.lang.management.ManagementFactory
 import java.time.LocalDateTime
+import org.elasticsearch.gradle.Architecture
+
+import static org.elasticsearch.gradle.internal.util.CiUtils.safeName
 
 // Resolving this early to avoid issues with the build scan plugin in combination with the configuration cache usage
 def taskNames = gradle.startParameter.taskNames.join(' ')
@@ -32,10 +30,10 @@ develocity {
     // Automatically publish scans from Elasticsearch CI
     if (onCI) {
       publishing.onlyIf { true }
-      if(server.isPresent() == false) {
+      if (server.isPresent() == false) {
         server = 'https://gradle-enterprise.elastic.co'
       }
-    } else if( server.isPresent() == false) {
+    } else if (server.isPresent() == false) {
       publishing.onlyIf { false }
     }
 
@@ -99,6 +97,15 @@ develocity {
           tag 'pull-request'
           link 'Source', "${prBaseUrl}/tree/${System.getenv('BUILDKITE_COMMIT')}"
           link 'Pull Request', "https://github.com/${repository}/pull/${prId}"
+        } else if (System.getenv('ELASTICSEARCH_PR_NUMBER')) {
+          // For tracking es-pr-check failures in the serverless repo caused by elasticsearch PRs
+          def esPr = System.getenv('ELASTICSEARCH_PR_NUMBER')
+          def esCommit = System.getenv('ELASTICSEARCH_SUBMODULE_COMMIT')
+          value 'Git Commit ID', esCommit
+          tag "pr/${esPr}"
+          tag 'pull-request'
+          link 'Source', "https://github.com/elastic/elasticsearch/pull/${esPr}/commits/${esCommit}"
+          link 'Pull Request', "https://github.com/elastic/elasticsearch/pull/${esPr}"
         } else {
           value 'Git Commit ID', gitRevision.get()
           link 'Source', "https://github.com/${repository}/tree/${gitRevision.get()}"