Merge branch 'main' into fix/single_project_format

Author: alexey-ivanov-es

.buildkite/pipelines/periodic-java-ea.template.yml

@@ -30,13 +30,12 @@ steps:
       buildDirectory: /dev/shm/bk
   - group: java-matrix
     steps:
-      - label: "{{matrix.GRADLE_TASK}} / java-ea"
+      - label: "{{matrix.GRADLE_TASK}} / {{matrix.image}} "
         command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true $$GRADLE_TASK
         timeout_in_minutes: 300
         matrix:
           setup:
             image:
-              - windows-2025
               - ubuntu-2404
             GRADLE_TASK:
               - checkPart1
@@ -53,6 +52,30 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+      - label: "{{matrix.GRADLE_TASK}} / {{matrix.image}} "
+        command: |
+          .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - windows-2025
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - checkPart4
+              - checkPart5
+              - checkPart6
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          GRADLE_PARAMS: "-Druntime.java=$$JAVA_EA_VERSION"
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
       - label: "{{matrix.BWC_VERSION}} / matrix-bwc"
         command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION  -Dbwc.checkout.align=true v$$BWC_VERSION#bwcTest
         timeout_in_minutes: 300

.buildkite/pipelines/periodic-java-ea.yml

@@ -468,13 +468,12 @@ steps:
       buildDirectory: /dev/shm/bk
   - group: java-matrix
     steps:
-      - label: "{{matrix.GRADLE_TASK}} / java-ea"
+      - label: "{{matrix.GRADLE_TASK}} / {{matrix.image}} "
         command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true $$GRADLE_TASK
         timeout_in_minutes: 300
         matrix:
           setup:
             image:
-              - windows-2025
               - ubuntu-2404
             GRADLE_TASK:
               - checkPart1
@@ -491,6 +490,30 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+      - label: "{{matrix.GRADLE_TASK}} / {{matrix.image}} "
+        command: |
+          .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - windows-2025
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - checkPart4
+              - checkPart5
+              - checkPart6
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          GRADLE_PARAMS: "-Druntime.java=${JAVA_EA_VERSION}"
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
       - label: "{{matrix.BWC_VERSION}} / matrix-bwc"
         command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION  -Dbwc.checkout.align=true v$$BWC_VERSION#bwcTest
         timeout_in_minutes: 300

.buildkite/scripts/windows-run-gradle.sh

@@ -2,4 +2,4 @@
 
 set -euo pipefail
 
-.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $GRADLE_TASK
+.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true ${GRADLE_PARAMS:-} $GRADLE_TASK

.github/CODEOWNERS

@@ -34,6 +34,11 @@ x-pack/plugin/apm-data/src/yamlRestTest/resources @elastic/obs-ds-intake-service
 x-pack/plugin/otel-data/src/main/resources @elastic/obs-ds-intake-services
 x-pack/plugin/otel-data/src/yamlRestTest/resources @elastic/obs-ds-intake-services
 
+# Storage Engine
+x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp @elastic/es-storage-engine
+x-pack/plugin/otel-data/src/test/java/org/elasticsearch/xpack/oteldata/otlp @elastic/es-storage-engine
+x-pack/plugin/otel-data/src/javaRestTest/java/org/elasticsearch/xpack/oteldata/otlp @elastic/es-storage-engine
+
 # Delivery
 gradle @elastic/es-delivery
 build-conventions @elastic/es-delivery

benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java

@@ -130,7 +130,7 @@ private ExponentialHistogram asCompressedHistogram(ExponentialHistogram histogra
             CompressedExponentialHistogram.writeHistogramBytes(histoBytes, histogram.scale(), negativeBuckets, positiveBuckets);
             CompressedExponentialHistogram result = new CompressedExponentialHistogram();
             BytesRef data = histoBytes.bytes().toBytesRef();
-            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), histogram.min(), data);
+            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), histogram.min(), histogram.max(), data);
             return result;
         } catch (IOException e) {
             throw new RuntimeException(e);

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OSQScorerBenchmark.java

@@ -14,9 +14,10 @@
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.MMapDirectory;
-import org.apache.lucene.util.VectorUtil;
+import org.apache.lucene.store.NIOFSDirectory;
 import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
 import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.simdvec.ES91OSQVectorsScorer;
 import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
 import org.openjdk.jmh.annotations.Benchmark;
@@ -29,6 +30,7 @@
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
 
@@ -66,9 +68,14 @@ public class OSQScorerBenchmark {
     float centroidDp;
 
     byte[] scratch;
-    ES91OSQVectorsScorer scorer;
+    ES91OSQVectorsScorer scorerMmap;
+    ES91OSQVectorsScorer scorerNfios;
 
-    IndexInput in;
+    Directory dirMmap;
+    IndexInput inMmap;
+
+    Directory dirNiofs;
+    IndexInput inNiofs;
 
     float[] scratchScores;
     float[] corrections;
@@ -84,18 +91,24 @@ public void setup() throws IOException {
             random.nextBytes(binaryVector);
         }
 
-        Directory dir = new MMapDirectory(Files.createTempDirectory("vectorData"));
-        IndexOutput out = dir.createOutput("vectors", IOContext.DEFAULT);
+        dirMmap = new MMapDirectory(Files.createTempDirectory("vectorDataMmap"));
+        dirNiofs = new NIOFSDirectory(Files.createTempDirectory("vectorDataNFIOS"));
+        IndexOutput outMmap = dirMmap.createOutput("vectors", IOContext.DEFAULT);
+        IndexOutput outNfios = dirNiofs.createOutput("vectors", IOContext.DEFAULT);
         byte[] correctionBytes = new byte[14 * ES91OSQVectorsScorer.BULK_SIZE];
         for (int i = 0; i < numVectors; i += ES91OSQVectorsScorer.BULK_SIZE) {
             for (int j = 0; j < ES91OSQVectorsScorer.BULK_SIZE; j++) {
-                out.writeBytes(binaryVectors[i + j], 0, binaryVectors[i + j].length);
+                outMmap.writeBytes(binaryVectors[i + j], 0, binaryVectors[i + j].length);
+                outNfios.writeBytes(binaryVectors[i + j], 0, binaryVectors[i + j].length);
             }
             random.nextBytes(correctionBytes);
-            out.writeBytes(correctionBytes, 0, correctionBytes.length);
+            outMmap.writeBytes(correctionBytes, 0, correctionBytes.length);
+            outNfios.writeBytes(correctionBytes, 0, correctionBytes.length);
         }
-        out.close();
-        in = dir.openInput("vectors", IOContext.DEFAULT);
+        outMmap.close();
+        outNfios.close();
+        inMmap = dirMmap.openInput("vectors", IOContext.DEFAULT);
+        inNiofs = dirNiofs.openInput("vectors", IOContext.DEFAULT);
 
         binaryQueries = new byte[numVectors][4 * length];
         for (byte[] binaryVector : binaryVectors) {
@@ -110,42 +123,40 @@ public void setup() throws IOException {
         centroidDp = random.nextFloat();
 
         scratch = new byte[length];
-        scorer = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(in, dims);
+        scorerMmap = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(inMmap, dims);
+        scorerNfios = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(inNiofs, dims);
         scratchScores = new float[16];
         corrections = new float[3];
     }
 
+    @TearDown
+    public void teardown() throws IOException {
+        IOUtils.close(dirMmap, inMmap, dirNiofs, inNiofs);
+    }
+
+    @Benchmark
+    public void scoreFromMemorySegmentOnlyVectorMmapScalar(Blackhole bh) throws IOException {
+        scoreFromMemorySegmentOnlyVector(bh, inMmap, scorerMmap);
+    }
+
     @Benchmark
     @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
-    public void scoreFromArray(Blackhole bh) throws IOException {
-        for (int j = 0; j < numQueries; j++) {
-            in.seek(0);
-            for (int i = 0; i < numVectors; i++) {
-                in.readBytes(scratch, 0, length);
-                float qDist = VectorUtil.int4BitDotProduct(binaryQueries[j], scratch);
-                in.readFloats(corrections, 0, corrections.length);
-                int addition = Short.toUnsignedInt(in.readShort());
-                float score = scorer.score(
-                    result.lowerInterval(),
-                    result.upperInterval(),
-                    result.quantizedComponentSum(),
-                    result.additionalCorrection(),
-                    VectorSimilarityFunction.EUCLIDEAN,
-                    centroidDp,
-                    corrections[0],
-                    corrections[1],
-                    addition,
-                    corrections[2],
-                    qDist
-                );
-                bh.consume(score);
-            }
-        }
+    public void scoreFromMemorySegmentOnlyVectorMmapVect(Blackhole bh) throws IOException {
+        scoreFromMemorySegmentOnlyVector(bh, inMmap, scorerMmap);
+    }
+
+    @Benchmark
+    public void scoreFromMemorySegmentOnlyVectorNiofsScalar(Blackhole bh) throws IOException {
+        scoreFromMemorySegmentOnlyVector(bh, inNiofs, scorerNfios);
     }
 
     @Benchmark
     @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
-    public void scoreFromMemorySegmentOnlyVector(Blackhole bh) throws IOException {
+    public void scoreFromMemorySegmentOnlyVectorNiofsVect(Blackhole bh) throws IOException {
+        scoreFromMemorySegmentOnlyVector(bh, inNiofs, scorerNfios);
+    }
+
+    private void scoreFromMemorySegmentOnlyVector(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
         for (int j = 0; j < numQueries; j++) {
             in.seek(0);
             for (int i = 0; i < numVectors; i++) {
@@ -170,9 +181,29 @@ public void scoreFromMemorySegmentOnlyVector(Blackhole bh) throws IOException {
         }
     }
 
+    @Benchmark
+    public void scoreFromMemorySegmentOnlyVectorBulkMmapScalar(Blackhole bh) throws IOException {
+        scoreFromMemorySegmentOnlyVectorBulk(bh, inMmap, scorerMmap);
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void scoreFromMemorySegmentOnlyVectorBulkMmapVect(Blackhole bh) throws IOException {
+        scoreFromMemorySegmentOnlyVectorBulk(bh, inMmap, scorerMmap);
+    }
+
+    @Benchmark
+    public void scoreFromMemorySegmentOnlyVectorBulkNiofsScalar(Blackhole bh) throws IOException {
+        scoreFromMemorySegmentOnlyVectorBulk(bh, inNiofs, scorerNfios);
+    }
+
     @Benchmark
     @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
-    public void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh) throws IOException {
+    public void scoreFromMemorySegmentOnlyVectorBulkNiofsVect(Blackhole bh) throws IOException {
+        scoreFromMemorySegmentOnlyVectorBulk(bh, inNiofs, scorerNfios);
+    }
+
+    private void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
         for (int j = 0; j < numQueries; j++) {
             in.seek(0);
             for (int i = 0; i < numVectors; i += 16) {
@@ -199,9 +230,29 @@ public void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh) throws IOExceptio
         }
     }
 
+    @Benchmark
+    public void scoreFromMemorySegmentAllBulkMmapScalar(Blackhole bh) throws IOException {
+        scoreFromMemorySegmentAllBulk(bh, inMmap, scorerMmap);
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void scoreFromMemorySegmentAllBulkMmapVect(Blackhole bh) throws IOException {
+        scoreFromMemorySegmentAllBulk(bh, inMmap, scorerMmap);
+    }
+
+    @Benchmark
+    public void scoreFromMemorySegmentAllBulkNiofsScalar(Blackhole bh) throws IOException {
+        scoreFromMemorySegmentAllBulk(bh, inNiofs, scorerNfios);
+    }
+
     @Benchmark
     @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
-    public void scoreFromMemorySegmentAllBulk(Blackhole bh) throws IOException {
+    public void scoreFromMemorySegmentAllBulkNiofsVect(Blackhole bh) throws IOException {
+        scoreFromMemorySegmentAllBulk(bh, inNiofs, scorerNfios);
+    }
+
+    private void scoreFromMemorySegmentAllBulk(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
         for (int j = 0; j < numQueries; j++) {
             in.seek(0);
             for (int i = 0; i < numVectors; i += 16) {

branches.json

@@ -1,23 +1,29 @@
 {
-  "notice": "This file is not maintained outside of the main branch and should only be used for tooling.",
-  "branches": [
+  "notice" : "This file is not maintained outside of the main branch and should only be used for tooling.",
+  "branches" : [
     {
-      "branch": "main"
+      "branch" : "main",
+      "version" : "9.2.0"
     },
     {
-      "branch": "9.1"
+      "branch" : "9.1",
+      "version" : "9.1.4"
     },
     {
-      "branch": "9.0"
+      "branch" : "9.0",
+      "version" : "9.0.7"
     },
     {
-      "branch": "8.19"
+      "branch" : "8.19",
+      "version" : "8.19.4"
     },
     {
-      "branch": "8.18"
+      "branch" : "8.18",
+      "version" : "8.18.7"
     },
     {
-      "branch": "7.17"
+      "branch" : "7.17",
+      "version" : "7.17.30"
     }
   ]
 }

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesPlugin.java

@@ -18,6 +18,8 @@
 
 public class VersionPropertiesPlugin implements Plugin<Project> {
 
+    public static final String VERSIONS_EXT = "versions";
+
     @Override
     public void apply(Project project) {
         File workspaceDir = Util.locateElasticsearchWorkspace(project.getGradle());
@@ -28,6 +30,6 @@ public void apply(Project project) {
                 .registerIfAbsent("versions", VersionPropertiesBuildService.class, spec -> {
             spec.getParameters().getInfoPath().set(infoPath);
         });
-        project.getExtensions().add("versions", serviceProvider.get().getProperties());
+        project.getExtensions().add(VERSIONS_EXT, serviceProvider.get().getProperties());
     }
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy

@@ -26,6 +26,17 @@ abstract class AbstractGitAwareGradleFuncTest extends AbstractGradleFuncTest {
         execute("git clone ${remoteGitRepo.absolutePath} cloned", testProjectDir.root)
         buildFile = new File(testProjectDir.root, 'cloned/build.gradle')
         settingsFile = new File(testProjectDir.root, 'cloned/settings.gradle')
+        versionPropertiesFile = new File(testProjectDir.root, 'cloned/build-tools-internal/version.properties')
+        versionPropertiesFile.text = """
+            elasticsearch     = 9.1.0
+            lucene            = 10.2.2
+
+            bundled_jdk_vendor = openjdk
+            bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90
+            minimumJdkVersion = 21
+            minimumRuntimeJava = 21
+            minimumCompilerJava = 21
+        """
     }
 
     File setupGitRemote() {