Merge branch 'main' into settings/watch_after_apply

Author: rjernst

.buildkite/pipelines/java-ea-check-new-build.yml

@@ -0,0 +1,7 @@
+steps:
+  - command: .buildkite/scripts/trigger-if-java-ea-new-build.sh
+    env:
+      RECENT_TIME_WINDOW: "24" # time window to consider a build as new in hours
+    agents:
+      image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5"
+      memory: "4G"

.buildkite/pipelines/pull-request-transport-versions.yml

@@ -0,0 +1 @@
+steps: []

.buildkite/scripts/trigger-if-java-ea-new-build.sh

@@ -0,0 +1,95 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the "Elastic License
+# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+# Public License v 1"; you may not use this file except in compliance with, at
+# your election, the "Elastic License 2.0", the "GNU Affero General Public
+# License v3.0 only", or the "Server Side Public License, v 1".
+#
+
+#!/bin/bash
+
+# Allow overriding the time window (in hours) to check for new builds, defaults to 24
+RECENT_TIME_WINDOW=${RECENT_TIME_WINDOW:-24}
+
+# Extract current JDK major version from bundled_jdk in version.properties
+CURRENT_JDK=$(grep "^bundled_jdk =" build-tools-internal/version.properties | cut -d'=' -f2 | tr -d ' ' | cut -d'.' -f1)
+TARGET_JDK=$((CURRENT_JDK + 1))
+
+echo "Current JDK major version: $CURRENT_JDK"
+echo "Target JDK major version: $TARGET_JDK"
+
+# Query Elasticsearch JDK archive for available JDKs
+JDK_ARCHIVE_URL="https://builds.es-jdk-archive.com/jdks/openjdk/recent.json"
+echo "Querying JDK archive: $JDK_ARCHIVE_URL"
+
+# Fetch JDK info and filter for target major version
+JDK_DATA=$(curl -s "$JDK_ARCHIVE_URL")
+
+if [[ -z "$JDK_DATA" ]]; then
+  echo "Failed to fetch JDK data from archive"
+  exit 1
+fi
+
+# Find the latest build for the target JDK version
+LATEST_BUILD=$(echo "$JDK_DATA" | jq -r --arg target "$TARGET_JDK" '
+  .majors[$target].builds |
+  sort_by(.archived_at) |
+  last'
+)
+
+if [[ "$LATEST_BUILD" == "null" || -z "$LATEST_BUILD" ]]; then
+  echo "No builds found for JDK $TARGET_JDK"
+  exit 1
+fi
+
+# Extract timestamp and JDK identifier
+TIMESTAMP=$(echo "$LATEST_BUILD" | jq -r '.archived_at')
+JDK_IDENTIFIER=$(echo "$LATEST_BUILD" | jq -r '.id')
+
+echo "Latest JDK ${TARGET_JDK} build from ES archive:"
+echo "  Timestamp: $TIMESTAMP"
+echo "  JDK Identifier: $JDK_IDENTIFIER"
+
+# Set variables for use in the pipeline trigger
+jdkbuild="$JDK_IDENTIFIER"
+jdk_timestamp="$TIMESTAMP"
+
+# Check if timestamp is within last 24 hours
+CURRENT_TIME=$(date +%s)
+BUILD_TIME=$(date -d "$TIMESTAMP" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%S" "${TIMESTAMP%Z}" +%s 2>/dev/null || echo "0")
+
+if [[ "$BUILD_TIME" == "0" ]]; then
+  echo "Failed to parse timestamp: $TIMESTAMP"
+  SHOULD_TRIGGER="false"
+else
+  TIME_DIFF=$((CURRENT_TIME - BUILD_TIME))
+  TIME_WINDOW=$((RECENT_TIME_WINDOW * 60 * 60))
+
+  if [[ $TIME_DIFF -lt $TIME_WINDOW ]]; then
+    echo "Build is recent (less than ${RECENT_TIME_WINDOW}h old)"
+    SHOULD_TRIGGER="true"
+  else
+    echo "Build is older than ${RECENT_TIME_WINDOW} hours"
+    SHOULD_TRIGGER="false"
+  fi
+fi
+
+echo "SHOULD_TRIGGER: $SHOULD_TRIGGER"
+
+
+if [[ "$SHOULD_TRIGGER" == "true" ]]; then
+  EFFECTIVE_START_DATE=$(date -u -d "@$BUILD_TIME" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -r "$BUILD_TIME" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "")
+  echo "Triggering performance-esbench-jdk for new jdk build $JDK_IDENTIFIER"
+  cat << EOF | buildkite-agent pipeline upload
+steps:
+- trigger: elasticsearch-performance-esbench-jdk
+  label: Triggering performance-esbench-jdk for new jdk build $JDK_IDENTIFIER
+  async: true
+  build:
+    branch: "$BUILDKITE_BRANCH"
+    env:
+      EFFECTIVE_START_DATE: "$EFFECTIVE_START_DATE"
+      EXECUTION_MODE: "start-run"
+EOF
+fi

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/ComputeNeighboursBenchmark.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.vector;
+
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.index.codec.vectors.cluster.NeighborHood;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@State(Scope.Benchmark)
+// first iteration is complete garbage, so make sure we really warmup
+@Warmup(iterations = 1, time = 1)
+// real iterations. not useful to spend tons of time here, better to fork more
+@Measurement(iterations = 3, time = 1)
+// engage some noise reduction
+@Fork(value = 1)
+public class ComputeNeighboursBenchmark {
+
+    static {
+        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+    }
+
+    @Param({ "1000", "2000", "3000", "5000", "10000", "20000", "50000" })
+    int numVectors;
+
+    @Param({ "384", "782", "1024" })
+    int dims;
+
+    float[][] vectors;
+    int clusterPerNeighbour = 128;
+
+    @Setup
+    public void setup() throws IOException {
+        Random random = new Random(123);
+        vectors = new float[numVectors][dims];
+        for (float[] vector : vectors) {
+            for (int i = 0; i < dims; i++) {
+                vector[i] = random.nextFloat();
+            }
+        }
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void bruteForce(Blackhole bh) {
+        bh.consume(NeighborHood.computeNeighborhoodsBruteForce(vectors, clusterPerNeighbour));
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void graph(Blackhole bh) throws IOException {
+        bh.consume(NeighborHood.computeNeighborhoodsGraph(vectors, clusterPerNeighbour));
+    }
+}

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy

@@ -14,6 +14,7 @@ import org.gradle.testkit.runner.BuildResult
 import org.gradle.testkit.runner.TaskOutcome
 
 class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
+
     def javaResource(String project, String path, String content) {
         file("${project}/src/main/resources/${path}").withWriter { writer ->
             writer << content
@@ -45,10 +46,18 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
         return referableAndReferencedTransportVersion(name, ids, "Test${name.capitalize()}")
     }
 
-    def referableAndReferencedTransportVersion(String name, String ids, String classname) {
+    def referencedTransportVersion(String name) {
+        referencedTransportVersion(name, "Test${name.capitalize()}")
+    }
+
+    def referencedTransportVersion(String name, String classname) {
         javaSource("myserver", "org.elasticsearch", classname, "", """
             static final TransportVersion usage = TransportVersion.fromName("${name}");
         """)
+    }
+
+    def referableAndReferencedTransportVersion(String name, String ids, String classname) {
+        referencedTransportVersion(name, classname)
         referableTransportVersion(name, ids)
     }
 
@@ -74,6 +83,20 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
         assertOutputContains(result.output, expectedOutput)
     }
 
+    void assertReferableDefinition(String name, String content) {
+        File definitionFile = file("myserver/src/main/resources/transport/definitions/referable/${name}.csv")
+        assert definitionFile.exists()
+        assert definitionFile.text.strip() == content
+    }
+
+    void assertReferableDefinitionDoesNotExist(String name) {
+        assert file("myserver/src/main/resources/transport/definitions/referable/${name}.csv").exists() == false
+    }
+
+    void assertUpperBound(String name, String content) {
+        assert file("myserver/src/main/resources/transport/upper_bounds/${name}.csv").text.strip() == content
+    }
+
     def setup() {
         configurationCacheCompatible = false
         internalBuild()
@@ -86,12 +109,17 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
             apply plugin: 'java-library'
             apply plugin: 'elasticsearch.transport-version-references'
             apply plugin: 'elasticsearch.transport-version-resources'
+
+            tasks.named('generateTransportVersionDefinition') {
+                currentUpperBoundName = '9.2'
+            }
         """
         referableTransportVersion("existing_91", "8012000")
         referableTransportVersion("existing_92", "8123000,8012001")
         unreferableTransportVersion("initial_9_0_0", "8000000")
         transportVersionUpperBound("9.2", "existing_92", "8123000")
         transportVersionUpperBound("9.1", "existing_92", "8012001")
+        transportVersionUpperBound("9.0", "initial_9_0_0", "8000000")
         // a mock version of TransportVersion, just here so we can compile Dummy.java et al
         javaSource("myserver", "org.elasticsearch", "TransportVersion", "", """
             public static TransportVersion fromName(String name) {