Merge remote-tracking branch 'origin/main' into lucene_snapshot_10_3

Author: romseygeek

.buildkite/pipelines/java-ea-check-new-build.yml

@@ -0,0 +1,7 @@
+steps:
+  - command: .buildkite/scripts/trigger-if-java-ea-new-build.sh
+    env:
+      RECENT_TIME_WINDOW: "24" # time window to consider a build as new in hours
+    agents:
+      image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5"
+      memory: "4G"

.buildkite/pipelines/periodic-fwc.template.yml

@@ -1,6 +1,6 @@
 steps:
-  - label: $FWC_VERSION / fwc
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
+  - label: "{{matrix.FWC_VERSION}} / fwc"
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
     timeout_in_minutes: 300
     agents:
       provider: gcp
@@ -11,4 +11,4 @@ steps:
       setup:
         FWC_VERSION: $FWC_LIST
     env:
-      FWC_VERSION: $FWC_VERSION
+      FWC_VERSION: "{{matrix.FWC_VERSION}}"

.buildkite/pipelines/periodic-fwc.yml

@@ -1,7 +1,7 @@
 # This file is auto-generated. See .buildkite/pipelines/periodic-fwc.template.yml
 steps:
-  - label: $FWC_VERSION / fwc
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
+  - label: "{{matrix.FWC_VERSION}} / fwc"
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
     timeout_in_minutes: 300
     agents:
       provider: gcp
@@ -12,4 +12,4 @@ steps:
       setup:
         FWC_VERSION: []
     env:
-      FWC_VERSION: $FWC_VERSION
+      FWC_VERSION: "{{matrix.FWC_VERSION}}"

.buildkite/pipelines/pull-request-transport-versions.yml

@@ -0,0 +1 @@
+steps: []

.buildkite/scripts/fwc-branches.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Configure FwC test branches
+# We do not want 7.x branch and only to run for branches that:
+# - have released at least one minor version (not main)
+# - have previous minor unreleased (not the oldest development branch)
+FWC_BRANCHES=()
+for branch in "${BRANCHES[@]}"; do
+  if [[ ! "$branch" =~ ^7\..* ]]; then
+    FWC_BRANCHES+=("$branch")
+  fi
+done
+# Remove first and last element
+FWC_BRANCHES=("${FWC_BRANCHES[@]:1:${#FWC_BRANCHES[@]}-2}")
+
+shouldRunFwcFor() {
+  local branch=$1
+  for fwc_branch in "${FWC_BRANCHES[@]}"; do
+    if [[ "$fwc_branch" == "$branch" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}

.buildkite/scripts/generate-pr-performance-benchmark.sh

@@ -2,39 +2,69 @@
 
 set -euo pipefail
 
+# uncomment for tests
+#function buildkite-agent {
+#  local command=$1
+#  echo "$@"
+#  if [ "$command" == "annotate" ]; then
+#    while read -r line; do
+#      echo "  read: $line";
+#    done
+#  fi
+#}
+
 env_id_baseline=$(python3 -c 'import uuid; print(uuid.uuid4())')
 env_id_contender=$(python3 -c 'import uuid; print(uuid.uuid4())')
 merge_base=$(git merge-base "${GITHUB_PR_TARGET_BRANCH}" HEAD)
 
+# PR comment
+buildkite-agent meta-data set pr_comment:early_comment_job_id "$BUILDKITE_JOB_ID"
 buildkite-agent meta-data set pr_comment:custom-body:body \
-  "This build attempted two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of this PR."
+  "This build attempts two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of this PR. \
+To estimate benchmark completion time inspect previous nightly runs [here](https://buildkite.com/elastic/elasticsearch-performance-esbench-nightly/builds?branch=master)."
 buildkite-agent meta-data set pr_comment:custom-baseline:head \
   "* Baseline: ${merge_base} (env ID ${env_id_baseline})"
 buildkite-agent meta-data set pr_comment:custom-contender:head \
   "* Contender: ${GITHUB_PR_TRIGGERED_SHA} (env ID ${env_id_contender})"
 
-cat << _EOF_
+# Buildkite annotation
+cat << _EOF1_ | buildkite-agent annotate --context "pr-benchmark-notification"
+  This build attempts two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of PR [${GITHUB_PR_NUMBER}](https://github.com/elastic/elasticsearch/pull/${GITHUB_PR_NUMBER}).
+  To estimate benchmark completion time inspect previous nightly runs [here](https://buildkite.com/elastic/elasticsearch-performance-esbench-nightly/builds?branch=master).
+  * Baseline: [${merge_base:0:7}](https://github.com/elastic/elasticsearch/commit/${merge_base}) (env ID ${env_id_baseline})
+  * Contender: [${GITHUB_PR_TRIGGERED_SHA:0:7}](https://github.com/elastic/elasticsearch/commit/${GITHUB_PR_TRIGGERED_SHA}) (env ID ${env_id_contender})
+_EOF1_
+
+cat << _EOF2_
 steps:
-  - label: Trigger baseline benchmark
+  - label: Trigger baseline benchmark with ${merge_base:0:7}
     trigger: elasticsearch-performance-esbench-pr
     build:
-      message: Baseline benchmark for PR${GITHUB_PR_NUMBER}
+      message: Baseline benchmark for PR ${GITHUB_PR_NUMBER} with ${merge_base:0:7}
       branch: master
       env:
         CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
         ENV_ID: ${env_id_baseline}
         REVISION: ${merge_base}
-  - label: Trigger contender benchmark
+  - label: Trigger contender benchmark with ${GITHUB_PR_TRIGGERED_SHA:0:7}
     trigger: elasticsearch-performance-esbench-pr
     build:
-      message: Contender benchmark for PR${GITHUB_PR_NUMBER}
+      message: Contender benchmark for PR ${GITHUB_PR_NUMBER} with ${GITHUB_PR_TRIGGERED_SHA:0:7}
       branch: master
       env:
         CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
         ENV_ID: ${env_id_contender}
         ES_REPO_URL: https://github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}.git
         REVISION: ${GITHUB_PR_TRIGGERED_SHA}
   - wait: ~
-  - label: Modify PR comment
-    command: buildkite-agent meta-data set pr_comment:custom-comparison:head "* [Benchmark results](<https://esbench-metrics.kb.us-east-2.aws.elastic-cloud.com:9243/app/dashboards#/view/d9079962-5866-49ef-b9f5-145f2141cd31?_a=(query:(language:kuery,query:'user-tags.env-id:${env_id_baseline} or user-tags.env-id:${env_id_contender}'))>)"
-_EOF_
+  - label: Update PR comment and Buildkite annotation
+    command: |
+      buildkite-agent meta-data set pr_comment:custom-body:body "This build ran two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of this PR."
+      buildkite-agent meta-data set pr_comment:custom-comparison:head "* [Benchmark results](<https://esbench-metrics.kb.us-east-2.aws.elastic-cloud.com:9243/app/dashboards#/view/d9079962-5866-49ef-b9f5-145f2141cd31?_a=(query:(language:kuery,query:'user-tags.env-id:${env_id_baseline} or user-tags.env-id:${env_id_contender}'))>)"
+      cat << _EOF3_ | buildkite-agent annotate --context "pr-benchmark-notification"
+        This build ran two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of PR [${GITHUB_PR_NUMBER}](https://github.com/elastic/elasticsearch/pull/${GITHUB_PR_NUMBER}).
+        * Baseline: [${merge_base:0:7}](https://github.com/elastic/elasticsearch/commit/${merge_base}) (env ID ${env_id_baseline})
+        * Contender: [${GITHUB_PR_TRIGGERED_SHA:0:7}](https://github.com/elastic/elasticsearch/commit/${GITHUB_PR_TRIGGERED_SHA}) (env ID ${env_id_contender})
+        * [Benchmark results](<https://esbench-metrics.kb.us-east-2.aws.elastic-cloud.com:9243/app/dashboards#/view/d9079962-5866-49ef-b9f5-145f2141cd31?_a=(query:(language:kuery,query:'user-tags.env-id:${env_id_baseline} or user-tags.env-id:${env_id_contender}'))>)
+      _EOF3_
+_EOF2_

.buildkite/scripts/periodic.trigger.sh

@@ -5,6 +5,7 @@ set -euo pipefail
 echo "steps:"
 
 source .buildkite/scripts/branches.sh
+source .buildkite/scripts/fwc-branches.sh
 
 IS_FIRST=true
 SKIP_DELAY="${SKIP_DELAY:-false}"
@@ -46,8 +47,7 @@ EOF
       branch: "$BRANCH"
       commit: "$LAST_GOOD_COMMIT"
 EOF
-# Include forward compatibility tests only for the bugfix branch
-if [[ "${BRANCH}" == "${BRANCHES[2]}" ]]; then
+if shouldRunFwcFor "$BRANCH"; then
   cat <<EOF
   - trigger: elasticsearch-periodic-fwc
     label: Trigger periodic-fwc pipeline for $BRANCH

.buildkite/scripts/trigger-if-java-ea-new-build.sh

@@ -0,0 +1,95 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the "Elastic License
+# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+# Public License v 1"; you may not use this file except in compliance with, at
+# your election, the "Elastic License 2.0", the "GNU Affero General Public
+# License v3.0 only", or the "Server Side Public License, v 1".
+#
+
+#!/bin/bash
+
+# Allow overriding the time window (in hours) to check for new builds, defaults to 24
+RECENT_TIME_WINDOW=${RECENT_TIME_WINDOW:-24}
+
+# Extract current JDK major version from bundled_jdk in version.properties
+CURRENT_JDK=$(grep "^bundled_jdk =" build-tools-internal/version.properties | cut -d'=' -f2 | tr -d ' ' | cut -d'.' -f1)
+TARGET_JDK=$((CURRENT_JDK + 1))
+
+echo "Current JDK major version: $CURRENT_JDK"
+echo "Target JDK major version: $TARGET_JDK"
+
+# Query Elasticsearch JDK archive for available JDKs
+JDK_ARCHIVE_URL="https://builds.es-jdk-archive.com/jdks/openjdk/recent.json"
+echo "Querying JDK archive: $JDK_ARCHIVE_URL"
+
+# Fetch JDK info and filter for target major version
+JDK_DATA=$(curl -s "$JDK_ARCHIVE_URL")
+
+if [[ -z "$JDK_DATA" ]]; then
+  echo "Failed to fetch JDK data from archive"
+  exit 1
+fi
+
+# Find the latest build for the target JDK version
+LATEST_BUILD=$(echo "$JDK_DATA" | jq -r --arg target "$TARGET_JDK" '
+  .majors[$target].builds |
+  sort_by(.archived_at) |
+  last'
+)
+
+if [[ "$LATEST_BUILD" == "null" || -z "$LATEST_BUILD" ]]; then
+  echo "No builds found for JDK $TARGET_JDK"
+  exit 1
+fi
+
+# Extract timestamp and JDK identifier
+TIMESTAMP=$(echo "$LATEST_BUILD" | jq -r '.archived_at')
+JDK_IDENTIFIER=$(echo "$LATEST_BUILD" | jq -r '.id')
+
+echo "Latest JDK ${TARGET_JDK} build from ES archive:"
+echo "  Timestamp: $TIMESTAMP"
+echo "  JDK Identifier: $JDK_IDENTIFIER"
+
+# Set variables for use in the pipeline trigger
+jdkbuild="$JDK_IDENTIFIER"
+jdk_timestamp="$TIMESTAMP"
+
+# Check if timestamp is within last 24 hours
+CURRENT_TIME=$(date +%s)
+BUILD_TIME=$(date -d "$TIMESTAMP" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%S" "${TIMESTAMP%Z}" +%s 2>/dev/null || echo "0")
+
+if [[ "$BUILD_TIME" == "0" ]]; then
+  echo "Failed to parse timestamp: $TIMESTAMP"
+  SHOULD_TRIGGER="false"
+else
+  TIME_DIFF=$((CURRENT_TIME - BUILD_TIME))
+  TIME_WINDOW=$((RECENT_TIME_WINDOW * 60 * 60))
+
+  if [[ $TIME_DIFF -lt $TIME_WINDOW ]]; then
+    echo "Build is recent (less than ${RECENT_TIME_WINDOW}h old)"
+    SHOULD_TRIGGER="true"
+  else
+    echo "Build is older than ${RECENT_TIME_WINDOW} hours"
+    SHOULD_TRIGGER="false"
+  fi
+fi
+
+echo "SHOULD_TRIGGER: $SHOULD_TRIGGER"
+
+
+if [[ "$SHOULD_TRIGGER" == "true" ]]; then
+  EFFECTIVE_START_DATE=$(date -u -d "@$BUILD_TIME" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -r "$BUILD_TIME" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "")
+  echo "Triggering performance-esbench-jdk for new jdk build $JDK_IDENTIFIER"
+  cat << EOF | buildkite-agent pipeline upload
+steps:
+- trigger: elasticsearch-performance-esbench-jdk
+  label: Triggering performance-esbench-jdk for new jdk build $JDK_IDENTIFIER
+  async: true
+  build:
+    branch: "$BUILDKITE_BRANCH"
+    env:
+      EFFECTIVE_START_DATE: "$EFFECTIVE_START_DATE"
+      EXECUTION_MODE: "start-run"
+EOF
+fi

TESTING.asciidoc

@@ -681,12 +681,15 @@ There are multiple base classes for tests:
   directly by unit tests.
 * **`ESSingleNodeTestCase`**: This test case sets up a cluster that has a
   single node.
-* **`ESIntegTestCase`**: An integration test case that creates a cluster that
-  might have multiple nodes.
-* **`ESRestTestCase`**: An integration tests that interacts with an external
-  cluster via the REST API. This is used for Java based REST tests.
-* **`ESClientYamlSuiteTestCase` **: A subclass of `ESRestTestCase` used to run
-  YAML based REST tests.
+* **`ESIntegTestCase`**: An internal integration test that starts nodes within the same JVM as the test.
+    These allow you to test functionality that is not exposed via the REST API, or for verifying a certain internal state.
+    Additionally, you can easily simulate tricky distributed setups that are difficult to do in REST tests.
+    If you only need to start one node, use `ESSingleNodeTestCase` instead, which is a much lighter test setup.
+* **`ESRestTestCase`**: An integration test that interacts with an external
+  cluster via the REST API. This is used for Java based REST tests. This should
+  be the first choice for writing integration tests as these tests run in a much more
+  realistic setup.
+* **`ESClientYamlSuiteTestCase` **: A subclass of `ESRestTestCase` used to run YAML based REST tests.
 
 === Good practices
 

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/ComputeNeighboursBenchmark.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.vector;
+
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.index.codec.vectors.cluster.NeighborHood;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@State(Scope.Benchmark)
+// first iteration is complete garbage, so make sure we really warmup
+@Warmup(iterations = 1, time = 1)
+// real iterations. not useful to spend tons of time here, better to fork more
+@Measurement(iterations = 3, time = 1)
+// engage some noise reduction
+@Fork(value = 1)
+public class ComputeNeighboursBenchmark {
+
+    static {
+        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+    }
+
+    @Param({ "1000", "2000", "3000", "5000", "10000", "20000", "50000" })
+    int numVectors;
+
+    @Param({ "384", "782", "1024" })
+    int dims;
+
+    float[][] vectors;
+    int clusterPerNeighbour = 128;
+
+    @Setup
+    public void setup() throws IOException {
+        Random random = new Random(123);
+        vectors = new float[numVectors][dims];
+        for (float[] vector : vectors) {
+            for (int i = 0; i < dims; i++) {
+                vector[i] = random.nextFloat();
+            }
+        }
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void bruteForce(Blackhole bh) {
+        bh.consume(NeighborHood.computeNeighborhoodsBruteForce(vectors, clusterPerNeighbour));
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void graph(Blackhole bh) throws IOException {
+        bh.consume(NeighborHood.computeNeighborhoodsGraph(vectors, clusterPerNeighbour));
+    }
+}