Merge branch 'main' into mtv24

Author: jdconrad

.buildkite/pipelines/cuvs-snapshot/run-tests.yml

@@ -0,0 +1,21 @@
+steps:
+  - label: "{{matrix.GRADLE_TASK}}"
+    command: .buildkite/scripts/cuvs-snapshot/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints {{matrix.GRADLE_TASK}}
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404-nvidia
+      machineType: g2-standard-32
+      buildDirectory: /dev/shm/bk
+      zones: us-central1-b,us-central1-c
+    env:
+      GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+    matrix:
+      setup:
+        GRADLE_TASK:
+          - checkPart1
+          - checkPart2
+          - checkPart3
+          - checkPart4
+          - checkPart5
+          - checkPart6

.buildkite/pipelines/cuvs-snapshot/update-snapshot.yml

@@ -0,0 +1,14 @@
+steps:
+  - label: "Smoke test and update new cuVS snapshot"
+    command: .buildkite/scripts/cuvs-snapshot/update-current-snapshot-version.sh
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404-nvidia
+      machineType: g2-standard-16
+      zones: us-central1-b,us-central1-c
+      diskSizeGb: 150
+  - wait: ~
+  - trigger: "elasticsearch-cuvs-run-tests"
+    build:
+      branch: "${BUILDKITE_BRANCH}"
+    async: true

.buildkite/pipelines/pull-request/.defaults.yml

@@ -1,5 +1,6 @@
 config:
-  skip-labels: ">test-mute"
+  skip-labels:
+    - ">test-mute"
   excluded-regions:
     - ^docs/.*
     - ^x-pack/docs/.*

.buildkite/pipelines/pull-request/bwc-snapshots.yml

@@ -78,4 +78,3 @@ steps:
           image: family/elasticsearch-ubuntu-2004
           machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
-

.buildkite/pipelines/pull-request/gpu.yml

@@ -0,0 +1,28 @@
+config:
+  allow-labels: test-gpu
+  skip-labels:
+    - ">test-mute"
+steps:
+  - group: gpu-tests
+    steps:
+      - label: "{{matrix.GRADLE_TASK}} / gpu-tests"
+        key: "packaging-tests-unix"
+        command: .buildkite/scripts/cuvs-snapshot/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints {{matrix.GRADLE_TASK}}
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404-nvidia
+          machineType: g2-standard-32
+          buildDirectory: /dev/shm/bk
+          zones: us-central1-b,us-central1-c
+        env:
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+        matrix:
+          setup:
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - checkPart4
+              - checkPart5
+              - checkPart6

.buildkite/scripts/cuvs-snapshot/configure.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [[ -f /etc/profile.d/elastic-nvidia.sh ]]; then
+  export JAVA_HOME="$HOME/.java/openjdk24"
+  export PATH="$JAVA_HOME/bin:$PATH"
+
+  # Setup LD_LIBRARY_PATH, PATH
+
+  export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}"
+  source /etc/profile.d/elastic-nvidia.sh
+fi
+
+# Not running this before the tests results in an error when running the tests
+# No idea why...
+nvidia-smi
+
+CURRENT_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ELASTICSEARCH_REPO_DIR="$(cd "$CURRENT_SCRIPT_DIR/../../.." && pwd)"
+
+CUVS_SNAPSHOT_VERSION="${CUVS_SNAPSHOT_VERSION:-$(cat "$CURRENT_SCRIPT_DIR"/current-snapshot-version)}"
+CUVS_ARCHIVE="cuvs-$CUVS_SNAPSHOT_VERSION.tar.gz"
+CUVS_URL="https://storage.googleapis.com/elasticsearch-cuvs-snapshots/$CUVS_ARCHIVE"
+
+CUVS_WORKSPACE=${CUVS_WORKSPACE:-$(cd "$(mktemp -d)")}
+CUVS_DIR="$(pwd)/cuvs-$CUVS_SNAPSHOT_VERSION"
+
+curl -O "$CUVS_URL"
+tar -xzf "$CUVS_ARCHIVE"
+
+CUVS_VERSION=$(cd "$CUVS_DIR/cuvs-java/target" && mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+
+LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v "libcuvs/linux-x64" | tr '\n' ':' | sed 's/:$//')
+LD_LIBRARY_PATH="$CUVS_DIR/libcuvs/linux-x64:$LD_LIBRARY_PATH"
+export LD_LIBRARY_PATH
+
+cd "$CUVS_DIR/cuvs-java/target"
+mvn install:install-file -Dfile="cuvs-java-$CUVS_VERSION.jar" -DartifactId=elastic-cuvs-java -DgeneratePom=true
+
+cd "$ELASTICSEARCH_REPO_DIR"
+PLUGIN_GRADLE_FILE=x-pack/plugin/gpu/build.gradle
+sed -i "s|implementation 'com.nvidia.cuvs:elastic-cuvs-java:.*'|implementation 'com.nvidia.cuvs:elastic-cuvs-java:$CUVS_VERSION'|" "$PLUGIN_GRADLE_FILE"

.buildkite/scripts/cuvs-snapshot/current-snapshot-version

@@ -0,0 +1 @@
+fdb8bfb8

.buildkite/scripts/cuvs-snapshot/run-gradle.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/cuvs-snapshot/configure.sh
+
+cd "$WORKSPACE"
+
+.ci/scripts/run-gradle.sh "$@"

.buildkite/scripts/cuvs-snapshot/update-current-snapshot-version.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -euo pipefail
+
+SNAPSHOT_VERSION_FILE=.buildkite/scripts/cuvs-snapshot/current-snapshot-version
+BRANCH_TO_UPDATE="${BRANCH_TO_UPDATE:-${BUILDKITE_BRANCH:-cuvs-snapshot}}"
+
+if [[ -z "${CUVS_SNAPSHOT_VERSION:-}" ]]; then
+  echo "CUVS_SNAPSHOT_VERSION not set. Set this to update the current snapshot version."
+  exit 1
+fi
+
+if [[ "$CUVS_SNAPSHOT_VERSION" == "$(cat $SNAPSHOT_VERSION_FILE)" ]]; then
+  echo "Current snapshot version already set to '$CUVS_SNAPSHOT_VERSION'. No need to update."
+  exit 0
+fi
+
+echo "--- Configuring libcuvs/cuvs-java"
+source .buildkite/scripts/cuvs-snapshot/configure.sh
+
+if [[ "${SKIP_TESTING:-}" != "true" ]]; then
+  echo "--- Testing snapshot before updating"
+  ./gradlew -Druntime.java=24 :x-pack:plugin:gpu:yamlRestTest -S
+fi
+
+echo "--- Updating snapshot"
+
+echo "$CUVS_SNAPSHOT_VERSION" > "$SNAPSHOT_VERSION_FILE"
+
+CURRENT_SHA="$(gh api "/repos/elastic/elasticsearch/contents/$SNAPSHOT_VERSION_FILE?ref=$BRANCH_TO_UPDATE" | jq -r .sha)" || true
+
+gh api -X PUT "/repos/elastic/elasticsearch/contents/$SNAPSHOT_VERSION_FILE" \
+  -f branch="$BRANCH_TO_UPDATE" \
+  -f message="Update cuvs snapshot version to $CUVS_VERSION" \
+  -f content="$(base64 -w 0 "$WORKSPACE/$SNAPSHOT_VERSION_FILE")" \
+  -f sha="$CURRENT_SHA"

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -64,7 +64,7 @@
 public class ElasticsearchCluster implements TestClusterConfiguration, Named {
 
     private static final Logger LOGGER = Logging.getLogger(ElasticsearchNode.class);
-    private static final int CLUSTER_UP_TIMEOUT = 40;
+    private static final int CLUSTER_UP_TIMEOUT = 120;
     private static final TimeUnit CLUSTER_UP_TIMEOUT_UNIT = TimeUnit.SECONDS;
 
     private final AtomicBoolean configurationFrozen = new AtomicBoolean(false);