Skip to content

Commit 33ef501

Browse files
jdconradjdconrad
committed
Merge branch 'main' into mtv24
2 parents a27a736 + 771aaff commit 33ef501

File tree

400 files changed

+12775
-5049
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

400 files changed

+12775
-5049
lines changed

‎.buildkite/pipelines/cuvs-snapshot/run-tests.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
steps:
2+
- label: "{{matrix.GRADLE_TASK}}"
3+
command: .buildkite/scripts/cuvs-snapshot/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints {{matrix.GRADLE_TASK}}
4+
timeout_in_minutes: 300
5+
agents:
6+
provider: gcp
7+
image: family/elasticsearch-ubuntu-2404-nvidia
8+
machineType: g2-standard-32
9+
buildDirectory: /dev/shm/bk
10+
zones: us-central1-b,us-central1-c
11+
env:
12+
GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
13+
matrix:
14+
setup:
15+
GRADLE_TASK:
16+
- checkPart1
17+
- checkPart2
18+
- checkPart3
19+
- checkPart4
20+
- checkPart5
21+
- checkPart6

‎.buildkite/pipelines/cuvs-snapshot/update-snapshot.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
steps:
2+
- label: "Smoke test and update new cuVS snapshot"
3+
command: .buildkite/scripts/cuvs-snapshot/update-current-snapshot-version.sh
4+
agents:
5+
provider: gcp
6+
image: family/elasticsearch-ubuntu-2404-nvidia
7+
machineType: g2-standard-16
8+
zones: us-central1-b,us-central1-c
9+
diskSizeGb: 150
10+
- wait: ~
11+
- trigger: "elasticsearch-cuvs-run-tests"
12+
build:
13+
branch: "${BUILDKITE_BRANCH}"
14+
async: true

‎.buildkite/pipelines/pull-request/.defaults.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
config:
2-
skip-labels: ">test-mute"
2+
skip-labels:
3+
- ">test-mute"
34
excluded-regions:
45
- ^docs/.*
56
- ^x-pack/docs/.*

‎.buildkite/pipelines/pull-request/bwc-snapshots.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,4 +78,3 @@ steps:
7878
image: family/elasticsearch-ubuntu-2004
7979
machineType: n1-standard-32
8080
buildDirectory: /dev/shm/bk
81-

‎.buildkite/pipelines/pull-request/gpu.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
config:
2+
allow-labels: test-gpu
3+
skip-labels:
4+
- ">test-mute"
5+
steps:
6+
- group: gpu-tests
7+
steps:
8+
- label: "{{matrix.GRADLE_TASK}} / gpu-tests"
9+
key: "packaging-tests-unix"
10+
command: .buildkite/scripts/cuvs-snapshot/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints {{matrix.GRADLE_TASK}}
11+
timeout_in_minutes: 300
12+
agents:
13+
provider: gcp
14+
image: family/elasticsearch-ubuntu-2404-nvidia
15+
machineType: g2-standard-32
16+
buildDirectory: /dev/shm/bk
17+
zones: us-central1-b,us-central1-c
18+
env:
19+
GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
20+
matrix:
21+
setup:
22+
GRADLE_TASK:
23+
- checkPart1
24+
- checkPart2
25+
- checkPart3
26+
- checkPart4
27+
- checkPart5
28+
- checkPart6

‎.buildkite/scripts/cuvs-snapshot/configure.sh‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
if [[ -f /etc/profile.d/elastic-nvidia.sh ]]; then
6+
export JAVA_HOME="$HOME/.java/openjdk24"
7+
export PATH="$JAVA_HOME/bin:$PATH"
8+
9+
# Setup LD_LIBRARY_PATH, PATH
10+
11+
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}"
12+
source /etc/profile.d/elastic-nvidia.sh
13+
fi
14+
15+
# Not running this before the tests results in an error when running the tests
16+
# No idea why...
17+
nvidia-smi
18+
19+
CURRENT_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
20+
ELASTICSEARCH_REPO_DIR="$(cd "$CURRENT_SCRIPT_DIR/../../.." && pwd)"
21+
22+
CUVS_SNAPSHOT_VERSION="${CUVS_SNAPSHOT_VERSION:-$(cat "$CURRENT_SCRIPT_DIR"/current-snapshot-version)}"
23+
CUVS_ARCHIVE="cuvs-$CUVS_SNAPSHOT_VERSION.tar.gz"
24+
CUVS_URL="https://storage.googleapis.com/elasticsearch-cuvs-snapshots/$CUVS_ARCHIVE"
25+
26+
CUVS_WORKSPACE=${CUVS_WORKSPACE:-$(cd "$(mktemp -d)")}
27+
CUVS_DIR="$(pwd)/cuvs-$CUVS_SNAPSHOT_VERSION"
28+
29+
curl -O "$CUVS_URL"
30+
tar -xzf "$CUVS_ARCHIVE"
31+
32+
CUVS_VERSION=$(cd "$CUVS_DIR/cuvs-java/target" && mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
33+
34+
LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v "libcuvs/linux-x64" | tr '\n' ':' | sed 's/:$//')
35+
LD_LIBRARY_PATH="$CUVS_DIR/libcuvs/linux-x64:$LD_LIBRARY_PATH"
36+
export LD_LIBRARY_PATH
37+
38+
cd "$CUVS_DIR/cuvs-java/target"
39+
mvn install:install-file -Dfile="cuvs-java-$CUVS_VERSION.jar" -DartifactId=elastic-cuvs-java -DgeneratePom=true
40+
41+
cd "$ELASTICSEARCH_REPO_DIR"
42+
PLUGIN_GRADLE_FILE=x-pack/plugin/gpu/build.gradle
43+
sed -i "s|implementation 'com.nvidia.cuvs:elastic-cuvs-java:.*'|implementation 'com.nvidia.cuvs:elastic-cuvs-java:$CUVS_VERSION'|" "$PLUGIN_GRADLE_FILE"

‎.buildkite/scripts/cuvs-snapshot/current-snapshot-version‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
fdb8bfb8

‎.buildkite/scripts/cuvs-snapshot/run-gradle.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
source .buildkite/scripts/cuvs-snapshot/configure.sh
6+
7+
cd "$WORKSPACE"
8+
9+
.ci/scripts/run-gradle.sh "$@"

‎.buildkite/scripts/cuvs-snapshot/update-current-snapshot-version.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
SNAPSHOT_VERSION_FILE=.buildkite/scripts/cuvs-snapshot/current-snapshot-version
6+
BRANCH_TO_UPDATE="${BRANCH_TO_UPDATE:-${BUILDKITE_BRANCH:-cuvs-snapshot}}"
7+
8+
if [[ -z "${CUVS_SNAPSHOT_VERSION:-}" ]]; then
9+
echo "CUVS_SNAPSHOT_VERSION not set. Set this to update the current snapshot version."
10+
exit 1
11+
fi
12+
13+
if [[ "$CUVS_SNAPSHOT_VERSION" == "$(cat $SNAPSHOT_VERSION_FILE)" ]]; then
14+
echo "Current snapshot version already set to '$CUVS_SNAPSHOT_VERSION'. No need to update."
15+
exit 0
16+
fi
17+
18+
echo "--- Configuring libcuvs/cuvs-java"
19+
source .buildkite/scripts/cuvs-snapshot/configure.sh
20+
21+
if [[ "${SKIP_TESTING:-}" != "true" ]]; then
22+
echo "--- Testing snapshot before updating"
23+
./gradlew -Druntime.java=24 :x-pack:plugin:gpu:yamlRestTest -S
24+
fi
25+
26+
echo "--- Updating snapshot"
27+
28+
echo "$CUVS_SNAPSHOT_VERSION" > "$SNAPSHOT_VERSION_FILE"
29+
30+
CURRENT_SHA="$(gh api "/repos/elastic/elasticsearch/contents/$SNAPSHOT_VERSION_FILE?ref=$BRANCH_TO_UPDATE" | jq -r .sha)" || true
31+
32+
gh api -X PUT "/repos/elastic/elasticsearch/contents/$SNAPSHOT_VERSION_FILE" \
33+
-f branch="$BRANCH_TO_UPDATE" \
34+
-f message="Update cuvs snapshot version to $CUVS_VERSION" \
35+
-f content="$(base64 -w 0 "$WORKSPACE/$SNAPSHOT_VERSION_FILE")" \
36+
-f sha="$CURRENT_SHA"

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,6 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
9090
assertUpperBound("9.2", "new_tv,8124000")
9191
}
9292

93-
/*
94-
temporarily muted, see https://github.com/elastic/elasticsearch/pull/135226
95-
9693
def "invalid changes to a upper bounds should be reverted"() {
9794
given:
9895
transportVersionUpperBound("9.2", "modification", "9000000")
@@ -147,7 +144,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
147144
assertReferableDefinitionDoesNotExist("test_tv")
148145
assertUpperBound("9.2", "existing_92,8123000")
149146
assertUpperBound("9.1", "existing_92,8012001")
150-
}*/
147+
}
151148

152149
def "a reference can be renamed"() {
153150
given:
@@ -245,11 +242,8 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
245242
def "unreferenced definitions are removed"() {
246243
given:
247244
referableTransportVersion("test_tv", "8124000,8012002")
248-
/*
249-
TODO: reset of upper bounds
250245
transportVersionUpperBound("9.2", "test_tv", "8124000")
251246
transportVersionUpperBound("9.1", "test_tv", "8012002")
252-
*/
253247

254248
when:
255249
def result = runGenerateAndValidateTask().build()
@@ -412,8 +406,6 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
412406
assertUpperBound("9.2", "new_tv,8124000")
413407
}
414408

415-
/*
416-
TODO: reset of upper bounds
417409
def "deleted upper bounds files are restored"() {
418410
given:
419411
file("myserver/src/main/resources/transport/upper_bounds/9.2.csv").delete()
@@ -424,7 +416,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
424416
then:
425417
assertGenerateAndValidateSuccess(result)
426418
assertUpperBound("9.2", "existing_92,8123000")
427-
}*/
419+
}
428420

429421
def "upper bounds files must exist for backport branches"() {
430422
when:

0 commit comments

Comments
 (0)