Skip to content

Commit 8753e8e

Browse files
pawankartik-elasticpawankartik-elastic
authored
Merge branch 'main' into pkar/resolve-index-force-reconn
2 parents fd8c4f6 + ea98a38 commit 8753e8e

File tree

1,990 files changed

+60408
-24363
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,990 files changed

+60408
-24363
lines changed

‎.buildkite/pipelines/cuvs-snapshot/run-tests.yml‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,12 @@ steps:
33
command: .buildkite/scripts/cuvs-snapshot/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints {{matrix.GRADLE_TASK}}
44
timeout_in_minutes: 300
55
agents:
6-
provider: gcp
7-
image: family/elasticsearch-ubuntu-2404-nvidia
8-
machineType: g2-standard-32
9-
buildDirectory: /dev/shm/bk
10-
zones: us-central1-b,us-central1-c
6+
provider: aws
7+
imagePrefix: elasticsearch-aws-ubuntu-2204-nvidia
8+
instanceType: g6.8xlarge
9+
diskSizeGb: 350
10+
diskType: gp3
11+
diskName: /dev/sda1
1112
env:
1213
GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
1314
matrix:

‎.buildkite/pipelines/cuvs-snapshot/update-snapshot.yml‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,12 @@ steps:
22
- label: "Smoke test and update new cuVS snapshot"
33
command: .buildkite/scripts/cuvs-snapshot/update-current-snapshot-version.sh
44
agents:
5-
provider: gcp
6-
image: family/elasticsearch-ubuntu-2404-nvidia
7-
machineType: g2-standard-16
8-
zones: us-central1-b,us-central1-c
9-
diskSizeGb: 150
5+
provider: aws
6+
imagePrefix: elasticsearch-aws-ubuntu-2204-nvidia
7+
instanceType: g6.2xlarge
8+
diskSizeGb: 350
9+
diskType: gp3
10+
diskName: /dev/sda1
1011
- wait: ~
1112
- trigger: "elasticsearch-cuvs-run-tests"
1213
build:

‎.buildkite/pipelines/intake.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ steps:
6565
timeout_in_minutes: 300
6666
matrix:
6767
setup:
68-
BWC_VERSION: ["8.19.8", "9.1.8", "9.2.2", "9.3.0"]
68+
BWC_VERSION: ["8.19.9", "9.1.9", "9.2.3", "9.3.0"]
6969
agents:
7070
provider: gcp
7171
image: family/elasticsearch-ubuntu-2404

‎.buildkite/pipelines/periodic-java-ea.yml‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -366,8 +366,8 @@ steps:
366366
- signal_reason: agent_stop
367367
limit: 3
368368

369-
- label: 8.19.8 / bwc
370-
command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v8.19.8#bwcTest
369+
- label: 8.19.9 / bwc
370+
command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v8.19.9#bwcTest
371371
timeout_in_minutes: 300
372372
agents:
373373
provider: gcp
@@ -376,7 +376,7 @@ steps:
376376
buildDirectory: /dev/shm/bk
377377
preemptible: true
378378
env:
379-
BWC_VERSION: 8.19.8
379+
BWC_VERSION: 8.19.9
380380
retry:
381381
automatic:
382382
- exit_status: "-1"
@@ -404,8 +404,8 @@ steps:
404404
- signal_reason: agent_stop
405405
limit: 3
406406

407-
- label: 9.1.8 / bwc
408-
command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.1.8#bwcTest
407+
- label: 9.1.9 / bwc
408+
command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.1.9#bwcTest
409409
timeout_in_minutes: 300
410410
agents:
411411
provider: gcp
@@ -414,7 +414,7 @@ steps:
414414
buildDirectory: /dev/shm/bk
415415
preemptible: true
416416
env:
417-
BWC_VERSION: 9.1.8
417+
BWC_VERSION: 9.1.9
418418
retry:
419419
automatic:
420420
- exit_status: "-1"
@@ -423,8 +423,8 @@ steps:
423423
- signal_reason: agent_stop
424424
limit: 3
425425

426-
- label: 9.2.2 / bwc
427-
command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.2.2#bwcTest
426+
- label: 9.2.3 / bwc
427+
command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.2.3#bwcTest
428428
timeout_in_minutes: 300
429429
agents:
430430
provider: gcp
@@ -433,7 +433,7 @@ steps:
433433
buildDirectory: /dev/shm/bk
434434
preemptible: true
435435
env:
436-
BWC_VERSION: 9.2.2
436+
BWC_VERSION: 9.2.3
437437
retry:
438438
automatic:
439439
- exit_status: "-1"
@@ -538,7 +538,7 @@ steps:
538538
timeout_in_minutes: 300
539539
matrix:
540540
setup:
541-
BWC_VERSION: ["8.19.8", "9.1.8", "9.2.2", "9.3.0"]
541+
BWC_VERSION: ["8.19.9", "9.1.9", "9.2.3", "9.3.0"]
542542
agents:
543543
provider: gcp
544544
image: family/elasticsearch-ubuntu-2404

‎.buildkite/pipelines/periodic-packaging.yml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -336,8 +336,8 @@ steps:
336336
env:
337337
BWC_VERSION: 8.18.8
338338

339-
- label: "{{matrix.image}} / 8.19.8 / packaging-tests-upgrade"
340-
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.8
339+
- label: "{{matrix.image}} / 8.19.9 / packaging-tests-upgrade"
340+
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.9
341341
timeout_in_minutes: 300
342342
matrix:
343343
setup:
@@ -350,7 +350,7 @@ steps:
350350
machineType: custom-16-32768
351351
buildDirectory: /dev/shm/bk
352352
env:
353-
BWC_VERSION: 8.19.8
353+
BWC_VERSION: 8.19.9
354354

355355
- label: "{{matrix.image}} / 9.0.8 / packaging-tests-upgrade"
356356
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.8
@@ -368,8 +368,8 @@ steps:
368368
env:
369369
BWC_VERSION: 9.0.8
370370

371-
- label: "{{matrix.image}} / 9.1.8 / packaging-tests-upgrade"
372-
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.8
371+
- label: "{{matrix.image}} / 9.1.9 / packaging-tests-upgrade"
372+
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.9
373373
timeout_in_minutes: 300
374374
matrix:
375375
setup:
@@ -382,10 +382,10 @@ steps:
382382
machineType: custom-16-32768
383383
buildDirectory: /dev/shm/bk
384384
env:
385-
BWC_VERSION: 9.1.8
385+
BWC_VERSION: 9.1.9
386386

387-
- label: "{{matrix.image}} / 9.2.2 / packaging-tests-upgrade"
388-
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.2.2
387+
- label: "{{matrix.image}} / 9.2.3 / packaging-tests-upgrade"
388+
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.2.3
389389
timeout_in_minutes: 300
390390
matrix:
391391
setup:
@@ -398,7 +398,7 @@ steps:
398398
machineType: custom-16-32768
399399
buildDirectory: /dev/shm/bk
400400
env:
401-
BWC_VERSION: 9.2.2
401+
BWC_VERSION: 9.2.3
402402

403403
- label: "{{matrix.image}} / 9.3.0 / packaging-tests-upgrade"
404404
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.3.0

‎.buildkite/pipelines/periodic.yml‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -363,8 +363,8 @@ steps:
363363
- signal_reason: agent_stop
364364
limit: 3
365365

366-
- label: 8.19.8 / bwc
367-
command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v8.19.8#bwcTest
366+
- label: 8.19.9 / bwc
367+
command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v8.19.9#bwcTest
368368
timeout_in_minutes: 300
369369
agents:
370370
provider: gcp
@@ -373,7 +373,7 @@ steps:
373373
buildDirectory: /dev/shm/bk
374374
preemptible: true
375375
env:
376-
BWC_VERSION: 8.19.8
376+
BWC_VERSION: 8.19.9
377377
retry:
378378
automatic:
379379
- exit_status: "-1"
@@ -401,8 +401,8 @@ steps:
401401
- signal_reason: agent_stop
402402
limit: 3
403403

404-
- label: 9.1.8 / bwc
405-
command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v9.1.8#bwcTest
404+
- label: 9.1.9 / bwc
405+
command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v9.1.9#bwcTest
406406
timeout_in_minutes: 300
407407
agents:
408408
provider: gcp
@@ -411,7 +411,7 @@ steps:
411411
buildDirectory: /dev/shm/bk
412412
preemptible: true
413413
env:
414-
BWC_VERSION: 9.1.8
414+
BWC_VERSION: 9.1.9
415415
retry:
416416
automatic:
417417
- exit_status: "-1"
@@ -420,8 +420,8 @@ steps:
420420
- signal_reason: agent_stop
421421
limit: 3
422422

423-
- label: 9.2.2 / bwc
424-
command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v9.2.2#bwcTest
423+
- label: 9.2.3 / bwc
424+
command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v9.2.3#bwcTest
425425
timeout_in_minutes: 300
426426
agents:
427427
provider: gcp
@@ -430,7 +430,7 @@ steps:
430430
buildDirectory: /dev/shm/bk
431431
preemptible: true
432432
env:
433-
BWC_VERSION: 9.2.2
433+
BWC_VERSION: 9.2.3
434434
retry:
435435
automatic:
436436
- exit_status: "-1"
@@ -530,7 +530,7 @@ steps:
530530
setup:
531531
ES_RUNTIME_JAVA:
532532
- openjdk21
533-
BWC_VERSION: ["8.19.8", "9.1.8", "9.2.2", "9.3.0"]
533+
BWC_VERSION: ["8.19.9", "9.1.9", "9.2.3", "9.3.0"]
534534
agents:
535535
provider: gcp
536536
image: family/elasticsearch-ubuntu-2404
@@ -575,7 +575,7 @@ steps:
575575
ES_RUNTIME_JAVA:
576576
- openjdk21
577577
- openjdk25
578-
BWC_VERSION: ["8.19.8", "9.1.8", "9.2.2", "9.3.0"]
578+
BWC_VERSION: ["8.19.9", "9.1.9", "9.2.3", "9.3.0"]
579579
agents:
580580
provider: gcp
581581
image: family/elasticsearch-ubuntu-2404

‎.buildkite/pipelines/pull-request/gpu.yml‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,12 @@ steps:
1010
command: .buildkite/scripts/cuvs-snapshot/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints {{matrix.GRADLE_TASK}}
1111
timeout_in_minutes: 300
1212
agents:
13-
provider: gcp
14-
image: family/elasticsearch-ubuntu-2404-nvidia
15-
machineType: g2-standard-32
16-
buildDirectory: /dev/shm/bk
17-
zones: us-central1-b,us-central1-c
13+
provider: aws
14+
imagePrefix: elasticsearch-aws-ubuntu-2204-nvidia
15+
instanceType: g6.8xlarge
16+
diskSizeGb: 350
17+
diskType: gp3
18+
diskName: /dev/sda1
1819
env:
1920
GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
2021
matrix:

‎.buildkite/scripts/cuvs-snapshot/configure.sh‎

Lines changed: 25 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
#!/bin/bash
22

3-
set -euo pipefail
4-
53
if [[ -f /etc/profile.d/elastic-nvidia.sh ]]; then
64
export JAVA_HOME="$HOME/.java/openjdk24"
75
export PATH="$JAVA_HOME/bin:$PATH"
@@ -10,34 +8,35 @@ if [[ -f /etc/profile.d/elastic-nvidia.sh ]]; then
108

119
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}"
1210
source /etc/profile.d/elastic-nvidia.sh
13-
fi
14-
15-
# Not running this before the tests results in an error when running the tests
16-
# No idea why...
17-
nvidia-smi
1811

19-
CURRENT_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
20-
ELASTICSEARCH_REPO_DIR="$(cd "$CURRENT_SCRIPT_DIR/../../.." && pwd)"
21-
22-
CUVS_SNAPSHOT_VERSION="${CUVS_SNAPSHOT_VERSION:-$(cat "$CURRENT_SCRIPT_DIR"/current-snapshot-version)}"
23-
CUVS_ARCHIVE="cuvs-$CUVS_SNAPSHOT_VERSION.tar.gz"
24-
CUVS_URL="https://storage.googleapis.com/elasticsearch-cuvs-snapshots/$CUVS_ARCHIVE"
12+
# Not running this before the tests results in an error when running the tests
13+
# No idea why...
14+
if [[ "${BUILDKITE:-}" != "" && "${CI:-}" == "true" ]]; then
15+
nvidia-smi
16+
fi
17+
fi
2518

26-
CUVS_WORKSPACE=${CUVS_WORKSPACE:-$(cd "$(mktemp -d)")}
27-
CUVS_DIR="$(pwd)/cuvs-$CUVS_SNAPSHOT_VERSION"
19+
LIBCUVS_GCS_BUCKET="elasticsearch-cuvs-snapshots"
2820

29-
curl -O "$CUVS_URL"
30-
tar -xzf "$CUVS_ARCHIVE"
21+
LIBCUVS_DIR="/opt/libcuvs"
22+
sudo mkdir -p "$LIBCUVS_DIR"
23+
sudo chmod 777 "$LIBCUVS_DIR"
3124

32-
CUVS_VERSION=$(cd "$CUVS_DIR/cuvs-java/target" && mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
25+
CUVS_VERSION=$(grep 'cuvs_java' build-tools-internal/version.properties | awk '{print $3}')
3326

34-
LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v "libcuvs/linux-x64" | tr '\n' ':' | sed 's/:$//')
35-
LD_LIBRARY_PATH="$CUVS_DIR/libcuvs/linux-x64:$LD_LIBRARY_PATH"
36-
export LD_LIBRARY_PATH
27+
LIBCUVS_VERSION_DIR="$LIBCUVS_DIR/$CUVS_VERSION"
3728

38-
cd "$CUVS_DIR/cuvs-java/target"
39-
mvn install:install-file -Dfile="cuvs-java-$CUVS_VERSION.jar" -DartifactId=elastic-cuvs-java -DgeneratePom=true
29+
if [[ ! -d "$LIBCUVS_VERSION_DIR" ]]; then
30+
mkdir -p $LIBCUVS_VERSION_DIR
31+
cd "$LIBCUVS_VERSION_DIR"
32+
CUVS_ARCHIVE="libcuvs-$CUVS_VERSION.tar.gz"
33+
curl -fO "https://storage.googleapis.com/$LIBCUVS_GCS_BUCKET/libcuvs/$CUVS_ARCHIVE"
34+
tar -xzf "$CUVS_ARCHIVE"
35+
rm -f "$CUVS_ARCHIVE"
36+
if [[ -d "$CUVS_VERSION" ]]; then
37+
mv "$CUVS_VERSION/*" ./
38+
fi
39+
cd -
40+
fi
4041

41-
cd "$ELASTICSEARCH_REPO_DIR"
42-
PLUGIN_GRADLE_FILE=x-pack/plugin/gpu/build.gradle
43-
sed -i "s|implementation 'com.nvidia.cuvs:elastic-cuvs-java:.*'|implementation 'com.nvidia.cuvs:elastic-cuvs-java:$CUVS_VERSION'|" "$PLUGIN_GRADLE_FILE"
42+
export LD_LIBRARY_PATH="$LIBCUVS_VERSION_DIR:${LD_LIBRARY_PATH:-}"

‎.buildkite/scripts/cuvs-snapshot/current-snapshot-version‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎.buildkite/scripts/cuvs-snapshot/remove-verification-metadata.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import re
2+
3+
regex = re.compile(
4+
r'<component group="com.nvidia.cuvs" name="cuvs-.*?</component>\s*',
5+
re.MULTILINE | re.DOTALL,
6+
)
7+
8+
with open("gradle/verification-metadata.xml", "r+") as f:
9+
text = f.read()
10+
text = regex.sub("", text)
11+
f.seek(0)
12+
f.truncate()
13+
f.write(text)

0 commit comments

Comments
 (0)