Skip to content

Commit fd1a517

Browse files
henriquepaes1henriquepaes1
authored
Merge branch 'main' into 112812-copy-to-non-existent-field
2 parents 1843499 + fd81c51 commit fd1a517

File tree

7,382 files changed

+270638
-97925
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

7,382 files changed

+270638
-97925
lines changed

‎.backportrc.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
22
"upstream" : "elastic/elasticsearch",
3-
"targetBranchChoices" : [ "main", "8.x", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
3+
"targetBranchChoices" : [ "main", "8.x", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
44
"targetPRLabels" : [ "backport" ],
55
"branchLabelMapping" : {
66
"^v9.0.0$" : "main",
7-
"^v8.16.0$" : "8.x",
7+
"^v8.18.0$" : "8.x",
88
"^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
99
}
1010
}

‎.buildkite/hooks/pre-command‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,10 @@ export COMPOSE_HTTP_TIMEOUT
1616
JOB_BRANCH="$BUILDKITE_BRANCH"
1717
export JOB_BRANCH
1818

19-
GRADLEW="./gradlew --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
19+
GRADLEW="./gradlew --console=plain --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
2020
export GRADLEW
2121

22-
GRADLEW_BAT="./gradlew.bat --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
22+
GRADLEW_BAT="./gradlew.bat --console=plain --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
2323
export GRADLEW_BAT
2424

2525
export $(cat .ci/java-versions.properties | grep '=' | xargs)
@@ -47,6 +47,8 @@ export GRADLE_BUILD_CACHE_PASSWORD
4747
BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token)
4848
export BUILDKITE_API_TOKEN
4949

50+
export GH_TOKEN="$VAULT_GITHUB_TOKEN"
51+
5052
if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
5153
data=$(.buildkite/scripts/get-legacy-secret.sh aws-elastic/creds/lucene-snapshots)
5254

@@ -78,11 +80,15 @@ if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then
7880
fi
7981

8082
if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then
81-
DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
82-
export DOCKER_REGISTRY_USERNAME
83+
if which docker > /dev/null 2>&1; then
84+
DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
85+
export DOCKER_REGISTRY_USERNAME
86+
87+
DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
88+
export DOCKER_REGISTRY_PASSWORD
8389

84-
DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
85-
export DOCKER_REGISTRY_PASSWORD
90+
docker login --username "$DOCKER_REGISTRY_USERNAME" --password "$DOCKER_REGISTRY_PASSWORD" docker.elastic.co
91+
fi
8692
fi
8793

8894
if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then
@@ -105,3 +111,13 @@ EOF
105111
<summary>Agent information from gobld</summary>
106112
EOF
107113
fi
114+
115+
# Amazon Linux 2 has DNS resolution issues with resource-based hostnames in EC2
116+
# We have many functional tests that try to lookup and resolve the hostname of the local machine in a particular way
117+
# And they fail. This sets up a manual entry for the hostname in dnsmasq.
118+
if [[ -f /etc/os-release ]] && grep -q '"Amazon Linux 2"' /etc/os-release; then
119+
echo "$(hostname -i | cut -d' ' -f 2) $(hostname -f)." | sudo tee /etc/dnsmasq.hosts
120+
sudo systemctl restart dnsmasq.service
121+
fi
122+
123+
.buildkite/scripts/get-latest-test-mutes.sh

‎.buildkite/hooks/pre-command.bat‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,12 @@ set BUILD_NUMBER=%BUILDKITE_BUILD_NUMBER%
1515
set COMPOSE_HTTP_TIMEOUT=120
1616
set JOB_BRANCH=%BUILDKITE_BRANCH%
1717

18+
set GH_TOKEN=%VAULT_GITHUB_TOKEN%
19+
1820
set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
1921
set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
2022

2123
bash.exe -c "nohup bash .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &"
24+
bash.exe -c "bash .buildkite/scripts/get-latest-test-mutes.sh"
2225

2326
exit /b 0

‎.buildkite/packer_cache.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ for branch in "${branches[@]}"; do
2929
fi
3030

3131
export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
32-
"checkout/${branch}/gradlew" --project-dir "$CHECKOUT_DIR" --parallel -s resolveAllDependencies -Dorg.gradle.warning.mode=none -DisCI
32+
"checkout/${branch}/gradlew" --project-dir "$CHECKOUT_DIR" --parallel -s resolveAllDependencies -Dorg.gradle.warning.mode=none -DisCI --max-workers=4
33+
"checkout/${branch}/gradlew" --stop
34+
pkill -f '.*GradleDaemon.*'
3335
rm -rf "checkout/${branch}"
3436
done

‎.buildkite/pipelines/dra-workflow.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@ steps:
22
- command: .buildkite/scripts/dra-workflow.sh
33
env:
44
USE_DRA_CREDENTIALS: "true"
5+
USE_PROD_DOCKER_CREDENTIALS: "true"
56
agents:
67
provider: gcp
78
image: family/elasticsearch-ubuntu-2204
89
machineType: custom-32-98304
9-
buildDirectory: /dev/shm/bk
10+
localSsds: 1
11+
localSsdInterface: nvme
1012
diskSizeGb: 350
1113
- wait
1214
# The hadoop build depends on the ES artifact
@@ -17,4 +19,5 @@ steps:
1719
branch: "${BUILDKITE_BRANCH}"
1820
env:
1921
DRA_WORKFLOW: staging
22+
USE_PROD_DOCKER_CREDENTIALS: "true"
2023
if: build.env('DRA_WORKFLOW') == 'staging'

‎.buildkite/pipelines/intake.template.yml‎

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,57 @@
11
steps:
22
- label: sanity-check
3-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit
3+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints precommit
44
timeout_in_minutes: 300
55
agents:
66
provider: gcp
77
image: family/elasticsearch-ubuntu-2004
88
machineType: custom-32-98304
99
buildDirectory: /dev/shm/bk
10-
diskSizeGb: 250
1110
- wait
1211
- label: part1
13-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1
12+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart1
1413
timeout_in_minutes: 300
1514
agents:
1615
provider: gcp
1716
image: family/elasticsearch-ubuntu-2004
1817
machineType: n1-standard-32
1918
buildDirectory: /dev/shm/bk
20-
diskSizeGb: 250
2119
- label: part2
22-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2
20+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart2
2321
timeout_in_minutes: 300
2422
agents:
2523
provider: gcp
2624
image: family/elasticsearch-ubuntu-2004
2725
machineType: n1-standard-32
2826
buildDirectory: /dev/shm/bk
29-
diskSizeGb: 250
3027
- label: part3
31-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3
28+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart3
3229
timeout_in_minutes: 300
3330
agents:
3431
provider: gcp
3532
image: family/elasticsearch-ubuntu-2004
3633
machineType: n1-standard-32
3734
buildDirectory: /dev/shm/bk
38-
diskSizeGb: 250
3935
- label: part4
40-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
36+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart4
4137
timeout_in_minutes: 300
4238
agents:
4339
provider: gcp
4440
image: family/elasticsearch-ubuntu-2004
4541
machineType: n1-standard-32
4642
buildDirectory: /dev/shm/bk
47-
diskSizeGb: 250
4843
- label: part5
49-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5
44+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart5
5045
timeout_in_minutes: 300
5146
agents:
5247
provider: gcp
5348
image: family/elasticsearch-ubuntu-2004
5449
machineType: n1-standard-32
5550
buildDirectory: /dev/shm/bk
56-
diskSizeGb: 250
5751
- group: bwc-snapshots
5852
steps:
5953
- label: "{{matrix.BWC_VERSION}} / bwc-snapshots"
60-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$$BWC_VERSION#bwcTest
54+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints v$$BWC_VERSION#bwcTest
6155
timeout_in_minutes: 300
6256
matrix:
6357
setup:
@@ -67,22 +61,21 @@ steps:
6761
image: family/elasticsearch-ubuntu-2004
6862
machineType: custom-32-98304
6963
buildDirectory: /dev/shm/bk
70-
diskSizeGb: 250
7164
env:
7265
BWC_VERSION: "{{matrix.BWC_VERSION}}"
7366
- label: rest-compat
74-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat
67+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkRestCompat
7568
timeout_in_minutes: 300
7669
agents:
7770
provider: gcp
7871
image: family/elasticsearch-ubuntu-2004
7972
machineType: custom-32-98304
8073
buildDirectory: /dev/shm/bk
81-
diskSizeGb: 250
8274
- wait
8375
- trigger: elasticsearch-dra-workflow
8476
label: Trigger DRA snapshot workflow
8577
async: true
78+
branches: "main 8.* 7.17"
8679
build:
8780
branch: "$BUILDKITE_BRANCH"
8881
commit: "$BUILDKITE_COMMIT"

‎.buildkite/pipelines/intake.yml‎

Lines changed: 10 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,89 +1,82 @@
11
# This file is auto-generated. See .buildkite/pipelines/intake.template.yml
22
steps:
33
- label: sanity-check
4-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit
4+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints precommit
55
timeout_in_minutes: 300
66
agents:
77
provider: gcp
88
image: family/elasticsearch-ubuntu-2004
99
machineType: custom-32-98304
1010
buildDirectory: /dev/shm/bk
11-
diskSizeGb: 250
1211
- wait
1312
- label: part1
14-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1
13+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart1
1514
timeout_in_minutes: 300
1615
agents:
1716
provider: gcp
1817
image: family/elasticsearch-ubuntu-2004
1918
machineType: n1-standard-32
2019
buildDirectory: /dev/shm/bk
21-
diskSizeGb: 250
2220
- label: part2
23-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2
21+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart2
2422
timeout_in_minutes: 300
2523
agents:
2624
provider: gcp
2725
image: family/elasticsearch-ubuntu-2004
2826
machineType: n1-standard-32
2927
buildDirectory: /dev/shm/bk
30-
diskSizeGb: 250
3128
- label: part3
32-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3
29+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart3
3330
timeout_in_minutes: 300
3431
agents:
3532
provider: gcp
3633
image: family/elasticsearch-ubuntu-2004
3734
machineType: n1-standard-32
3835
buildDirectory: /dev/shm/bk
39-
diskSizeGb: 250
4036
- label: part4
41-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
37+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart4
4238
timeout_in_minutes: 300
4339
agents:
4440
provider: gcp
4541
image: family/elasticsearch-ubuntu-2004
4642
machineType: n1-standard-32
4743
buildDirectory: /dev/shm/bk
48-
diskSizeGb: 250
4944
- label: part5
50-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5
45+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart5
5146
timeout_in_minutes: 300
5247
agents:
5348
provider: gcp
5449
image: family/elasticsearch-ubuntu-2004
5550
machineType: n1-standard-32
5651
buildDirectory: /dev/shm/bk
57-
diskSizeGb: 250
5852
- group: bwc-snapshots
5953
steps:
6054
- label: "{{matrix.BWC_VERSION}} / bwc-snapshots"
61-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$$BWC_VERSION#bwcTest
55+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints v$$BWC_VERSION#bwcTest
6256
timeout_in_minutes: 300
6357
matrix:
6458
setup:
65-
BWC_VERSION: ["8.15.2", "8.16.0", "9.0.0"]
59+
BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"]
6660
agents:
6761
provider: gcp
6862
image: family/elasticsearch-ubuntu-2004
6963
machineType: custom-32-98304
7064
buildDirectory: /dev/shm/bk
71-
diskSizeGb: 250
7265
env:
7366
BWC_VERSION: "{{matrix.BWC_VERSION}}"
7467
- label: rest-compat
75-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat
68+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkRestCompat
7669
timeout_in_minutes: 300
7770
agents:
7871
provider: gcp
7972
image: family/elasticsearch-ubuntu-2004
8073
machineType: custom-32-98304
8174
buildDirectory: /dev/shm/bk
82-
diskSizeGb: 250
8375
- wait
8476
- trigger: elasticsearch-dra-workflow
8577
label: Trigger DRA snapshot workflow
8678
async: true
79+
branches: "main 8.* 7.17"
8780
build:
8881
branch: "$BUILDKITE_BRANCH"
8982
commit: "$BUILDKITE_COMMIT"

‎.buildkite/pipelines/lucene-snapshot/build-snapshot.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ steps:
1515
image: family/elasticsearch-ubuntu-2004
1616
machineType: custom-32-98304
1717
buildDirectory: /dev/shm/bk
18-
diskSizeGb: 250
1918
- wait
2019
- trigger: "elasticsearch-lucene-snapshot-tests"
2120
build:

0 commit comments

Comments
 (0)