Skip to content

Commit c0513dd

Browse files
author
Shiming Li
committed
Merge branch 'main' into script_rescorer
2 parents 789b1b8 + fe59163 commit c0513dd

File tree

8,698 files changed

+349329
-127281
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

8,698 files changed

+349329
-127281
lines changed

‎.backportrc.json‎

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,9 @@
11
{
2-
"upstream": "elastic/elasticsearch",
3-
"targetBranchChoices": [
4-
"main",
5-
"8.3",
6-
"8.2",
7-
"8.1",
8-
"8.0",
9-
"7.17",
10-
"6.8"
11-
],
12-
"targetPRLabels": ["backport"],
13-
"branchLabelMapping": {
14-
"^v8.4.0$": "main",
15-
"^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$": "$1.$2"
2+
"upstream" : "elastic/elasticsearch",
3+
"targetBranchChoices" : [ "main", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
4+
"targetPRLabels" : [ "backport" ],
5+
"branchLabelMapping" : {
6+
"^v8.8.0$" : "main",
7+
"^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
168
}
17-
}
9+
}

‎.ci/bwcVersions‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,10 @@ BWC_VERSION:
5555
- "7.17.4"
5656
- "7.17.5"
5757
- "7.17.6"
58+
- "7.17.7"
59+
- "7.17.8"
60+
- "7.17.9"
61+
- "7.17.10"
5862
- "8.0.0"
5963
- "8.0.1"
6064
- "8.1.0"
@@ -70,3 +74,16 @@ BWC_VERSION:
7074
- "8.3.2"
7175
- "8.3.3"
7276
- "8.4.0"
77+
- "8.4.1"
78+
- "8.4.2"
79+
- "8.4.3"
80+
- "8.5.0"
81+
- "8.5.1"
82+
- "8.5.2"
83+
- "8.5.3"
84+
- "8.6.0"
85+
- "8.6.1"
86+
- "8.6.2"
87+
- "8.6.3"
88+
- "8.7.0"
89+
- "8.8.0"

‎.ci/jobs.t/defaults.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,8 @@
5656
COMPOSE_HTTP_TIMEOUT=120
5757
JOB_BRANCH=%BRANCH%
5858
HOME=$JENKINS_HOME
59-
GRADLEW=./gradlew --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
60-
GRADLEW_BAT=./gradlew.bat --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
59+
GRADLEW=./gradlew --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
60+
GRADLEW_BAT=./gradlew.bat --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
6161
publishers:
6262
# Disable Junit publisher for now since it's causing OOME on our Jenkins instance
6363
# - junit:

‎.ci/jobs.t/elastic+elasticsearch+dra-snapshot.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
---
2+
- job:
3+
name: elastic+elasticsearch+%BRANCH%+dra-snapshot
4+
workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+dra-snapshot
5+
display-name: "elastic / elasticsearch # %BRANCH% - DRA snapshot"
6+
description: "Publishing Daily Releasable Artifacts (DRAs) of Elasticsearch %BRANCH% snapshots.\n"
7+
node: "ubuntu-20.04"
8+
builders:
9+
- inject:
10+
properties-file: '.ci/java-versions.properties'
11+
properties-content: |
12+
JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
13+
RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
14+
- shell: |
15+
#!/usr/local/bin/runbld --redirect-stderr
16+
WORKFLOW="snapshot"
17+
RM_BRANCH="%BRANCH%" && [[ "%BRANCH%" == "main" ]] && RM_BRANCH=master
18+
ES_VERSION=$(cat build-tools-internal/version.properties \
19+
| grep elasticsearch \
20+
| sed "s/elasticsearch *= *//g")
21+
VERSION_SUFFIX="" && [[ "$WORKFLOW" == "snapshot" ]] && VERSION_SUFFIX="-SNAPSHOT"
22+
BEATS_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh beats $RM_BRANCH $ES_VERSION $WORKFLOW)"
23+
ML_CPP_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh ml-cpp $RM_BRANCH $ES_VERSION $WORKFLOW)"
24+
set -euo pipefail
25+
set +x
26+
VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID)
27+
export VAULT_TOKEN
28+
$WORKSPACE/.ci/scripts/run-gradle.sh -Ddra.artifacts=true \
29+
-Ddra.artifacts.dependency.beats=${BEATS_BUILD_ID} \
30+
-Ddra.artifacts.dependency.ml-cpp=${ML_CPP_BUILD_ID} \
31+
-Ddra.workflow=$WORKFLOW \
32+
-Dcsv=$WORKSPACE/build/distributions/dependencies-${ES_VERSION}${VERSION_SUFFIX}.csv \
33+
buildReleaseArtifacts \
34+
exportCompressedDockerImages \
35+
:distribution:generateDependenciesReport
36+
37+
unset VAULT_TOKEN
38+
set -x
39+
$WORKSPACE/x-pack/plugin/sql/connectors/tableau/package.sh asm qualifier="$VERSION_SUFFIX"
40+
41+
# we regenerate this file as part of the release manager invocation
42+
rm $WORKSPACE/build/distributions/elasticsearch-jdbc-${ES_VERSION}${VERSION_SUFFIX}.taco.sha512
43+
44+
# Allow other users access to read the artifacts so they are readable in the
45+
# container
46+
find $WORKSPACE -type f -path "*/build/distributions/*" -exec chmod a+r {} \;
47+
48+
# Allow other users write access to create checksum files
49+
find $WORKSPACE -type d -path "*/build/distributions" -exec chmod a+w {} \;
50+
51+
# Artifacts should be generated
52+
docker run --rm \
53+
--name release-manager \
54+
-e VAULT_ADDR \
55+
-e VAULT_ROLE_ID \
56+
-e VAULT_SECRET_ID \
57+
--mount type=bind,readonly=false,src="$PWD",target=/artifacts \
58+
docker.elastic.co/infra/release-manager:latest \
59+
cli collect \
60+
--project elasticsearch \
61+
--branch "$RM_BRANCH" \
62+
--commit "$GIT_COMMIT" \
63+
--workflow "$WORKFLOW" \
64+
--version "$ES_VERSION" \
65+
--artifact-set main \
66+
--dependency beats:https://artifacts-${WORKFLOW}.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json \
67+
--dependency ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json

‎.ci/jobs.t/elastic+elasticsearch+dra-staging-trigger.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
jjbb-template: periodic-trigger-lgc.yml
3+
vars:
4+
- periodic-job: elastic+elasticsearch+%BRANCH%+dra-staging
5+
- lgc-job: elastic+elasticsearch+%BRANCH%+intake
6+
- cron: "H H/12 * * *"

‎.ci/jobs.t/elastic+elasticsearch+dra-staging.yml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
---
2+
- job:
3+
name: elastic+elasticsearch+%BRANCH%+dra-staging
4+
workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+dra-staging
5+
display-name: "elastic / elasticsearch # %BRANCH% - DRA staging"
6+
description: "Publishing Daily Releasable Artifacts (DRAs) of Elasticsearch %BRANCH% staging.\n"
7+
node: "ubuntu-20.04"
8+
builders:
9+
- inject:
10+
properties-file: '.ci/java-versions.properties'
11+
properties-content: |
12+
JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
13+
RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
14+
- shell: |
15+
#!/usr/local/bin/runbld --redirect-stderr
16+
17+
# Don't publish main branch to staging
18+
if [ "%BRANCH%" == "main" ]; then
19+
exit 0
20+
fi
21+
22+
WORKFLOW="staging"
23+
RM_BRANCH="%BRANCH%" && [[ "%BRANCH%" == "main" ]] && RM_BRANCH=master
24+
ES_VERSION=$(cat build-tools-internal/version.properties \
25+
| grep elasticsearch \
26+
| sed "s/elasticsearch *= *//g")
27+
VERSION_SUFFIX="" && [[ "$WORKFLOW" == "snapshot" ]] && VERSION_SUFFIX="-SNAPSHOT"
28+
BEATS_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh beats $RM_BRANCH $ES_VERSION $WORKFLOW)"
29+
ML_CPP_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh ml-cpp $RM_BRANCH $ES_VERSION $WORKFLOW)"
30+
set -euo pipefail
31+
set +x
32+
VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID)
33+
LICENSE_KEY=$(mktemp -d)/license.key
34+
export VAULT_TOKEN
35+
vault read -field pubkey secret/elasticsearch-ci/license | base64 --decode > $LICENSE_KEY
36+
unset VAULT_TOKEN
37+
38+
$WORKSPACE/.ci/scripts/run-gradle.sh -Ddra.artifacts=true \
39+
-Ddra.artifacts.dependency.beats=${BEATS_BUILD_ID} \
40+
-Ddra.artifacts.dependency.ml-cpp=${ML_CPP_BUILD_ID} \
41+
-Ddra.workflow=$WORKFLOW \
42+
-Dcsv=$WORKSPACE/build/distributions/dependencies-${ES_VERSION}${VERSION_SUFFIX}.csv \
43+
-Dbuild.snapshot=false \
44+
-Dlicense.key=$LICENSE_KEY \
45+
buildReleaseArtifacts \
46+
exportCompressedDockerImages \
47+
:distribution:generateDependenciesReport
48+
49+
set -x
50+
$WORKSPACE/x-pack/plugin/sql/connectors/tableau/package.sh asm qualifier="$VERSION_SUFFIX"
51+
52+
# we regenerate this file as part of the release manager invocation
53+
rm $WORKSPACE/build/distributions/elasticsearch-jdbc-${ES_VERSION}${VERSION_SUFFIX}.taco.sha512
54+
55+
# Allow other users access to read the artifacts so they are readable in the
56+
# container
57+
find $WORKSPACE -type f -path "*/build/distributions/*" -exec chmod a+r {} \;
58+
59+
# Allow other users write access to create checksum files
60+
find $WORKSPACE -type d -path "*/build/distributions" -exec chmod a+w {} \;
61+
62+
# Artifacts should be generated
63+
docker run --rm \
64+
--name release-manager \
65+
-e VAULT_ADDR \
66+
-e VAULT_ROLE_ID \
67+
-e VAULT_SECRET_ID \
68+
--mount type=bind,readonly=false,src="$PWD",target=/artifacts \
69+
docker.elastic.co/infra/release-manager:latest \
70+
cli collect \
71+
--project elasticsearch \
72+
--branch "$RM_BRANCH" \
73+
--commit "$GIT_COMMIT" \
74+
--workflow "$WORKFLOW" \
75+
--version "$ES_VERSION" \
76+
--artifact-set main \
77+
--dependency beats:https://artifacts-${WORKFLOW}.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json \
78+
--dependency ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json

‎.ci/jobs.t/elastic+elasticsearch+intake.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,13 @@
5454
kill-phase-on: NEVER
5555
current-parameters: true
5656
git-revision: true
57+
- multijob:
58+
name: Publish snapshot artifacts
59+
projects:
60+
- name: elastic+elasticsearch+%BRANCH%+dra-snapshot
61+
kill-phase-on: NEVER
62+
current-parameters: true
63+
git-revision: true
5764
- multijob:
5865
name: Update last good commit
5966
projects:

‎.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,7 @@
33
name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm
44
display-name: "elastic / elasticsearch # %BRANCH% - arm compatibility"
55
description: "Elasticsearch %BRANCH% ARM (aarch64) compatibility testing.\n"
6-
# Don't use ramdisk for now as we are exhausting memory and causing oomkiller to trigger
7-
# child-workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm"
6+
child-workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm"
87
project-type: matrix
98
node: master
109
scm:
@@ -17,6 +16,15 @@
1716
values:
1817
- "almalinux-8-aarch64&&immutable"
1918
- "ubuntu-1804-aarch64&&immutable"
19+
- axis:
20+
type: user-defined
21+
name: GRADLE_TASK
22+
values:
23+
- 'checkPart1'
24+
- 'checkPart2'
25+
- 'checkPart3'
26+
- 'bwcTestSnapshots'
27+
- 'checkRestCompat'
2028
builders:
2129
- inject:
2230
properties-file: '.ci/java-versions-aarch64.properties'
@@ -28,4 +36,4 @@
2836
JAVA16_HOME=$HOME/.java/jdk16
2937
- shell: |
3038
#!/usr/local/bin/runbld --redirect-stderr
31-
$WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check
39+
$WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $GRADLE_TASK

‎.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
description: "Elasticsearch %BRANCH% unix compatibility testing.\n"
66
project-type: matrix
77
node: master
8+
child-workspace: "/var/lib/jenkins/workspace/elastic+elasticsearch+%BRANCH%+multijob+platform-support-unix"
89
scm:
910
- git:
1011
wipe-workspace: false
@@ -14,7 +15,8 @@
1415
name: os
1516
values:
1617
- "centos-7&&immutable"
17-
- "amazon&&immutable"
18+
- "amazon-2&&immutable"
19+
- "amazon-2022&&immutable"
1820
- "debian-10&&immutable"
1921
- "debian-11&&immutable"
2022
- "opensuse-15-1&&immutable"

‎.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-geoip.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
properties-content: |
1111
JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
1212
RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
13-
geoip_use_service=true
1413
- shell: |
1514
#!/usr/local/bin/runbld --redirect-stderr
16-
$WORKSPACE/.ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest
15+
$WORKSPACE/.ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dgeoip_use_service=true

0 commit comments

Comments
 (0)