Skip to content

Commit 45814ad

Browse files
piergmpiergm
committed
merged main
2 parents cceb136 + 5f146f8 commit 45814ad

File tree

15,304 files changed

+755318
-272972
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

15,304 files changed

+755318
-272972
lines changed

‎.backportrc.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
22
"upstream" : "elastic/elasticsearch",
3-
"targetBranchChoices" : [ "main", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
3+
"targetBranchChoices" : [ "main", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
44
"targetPRLabels" : [ "backport" ],
55
"branchLabelMapping" : {
6-
"^v8.11.0$" : "main",
6+
"^v8.15.0$" : "main",
77
"^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
88
}
99
}

‎.buildkite/.editorconfig‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[*.ts]
2+
max_line_length = 120

‎.buildkite/.gitignore‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
2+
3+
# Logs
4+
5+
logs
6+
_.log
7+
npm-debug.log_
8+
yarn-debug.log*
9+
yarn-error.log*
10+
lerna-debug.log*
11+
.pnpm-debug.log*
12+
13+
# Diagnostic reports (https://nodejs.org/api/report.html)
14+
15+
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
16+
17+
# Runtime data
18+
19+
pids
20+
_.pid
21+
_.seed
22+
\*.pid.lock
23+
24+
# Directory for instrumented libs generated by jscoverage/JSCover
25+
26+
lib-cov
27+
28+
# Coverage directory used by tools like istanbul
29+
30+
coverage
31+
\*.lcov
32+
33+
# nyc test coverage
34+
35+
.nyc_output
36+
37+
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
38+
39+
.grunt
40+
41+
# Bower dependency directory (https://bower.io/)
42+
43+
bower_components
44+
45+
# node-waf configuration
46+
47+
.lock-wscript
48+
49+
# Compiled binary addons (https://nodejs.org/api/addons.html)
50+
51+
build/Release
52+
53+
# Dependency directories
54+
55+
node_modules/
56+
jspm_packages/
57+
58+
# Snowpack dependency directory (https://snowpack.dev/)
59+
60+
web_modules/
61+
62+
# TypeScript cache
63+
64+
\*.tsbuildinfo
65+
66+
# Optional npm cache directory
67+
68+
.npm
69+
70+
# Optional eslint cache
71+
72+
.eslintcache
73+
74+
# Optional stylelint cache
75+
76+
.stylelintcache
77+
78+
# Microbundle cache
79+
80+
.rpt2_cache/
81+
.rts2_cache_cjs/
82+
.rts2_cache_es/
83+
.rts2_cache_umd/
84+
85+
# Optional REPL history
86+
87+
.node_repl_history
88+
89+
# Output of 'npm pack'
90+
91+
\*.tgz
92+
93+
# Yarn Integrity file
94+
95+
.yarn-integrity
96+
97+
# dotenv environment variable files
98+
99+
.env
100+
.env.development.local
101+
.env.test.local
102+
.env.production.local
103+
.env.local
104+
105+
# parcel-bundler cache (https://parceljs.org/)
106+
107+
.cache
108+
.parcel-cache
109+
110+
# Next.js build output
111+
112+
.next
113+
out
114+
115+
# Nuxt.js build / generate output
116+
117+
.nuxt
118+
dist
119+
120+
# Gatsby files
121+
122+
.cache/
123+
124+
# Comment in the public line in if your project uses Gatsby and not Next.js
125+
126+
# https://nextjs.org/blog/next-9-1#public-directory-support
127+
128+
# public
129+
130+
# vuepress build output
131+
132+
.vuepress/dist
133+
134+
# vuepress v2.x temp and cache directory
135+
136+
.temp
137+
.cache
138+
139+
# Docusaurus cache and generated files
140+
141+
.docusaurus
142+
143+
# Serverless directories
144+
145+
.serverless/
146+
147+
# FuseBox cache
148+
149+
.fusebox/
150+
151+
# DynamoDB Local files
152+
153+
.dynamodb/
154+
155+
# TernJS port file
156+
157+
.tern-port
158+
159+
# Stores VSCode versions used for testing VSCode extensions
160+
161+
.vscode-test
162+
163+
# yarn v2
164+
165+
.yarn/cache
166+
.yarn/unplugged
167+
.yarn/build-state.yml
168+
.yarn/install-state.gz
169+
.pnp.\*

‎.buildkite/README.md‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Elasticsearch CI Pipelines
2+
3+
This directory contains pipeline definitions and scripts for running Elasticsearch CI on Buildkite.
4+
5+
## Directory Structure
6+
7+
- [pipelines](pipelines/) - pipeline definitions/yml
8+
- [scripts](scripts/) - scripts used by pipelines, inside steps
9+
- [hooks](hooks/) - [Buildkite hooks](https://buildkite.com/docs/agent/v3/hooks), where global env vars and secrets are set
10+
11+
## Pipeline Definitions
12+
13+
Pipelines are defined using YAML files residing in [pipelines](pipelines/). These are mostly static definitions that are used as-is, but there are a few dynamically-generated exceptions (see below).
14+
15+
### Dynamically Generated Pipelines
16+
17+
Pull request pipelines are generated dynamically based on labels, files changed, and other properties of pull requests.
18+
19+
Non-pull request pipelines that include BWC version matrices must also be generated whenever the [list of BWC versions](../.ci/bwcVersions) is updated.
20+
21+
#### Pull Request Pipelines
22+
23+
Pull request pipelines are generated dynamically at CI time based on numerous properties of the pull request. See [scripts/pull-request](scripts/pull-request) for details.
24+
25+
#### BWC Version Matrices
26+
27+
For pipelines that include BWC version matrices, you will see one or more template files (e.g. [periodic.template.yml](pipelines/periodic.template.yml)) and a corresponding generated file (e.g. [periodic.yml](pipelines/periodic.yml)). The generated file is the one that is actually used by Buildkite.
28+
29+
These files are updated by running:
30+
31+
```bash
32+
./gradlew updateCIBwcVersions
33+
```
34+
35+
This also runs automatically during release procedures.
36+
37+
You should always make changes to the template files, and run the above command to update the generated files.
38+
39+
## Node / TypeScript
40+
41+
Node (technically `bun`), TypeScript, and related files are currently used to generate pipelines for pull request CI. See [scripts/pull-request](scripts/pull-request) for details.

‎.buildkite/bun.lockb‎

1.92 KB
Binary file not shown.

‎.buildkite/check-es-serverless.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
steps:
2-
- trigger: elasticsearch-serverless-update-submodule
2+
- trigger: elasticsearch-serverless-validate-submodule
33
label: ":elasticsearch: Check elasticsearch changes against serverless"
44
build:
55
message: "Validate latest elasticsearch changes"

‎.buildkite/hooks/pre-command‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/bu
4848
export BUILDKITE_API_TOKEN
4949

5050
if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
51-
data=$(.buildkite/scripts/lucene-snapshot/get-credentials.sh)
51+
data=$(.buildkite/scripts/get-legacy-secret.sh aws-elastic/creds/lucene-snapshots)
5252

5353
AWS_ACCESS_KEY_ID=$(echo "$data" | jq -r .data.access_key)
5454
export AWS_ACCESS_KEY_ID
@@ -70,12 +70,38 @@ if [[ "${USE_DRA_CREDENTIALS:-}" == "true" ]]; then
7070
export DRA_VAULT_ADDR
7171
fi
7272

73+
source .buildkite/scripts/third-party-test-credentials.sh
74+
7375
if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then
7476
SNYK_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/migrated/snyk)
7577
export SNYK_TOKEN
7678
fi
7779

80+
if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then
81+
DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
82+
export DOCKER_REGISTRY_USERNAME
83+
84+
DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
85+
export DOCKER_REGISTRY_PASSWORD
86+
fi
87+
7888
if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then
7989
# Run in the background, while the job continues
8090
nohup .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &
8191
fi
92+
93+
# Initialize the build scan and gobld annotations with empty/open <details> tags
94+
# This ensures that they are collapsible when they get appended to
95+
if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* || "${BUILDKITE_LABEL:-}" == *"Upload Pipeline"* ]]; then
96+
cat << EOF | buildkite-agent annotate --context "gradle-build-scans" --style "info"
97+
<details>
98+
99+
<summary>Gradle build scan links</summary>
100+
EOF
101+
102+
cat << EOF | buildkite-agent annotate --context "ctx-gobld-metrics" --style "info"
103+
<details>
104+
105+
<summary>Agent information from gobld</summary>
106+
EOF
107+
fi

‎.buildkite/hooks/pre-command.bat‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,6 @@ set JOB_BRANCH=%BUILDKITE_BRANCH%
1818
set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
1919
set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
2020

21+
bash.exe -c "nohup bash .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &"
22+
2123
exit /b 0

‎.buildkite/package.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"name": "buildkite-pipelines",
3+
"type": "module",
4+
"devDependencies": {
5+
"@types/node": "^20.6.0",
6+
"bun-types": "latest",
7+
"yaml": "^2.3.2"
8+
},
9+
"peerDependencies": {
10+
"typescript": "^5.0.0"
11+
}
12+
}

‎.buildkite/packer_cache.sh‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
3+
ROOT_DIR=$(cd "$(dirname "$0")/.." && pwd)
4+
5+
branches=($(cat "$ROOT_DIR/branches.json" | jq -r '.branches[].branch'))
6+
for branch in "${branches[@]}"; do
7+
echo "Resolving dependencies for ${branch} branch"
8+
rm -rf "checkout/$branch"
9+
git clone /opt/git-mirrors/elastic-elasticsearch --branch "$branch" --single-branch "checkout/$branch"
10+
11+
CHECKOUT_DIR=$(cd "./checkout/${branch}" && pwd)
12+
CI_DIR="$CHECKOUT_DIR/.ci"
13+
14+
if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
15+
## On ARM we use a different properties file for setting java home
16+
## Also, we don't bother attempting to resolve dependencies for the 6.8 branch
17+
source "$CI_DIR/java-versions-aarch64.properties"
18+
export JAVA16_HOME="$HOME/.java/jdk16"
19+
else
20+
source "$CI_DIR/java-versions.properties"
21+
## We are caching BWC versions too, need these so we can build those
22+
export JAVA8_HOME="$HOME/.java/java8"
23+
export JAVA11_HOME="$HOME/.java/java11"
24+
export JAVA12_HOME="$HOME/.java/openjdk12"
25+
export JAVA13_HOME="$HOME/.java/openjdk13"
26+
export JAVA14_HOME="$HOME/.java/openjdk14"
27+
export JAVA15_HOME="$HOME/.java/openjdk15"
28+
export JAVA16_HOME="$HOME/.java/openjdk16"
29+
fi
30+
31+
export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
32+
"checkout/${branch}/gradlew" --project-dir "$CHECKOUT_DIR" --parallel -s resolveAllDependencies -Dorg.gradle.warning.mode=none -DisCI
33+
rm -rf "checkout/${branch}"
34+
done

0 commit comments

Comments
 (0)