Skip to content

Commit d04e400

Browse files
promise2mmpromise2mm
authored
Merge pull request #1 from elastic/main
merge
2 parents febf22c + f66ca2d commit d04e400

File tree

23,970 files changed

+1727945
-631231
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

23,970 files changed

+1727945
-631231
lines changed

‎.backportrc.json‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,9 @@
11
{
2-
"upstream": "elastic/elasticsearch",
3-
"targetBranchChoices": [
4-
"master",
5-
"8.0",
6-
"7.17",
7-
"7.16",
8-
"6.8"
9-
],
10-
"targetPRLabels": ["backport"],
11-
"branchLabelMapping": {
12-
"^v8.1.0$": "master",
13-
"^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$": "$1.$2"
2+
"upstream" : "elastic/elasticsearch",
3+
"targetBranchChoices" : [ "main", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
4+
"targetPRLabels" : [ "backport" ],
5+
"branchLabelMapping" : {
6+
"^v8.14.0$" : "main",
7+
"^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
148
}
15-
}
9+
}

‎.buildkite/.editorconfig‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[*.ts]
2+
max_line_length = 120

‎.buildkite/.gitignore‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
2+
3+
# Logs
4+
5+
logs
6+
_.log
7+
npm-debug.log_
8+
yarn-debug.log*
9+
yarn-error.log*
10+
lerna-debug.log*
11+
.pnpm-debug.log*
12+
13+
# Diagnostic reports (https://nodejs.org/api/report.html)
14+
15+
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
16+
17+
# Runtime data
18+
19+
pids
20+
_.pid
21+
_.seed
22+
\*.pid.lock
23+
24+
# Directory for instrumented libs generated by jscoverage/JSCover
25+
26+
lib-cov
27+
28+
# Coverage directory used by tools like istanbul
29+
30+
coverage
31+
\*.lcov
32+
33+
# nyc test coverage
34+
35+
.nyc_output
36+
37+
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
38+
39+
.grunt
40+
41+
# Bower dependency directory (https://bower.io/)
42+
43+
bower_components
44+
45+
# node-waf configuration
46+
47+
.lock-wscript
48+
49+
# Compiled binary addons (https://nodejs.org/api/addons.html)
50+
51+
build/Release
52+
53+
# Dependency directories
54+
55+
node_modules/
56+
jspm_packages/
57+
58+
# Snowpack dependency directory (https://snowpack.dev/)
59+
60+
web_modules/
61+
62+
# TypeScript cache
63+
64+
\*.tsbuildinfo
65+
66+
# Optional npm cache directory
67+
68+
.npm
69+
70+
# Optional eslint cache
71+
72+
.eslintcache
73+
74+
# Optional stylelint cache
75+
76+
.stylelintcache
77+
78+
# Microbundle cache
79+
80+
.rpt2_cache/
81+
.rts2_cache_cjs/
82+
.rts2_cache_es/
83+
.rts2_cache_umd/
84+
85+
# Optional REPL history
86+
87+
.node_repl_history
88+
89+
# Output of 'npm pack'
90+
91+
\*.tgz
92+
93+
# Yarn Integrity file
94+
95+
.yarn-integrity
96+
97+
# dotenv environment variable files
98+
99+
.env
100+
.env.development.local
101+
.env.test.local
102+
.env.production.local
103+
.env.local
104+
105+
# parcel-bundler cache (https://parceljs.org/)
106+
107+
.cache
108+
.parcel-cache
109+
110+
# Next.js build output
111+
112+
.next
113+
out
114+
115+
# Nuxt.js build / generate output
116+
117+
.nuxt
118+
dist
119+
120+
# Gatsby files
121+
122+
.cache/
123+
124+
# Comment in the public line in if your project uses Gatsby and not Next.js
125+
126+
# https://nextjs.org/blog/next-9-1#public-directory-support
127+
128+
# public
129+
130+
# vuepress build output
131+
132+
.vuepress/dist
133+
134+
# vuepress v2.x temp and cache directory
135+
136+
.temp
137+
.cache
138+
139+
# Docusaurus cache and generated files
140+
141+
.docusaurus
142+
143+
# Serverless directories
144+
145+
.serverless/
146+
147+
# FuseBox cache
148+
149+
.fusebox/
150+
151+
# DynamoDB Local files
152+
153+
.dynamodb/
154+
155+
# TernJS port file
156+
157+
.tern-port
158+
159+
# Stores VSCode versions used for testing VSCode extensions
160+
161+
.vscode-test
162+
163+
# yarn v2
164+
165+
.yarn/cache
166+
.yarn/unplugged
167+
.yarn/build-state.yml
168+
.yarn/install-state.gz
169+
.pnp.\*

‎.buildkite/README.md‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Elasticsearch CI Pipelines
2+
3+
This directory contains pipeline definitions and scripts for running Elasticsearch CI on Buildkite.
4+
5+
## Directory Structure
6+
7+
- [pipelines](pipelines/) - pipeline definitions/yml
8+
- [scripts](scripts/) - scripts used by pipelines, inside steps
9+
- [hooks](hooks/) - [Buildkite hooks](https://buildkite.com/docs/agent/v3/hooks), where global env vars and secrets are set
10+
11+
## Pipeline Definitions
12+
13+
Pipelines are defined using YAML files residing in [pipelines](pipelines/). These are mostly static definitions that are used as-is, but there are a few dynamically-generated exceptions (see below).
14+
15+
### Dynamically Generated Pipelines
16+
17+
Pull request pipelines are generated dynamically based on labels, files changed, and other properties of pull requests.
18+
19+
Non-pull request pipelines that include BWC version matrices must also be generated whenever the [list of BWC versions](../.ci/bwcVersions) is updated.
20+
21+
#### Pull Request Pipelines
22+
23+
Pull request pipelines are generated dynamically at CI time based on numerous properties of the pull request. See [scripts/pull-request](scripts/pull-request) for details.
24+
25+
#### BWC Version Matrices
26+
27+
For pipelines that include BWC version matrices, you will see one or more template files (e.g. [periodic.template.yml](pipelines/periodic.template.yml)) and a corresponding generated file (e.g. [periodic.yml](pipelines/periodic.yml)). The generated file is the one that is actually used by Buildkite.
28+
29+
These files are updated by running:
30+
31+
```bash
32+
./gradlew updateCIBwcVersions
33+
```
34+
35+
This also runs automatically during release procedures.
36+
37+
You should always make changes to the template files, and run the above command to update the generated files.
38+
39+
## Node / TypeScript
40+
41+
Node (technically `bun`), TypeScript, and related files are currently used to generate pipelines for pull request CI. See [scripts/pull-request](scripts/pull-request) for details.

‎.buildkite/bun.lockb‎

1.92 KB
Binary file not shown.

‎.buildkite/check-es-serverless.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
steps:
2+
- trigger: elasticsearch-serverless-validate-submodule
3+
label: ":elasticsearch: Check elasticsearch changes against serverless"
4+
build:
5+
message: "Validate latest elasticsearch changes"
6+
env:
7+
ELASTICSEARCH_SUBMODULE_COMMIT: "${BUILDKITE_COMMIT}"
8+
UPDATE_SUBMODULE: "false"
9+

‎.buildkite/hooks/pre-command‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
#!/bin/bash
2+
3+
# On some distros, this directory ends up not readable by the `elasticsearch` user that gets created during tests
4+
# This fixes that
5+
chmod 755 ~
6+
7+
WORKSPACE="$(pwd)"
8+
export WORKSPACE
9+
10+
BUILD_NUMBER="$BUILDKITE_BUILD_NUMBER"
11+
export BUILD_NUMBER
12+
13+
COMPOSE_HTTP_TIMEOUT="120"
14+
export COMPOSE_HTTP_TIMEOUT
15+
16+
JOB_BRANCH="$BUILDKITE_BRANCH"
17+
export JOB_BRANCH
18+
19+
GRADLEW="./gradlew --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
20+
export GRADLEW
21+
22+
GRADLEW_BAT="./gradlew.bat --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
23+
export GRADLEW_BAT
24+
25+
export $(cat .ci/java-versions.properties | grep '=' | xargs)
26+
27+
JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
28+
export JAVA_HOME
29+
30+
JAVA11_HOME="$HOME/.java/java11"
31+
export JAVA11_HOME
32+
33+
JAVA16_HOME="$HOME/.java/openjdk16"
34+
export JAVA16_HOME
35+
36+
if [[ "${ES_RUNTIME_JAVA:-}" ]]; then
37+
RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
38+
export RUNTIME_JAVA_HOME
39+
fi
40+
41+
GRADLE_BUILD_CACHE_USERNAME=$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
42+
export GRADLE_BUILD_CACHE_USERNAME
43+
44+
GRADLE_BUILD_CACHE_PASSWORD=$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
45+
export GRADLE_BUILD_CACHE_PASSWORD
46+
47+
BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token)
48+
export BUILDKITE_API_TOKEN
49+
50+
if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
51+
data=$(.buildkite/scripts/get-legacy-secret.sh aws-elastic/creds/lucene-snapshots)
52+
53+
AWS_ACCESS_KEY_ID=$(echo "$data" | jq -r .data.access_key)
54+
export AWS_ACCESS_KEY_ID
55+
56+
AWS_SECRET_ACCESS_KEY=$(echo "$data" | jq -r .data.secret_key)
57+
export AWS_SECRET_ACCESS_KEY
58+
59+
unset data
60+
fi
61+
62+
if [[ "${USE_DRA_CREDENTIALS:-}" == "true" ]]; then
63+
DRA_VAULT_ROLE_ID_SECRET=$(vault read -field=role-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
64+
export DRA_VAULT_ROLE_ID_SECRET
65+
66+
DRA_VAULT_SECRET_ID_SECRET=$(vault read -field=secret-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
67+
export DRA_VAULT_SECRET_ID_SECRET
68+
69+
DRA_VAULT_ADDR=https://secrets.elastic.co:8200
70+
export DRA_VAULT_ADDR
71+
fi
72+
73+
source .buildkite/scripts/third-party-test-credentials.sh
74+
75+
if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then
76+
SNYK_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/migrated/snyk)
77+
export SNYK_TOKEN
78+
fi
79+
80+
if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then
81+
DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
82+
export DOCKER_REGISTRY_USERNAME
83+
84+
DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
85+
export DOCKER_REGISTRY_PASSWORD
86+
fi
87+
88+
if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then
89+
# Run in the background, while the job continues
90+
nohup .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &
91+
fi
92+
93+
# Initialize the build scan and gobld annotations with empty/open <details> tags
94+
# This ensures that they are collapsible when they get appended to
95+
if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* || "${BUILDKITE_LABEL:-}" == *"Upload Pipeline"* ]]; then
96+
cat << EOF | buildkite-agent annotate --context "gradle-build-scans" --style "info"
97+
<details>
98+
99+
<summary>Gradle build scan links</summary>
100+
EOF
101+
102+
cat << EOF | buildkite-agent annotate --context "ctx-gobld-metrics" --style "info"
103+
<details>
104+
105+
<summary>Agent information from gobld</summary>
106+
EOF
107+
fi

‎.buildkite/hooks/pre-command.bat‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
@ECHO OFF
2+
3+
FOR /F "tokens=* eol=#" %%i in ('type .ci\java-versions.properties') do set %%i
4+
5+
SET JAVA_HOME=%USERPROFILE%\.java\%ES_BUILD_JAVA%
6+
SET JAVA16_HOME=%USERPROFILE%\.java\openjdk16
7+
8+
SET GRADLEW=./gradlew --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
9+
SET GRADLEW_BAT=./gradlew.bat --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
10+
11+
(if not exist "%USERPROFILE%/.gradle" mkdir "%USERPROFILE%/.gradle") && (echo. >> "%USERPROFILE%/.gradle/gradle.properties" && echo org.gradle.daemon=false >> "%USERPROFILE%/.gradle/gradle.properties")
12+
13+
set WORKSPACE=%cd%
14+
set BUILD_NUMBER=%BUILDKITE_BUILD_NUMBER%
15+
set COMPOSE_HTTP_TIMEOUT=120
16+
set JOB_BRANCH=%BUILDKITE_BRANCH%
17+
18+
set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
19+
set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
20+
21+
exit /b 0

‎.buildkite/package.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"name": "buildkite-pipelines",
3+
"type": "module",
4+
"devDependencies": {
5+
"@types/node": "^20.6.0",
6+
"bun-types": "latest",
7+
"yaml": "^2.3.2"
8+
},
9+
"peerDependencies": {
10+
"typescript": "^5.0.0"
11+
}
12+
}

0 commit comments

Comments
 (0)