Skip to content

Commit e303587

Browse files
pazonemergify[bot]
pazone
authored and
mergify[bot]
committed
[CI] BK Vault plugin for EC access (#8377)
* [CI] BK Vault plugin for ES access * Typo * Typo * Quick Windows test * Quick test Windows * Revert last two commits * Applied proposed changes * Fixed indentation * revert buildkite_analytics_token deletion * Remaned the anchor * Added the issue to comments * Updated FIPS pipeline (cherry picked from commit e2505e4)
1 parent 4e49dee commit e303587

File tree

5 files changed

+62
-25
lines changed

5 files changed

+62
-25
lines changed

‎.buildkite/bk.integration-fips.pipeline.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,15 @@ env:
88
IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1751072471"
99
IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1751072471"
1010

11+
# This section is used to define the plugins that will be used in the pipeline.
12+
# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
13+
common:
14+
- vault_ec_key_prod: &vault_ec_key_prod
15+
elastic/vault-secrets#v0.1.0:
16+
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
17+
field: "apiKey"
18+
env_var: "EC_API_KEY"
19+
1120
steps:
1221
- label: Build and push custom elastic-agent image
1322
depends_on:
@@ -45,6 +54,8 @@ steps:
4554
agents:
4655
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
4756
useCustomGlobalHooks: true
57+
plugins:
58+
- *vault_ec_key_prod
4859

4960
- group: "fips:Stateful:Ubuntu"
5061
key: integration-tests-ubuntu-fips
@@ -71,6 +82,8 @@ steps:
7182
provider: "aws"
7283
image: "${IMAGE_UBUNTU_X86_64_FIPS}"
7384
instanceType: "m5.2xlarge"
85+
plugins:
86+
- *vault_ec_key_prod
7487
matrix:
7588
setup:
7689
sudo:
@@ -99,6 +112,8 @@ steps:
99112
provider: "aws"
100113
image: "${IMAGE_UBUNTU_ARM64_FIPS}"
101114
instanceType: "m6g.2xlarge"
115+
plugins:
116+
- *vault_ec_key_prod
102117
matrix:
103118
setup:
104119
sudo:
@@ -137,6 +152,8 @@ steps:
137152
agents:
138153
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
139154
useCustomGlobalHooks: true
155+
plugins:
156+
- *vault_ec_key_prod
140157

141158
- label: Aggregate test reports
142159
depends_on:

‎.buildkite/bk.integration.pipeline.yml‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,11 @@ common:
3737
KIBANA_HOST: ea-serverless-it-kibana-hostname
3838
KIBANA_USERNAME: ea-serverless-it-kibana-username
3939
KIBANA_PASSWORD: ea-serverless-it-kibana-password
40+
- vault_ec_key_prod: &vault_ec_key_prod
41+
elastic/vault-secrets#v0.1.0:
42+
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
43+
field: "apiKey"
44+
env_var: "EC_API_KEY"
4045

4146
steps:
4247
- label: Start ESS stack for integration tests
@@ -56,6 +61,8 @@ steps:
5661
agents:
5762
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
5863
useCustomGlobalHooks: true
64+
plugins:
65+
- *vault_ec_key_prod
5966

6067
- group: "Extended runtime leak tests"
6168
key: extended-integration-tests
@@ -83,6 +90,9 @@ steps:
8390
retry:
8491
automatic:
8592
limit: 1
93+
plugins:
94+
- *vault_ec_key_prod
95+
8696
- label: "Windows:2025:amd64:sudo"
8797
depends_on:
8898
- packaging-windows
@@ -101,6 +111,9 @@ steps:
101111
provider: "gcp"
102112
machineType: "n2-standard-8"
103113
image: "${IMAGE_WIN_2025}"
114+
plugins:
115+
- *vault_ec_key_prod
116+
104117
- label: "Ubuntu:2404:amd64:sudo"
105118
depends_on: packaging-ubuntu-x86-64
106119
env:
@@ -118,6 +131,8 @@ steps:
118131
provider: "gcp"
119132
machineType: "n2-standard-8"
120133
image: "${IMAGE_UBUNTU_2404_X86_64}"
134+
plugins:
135+
- *vault_ec_key_prod
121136

122137
- group: "Stateful: Windows"
123138
key: integration-tests-win
@@ -145,6 +160,8 @@ steps:
145160
retry:
146161
automatic:
147162
limit: 1
163+
plugins:
164+
- *vault_ec_key_prod
148165
matrix:
149166
- default
150167
- fleet
@@ -173,6 +190,8 @@ steps:
173190
retry:
174191
automatic:
175192
limit: 1
193+
plugins:
194+
- *vault_ec_key_prod
176195
matrix:
177196
- default
178197

@@ -194,6 +213,8 @@ steps:
194213
retry:
195214
automatic:
196215
limit: 1
216+
plugins:
217+
- *vault_ec_key_prod
197218
matrix:
198219
- default
199220
- fleet
@@ -222,6 +243,8 @@ steps:
222243
provider: "gcp"
223244
machineType: "n2-standard-8"
224245
image: "${IMAGE_WIN_2025}"
246+
plugins:
247+
- *vault_ec_key_prod
225248
matrix:
226249
- default
227250

@@ -250,6 +273,8 @@ steps:
250273
provider: "gcp"
251274
machineType: "n2-standard-8"
252275
image: "${IMAGE_UBUNTU_2404_X86_64}"
276+
plugins:
277+
- *vault_ec_key_prod
253278
matrix:
254279
- default
255280

@@ -272,6 +297,8 @@ steps:
272297
provider: "gcp"
273298
machineType: "n2-standard-8"
274299
image: "${IMAGE_UBUNTU_2404_X86_64}"
300+
plugins:
301+
- *vault_ec_key_prod
275302
matrix:
276303
- default
277304
- upgrade
@@ -306,6 +333,8 @@ steps:
306333
retry:
307334
automatic:
308335
limit: 1
336+
plugins:
337+
- *vault_ec_key_prod
309338
matrix:
310339
- default
311340
- upgrade
@@ -342,6 +371,8 @@ steps:
342371
provider: "aws"
343372
image: "${IMAGE_UBUNTU_2404_ARM_64}"
344373
instanceType: "m6g.xlarge"
374+
plugins:
375+
- *vault_ec_key_prod
345376
matrix:
346377
- default
347378

@@ -370,6 +401,8 @@ steps:
370401
provider: "gcp"
371402
machineType: "n2-standard-8"
372403
image: "${IMAGE_DEBIAN_12}"
404+
plugins:
405+
- *vault_ec_key_prod
373406
matrix:
374407
- default
375408

@@ -392,6 +425,8 @@ steps:
392425
provider: "gcp"
393426
machineType: "n2-standard-8"
394427
image: "${IMAGE_DEBIAN_12}"
428+
plugins:
429+
- *vault_ec_key_prod
395430
matrix:
396431
- default
397432
- upgrade
@@ -431,6 +466,8 @@ steps:
431466
retry:
432467
automatic:
433468
limit: 1
469+
plugins:
470+
- *vault_ec_key_prod
434471
agents:
435472
provider: "gcp"
436473
machineType: "n2-standard-8"
@@ -466,6 +503,8 @@ steps:
466503
machineType: "n2-standard-4"
467504
image: "${IMAGE_UBUNTU_2404_X86_64}"
468505
diskSizeGb: 80
506+
plugins:
507+
- *vault_ec_key_prod
469508
matrix:
470509
setup:
471510
variants:
@@ -569,7 +608,8 @@ steps:
569608
agents:
570609
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
571610
useCustomGlobalHooks: true
572-
611+
plugins:
612+
- *vault_ec_key_prod
573613
- label: Aggregate test reports
574614
# Warning: The key has a hook in pre-command
575615
key: aggregate-reports

‎.buildkite/hooks/pre-command‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ fi
1515

1616
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
1717
CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp"
18+
# This key exists for backward compatibility with OGC framework
19+
# see https://github.com/elastic/elastic-agent/issues/8536
1820
CI_ESS_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
1921
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
2022

‎.buildkite/scripts/steps/ess.ps1‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,7 @@ function ess_up {
1313
Write-Error "Error: Specify stack version: ess_up [stack_version]"
1414
return 1
1515
}
16-
17-
$Env:EC_API_KEY = Retry-Command -ScriptBlock {
18-
vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
19-
}
20-
21-
if (-not $Env:EC_API_KEY) {
22-
Write-Error "Error: Failed to get EC API key from vault"
23-
exit 1
24-
}
25-
16+
2617
$BuildkiteBuildCreator = if ($Env:BUILDKITE_BUILD_CREATOR) { $Env:BUILDKITE_BUILD_CREATOR } else { get_git_user_email }
2718
$BuildkiteBuildNumber = if ($Env:BUILDKITE_BUILD_NUMBER) { $Env:BUILDKITE_BUILD_NUMBER } else { "0" }
2819
$BuildkitePipelineSlug = if ($Env:BUILDKITE_PIPELINE_SLUG) { $Env:BUILDKITE_PIPELINE_SLUG } else { "elastic-agent-integration-tests" }
@@ -56,10 +47,7 @@ function ess_down {
5647
return 0
5748
}
5849
Write-Output "~~~ Tearing down the ESS Stack(created for this step)"
59-
try {
60-
$Env:EC_API_KEY = Retry-Command -ScriptBlock {
61-
vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
62-
}
50+
try {
6351
Push-Location -Path $TfDir
6452
& terraform init
6553
& terraform destroy -auto-approve

‎.buildkite/scripts/steps/ess.sh‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,6 @@ function ess_up() {
1313
return 1
1414
fi
1515

16-
export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
17-
18-
if [[ -z "${EC_API_KEY}" ]]; then
19-
echo "Error: Failed to get EC API key from vault" >&2
20-
exit 1
21-
fi
22-
2316
BUILDKITE_BUILD_CREATOR="${BUILDKITE_BUILD_CREATOR:-"$(get_git_user_email)"}"
2417
BUILDKITE_BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-"0"}"
2518
BUILDKITE_PIPELINE_SLUG="${BUILDKITE_PIPELINE_SLUG:-"elastic-agent-integration-tests"}"
@@ -48,9 +41,6 @@ function ess_down() {
4841
echo "~~~ Tearing down the ESS Stack"
4942
local WORKSPACE=$(git rev-parse --show-toplevel)
5043
local TF_DIR="${WORKSPACE}/test_infra/ess/"
51-
if [ -z "${EC_API_KEY:-}" ]; then
52-
export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
53-
fi
5444

5545
pushd "${TF_DIR}"
5646
terraform init

0 commit comments

Comments
 (0)