Skip to content

Commit 9cc8aeb

Browse files
pazonemergify[bot]
pazone
authored and
mergify[bot]
committed
[CI] BK Vault plugin for EC access (#8377)
* [CI] BK Vault plugin for ES access * Typo * Typo * Quick Windows test * Quick test Windows * Revert last two commits * Applied proposed changes * Fixed indentation * revert buildkite_analytics_token deletion * Remaned the anchor * Added the issue to comments * Updated FIPS pipeline (cherry picked from commit e2505e4)
1 parent 3758211 commit 9cc8aeb

File tree

5 files changed

+62
-25
lines changed

5 files changed

+62
-25
lines changed

‎.buildkite/bk.integration-fips.pipeline.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,15 @@ env:
88
IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1751072471"
99
IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1751072471"
1010

11+
# This section is used to define the plugins that will be used in the pipeline.
12+
# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
13+
common:
14+
- vault_ec_key_prod: &vault_ec_key_prod
15+
elastic/vault-secrets#v0.1.0:
16+
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
17+
field: "apiKey"
18+
env_var: "EC_API_KEY"
19+
1120
steps:
1221
- label: Build and push custom elastic-agent image
1322
depends_on:
@@ -45,6 +54,8 @@ steps:
4554
agents:
4655
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
4756
useCustomGlobalHooks: true
57+
plugins:
58+
- *vault_ec_key_prod
4859

4960
- group: "fips:Stateful:Ubuntu"
5061
key: integration-tests-ubuntu-fips
@@ -71,6 +82,8 @@ steps:
7182
provider: "aws"
7283
image: "${IMAGE_UBUNTU_X86_64_FIPS}"
7384
instanceType: "m5.2xlarge"
85+
plugins:
86+
- *vault_ec_key_prod
7487
matrix:
7588
setup:
7689
sudo:
@@ -99,6 +112,8 @@ steps:
99112
provider: "aws"
100113
image: "${IMAGE_UBUNTU_ARM64_FIPS}"
101114
instanceType: "m6g.2xlarge"
115+
plugins:
116+
- *vault_ec_key_prod
102117
matrix:
103118
setup:
104119
sudo:
@@ -137,6 +152,8 @@ steps:
137152
agents:
138153
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
139154
useCustomGlobalHooks: true
155+
plugins:
156+
- *vault_ec_key_prod
140157

141158
- label: Aggregate test reports
142159
depends_on:

‎.buildkite/bk.integration.pipeline.yml‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,11 @@ common:
3737
KIBANA_HOST: ea-serverless-it-kibana-hostname
3838
KIBANA_USERNAME: ea-serverless-it-kibana-username
3939
KIBANA_PASSWORD: ea-serverless-it-kibana-password
40+
- vault_ec_key_prod: &vault_ec_key_prod
41+
elastic/vault-secrets#v0.1.0:
42+
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
43+
field: "apiKey"
44+
env_var: "EC_API_KEY"
4045

4146
steps:
4247
- label: Start ESS stack for integration tests
@@ -56,6 +61,8 @@ steps:
5661
agents:
5762
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
5863
useCustomGlobalHooks: true
64+
plugins:
65+
- *vault_ec_key_prod
5966

6067
- group: "Extended runtime leak tests"
6168
key: extended-integration-tests
@@ -83,6 +90,9 @@ steps:
8390
retry:
8491
automatic:
8592
limit: 1
93+
plugins:
94+
- *vault_ec_key_prod
95+
8696
- label: "Windows:2025:amd64:sudo"
8797
depends_on:
8898
- packaging-windows
@@ -101,6 +111,9 @@ steps:
101111
provider: "gcp"
102112
machineType: "n2-standard-8"
103113
image: "${IMAGE_WIN_2025}"
114+
plugins:
115+
- *vault_ec_key_prod
116+
104117
- label: "Ubuntu:2404:amd64:sudo"
105118
depends_on: packaging-ubuntu-x86-64
106119
env:
@@ -118,6 +131,8 @@ steps:
118131
provider: "gcp"
119132
machineType: "n2-standard-8"
120133
image: "${IMAGE_UBUNTU_2404_X86_64}"
134+
plugins:
135+
- *vault_ec_key_prod
121136

122137
- group: "Stateful: Windows"
123138
key: integration-tests-win
@@ -145,6 +160,8 @@ steps:
145160
retry:
146161
automatic:
147162
limit: 1
163+
plugins:
164+
- *vault_ec_key_prod
148165
matrix:
149166
- default
150167
- fleet
@@ -172,6 +189,8 @@ steps:
172189
retry:
173190
automatic:
174191
limit: 1
192+
plugins:
193+
- *vault_ec_key_prod
175194
matrix:
176195
- default
177196

@@ -193,6 +212,8 @@ steps:
193212
retry:
194213
automatic:
195214
limit: 1
215+
plugins:
216+
- *vault_ec_key_prod
196217
matrix:
197218
- default
198219
- fleet
@@ -221,6 +242,8 @@ steps:
221242
provider: "gcp"
222243
machineType: "n2-standard-8"
223244
image: "${IMAGE_WIN_2025}"
245+
plugins:
246+
- *vault_ec_key_prod
224247
matrix:
225248
- default
226249

@@ -249,6 +272,8 @@ steps:
249272
provider: "gcp"
250273
machineType: "n2-standard-8"
251274
image: "${IMAGE_UBUNTU_2404_X86_64}"
275+
plugins:
276+
- *vault_ec_key_prod
252277
matrix:
253278
- default
254279

@@ -271,6 +296,8 @@ steps:
271296
provider: "gcp"
272297
machineType: "n2-standard-8"
273298
image: "${IMAGE_UBUNTU_2404_X86_64}"
299+
plugins:
300+
- *vault_ec_key_prod
274301
matrix:
275302
- default
276303
- upgrade
@@ -304,6 +331,8 @@ steps:
304331
retry:
305332
automatic:
306333
limit: 1
334+
plugins:
335+
- *vault_ec_key_prod
307336
matrix:
308337
- default
309338
- upgrade
@@ -339,6 +368,8 @@ steps:
339368
provider: "aws"
340369
image: "${IMAGE_UBUNTU_2404_ARM_64}"
341370
instanceType: "m6g.xlarge"
371+
plugins:
372+
- *vault_ec_key_prod
342373
matrix:
343374
- default
344375

@@ -367,6 +398,8 @@ steps:
367398
provider: "gcp"
368399
machineType: "n2-standard-8"
369400
image: "${IMAGE_DEBIAN_12}"
401+
plugins:
402+
- *vault_ec_key_prod
370403
matrix:
371404
- default
372405

@@ -389,6 +422,8 @@ steps:
389422
provider: "gcp"
390423
machineType: "n2-standard-8"
391424
image: "${IMAGE_DEBIAN_12}"
425+
plugins:
426+
- *vault_ec_key_prod
392427
matrix:
393428
- default
394429
- upgrade
@@ -428,6 +463,8 @@ steps:
428463
retry:
429464
automatic:
430465
limit: 1
466+
plugins:
467+
- *vault_ec_key_prod
431468
agents:
432469
provider: "gcp"
433470
machineType: "n2-standard-8"
@@ -463,6 +500,8 @@ steps:
463500
machineType: "n2-standard-4"
464501
image: "${IMAGE_UBUNTU_2404_X86_64}"
465502
diskSizeGb: 80
503+
plugins:
504+
- *vault_ec_key_prod
466505
matrix:
467506
setup:
468507
variants:
@@ -565,7 +604,8 @@ steps:
565604
agents:
566605
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
567606
useCustomGlobalHooks: true
568-
607+
plugins:
608+
- *vault_ec_key_prod
569609
- label: Aggregate test reports
570610
# Warning: The key has a hook in pre-command
571611
key: aggregate-reports

‎.buildkite/hooks/pre-command‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ fi
1515

1616
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
1717
CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp"
18+
# This key exists for backward compatibility with OGC framework
19+
# see https://github.com/elastic/elastic-agent/issues/8536
1820
CI_ESS_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
1921
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
2022

‎.buildkite/scripts/steps/ess.ps1‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,7 @@ function ess_up {
1313
Write-Error "Error: Specify stack version: ess_up [stack_version]"
1414
return 1
1515
}
16-
17-
$Env:EC_API_KEY = Retry-Command -ScriptBlock {
18-
vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
19-
}
20-
21-
if (-not $Env:EC_API_KEY) {
22-
Write-Error "Error: Failed to get EC API key from vault"
23-
exit 1
24-
}
25-
16+
2617
$BuildkiteBuildCreator = if ($Env:BUILDKITE_BUILD_CREATOR) { $Env:BUILDKITE_BUILD_CREATOR } else { get_git_user_email }
2718
$BuildkiteBuildNumber = if ($Env:BUILDKITE_BUILD_NUMBER) { $Env:BUILDKITE_BUILD_NUMBER } else { "0" }
2819
$BuildkitePipelineSlug = if ($Env:BUILDKITE_PIPELINE_SLUG) { $Env:BUILDKITE_PIPELINE_SLUG } else { "elastic-agent-integration-tests" }
@@ -56,10 +47,7 @@ function ess_down {
5647
return 0
5748
}
5849
Write-Output "~~~ Tearing down the ESS Stack(created for this step)"
59-
try {
60-
$Env:EC_API_KEY = Retry-Command -ScriptBlock {
61-
vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
62-
}
50+
try {
6351
Push-Location -Path $TfDir
6452
& terraform init
6553
& terraform destroy -auto-approve

‎.buildkite/scripts/steps/ess.sh‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,6 @@ function ess_up() {
1313
return 1
1414
fi
1515

16-
export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
17-
18-
if [[ -z "${EC_API_KEY}" ]]; then
19-
echo "Error: Failed to get EC API key from vault" >&2
20-
exit 1
21-
fi
22-
2316
BUILDKITE_BUILD_CREATOR="${BUILDKITE_BUILD_CREATOR:-"$(get_git_user_email)"}"
2417
BUILDKITE_BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-"0"}"
2518
BUILDKITE_PIPELINE_SLUG="${BUILDKITE_PIPELINE_SLUG:-"elastic-agent-integration-tests"}"
@@ -48,9 +41,6 @@ function ess_down() {
4841
echo "~~~ Tearing down the ESS Stack"
4942
local WORKSPACE=$(git rev-parse --show-toplevel)
5043
local TF_DIR="${WORKSPACE}/test_infra/ess/"
51-
if [ -z "${EC_API_KEY:-}" ]; then
52-
export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
53-
fi
5444

5545
pushd "${TF_DIR}"
5646
terraform init

0 commit comments

Comments
 (0)