Skip to content

Commit e2505e4

Browse files
pazonepazone
authored
[CI] BK Vault plugin for EC access (#8377)
* [CI] BK Vault plugin for ES access * Typo * Typo * Quick Windows test * Quick test Windows * Revert last two commits * Applied proposed changes * Fixed indentation * revert buildkite_analytics_token deletion * Remaned the anchor * Added the issue to comments * Updated FIPS pipeline
1 parent 2ec9992 commit e2505e4

File tree

5 files changed

+62
-25
lines changed

5 files changed

+62
-25
lines changed

‎.buildkite/bk.integration-fips.pipeline.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,15 @@ env:
1010
# Remove AGENT_VERSION pinning once 9.2.0 DRA and stack are released
1111
AGENT_VERSION: "9.1.0-SNAPSHOT"
1212

13+
# This section is used to define the plugins that will be used in the pipeline.
14+
# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
15+
common:
16+
- vault_ec_key_prod: &vault_ec_key_prod
17+
elastic/vault-secrets#v0.1.0:
18+
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
19+
field: "apiKey"
20+
env_var: "EC_API_KEY"
21+
1322
steps:
1423
- label: Build and push custom elastic-agent image
1524
depends_on:
@@ -47,6 +56,8 @@ steps:
4756
agents:
4857
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
4958
useCustomGlobalHooks: true
59+
plugins:
60+
- *vault_ec_key_prod
5061

5162
- group: "fips:Stateful:Ubuntu"
5263
key: integration-tests-ubuntu-fips
@@ -73,6 +84,8 @@ steps:
7384
provider: "aws"
7485
image: "${IMAGE_UBUNTU_X86_64_FIPS}"
7586
instanceType: "m5.2xlarge"
87+
plugins:
88+
- *vault_ec_key_prod
7689
matrix:
7790
setup:
7891
sudo:
@@ -101,6 +114,8 @@ steps:
101114
provider: "aws"
102115
image: "${IMAGE_UBUNTU_ARM64_FIPS}"
103116
instanceType: "m6g.2xlarge"
117+
plugins:
118+
- *vault_ec_key_prod
104119
matrix:
105120
setup:
106121
sudo:
@@ -139,6 +154,8 @@ steps:
139154
agents:
140155
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
141156
useCustomGlobalHooks: true
157+
plugins:
158+
- *vault_ec_key_prod
142159

143160
- label: Aggregate test reports
144161
depends_on:

‎.buildkite/bk.integration.pipeline.yml‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,11 @@ common:
3939
KIBANA_HOST: ea-serverless-it-kibana-hostname
4040
KIBANA_USERNAME: ea-serverless-it-kibana-username
4141
KIBANA_PASSWORD: ea-serverless-it-kibana-password
42+
- vault_ec_key_prod: &vault_ec_key_prod
43+
elastic/vault-secrets#v0.1.0:
44+
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
45+
field: "apiKey"
46+
env_var: "EC_API_KEY"
4247

4348
steps:
4449
- label: Start ESS stack for integration tests
@@ -58,6 +63,8 @@ steps:
5863
agents:
5964
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
6065
useCustomGlobalHooks: true
66+
plugins:
67+
- *vault_ec_key_prod
6168

6269
- group: "Extended runtime leak tests"
6370
key: extended-integration-tests
@@ -85,6 +92,9 @@ steps:
8592
retry:
8693
automatic:
8794
limit: 1
95+
plugins:
96+
- *vault_ec_key_prod
97+
8898
- label: "Windows:2025:amd64:sudo"
8999
depends_on:
90100
- packaging-windows
@@ -103,6 +113,9 @@ steps:
103113
provider: "gcp"
104114
machineType: "n2-standard-8"
105115
image: "${IMAGE_WIN_2025}"
116+
plugins:
117+
- *vault_ec_key_prod
118+
106119
- label: "Ubuntu:2404:amd64:sudo"
107120
depends_on: packaging-ubuntu-x86-64
108121
env:
@@ -120,6 +133,8 @@ steps:
120133
provider: "gcp"
121134
machineType: "n2-standard-8"
122135
image: "${IMAGE_UBUNTU_2404_X86_64}"
136+
plugins:
137+
- *vault_ec_key_prod
123138

124139
- group: "Stateful: Windows"
125140
key: integration-tests-win
@@ -147,6 +162,8 @@ steps:
147162
retry:
148163
automatic:
149164
limit: 1
165+
plugins:
166+
- *vault_ec_key_prod
150167
matrix:
151168
- default
152169
- fleet
@@ -175,6 +192,8 @@ steps:
175192
retry:
176193
automatic:
177194
limit: 1
195+
plugins:
196+
- *vault_ec_key_prod
178197
matrix:
179198
- default
180199

@@ -196,6 +215,8 @@ steps:
196215
retry:
197216
automatic:
198217
limit: 1
218+
plugins:
219+
- *vault_ec_key_prod
199220
matrix:
200221
- default
201222
- fleet
@@ -224,6 +245,8 @@ steps:
224245
provider: "gcp"
225246
machineType: "n2-standard-8"
226247
image: "${IMAGE_WIN_2025}"
248+
plugins:
249+
- *vault_ec_key_prod
227250
matrix:
228251
- default
229252

@@ -252,6 +275,8 @@ steps:
252275
provider: "gcp"
253276
machineType: "n2-standard-8"
254277
image: "${IMAGE_UBUNTU_2404_X86_64}"
278+
plugins:
279+
- *vault_ec_key_prod
255280
matrix:
256281
- default
257282

@@ -274,6 +299,8 @@ steps:
274299
provider: "gcp"
275300
machineType: "n2-standard-8"
276301
image: "${IMAGE_UBUNTU_2404_X86_64}"
302+
plugins:
303+
- *vault_ec_key_prod
277304
matrix:
278305
- default
279306
- upgrade
@@ -308,6 +335,8 @@ steps:
308335
retry:
309336
automatic:
310337
limit: 1
338+
plugins:
339+
- *vault_ec_key_prod
311340
matrix:
312341
- default
313342
- upgrade
@@ -344,6 +373,8 @@ steps:
344373
provider: "aws"
345374
image: "${IMAGE_UBUNTU_2404_ARM_64}"
346375
instanceType: "m6g.xlarge"
376+
plugins:
377+
- *vault_ec_key_prod
347378
matrix:
348379
- default
349380

@@ -372,6 +403,8 @@ steps:
372403
provider: "gcp"
373404
machineType: "n2-standard-8"
374405
image: "${IMAGE_DEBIAN_12}"
406+
plugins:
407+
- *vault_ec_key_prod
375408
matrix:
376409
- default
377410

@@ -394,6 +427,8 @@ steps:
394427
provider: "gcp"
395428
machineType: "n2-standard-8"
396429
image: "${IMAGE_DEBIAN_12}"
430+
plugins:
431+
- *vault_ec_key_prod
397432
matrix:
398433
- default
399434
- upgrade
@@ -433,6 +468,8 @@ steps:
433468
retry:
434469
automatic:
435470
limit: 1
471+
plugins:
472+
- *vault_ec_key_prod
436473
agents:
437474
provider: "gcp"
438475
machineType: "n2-standard-8"
@@ -468,6 +505,8 @@ steps:
468505
machineType: "n2-standard-4"
469506
image: "${IMAGE_UBUNTU_2404_X86_64}"
470507
diskSizeGb: 80
508+
plugins:
509+
- *vault_ec_key_prod
471510
matrix:
472511
setup:
473512
variants:
@@ -571,7 +610,8 @@ steps:
571610
agents:
572611
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
573612
useCustomGlobalHooks: true
574-
613+
plugins:
614+
- *vault_ec_key_prod
575615
- label: Aggregate test reports
576616
# Warning: The key has a hook in pre-command
577617
key: aggregate-reports

‎.buildkite/hooks/pre-command‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ fi
1515

1616
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
1717
CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp"
18+
# This key exists for backward compatibility with OGC framework
19+
# see https://github.com/elastic/elastic-agent/issues/8536
1820
CI_ESS_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
1921
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
2022

‎.buildkite/scripts/steps/ess.ps1‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,7 @@ function ess_up {
1313
Write-Error "Error: Specify stack version: ess_up [stack_version]"
1414
return 1
1515
}
16-
17-
$Env:EC_API_KEY = Retry-Command -ScriptBlock {
18-
vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
19-
}
20-
21-
if (-not $Env:EC_API_KEY) {
22-
Write-Error "Error: Failed to get EC API key from vault"
23-
exit 1
24-
}
25-
16+
2617
$BuildkiteBuildCreator = if ($Env:BUILDKITE_BUILD_CREATOR) { $Env:BUILDKITE_BUILD_CREATOR } else { get_git_user_email }
2718
$BuildkiteBuildNumber = if ($Env:BUILDKITE_BUILD_NUMBER) { $Env:BUILDKITE_BUILD_NUMBER } else { "0" }
2819
$BuildkitePipelineSlug = if ($Env:BUILDKITE_PIPELINE_SLUG) { $Env:BUILDKITE_PIPELINE_SLUG } else { "elastic-agent-integration-tests" }
@@ -56,10 +47,7 @@ function ess_down {
5647
return 0
5748
}
5849
Write-Output "~~~ Tearing down the ESS Stack(created for this step)"
59-
try {
60-
$Env:EC_API_KEY = Retry-Command -ScriptBlock {
61-
vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
62-
}
50+
try {
6351
Push-Location -Path $TfDir
6452
& terraform init
6553
& terraform destroy -auto-approve

‎.buildkite/scripts/steps/ess.sh‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,6 @@ function ess_up() {
1313
return 1
1414
fi
1515

16-
export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
17-
18-
if [[ -z "${EC_API_KEY}" ]]; then
19-
echo "Error: Failed to get EC API key from vault" >&2
20-
exit 1
21-
fi
22-
2316
BUILDKITE_BUILD_CREATOR="${BUILDKITE_BUILD_CREATOR:-"$(get_git_user_email)"}"
2417
BUILDKITE_BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-"0"}"
2518
BUILDKITE_PIPELINE_SLUG="${BUILDKITE_PIPELINE_SLUG:-"elastic-agent-integration-tests"}"
@@ -48,9 +41,6 @@ function ess_down() {
4841
echo "~~~ Tearing down the ESS Stack"
4942
local WORKSPACE=$(git rev-parse --show-toplevel)
5043
local TF_DIR="${WORKSPACE}/test_infra/ess/"
51-
if [ -z "${EC_API_KEY:-}" ]; then
52-
export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
53-
fi
5444

5545
pushd "${TF_DIR}"
5646
terraform init

0 commit comments

Comments
 (0)