Skip to content

[CI] BK Vault plugin for EC access #8377

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Jul 2, 2025
Merged

[CI] BK Vault plugin for EC access #8377

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
71d873d
[CI] BK Vault plugin for ES access
pazone Jun 6, 2025
5c40b3d
Typo
pazone Jun 6, 2025
a38103f
Typo
pazone Jun 6, 2025
15120a0
Quick Windows test
pazone Jun 6, 2025
cfc8a1e
Quick test Windows
pazone Jun 6, 2025
ccf2f38
Revert last two commits
pazone Jun 6, 2025
df2e29a
Applied proposed changes
pazone Jun 6, 2025
c926ff7
Fixed indentation
pazone Jun 6, 2025
576f34c
revert buildkite_analytics_token deletion
pazone Jun 16, 2025
e3db7ab
Remaned the anchor
pazone Jun 16, 2025
924f741
Added the issue to comments
pazone Jun 16, 2025
c874de2
Merge branch 'main' of github.com:elastic/elastic-agent into ci_use_v…
pazone Jun 16, 2025
e94a92d
Updated FIPS pipeline
pazone Jun 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 87 additions & 0 deletions .buildkite/bk.integration.pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ steps:
agents:
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
useCustomGlobalHooks: true
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"

- group: "Extended runtime leak tests"
key: extended-integration-tests
Expand All @@ -53,6 +58,12 @@ steps:
retry:
automatic:
limit: 1
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"

- label: "Windows:2025:amd64:sudo"
depends_on:
- packaging-windows
Expand All @@ -71,6 +82,12 @@ steps:
provider: "gcp"
machineType: "n2-standard-8"
image: "${IMAGE_WIN_2025}"
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"

- label: "Ubuntu:2404:amd64:sudo"
depends_on: packaging-ubuntu-x86-64
env:
Expand All @@ -88,6 +105,11 @@ steps:
provider: "gcp"
machineType: "n2-standard-8"
image: "${IMAGE_UBUNTU_2404_X86_64}"
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"

- group: "Stateful: Windows"
key: integration-tests-win
Expand All @@ -110,6 +132,11 @@ steps:
retry:
automatic:
limit: 1
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"
matrix:
- default
- fleet
Expand All @@ -136,6 +163,11 @@ steps:
retry:
automatic:
limit: 1
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"
matrix:
- default

Expand All @@ -155,6 +187,11 @@ steps:
retry:
automatic:
limit: 1
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"
matrix:
- default
- fleet
Expand All @@ -181,6 +218,11 @@ steps:
provider: "gcp"
machineType: "n2-standard-8"
image: "${IMAGE_WIN_2025}"
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"
matrix:
- default

Expand All @@ -204,6 +246,11 @@ steps:
provider: "gcp"
machineType: "n2-standard-8"
image: "${IMAGE_UBUNTU_2404_X86_64}"
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"
matrix:
- default

Expand All @@ -224,6 +271,11 @@ steps:
provider: "gcp"
machineType: "n2-standard-8"
image: "${IMAGE_UBUNTU_2404_X86_64}"
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"
matrix:
- default
- upgrade
Expand Down Expand Up @@ -256,6 +308,11 @@ steps:
retry:
automatic:
limit: 1
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"
matrix:
- default
- upgrade
Expand Down Expand Up @@ -290,6 +347,11 @@ steps:
provider: "aws"
image: "${IMAGE_UBUNTU_2404_ARM_64}"
instanceType: "m6g.xlarge"
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"
matrix:
- default

Expand All @@ -313,6 +375,11 @@ steps:
provider: "gcp"
machineType: "n2-standard-8"
image: "${IMAGE_DEBIAN_12}"
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"
matrix:
- default

Expand All @@ -333,6 +400,11 @@ steps:
provider: "gcp"
machineType: "n2-standard-8"
image: "${IMAGE_DEBIAN_12}"
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"
matrix:
- default
- upgrade
Expand Down Expand Up @@ -367,6 +439,11 @@ steps:
retry:
automatic:
limit: 1
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"
agents:
provider: "gcp"
machineType: "n2-standard-8"
Expand Down Expand Up @@ -399,6 +476,11 @@ steps:
machineType: "n2-standard-4"
image: "${IMAGE_UBUNTU_2404_X86_64}"
diskSizeGb: 80
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"
matrix:
setup:
variants:
Expand Down Expand Up @@ -432,6 +514,11 @@ steps:
agents:
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
useCustomGlobalHooks: true
plugins:
- elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"

- label: Aggregate test reports
# Warning: The key has a hook in pre-command
Expand Down
1 change: 1 addition & 0 deletions .buildkite/hooks/pre-command
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ DOCKER_REGISTRY="docker.elastic.co"
DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp"
# This key exists for backward compatibility with OGC framework
CI_ESS_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"

Expand Down
23 changes: 11 additions & 12 deletions .buildkite/scripts/steps/ess.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
function check_ec_access {
# EC_API_KEY provided by buildkite-vault plugin. See the pipeline step definition.
if (-not $Env:EC_API_KEY) {
Write-Error "Error: EC_API_KEY is empty"
exit 1
}
}

function ess_up {
param (
[string]$StackVersion,
Expand All @@ -14,14 +22,7 @@ function ess_up {
return 1
}

$Env:EC_API_KEY = Retry-Command -ScriptBlock {
vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
}

if (-not $Env:EC_API_KEY) {
Write-Error "Error: Failed to get EC API key from vault"
exit 1
}
check_ec_access

$BuildkiteBuildCreator = if ($Env:BUILDKITE_BUILD_CREATOR) { $Env:BUILDKITE_BUILD_CREATOR } else { get_git_user_email }
$BuildkiteBuildNumber = if ($Env:BUILDKITE_BUILD_NUMBER) { $Env:BUILDKITE_BUILD_NUMBER } else { "0" }
Expand Down Expand Up @@ -56,10 +57,8 @@ function ess_down {
return 0
}
Write-Output "~~~ Tearing down the ESS Stack(created for this step)"
try {
$Env:EC_API_KEY = Retry-Command -ScriptBlock {
vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
}
try {
check_ec_access
Push-Location -Path $TfDir
& terraform init
& terraform destroy -auto-approve
Expand Down
19 changes: 10 additions & 9 deletions .buildkite/scripts/steps/ess.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail

function check_ec_access() {
# EC_API_KEY provided by buildkite-vault plugin. See the pipeline step definition.
if [[ -z "${EC_API_KEY}" ]]; then
echo "Error: EC_API_KEY is empty" >&2
exit 1
fi
}

function ess_up() {
echo "~~~ Starting ESS Stack"
local WORKSPACE=$(git rev-parse --show-toplevel)
Expand All @@ -13,12 +21,7 @@ function ess_up() {
return 1
fi

export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)

if [[ -z "${EC_API_KEY}" ]]; then
echo "Error: Failed to get EC API key from vault" >&2
exit 1
fi
check_ec_access

BUILDKITE_BUILD_CREATOR="${BUILDKITE_BUILD_CREATOR:-"$(get_git_user_email)"}"
BUILDKITE_BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-"0"}"
Expand Down Expand Up @@ -48,9 +51,7 @@ function ess_down() {
echo "~~~ Tearing down the ESS Stack"
local WORKSPACE=$(git rev-parse --show-toplevel)
local TF_DIR="${WORKSPACE}/test_infra/ess/"
if [ -z "${EC_API_KEY:-}" ]; then
export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
fi
check_ec_access

pushd "${TF_DIR}"
terraform init
Expand Down