Skip to content

[9.0] (backport #8377) [CI] BK Vault plugin for EC access #8795

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jul 4, 2025
Merged

[9.0] (backport #8377) [CI] BK Vault plugin for EC access #8795

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f852347
[CI] BK Vault plugin for EC access (#8377)
pazone Jul 2, 2025
57caa5a
Remove backported fips pipeline
pazone Jul 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 41 additions & 1 deletion .buildkite/bk.integration.pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@ common:
KIBANA_HOST: ea-serverless-it-kibana-hostname
KIBANA_USERNAME: ea-serverless-it-kibana-username
KIBANA_PASSWORD: ea-serverless-it-kibana-password
- vault_ec_key_prod: &vault_ec_key_prod
elastic/vault-secrets#v0.1.0:
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
field: "apiKey"
env_var: "EC_API_KEY"

steps:
- label: Start ESS stack for integration tests
Expand All @@ -58,6 +63,8 @@ steps:
agents:
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
useCustomGlobalHooks: true
plugins:
- *vault_ec_key_prod

- group: "Extended runtime leak tests"
key: extended-integration-tests
Expand Down Expand Up @@ -85,6 +92,9 @@ steps:
retry:
automatic:
limit: 1
plugins:
- *vault_ec_key_prod

- label: "Windows:2025:amd64:sudo"
depends_on:
- packaging-windows
Expand All @@ -103,6 +113,9 @@ steps:
provider: "gcp"
machineType: "n2-standard-8"
image: "${IMAGE_WIN_2025}"
plugins:
- *vault_ec_key_prod

- label: "Ubuntu:2404:amd64:sudo"
depends_on: packaging-ubuntu-x86-64
env:
Expand All @@ -120,6 +133,8 @@ steps:
provider: "gcp"
machineType: "n2-standard-8"
image: "${IMAGE_UBUNTU_2404_X86_64}"
plugins:
- *vault_ec_key_prod

- group: "Stateful: Windows"
key: integration-tests-win
Expand Down Expand Up @@ -147,6 +162,8 @@ steps:
retry:
automatic:
limit: 1
plugins:
- *vault_ec_key_prod
matrix:
- default
- fleet
Expand Down Expand Up @@ -175,6 +192,8 @@ steps:
retry:
automatic:
limit: 1
plugins:
- *vault_ec_key_prod
matrix:
- default

Expand All @@ -196,6 +215,8 @@ steps:
retry:
automatic:
limit: 1
plugins:
- *vault_ec_key_prod
matrix:
- default
- fleet
Expand Down Expand Up @@ -224,6 +245,8 @@ steps:
provider: "gcp"
machineType: "n2-standard-8"
image: "${IMAGE_WIN_2025}"
plugins:
- *vault_ec_key_prod
matrix:
- default

Expand Down Expand Up @@ -252,6 +275,8 @@ steps:
provider: "gcp"
machineType: "n2-standard-8"
image: "${IMAGE_UBUNTU_2404_X86_64}"
plugins:
- *vault_ec_key_prod
matrix:
- default

Expand All @@ -274,6 +299,8 @@ steps:
provider: "gcp"
machineType: "n2-standard-8"
image: "${IMAGE_UBUNTU_2404_X86_64}"
plugins:
- *vault_ec_key_prod
matrix:
- default
- upgrade
Expand Down Expand Up @@ -308,6 +335,8 @@ steps:
retry:
automatic:
limit: 1
plugins:
- *vault_ec_key_prod
matrix:
- default
- upgrade
Expand Down Expand Up @@ -344,6 +373,8 @@ steps:
provider: "aws"
image: "${IMAGE_UBUNTU_2404_ARM_64}"
instanceType: "m6g.xlarge"
plugins:
- *vault_ec_key_prod
matrix:
- default

Expand Down Expand Up @@ -372,6 +403,8 @@ steps:
provider: "gcp"
machineType: "n2-standard-8"
image: "${IMAGE_DEBIAN_12}"
plugins:
- *vault_ec_key_prod
matrix:
- default

Expand All @@ -394,6 +427,8 @@ steps:
provider: "gcp"
machineType: "n2-standard-8"
image: "${IMAGE_DEBIAN_12}"
plugins:
- *vault_ec_key_prod
matrix:
- default
- upgrade
Expand Down Expand Up @@ -433,6 +468,8 @@ steps:
retry:
automatic:
limit: 1
plugins:
- *vault_ec_key_prod
agents:
provider: "gcp"
machineType: "n2-standard-8"
Expand Down Expand Up @@ -468,6 +505,8 @@ steps:
machineType: "n2-standard-4"
image: "${IMAGE_UBUNTU_2404_X86_64}"
diskSizeGb: 80
plugins:
- *vault_ec_key_prod
matrix:
setup:
variants:
Expand Down Expand Up @@ -571,7 +610,8 @@ steps:
agents:
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
useCustomGlobalHooks: true

plugins:
- *vault_ec_key_prod
- label: Aggregate test reports
# Warning: The key has a hook in pre-command
key: aggregate-reports
Expand Down
2 changes: 2 additions & 0 deletions .buildkite/hooks/pre-command
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ fi

CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp"
# This key exists for backward compatibility with OGC framework
# see https://github.com/elastic/elastic-agent/issues/8536
CI_ESS_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"

Expand Down
16 changes: 2 additions & 14 deletions .buildkite/scripts/steps/ess.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,7 @@ function ess_up {
Write-Error "Error: Specify stack version: ess_up [stack_version]"
return 1
}

$Env:EC_API_KEY = Retry-Command -ScriptBlock {
vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
}

if (-not $Env:EC_API_KEY) {
Write-Error "Error: Failed to get EC API key from vault"
exit 1
}


$BuildkiteBuildCreator = if ($Env:BUILDKITE_BUILD_CREATOR) { $Env:BUILDKITE_BUILD_CREATOR } else { get_git_user_email }
$BuildkiteBuildNumber = if ($Env:BUILDKITE_BUILD_NUMBER) { $Env:BUILDKITE_BUILD_NUMBER } else { "0" }
$BuildkitePipelineSlug = if ($Env:BUILDKITE_PIPELINE_SLUG) { $Env:BUILDKITE_PIPELINE_SLUG } else { "elastic-agent-integration-tests" }
Expand Down Expand Up @@ -55,10 +46,7 @@ function ess_down {
return 0
}
Write-Output "~~~ Tearing down the ESS Stack(created for this step)"
try {
$Env:EC_API_KEY = Retry-Command -ScriptBlock {
vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
}
try {
Push-Location -Path $TfDir
& terraform init
& terraform destroy -auto-approve
Expand Down
10 changes: 0 additions & 10 deletions .buildkite/scripts/steps/ess.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,6 @@ function ess_up() {
return 1
fi

export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)

if [[ -z "${EC_API_KEY}" ]]; then
echo "Error: Failed to get EC API key from vault" >&2
exit 1
fi

BUILDKITE_BUILD_CREATOR="${BUILDKITE_BUILD_CREATOR:-"$(get_git_user_email)"}"
BUILDKITE_BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-"0"}"
BUILDKITE_PIPELINE_SLUG="${BUILDKITE_PIPELINE_SLUG:-"elastic-agent-integration-tests"}"
Expand Down Expand Up @@ -47,9 +40,6 @@ function ess_down() {
echo "~~~ Tearing down the ESS Stack"
local WORKSPACE=$(git rev-parse --show-toplevel)
local TF_DIR="${WORKSPACE}/test_infra/ess/"
if [ -z "${EC_API_KEY:-}" ]; then
export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
fi

pushd "${TF_DIR}"
terraform init
Expand Down