[9.0] (backport #8377) [CI] BK Vault plugin for EC access (#8795)

* [CI] BK Vault plugin for EC access (#8377)

* [CI] BK Vault plugin for ES access

* Typo

* Typo

* Quick Windows test

* Quick test Windows

* Revert last two commits

* Applied proposed changes

* Fixed indentation

* revert buildkite_analytics_token deletion

* Remaned the anchor

* Added the issue to comments

* Updated FIPS pipeline

(cherry picked from commit e2505e4)

# Conflicts:
#	.buildkite/bk.integration-fips.pipeline.yml

* Remove backported fips pipeline

---------

Co-authored-by: Pavel Zorin <[email protected]>

Author: mergify[bot]

.buildkite/bk.integration.pipeline.yml

@@ -37,6 +37,11 @@ common:
           KIBANA_HOST: ea-serverless-it-kibana-hostname
           KIBANA_USERNAME: ea-serverless-it-kibana-username
           KIBANA_PASSWORD: ea-serverless-it-kibana-password
+  - vault_ec_key_prod: &vault_ec_key_prod
+      elastic/vault-secrets#v0.1.0:
+        path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
+        field: "apiKey"
+        env_var: "EC_API_KEY"  
 
 steps:
   - label: Start ESS stack for integration tests
@@ -56,6 +61,8 @@ steps:
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
       useCustomGlobalHooks: true
+    plugins:
+      - *vault_ec_key_prod
 
   - group: "Extended runtime leak tests"
     key: extended-integration-tests
@@ -83,6 +90,9 @@ steps:
         retry:
           automatic:
             limit: 1
+        plugins:
+          - *vault_ec_key_prod
+
       - label: "Windows:2025:amd64:sudo"
         depends_on:
           - packaging-windows
@@ -101,6 +111,9 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_WIN_2025}"
+        plugins:
+          - *vault_ec_key_prod
+  
       - label: "Ubuntu:2404:amd64:sudo"
         depends_on: packaging-ubuntu-x86-64
         env:
@@ -118,6 +131,8 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_UBUNTU_2404_X86_64}"
+        plugins:
+          - *vault_ec_key_prod
 
   - group: "Stateful: Windows"
     key: integration-tests-win
@@ -145,6 +160,8 @@ steps:
         retry:
           automatic:
             limit: 1
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
           - fleet
@@ -173,6 +190,8 @@ steps:
         retry:
           automatic:
             limit: 1
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
 
@@ -194,6 +213,8 @@ steps:
         retry:
           automatic:
             limit: 1
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
           - fleet
@@ -222,6 +243,8 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_WIN_2025}"
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
 
@@ -250,6 +273,8 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_UBUNTU_2404_X86_64}"
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
 
@@ -272,6 +297,8 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_UBUNTU_2404_X86_64}"
+        plugins:
+          - *vault_ec_key_prod  
         matrix:
           - default
           - upgrade
@@ -306,6 +333,8 @@ steps:
         retry:
           automatic:
             limit: 1
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
           - upgrade
@@ -342,6 +371,8 @@ steps:
           provider: "aws"
           image: "${IMAGE_UBUNTU_2404_ARM_64}"
           instanceType: "m6g.xlarge"
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
 
@@ -370,6 +401,8 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_DEBIAN_12}"
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
 
@@ -392,6 +425,8 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_DEBIAN_12}"
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
           - upgrade
@@ -431,6 +466,8 @@ steps:
         retry:
           automatic:
             limit: 1
+        plugins:
+          - *vault_ec_key_prod
         agents:
           provider: "gcp"
           machineType: "n2-standard-8"
@@ -466,6 +503,8 @@ steps:
           machineType: "n2-standard-4"
           image: "${IMAGE_UBUNTU_2404_X86_64}"
           diskSizeGb: 80
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           setup:
             variants:
@@ -569,7 +608,8 @@ steps:
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
       useCustomGlobalHooks: true
-
+    plugins:
+      - *vault_ec_key_prod
   - label: Aggregate test reports
     # Warning: The key has a hook in pre-command
     key: aggregate-reports

.buildkite/hooks/pre-command

@@ -15,6 +15,8 @@ fi
 
 CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
 CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp"
+# This key exists for backward compatibility with OGC framework
+# see https://github.com/elastic/elastic-agent/issues/8536
 CI_ESS_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
 CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
 

.buildkite/scripts/steps/ess.ps1

@@ -13,16 +13,7 @@ function ess_up {
       Write-Error "Error: Specify stack version: ess_up [stack_version]"
       return 1
   }
-
-  $Env:EC_API_KEY = Retry-Command -ScriptBlock {  
-    vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
-  }
-
-  if (-not $Env:EC_API_KEY) {
-      Write-Error "Error: Failed to get EC API key from vault"
-      exit 1
-  }
-
+  
   $BuildkiteBuildCreator = if ($Env:BUILDKITE_BUILD_CREATOR) { $Env:BUILDKITE_BUILD_CREATOR } else { get_git_user_email }
   $BuildkiteBuildNumber = if ($Env:BUILDKITE_BUILD_NUMBER) { $Env:BUILDKITE_BUILD_NUMBER } else { "0" }
   $BuildkitePipelineSlug = if ($Env:BUILDKITE_PIPELINE_SLUG) { $Env:BUILDKITE_PIPELINE_SLUG } else { "elastic-agent-integration-tests" }
@@ -55,10 +46,7 @@ function ess_down {
     return 0
   }
   Write-Output "~~~ Tearing down the ESS Stack(created for this step)"
-  try {  
-    $Env:EC_API_KEY = Retry-Command -ScriptBlock {  
-      vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
-    }
+  try {
     Push-Location -Path $TfDir
     & terraform init
     & terraform destroy -auto-approve

.buildkite/scripts/steps/ess.sh

@@ -13,13 +13,6 @@ function ess_up() {
     return 1
   fi
 
-  export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
-  
-  if [[ -z "${EC_API_KEY}" ]]; then
-    echo "Error: Failed to get EC API key from vault" >&2
-    exit 1
-  fi
-
   BUILDKITE_BUILD_CREATOR="${BUILDKITE_BUILD_CREATOR:-"$(get_git_user_email)"}"
   BUILDKITE_BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-"0"}"
   BUILDKITE_PIPELINE_SLUG="${BUILDKITE_PIPELINE_SLUG:-"elastic-agent-integration-tests"}"
@@ -47,9 +40,6 @@ function ess_down() {
   echo "~~~ Tearing down the ESS Stack"  
   local WORKSPACE=$(git rev-parse --show-toplevel)
   local TF_DIR="${WORKSPACE}/test_infra/ess/"
-  if [ -z "${EC_API_KEY:-}" ]; then
-    export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)    
-  fi
 
   pushd "${TF_DIR}"
   terraform init