[CI] BK Vault plugin for EC access (#8377)

* [CI] BK Vault plugin for ES access

* Typo

* Typo

* Quick Windows test

* Quick test Windows

* Revert last two commits

* Applied proposed changes

* Fixed indentation

* revert buildkite_analytics_token deletion

* Remaned the anchor

* Added the issue to comments

* Updated FIPS pipeline

(cherry picked from commit e2505e4)

# Conflicts:
#	.buildkite/bk.integration-fips.pipeline.yml

Author: pazone

.buildkite/bk.integration-fips.pipeline.yml

@@ -0,0 +1,180 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  ASDF_MAGE_VERSION: 1.14.0
+  MS_GOTOOLCHAIN_TELEMETRY_ENABLED: "0"
+
+  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1751072471"
+  IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1751072471"
+  IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1751072471"
+  # Remove AGENT_VERSION pinning once 9.2.0 DRA and stack are released
+  AGENT_VERSION: "9.1.0-SNAPSHOT"
+
+# This section is used to define the plugins that will be used in the pipeline.
+# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
+common:
+  - vault_ec_key_prod: &vault_ec_key_prod
+      elastic/vault-secrets#v0.1.0:
+        path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
+        field: "apiKey"
+        env_var: "EC_API_KEY"
+
+steps:
+  - label: Build and push custom elastic-agent image
+    depends_on:
+      - 'packaging-containers-x86-64-fips' # Reuse artifacts produced in .buildkite/integration.pipeline.yml
+    key: integration-fips-cloud-image
+    env:
+      FIPS: "true"
+      CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
+      CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips"
+      TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
+    command: |
+      buildkite-agent artifact download build/distributions/elastic-agent-cloud-fips-*-linux-amd64.docker.tar.gz . --step 'packaging-containers-x86-64-fips'
+      mage cloud:load
+      mage cloud:push
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+      image: "${IMAGE_UBUNTU_2404_X86_64}"
+    plugins:
+      - elastic/vault-docker-login#v0.5.2:
+          secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry'
+
+  - label: Start ESS stack for FIPS integration tests
+    key: integration-fips-ess
+    depends_on:
+      - integration-fips-cloud-image
+    env:
+      ASDF_TERRAFORM_VERSION: 1.9.2
+      TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
+    command: |
+      source .buildkite/scripts/steps/ess_start.sh
+    artifact_paths:
+      - test_infra/ess/*.tfstate
+      - test_infra/ess/*.lock.hcl
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true
+    plugins:
+      - *vault_ec_key_prod  
+
+  - group: "fips:Stateful:Ubuntu"
+    key: integration-tests-ubuntu-fips
+    depends_on:
+      - integration-fips-ess
+    steps:
+      - label: "fips:x86_64:sudo-{{matrix.sudo}}:{{matrix.groups}}"
+        depends_on:
+          - packaging-ubuntu-x86-64-fips # Reuse artifacts produced in .buildkite/integration.pipeline.yml
+        env:
+          FIPS: "true"
+          TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64-fips'
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}}
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        retry:
+          automatic:
+            limit: 1
+        agents:
+          provider: "aws"
+          image: "${IMAGE_UBUNTU_X86_64_FIPS}"
+          instanceType: "m5.2xlarge"
+        plugins:
+          - *vault_ec_key_prod  
+        matrix:
+          setup:
+            sudo:
+              - "false"
+              - "true"
+            groups:
+              - fleet # currently there is only a single test in the fleet group, add more tests once they have been defined
+
+      - label: "fips:arm64:sudo-{{matrix.sudo}}:{{matrix.groups}}"
+        depends_on:
+          - packaging-ubuntu-arm64-fips
+        env:
+          FIPS: "true"
+          TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64-fips'
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}}
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        retry:
+          automatic:
+            limit: 1
+        agents:
+          provider: "aws"
+          image: "${IMAGE_UBUNTU_ARM64_FIPS}"
+          instanceType: "m6g.2xlarge"
+        plugins:
+          - *vault_ec_key_prod  
+        matrix:
+          setup:
+            sudo:
+              - "false"
+              - "true"
+            groups:
+              - fleet
+
+      - label: "fips:upgrade-ech-deployment"
+        if: build.env("BUILDKITE_PULL_REQUEST") != "false" &&  build.env("GITHUB_PR_LABELS") =~ /.*(Testing:run:TestUpgradeIntegrationsServer).*/
+        env:
+          FIPS: "true"
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
+        command: |
+          export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
+          .buildkite/scripts/buildkite-integration-tests.sh ech-deployment false
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        retry:
+          automatic:
+            limit: 1
+        agents:
+          provider: "aws"
+          image: "${IMAGE_UBUNTU_X86_64_FIPS}"
+          instanceType: "m5.2xlarge"
+
+  - label: ESS FIPS stack cleanup
+    depends_on:
+      - integration-tests-ubuntu-fips
+    allow_dependency_failure: true
+    command: |
+      buildkite-agent artifact download "test_infra/ess/**" . --step "integration-fips-ess"
+      ls -lah test_infra/ess
+      .buildkite/scripts/steps/ess_down.sh
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true
+    plugins:
+      - *vault_ec_key_prod  
+
+  - label: Aggregate test reports
+    depends_on:
+      - integration-tests-ubuntu-fips
+    allow_dependency_failure: true
+    command: |
+      buildkite-agent artifact download "build/*.xml" .
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true
+    soft_fail:
+      - exit_status: "*"
+    plugins:
+      - elastic/vault-secrets#v0.1.0:
+          path: "kv/ci-shared/platform-ingest/buildkite_analytics_token"
+          field: "token"
+          env_var: "BUILDKITE_ANALYTICS_TOKEN"
+      - test-collector#v1.11.0:
+          files: "build/*.xml"
+          format: "junit"
+          branches: "main"
+          debug: true

.buildkite/bk.integration.pipeline.yml

@@ -39,6 +39,11 @@ common:
           KIBANA_HOST: ea-serverless-it-kibana-hostname
           KIBANA_USERNAME: ea-serverless-it-kibana-username
           KIBANA_PASSWORD: ea-serverless-it-kibana-password
+  - vault_ec_key_prod: &vault_ec_key_prod
+      elastic/vault-secrets#v0.1.0:
+        path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
+        field: "apiKey"
+        env_var: "EC_API_KEY"  
 
 steps:
   - label: Start ESS stack for integration tests
@@ -58,6 +63,8 @@ steps:
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
       useCustomGlobalHooks: true
+    plugins:
+      - *vault_ec_key_prod
 
   - group: "Extended runtime leak tests"
     key: extended-integration-tests
@@ -85,6 +92,9 @@ steps:
         retry:
           automatic:
             limit: 1
+        plugins:
+          - *vault_ec_key_prod
+
       - label: "Windows:2025:amd64:sudo"
         depends_on:
           - packaging-windows
@@ -103,6 +113,9 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_WIN_2025}"
+        plugins:
+          - *vault_ec_key_prod
+  
       - label: "Ubuntu:2404:amd64:sudo"
         depends_on: packaging-ubuntu-x86-64
         env:
@@ -120,6 +133,8 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_UBUNTU_2404_X86_64}"
+        plugins:
+          - *vault_ec_key_prod
 
   - group: "Stateful: Windows"
     key: integration-tests-win
@@ -147,6 +162,8 @@ steps:
         retry:
           automatic:
             limit: 1
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
           - fleet
@@ -175,6 +192,8 @@ steps:
         retry:
           automatic:
             limit: 1
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
 
@@ -196,6 +215,8 @@ steps:
         retry:
           automatic:
             limit: 1
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
           - fleet
@@ -224,6 +245,8 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_WIN_2025}"
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
 
@@ -252,6 +275,8 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_UBUNTU_2404_X86_64}"
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
 
@@ -274,6 +299,8 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_UBUNTU_2404_X86_64}"
+        plugins:
+          - *vault_ec_key_prod  
         matrix:
           - default
           - upgrade
@@ -308,6 +335,8 @@ steps:
         retry:
           automatic:
             limit: 1
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
           - upgrade
@@ -344,6 +373,8 @@ steps:
           provider: "aws"
           image: "${IMAGE_UBUNTU_2404_ARM_64}"
           instanceType: "m6g.xlarge"
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
 
@@ -372,6 +403,8 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_DEBIAN_12}"
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
 
@@ -394,6 +427,8 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_DEBIAN_12}"
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           - default
           - upgrade
@@ -433,6 +468,8 @@ steps:
         retry:
           automatic:
             limit: 1
+        plugins:
+          - *vault_ec_key_prod
         agents:
           provider: "gcp"
           machineType: "n2-standard-8"
@@ -468,6 +505,8 @@ steps:
           machineType: "n2-standard-4"
           image: "${IMAGE_UBUNTU_2404_X86_64}"
           diskSizeGb: 80
+        plugins:
+          - *vault_ec_key_prod
         matrix:
           setup:
             variants:
@@ -571,7 +610,8 @@ steps:
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
       useCustomGlobalHooks: true
-
+    plugins:
+      - *vault_ec_key_prod
   - label: Aggregate test reports
     # Warning: The key has a hook in pre-command
     key: aggregate-reports

.buildkite/hooks/pre-command

@@ -15,6 +15,8 @@ fi
 
 CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
 CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp"
+# This key exists for backward compatibility with OGC framework
+# see https://github.com/elastic/elastic-agent/issues/8536
 CI_ESS_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
 CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
 

.buildkite/scripts/steps/ess.ps1

@@ -13,16 +13,7 @@ function ess_up {
       Write-Error "Error: Specify stack version: ess_up [stack_version]"
       return 1
   }
-
-  $Env:EC_API_KEY = Retry-Command -ScriptBlock {  
-    vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
-  }
-
-  if (-not $Env:EC_API_KEY) {
-      Write-Error "Error: Failed to get EC API key from vault"
-      exit 1
-  }
-
+  
   $BuildkiteBuildCreator = if ($Env:BUILDKITE_BUILD_CREATOR) { $Env:BUILDKITE_BUILD_CREATOR } else { get_git_user_email }
   $BuildkiteBuildNumber = if ($Env:BUILDKITE_BUILD_NUMBER) { $Env:BUILDKITE_BUILD_NUMBER } else { "0" }
   $BuildkitePipelineSlug = if ($Env:BUILDKITE_PIPELINE_SLUG) { $Env:BUILDKITE_PIPELINE_SLUG } else { "elastic-agent-integration-tests" }
@@ -55,10 +46,7 @@ function ess_down {
     return 0
   }
   Write-Output "~~~ Tearing down the ESS Stack(created for this step)"
-  try {  
-    $Env:EC_API_KEY = Retry-Command -ScriptBlock {  
-      vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
-    }
+  try {
     Push-Location -Path $TfDir
     & terraform init
     & terraform destroy -auto-approve