oblt-cli(hosted): support running ITs using oblt-cli

with OIDC support

Author: v1v

.buildkite/bk.integration.pipeline.yml

@@ -23,6 +23,11 @@ common:
         lifetime: 10800 # seconds
         project-id: "elastic-observability-ci"
         project-number: "911195782929"
+  - google_oidc_observability_plugin: &google_oidc_observability_plugin
+      # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/elastic-agent/01-gcp-oidc.tf
+      # This plugin authenticates to Google Cloud using the OIDC token.
+      elastic/oblt-google-auth#v1.3.0:
+        lifetime: 10800 # seconds
 # see https://github.com/avaly/gcp-secret-manager-buildkite-plugin/pull/10
 # see https://github.com/avaly/gcp-secret-manager-buildkite-plugin/pull/11
 #  - gcp_serverless_secrets_plugin: &gcp_serverless_secrets_plugin
@@ -37,24 +42,28 @@ common:
           KIBANA_HOST: ea-serverless-it-kibana-hostname
           KIBANA_USERNAME: ea-serverless-it-kibana-username
           KIBANA_PASSWORD: ea-serverless-it-kibana-password
+  - gcp_hosted_secrets_plugin: &gcp_hosted_secrets_plugin
+      elastic/gcp-secret-manager#v1.3.0-elastic:
+        env:
+          # These secrets are created in the step called Start ESS stack for integration tests
+          # TODO: need to find a way to use dynamic names in the secrets
+          ELASTICSEARCH_HOST: ea-hosted-it-elasticsearch-hostname
+          ELASTICSEARCH_PASSWORD: ea-hosted-it-elasticsearch-password
+          ELASTICSEARCH_USERNAME: ea-hosted-it-elasticsearch-username
+          KIBANA_HOST: ea-hosted-it-kibana-hostname
+          KIBANA_USERNAME: ea-hosted-it-kibana-username
+          KIBANA_PASSWORD: ea-hosted-it-kibana-password
+          INTEGRATIONS_SERVER_HOST: ea-hosted-it-integration-hostname
 
 steps:
   - label: Start ESS stack for integration tests
     key: integration-ess
     notify:
       - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - ESS stack provision"
-    env:
-      ASDF_TERRAFORM_VERSION: 1.9.2
-    command: |
-      #!/usr/bin/env bash
-      set -euo pipefail
-      source .buildkite/scripts/steps/ess_start.sh
-    artifact_paths:
-      - test_infra/ess/*.tfstate
-      - test_infra/ess/*.lock.hcl
+          context: "buildkite/elastic-agent-extended-testing - ESS stack provision using oblt-cli"
+    command: .buildkite/scripts/steps/ess_start.sh
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/oblt-cli:latest"
       useCustomGlobalHooks: true
 
   - group: "Extended runtime leak tests"
@@ -152,6 +161,9 @@ steps:
           - upgrade
           - upgrade-flavor
           - install-uninstall
+        plugins:
+          - *google_oidc_observability_plugin
+          - *gcp_hosted_secrets_plugin
 
       - label: "Win2022:non-sudo:{{matrix}}"
         depends_on:
@@ -171,6 +183,9 @@ steps:
             limit: 1
         matrix:
           - default
+        plugins:
+          - *google_oidc_observability_plugin
+          - *gcp_hosted_secrets_plugin
 
       - label: "Win2025:sudo:{{matrix}}"
         depends_on:
@@ -197,6 +212,9 @@ steps:
           - upgrade
           - upgrade-flavor
           - install-uninstall
+        plugins:
+          - *google_oidc_observability_plugin
+          - *gcp_hosted_secrets_plugin
 
       - label: "Win2025:non-sudo:{{matrix}}"
         depends_on:
@@ -216,6 +234,9 @@ steps:
           image: "${IMAGE_WIN_2025}"
         matrix:
           - default
+        plugins:
+          - *google_oidc_observability_plugin
+          - *gcp_hosted_secrets_plugin
 
   - group: "Stateful:Ubuntu"
     key: integration-tests-ubuntu
@@ -242,6 +263,9 @@ steps:
           image: "${IMAGE_UBUNTU_2404_X86_64}"
         matrix:
           - default
+        plugins:
+          - *google_oidc_observability_plugin
+          - *gcp_hosted_secrets_plugin
 
       - label: "x86_64:sudo: {{matrix}}"
         depends_on:
@@ -275,6 +299,9 @@ steps:
           - fqdn
           - deb
           - container
+        plugins:
+          - *google_oidc_observability_plugin
+          - *gcp_hosted_secrets_plugin
 
       - label: "arm:sudo: {{matrix}}"
         depends_on:
@@ -308,6 +335,9 @@ steps:
           # - fqdn
           # - deb
           # - container
+        plugins:
+          - *google_oidc_observability_plugin
+          - *gcp_hosted_secrets_plugin
 
       - label: "arm:non-sudo: {{matrix}}"
         skip: true
@@ -328,6 +358,9 @@ steps:
           instanceType: "m6g.xlarge"
         matrix:
           - default
+        plugins:
+          - *google_oidc_observability_plugin
+          - *gcp_hosted_secrets_plugin
 
   - group: "Stateful:Debian"
     key: integration-tests-debian
@@ -354,6 +387,9 @@ steps:
           image: "${IMAGE_DEBIAN_12}"
         matrix:
           - default
+        plugins:
+          - *google_oidc_observability_plugin
+          - *gcp_hosted_secrets_plugin
 
       - label: "x86_64:sudo: {{matrix}}"
         depends_on:
@@ -388,6 +424,9 @@ steps:
           #- fqdn
           - deb
           - container
+        plugins:
+          - *google_oidc_observability_plugin
+          - *gcp_hosted_secrets_plugin
 
   - group: "Stateful(Sudo):RHEL8"
     key: integration-tests-rhel8
@@ -409,6 +448,9 @@ steps:
         retry:
           automatic:
             limit: 1
+        plugins:
+          - *google_oidc_observability_plugin
+          - *gcp_hosted_secrets_plugin
         agents:
           provider: "gcp"
           machineType: "n2-standard-8"
@@ -444,6 +486,9 @@ steps:
           machineType: "n2-standard-4"
           image: "${IMAGE_UBUNTU_2404_X86_64}"
           diskSizeGb: 80
+        plugins:
+          - *google_oidc_observability_plugin
+          - *gcp_hosted_secrets_plugin
         matrix:
           setup:
             variants:
@@ -540,12 +585,9 @@ steps:
       - integration-tests-kubernetes
       - extended-integration-tests
     allow_dependency_failure: true
-    command: |
-      buildkite-agent artifact download "test_infra/ess/**" . --step "integration-ess"
-      ls -lah test_infra/ess
-      .buildkite/scripts/steps/ess_down.sh
+    command: .buildkite/scripts/steps/oblt-cli-teardown.sh
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/oblt-cli:latest"
       useCustomGlobalHooks: true
 
   - label: Aggregate test reports

.buildkite/scripts/steps/ess.ps1

@@ -5,84 +5,24 @@ function ess_up {
   )
 
   Write-Output "~~~ Starting ESS Stack"
-  
-  $Workspace = & git rev-parse --show-toplevel
-  $TfDir = Join-Path -Path $Workspace -ChildPath "test_infra/ess/"
 
   if (-not $StackVersion) {
       Write-Error "Error: Specify stack version: ess_up [stack_version]"
       return 1
   }
 
-  $Env:EC_API_KEY = Retry-Command -ScriptBlock {  
-    vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
-  }
-
-  if (-not $Env:EC_API_KEY) {
-      Write-Error "Error: Failed to get EC API key from vault"
-      exit 1
-  }
-
-  $BuildkiteBuildCreator = if ($Env:BUILDKITE_BUILD_CREATOR) { $Env:BUILDKITE_BUILD_CREATOR } else { get_git_user_email }
-  $BuildkiteBuildNumber = if ($Env:BUILDKITE_BUILD_NUMBER) { $Env:BUILDKITE_BUILD_NUMBER } else { "0" }
-  $BuildkitePipelineSlug = if ($Env:BUILDKITE_PIPELINE_SLUG) { $Env:BUILDKITE_PIPELINE_SLUG } else { "elastic-agent-integration-tests" }
-
-  Push-Location -Path $TfDir
-  & terraform init
-  & terraform apply -auto-approve `
-      -var="stack_version=$StackVersion" `
-      -var="ess_region=$EssRegion" `
-      -var="creator=$BuildkiteBuildCreator" `
-      -var="buildkite_id=$BuildkiteBuildNumber" `
-      -var="pipeline=$BuildkitePipelineSlug"
-
-  $Env:ELASTICSEARCH_HOST = & terraform output -raw es_host
-  $Env:ELASTICSEARCH_USERNAME = & terraform output -raw es_username
-  $Env:ELASTICSEARCH_PASSWORD = & terraform output -raw es_password
-  $Env:KIBANA_HOST = & terraform output -raw kibana_endpoint
-  $Env:KIBANA_USERNAME = $Env:ELASTICSEARCH_USERNAME
-  $Env:KIBANA_PASSWORD = $Env:ELASTICSEARCH_PASSWORD
-  $Env:INTEGRATIONS_SERVER_HOST = & terraform output -raw integrations_server_endpoint
-  Pop-Location
+  & oblt-cli
 }
 
 function ess_down {  
-  $Workspace = & git rev-parse --show-toplevel
-  $TfDir = Join-Path -Path $Workspace -ChildPath "test_infra/ess/"
-  $stateFilePath = Join-Path -Path $TfDir -ChildPath "terraform.tfstate"
-
-  if (-not (Test-Path -Path $stateFilePath)) {
-    Write-Output "Terraform state file not found. Skipping ESS destroy."
-    return 0
-  }
   Write-Output "~~~ Tearing down the ESS Stack(created for this step)"
   try {  
-    $Env:EC_API_KEY = Retry-Command -ScriptBlock {  
-      vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod
-    }
-    Push-Location -Path $TfDir
-    & terraform init
-    & terraform destroy -auto-approve
-    Pop-Location
+    & oblt-cli
   } catch {
     Write-Output "Error: Failed to destroy ESS stack(it will be auto-deleted later): $_"
   }
 }
 
-function get_git_user_email {
-  if (!(git rev-parse --is-inside-work-tree *>&1)) {
-      return "unknown"
-  }
-
-  $email = & git config --get user.email
-
-  if (-not $email) {
-      return "unknown"
-  } else {
-      return $email
-  }
-}
-
 function Retry-Command {
   param (
       [scriptblock]$ScriptBlock,
@@ -115,18 +55,7 @@ function Get-Ess-Stack {
   )
 
   if ($Env:BUILDKITE_RETRY_COUNT -gt 0) {
-      Write-Output "The step is retried, starting the ESS stack again"        
+      Write-Output "The step is retried, starting the ESS stack again"
       ess_up $StackVersion
-      Write-Output "ESS stack is up. ES_HOST: $Env:ELASTICSEARCH_HOST"
-  } else {
-      # For the first run, we retrieve ESS stack metadata
-      Write-Output "~~~ Receiving ESS stack metadata"
-      $Env:ELASTICSEARCH_HOST = & buildkite-agent meta-data get "es.host"
-      $Env:ELASTICSEARCH_USERNAME = & buildkite-agent meta-data get "es.username"
-      $Env:ELASTICSEARCH_PASSWORD = & buildkite-agent meta-data get "es.pwd"
-      $Env:KIBANA_HOST = & buildkite-agent meta-data get "kibana.host"
-      $Env:KIBANA_USERNAME = & buildkite-agent meta-data get "kibana.username"
-      $Env:KIBANA_PASSWORD = & buildkite-agent meta-data get "kibana.pwd"
-      Write-Output "Received ESS stack data from previous step. ES_HOST: $Env:ELASTICSEARCH_HOST"
   }
 }

.buildkite/scripts/steps/ess.sh

@@ -3,8 +3,6 @@ set -euo pipefail
 
 function ess_up() {
   echo "~~~ Starting ESS Stack"
-  local WORKSPACE=$(git rev-parse --show-toplevel)
-  local TF_DIR="${WORKSPACE}/test_infra/ess/"
   local STACK_VERSION=$1
   local ESS_REGION=${2:-"gcp-us-west2"}
 
@@ -13,64 +11,10 @@ function ess_up() {
     return 1
   fi
 
-  export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
-  
-  if [[ -z "${EC_API_KEY}" ]]; then
-    echo "Error: Failed to get EC API key from vault" >&2
-    exit 1
-  fi
-
-  BUILDKITE_BUILD_CREATOR="${BUILDKITE_BUILD_CREATOR:-"$(get_git_user_email)"}"
-  BUILDKITE_BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-"0"}"
-  BUILDKITE_PIPELINE_SLUG="${BUILDKITE_PIPELINE_SLUG:-"elastic-agent-integration-tests"}"
-  
-  pushd "${TF_DIR}"    
-  terraform init
-  terraform apply \
-    -auto-approve \
-    -var="stack_version=${STACK_VERSION}" \
-    -var="ess_region=${ESS_REGION}" \
-    -var="creator=${BUILDKITE_BUILD_CREATOR}" \
-    -var="buildkite_id=${BUILDKITE_BUILD_NUMBER}" \
-    -var="pipeline=${BUILDKITE_PIPELINE_SLUG}"
-
-  export ELASTICSEARCH_HOST=$(terraform output -raw es_host)
-  export ELASTICSEARCH_USERNAME=$(terraform output -raw es_username)
-  export ELASTICSEARCH_PASSWORD=$(terraform output -raw es_password)
-  export KIBANA_HOST=$(terraform output -raw kibana_endpoint)
-  export KIBANA_USERNAME=$ELASTICSEARCH_USERNAME
-  export KIBANA_PASSWORD=$ELASTICSEARCH_PASSWORD
-  export INTEGRATIONS_SERVER_HOST=$(terraform output -raw integrations_server_endpoint)
-  popd
+  oblt-cli
 }
 
 function ess_down() {
   echo "~~~ Tearing down the ESS Stack"  
-  local WORKSPACE=$(git rev-parse --show-toplevel)
-  local TF_DIR="${WORKSPACE}/test_infra/ess/"
-  if [ -z "${EC_API_KEY:-}" ]; then
-    export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)    
-  fi
-  
-  pushd "${TF_DIR}"
-  terraform init
-  terraform destroy -auto-approve
-  popd
-}
-
-function get_git_user_email() {
-  if ! git rev-parse --is-inside-work-tree &>/dev/null; then
-    echo "unknown"  
-    return
-  fi
-
-  local email
-  email=$(git config --get user.email)
-  
-  if [ -z "$email" ]; then
-    echo "unknown"  
-  else
-    echo "$email"
-  fi
+  oblt-cli
 }
-

.buildkite/scripts/steps/ess_start.sh

@@ -10,12 +10,3 @@ OVERRIDE_STACK_VERSION=${OVERRIDE_STACK_VERSION}"-SNAPSHOT"
 ess_up $OVERRIDE_STACK_VERSION
 
 preinstall_fleet_packages
-
-echo "ES_HOST: ${ELASTICSEARCH_HOST}"
-buildkite-agent meta-data set "es.host" $ELASTICSEARCH_HOST
-buildkite-agent meta-data set "es.username" $ELASTICSEARCH_USERNAME
-buildkite-agent meta-data set "es.pwd" $ELASTICSEARCH_PASSWORD
-buildkite-agent meta-data set "kibana.host" $KIBANA_HOST
-buildkite-agent meta-data set "kibana.username" $KIBANA_USERNAME
-buildkite-agent meta-data set "kibana.pwd" $KIBANA_PASSWORD
-buildkite-agent meta-data set "integrations_server.host" $INTEGRATIONS_SERVER_HOST

.buildkite/scripts/steps/integration_tests_tf.sh

@@ -37,16 +37,6 @@ if [[ "${BUILDKITE_RETRY_COUNT}" -gt 0 ]]; then
   trap 'ess_down' EXIT
   ess_up $OVERRIDE_STACK_VERSION || (echo -e "^^^ +++\nFailed to start ESS stack")
   preinstall_fleet_packages
-else
-  # For the first run, we start the stack in the start_ess.sh step and it sets the meta-data
-  echo "~~~ Receiving ESS stack metadata"
-  export ELASTICSEARCH_HOST=$(buildkite-agent meta-data get "es.host")
-  export ELASTICSEARCH_USERNAME=$(buildkite-agent meta-data get "es.username")
-  export ELASTICSEARCH_PASSWORD=$(buildkite-agent meta-data get "es.pwd")
-  export KIBANA_HOST=$(buildkite-agent meta-data get "kibana.host")
-  export KIBANA_USERNAME=$(buildkite-agent meta-data get "kibana.username")
-  export KIBANA_PASSWORD=$(buildkite-agent meta-data get "kibana.pwd")
-  export INTEGRATIONS_SERVER_HOST=$(buildkite-agent meta-data get "integrations_server.host")
 fi
 
 # Run integration tests