initial attempt to create/destroy/read

Author: v1v

.buildkite/bk.integration-fips.pipeline.yml

@@ -7,6 +7,19 @@ env:
   IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1749862860"
   IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1749862860"
 
+# This section is used to define the plugins that will be used in the pipeline.
+# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
+common:
+  - google_oidc_observability_plugin: &google_oidc_observability_plugin
+      # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/elastic-agent/01-gcp-oidc.tf
+      # This plugin authenticates to Google Cloud using the OIDC token.
+      elastic/oblt-google-auth#v1.3.0:
+        lifetime: 10800 # seconds
+  # TODO: maybe we can rely on the VM and install only if not present?
+  - oblt_cli_plugin: &oblt_cli_plugin:
+      elastic/oblt-cli#v0.2.0:
+        version-file: .oblt-cli-version
+
 steps:
   - label: Build and push custom elastic-agent image
     depends_on:
@@ -29,7 +42,7 @@ steps:
       - elastic/vault-docker-login#v0.5.2:
           secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry'
 
-  - label: Start ESS stack for FIPS integration tests
+  - label: Start ESS stack for FIPS integration tests using oblt-cli
     key: integration-fips-ess
     depends_on:
       - integration-fips-cloud-image
@@ -38,14 +51,9 @@ steps:
       CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
       CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips"
       TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
-    command: |
-      source .buildkite/scripts/steps/ess_start.sh
-    artifact_paths:
-      - test_infra/ess/*.tfstate
-      - test_infra/ess/*.lock.hcl
+    command: .buildkite/scripts/steps/ess_start.sh
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
-      useCustomGlobalHooks: true
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/oblt-cli:latest"
 
   - group: "fips:Stateful:Ubuntu"
     key: integration-tests-ubuntu-fips
@@ -80,6 +88,9 @@ steps:
               - "true"
             groups:
               - fleet # currently there is only a single test in the fleet group, add more tests once they have been defined
+        plugins:
+          - *google_oidc_observability_plugin
+          - *oblt_cli_plugin
 
       - label: "fips:arm64:sudo-{{matrix.sudo}}:{{matrix.groups}}"
         depends_on:
@@ -109,18 +120,17 @@ steps:
               - "true"
             groups:
               - fleet
+        plugins:
+          - *google_oidc_observability_plugin
+          - *oblt_cli_plugin
 
   - label: ESS FIPS stack cleanup
     depends_on:
       - integration-tests-ubuntu-fips
     allow_dependency_failure: true
-    command: |
-      buildkite-agent artifact download "test_infra/ess/**" . --step "integration-fips-ess"
-      ls -lah test_infra/ess
-      .buildkite/scripts/steps/ess_down.sh
+    command: .buildkite/scripts/steps/ess_down.sh
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
-      useCustomGlobalHooks: true
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/oblt-cli:latest"
 
   - label: Aggregate test reports
     depends_on:

.buildkite/bk.integration.pipeline.yml

@@ -42,29 +42,20 @@ common:
           KIBANA_HOST: ea-serverless-it-kibana-hostname
           KIBANA_USERNAME: ea-serverless-it-kibana-username
           KIBANA_PASSWORD: ea-serverless-it-kibana-password
-  - gcp_hosted_secrets_plugin: &gcp_hosted_secrets_plugin
-      elastic/gcp-secret-manager#v1.3.0-elastic:
-        env:
-          # These secrets are created in the step called Start ESS stack for integration tests
-          # TODO: need to find a way to use dynamic names in the secrets
-          ELASTICSEARCH_HOST: ea-hosted-it-elasticsearch-hostname
-          ELASTICSEARCH_PASSWORD: ea-hosted-it-elasticsearch-password
-          ELASTICSEARCH_USERNAME: ea-hosted-it-elasticsearch-username
-          KIBANA_HOST: ea-hosted-it-kibana-hostname
-          KIBANA_USERNAME: ea-hosted-it-kibana-username
-          KIBANA_PASSWORD: ea-hosted-it-kibana-password
-          INTEGRATIONS_SERVER_HOST: ea-hosted-it-integration-hostname
+  # TODO: maybe we can rely on the VM and install only if not present?
+  - oblt_cli_plugin: &oblt_cli_plugin:
+      elastic/oblt-cli#v0.2.0:
+        version-file: .oblt-cli-version
 
 steps:
-  - label: Start ESS stack for integration tests
+  - label: Start ESS stack for integration tests using oblt-cli
     key: integration-ess
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - ESS stack provision using oblt-cli"
     command: .buildkite/scripts/steps/ess_start.sh
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/oblt-cli:latest"
-      useCustomGlobalHooks: true
 
   - group: "Extended runtime leak tests"
     key: extended-integration-tests
@@ -92,6 +83,10 @@ steps:
         retry:
           automatic:
             limit: 1
+        plugins:
+          - *google_oidc_observability_plugin
+          - *oblt_cli_plugin
+
       - label: "Windows:2025:amd64:sudo"
         depends_on:
           - packaging-windows
@@ -110,6 +105,10 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_WIN_2025}"
+        plugins:
+          - *google_oidc_observability_plugin
+          - *oblt_cli_plugin
+
       - label: "Ubuntu:2404:amd64:sudo"
         depends_on: packaging-ubuntu-x86-64
         env:
@@ -127,6 +126,9 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_UBUNTU_2404_X86_64}"
+        plugins:
+          - *google_oidc_observability_plugin
+          - *oblt_cli_plugin
 
   - group: "Stateful: Windows"
     key: integration-tests-win
@@ -163,7 +165,7 @@ steps:
           - install-uninstall
         plugins:
           - *google_oidc_observability_plugin
-          - *gcp_hosted_secrets_plugin
+          - *oblt_cli_plugin
 
       - label: "Win2022:non-sudo:{{matrix}}"
         depends_on:
@@ -185,7 +187,7 @@ steps:
           - default
         plugins:
           - *google_oidc_observability_plugin
-          - *gcp_hosted_secrets_plugin
+          - *oblt_cli_plugin
 
       - label: "Win2025:sudo:{{matrix}}"
         depends_on:
@@ -214,7 +216,7 @@ steps:
           - install-uninstall
         plugins:
           - *google_oidc_observability_plugin
-          - *gcp_hosted_secrets_plugin
+          - *oblt_cli_plugin
 
       - label: "Win2025:non-sudo:{{matrix}}"
         depends_on:
@@ -236,7 +238,7 @@ steps:
           - default
         plugins:
           - *google_oidc_observability_plugin
-          - *gcp_hosted_secrets_plugin
+          - *oblt_cli_plugin
 
   - group: "Stateful:Ubuntu"
     key: integration-tests-ubuntu
@@ -265,7 +267,7 @@ steps:
           - default
         plugins:
           - *google_oidc_observability_plugin
-          - *gcp_hosted_secrets_plugin
+          - *oblt_cli_plugin
 
       - label: "x86_64:sudo: {{matrix}}"
         depends_on:
@@ -301,7 +303,7 @@ steps:
           - container
         plugins:
           - *google_oidc_observability_plugin
-          - *gcp_hosted_secrets_plugin
+          - *oblt_cli_plugin
 
       - label: "arm:sudo: {{matrix}}"
         depends_on:
@@ -337,7 +339,7 @@ steps:
           # - container
         plugins:
           - *google_oidc_observability_plugin
-          - *gcp_hosted_secrets_plugin
+          - *oblt_cli_plugin
 
       - label: "arm:non-sudo: {{matrix}}"
         skip: true
@@ -360,7 +362,7 @@ steps:
           - default
         plugins:
           - *google_oidc_observability_plugin
-          - *gcp_hosted_secrets_plugin
+          - *oblt_cli_plugin
 
   - group: "Stateful:Debian"
     key: integration-tests-debian
@@ -389,7 +391,7 @@ steps:
           - default
         plugins:
           - *google_oidc_observability_plugin
-          - *gcp_hosted_secrets_plugin
+          - *oblt_cli_plugin
 
       - label: "x86_64:sudo: {{matrix}}"
         depends_on:
@@ -426,7 +428,7 @@ steps:
           - container
         plugins:
           - *google_oidc_observability_plugin
-          - *gcp_hosted_secrets_plugin
+          - *oblt_cli_plugin
 
   - group: "Stateful(Sudo):RHEL8"
     key: integration-tests-rhel8
@@ -448,13 +450,13 @@ steps:
         retry:
           automatic:
             limit: 1
-        plugins:
-          - *google_oidc_observability_plugin
-          - *gcp_hosted_secrets_plugin
         agents:
           provider: "gcp"
           machineType: "n2-standard-8"
           image: "${IMAGE_RHEL_8}"
+        plugins:
+          - *google_oidc_observability_plugin
+          - *oblt_cli_plugin
 
   - group: "Kubernetes"
     key: integration-tests-kubernetes
@@ -486,9 +488,6 @@ steps:
           machineType: "n2-standard-4"
           image: "${IMAGE_UBUNTU_2404_X86_64}"
           diskSizeGb: 80
-        plugins:
-          - *google_oidc_observability_plugin
-          - *gcp_hosted_secrets_plugin
         matrix:
           setup:
             variants:
@@ -506,6 +505,9 @@ steps:
               - v1.31.0
               - v1.32.0
               - v1.33.0
+        plugins:
+          - *google_oidc_observability_plugin
+          - *oblt_cli_plugin
 
   - group: "Serverless integration test"
     key: integration-tests-serverless
@@ -585,10 +587,9 @@ steps:
       - integration-tests-kubernetes
       - extended-integration-tests
     allow_dependency_failure: true
-    command: .buildkite/scripts/steps/oblt-cli-teardown.sh
+    command: .buildkite/scripts/steps/ess_down.sh
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/oblt-cli:latest"
-      useCustomGlobalHooks: true
 
   - label: Aggregate test reports
     # Warning: The key has a hook in pre-command

.buildkite/scripts/steps/ess.ps1

@@ -11,13 +11,27 @@ function ess_up {
       return 1
   }
 
-  & oblt-cli
+  & oblt-cli cluster create ess `
+      --stack-version "$StackVersion" `
+      --cluster-name-prefix ea-hosted-it `
+      --output-file="cluster-info.json" `
+      --wait 15
+
+  $ClusterName = (Get-Content -Path "cluster-info.json" | ConvertFrom-Json).ClusterName
+  if (-not $ClusterName) {
+      Write-Error "Error: Failed to retrieve cluster name from cluster-info.json"
+      return 1
+  }
+
+  # Store the cluster name as a meta-data
+  & buildkite-agent meta-data set cluster-name $ClusterName
 }
 
 function ess_down {  
   Write-Output "~~~ Tearing down the ESS Stack(created for this step)"
-  try {  
-    & oblt-cli
+  try {
+    $ClusterName = & buildkite-agent meta-data get cluster-name
+    & oblt-cli cluster destroy --cluster-name "$ClusterName" --force
   } catch {
     Write-Output "Error: Failed to destroy ESS stack(it will be auto-deleted later): $_"
   }

.buildkite/scripts/steps/ess.sh

@@ -11,10 +11,26 @@ function ess_up() {
     return 1
   fi
 
-  oblt-cli
+  # Create a cluster with the specified stack version and store the cluster information in a file
+  GITHUB_TOKEN="$VAULT_GITHUB_TOKEN" \
+    oblt-cli cluster create ess \
+      --stack-version "$STACK_VERSION" \
+      --cluster-name-prefix ea-hosted-it \
+      --output-file="${PWD}/cluster-info.json" \
+      --wait 15
+
+  # Extract the cluster name from the cluster information file
+  CLUSTER_NAME=$(jq -r '.ClusterName' cluster-info.json)
+
+  # Store the cluster name as a meta-data
+  buildkite-agent meta-data set cluster-name "${CLUSTER_NAME}"
 }
 
 function ess_down() {
-  echo "~~~ Tearing down the ESS Stack"  
-  oblt-cli
+  echo "~~~ Tearing down the ESS Stack"
+  # Get the cluster name from the meta-data
+  CLUSTER_NAME="$(buildkite-agent meta-data get cluster-name)"
+
+  # Destroy the cluster
+  oblt-cli cluster destroy --cluster-name "${CLUSTER_NAME}" --force
 }

.buildkite/scripts/steps/integration_tests_tf.ps1

@@ -28,6 +28,16 @@ try {
     Write-Output "~~~ Running integration tests"
     # Get-Ess-Stack will start the ESS stack if it is a BK retry, otherwise it will retrieve ESS stack metadata
     Get-Ess-Stack -StackVersion $PACKAGE_VERSION
+
+    # Load the ESS stack secrets
+    # Get the cluster name from the meta-data (CI specific)
+    # QUESTION: should we support the case when using the ESS stack in local environment?
+    $ClusterName = & buildkite-agent meta-data get cluster-name
+    & oblt-cli cluster secrets env --cluster-nam $ClusterName --output-file="env.sh"
+
+    # TODO: source the secrets file
+    source "${PWD}/env.sh" || rm "${PWD}/env.sh"
+
     & "$PWD\.buildkite\scripts\buildkite-integration-tests.ps1" $GROUP_NAME $TEST_SUDO
     $TestsExitCode = $LASTEXITCODE
     if ($TestsExitCode -ne 0)

.buildkite/scripts/steps/integration_tests_tf.sh

@@ -39,6 +39,16 @@ if [[ "${BUILDKITE_RETRY_COUNT}" -gt 0 ]]; then
   preinstall_fleet_packages
 fi
 
+# Load the ESS stack secrets
+# Get the cluster name from the meta-data (CI specific)
+# QUESTION: should we support the case when using the ESS stack in local environment?
+CLUSTER_NAME="$(buildkite-agent meta-data get cluster-name)"
+
+oblt-cli cluster secrets env --cluster-name="${CLUSTER_NAME}" --output-file="${PWD}/env.sh"
+
+# Source the secrets file
+source "${PWD}/env.sh" || rm "${PWD}/env.sh"
+
 # Run integration tests
 echo "~~~ Running integration tests"
 

.oblt-cli-version

@@ -0,0 +1 @@
+7.19.0