Merge branch 'feature/hosted-stack-using-oblt-cli' into test/oblt-cli-hosted

* feature/hosted-stack-using-oblt-cli: (26 commits)
  Use the current official docker image for oblt-cli
  Mark the elasticinframetrics processor as deprecated and schedule for removal (#8659)
  [main][Automation] Update versions (#8668)
  chore: Update create_deployment_csp_configuration.yaml (#8669)
  Attempt to make test more reliable by querying ES directly (#8422)
  [test] split up ess and beats serverless integration tests (#8551)
  Remove resource/k8s processor and use k8sattributes processor for service attributes (#8599)
  fix: use --force-confold for deb tests in TestUpgradeAgentWithTamperProtectedEndpoint_DEB (#8649)
  [main][Automation] Bump stack images versions to 9.1.0-ea0b7542 (#8612)
  chore: Update to elastic/beats@f6594fb72670 (#8640)
  [deb/rpm] restart endpoint with tamper protection after elastic-agent  (#8637)
  ci: don't preinstall fleet packages on retried CI steps (#8636)
  chore: Update to elastic/beats@6b6941eed496 (#8619)
  [main][Automation] Bump VM Image version to 1750467641 (#8617)
  flaky: skip TestUpgradeAgentWithTamperProtectedEndpoint_RPM (#8626)
  Add skip-changelog PR label for bump VM PRs (#8627)
  build(deps): bump github.com/elastic/go-seccomp-bpf from 1.5.0 to 1.6.0 (#8611)
  [ci] fix k8s integration tests flakiness (#8575)
  bump apmconfig Otel extension to v0.3.0 (#8600)
  Enhancement/6394 allow deb rpm to upgrade with endpoint tamper protection (#6907)
  ...

Author: v1v

.buildkite/bk.integration-fips.pipeline.yml

@@ -2,10 +2,11 @@
 
 env:
   ASDF_MAGE_VERSION: 1.14.0
+  MS_GOTOOLCHAIN_TELEMETRY_ENABLED: "0"
 
-  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1749862860"
-  IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1749862860"
-  IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1749862860"
+  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1750467641"
+  IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1750467641"
+  IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1750467641"
 
 # This section is used to define the plugins that will be used in the pipeline.
 # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
@@ -53,7 +54,8 @@ steps:
       TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
     command: .buildkite/scripts/steps/ess_start.sh
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/oblt-cli:latest"
+      # image: "docker.elastic.co/ci-agent-images/platform-ingest/oblt-cli:latest"
+      image: "docker.elastic.co/observability-ci/oblt-cli:7.21.0"
 
   - group: "fips:Stateful:Ubuntu"
     key: integration-tests-ubuntu-fips
@@ -68,6 +70,7 @@ steps:
           CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
           CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips"
           TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64-fips'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}}
@@ -100,6 +103,7 @@ steps:
           CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
           CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips"
           TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64-fips'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}}
@@ -130,7 +134,8 @@ steps:
     allow_dependency_failure: true
     command: .buildkite/scripts/steps/ess_down.sh
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/oblt-cli:latest"
+      # image: "docker.elastic.co/ci-agent-images/platform-ingest/oblt-cli:latest"
+      image: "docker.elastic.co/observability-ci/oblt-cli:7.21.0"
 
   - label: Aggregate test reports
     depends_on:

.buildkite/bk.integration.pipeline.yml

@@ -6,12 +6,12 @@ env:
 
   # The following images are defined here and their values will be updated by updatecli
   # Please do not change them manually.
-  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1749862860"
-  IMAGE_UBUNTU_2404_ARM_64: "platform-ingest-elastic-agent-ubuntu-2404-aarch64-1749862860"
-  IMAGE_RHEL_8: "platform-ingest-elastic-agent-rhel-8-1749862860"
-  IMAGE_DEBIAN_12: "platform-ingest-elastic-agent-debian-12-1749862860"
-  IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1749862860"
-  IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1749862860"
+  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1750467641"
+  IMAGE_UBUNTU_2404_ARM_64: "platform-ingest-elastic-agent-ubuntu-2404-aarch64-1750467641"
+  IMAGE_RHEL_8: "platform-ingest-elastic-agent-rhel-8-1750467641"
+  IMAGE_DEBIAN_12: "platform-ingest-elastic-agent-debian-12-1750467641"
+  IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1750467641"
+  IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1750467641"
 
 # This section is used to define the plugins that will be used in the pipeline.
 # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
@@ -55,7 +55,8 @@ steps:
           context: "buildkite/elastic-agent-extended-testing - ESS stack provision using oblt-cli"
     command: .buildkite/scripts/steps/ess_start.sh
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/oblt-cli:latest"
+      # image: "docker.elastic.co/ci-agent-images/platform-ingest/oblt-cli:latest"
+      image: "docker.elastic.co/observability-ci/oblt-cli:7.21.0"
 
   - group: "Extended runtime leak tests"
     key: extended-integration-tests
@@ -144,6 +145,8 @@ steps:
       - label: "Win2022:sudo:{{matrix}}"
         depends_on:
           - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} true
@@ -173,6 +176,8 @@ steps:
       - label: "Win2022:non-sudo:{{matrix}}"
         depends_on:
           - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} false
@@ -195,6 +200,8 @@ steps:
       - label: "Win2025:sudo:{{matrix}}"
         depends_on:
           - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} true
@@ -224,6 +231,8 @@ steps:
       - label: "Win2025:non-sudo:{{matrix}}"
         depends_on:
           - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} false
@@ -253,6 +262,8 @@ steps:
     steps:
       - label: "x86_64:non-sudo: {{matrix}}"
         depends_on: packaging-ubuntu-x86-64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
@@ -276,6 +287,8 @@ steps:
         depends_on:
           - packaging-ubuntu-x86-64
         # due to deb group present in matrix tar.gz and deb packages artifacts are required
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
@@ -311,6 +324,8 @@ steps:
       - label: "arm:sudo: {{matrix}}"
         depends_on:
           - packaging-ubuntu-arm64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
@@ -348,6 +363,8 @@ steps:
         skip: true
         depends_on:
           - packaging-ubuntu-arm64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
@@ -377,6 +394,8 @@ steps:
     steps:
       - label: "x86_64:non-sudo: {{matrix}}"
         depends_on: packaging-ubuntu-x86-64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
@@ -400,6 +419,8 @@ steps:
         depends_on:
           - packaging-ubuntu-x86-64
         # due to deb group present in matrix tar.gz and deb packages artifacts are required
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
@@ -444,6 +465,8 @@ steps:
       - label: "x86_64:sudo:rpm"
         depends_on:
           - packaging-ubuntu-x86-64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64
           .buildkite/scripts/steps/integration_tests_tf.sh rpm true
@@ -593,7 +616,8 @@ steps:
     allow_dependency_failure: true
     command: .buildkite/scripts/steps/ess_down.sh
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/oblt-cli:latest"
+      # image: "docker.elastic.co/ci-agent-images/platform-ingest/oblt-cli:latest"
+      image: "docker.elastic.co/observability-ci/oblt-cli:7.21.0"
 
   - label: Aggregate test reports
     # Warning: The key has a hook in pre-command

.buildkite/hooks/pre-command

@@ -18,6 +18,13 @@ CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp"
 CI_ESS_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
 CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
 
+# NOTE: oblt-cli requires the GITHUB_TOKEN to be set in the environment
+#       unfortunatelym, we cannot use the BK pipeline itself as it will
+#       print the content, so we need to set it in the pre-command hook.
+#       I decided to set it here for all the steps regardless to avoid
+#       complicated logic in the pre-command hook.
+export GITHUB_TOKEN=$VAULT_GITHUB_TOKEN
+
 function release_manager_login {
   DRA_CREDS_SECRET=$(retry 5 vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
   VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')

.buildkite/pipeline.yml

@@ -5,12 +5,12 @@ env:
 
   # The following images are defined here and their values will be updated by updatecli
   # Please do not change them manually.
-  IMAGE_UBUNTU_2204_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1749862860"
-  IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1749862860"
-  IMAGE_WIN_2016: "platform-ingest-elastic-agent-windows-2016-1749862860"
-  IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1749862860"
-  IMAGE_WIN_10: "platform-ingest-elastic-agent-windows-10-1749862860"
-  IMAGE_WIN_11: "platform-ingest-elastic-agent-windows-11-1749862860"
+  IMAGE_UBUNTU_2204_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1750467641"
+  IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1750467641"
+  IMAGE_WIN_2016: "platform-ingest-elastic-agent-windows-2016-1750467641"
+  IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1750467641"
+  IMAGE_WIN_10: "platform-ingest-elastic-agent-windows-10-1750467641"
+  IMAGE_WIN_11: "platform-ingest-elastic-agent-windows-11-1750467641"
 
 steps:
   # Trigger for pull requests

.buildkite/scripts/steps/ess.sh

@@ -12,8 +12,7 @@ function ess_up() {
   fi
 
   # Create a cluster with the specified stack version and store the cluster information in a file
-  GITHUB_TOKEN="$VAULT_GITHUB_TOKEN" \
-    oblt-cli cluster create ess \
+  oblt-cli cluster create ess \
       --stack-version "$STACK_VERSION" \
       --cluster-name-prefix ea-hosted-it \
       --output-file="${PWD}/cluster-info.json" \

.buildkite/scripts/steps/integration_tests_tf.sh

@@ -36,7 +36,16 @@ if [[ "${BUILDKITE_RETRY_COUNT}" -gt 0 ]]; then
   echo "~~~ The steps is retried, starting the ESS stack again"
   trap 'ess_down' EXIT
   ess_up $OVERRIDE_STACK_VERSION || (echo -e "^^^ +++\nFailed to start ESS stack")
-  preinstall_fleet_packages
+else
+  # For the first run, we start the stack in the start_ess.sh step and it sets the meta-data
+  echo "~~~ Receiving ESS stack metadata"
+  export ELASTICSEARCH_HOST=$(buildkite-agent meta-data get "es.host")
+  export ELASTICSEARCH_USERNAME=$(buildkite-agent meta-data get "es.username")
+  export ELASTICSEARCH_PASSWORD=$(buildkite-agent meta-data get "es.pwd")
+  export KIBANA_HOST=$(buildkite-agent meta-data get "kibana.host")
+  export KIBANA_USERNAME=$(buildkite-agent meta-data get "kibana.username")
+  export KIBANA_PASSWORD=$(buildkite-agent meta-data get "kibana.pwd")
+  export INTEGRATIONS_SERVER_HOST=$(buildkite-agent meta-data get "integrations_server.host")
 fi
 
 # Run integration tests

.ci/updatecli/updatecli-bump-vm-images.yml

@@ -26,6 +26,7 @@ actions:
       labels:
         - dependencies
         - backport-skip
+        - skip-changelog
       title: '[{{ requiredEnv "BRANCH_NAME" }}][Automation] Bump VM Image version to {{ source "latestVersion" }}'
 
 sources:

.oblt-cli-version

@@ -1 +1 @@
-7.19.0
+7.21.0

Makefile

@@ -41,6 +41,7 @@ check-ci:
 	@mage -v helm:lint
 	@mage -v helm:updateAgentVersion
 	@mage -v helm:renderExamples
+	@mage -v integration:buildKubernetesTestData
 	@$(MAKE) check-no-changes
 
 ## check: run all the checks including linting using golangci-lint.