Merge branch 'main' into feature/hosted-stack-using-oblt-cli

Author: v1v

.buildkite/bk.integration-fips.pipeline.yml

@@ -2,10 +2,11 @@
 
 env:
   ASDF_MAGE_VERSION: 1.14.0
+  MS_GOTOOLCHAIN_TELEMETRY_ENABLED: "0"
 
-  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1749862860"
-  IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1749862860"
-  IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1749862860"
+  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1750467641"
+  IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1750467641"
+  IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1750467641"
 
 # This section is used to define the plugins that will be used in the pipeline.
 # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
@@ -68,6 +69,7 @@ steps:
           CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
           CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips"
           TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64-fips'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}}
@@ -100,6 +102,7 @@ steps:
           CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
           CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips"
           TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64-fips'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}}

.buildkite/bk.integration.pipeline.yml

@@ -6,12 +6,12 @@ env:
 
   # The following images are defined here and their values will be updated by updatecli
   # Please do not change them manually.
-  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1749862860"
-  IMAGE_UBUNTU_2404_ARM_64: "platform-ingest-elastic-agent-ubuntu-2404-aarch64-1749862860"
-  IMAGE_RHEL_8: "platform-ingest-elastic-agent-rhel-8-1749862860"
-  IMAGE_DEBIAN_12: "platform-ingest-elastic-agent-debian-12-1749862860"
-  IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1749862860"
-  IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1749862860"
+  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1750467641"
+  IMAGE_UBUNTU_2404_ARM_64: "platform-ingest-elastic-agent-ubuntu-2404-aarch64-1750467641"
+  IMAGE_RHEL_8: "platform-ingest-elastic-agent-rhel-8-1750467641"
+  IMAGE_DEBIAN_12: "platform-ingest-elastic-agent-debian-12-1750467641"
+  IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1750467641"
+  IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1750467641"
 
 # This section is used to define the plugins that will be used in the pipeline.
 # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
@@ -141,6 +141,8 @@ steps:
       - label: "Win2022:sudo:{{matrix}}"
         depends_on:
           - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} true
@@ -170,6 +172,8 @@ steps:
       - label: "Win2022:non-sudo:{{matrix}}"
         depends_on:
           - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} false
@@ -192,6 +196,8 @@ steps:
       - label: "Win2025:sudo:{{matrix}}"
         depends_on:
           - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} true
@@ -221,6 +227,8 @@ steps:
       - label: "Win2025:non-sudo:{{matrix}}"
         depends_on:
           - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} false
@@ -250,6 +258,8 @@ steps:
     steps:
       - label: "x86_64:non-sudo: {{matrix}}"
         depends_on: packaging-ubuntu-x86-64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
@@ -273,6 +283,8 @@ steps:
         depends_on:
           - packaging-ubuntu-x86-64
         # due to deb group present in matrix tar.gz and deb packages artifacts are required
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
@@ -308,6 +320,8 @@ steps:
       - label: "arm:sudo: {{matrix}}"
         depends_on:
           - packaging-ubuntu-arm64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
@@ -345,6 +359,8 @@ steps:
         skip: true
         depends_on:
           - packaging-ubuntu-arm64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
@@ -374,6 +390,8 @@ steps:
     steps:
       - label: "x86_64:non-sudo: {{matrix}}"
         depends_on: packaging-ubuntu-x86-64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
@@ -397,6 +415,8 @@ steps:
         depends_on:
           - packaging-ubuntu-x86-64
         # due to deb group present in matrix tar.gz and deb packages artifacts are required
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
@@ -441,6 +461,8 @@ steps:
       - label: "x86_64:sudo:rpm"
         depends_on:
           - packaging-ubuntu-x86-64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64
           .buildkite/scripts/steps/integration_tests_tf.sh rpm true

.buildkite/pipeline.yml

@@ -5,12 +5,12 @@ env:
 
   # The following images are defined here and their values will be updated by updatecli
   # Please do not change them manually.
-  IMAGE_UBUNTU_2204_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1749862860"
-  IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1749862860"
-  IMAGE_WIN_2016: "platform-ingest-elastic-agent-windows-2016-1749862860"
-  IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1749862860"
-  IMAGE_WIN_10: "platform-ingest-elastic-agent-windows-10-1749862860"
-  IMAGE_WIN_11: "platform-ingest-elastic-agent-windows-11-1749862860"
+  IMAGE_UBUNTU_2204_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1750467641"
+  IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1750467641"
+  IMAGE_WIN_2016: "platform-ingest-elastic-agent-windows-2016-1750467641"
+  IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1750467641"
+  IMAGE_WIN_10: "platform-ingest-elastic-agent-windows-10-1750467641"
+  IMAGE_WIN_11: "platform-ingest-elastic-agent-windows-11-1750467641"
 
 steps:
   - label: "check-ci"

.buildkite/scripts/steps/integration_tests_tf.sh

@@ -36,7 +36,16 @@ if [[ "${BUILDKITE_RETRY_COUNT}" -gt 0 ]]; then
   echo "~~~ The steps is retried, starting the ESS stack again"
   trap 'ess_down' EXIT
   ess_up $OVERRIDE_STACK_VERSION || (echo -e "^^^ +++\nFailed to start ESS stack")
-  preinstall_fleet_packages
+else
+  # For the first run, we start the stack in the start_ess.sh step and it sets the meta-data
+  echo "~~~ Receiving ESS stack metadata"
+  export ELASTICSEARCH_HOST=$(buildkite-agent meta-data get "es.host")
+  export ELASTICSEARCH_USERNAME=$(buildkite-agent meta-data get "es.username")
+  export ELASTICSEARCH_PASSWORD=$(buildkite-agent meta-data get "es.pwd")
+  export KIBANA_HOST=$(buildkite-agent meta-data get "kibana.host")
+  export KIBANA_USERNAME=$(buildkite-agent meta-data get "kibana.username")
+  export KIBANA_PASSWORD=$(buildkite-agent meta-data get "kibana.pwd")
+  export INTEGRATIONS_SERVER_HOST=$(buildkite-agent meta-data get "integrations_server.host")
 fi
 
 # Run integration tests

.ci/updatecli/updatecli-bump-vm-images.yml

@@ -26,6 +26,7 @@ actions:
       labels:
         - dependencies
         - backport-skip
+        - skip-changelog
       title: '[{{ requiredEnv "BRANCH_NAME" }}][Automation] Bump VM Image version to {{ source "latestVersion" }}'
 
 sources:

Makefile

@@ -41,6 +41,7 @@ check-ci:
 	@mage -v helm:lint
 	@mage -v helm:updateAgentVersion
 	@mage -v helm:renderExamples
+	@mage -v integration:buildKubernetesTestData
 	@$(MAKE) check-no-changes
 
 ## check: run all the checks including linting using golangci-lint.