[8.17] (backport #7931) [ci] migrate extended runtime leak tests to buildkite (#8307)

* ci: migrate extended runtime leak tests to buildkite (#7931)

(cherry picked from commit f108618)

# Conflicts:
#	.buildkite/scripts/buildkite-integration-tests.sh
#	.buildkite/scripts/integration-tests.ps1

* fix: resolve conflicts

* fix: remove --junitfile-hide-skipped-tests from gotestsum

---------

Co-authored-by: Panos Koutsovasilis <[email protected]>

Author: mergify[bot]

.buildkite/bk.integration.pipeline.yml

@@ -28,6 +28,65 @@ steps:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
       useCustomGlobalHooks: true
 
+  - group: "Extended runtime leak tests"
+    key: extended-integration-tests
+    depends_on:
+      - integration-ess
+    steps:
+      - label: "Windows:2022:amd64:sudo"
+        depends_on:
+          - packaging-windows
+        env:
+          TEST_LONG_RUNNING: "true"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
+          .buildkite/scripts/integration-tests.ps1 fleet true TestLongRunningAgentForLeaks
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          image: "${IMAGE_WIN_2022}"
+        retry:
+          automatic:
+            limit: 1
+      - label: "Windows:2025:amd64:sudo"
+        depends_on:
+          - packaging-windows
+        env:
+          TEST_LONG_RUNNING: "true"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
+          .buildkite/scripts/integration-tests.ps1 fleet true TestLongRunningAgentForLeaks
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        retry:
+          automatic:
+            limit: 1
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          image: "${IMAGE_WIN_2025}"
+      - label: "Ubuntu:2404:amd64:sudo"
+        depends_on: packaging-ubuntu-x86-64
+        env:
+          TEST_LONG_RUNNING: "true"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
+          .buildkite/scripts/steps/integration_tests_tf.sh fleet true TestLongRunningAgentForLeaks
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        retry:
+          automatic:
+            limit: 1
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          image: "${IMAGE_UBUNTU_2404_X86_64}"
+
   - group: "Stateful: Windows"
     key: integration-tests-win
     depends_on:
@@ -352,6 +411,7 @@ steps:
       - integration-tests-win
       - integration-tests-rhel8
       - integration-tests-kubernetes
+      - extended-integration-tests
     allow_dependency_failure: true
     command: |
       buildkite-agent artifact download "test_infra/ess/**" . --step "integration-ess"

.buildkite/integration.pipeline.yml

@@ -110,30 +110,6 @@ steps:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
 
-  - label: "Extended runtime leak tests"
-    key: "extended-integration-tests"
-    depends_on:
-      - int-packaging
-    concurrency_group: elastic-agent-extended-testing/leak-tests
-    concurrency: 8
-    env:
-      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
-    command: |
-      buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
-      .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n2-standard-8"
-    retry:
-      automatic:
-        limit: 1
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests"
-
   - label: "Triggering Integration tests"
     depends_on:
       - int-packaging

.buildkite/scripts/buildkite-integration-tests.sh

@@ -3,6 +3,7 @@
 
 GROUP_NAME=$1
 TEST_SUDO=$2
+TEST_NAME_PATTERN=${3:-""}
 
 if [ -z "$GROUP_NAME" ]; then
   echo "Error: Specify the group name: sudo-integration-tests.sh [group_name]" >&2
@@ -46,9 +47,15 @@ outputXML="build/${fully_qualified_group_name}.integration.xml"
 outputJSON="build/${fully_qualified_group_name}.integration.out.json"
 
 echo "~~~ Integration tests: ${GROUP_NAME}"
+GOTEST_ARGS=(-tags integration -test.shuffle on -test.timeout 2h0m0s)
+if [ -n "$TEST_NAME_PATTERN" ]; then
+  GOTEST_ARGS+=(-run="${TEST_NAME_PATTERN}")
+fi
+GOTEST_ARGS+=("github.com/elastic/elastic-agent/testing/integration" -v -args "-integration.groups=${GROUP_NAME}" "-integration.sudo=${TEST_SUDO}")
 
 set +e
-TEST_BINARY_NAME="elastic-agent" AGENT_VERSION="${AGENT_VERSION}" SNAPSHOT=true gotestsum --no-color -f standard-quiet --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- -tags integration -test.shuffle on -test.timeout 2h0m0s github.com/elastic/elastic-agent/testing/integration -v -args -integration.groups="${GROUP_NAME}" -integration.sudo="${TEST_SUDO}"
+TEST_BINARY_NAME="elastic-agent" AGENT_VERSION="${AGENT_VERSION}" SNAPSHOT=true \
+  gotestsum --no-color -f standard-quiet --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- "${GOTEST_ARGS[@]}"
 TESTS_EXIT_STATUS=$?
 set -e
 

.buildkite/scripts/integration-tests.ps1

@@ -1,6 +1,7 @@
 param (
     [string]$GROUP_NAME,
-    [string]$TEST_SUDO
+    [string]$TEST_SUDO,
+    [string]$TEST_NAME_PATTERN = ""
 )
 
 echo "~~~ Preparing environment"
@@ -42,8 +43,13 @@ $TestsExitCode = 0
 try {
     Get-Ess-Stack -StackVersion $PACKAGE_VERSION
     Write-Output "~~~ Running integration test group: $GROUP_NAME as user: $env:USERNAME"
-    & gotestsum --no-color -f standard-quiet --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- -tags=integration -shuffle=on -timeout=2h0m0s "github.com/elastic/elastic-agent/testing/integration" -v -args "-integration.groups=$GROUP_NAME" "-integration.sudo=$TEST_SUDO"
-    $TestsExitCode = $LASTEXITCODE    
+    $gotestArgs = @("-tags=integration", "-shuffle=on", "-timeout=2h0m0s")
+    if ($TEST_NAME_PATTERN -ne "") {
+        $gotestArgs += "-run=${TEST_NAME_PATTERN}"
+    }
+    $gotestArgs += @("github.com/elastic/elastic-agent/testing/integration", "-v", "-args", "-integration.groups=$GROUP_NAME", "-integration.sudo=$TEST_SUDO")
+    & gotestsum --no-color -f standard-quiet --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- @gotestArgs
+    $TestsExitCode = $LASTEXITCODE
 } finally {
     ess_down
 

.buildkite/scripts/steps/integration_tests_tf.sh

@@ -11,6 +11,9 @@ asdf install
 
 GROUP_NAME=$1
 TEST_SUDO=$2
+# NOTE: This argument is not used in this script, but is declared to show that it can be set
+# and passed down to downstream scripts where it may be used.
+TEST_NAME_PATTERN=${3:-""}
 if [ -z "$GROUP_NAME" ]; then
   echo "Error: Specify the group name: integration_tests_tf.sh [group_name]" >&2
   exit 1