[8.17] (backport #8551) [test] split up ess and beats serverless integration tests (#8655)

* [test] split up ess and beats serverless integration tests (#8551)

* feat: split up ess and beats serverless integration tests

* feat: add the new integration test packages in the CI pipelines

* feat: migrate serveless_beats to beats/serverless package

(cherry picked from commit 73737c9)

# Conflicts:
#	.buildkite/bk.integration-fips.pipeline.yml
#	testing/integration/beats/serverless/beats_serverless_test.go
#	testing/integration/ess/auditd_monitoring_test.go
#	testing/integration/ess/beat_receivers_test.go
#	testing/integration/ess/endpoint_security_test.go
#	testing/integration/ess/enroll_replace_token_test.go
#	testing/integration/ess/fleetserver_fips_test.go
#	testing/integration/ess/fleetserver_test.go
#	testing/integration/ess/install_test.go
#	testing/integration/ess/linux_deb_test.go
#	testing/integration/ess/linux_rpm_test.go
#	testing/integration/ess/logs_ingestion_test.go
#	testing/integration/ess/metrics_monitoring_test.go
#	testing/integration/ess/network_traffic_monitoring_test.go
#	testing/integration/ess/osquery_monitoring_test.go
#	testing/integration/ess/otel_test.go
#	testing/integration/ess/restrict_upgrade_deb_test.go
#	testing/integration/ess/restrict_upgrade_rpm_test.go
#	testing/integration/ess/testdata/auditd_package.json
#	testing/integration/ess/testdata/fleet-server.json
#	testing/integration/ess/testdata/network_traffic_package.json
#	testing/integration/ess/testdata/osquery_package.json
#	testing/integration/ess/upgrade_fleet_test.go
#	testing/integration/ess/upgrade_standalone_flavors_test.go
#	testing/integration/ess/validate_items_deprecated_list_test.go
#	testing/integration/fleet-server.json
#	testing/integration/testdata/fleet-server.json

* fix: resolve conflicts

* fix: mage fmt

* fix: move fleet-server.json

---------

Co-authored-by: Panos Koutsovasilis <[email protected]>

Author: mergify[bot]

.buildkite/bk.integration.pipeline.yml

@@ -132,6 +132,8 @@ steps:
       - label: "Win2022:sudo:{{matrix}}"
         depends_on:
           - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} true
@@ -157,6 +159,8 @@ steps:
       - label: "Win2022:non-sudo:{{matrix}}"
         depends_on:
           - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} false
@@ -176,6 +180,8 @@ steps:
       - label: "Win2025:sudo:{{matrix}}"
         depends_on:
           - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} true
@@ -202,6 +208,8 @@ steps:
       - label: "Win2025:non-sudo:{{matrix}}"
         depends_on:
           - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} false
@@ -228,6 +236,8 @@ steps:
     steps:
       - label: "x86_64:non-sudo: {{matrix}}"
         depends_on: packaging-ubuntu-x86-64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
@@ -248,6 +258,8 @@ steps:
         depends_on:
           - packaging-ubuntu-x86-64
         # due to deb group present in matrix tar.gz and deb packages artifacts are required
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
@@ -279,6 +291,8 @@ steps:
       - label: "arm:sudo: {{matrix}}"
         depends_on:
           - packaging-ubuntu-arm64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
@@ -312,6 +326,8 @@ steps:
         skip: true
         depends_on:
           - packaging-ubuntu-arm64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
@@ -338,6 +354,8 @@ steps:
     steps:
       - label: "x86_64:non-sudo: {{matrix}}"
         depends_on: packaging-ubuntu-x86-64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
@@ -358,6 +376,8 @@ steps:
         depends_on:
           - packaging-ubuntu-x86-64
         # due to deb group present in matrix tar.gz and deb packages artifacts are required
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
@@ -399,6 +419,8 @@ steps:
       - label: "x86_64:sudo:rpm"
         depends_on:
           - packaging-ubuntu-x86-64
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
         command: |
           buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64
           .buildkite/scripts/steps/integration_tests_tf.sh rpm true

magefile.go

@@ -1920,8 +1920,9 @@ func (Integration) Clean() error {
 func (Integration) Check() error {
 	fmt.Println(">> check: Checking for define.Require in integration tests") // nolint:forbidigo // it's ok to use fmt.println in mage
 	return errors.Join(
-		define.ValidateDir("testing/integration"),
+		define.ValidateDir("testing/integration/ess"),
 		define.ValidateDir("testing/integration/serverless"),
+		define.ValidateDir("testing/integration/beats/serverless"),
 		define.ValidateDir("testing/integration/leak"),
 		define.ValidateDir("testing/integration/k8s"),
 	)
@@ -1981,17 +1982,17 @@ func (Integration) Auth(ctx context.Context) error {
 
 // Test runs integration tests on remote hosts
 func (Integration) Test(ctx context.Context) error {
-	return integRunner(ctx, "testing/integration", false, "")
+	return integRunner(ctx, "testing/integration/ess", false, "")
 }
 
 // Matrix runs integration tests on a matrix of all supported remote hosts
 func (Integration) Matrix(ctx context.Context) error {
-	return integRunner(ctx, "testing/integration", true, "")
+	return integRunner(ctx, "testing/integration/ess", true, "")
 }
 
 // Single runs single integration test on remote host
 func (Integration) Single(ctx context.Context, testName string) error {
-	return integRunner(ctx, "testing/integration", false, testName)
+	return integRunner(ctx, "testing/integration/ess", false, testName)
 }
 
 // TestServerless runs the integration tests defined in testing/integration/serverless
@@ -2546,7 +2547,7 @@ func (Integration) TestBeatServerless(ctx context.Context, beatname string) erro
 	if err != nil {
 		return fmt.Errorf("error setting binary name: %w", err)
 	}
-	return integRunner(ctx, "testing/integration", false, "TestBeatsServerless")
+	return integRunner(ctx, "testing/integration/beats/serverless", false, "TestBeatsServerless")
 }
 
 // TestForResourceLeaks runs tests that check for resource leaks
@@ -2629,7 +2630,7 @@ func (Integration) TestOnRemote(ctx context.Context) error {
 
 func (Integration) Buildkite() error {
 	goTestFlags := os.Getenv("GOTEST_FLAGS")
-	batches, err := define.DetermineBatches("testing/integration", goTestFlags, "integration")
+	batches, err := define.DetermineBatches("testing/integration/ess", goTestFlags, "integration")
 	if err != nil {
 		return fmt.Errorf("failed to determine batches: %w", err)
 	}

testing/integration/beats_serverless_test.go renamed to testing/integration/beats/serverless/beats_serverless_test.go

@@ -4,7 +4,7 @@
 
 //go:build integration
 
-package integration
+package serverless
 
 import (
 	"bytes"
@@ -29,6 +29,7 @@ import (
 	"github.com/elastic/elastic-agent/pkg/testing/define"
 	"github.com/elastic/elastic-agent/pkg/testing/tools"
 	"github.com/elastic/elastic-agent/pkg/testing/tools/estools"
+	"github.com/elastic/elastic-agent/testing/integration"
 )
 
 type BeatRunner struct {
@@ -50,7 +51,7 @@ type BeatRunner struct {
 
 func TestBeatsServerless(t *testing.T) {
 	info := define.Require(t, define.Requirements{
-		Group: Default,
+		Group: integration.Default,
 		OS: []define.OS{
 			{Type: define.Linux},
 		},

testing/integration/beats/serverless/main_test.go

@@ -0,0 +1,46 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+//go:build integration
+
+package serverless
+
+import (
+	"flag"
+	"log"
+	"os"
+	"testing"
+
+	"github.com/elastic/elastic-agent/pkg/testing/define"
+)
+
+var flagSet = flag.CommandLine
+
+func init() {
+	define.RegisterFlags("integration.", flagSet)
+}
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	if define.AutoDiscover {
+		define.InitAutodiscovery(nil)
+	}
+
+	runExitCode := m.Run()
+
+	if define.AutoDiscover {
+		discoveredTests, err := define.DumpAutodiscoveryYAML()
+		if err != nil {
+			log.Fatalf("Error dumping autodiscovery YAML: %v\n", err)
+		}
+
+		err = os.WriteFile(define.AutoDiscoveryOutput, discoveredTests, 0644)
+		if err != nil {
+			log.Fatalf("Error writing autodiscovery data in %q: %v\n", define.AutoDiscoveryOutput, err)
+		}
+	}
+
+	os.Exit(runExitCode)
+}

testing/integration/logs_ingestion.go renamed to testing/integration/common_logs_ingestion.go

@@ -102,19 +102,19 @@ func LogIngestionFleetManaged(t *testing.T, info *define.Info) {
 	checkHealthAtStartup(t, ctx, agentFixture)
 
 	t.Run("Monitoring logs are shipped", func(t *testing.T) {
-		testMonitoringLogsAreShipped(t, ctx, info, agentFixture, policy)
+		TestMonitoringLogsAreShipped(t, ctx, info, agentFixture, policy)
 	})
 
 	t.Run("Normal logs with flattened data_stream are shipped", func(t *testing.T) {
-		testFlattenedDatastreamFleetPolicy(t, ctx, info, policy)
+		TestFlattenedDatastreamFleetPolicy(t, ctx, info, policy)
 	})
 }
 
-// startMockES starts a MockES on a random port using httptest.NewServer.
+// StartMockES starts a MockES on a random port using httptest.NewServer.
 // It registers a cleanup function to close the server when the test finishes.
 // The server will respond with the passed error probabilities. If they add
 // up to zero, all requests are a success.
-func startMockES(t *testing.T, percentDuplicate, percentTooMany, percentNonIndex, percentTooLarge uint) string {
+func StartMockES(t *testing.T, percentDuplicate, percentTooMany, percentNonIndex, percentTooLarge uint) string {
 	uid := uuid.Must(uuid.NewV4())
 	clusterUUID := uuid.Must(uuid.NewV4()).String()
 
@@ -161,7 +161,7 @@ func checkHealthAtStartup(t *testing.T, ctx context.Context, agentFixture *atest
 	}, 3*time.Minute, 3*time.Second, "install never became healthy: components did not return a healthy state: %s", compDebugName)
 }
 
-func testMonitoringLogsAreShipped(
+func TestMonitoringLogsAreShipped(
 	t *testing.T,
 	ctx context.Context,
 	info *define.Info,
@@ -299,7 +299,7 @@ func queryESDocs(t *testing.T, findFn func() (estools.Documents, error)) estools
 	return docs
 }
 
-func testFlattenedDatastreamFleetPolicy(
+func TestFlattenedDatastreamFleetPolicy(
 	t *testing.T,
 	ctx context.Context,
 	info *define.Info,
@@ -322,18 +322,18 @@ func testFlattenedDatastreamFleetPolicy(
 		t.Fatalf("failed to chmod temp directory %s: %s", tempDir, err)
 	}
 	logFilePath := filepath.Join(tempDir, "log.log")
-	generateLogFile(t, logFilePath, 2*time.Millisecond, numEvents)
+	GenerateLogFile(t, logFilePath, 2*time.Millisecond, numEvents)
 
 	// 1. Prepare a request to add an integration to the policy
-	tmpl, err := template.New(t.Name() + "custom-log-policy").Parse(policyJSON)
+	tmpl, err := template.New(t.Name() + "custom-log-policy").Parse(PolicyJSON)
 	if err != nil {
 		t.Fatalf("cannot parse template: %s", err)
 	}
 
 	// The time here ensures there are no conflicts with the integration name
 	// in Fleet.
 	agentPolicyBuilder := strings.Builder{}
-	err = tmpl.Execute(&agentPolicyBuilder, policyVars{
+	err = tmpl.Execute(&agentPolicyBuilder, PolicyVars{
 		Name:              "Log-Input-" + t.Name() + "-" + time.Now().Format(time.RFC3339),
 		PolicyID:          policy.ID,
 		LogFilePath:       logFilePath,
@@ -413,10 +413,10 @@ func ensureDocumentsInES(
 	return f
 }
 
-// generateLogFile generates a log file by appending new lines every tick
+// GenerateLogFile generates a log file by appending new lines every tick
 // the lines are composed by the test name and the current time in RFC3339Nano
 // This function spans a new goroutine and does not block
-func generateLogFile(t *testing.T, fullPath string, tick time.Duration, events int) {
+func GenerateLogFile(t *testing.T, fullPath string, tick time.Duration, events int) {
 	t.Helper()
 	f, err := os.Create(fullPath)
 	if err != nil {
@@ -473,7 +473,7 @@ func cleanString(s string) string {
 
 var nonAlphanumericRegex = regexp.MustCompile(`[^a-zA-Z0-9 ]+`)
 
-var policyJSON = `
+var PolicyJSON = `
 {
   "policy_id": "{{.PolicyID}}",
   "package": {
@@ -500,7 +500,7 @@ var policyJSON = `
   }
 }`
 
-type policyVars struct {
+type PolicyVars struct {
 	Name              string
 	PolicyID          string
 	LogFilePath       string

testing/integration/apm_propagation_test.go renamed to testing/integration/ess/apm_propagation_test.go

@@ -4,7 +4,7 @@
 
 //go:build integration
 
-package integration
+package ess
 
 import (
 	"bytes"
@@ -27,6 +27,7 @@ import (
 	"github.com/elastic/elastic-agent/pkg/control/v2/client"
 	atesting "github.com/elastic/elastic-agent/pkg/testing"
 	"github.com/elastic/elastic-agent/pkg/testing/define"
+	"github.com/elastic/elastic-agent/testing/integration"
 )
 
 const agentConfigTemplateString = `
@@ -52,7 +53,7 @@ agent.monitoring:
 
 func TestAPMConfig(t *testing.T) {
 	info := define.Require(t, define.Requirements{
-		Group: Default,
+		Group: integration.Default,
 		Stack: &define.Stack{},
 	})
 	f, err := define.NewFixtureFromLocalBuild(t, define.Version())

testing/integration/container_cmd_test.go renamed to testing/integration/ess/container_cmd_test.go

@@ -4,7 +4,7 @@
 
 //go:build integration
 
-package integration
+package ess
 
 import (
 	"bufio"
@@ -31,6 +31,7 @@ import (
 	atesting "github.com/elastic/elastic-agent/pkg/testing"
 	"github.com/elastic/elastic-agent/pkg/testing/define"
 	"github.com/elastic/elastic-agent/pkg/testing/tools/fleettools"
+	"github.com/elastic/elastic-agent/testing/integration"
 )
 
 func createPolicy(
@@ -355,7 +356,7 @@ func TestContainerCMDEventToStderr(t *testing.T) {
 
 	cmd, agentOutput := prepareAgentCMD(t, ctx, agentFixture, []string{"container"}, env)
 	addLogIntegration(t, info, policyID, "/tmp/flog.log")
-	generateLogFile(t, "/tmp/flog.log", time.Second/2, 100)
+	integration.GenerateLogFile(t, "/tmp/flog.log", time.Second/2, 100)
 
 	t.Logf(">> running binary with: %v", cmd.Args)
 	if err := cmd.Start(); err != nil {
@@ -396,7 +397,7 @@ func TestContainerCMDEventToStderr(t *testing.T) {
 // The server will respond with the passed error probabilities. If they add
 // up to zero, all requests are a success.
 func createMockESOutput(t *testing.T, info *define.Info, percentDuplicate, percentTooMany, percentNonIndex, percentTooLarge uint) (string, string) {
-	mockesURL := startMockES(t, percentDuplicate, percentTooMany, percentNonIndex, percentTooLarge)
+	mockesURL := integration.StartMockES(t, percentDuplicate, percentTooMany, percentNonIndex, percentTooLarge)
 	createOutputBody := `
 {
   "id": "mock-es-%[1]s",
@@ -445,12 +446,12 @@ func createMockESOutput(t *testing.T, info *define.Info, percentDuplicate, perce
 
 func addLogIntegration(t *testing.T, info *define.Info, policyID, logFilePath string) {
 	agentPolicyBuilder := strings.Builder{}
-	tmpl, err := template.New(t.Name() + "custom-log-policy").Parse(policyJSON)
+	tmpl, err := template.New(t.Name() + "custom-log-policy").Parse(integration.PolicyJSON)
 	if err != nil {
 		t.Fatalf("cannot parse template: %s", err)
 	}
 
-	err = tmpl.Execute(&agentPolicyBuilder, policyVars{
+	err = tmpl.Execute(&agentPolicyBuilder, integration.PolicyVars{
 		Name:        "Log-Input-" + t.Name() + "-" + time.Now().Format(time.RFC3339),
 		PolicyID:    policyID,
 		LogFilePath: logFilePath,