[test] split up k8s integration tests

.buildkite/scripts/buildkite-k8s-integration-tests.sh

@@ -99,10 +99,14 @@ EOF
   pod_logs_base="${PWD}/build/${fully_qualified_group_name}.pod_logs_dump"
 
   set +e
-  K8S_TESTS_POD_LOGS_BASE="${pod_logs_base}" AGENT_IMAGE="${image}" DOCKER_VARIANT="${variant}" gotestsum --hide-summary=skipped --format testname --no-color -f standard-quiet --junitfile-hide-skipped-tests --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- -tags kubernetes,integration -test.shuffle on -test.timeout 2h0m0s github.com/elastic/elastic-agent/testing/integration -v -args -integration.groups="${group_name}" -integration.sudo="false"
+  K8S_TESTS_POD_LOGS_BASE="${pod_logs_base}" AGENT_IMAGE="${image}" DOCKER_VARIANT="${variant}" gotestsum --hide-summary=skipped --format testname --no-color -f standard-quiet --junitfile-hide-skipped-tests --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- -tags kubernetes,integration -test.shuffle on -test.timeout 2h0m0s github.com/elastic/elastic-agent/testing/integration/k8s -v -args -integration.groups="${group_name}" -integration.sudo="false"
   exit_status=$?
   set -e
 
+  if [[ $exit_status -ne 0 ]]; then
+     echo "^^^ +++"
+  fi
+
   if [[ $TESTS_EXIT_STATUS -eq 0 && $exit_status -ne 0 ]]; then
     TESTS_EXIT_STATUS=$exit_status
   fi

.buildkite/scripts/steps/k8s-extended-tests.sh

@@ -26,7 +26,7 @@ else
 fi
 
 SNAPSHOT=true EXTERNAL=true PACKAGES=docker mage -v package
-TEST_INTEG_CLEAN_ON_EXIT=true INSTANCE_PROVISIONER=kind STACK_PROVISIONER=stateful SNAPSHOT=true mage integration:kubernetesMatrix
+TEST_INTEG_CLEAN_ON_EXIT=true INSTANCE_PROVISIONER=kind STACK_PROVISIONER=stateful SNAPSHOT=true mage integration:testKubernetesMatrix
 TESTS_EXIT_STATUS=$?
 set -e
 

docs/test-framework-dev-guide.md

@@ -65,9 +65,9 @@ The test are run with mage using the `integration` namespace:
 
 - `mage integration:matrix` to run all tests on the complete matrix of supported operating systems and architectures of the Elastic Agent.
 
-- `mage integration:kubernetes` to run kubernetes tests for the default image on the default version of kubernetes (all previous commands will not run any kubernetes tests).
+- `mage integration:testKubernetes` to run kubernetes tests for the default image on the default version of kubernetes (all previous commands will not run any kubernetes tests).
 
-- `mage integration:kubernetesMatrix` to run a matrix of kubernetes tests for all image types and supported versions of kubernetes.
+- `mage integration:testKubernetesMatrix` to run a matrix of kubernetes tests for all image types and supported versions of kubernetes.
 
 #### Selecting specific platform
 
@@ -83,7 +83,7 @@ between, and it can be very specific or not very specific.
 - `TEST_PLATFORMS="linux/amd64/ubuntu/20.04 mage integration:test` to execute tests only on Ubuntu 20.04 ARM64.
 - `TEST_PLATFORMS="windows/amd64/2022 mage integration:test` to execute tests only on Windows Server 2022.
 - `TEST_PLATFORMS="linux/amd64 windows/amd64/2022 mage integration:test` to execute tests on Linux AMD64 and Windows Server 2022.
-- `TEST_PLATFORMS="kubernetes/arm64/1.33.0/wolfi" mage integration:kubernetes` to execute kubernetes tests on Kubernetes version 1.33.0 with wolfi docker variant.
+- `INSTANCE_PROVISIONER="kind" TEST_PLATFORMS="kubernetes/arm64/1.33.0/wolfi" mage integration:testKubernetes` to execute kubernetes tests on Kubernetes version 1.33.0 with wolfi docker variant under kind cluster.
 
 > [!NOTE]
 > This only filters down the tests based on the platform. It will not execute a tests on a platform unless
@@ -411,7 +411,7 @@ not cause already provisioned resources to be replaced with an instance created
 ### Kind Instance Provisioner
 Use only when running Kubernetes tests. Uses local installed kind to create Kubernetes clusters on the fly.
 
-- `INSTANCE_PROVISIONER="kind" mage integration:kubernetes`
+- `INSTANCE_PROVISIONER="kind" mage integration:testKubernetes`
 
 ## Troubleshooting Tips
 

go.mod

@@ -103,7 +103,6 @@ require (
 	kernel.org/pub/linux/libs/security/libcap/cap v1.2.70
 	sigs.k8s.io/e2e-framework v0.4.0
 	sigs.k8s.io/kustomize/api v0.18.0
-	sigs.k8s.io/kustomize/kyaml v0.18.1
 )
 
 require (
@@ -688,6 +687,7 @@ require (
 	oras.land/oras-go v1.2.5 // indirect
 	sigs.k8s.io/controller-runtime v0.20.4 // indirect
 	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.18.1 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.3 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )

magefile.go

@@ -2261,6 +2261,7 @@ func (Integration) Check() error {
 		define.ValidateDir("testing/integration"),
 		define.ValidateDir("testing/integration/serverless"),
 		define.ValidateDir("testing/integration/leak"),
+		define.ValidateDir("testing/integration/k8s"),
 	)
 }
 
@@ -2341,34 +2342,34 @@ func (Integration) TestServerless(ctx context.Context) error {
 	return integRunner(ctx, "testing/integration/serverless", false, "")
 }
 
-// Kubernetes runs kubernetes integration tests
-func (Integration) Kubernetes(ctx context.Context) error {
+// TestKubernetes runs kubernetes integration tests
+func (Integration) TestKubernetes(ctx context.Context) error {
 	// invoke integration tests
 	if err := os.Setenv("TEST_GROUPS", "kubernetes"); err != nil {
 		return err
 	}
 
-	return integRunner(ctx, "testing/integration", false, "")
+	return integRunner(ctx, "testing/integration/k8s", false, "")
 }
 
-// KubernetesSingle runs a single Kubernetes integration test
-func (Integration) KubernetesSingle(ctx context.Context, testName string) error {
+// TestKubernetesSingle runs single k8s integration test
+func (Integration) TestKubernetesSingle(ctx context.Context, testName string) error {
 	// invoke integration tests
 	if err := os.Setenv("TEST_GROUPS", "kubernetes"); err != nil {
 		return err
 	}
 
-	return integRunner(ctx, "testing/integration", false, testName)
+	return integRunner(ctx, "testing/integration/k8s", false, testName)
 }
 
-// KubernetesMatrix runs a matrix of kubernetes integration tests
-func (Integration) KubernetesMatrix(ctx context.Context) error {
+// TestKubernetesMatrix runs a matrix of kubernetes integration tests
+func (Integration) TestKubernetesMatrix(ctx context.Context) error {
 	// invoke integration tests
 	if err := os.Setenv("TEST_GROUPS", "kubernetes"); err != nil {
 		return err
 	}
 
-	return integRunner(ctx, "testing/integration", true, "")
+	return integRunner(ctx, "testing/integration/k8s", true, "")
 }
 
 // UpdateVersions runs an update on the `.agent-versions.yml` fetching

pkg/testing/define/define.go

@@ -39,6 +39,7 @@ var osInfo *types.OSInfo
 var osInfoErr error
 var osInfoOnce sync.Once
 var noSpecialCharsRegexp = regexp.MustCompile("[^a-zA-Z0-9]+")
+var kubernetesSupported = false
 
 // Require defines what this test requires for it to be run by the test runner.
 //
@@ -48,6 +49,12 @@ func Require(t *testing.T, req Requirements) *Info {
 	return defineAction(t, req)
 }
 
+// SetKubernetesSupported sets the kubernetesSupported flag to true
+// to allow kubernetes tests to be run.
+func SetKubernetesSupported() {
+	kubernetesSupported = true
+}
+
 type Info struct {
 	// ESClient is the elasticsearch client to communicate with elasticsearch.
 	// This is only present if you say a cloud is required in the `define.Require`.
@@ -139,7 +146,7 @@ func findProjectRoot() (string, error) {
 	}
 }
 
-func runOrSkip(t *testing.T, req Requirements, local bool, kubernetes bool) *Info {
+func runOrSkip(t *testing.T, req Requirements, local bool) *Info {
 	// always validate requirement is valid
 	if err := req.Validate(); err != nil {
 		panic(fmt.Sprintf("test %s has invalid requirements: %s", t.Name(), err))
@@ -169,7 +176,7 @@ func runOrSkip(t *testing.T, req Requirements, local bool, kubernetes bool) *Inf
 		return nil
 	}
 	for _, o := range req.OS {
-		if o.Type == Kubernetes && !kubernetes {
+		if o.Type == Kubernetes && !kubernetesSupported {
 			t.Skip("test requires kubernetes")
 			return nil
 		}

pkg/testing/define/define_all.go

@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License 2.0;
 // you may not use this file except in compliance with the Elastic License 2.0.
 
-//go:build !define && !local && !kubernetes
+//go:build !define && !local
 
 package define
 
@@ -11,5 +11,5 @@ import (
 )
 
 func defineAction(t *testing.T, req Requirements) *Info {
-	return runOrSkip(t, req, false, false)
+	return runOrSkip(t, req, false)
 }

pkg/testing/define/define_local.go

@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License 2.0;
 // you may not use this file except in compliance with the Elastic License 2.0.
 
-//go:build local && !define && !kubernetes
+//go:build local && !define
 
 package define
 
@@ -11,5 +11,5 @@ import (
 )
 
 func defineAction(t *testing.T, req Requirements) *Info {
-	return runOrSkip(t, req, true, false)
+	return runOrSkip(t, req, true)
 }

testing/integration/beat_receivers_test.go

@@ -477,7 +477,7 @@ outputs:
     api_key: {{.BeatsESApiKey}}
 `
 
-	esEndpoint, err := getESHost()
+	esEndpoint, err := GetESHost()
 	require.NoError(t, err, "error getting elasticsearch endpoint")
 	esApiKey, err := createESApiKey(info.ESClient)
 	require.NoError(t, err, "error creating API key")

testing/integration/common.go

@@ -0,0 +1,53 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+//go:build integration
+
+package integration
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/elastic-agent-libs/testing/estools"
+)
+
+func GetESHost() (string, error) {
+	fixedESHost := os.Getenv("ELASTICSEARCH_HOST")
+	parsedES, err := url.Parse(fixedESHost)
+	if err != nil {
+		return "", err
+	}
+	if parsedES.Port() == "" {
+		fixedESHost = fmt.Sprintf("%s:443", fixedESHost)
+	}
+	return fixedESHost, nil
+}
+
+// FindESDocs runs `findFn` until at least one document is returned and there is no error
+func FindESDocs(t *testing.T, findFn func() (estools.Documents, error)) estools.Documents {
+	var docs estools.Documents
+	require.Eventually(
+		t,
+		func() bool {
+			var err error
+			docs, err = findFn()
+			if err != nil {
+				t.Logf("got an error querying ES, retrying. Error: %s", err)
+				return false
+			}
+
+			return docs.Hits.Total.Value != 0
+		},
+		3*time.Minute,
+		15*time.Second,
+	)
+
+	return docs
+}