e2e: add step at the end of the e2e tests to print all logs (with a high hard limit)

Signed-off-by: Peter Wilcsinszky <[email protected]>

Author: pepov

.github/workflows/e2e.yaml

@@ -58,6 +58,9 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
 
+      - name: Stern
+        run: make stern && test -x bin/stern
+
       - name: Build controller image
         run: make docker-build IMG='controller:local'
 
@@ -88,3 +91,6 @@ jobs:
 
       - name: Test
         run: hack/test.sh
+
+      - name: Print last 10k kubernetes logs from default and logging namespaces
+        run: bin/stern -n default,logging ".*" --tail 10000 --no-follow

Makefile

@@ -156,8 +156,8 @@ test: generate fmt vet manifests ${ENVTEST_BINARY_ASSETS} ${KUBEBUILDER} ## Run
 	ENVTEST_BINARY_ASSETS=${ENVTEST_BINARY_ASSETS} go test ./controllers/extensions/... ./pkg/... -coverprofile cover.out
 
 .PHONY: test-e2e
-test-e2e: ${KIND} docker-build generate fmt vet manifests ## Run E2E tests
-	cd e2e && LOGGING_OPERATOR_IMAGE="${IMG}" go test -timeout ${E2E_TEST_TIMEOUT} ./...
+test-e2e: ${KIND} docker-build generate fmt vet manifests stern ## Run E2E tests
+	cd e2e && LOGGING_OPERATOR_IMAGE="${IMG}" go test -v -timeout ${E2E_TEST_TIMEOUT} ./...
 
 .PHONY: tidy
 tidy: ## Tidy Go modules
@@ -229,6 +229,9 @@ ${SETUP_ENVTEST}: VERSION := latest
 ${SETUP_ENVTEST}: | ${BIN}
 	GOBIN=${BIN} go install ${IMPORT_PATH}@${VERSION}
 
+stern: | ${BIN}
+	GOBIN=${BIN} go install github.com/stern/stern@latest
+
 ${ENVTEST_BIN_DIR}: | ${BIN}
 	mkdir -p $@
 

e2e/common/cluster.go

@@ -16,10 +16,16 @@ package common
 
 import (
 	"context"
+	"os"
+	"os/exec"
+	"strings"
 	"testing"
 
 	"emperror.dev/errors"
+	"github.com/spf13/cast"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/controller-runtime/pkg/cluster"
 
@@ -28,9 +34,13 @@ import (
 
 const defaultClusterName = "e2e-test"
 
+var NamespacesUsed []string
+
 type Cluster interface {
 	cluster.Cluster
 	LoadImages(images ...string) error
+	Cleanup() error
+	PrintLogs(int) error
 }
 
 func WithCluster(t *testing.T, fn func(*testing.T, Cluster), opts ...cluster.Option) {
@@ -43,6 +53,10 @@ func WithCluster(t *testing.T, fn func(*testing.T, Cluster), opts ...cluster.Opt
 	}()
 
 	defer func() {
+		t.Log("CLUSTER LOGS START")
+		assert.NoError(t, cluster.PrintLogs(10000))
+		t.Log("CLUSTER LOGS END")
+		assert.NoError(t, cluster.Cleanup())
 		cancel()
 		require.NoError(t, DeleteTestCluster(defaultClusterName))
 	}()
@@ -51,22 +65,38 @@ func WithCluster(t *testing.T, fn func(*testing.T, Cluster), opts ...cluster.Opt
 }
 
 func GetTestCluster(clusterName string, opts ...cluster.Option) (Cluster, error) {
-	kubeconfig, err := KindClusterKubeconfig(clusterName)
+	var err error
+	var kubeconfig []byte
+	var kubeconfigFile *os.File
+	var clientCfg clientcmd.ClientConfig
+	var restCfg *rest.Config
+	var c cluster.Cluster
+
+	kubeconfig, err = KindClusterKubeconfig(clusterName)
 	if err != nil {
 		return nil, errors.WrapIfWithDetails(err, "getting kubeconfig of kind cluster", "clusterName", clusterName)
 	}
-	clientCfg, err := clientcmd.NewClientConfigFromBytes(kubeconfig)
+	kubeconfigFile, err = os.CreateTemp("", "kind-kind-kubeconfig")
+	if err != nil {
+		return nil, errors.WrapIfWithDetails(err, "unable to create temp file for kubeconfig", "clusterName", clusterName)
+	}
+	err = os.WriteFile(kubeconfigFile.Name(), kubeconfig, os.FileMode(0600))
+	if err != nil {
+		return nil, errors.WrapIfWithDetails(err, "failed to write kubeconfig", "clusterName", clusterName, "path", kubeconfigFile.Name())
+	}
+	clientCfg, err = clientcmd.NewClientConfigFromBytes(kubeconfig)
 	if err != nil {
 		return nil, errors.WrapIfWithDetails(err, "creating client config from kubeconfig bytes", "kubeconfig", kubeconfig)
 	}
-	restCfg, err := clientCfg.ClientConfig()
+	restCfg, err = clientCfg.ClientConfig()
 	if err != nil {
 		return nil, errors.WrapIfWithDetails(err, "creating rest config from client config", "cfg", clientCfg)
 	}
-	c, err := cluster.New(restCfg, opts...)
+	c, err = cluster.New(restCfg, opts...)
 	return &kindCluster{
-		Cluster:     c,
-		clusterName: clusterName,
+		Cluster:            c,
+		kubeconfigFilePath: kubeconfigFile.Name(),
+		clusterName:        clusterName,
 	}, errors.WrapIfWithDetails(err, "creating cluster with rest config", "cfg", restCfg)
 }
 
@@ -78,7 +108,19 @@ func DeleteTestCluster(clusterName string) error {
 
 type kindCluster struct {
 	cluster.Cluster
-	clusterName string
+	kubeconfigFilePath string
+	clusterName        string
+}
+
+func (c kindCluster) PrintLogs(tail int) error {
+	cmd := exec.Command("stern", "-n", strings.Join(NamespacesUsed, ","), ".*", "--no-follow", "--tail", cast.ToString(tail), "--kubeconfig", c.kubeconfigFilePath)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func (c kindCluster) Cleanup() error {
+	return os.Remove(c.kubeconfigFilePath)
 }
 
 func (c kindCluster) LoadImages(images ...string) error {

e2e/go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/cisco-open/operator-tools v0.29.0
 	github.com/go-logr/logr v1.2.4
 	github.com/kube-logging/logging-operator/pkg/sdk v0.9.1
+	github.com/spf13/cast v1.5.0
 	github.com/stretchr/testify v1.8.2
 	k8s.io/api v0.26.4
 	k8s.io/apiextensions-apiserver v0.26.4
@@ -56,7 +57,6 @@ require (
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
-	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/wayneashleyberry/terminal-dimensions v1.0.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect

e2e/volumedrain/volumedrain_test.go

@@ -22,8 +22,6 @@ import (
 	"time"
 
 	"github.com/cisco-open/operator-tools/pkg/utils"
-	"github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1"
-	"github.com/kube-logging/logging-operator/pkg/sdk/logging/model/output"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	appsv1 "k8s.io/api/apps/v1"
@@ -37,15 +35,18 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/cluster"
 
+	"github.com/kube-logging/logging-operator/pkg/sdk/logging/api/v1beta1"
+	"github.com/kube-logging/logging-operator/pkg/sdk/logging/model/output"
+
 	"github.com/kube-logging/logging-operator/e2e/common"
 	"github.com/kube-logging/logging-operator/e2e/common/cond"
 	"github.com/kube-logging/logging-operator/e2e/common/setup"
 )
 
 func TestVolumeDrain_Downscale(t *testing.T) {
+	ns := "testing-1"
+	common.NamespacesUsed = []string{ns, "default"}
 	common.WithCluster(t, func(t *testing.T, c common.Cluster) {
-		ns := "testing-1"
-
 		setup.LoggingOperator(t, c, setup.LoggingOperatorOptionFunc(func(options *setup.LoggingOperatorOptions) {
 			options.Config.DisableWebhook = true
 			options.Config.Namespace = ns
@@ -196,9 +197,9 @@ func TestVolumeDrain_Downscale(t *testing.T) {
 }
 
 func TestVolumeDrain_Downscale_DeleteVolume(t *testing.T) {
+	ns := "testing-2"
+	common.NamespacesUsed = []string{ns, "default"}
 	common.WithCluster(t, func(t *testing.T, c common.Cluster) {
-		ns := "testing-2"
-
 		setup.LoggingOperator(t, c, setup.LoggingOperatorOptionFunc(func(options *setup.LoggingOperatorOptions) {
 			options.Config.DisableWebhook = true
 			options.Config.Namespace = ns

hack/test.sh

@@ -16,8 +16,6 @@ function main()
     helm_deploy_logging_operator
     configure_logging
 
-
-
     wait_for_log_files "${mc_pod}" 300
     print_logs "${mc_pod}"
 }
@@ -53,7 +51,7 @@ function helm_add_repo()
 function helm_deploy_logging_operator()
 {
     # TODO: remove version once there is a semver stable version in the repo
-    helm install \
+    helm upgrade --install \
         --debug \
         --wait \
         --set image.tag='local' \
@@ -66,7 +64,7 @@ function helm_deploy_logging_operator()
 function configure_logging()
 {
     # TODO: remove version once there is a semver stable version in the repo
-    helm install \
+    helm upgrade --install \
         --debug \
         --wait \
         --create-namespace \