Rework existing e2e test setup (#516)

* Simplify komega usage in existing e2e tests

* Split into multiple ordered containers

* Retry secret creation

* Use dedicated namespaces for e2e tests

Author: timebertt

cmd/checksum-controller/main.go

@@ -80,6 +80,7 @@ This example sharded controller is also useful for developing the sharding compo
 type options struct {
 	zapOptions         *zap.Options
 	controllerRingName string
+	namespace          string
 	leaseNamespace     string
 	shardName          string
 }
@@ -92,11 +93,13 @@ func newOptions() *options {
 		},
 
 		controllerRingName: "checksum-controller",
+		namespace:          metav1.NamespaceDefault,
 	}
 }
 
 func (o *options) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&o.controllerRingName, "controllerring", o.controllerRingName, "Name of the ControllerRing the shard belongs to.")
+	fs.StringVar(&o.namespace, "namespace", o.namespace, "Namespace to watch objects in.")
 	fs.StringVar(&o.leaseNamespace, "lease-namespace", o.leaseNamespace, "Namespace to use for the shard lease. Defaults to the pod's namespace if running in-cluster.")
 	fs.StringVar(&o.shardName, "shard-name", o.shardName, "Name of the shard. Defaults to the instance's hostname.")
 
@@ -153,8 +156,8 @@ func (o *options) run(ctx context.Context) error {
 
 		// FILTERED WATCH CACHE
 		Cache: cache.Options{
-			// This controller only acts on objects in the default namespace.
-			DefaultNamespaces: map[string]cache.Config{metav1.NamespaceDefault: {}},
+			// This controller only acts on objects in a single configured namespace.
+			DefaultNamespaces: map[string]cache.Config{o.namespace: {}},
 			// Configure cache to only watch objects that are assigned to this shard.
 			// This controller only watches sharded objects, so we can configure the label selector on the cache's global level.
 			// If your controller watches sharded objects as well as non-sharded objects, use cache.Options.ByObject to configure

hack/config/checksum-controller/controller/deployment.yaml

@@ -13,7 +13,6 @@ spec:
       - name: checksum-controller
         image: checksum-controller:latest
         args:
-        - --controllerring=checksum-controller
         - --zap-log-level=debug
         env:
         - name: DISABLE_HTTP2

pkg/utils/test/matchers/condition.go

@@ -18,7 +18,6 @@ package matchers
 
 import (
 	. "github.com/onsi/gomega"
-	"github.com/onsi/gomega/gstruct"
 	gomegatypes "github.com/onsi/gomega/types"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -35,28 +34,20 @@ var MatchCondition = And
 
 // OfType returns a matcher for checking whether a condition has a certain type.
 func OfType(conditionType string) gomegatypes.GomegaMatcher {
-	return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
-		"Type": Equal(conditionType),
-	})
+	return HaveField("Type", Equal(conditionType))
 }
 
 // WithStatus returns a matcher for checking whether a condition has a certain status.
 func WithStatus(status metav1.ConditionStatus) gomegatypes.GomegaMatcher {
-	return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
-		"Status": Equal(status),
-	})
+	return HaveField("Status", Equal(status))
 }
 
 // WithReason returns a matcher for checking whether a condition has a certain reason.
 func WithReason(reason string) gomegatypes.GomegaMatcher {
-	return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
-		"Reason": Equal(reason),
-	})
+	return HaveField("Reason", Equal(reason))
 }
 
 // WithMessage returns a matcher for checking whether a condition has a certain message.
 func WithMessage(message string) gomegatypes.GomegaMatcher {
-	return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
-		"Message": ContainSubstring(message),
-	})
+	return HaveField("Message", ContainSubstring(message))
 }

test/e2e/e2e_suite_test.go

@@ -18,12 +18,19 @@ package e2e
 
 import (
 	"context"
+	"fmt"
+	"maps"
 	"testing"
 	"time"
 
 	"github.com/go-logr/logr"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -32,6 +39,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	shardingv1alpha1 "github.com/timebertt/kubernetes-controller-sharding/pkg/apis/sharding/v1alpha1"
+	"github.com/timebertt/kubernetes-controller-sharding/pkg/utils/test"
+	. "github.com/timebertt/kubernetes-controller-sharding/pkg/utils/test/matchers"
 )
 
 func TestE2E(t *testing.T) {
@@ -40,17 +49,23 @@ func TestE2E(t *testing.T) {
 }
 
 const (
-	ShortTimeout = 10 * time.Second
+	testID = "e2e-controller-sharding"
+
+	ShortTimeout  = 10 * time.Second
+	MediumTimeout = time.Minute
 )
 
 var (
 	log logr.Logger
 
 	testClient client.Client
+
+	testRunID     string
+	testRunLabels map[string]string
 )
 
 var _ = BeforeSuite(func() {
-	log = zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter))
+	log = zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter)).WithName(testID)
 
 	restConfig, err := config.GetConfig()
 	Expect(err).NotTo(HaveOccurred())
@@ -69,4 +84,87 @@ var _ = BeforeSuite(func() {
 	komega.SetClient(testClient)
 	komega.SetContext(clientContext)
 	DeferCleanup(clientCancel)
+
+	testRunID = testID + "-" + test.RandomSuffix()
+	testRunLabels = map[string]string{
+		testID: testRunID,
+	}
+	log = log.WithValues("testRun", testRunID)
 })
+
+var (
+	controllerRing *shardingv1alpha1.ControllerRing
+	namespace      *corev1.Namespace
+)
+
+var _ = BeforeEach(func(ctx SpecContext) {
+	By("Set up test Namespace")
+	namespace = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{
+		GenerateName: testRunID + "-",
+		Labels:       maps.Clone(testRunLabels),
+	}}
+
+	// We create a dedicated test namespace and clean it up for every test case to ensure a clean test environment.
+	Expect(testClient.Create(ctx, namespace)).To(Succeed())
+	log.Info("Created test Namespace", "namespace", namespace.Name)
+
+	DeferCleanup(func(ctx SpecContext) {
+		By("Delete test Namespace")
+		Eventually(ctx, func() error {
+			return testClient.Delete(ctx, namespace)
+		}).Should(Or(Succeed(), BeNotFoundError()))
+	}, NodeTimeout(MediumTimeout))
+
+	By("Set up test ControllerRing")
+	// Deploy a dedicated ControllerRing instance for this test case
+	defaultControllerRing := &shardingv1alpha1.ControllerRing{ObjectMeta: metav1.ObjectMeta{Name: "checksum-controller"}}
+	Expect(komega.Get(defaultControllerRing)()).To(Succeed())
+
+	controllerRing = defaultControllerRing.DeepCopy()
+	controllerRing.Name = namespace.Name
+	controllerRing.ResourceVersion = ""
+	maps.Copy(controllerRing.Labels, testRunLabels)
+	controllerRing.Spec.NamespaceSelector.MatchLabels[corev1.LabelMetadataName] = namespace.Name
+	Expect(testClient.Create(ctx, controllerRing)).To(Succeed())
+
+	DeferCleanup(func(ctx SpecContext) {
+		By("Delete test ControllerRing")
+		Eventually(ctx, func() error {
+			return testClient.Delete(ctx, controllerRing)
+		}).Should(Or(Succeed(), BeNotFoundError()))
+	}, NodeTimeout(ShortTimeout))
+
+	By("Set up test controller")
+	// Deploy a dedicated controller instance to this test case's namespace.
+	// Copy all relevant objects from the default namespace.
+	for _, objList := range []client.ObjectList{
+		&appsv1.DeploymentList{},
+		&corev1.ServiceAccountList{},
+		&rbacv1.RoleList{},
+		&rbacv1.RoleBindingList{},
+	} {
+		Expect(testClient.List(ctx, objList, client.InNamespace(metav1.NamespaceDefault), client.MatchingLabels{"app.kubernetes.io/component": "checksum-controller"})).
+			Should(Succeed(), "should list %T in default namespace", objList)
+
+		Expect(meta.EachListItem(objList, func(object runtime.Object) error {
+			obj := object.DeepCopyObject().(client.Object)
+			obj.SetNamespace(namespace.Name)
+			obj.SetResourceVersion("")
+
+			switch o := obj.(type) {
+			case *appsv1.Deployment:
+				o.Spec.Template.Spec.Containers[0].Args = append(o.Spec.Template.Spec.Containers[0].Args,
+					"--controllerring="+controllerRing.Name,
+					"--namespace="+namespace.Name,
+				)
+			case *rbacv1.RoleBinding:
+				o.Subjects[0].Namespace = namespace.Name
+			}
+
+			if err := testClient.Create(ctx, obj); err != nil {
+				return fmt.Errorf("error copying object %T %q to %s namespace: %w", obj, client.ObjectKeyFromObject(obj), namespace.Name, err)
+			}
+			return nil
+		})).To(Succeed(), "should copy %T", objList)
+	}
+}, NodeTimeout(MediumTimeout), OncePerOrdered)