Add integration test for shard controller (#490)

* Rework example shard controller

* Use `HaveLabel*` matcher in more places

* Add integration test for `shard` controller

* Deflake `sharder` webhook integration test

Author: timebertt

cmd/shard/reconciler.go

@@ -18,6 +18,8 @@ package main
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 
 	corev1 "k8s.io/api/core/v1"
@@ -37,7 +39,7 @@ import (
 	shardcontroller "github.com/timebertt/kubernetes-controller-sharding/pkg/shard/controller"
 )
 
-// Reconciler is a reconciler for ConfigMaps that creates a dummy secret for every ConfigMap.
+// Reconciler watches Secrets and creates a ConfigMap for every Secret containing the Secret data's checksums.
 // It handles the shard and drain label.
 type Reconciler struct {
 	Client client.Client
@@ -54,27 +56,27 @@ func (r *Reconciler) AddToManager(mgr manager.Manager, controllerRingName, shard
 	// - a predicate that triggers when the drain label is present (even if the actual predicates don't trigger)
 	// - wrapping the actual reconciler a reconciler that handles the drain operation for us
 	return builder.ControllerManagedBy(mgr).
-		Named("configmap").
-		For(&corev1.ConfigMap{}, builder.WithPredicates(shardcontroller.Predicate(controllerRingName, shardName, ConfigMapDataChanged(), predicate.GenerationChangedPredicate{}))).
-		Owns(&corev1.Secret{}, builder.WithPredicates(ObjectDeleted())).
+		Named("secret-checksums").
+		For(&corev1.Secret{}, builder.WithPredicates(shardcontroller.Predicate(controllerRingName, shardName, SecretDataChanged()))).
+		Owns(&corev1.ConfigMap{}, builder.WithPredicates(ObjectDeleted())).
 		WithOptions(controller.Options{
 			MaxConcurrentReconciles: 5,
 		}).
 		Complete(
 			shardcontroller.NewShardedReconciler(mgr).
-				For(&corev1.ConfigMap{}).
+				For(&corev1.Secret{}).
 				InControllerRing(controllerRingName).
 				WithShardName(shardName).
 				MustBuild(r),
 		)
 }
 
-// ConfigMapDataChanged returns a predicate that is similar to predicate.GenerationChangedPredicate but for ConfigMaps
+// SecretDataChanged returns a predicate that is similar to predicate.GenerationChangedPredicate but for Secrets
 // that don't have a metadata.generation field.
-func ConfigMapDataChanged() predicate.Predicate {
+func SecretDataChanged() predicate.Predicate {
 	return predicate.Funcs{
 		UpdateFunc: func(e event.UpdateEvent) bool {
-			return apiequality.Semantic.DeepEqual(e.ObjectOld.(*corev1.ConfigMap).Data, e.ObjectNew.(*corev1.ConfigMap).Data)
+			return apiequality.Semantic.DeepEqual(e.ObjectOld.(*corev1.Secret).Data, e.ObjectNew.(*corev1.Secret).Data)
 		},
 	}
 }
@@ -93,29 +95,36 @@ func ObjectDeleted() predicate.Predicate {
 func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
 	log := logf.FromContext(ctx)
 
-	configMap := &corev1.ConfigMap{}
-	if err := r.Client.Get(ctx, req.NamespacedName, configMap); err != nil {
+	secret := &corev1.Secret{}
+	if err := r.Client.Get(ctx, req.NamespacedName, secret); err != nil {
 		if apierrors.IsNotFound(err) {
 			log.V(1).Info("Object is gone, stop reconciling")
 			return reconcile.Result{}, nil
 		}
 		return reconcile.Result{}, fmt.Errorf("error retrieving object from store: %w", err)
 	}
 
-	// perform some dummy operation, this is just an example controller, we don't really care what it actually does
-	// create a secret with controller reference so that the controller also serves as an example with controlled objects
+	// Perform a typical operation in this example controller.
+	// Create a ConfigMap with a controller reference to the watched Secret.
 	log.V(1).Info("Reconciling object")
 
-	secret := &corev1.Secret{
+	configMap := &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "dummy-" + configMap.Name,
-			Namespace: configMap.Namespace,
+			Name:      "checksums-" + secret.Name,
+			Namespace: secret.Namespace,
 		},
+		Data: make(map[string]string, len(secret.Data)),
 	}
 
-	if err := controllerutil.SetControllerReference(configMap, secret, r.Client.Scheme()); err != nil {
+	// Calculate the checksum for every Secret key and populate it in the ConfigMap.
+	for key, data := range secret.Data {
+		checksum := sha256.Sum256(data)
+		configMap.Data[key] = hex.EncodeToString(checksum[:])
+	}
+
+	if err := controllerutil.SetControllerReference(secret, configMap, r.Client.Scheme()); err != nil {
 		return reconcile.Result{}, err
 	}
 
-	return reconcile.Result{}, client.IgnoreAlreadyExists(r.Client.Create(ctx, secret))
+	return reconcile.Result{}, client.IgnoreAlreadyExists(r.Client.Create(ctx, configMap))
 }

docs/design.md

@@ -32,7 +32,7 @@ The sharder webhook is called on `CREATE` and `UPDATE` requests for configured r
 The sharder uses the consistent hashing ring to determine the desired shard and adds the shard label during admission accordingly.
 Shards then use a label selector for the shard label with their own instance name to restrict the cache and controller to the subset of objects assigned to them.
 
-For the controller's "main" object (configured in `ControllerRing.spec.resources[]`), the object's `apiVersion`, `kind`, `namespace`, and `name` are concatenated to form its hash key.
+For the controller's "main" object (configured in `ControllerRing.spec.resources[]`), the object's API group, `kind`, `namespace`, and `name` are concatenated to form its hash key.
 For objects controlled by other objects (configured in `ControllerRing.spec.resources[].controlledResources[]`), the sharder utilizes information about the controlling object (`ownerReference` with `controller=true`) to calculate the object's hash key.
 This ensures that owned objects are consistently assigned to the same shard as their owner.
 

docs/development.md

@@ -94,7 +94,7 @@ You should see that the shard successfully announced itself to the sharder:
 ```bash
 $ kubectl get lease -L alpha.sharding.timebertt.dev/controllerring,alpha.sharding.timebertt.dev/state
 NAME             HOLDER           AGE   CONTROLLERRING   STATE
-shard-fkpxhjk8   shard-fkpxhjk8   18s   example          ready
+shard-5pv57c6c   shard-5pv57c6c   18s   example          ready
 
 $ kubectl get controllerring
 NAME      READY   AVAILABLE   SHARDS   AGE
@@ -113,19 +113,19 @@ make run-shard
 
 ## Testing the Sharding Setup
 
-Independent of the used setup (skaffold-based or running on the host machine), you should be able to create sharded `ConfigMaps` in the `default` namespace as configured in the `example` `ControllerRing`.
-The `Secrets` created by the example shard controller should be assigned to the same shard as the owning `ConfigMap`:
+Independent of the used setup (skaffold-based or running on the host machine), you should be able to create sharded `Secrets` in the `default` namespace as configured in the `example` `ControllerRing`.
+The `ConfigMaps` created by the example shard controller should be assigned to the same shard as the owning `Secret`:
 
 ```bash
-$ kubectl create cm foo
-configmap/foo created
+$ kubectl create secret generic foo --from-literal foo=bar
+secret/foo created
 
 $ kubectl get cm,secret -L shard.alpha.sharding.timebertt.dev/example
-NAME                         DATA   AGE     EXAMPLE
-configmap/foo                0      1s      shard-656d588475-5746d
+NAME                      DATA   AGE   EXAMPLE
+configmap/checksums-foo   1      1s    shard-5pv57c6c
 
-NAME                            TYPE     DATA   AGE     EXAMPLE
-secret/dummy-foo                Opaque   0      1s      shard-656d588475-5746d
+NAME         TYPE     DATA   AGE   EXAMPLE
+secret/foo   Opaque   1      1s    shard-5pv57c6c
 ```
 
 ## Monitoring