feat(failuredomains): add failure domain rollout controller (#1207)

Adds a new controller that monitors cluster.status.failureDomains and
triggers rollouts on KubeAdmControlPlane when failure domains currently
in use by control plane machines are disabled or removed.

The controller is enabled by default and can be disabled with
--failure-domain-rollout-enabled=false.

**How has this been tested?**
* Create 4 failure domains: fd-1, fd-2, fd-3, fd-4
* Create a cluster with control plane on 3 failure domains fd-1, fd-2,
fd-3
* Remove fd-3 from the control plane configuration and observe a rollout
to fd-1 & fd-2 only
* Add fd-3 again and observe a rollout to fd-1, fd-2, and fd-3
* Add fd-4 to control plane configuration and observe no rollout takes
place

Author: thunderboltsid

charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml

@@ -75,10 +75,17 @@ rules:
       - cluster.x-k8s.io
     resources:
       - clusters
+      - machines
     verbs:
       - get
       - list
       - watch
+  - apiGroups:
+      - cluster.x-k8s.io
+    resources:
+      - clusters/status
+    verbs:
+      - get
   - apiGroups:
       - cluster.x-k8s.io
     resources:
@@ -89,6 +96,16 @@ rules:
       - patch
       - update
       - watch
+  - apiGroups:
+      - controlplane.cluster.x-k8s.io
+    resources:
+      - kubeadmcontrolplanes
+    verbs:
+      - get
+      - list
+      - patch
+      - update
+      - watch
   - apiGroups:
       - storage.k8s.io
     resources:

cmd/main.go

@@ -19,6 +19,7 @@ import (
 	"k8s.io/klog/v2"
 	crsv1 "sigs.k8s.io/cluster-api/api/addons/v1beta1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -33,6 +34,7 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/server"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/controllers/enforceclusterautoscalerlimits"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/controllers/failuredomainrollout"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/controllers/namespacesync"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/feature"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws"
@@ -56,6 +58,7 @@ func main() {
 	utilruntime.Must(clientgoscheme.AddToScheme(clientScheme))
 	utilruntime.Must(crsv1.AddToScheme(clientScheme))
 	utilruntime.Must(clusterv1.AddToScheme(clientScheme))
+	utilruntime.Must(controlplanev1.AddToScheme(clientScheme))
 	utilruntime.Must(caaphv1.AddToScheme(clientScheme))
 	utilruntime.Must(capxv1.AddToScheme(clientScheme))
 
@@ -123,6 +126,7 @@ func main() {
 
 	namespacesyncOptions := namespacesync.Options{}
 	enforceClusterAutoscalerLimitsOptions := enforceclusterautoscalerlimits.Options{}
+	failureDomainRolloutOptions := failuredomainrollout.Options{}
 
 	// Initialize and parse command line flags.
 	logs.AddFlags(pflag.CommandLine, logs.SkipLoggingConfigurationFlags())
@@ -135,6 +139,7 @@ func main() {
 	nutanixMetaHandlers.AddFlags(pflag.CommandLine)
 	namespacesyncOptions.AddFlags(pflag.CommandLine)
 	enforceClusterAutoscalerLimitsOptions.AddFlags(pflag.CommandLine)
+	failureDomainRolloutOptions.AddFlags(pflag.CommandLine)
 	pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
 	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
 
@@ -234,6 +239,23 @@ func main() {
 		}
 	}
 
+	if failureDomainRolloutOptions.Enabled {
+		if err := (&failuredomainrollout.Reconciler{
+			Client: mgr.GetClient(),
+		}).SetupWithManager(
+			mgr,
+			&controller.Options{MaxConcurrentReconciles: failureDomainRolloutOptions.Concurrency},
+		); err != nil {
+			setupLog.Error(
+				err,
+				"unable to create controller",
+				"controller",
+				"failuredomainrollout.Reconciler",
+			)
+			os.Exit(1)
+		}
+	}
+
 	mgr.GetWebhookServer().Register("/mutate-v1beta1-cluster", &webhook.Admission{
 		Handler: cluster.NewDefaulter(mgr.GetClient(), admission.NewDecoder(mgr.GetScheme())),
 	})