fix: add tolerations and nodeAffinity overrides for registry addon

charts/cluster-api-runtime-extensions-nutanix/addons/registry/cncf-distribution/values-template.yaml

@@ -25,3 +25,26 @@ statefulSet:
         cpu: 100m
         memory: 75Mi
 tlsSecretName: {{ .TLSSecretName }}
+tolerations:
+  - key: "node-role.kubernetes.io/control-plane"
+    operator: Exists
+    effect: "NoSchedule"
+
+podLabels:
+  cncf-distribution-registry: "true" # ensure the labels match with pod AntiAffinity.
+
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: node-role.kubernetes.io/control-plane
+              operator: Exists
+  podAntiAffinity:
+    preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 100
+        podAffinityTerm:
+          labelSelector:
+            matchLabels:
+              cncf-distribution-registry: "true"
+          topologyKey: kubernetes.io/hostname

test/e2e/quick_start_test.go

@@ -315,6 +315,15 @@ var _ = Describe("Quick start", func() {
 														ClusterProxy:    proxy,
 													},
 												)
+
+												EnsureAntiAffnityForRegistryAddon(
+													ctx,
+													EnsureAntiAffnityForRegistryAddonInput{
+														Registry:        addonsConfig.Registry,
+														WorkloadCluster: workloadCluster,
+														ClusterProxy:    proxy,
+													},
+												)
 											},
 										}
 									})

test/e2e/registry.go

@@ -101,3 +101,57 @@ func EnsureClusterCAForRegistryAddon(
 	const caCrtKey = "ca.crt"
 	Expect(rootCASecret.Data[caCrtKey]).To(Equal(rootCASecret.Data[caCrtKey]))
 }
+
+type EnsureAntiAffnityForRegistryAddonInput struct {
+	Registry        *v1alpha1.RegistryAddon
+	WorkloadCluster *clusterv1.Cluster
+	ClusterProxy    framework.ClusterProxy
+}
+
+func EnsureAntiAffnityForRegistryAddon(
+	ctx context.Context,
+	input EnsureAntiAffnityForRegistryAddonInput,
+) {
+	if input.Registry == nil {
+		return
+	}
+	Log("Ensuring anti-affinity for registry addon in workload cluster")
+	workloadClusterClient := input.ClusterProxy.GetWorkloadCluster(
+		ctx, input.WorkloadCluster.Namespace, input.WorkloadCluster.Name,
+	).GetClient()
+
+	sts := &appsv1.StatefulSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "cncf-distribution-registry-docker-registry",
+			Namespace: "registry-system",
+		},
+	}
+	err := workloadClusterClient.Get(ctx, ctrlclient.ObjectKeyFromObject(sts), sts)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(sts.Spec.Template.Spec.Affinity).ToNot(BeNil())
+	Expect(sts.Spec.Template.Spec.Affinity.PodAntiAffinity).ToNot(BeNil())
+	podAntiAffinity := sts.Spec.Template.Spec.Affinity.PodAntiAffinity
+	Expect(
+		podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution,
+	).ToNot(BeEmpty())
+	Expect(
+		podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight,
+	).To(Equal(int32(100)))
+	podAffinityTerm := podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm
+	Expect(podAffinityTerm).ToNot(BeNil())
+	Expect(podAffinityTerm.TopologyKey).To(Equal("kubernetes.io/hostname"))
+	Expect(podAffinityTerm.LabelSelector).ToNot(BeNil())
+	affinityLabels := podAffinityTerm.LabelSelector.MatchLabels
+	Expect(
+		affinityLabels["cncf-distribution-registry"],
+	).To(Equal("true")) // Ensure the label matches the pod AntiAffinity.
+
+	// test node affinity
+	nodeAffinity := sts.Spec.Template.Spec.Affinity.NodeAffinity
+	Expect(nodeAffinity).ToNot(BeNil())
+	Expect(nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution).ToNot(BeNil())
+	nodeSelectorTerm := nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0]
+	Expect(nodeSelectorTerm).ToNot(BeNil())
+	Expect(nodeSelectorTerm.MatchExpressions).ToNot(BeEmpty())
+	Expect(nodeSelectorTerm.MatchExpressions[0].Key).To(Equal("node-role.kubernetes.io/control-plane"))
+}

test/e2e/self_hosted_test.go

@@ -177,6 +177,15 @@ var _ = Describe("Self-hosted", Serial, func() {
 															ClusterProxy:    proxy,
 														},
 													)
+
+													EnsureAntiAffnityForRegistryAddon(
+														ctx,
+														EnsureAntiAffnityForRegistryAddonInput{
+															Registry:        addonsConfig.Registry,
+															WorkloadCluster: workloadCluster,
+															ClusterProxy:    proxy,
+														},
+													)
 												},
 											}
 										},

test/e2e/statefulset_helpers.go

@@ -21,7 +21,7 @@ type WaitForStatefulSetAvailableInput struct {
 	StatefulSet *appsv1.StatefulSet
 }
 
-// WaitForStatefulSetsAvailable waits until the Deployment has observedGeneration equal to generation and
+// WaitForStatefulSetsAvailable waits until the Statefulset has observedGeneration equal to generation and
 // status.Available = True, that signals that all the desired replicas are in place.
 func WaitForStatefulSetsAvailable(
 	ctx context.Context, input WaitForStatefulSetAvailableInput, intervals ...interface{},