diff --git a/charts/cluster-api-runtime-extensions-nutanix/addons/registry/cncf-distribution/values-template.yaml b/charts/cluster-api-runtime-extensions-nutanix/addons/registry/cncf-distribution/values-template.yaml index 3c37dbdb1..62089fe1a 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/addons/registry/cncf-distribution/values-template.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/addons/registry/cncf-distribution/values-template.yaml @@ -25,3 +25,26 @@ statefulSet: cpu: 100m memory: 75Mi tlsSecretName: {{ .TLSSecretName }} +tolerations: + - key: "node-role.kubernetes.io/control-plane" + operator: Exists + effect: "NoSchedule" + +podLabels: + cncf-distribution-registry: "true" # ensure the labels match with pod AntiAffinity. + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + cncf-distribution-registry: "true" + topologyKey: kubernetes.io/hostname diff --git a/test/e2e/quick_start_test.go b/test/e2e/quick_start_test.go index f0b89865d..9af412bb6 100644 --- a/test/e2e/quick_start_test.go +++ b/test/e2e/quick_start_test.go @@ -315,6 +315,15 @@ var _ = Describe("Quick start", func() { ClusterProxy: proxy, }, ) + + EnsureAntiAffnityForRegistryAddon( + ctx, + EnsureAntiAffnityForRegistryAddonInput{ + Registry: addonsConfig.Registry, + WorkloadCluster: workloadCluster, + ClusterProxy: proxy, + }, + ) }, } }) diff --git a/test/e2e/registry.go b/test/e2e/registry.go index ae9668d03..357048097 100644 --- a/test/e2e/registry.go +++ b/test/e2e/registry.go @@ -101,3 +101,57 @@ func EnsureClusterCAForRegistryAddon( const caCrtKey = "ca.crt" Expect(rootCASecret.Data[caCrtKey]).To(Equal(rootCASecret.Data[caCrtKey])) } + +type EnsureAntiAffnityForRegistryAddonInput struct { + Registry *v1alpha1.RegistryAddon + WorkloadCluster *clusterv1.Cluster + ClusterProxy framework.ClusterProxy +} + +func EnsureAntiAffnityForRegistryAddon( + ctx context.Context, + input EnsureAntiAffnityForRegistryAddonInput, +) { + if input.Registry == nil { + return + } + Log("Ensuring anti-affinity for registry addon in workload cluster") + workloadClusterClient := input.ClusterProxy.GetWorkloadCluster( + ctx, input.WorkloadCluster.Namespace, input.WorkloadCluster.Name, + ).GetClient() + + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cncf-distribution-registry-docker-registry", + Namespace: "registry-system", + }, + } + err := workloadClusterClient.Get(ctx, ctrlclient.ObjectKeyFromObject(sts), sts) + Expect(err).NotTo(HaveOccurred()) + Expect(sts.Spec.Template.Spec.Affinity).ToNot(BeNil()) + Expect(sts.Spec.Template.Spec.Affinity.PodAntiAffinity).ToNot(BeNil()) + podAntiAffinity := sts.Spec.Template.Spec.Affinity.PodAntiAffinity + Expect( + podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, + ).ToNot(BeEmpty()) + Expect( + podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight, + ).To(Equal(int32(100))) + podAffinityTerm := podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm + Expect(podAffinityTerm).ToNot(BeNil()) + Expect(podAffinityTerm.TopologyKey).To(Equal("kubernetes.io/hostname")) + Expect(podAffinityTerm.LabelSelector).ToNot(BeNil()) + affinityLabels := podAffinityTerm.LabelSelector.MatchLabels + Expect( + affinityLabels["cncf-distribution-registry"], + ).To(Equal("true")) // Ensure the label matches the pod AntiAffinity. + + // test node affinity + nodeAffinity := sts.Spec.Template.Spec.Affinity.NodeAffinity + Expect(nodeAffinity).ToNot(BeNil()) + Expect(nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution).ToNot(BeNil()) + nodeSelectorTerm := nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0] + Expect(nodeSelectorTerm).ToNot(BeNil()) + Expect(nodeSelectorTerm.MatchExpressions).ToNot(BeEmpty()) + Expect(nodeSelectorTerm.MatchExpressions[0].Key).To(Equal("node-role.kubernetes.io/control-plane")) +} diff --git a/test/e2e/self_hosted_test.go b/test/e2e/self_hosted_test.go index 26cc9b4a7..d9b44772c 100644 --- a/test/e2e/self_hosted_test.go +++ b/test/e2e/self_hosted_test.go @@ -177,6 +177,15 @@ var _ = Describe("Self-hosted", Serial, func() { ClusterProxy: proxy, }, ) + + EnsureAntiAffnityForRegistryAddon( + ctx, + EnsureAntiAffnityForRegistryAddonInput{ + Registry: addonsConfig.Registry, + WorkloadCluster: workloadCluster, + ClusterProxy: proxy, + }, + ) }, } }, diff --git a/test/e2e/statefulset_helpers.go b/test/e2e/statefulset_helpers.go index 1ed7344bd..1fa0b08cc 100644 --- a/test/e2e/statefulset_helpers.go +++ b/test/e2e/statefulset_helpers.go @@ -21,7 +21,7 @@ type WaitForStatefulSetAvailableInput struct { StatefulSet *appsv1.StatefulSet } -// WaitForStatefulSetsAvailable waits until the Deployment has observedGeneration equal to generation and +// WaitForStatefulSetsAvailable waits until the Statefulset has observedGeneration equal to generation and // status.Available = True, that signals that all the desired replicas are in place. func WaitForStatefulSetsAvailable( ctx context.Context, input WaitForStatefulSetAvailableInput, intervals ...interface{},