Remove CriticalAddonsOnly toleration from the CDI pods (#3909) (#3910)

The CriticalAddonsOnly toleration is added to the CDI pods which
affects pod scheduling and is against k8s best practices

Signed-off-by: Howard Gao <[email protected]>
Co-authored-by: Howard Gao <[email protected]>

Author: gaohoward

doc/quota.md

@@ -48,9 +48,6 @@ spec:
   infra:
     nodeSelector:
       kubernetes.io/os: linux
-    tolerations:
-    - key: CriticalAddonsOnly
-      operator: Exists
   workload:
     nodeSelector:
       kubernetes.io/os: linux

manifests/templates/release/cdi-cr.yaml.in

@@ -11,9 +11,6 @@ spec:
   infra:
     nodeSelector:
       kubernetes.io/os: linux
-    tolerations:
-    - key: CriticalAddonsOnly
-      operator: Exists
   workload:
     nodeSelector:
       kubernetes.io/os: linux

pkg/operator/resources/utils/common.go

@@ -117,12 +117,6 @@ func CreateOperatorDeployment(name, namespace, matchKey, matchValue, serviceAcco
 		},
 		ImagePullSecrets: imagePullSecrets,
 		NodeSelector:     map[string]string{"kubernetes.io/os": "linux"},
-		Tolerations: []corev1.Toleration{
-			{
-				Key:      "CriticalAddonsOnly",
-				Operator: corev1.TolerationOpExists,
-			},
-		},
 		Affinity: &corev1.Affinity{
 			PodAffinity: &corev1.PodAffinity{
 				PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{

tests/operator_test.go

@@ -317,71 +317,6 @@ var _ = Describe("ALL Operator tests", func() {
 					return true
 				}, 5*time.Minute, 2*time.Second).Should(BeTrue())
 			})
-
-			It("should deploy components that tolerate CriticalAddonsOnly taint", func() {
-				cr := getCDI(f)
-				criticalAddonsToleration := corev1.Toleration{
-					Key:      "CriticalAddonsOnly",
-					Operator: corev1.TolerationOpExists,
-				}
-
-				if !tolerationExists(cr.Spec.Infra.NodePlacement.Tolerations, criticalAddonsToleration) {
-					Skip("Unexpected CDI CR (not from cdi-cr.yaml), doesn't tolerate CriticalAddonsOnly")
-				}
-
-				labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"cdi.kubevirt.io/testing": ""}}
-				cdiTestPods, err := f.K8sClient.CoreV1().Pods(f.CdiInstallNs).List(context.TODO(), metav1.ListOptions{
-					LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
-				})
-				Expect(err).ToNot(HaveOccurred(), "failed listing cdi testing pods")
-				Expect(cdiTestPods.Items).ToNot(BeEmpty(), "no cdi testing pods found")
-
-				By("adding taints to all nodes")
-				criticalPodTaint := corev1.Taint{
-					Key:    "CriticalAddonsOnly",
-					Value:  "",
-					Effect: corev1.TaintEffectNoExecute,
-				}
-
-				for _, node := range nodes.Items {
-					Eventually(func() bool {
-						nodeCopy, err := f.K8sClient.CoreV1().Nodes().Get(context.TODO(), node.Name, metav1.GetOptions{})
-						Expect(err).ToNot(HaveOccurred())
-
-						if nodeHasTaint(*nodeCopy, criticalPodTaint) {
-							return true
-						}
-
-						nodeCopy.Spec.Taints = append(nodeCopy.Spec.Taints, criticalPodTaint)
-						_, _ = f.K8sClient.CoreV1().Nodes().Update(context.TODO(), nodeCopy, metav1.UpdateOptions{})
-						return false
-					}, 5*time.Minute, 2*time.Second).Should(BeTrue())
-				}
-
-				By("Waiting for all CDI testing pods to terminate")
-				Eventually(func() bool {
-					for _, cdiTestPod := range cdiTestPods.Items {
-						By(fmt.Sprintf("CDI test pod: %s", cdiTestPod.Name))
-						_, err := f.K8sClient.CoreV1().Pods(cdiTestPod.Namespace).Get(context.TODO(), cdiTestPod.Name, metav1.GetOptions{})
-						if !errors.IsNotFound(err) {
-							return false
-						}
-					}
-					return true
-				}, 5*time.Minute, 2*time.Second).Should(BeTrue())
-
-				By("Checking that all the non-testing pods are running")
-				for _, cdiPod := range cdiPods.Items {
-					if _, isTestingComponent := cdiPod.Labels["cdi.kubevirt.io/testing"]; isTestingComponent {
-						continue
-					}
-					By(fmt.Sprintf("Non-test CDI pod: %s", cdiPod.Name))
-					podUpdated, err := f.K8sClient.CoreV1().Pods(cdiPod.Namespace).Get(context.TODO(), cdiPod.Name, metav1.GetOptions{})
-					Expect(err).ToNot(HaveOccurred(), "failed setting taint on node")
-					Expect(podUpdated.Status.Phase).To(Equal(corev1.PodRunning))
-				}
-			})
-
 		})
 
 		var _ = Describe("Operator delete CDI CR tests", func() {
@@ -1356,24 +1291,6 @@ func waitCDI(f *framework.Framework, cr *cdiv1.CDI, cdiPods *corev1.PodList) {
 	}
 }
 
-func tolerationExists(tolerations []corev1.Toleration, testValue corev1.Toleration) bool {
-	for _, toleration := range tolerations {
-		if reflect.DeepEqual(toleration, testValue) {
-			return true
-		}
-	}
-	return false
-}
-
-func nodeHasTaint(node corev1.Node, testedTaint corev1.Taint) bool {
-	for _, taint := range node.Spec.Taints {
-		if reflect.DeepEqual(taint, testedTaint) {
-			return true
-		}
-	}
-	return false
-}
-
 func infraDeploymentAvailable(f *framework.Framework, cr *cdiv1.CDI) bool {
 	cdi, _ := f.CdiClient.CdiV1beta1().CDIs().Get(context.TODO(), cr.Name, metav1.GetOptions{})
 	if !conditions.IsStatusConditionTrue(cdi.Status.Conditions, conditions.ConditionAvailable) {