Add CriticalAddonsOnly toleration for all dependencies

Author: chnnmz

helm_chart/HyperPodHelmChart/charts/health-monitoring-agent/templates/health-monitoring-agent.yaml

@@ -164,3 +164,5 @@ spec:
           operator: Exists
         - effect: NoExecute
           operator: Exists
+        - key: CriticalAddonsOnly
+          operator: Exists

helm_chart/HyperPodHelmChart/charts/mpi-operator/values.yaml

@@ -22,6 +22,8 @@ mpiOperator:
   ## Tolerations for pod assignment
   ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
   tolerations:
+  - key: CriticalAddonsOnly
+    operator: Exists
   - key: sagemaker.amazonaws.com/node-health-status
     operator: "Equal"
     value: "Unschedulable"
@@ -35,4 +37,4 @@ mpiOperator:
   imagePullPolicy: IfNotPresent
 
 ## Apply extra labels to all created resources
-extraLabels: {}
+extraLabels: {}

helm_chart/HyperPodHelmChart/charts/training-operators/templates/Deployment/training-operator-kubeflow-Deployment.yaml

@@ -54,5 +54,8 @@ spec:
           timeoutSeconds: 3
         securityContext:
           allowPrivilegeEscalation: false
+        tolerations:
+          - key: CriticalAddonsOnly
+            operator: Exists
       serviceAccountName: training-operator
-      terminationGracePeriodSeconds: 10
+      terminationGracePeriodSeconds: 10

helm_chart/HyperPodHelmChart/values.yaml

@@ -180,6 +180,8 @@ nvidia-device-plugin:
       operator: Equal
       value: Unschedulable
       effect: NoSchedule
+    - key: CriticalAddonsOnly
+      operator: Exists
 
 neuron-device-plugin:
   devicePlugin: