Skip draining failed DaemonSet pods to prevent recreation loops

In scenarios where nodes experience DiskPressure, DaemonSet pods may be
evicted and enter Failed state. When deleting Machines, cluster-api attempts
to delete these failed pods, but this triggers DaemonSet to create new pods
which can then be evicted again due to persistent DiskPressure, creating an
infinite loop.

Signed-off-by: liuxu <[email protected]>

Author: liuxu623

internal/controllers/machine/drain/drain.go

@@ -137,7 +137,7 @@ func (d *Helper) GetPodsForEviction(ctx context.Context, cluster *clusterv1.Clus
 		// Skip Pods with deletionTimestamp (if Node is unreachable & time.Since(deletionTimestamp) > 1s)
 		d.skipDeletedFilter,
 
-		// Skip DaemonSet Pods (if they are not finished)
+		// Skip DaemonSet Pods
 		d.daemonSetFilter,
 
 		// Skip static Pods

internal/controllers/machine/drain/drain_test.go

@@ -268,7 +268,7 @@ func TestGetPodsForEviction(t *testing.T) {
 				},
 				{
 					ObjectMeta: metav1.ObjectMeta{
-						Name:      "pod-2-delete-succeeded-daemonset-pod",
+						Name:      "pod-2-skip-succeeded-daemonset-pod",
 						Namespace: metav1.NamespaceDefault,
 						OwnerReferences: []metav1.OwnerReference{
 							{
@@ -283,7 +283,22 @@ func TestGetPodsForEviction(t *testing.T) {
 				},
 				{
 					ObjectMeta: metav1.ObjectMeta{
-						Name:      "pod-3-delete-orphaned-daemonset-pod",
+						Name:      "pod-3-skip-failed-daemonset-pod",
+						Namespace: metav1.NamespaceDefault,
+						OwnerReferences: []metav1.OwnerReference{
+							{
+								Kind:       "DaemonSet",
+								Controller: ptr.To(true),
+							},
+						},
+					},
+					Status: corev1.PodStatus{
+						Phase: corev1.PodFailed,
+					},
+				},
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "pod-4-delete-orphaned-daemonset-pod",
 						Namespace: metav1.NamespaceDefault,
 						OwnerReferences: []metav1.OwnerReference{
 							{
@@ -299,7 +314,7 @@ func TestGetPodsForEviction(t *testing.T) {
 				},
 				{
 					ObjectMeta: metav1.ObjectMeta{
-						Name:      "pod-4-skip-daemonset-pod",
+						Name:      "pod-5-skip-daemonset-pod",
 						Namespace: metav1.NamespaceDefault,
 						OwnerReferences: []metav1.OwnerReference{
 							{
@@ -332,21 +347,7 @@ func TestGetPodsForEviction(t *testing.T) {
 				{
 					Pod: &corev1.Pod{
 						ObjectMeta: metav1.ObjectMeta{
-							Name:      "pod-2-delete-succeeded-daemonset-pod",
-							Namespace: metav1.NamespaceDefault,
-						},
-					},
-					// Delete this DaemonSet Pod because it is succeeded.
-					Status: PodDeleteStatus{
-						DrainBehavior: clusterv1.MachineDrainRuleDrainBehaviorDrain,
-						DrainOrder:    ptr.To[int32](0),
-						Reason:        PodDeleteStatusTypeOkay,
-					},
-				},
-				{
-					Pod: &corev1.Pod{
-						ObjectMeta: metav1.ObjectMeta{
-							Name:      "pod-3-delete-orphaned-daemonset-pod",
+							Name:      "pod-4-delete-orphaned-daemonset-pod",
 							Namespace: metav1.NamespaceDefault,
 						},
 					},
@@ -361,7 +362,7 @@ func TestGetPodsForEviction(t *testing.T) {
 				{
 					Pod: &corev1.Pod{
 						ObjectMeta: metav1.ObjectMeta{
-							Name:      "pod-4-skip-daemonset-pod",
+							Name:      "pod-5-skip-daemonset-pod",
 							Namespace: metav1.NamespaceDefault,
 						},
 					},

internal/controllers/machine/drain/filters.go

@@ -212,10 +212,6 @@ func (d *Helper) daemonSetFilter(ctx context.Context, pod *corev1.Pod) PodDelete
 	if controllerRef == nil || controllerRef.Kind != appsv1.SchemeGroupVersion.WithKind("DaemonSet").Kind {
 		return MakePodDeleteStatusOkay()
 	}
-	// Any finished pod can be removed.
-	if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed {
-		return MakePodDeleteStatusOkay()
-	}
 
 	if err := d.RemoteClient.Get(ctx, client.ObjectKey{Namespace: pod.Namespace, Name: controllerRef.Name}, &appsv1.DaemonSet{}); err != nil {
 		// remove orphaned pods with a warning