From 5f889ae1591ae783e54f266adfc2b4aa5878c5d5 Mon Sep 17 00:00:00 2001 From: liuxu Date: Mon, 18 Aug 2025 11:46:30 +0800 Subject: [PATCH] Skip draining failed DaemonSet pods to prevent recreation loops In scenarios where nodes experience DiskPressure, DaemonSet pods may be evicted and enter Failed state. When deleting Machines, cluster-api attempts to delete these failed pods, but this triggers DaemonSet to create new pods which can then be evicted again due to persistent DiskPressure, creating an infinite loop. Signed-off-by: liuxu --- internal/controllers/machine/drain/drain.go | 2 +- .../controllers/machine/drain/drain_test.go | 39 ++++++++++--------- internal/controllers/machine/drain/filters.go | 4 -- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/internal/controllers/machine/drain/drain.go b/internal/controllers/machine/drain/drain.go index 30511c0bd7d2..b7b04d7dcf7f 100644 --- a/internal/controllers/machine/drain/drain.go +++ b/internal/controllers/machine/drain/drain.go @@ -137,7 +137,7 @@ func (d *Helper) GetPodsForEviction(ctx context.Context, cluster *clusterv1.Clus // Skip Pods with deletionTimestamp (if Node is unreachable & time.Since(deletionTimestamp) > 1s) d.skipDeletedFilter, - // Skip DaemonSet Pods (if they are not finished) + // Skip DaemonSet Pods d.daemonSetFilter, // Skip static Pods diff --git a/internal/controllers/machine/drain/drain_test.go b/internal/controllers/machine/drain/drain_test.go index 29ca70588c91..5cd0572b87d3 100644 --- a/internal/controllers/machine/drain/drain_test.go +++ b/internal/controllers/machine/drain/drain_test.go @@ -268,7 +268,7 @@ func TestGetPodsForEviction(t *testing.T) { }, { ObjectMeta: metav1.ObjectMeta{ - Name: "pod-2-delete-succeeded-daemonset-pod", + Name: "pod-2-skip-succeeded-daemonset-pod", Namespace: metav1.NamespaceDefault, OwnerReferences: []metav1.OwnerReference{ { @@ -283,7 +283,22 @@ func TestGetPodsForEviction(t *testing.T) { }, { ObjectMeta: metav1.ObjectMeta{ - Name: "pod-3-delete-orphaned-daemonset-pod", + Name: "pod-3-skip-failed-daemonset-pod", + Namespace: metav1.NamespaceDefault, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "DaemonSet", + Controller: ptr.To(true), + }, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodFailed, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-4-delete-orphaned-daemonset-pod", Namespace: metav1.NamespaceDefault, OwnerReferences: []metav1.OwnerReference{ { @@ -299,7 +314,7 @@ func TestGetPodsForEviction(t *testing.T) { }, { ObjectMeta: metav1.ObjectMeta{ - Name: "pod-4-skip-daemonset-pod", + Name: "pod-5-skip-daemonset-pod", Namespace: metav1.NamespaceDefault, OwnerReferences: []metav1.OwnerReference{ { @@ -332,21 +347,7 @@ func TestGetPodsForEviction(t *testing.T) { { Pod: &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: "pod-2-delete-succeeded-daemonset-pod", - Namespace: metav1.NamespaceDefault, - }, - }, - // Delete this DaemonSet Pod because it is succeeded. - Status: PodDeleteStatus{ - DrainBehavior: clusterv1.MachineDrainRuleDrainBehaviorDrain, - DrainOrder: ptr.To[int32](0), - Reason: PodDeleteStatusTypeOkay, - }, - }, - { - Pod: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-3-delete-orphaned-daemonset-pod", + Name: "pod-4-delete-orphaned-daemonset-pod", Namespace: metav1.NamespaceDefault, }, }, @@ -361,7 +362,7 @@ func TestGetPodsForEviction(t *testing.T) { { Pod: &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: "pod-4-skip-daemonset-pod", + Name: "pod-5-skip-daemonset-pod", Namespace: metav1.NamespaceDefault, }, }, diff --git a/internal/controllers/machine/drain/filters.go b/internal/controllers/machine/drain/filters.go index 794d3d9547cc..f90d32f0a6e5 100644 --- a/internal/controllers/machine/drain/filters.go +++ b/internal/controllers/machine/drain/filters.go @@ -212,10 +212,6 @@ func (d *Helper) daemonSetFilter(ctx context.Context, pod *corev1.Pod) PodDelete if controllerRef == nil || controllerRef.Kind != appsv1.SchemeGroupVersion.WithKind("DaemonSet").Kind { return MakePodDeleteStatusOkay() } - // Any finished pod can be removed. - if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { - return MakePodDeleteStatusOkay() - } if err := d.RemoteClient.Get(ctx, client.ObjectKey{Namespace: pod.Namespace, Name: controllerRef.Name}, &appsv1.DaemonSet{}); err != nil { // remove orphaned pods with a warning