Skip to content

Commit 3ebd3fe

Browse files
fabriziopandinifabriziopandini
authored
🌱 Improve Machine create and delete logs (#11693)
* Improve Machine create and delete logs * Add k/v pairs describing the overall status of the control plane * Fix unit test * Address comments * More feedback * More comments
1 parent d01f2a8 commit 3ebd3fe

File tree

9 files changed

+212
-18
lines changed

9 files changed

+212
-18
lines changed

‎controlplane/kubeadm/internal/control_plane.go‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ package internal
1818

1919
import (
2020
"context"
21+
"fmt"
22+
"sort"
23+
"strings"
2124

2225
"github.com/pkg/errors"
2326
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -32,6 +35,7 @@ import (
3235
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
3336
"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal/etcd"
3437
"sigs.k8s.io/cluster-api/util/collections"
38+
"sigs.k8s.io/cluster-api/util/conditions"
3539
"sigs.k8s.io/cluster-api/util/failuredomains"
3640
"sigs.k8s.io/cluster-api/util/patch"
3741
)
@@ -385,3 +389,81 @@ func (c *ControlPlane) InjectTestManagementCluster(managementCluster ManagementC
385389
c.managementCluster = managementCluster
386390
c.workloadCluster = nil
387391
}
392+
393+
// StatusToLogKeyAndValues returns the following key/value pairs describing the overall status of the control plane:
394+
// - machines is the list of KCP machines; each machine might have additional notes surfacing
395+
// - if the machine has been created in the current reconcile
396+
// - if machines node ref is not yet set
397+
// - if the machine has been marked for remediation
398+
// - if there are unhealthy control plane component on the machine
399+
// - if the machine has a deletion timestamp/has been deleted in the current reconcile
400+
// - if the machine is not up to date with the KCP spec
401+
//
402+
// - etcdMembers list as reported by etcd.
403+
func (c *ControlPlane) StatusToLogKeyAndValues(newMachine, deletedMachine *clusterv1.Machine) []any {
404+
controlPlaneMachineHealthConditions := []clusterv1.ConditionType{
405+
controlplanev1.MachineAPIServerPodHealthyCondition,
406+
controlplanev1.MachineControllerManagerPodHealthyCondition,
407+
controlplanev1.MachineSchedulerPodHealthyCondition,
408+
}
409+
if c.IsEtcdManaged() {
410+
controlPlaneMachineHealthConditions = append(controlPlaneMachineHealthConditions,
411+
controlplanev1.MachineEtcdPodHealthyCondition,
412+
controlplanev1.MachineEtcdMemberHealthyCondition,
413+
)
414+
}
415+
416+
machines := []string{}
417+
for _, m := range c.Machines {
418+
notes := []string{}
419+
420+
if m.Status.NodeRef == nil {
421+
notes = append(notes, "status.nodeRef not set")
422+
}
423+
424+
if c.MachinesToBeRemediatedByKCP().Has(m) {
425+
notes = append(notes, "marked for remediation")
426+
}
427+
428+
for _, condition := range controlPlaneMachineHealthConditions {
429+
if conditions.IsUnknown(m, condition) {
430+
notes = append(notes, strings.Replace(string(condition), "Healthy", " health unknown", -1))
431+
}
432+
if conditions.IsFalse(m, condition) {
433+
notes = append(notes, strings.Replace(string(condition), "Healthy", " not healthy", -1))
434+
}
435+
}
436+
437+
if !c.UpToDateMachines().Has(m) {
438+
notes = append(notes, "not up-to-date")
439+
}
440+
441+
if deletedMachine != nil && m.Name == deletedMachine.Name {
442+
notes = append(notes, "just deleted")
443+
} else if !m.DeletionTimestamp.IsZero() {
444+
notes = append(notes, "deleting")
445+
}
446+
447+
name := m.Name
448+
if len(notes) > 0 {
449+
name = fmt.Sprintf("%s (%s)", name, strings.Join(notes, ", "))
450+
}
451+
machines = append(machines, name)
452+
}
453+
454+
if newMachine != nil {
455+
machines = append(machines, fmt.Sprintf("%s (just created)", newMachine.Name))
456+
}
457+
sort.Strings(machines)
458+
459+
etcdMembers := []string{}
460+
for _, m := range c.EtcdMembers {
461+
etcdMembers = append(etcdMembers, m.Name)
462+
}
463+
sort.Strings(etcdMembers)
464+
465+
return []any{
466+
"machines", strings.Join(machines, ", "),
467+
"etcdMembers", strings.Join(etcdMembers, ", "),
468+
}
469+
}

‎controlplane/kubeadm/internal/control_plane_test.go‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,18 @@ limitations under the License.
1717
package internal
1818

1919
import (
20+
"strings"
2021
"testing"
2122

23+
"github.com/google/go-cmp/cmp"
2224
. "github.com/onsi/gomega"
2325
corev1 "k8s.io/api/core/v1"
2426
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2527
"k8s.io/utils/ptr"
2628

2729
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
2830
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
31+
"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal/etcd"
2932
"sigs.k8s.io/cluster-api/util/collections"
3033
"sigs.k8s.io/cluster-api/util/conditions"
3134
)
@@ -252,6 +255,75 @@ func TestHasHealthyMachineStillProvisioning(t *testing.T) {
252255
})
253256
}
254257

258+
func TestStatusToLogKeyAndValues(t *testing.T) {
259+
healthyMachine := &clusterv1.Machine{
260+
ObjectMeta: metav1.ObjectMeta{Name: "healthy"},
261+
Status: clusterv1.MachineStatus{
262+
NodeRef: &corev1.ObjectReference{Name: "healthy-node"},
263+
Conditions: []clusterv1.Condition{
264+
{Type: controlplanev1.MachineAPIServerPodHealthyCondition, Status: corev1.ConditionTrue},
265+
{Type: controlplanev1.MachineControllerManagerPodHealthyCondition, Status: corev1.ConditionTrue},
266+
{Type: controlplanev1.MachineSchedulerPodHealthyCondition, Status: corev1.ConditionTrue},
267+
{Type: controlplanev1.MachineEtcdPodHealthyCondition, Status: corev1.ConditionTrue},
268+
{Type: controlplanev1.MachineEtcdMemberHealthyCondition, Status: corev1.ConditionTrue},
269+
},
270+
},
271+
}
272+
273+
machineWithoutNode := &clusterv1.Machine{
274+
ObjectMeta: metav1.ObjectMeta{Name: "without-node"},
275+
Status: clusterv1.MachineStatus{
276+
NodeRef: nil,
277+
Conditions: []clusterv1.Condition{
278+
{Type: controlplanev1.MachineAPIServerPodHealthyCondition, Status: corev1.ConditionUnknown},
279+
{Type: controlplanev1.MachineControllerManagerPodHealthyCondition, Status: corev1.ConditionUnknown},
280+
{Type: controlplanev1.MachineSchedulerPodHealthyCondition, Status: corev1.ConditionUnknown},
281+
{Type: controlplanev1.MachineEtcdPodHealthyCondition, Status: corev1.ConditionUnknown},
282+
{Type: controlplanev1.MachineEtcdMemberHealthyCondition, Status: corev1.ConditionFalse}, // not a real use case, but used to test a code branch.
283+
},
284+
},
285+
}
286+
287+
machineJustCreated := &clusterv1.Machine{ObjectMeta: metav1.ObjectMeta{Name: "just-created"}}
288+
289+
machineJustDeleted := healthyMachine.DeepCopy()
290+
machineJustDeleted.Name = "just-deleted"
291+
292+
machineNotUpToDate := healthyMachine.DeepCopy()
293+
machineNotUpToDate.Name = "not-up-to-date"
294+
295+
machineMarkedForRemediation := healthyMachine.DeepCopy()
296+
machineMarkedForRemediation.Name = "marked-for-remediation"
297+
machineMarkedForRemediation.Status.Conditions = append(machineMarkedForRemediation.Status.Conditions,
298+
clusterv1.Condition{Type: clusterv1.MachineHealthCheckSucceededCondition, Status: corev1.ConditionFalse},
299+
clusterv1.Condition{Type: clusterv1.MachineOwnerRemediatedCondition, Status: corev1.ConditionFalse},
300+
)
301+
302+
g := NewWithT(t)
303+
c := &ControlPlane{
304+
KCP: &controlplanev1.KubeadmControlPlane{},
305+
Machines: collections.FromMachines(healthyMachine, machineWithoutNode, machineJustDeleted, machineNotUpToDate, machineMarkedForRemediation),
306+
machinesNotUptoDate: collections.FromMachines(machineNotUpToDate),
307+
EtcdMembers: []*etcd.Member{{Name: "m1"}, {Name: "m2"}, {Name: "m3"}},
308+
}
309+
310+
got := c.StatusToLogKeyAndValues(machineJustCreated, machineJustDeleted)
311+
312+
g.Expect(got).To(HaveLen(4))
313+
g.Expect(got[0]).To(Equal("machines"))
314+
machines := strings.Join([]string{
315+
"healthy",
316+
"just-created (just created)",
317+
"just-deleted (just deleted)",
318+
"marked-for-remediation (marked for remediation)",
319+
"not-up-to-date (not up-to-date)",
320+
"without-node (status.nodeRef not set, APIServerPod health unknown, ControllerManagerPod health unknown, SchedulerPod health unknown, EtcdPod health unknown, EtcdMember not healthy)",
321+
}, ", ")
322+
g.Expect(got[1]).To(Equal(machines), cmp.Diff(got[1], machines))
323+
g.Expect(got[2]).To(Equal("etcdMembers"))
324+
g.Expect(got[3]).To(Equal("m1, m2, m3"))
325+
}
326+
255327
type machineOpt func(*clusterv1.Machine)
256328

257329
func failureDomain(controlPlane bool) clusterv1.FailureDomainSpec {

‎controlplane/kubeadm/internal/controllers/controller.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -672,10 +672,13 @@ func (r *KubeadmControlPlaneReconciler) reconcileDelete(ctx context.Context, con
672672
continue
673673
}
674674

675-
log.Info("Deleting control plane Machine")
676675
if err := r.Client.Delete(ctx, machineToDelete); err != nil && !apierrors.IsNotFound(err) {
677676
errs = append(errs, errors.Wrapf(err, "failed to delete control plane Machine %s", klog.KObj(machineToDelete)))
678677
}
678+
// Note: We intentionally log after Delete because we want this log line to show up only after DeletionTimestamp has been set.
679+
// Also, setting DeletionTimestamp doesn't mean the Machine is actually deleted (deletion takes some time).
680+
log.WithValues(controlPlane.StatusToLogKeyAndValues(nil, machineToDelete)...).
681+
Info("Deleting Machine (KubeadmControlPlane deleted)")
679682
}
680683
if len(errs) > 0 {
681684
err := kerrors.NewAggregate(errs)

‎controlplane/kubeadm/internal/controllers/helpers.go‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -182,13 +182,13 @@ func (r *KubeadmControlPlaneReconciler) reconcileExternalReference(ctx context.C
182182
return patchHelper.Patch(ctx, obj)
183183
}
184184

185-
func (r *KubeadmControlPlaneReconciler) cloneConfigsAndGenerateMachine(ctx context.Context, cluster *clusterv1.Cluster, kcp *controlplanev1.KubeadmControlPlane, bootstrapSpec *bootstrapv1.KubeadmConfigSpec, failureDomain *string) error {
185+
func (r *KubeadmControlPlaneReconciler) cloneConfigsAndGenerateMachine(ctx context.Context, cluster *clusterv1.Cluster, kcp *controlplanev1.KubeadmControlPlane, bootstrapSpec *bootstrapv1.KubeadmConfigSpec, failureDomain *string) (*clusterv1.Machine, error) {
186186
var errs []error
187187

188188
// Compute desired Machine
189189
machine, err := r.computeDesiredMachine(kcp, cluster, failureDomain, nil)
190190
if err != nil {
191-
return errors.Wrap(err, "failed to create Machine: failed to compute desired Machine")
191+
return nil, errors.Wrap(err, "failed to create Machine: failed to compute desired Machine")
192192
}
193193

194194
// Since the cloned resource should eventually have a controller ref for the Machine, we create an
@@ -220,7 +220,7 @@ func (r *KubeadmControlPlaneReconciler) cloneConfigsAndGenerateMachine(ctx conte
220220
// Safe to return early here since no resources have been created yet.
221221
conditions.MarkFalse(kcp, controlplanev1.MachinesCreatedCondition, controlplanev1.InfrastructureTemplateCloningFailedReason,
222222
clusterv1.ConditionSeverityError, err.Error())
223-
return errors.Wrap(err, "failed to clone infrastructure template")
223+
return nil, errors.Wrap(err, "failed to clone infrastructure template")
224224
}
225225
machine.Spec.InfrastructureRef = *infraRef
226226

@@ -248,11 +248,10 @@ func (r *KubeadmControlPlaneReconciler) cloneConfigsAndGenerateMachine(ctx conte
248248
if err := r.cleanupFromGeneration(ctx, infraRef, bootstrapRef); err != nil {
249249
errs = append(errs, errors.Wrap(err, "failed to cleanup generated resources"))
250250
}
251-
252-
return kerrors.NewAggregate(errs)
251+
return nil, kerrors.NewAggregate(errs)
253252
}
254253

255-
return nil
254+
return machine, nil
256255
}
257256

258257
func (r *KubeadmControlPlaneReconciler) cleanupFromGeneration(ctx context.Context, remoteRefs ...*corev1.ObjectReference) error {

‎controlplane/kubeadm/internal/controllers/helpers_test.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,8 @@ func TestCloneConfigsAndGenerateMachine(t *testing.T) {
374374
bootstrapSpec := &bootstrapv1.KubeadmConfigSpec{
375375
JoinConfiguration: &bootstrapv1.JoinConfiguration{},
376376
}
377-
g.Expect(r.cloneConfigsAndGenerateMachine(ctx, cluster, kcp, bootstrapSpec, nil)).To(Succeed())
377+
_, err := r.cloneConfigsAndGenerateMachine(ctx, cluster, kcp, bootstrapSpec, nil)
378+
g.Expect(err).To(Succeed())
378379

379380
machineList := &clusterv1.MachineList{}
380381
g.Expect(env.GetAPIReader().List(ctx, machineList, client.InNamespace(cluster.Namespace))).To(Succeed())
@@ -463,7 +464,8 @@ func TestCloneConfigsAndGenerateMachineFail(t *testing.T) {
463464

464465
// Try to break Infra Cloning
465466
kcp.Spec.MachineTemplate.InfrastructureRef.Name = "something_invalid"
466-
g.Expect(r.cloneConfigsAndGenerateMachine(ctx, cluster, kcp, bootstrapSpec, nil)).To(HaveOccurred())
467+
_, err := r.cloneConfigsAndGenerateMachine(ctx, cluster, kcp, bootstrapSpec, nil)
468+
g.Expect(err).To(HaveOccurred())
467469
g.Expect(&kcp.GetConditions()[0]).Should(conditions.HaveSameStateOf(&clusterv1.Condition{
468470
Type: controlplanev1.MachinesCreatedCondition,
469471
Status: corev1.ConditionFalse,

‎controlplane/kubeadm/internal/controllers/remediation.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -311,7 +311,10 @@ func (r *KubeadmControlPlaneReconciler) reconcileUnhealthyMachines(ctx context.C
311311
}
312312

313313
// Surface the operation is in progress.
314-
log.Info("Remediating unhealthy machine")
314+
// Note: We intentionally log after Delete because we want this log line to show up only after DeletionTimestamp has been set.
315+
// Also, setting DeletionTimestamp doesn't mean the Machine is actually deleted (deletion takes some time).
316+
log.WithValues(controlPlane.StatusToLogKeyAndValues(nil, machineToBeRemediated)...).
317+
Info("Deleting Machine (remediating unhealthy Machine)")
315318
conditions.MarkFalse(machineToBeRemediated, clusterv1.MachineOwnerRemediatedCondition, clusterv1.RemediationInProgressReason, clusterv1.ConditionSeverityWarning, "")
316319

317320
v1beta2conditions.Set(machineToBeRemediated, metav1.Condition{

‎controlplane/kubeadm/internal/controllers/scale.go‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,19 @@ func (r *KubeadmControlPlaneReconciler) initializeControlPlane(ctx context.Conte
5353
return ctrl.Result{}, err
5454
}
5555

56-
if err := r.cloneConfigsAndGenerateMachine(ctx, controlPlane.Cluster, controlPlane.KCP, bootstrapSpec, fd); err != nil {
56+
newMachine, err := r.cloneConfigsAndGenerateMachine(ctx, controlPlane.Cluster, controlPlane.KCP, bootstrapSpec, fd)
57+
if err != nil {
5758
logger.Error(err, "Failed to create initial control plane Machine")
5859
r.recorder.Eventf(controlPlane.KCP, corev1.EventTypeWarning, "FailedInitialization", "Failed to create initial control plane Machine for cluster %s control plane: %v", klog.KObj(controlPlane.Cluster), err)
5960
return ctrl.Result{}, err
6061
}
6162

63+
logger.WithValues(controlPlane.StatusToLogKeyAndValues(newMachine, nil)...).
64+
Info("Machine created (scale up)",
65+
"Machine", klog.KObj(newMachine),
66+
newMachine.Spec.InfrastructureRef.Kind, klog.KRef(newMachine.Spec.InfrastructureRef.Namespace, newMachine.Spec.InfrastructureRef.Name),
67+
newMachine.Spec.Bootstrap.ConfigRef.Kind, klog.KRef(newMachine.Spec.Bootstrap.ConfigRef.Namespace, newMachine.Spec.Bootstrap.ConfigRef.Name))
68+
6269
// Requeue the control plane, in case there are additional operations to perform
6370
return ctrl.Result{Requeue: true}, nil
6471
}
@@ -87,12 +94,19 @@ func (r *KubeadmControlPlaneReconciler) scaleUpControlPlane(ctx context.Context,
8794
return ctrl.Result{}, err
8895
}
8996

90-
if err := r.cloneConfigsAndGenerateMachine(ctx, controlPlane.Cluster, controlPlane.KCP, bootstrapSpec, fd); err != nil {
97+
newMachine, err := r.cloneConfigsAndGenerateMachine(ctx, controlPlane.Cluster, controlPlane.KCP, bootstrapSpec, fd)
98+
if err != nil {
9199
logger.Error(err, "Failed to create additional control plane Machine")
92100
r.recorder.Eventf(controlPlane.KCP, corev1.EventTypeWarning, "FailedScaleUp", "Failed to create additional control plane Machine for cluster % control plane: %v", klog.KObj(controlPlane.Cluster), err)
93101
return ctrl.Result{}, err
94102
}
95103

104+
logger.WithValues(controlPlane.StatusToLogKeyAndValues(newMachine, nil)...).
105+
Info("Machine created (scale up)",
106+
"Machine", klog.KObj(newMachine),
107+
newMachine.Spec.InfrastructureRef.Kind, klog.KRef(newMachine.Spec.InfrastructureRef.Namespace, newMachine.Spec.InfrastructureRef.Name),
108+
newMachine.Spec.Bootstrap.ConfigRef.Kind, klog.KRef(newMachine.Spec.Bootstrap.ConfigRef.Namespace, newMachine.Spec.Bootstrap.ConfigRef.Name))
109+
96110
// Requeue the control plane, in case there are other operations to perform
97111
return ctrl.Result{Requeue: true}, nil
98112
}
@@ -138,13 +152,16 @@ func (r *KubeadmControlPlaneReconciler) scaleDownControlPlane(
138152
// NOTE: etcd member removal will be performed by the kcp-cleanup hook after machine completes drain & all volumes are detached.
139153
}
140154

141-
logger = logger.WithValues("Machine", klog.KObj(machineToDelete))
142155
if err := r.Client.Delete(ctx, machineToDelete); err != nil && !apierrors.IsNotFound(err) {
143156
logger.Error(err, "Failed to delete control plane machine")
144157
r.recorder.Eventf(controlPlane.KCP, corev1.EventTypeWarning, "FailedScaleDown",
145158
"Failed to delete control plane Machine %s for cluster %s control plane: %v", machineToDelete.Name, klog.KObj(controlPlane.Cluster), err)
146159
return ctrl.Result{}, err
147160
}
161+
// Note: We intentionally log after Delete because we want this log line to show up only after DeletionTimestamp has been set.
162+
// Also, setting DeletionTimestamp doesn't mean the Machine is actually deleted (deletion takes some time).
163+
logger.WithValues(controlPlane.StatusToLogKeyAndValues(nil, machineToDelete)...).
164+
Info("Deleting Machine (scale down)", "Machine", klog.KObj(machineToDelete))
148165

149166
// Requeue the control plane, in case there are additional operations to perform
150167
return ctrl.Result{Requeue: true}, nil

‎internal/controllers/machineset/machineset_controller.go‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -427,10 +427,12 @@ func (r *Reconciler) reconcileDelete(ctx context.Context, s *scope) (ctrl.Result
427427
// else delete owned machines.
428428
for _, machine := range machineList {
429429
if machine.DeletionTimestamp.IsZero() {
430-
log.Info("Deleting Machine", "Machine", klog.KObj(machine))
431430
if err := r.Client.Delete(ctx, machine); err != nil && !apierrors.IsNotFound(err) {
432431
return ctrl.Result{}, errors.Wrapf(err, "failed to delete Machine %s", klog.KObj(machine))
433432
}
433+
// Note: We intentionally log after Delete because we want this log line to show up only after DeletionTimestamp has been set.
434+
// Also, setting DeletionTimestamp doesn't mean the Machine is actually deleted (deletion takes some time).
435+
log.Info("Deleting Machine (MachineSet deleted)", "Machine", klog.KObj(machine))
434436
}
435437
}
436438

@@ -794,7 +796,7 @@ func (r *Reconciler) syncReplicas(ctx context.Context, s *scope) (ctrl.Result, e
794796
continue
795797
}
796798

797-
log.Info(fmt.Sprintf("Created machine %d of %d", i+1, diff), "Machine", klog.KObj(machine))
799+
log.Info(fmt.Sprintf("Machine created (scale up, creating %d of %d)", i+1, diff), "Machine", klog.KObj(machine))
798800
r.recorder.Eventf(ms, corev1.EventTypeNormal, "SuccessfulCreate", "Created machine %q", machine.Name)
799801
machineList = append(machineList, machine)
800802
}
@@ -816,16 +818,18 @@ func (r *Reconciler) syncReplicas(ctx context.Context, s *scope) (ctrl.Result, e
816818
for i, machine := range machinesToDelete {
817819
log := log.WithValues("Machine", klog.KObj(machine))
818820
if machine.GetDeletionTimestamp().IsZero() {
819-
log.Info(fmt.Sprintf("Deleting machine %d of %d", i+1, diff))
820821
if err := r.Client.Delete(ctx, machine); err != nil {
821822
log.Error(err, "Unable to delete Machine")
822823
r.recorder.Eventf(ms, corev1.EventTypeWarning, "FailedDelete", "Failed to delete machine %q: %v", machine.Name, err)
823824
errs = append(errs, err)
824825
continue
825826
}
827+
// Note: We intentionally log after Delete because we want this log line to show up only after DeletionTimestamp has been set.
828+
// Also, setting DeletionTimestamp doesn't mean the Machine is actually deleted (deletion takes some time).
829+
log.Info(fmt.Sprintf("Deleting Machine (scale down, deleting %d of %d)", i+1, diff))
826830
r.recorder.Eventf(ms, corev1.EventTypeNormal, "SuccessfulDelete", "Deleted machine %q", machine.Name)
827831
} else {
828-
log.Info(fmt.Sprintf("Waiting for machine %d of %d to be deleted", i+1, diff))
832+
log.Info(fmt.Sprintf("Waiting for Machine to be deleted (scale down, deleting %d of %d)", i+1, diff))
829833
}
830834
}
831835

@@ -1492,10 +1496,12 @@ func (r *Reconciler) reconcileUnhealthyMachines(ctx context.Context, s *scope) (
14921496
}
14931497
var errs []error
14941498
for _, m := range machinesToRemediate {
1495-
log.Info("Deleting unhealthy Machine", "Machine", klog.KObj(m))
14961499
if err := r.Client.Delete(ctx, m); err != nil && !apierrors.IsNotFound(err) {
14971500
errs = append(errs, errors.Wrapf(err, "failed to delete Machine %s", klog.KObj(m)))
14981501
}
1502+
// Note: We intentionally log after Delete because we want this log line to show up only after DeletionTimestamp has been set.
1503+
// Also, setting DeletionTimestamp doesn't mean the Machine is actually deleted (deletion takes some time).
1504+
log.Info("Deleting Machine (remediating unhealthy Machine)", "Machine", klog.KObj(m))
14991505
}
15001506
if len(errs) > 0 {
15011507
return ctrl.Result{}, errors.Wrapf(kerrors.NewAggregate(errs), "failed to delete unhealthy Machines")

0 commit comments

Comments
 (0)