Skip to content

Commit df5775a

Browse files
johscheuerjohscheuer
authored
Simplify the confirmRemoval code (#2148)
1 parent be79b33 commit df5775a

File tree

3 files changed

+76
-77
lines changed

3 files changed

+76
-77
lines changed

‎controllers/remove_process_groups.go

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -198,21 +198,21 @@ func removeProcessGroup(ctx context.Context, r *FoundationDBClusterReconciler, c
198198
return deletionError
199199
}
200200

201-
func confirmRemoval(ctx context.Context, logger logr.Logger, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, processGroup *fdbv1beta2.ProcessGroupStatus) (bool, bool, error) {
201+
func confirmRemoval(ctx context.Context, logger logr.Logger, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, processGroup *fdbv1beta2.ProcessGroupStatus) (bool, error) {
202202
canBeIncluded := true
203203

204204
podName := processGroup.GetPodName(cluster)
205205
pod, err := r.PodLifecycleManager.GetPod(ctx, r, cluster, podName)
206206
// If we get an error different from not found we will return the error.
207207
if err != nil && !k8serrors.IsNotFound(err) {
208-
return false, false, err
208+
return false, err
209209
}
210210

211211
// The Pod resource still exists, so we have to validate the deletion timestamp.
212212
if err == nil {
213213
if pod.DeletionTimestamp.IsZero() {
214214
logger.Info("Waiting for process group to get torn down", "processGroupID", processGroup.ProcessGroupID, "pod", podName)
215-
return false, false, nil
215+
return false, internal.ResourceNotDeleted{Resource: pod}
216216
}
217217

218218
// Pod is in terminating state so we don't want to block, but we also don't want to include it
@@ -223,40 +223,41 @@ func confirmRemoval(ctx context.Context, logger logr.Logger, r *FoundationDBClus
223223
pvcs := &corev1.PersistentVolumeClaimList{}
224224
err = r.List(ctx, pvcs, internal.GetSinglePodListOptions(cluster, processGroup.ProcessGroupID)...)
225225
if err != nil {
226-
return false, canBeIncluded, err
226+
return false, err
227227
}
228228

229229
if len(pvcs.Items) == 1 {
230-
if pvcs.Items[0].DeletionTimestamp == nil {
231-
logger.Info("Waiting for volume claim to get torn down", "processGroupID", processGroup.ProcessGroupID, "pvc", pvcs.Items[0].Name)
232-
return false, false, nil
230+
pvc := pvcs.Items[0]
231+
if pvc.DeletionTimestamp == nil {
232+
logger.Info("Waiting for volume claim to get torn down", "processGroupID", processGroup.ProcessGroupID, "pvc", pvc.Name)
233+
return false, internal.ResourceNotDeleted{Resource: &pvc}
233234
}
234235

235236
// PVC is in terminating state so we don't want to block, but we also don't want to include it
236237
canBeIncluded = false
237238
} else if len(pvcs.Items) > 1 {
238-
return false, false, fmt.Errorf("multiple PVCs found for cluster %s, processGroupID %s", cluster.Name, processGroup.ProcessGroupID)
239+
return false, fmt.Errorf("multiple PVCs found for cluster %s, processGroupID %s", cluster.Name, processGroup.ProcessGroupID)
239240
}
240241

241242
service := &corev1.Service{}
242243
err = r.Get(ctx, client.ObjectKey{Name: podName, Namespace: cluster.Namespace}, service)
243244
// If we get an error different from not found we will return the error.
244245
if err != nil && !k8serrors.IsNotFound(err) {
245-
return false, false, err
246+
return false, err
246247
}
247248

248249
// The Pod resource still exists, so we have to validate the deletion timestamp.
249250
if err == nil {
250251
if service.DeletionTimestamp.IsZero() {
251252
logger.Info("Waiting for process group to get torn down", "processGroupID", processGroup.ProcessGroupID, "service", podName)
252-
return false, false, nil
253+
return false, internal.ResourceNotDeleted{Resource: service}
253254
}
254255

255256
// Service is in terminating state so we don't want to block, but we also don't want to include it
256257
canBeIncluded = false
257258
}
258259

259-
return true, canBeIncluded, nil
260+
return canBeIncluded, nil
260261
}
261262

262263
func includeProcessGroup(ctx context.Context, logger logr.Logger, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster, removedProcessGroups map[fdbv1beta2.ProcessGroupID]bool, status *fdbv1beta2.FoundationDBStatus, adminClient fdbadminclient.AdminClient) error {
@@ -420,18 +421,15 @@ func (r *FoundationDBClusterReconciler) removeProcessGroups(ctx context.Context,
420421
// We have to check if the currently removed process groups are completely removed.
421422
// In addition, we have to check if one of the terminating process groups has been cleaned up.
422423
for _, processGroup := range processGroups {
423-
removed, include, err := confirmRemoval(ctx, logger, r, cluster, processGroup)
424-
if err != nil {
424+
include, err := confirmRemoval(ctx, logger, r, cluster, processGroup)
425+
if err != nil && !internal.IsResourceNotDeleted(err) {
425426
logger.Error(err, "Error during confirm process group removal", "processGroupID", processGroup.ProcessGroupID)
426427
continue
427428
}
428429

429-
if removed {
430-
// Pods that are stuck in terminating shouldn't block reconciliation, but we also
431-
// don't want to include them since they have an unknown state.
432-
removedProcessGroups[processGroup.ProcessGroupID] = include
433-
continue
434-
}
430+
// Pods that are stuck in terminating shouldn't block reconciliation, but we also
431+
// don't want to include them since they have an unknown state.
432+
removedProcessGroups[processGroup.ProcessGroupID] = include
435433
}
436434

437435
return removedProcessGroups

‎controllers/remove_process_groups_test.go

Lines changed: 40 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -130,9 +130,9 @@ var _ = Describe("remove_process_groups", func() {
130130
It("should not remove that process group", func() {
131131
Expect(result).To(BeNil())
132132
// Ensure resources are not deleted
133-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
134-
Expect(err).To(BeNil())
135-
Expect(removed).To(BeFalse())
133+
include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
134+
Expect(err).NotTo(BeNil())
135+
Expect(internal.IsResourceNotDeleted(err)).To(BeTrue())
136136
Expect(include).To(BeFalse())
137137
})
138138
})
@@ -149,9 +149,8 @@ var _ = Describe("remove_process_groups", func() {
149149
It("should successfully remove that process group", func() {
150150
Expect(result).To(BeNil())
151151
// Ensure resources are deleted
152-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
152+
include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
153153
Expect(err).To(BeNil())
154-
Expect(removed).To(BeTrue())
155154
Expect(include).To(BeTrue())
156155
})
157156
})
@@ -161,9 +160,8 @@ var _ = Describe("remove_process_groups", func() {
161160
It("should successfully remove that process group", func() {
162161
Expect(result).To(BeNil())
163162
// Ensure resources are deleted
164-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
163+
include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
165164
Expect(err).To(BeNil())
166-
Expect(removed).To(BeTrue())
167165
Expect(include).To(BeTrue())
168166
})
169167
})
@@ -187,9 +185,9 @@ var _ = Describe("remove_process_groups", func() {
187185
Expect(result).NotTo(BeNil())
188186
Expect(result.message).To(Equal("Removals cannot proceed because cluster has degraded fault tolerance"))
189187
// Ensure resources are not deleted
190-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
191-
Expect(err).To(BeNil())
192-
Expect(removed).To(BeFalse())
188+
include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
189+
Expect(err).NotTo(BeNil())
190+
Expect(internal.IsResourceNotDeleted(err)).To(BeTrue())
193191
Expect(include).To(BeFalse())
194192
})
195193
})
@@ -211,9 +209,9 @@ var _ = Describe("remove_process_groups", func() {
211209
Expect(result).NotTo(BeNil())
212210
Expect(result.message).To(Equal("Removals cannot proceed because cluster has degraded fault tolerance"))
213211
// Ensure resources are not deleted
214-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
215-
Expect(err).To(BeNil())
216-
Expect(removed).To(BeFalse())
212+
include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
213+
Expect(err).NotTo(BeNil())
214+
Expect(internal.IsResourceNotDeleted(err)).To(BeTrue())
217215
Expect(include).To(BeFalse())
218216
})
219217
})
@@ -235,9 +233,9 @@ var _ = Describe("remove_process_groups", func() {
235233
Expect(result).NotTo(BeNil())
236234
Expect(result.curError).To(HaveOccurred())
237235
// Ensure resources are not deleted
238-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
239-
Expect(err).To(BeNil())
240-
Expect(removed).To(BeFalse())
236+
include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
237+
Expect(err).NotTo(BeNil())
238+
Expect(internal.IsResourceNotDeleted(err)).To(BeTrue())
241239
Expect(include).To(BeFalse())
242240
})
243241
})
@@ -267,9 +265,8 @@ var _ = Describe("remove_process_groups", func() {
267265
It("should successfully remove that process group", func() {
268266
Expect(result).To(BeNil())
269267
// Ensure resources are deleted
270-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
268+
include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
271269
Expect(err).To(BeNil())
272-
Expect(removed).To(BeTrue())
273270
Expect(include).To(BeTrue())
274271
})
275272
})
@@ -291,9 +288,9 @@ var _ = Describe("remove_process_groups", func() {
291288
Expect(result).NotTo(BeNil())
292289
Expect(result.message).To(Equal("Removals cannot proceed because cluster has degraded fault tolerance"))
293290
// Ensure resources are not deleted
294-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
295-
Expect(err).To(BeNil())
296-
Expect(removed).To(BeFalse())
291+
include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
292+
Expect(err).NotTo(BeNil())
293+
Expect(internal.IsResourceNotDeleted(err)).To(BeTrue())
297294
Expect(include).To(BeFalse())
298295
})
299296
})
@@ -323,13 +320,11 @@ var _ = Describe("remove_process_groups", func() {
323320
Expect(result).To(BeNil())
324321
Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 1))
325322
// Check if resources are deleted
326-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
327-
Expect(err).To(BeNil())
323+
include, errFirst := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
328324
// Check if resources are deleted
329-
removedSecondary, includeSecondary, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
330-
Expect(err).To(BeNil())
325+
includeSecondary, errSecond := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
331326
// Make sure only one of the process groups was deleted.
332-
Expect(removed).NotTo(Equal(removedSecondary))
327+
Expect(errFirst).NotTo(Equal(errSecond))
333328
Expect(include).NotTo(Equal(includeSecondary))
334329
})
335330

@@ -343,14 +338,12 @@ var _ = Describe("remove_process_groups", func() {
343338
Expect(result).To(BeNil())
344339
Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
345340
// Ensure resources are deleted
346-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
341+
include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
347342
Expect(err).To(BeNil())
348-
Expect(removed).To(BeTrue())
349343
Expect(include).To(BeTrue())
350344
// Ensure resources are deleted
351-
removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
345+
include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
352346
Expect(err).To(BeNil())
353-
Expect(removed).To(BeTrue())
354347
Expect(include).To(BeTrue())
355348
})
356349
})
@@ -364,14 +357,12 @@ var _ = Describe("remove_process_groups", func() {
364357
Expect(result).To(BeNil())
365358
Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
366359
// Ensure resources are deleted
367-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
360+
include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
368361
Expect(err).To(BeNil())
369-
Expect(removed).To(BeTrue())
370362
Expect(include).To(BeTrue())
371363
// Ensure resources are deleted
372-
removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
364+
include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
373365
Expect(err).To(BeNil())
374-
Expect(removed).To(BeTrue())
375366
Expect(include).To(BeTrue())
376367
})
377368
})
@@ -388,14 +379,14 @@ var _ = Describe("remove_process_groups", func() {
388379
Expect(result.message).To(HavePrefix("not allowed to remove process groups, waiting:"))
389380
Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 0))
390381
// Ensure resources are not deleted
391-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
392-
Expect(err).To(BeNil())
393-
Expect(removed).To(BeFalse())
382+
include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
383+
Expect(err).NotTo(BeNil())
384+
Expect(internal.IsResourceNotDeleted(err)).To(BeTrue())
394385
Expect(include).To(BeFalse())
395-
// Ensure resources are deleted
396-
removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
397-
Expect(err).To(BeNil())
398-
Expect(removed).To(BeFalse())
386+
// Ensure resources are not deleted
387+
include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
388+
Expect(err).NotTo(BeNil())
389+
Expect(internal.IsResourceNotDeleted(err)).To(BeTrue())
399390
Expect(include).To(BeFalse())
400391
})
401392
})
@@ -425,14 +416,12 @@ var _ = Describe("remove_process_groups", func() {
425416
Expect(result).To(BeNil())
426417
Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
427418
// Ensure resources are deleted
428-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
419+
include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
429420
Expect(err).To(BeNil())
430-
Expect(removed).To(BeTrue())
431421
Expect(include).To(BeTrue())
432422
// Ensure resources are deleted as the RemovalMode is PodUpdateModeAll
433-
removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
423+
include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
434424
Expect(err).To(BeNil())
435-
Expect(removed).To(BeTrue())
436425
Expect(include).To(BeTrue())
437426
})
438427

@@ -446,14 +435,12 @@ var _ = Describe("remove_process_groups", func() {
446435
Expect(result).To(BeNil())
447436
Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
448437
// Ensure resources are deleted
449-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
438+
include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
450439
Expect(err).To(BeNil())
451-
Expect(removed).To(BeTrue())
452440
Expect(include).To(BeTrue())
453441
// Ensure resources are deleted
454-
removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
442+
include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
455443
Expect(err).To(BeNil())
456-
Expect(removed).To(BeTrue())
457444
Expect(include).To(BeTrue())
458445
})
459446
})
@@ -467,14 +454,12 @@ var _ = Describe("remove_process_groups", func() {
467454
Expect(result).To(BeNil())
468455
Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
469456
// Ensure resources are deleted
470-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
457+
include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
471458
Expect(err).To(BeNil())
472-
Expect(removed).To(BeTrue())
473459
Expect(include).To(BeTrue())
474460
// Ensure resources are deleted
475-
removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
461+
include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
476462
Expect(err).To(BeNil())
477-
Expect(removed).To(BeTrue())
478463
Expect(include).To(BeTrue())
479464
})
480465
})
@@ -490,14 +475,12 @@ var _ = Describe("remove_process_groups", func() {
490475
Expect(result).To(BeNil())
491476
Expect(initialCnt - len(cluster.Status.ProcessGroups)).To(BeNumerically("==", 2))
492477
// Ensure resources are not deleted
493-
removed, include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
478+
include, err := confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, removedProcessGroup)
494479
Expect(err).To(BeNil())
495-
Expect(removed).To(BeTrue())
496480
Expect(include).To(BeTrue())
497481
// Ensure resources are deleted
498-
removed, include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
482+
include, err = confirmRemoval(context.Background(), globalControllerLogger, clusterReconciler, cluster, secondRemovedProcessGroup)
499483
Expect(err).To(BeNil())
500-
Expect(removed).To(BeTrue())
501484
Expect(include).To(BeTrue())
502485
})
503486
})

‎internal/error_helper.go

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,32 @@ package internal
2222

2323
import (
2424
"errors"
25+
"fmt"
2526
"net"
2627
"strings"
2728

28-
fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
29+
"sigs.k8s.io/controller-runtime/pkg/client"
2930

31+
fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
3032
k8serrors "k8s.io/apimachinery/pkg/api/errors"
3133
)
3234

35+
// ResourceNotDeleted can be returned if the resource is not yet deleted.
36+
type ResourceNotDeleted struct {
37+
Resource client.Object
38+
}
39+
40+
// Error returns the error message of the internal timeout error.
41+
func (err ResourceNotDeleted) Error() string {
42+
return fmt.Sprintf("resource: %s is missing the deletion timestamp.", err.Resource.GetName())
43+
}
44+
45+
// IsResourceNotDeleted returns true if provided error is of the type IsResourceNotDeleted.
46+
func IsResourceNotDeleted(err error) bool {
47+
var targetErr ResourceNotDeleted
48+
return errors.As(err, &targetErr)
49+
}
50+
3351
// IsNetworkError returns true if the network is a network error net.Error
3452
func IsNetworkError(err error) bool {
3553
var netError net.Error

0 commit comments

Comments
 (0)