fix: finalizer handling and improve return value semantics across con… (#148)

0x5457 · web-flow · commit 341b4f912791 · 2025-04-24T16:21:26.000+08:00
* fix: finalizer handling and improve return value semantics across controllers

* fix lint

* chore: add Eventually block to handle async cleanup in workload test
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -21,11 +21,13 @@
         "cycjimmy",
         "dylib",
         "essd",
+        "envtest",
         "Eventf",
         "finalizer",
         "Finalizers",
         "goconst",
         "golint",
+        "Gomega",
         "gopsutil",
         "gosec",
         "gpunode",
diff --git a/internal/controller/gpunode_controller.go b/internal/controller/gpunode_controller.go
@@ -71,7 +71,7 @@ func (r *GPUNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		return ctrl.Result{}, err
 	}
 
-	deleted, err := utils.HandleFinalizer(ctx, node, r.Client, func(ctx context.Context, node *tfv1.GPUNode) (bool, error) {
+	shouldReturn, err := utils.HandleFinalizer(ctx, node, r.Client, func(ctx context.Context, node *tfv1.GPUNode) (bool, error) {
 		if node.Status.Phase != tfv1.TensorFusionGPUNodePhaseDestroying {
 			node.Status.Phase = tfv1.TensorFusionGPUNodePhaseDestroying
 			if err := r.Status().Update(ctx, node); err != nil {
@@ -121,8 +121,8 @@ func (r *GPUNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 	if err != nil {
 		return ctrl.Result{}, err
 	}
-	if deleted {
-		return ctrl.Result{RequeueAfter: constants.PendingRequeueDuration}, nil
+	if shouldReturn {
+		return ctrl.Result{}, nil
 	}
 
 	var poolName string
diff --git a/internal/controller/gpupool_controller.go b/internal/controller/gpupool_controller.go
@@ -76,7 +76,7 @@ func (r *GPUPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 	}
 
 	// TODO: if phase is destroying, stop all existing workers and hypervisors, stop time series flow aggregations
-	deleted, err := utils.HandleFinalizer(ctx, pool, r.Client, func(ctx context.Context, pool *tfv1.GPUPool) (bool, error) {
+	shouldReturn, err := utils.HandleFinalizer(ctx, pool, r.Client, func(ctx context.Context, pool *tfv1.GPUPool) (bool, error) {
 		log.Info("TensorFusionGPUPool is being deleted", "name", pool.Name)
 		if pool.Status.Phase != tfv1.TensorFusionPoolPhaseDestroying {
 			pool.Status.Phase = tfv1.TensorFusionPoolPhaseDestroying
@@ -94,8 +94,10 @@ func (r *GPUPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 	if err != nil {
 		return ctrl.Result{}, err
 	}
-	if deleted {
-		return ctrl.Result{RequeueAfter: constants.PendingRequeueDuration}, nil
+	if shouldReturn {
+		// requeue for next loop
+		// we need manually requeue cause GenerationChangedPredicate
+		return ctrl.Result{Requeue: true}, nil
 	}
 
 	if err := r.reconcilePoolCurrentCapacityAndReadiness(ctx, pool); err != nil {
diff --git a/internal/controller/pod_controller.go b/internal/controller/pod_controller.go
@@ -52,9 +52,16 @@ type PodReconciler struct {
 func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := log.FromContext(ctx)
 	pod := &corev1.Pod{}
+	if err := r.Get(ctx, req.NamespacedName, pod); err != nil {
+		if errors.IsNotFound(err) {
+			return ctrl.Result{}, nil
+		}
+		log.Error(err, "Failed to get Pod")
+		return ctrl.Result{}, err
+	}
 
 	if _, ok := pod.Annotations[constants.TensorFusionEnabledReplicasAnnotation]; ok {
-		deleted, err := utils.HandleFinalizer(ctx, pod, r.Client, func(context context.Context, pod *corev1.Pod) (bool, error) {
+		shouldReturn, err := utils.HandleFinalizer(ctx, pod, r.Client, func(context context.Context, pod *corev1.Pod) (bool, error) {
 			counter := &v1.TensorFusionPodCounter{Client: r.Client}
 			if err := counter.Decrease(ctx, pod); err != nil {
 				return false, err
@@ -64,19 +71,11 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R
 		if err != nil {
 			return ctrl.Result{}, err
 		}
-		if deleted {
+		if shouldReturn {
 			return ctrl.Result{}, nil
 		}
 	}
 
-	if err := r.Get(ctx, req.NamespacedName, pod); err != nil {
-		if errors.IsNotFound(err) {
-			return ctrl.Result{}, nil
-		}
-		log.Error(err, "Failed to get Pod")
-		return ctrl.Result{}, err
-	}
-
 	// generate tensor fusion connections and apply to cluster
 	tfConnection := generateTensorFusionConnection(pod)
 	existConn := &tfv1.TensorFusionConnection{}
diff --git a/internal/controller/schedulingconfigtemplate_controller_test.go b/internal/controller/schedulingconfigtemplate_controller_test.go
@@ -37,7 +37,7 @@ var _ = Describe("SchedulingConfigTemplate Controller", func() {
 
 		typeNamespacedName := types.NamespacedName{
 			Name:      resourceName,
-			Namespace: "default", // TODO(user):Modify as needed
+			Namespace: "default",
 		}
 		schedulingconfigtemplate := &tfv1.SchedulingConfigTemplate{}
 
@@ -75,7 +75,6 @@ var _ = Describe("SchedulingConfigTemplate Controller", func() {
 				Client: k8sClient,
 				Scheme: k8sClient.Scheme(),
 			}
-
 			_, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
 				NamespacedName: typeNamespacedName,
 			})
diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go
@@ -103,9 +103,6 @@ var _ = BeforeSuite(func() {
 	err = corev1.AddToScheme(scheme.Scheme)
 	Expect(err).NotTo(HaveOccurred())
 
-	err = tfv1.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-
 	// +kubebuilder:scaffold:scheme
 
 	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
@@ -122,7 +119,6 @@ var _ = BeforeSuite(func() {
 		Scheme: scheme.Scheme,
 	})
 	Expect(err).ToNot(HaveOccurred())
-
 	err = (&TensorFusionClusterReconciler{
 		Client:   mgr.GetClient(),
 		Scheme:   mgr.GetScheme(),
@@ -200,7 +196,7 @@ var _ = BeforeSuite(func() {
 		Client:    mgr.GetClient(),
 		Scheme:    mgr.GetScheme(),
 		Scheduler: scheduler,
-		Recorder:  mgr.GetEventRecorderFor("tensorfusionworkload"),
+		Recorder:  mgr.GetEventRecorderFor("TensorFusionWorkload"),
 		GpuInfos:  config.MockGpuInfo(),
 	}).SetupWithManager(mgr)
 	Expect(err).ToNot(HaveOccurred())
@@ -455,6 +451,15 @@ func (b *TensorFusionEnvBuilder) Build() *TensorFusionEnv {
 	tfc.Spec.GPUPools = gpuPools
 	Expect(k8sClient.Create(ctx, tfc)).To(Succeed())
 
+	// wait for pools are created
+	Eventually(func(g Gomega) {
+		gpuPoolList := &tfv1.GPUPoolList{}
+		g.Expect(k8sClient.List(ctx, gpuPoolList, client.MatchingLabels(map[string]string{
+			constants.LabelKeyOwner: tfc.Name,
+		}))).Should(Succeed())
+		g.Expect(gpuPoolList.Items).Should(HaveLen(b.poolCount))
+	}, timeout*1000, interval).Should(Succeed())
+
 	// generate nodes
 	selectors := strings.Split(constants.InitialGPUNodeSelector, "=")
 	for poolIndex, nodeGpuMap := range b.poolNodeMap {
diff --git a/internal/controller/tensorfusioncluster_controller.go b/internal/controller/tensorfusioncluster_controller.go
@@ -93,7 +93,7 @@ func (r *TensorFusionClusterReconciler) Reconcile(ctx context.Context, req ctrl.
 	}
 	originalStatus := tfc.Status.DeepCopy()
 
-	deleted, err := utils.HandleFinalizer(ctx, tfc, r.Client, func(context context.Context, tfc *tfv1.TensorFusionCluster) (bool, error) {
+	shouldReturn, err := utils.HandleFinalizer(ctx, tfc, r.Client, func(context context.Context, tfc *tfv1.TensorFusionCluster) (bool, error) {
 		log.Info("TensorFusionCluster is being deleted", "name", tfc.Name)
 		if tfc.Status.Phase != tfv1.TensorFusionClusterDestroying {
 			tfc.Status.Phase = tfv1.TensorFusionClusterDestroying
@@ -110,8 +110,10 @@ func (r *TensorFusionClusterReconciler) Reconcile(ctx context.Context, req ctrl.
 	if err != nil {
 		return ctrl.Result{}, err
 	}
-	if deleted {
-		return ctrl.Result{RequeueAfter: constants.PendingRequeueDuration}, nil
+	if shouldReturn {
+		// requeue for next loop
+		// we need manually requeue cause GenerationChangedPredicate
+		return ctrl.Result{Requeue: true}, nil
 	}
 
 	if tfc.Status.Phase == "" || tfc.Status.Phase == constants.PhaseUnknown {
diff --git a/internal/controller/tensorfusionconnection_controller_test.go b/internal/controller/tensorfusionconnection_controller_test.go
@@ -19,7 +19,6 @@ package controller
 import (
 	"context"
 	"fmt"
-	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -36,7 +35,7 @@ import (
 var _ = Describe("TensorFusionConnection Controller", func() {
 	Context("When reconciling a resource", func() {
 		const resourceName = "test-resource"
-		const workloadName = "test-workload"
+		const workloadName = "test-workload-1"
 
 		ctx := context.Background()
 
@@ -122,8 +121,8 @@ var _ = Describe("TensorFusionConnection Controller", func() {
 			Expect(k8sClient.Create(ctx, connectionNoLabel)).To(Succeed())
 			Consistently(func(g Gomega) {
 				g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(connectionNoLabel), connectionNoLabel)).Should(Succeed())
-				g.Expect(connectionNoLabel.Status.WorkerName).Should(BeEmpty())
-			}, 5*time.Second, interval).Should(Succeed())
+				g.Expect(connectionNoLabel.Status.WorkerName).Should(Equal(""))
+			}, timeout, interval).Should(Succeed())
 
 			// Clean up the test connection
 			Expect(k8sClient.Delete(ctx, connectionNoLabel)).To(Succeed())
diff --git a/internal/controller/tensorfusionworkload_controller.go b/internal/controller/tensorfusionworkload_controller.go
@@ -79,14 +79,14 @@ func (r *TensorFusionWorkloadReconciler) Reconcile(ctx context.Context, req ctrl
 		return ctrl.Result{}, fmt.Errorf("list pods: %w", err)
 	}
 
-	deleted, err := utils.HandleFinalizer(ctx, workload, r.Client, func(ctx context.Context, _ *tfv1.TensorFusionWorkload) (bool, error) {
+	shouldReturn, err := utils.HandleFinalizer(ctx, workload, r.Client, func(ctx context.Context, _ *tfv1.TensorFusionWorkload) (bool, error) {
 		// check if all pods are deleted
 		return len(podList.Items) == 0, nil
 	})
 	if err != nil {
 		return ctrl.Result{}, fmt.Errorf("handle finalizer: %w", err)
 	}
-	if deleted {
+	if shouldReturn {
 		return ctrl.Result{}, nil
 	}
 
@@ -95,8 +95,9 @@ func (r *TensorFusionWorkloadReconciler) Reconcile(ctx context.Context, req ctrl
 	// Process pods with our finalizer
 	for i := range podList.Items {
 		pod := &podList.Items[i]
+		deleted := pod.DeletionTimestamp != nil
 		// Handle our GPU resource cleanup finalizer
-		deleted, err := utils.HandleFinalizer(ctx, pod, r.Client, func(ctx context.Context, obj *corev1.Pod) (bool, error) {
+		_, err := utils.HandleFinalizer(ctx, pod, r.Client, func(ctx context.Context, obj *corev1.Pod) (bool, error) {
 			return r.handlePodGPUCleanup(ctx, pod, workload)
 		})
 
diff --git a/internal/controller/tensorfusionworkload_controller_test.go b/internal/controller/tensorfusionworkload_controller_test.go
@@ -192,7 +192,7 @@ var _ = Describe("TensorFusionWorkload Controller", func() {
 				g.Expect(k8sClient.List(ctx, podList,
 					client.InNamespace(workload.Namespace),
 					client.MatchingLabels{constants.WorkloadKey: workload.Name})).Should(Succeed())
-				g.Expect(podList.Items).Should(BeNil())
+				g.Expect(podList.Items).Should(BeEmpty())
 			}, 5*time.Second, 100*time.Millisecond).Should(Succeed())
 		})
 	})
@@ -261,10 +261,14 @@ func createTensorFusionWorkload(poolName string, key client.ObjectKey, replicas
 func cleanupWorkload(key client.ObjectKey) {
 	GinkgoHelper()
 	workload := &tfv1.TensorFusionWorkload{}
-	Expect(k8sClient.Get(ctx, key, workload)).Should(Succeed())
-	workloadCopy := workload.DeepCopy()
-	workloadCopy.Spec.Replicas = ptr.Int32(0)
-	Expect(k8sClient.Update(ctx, workloadCopy)).To(Succeed())
+
+	// Set replicas to 0
+	Eventually(func(g Gomega) {
+		g.Expect(k8sClient.Get(ctx, key, workload)).Should(Succeed())
+		workload.Spec.Replicas = ptr.Int32(0)
+		g.Expect(k8sClient.Update(ctx, workload)).To(Succeed())
+	}, timeout, interval).Should(Succeed())
+
 	Eventually(func(g Gomega) {
 		podList := &corev1.PodList{}
 		g.Expect(k8sClient.List(ctx, podList,
diff --git a/internal/controller/workloadprofile_controller_test.go b/internal/controller/workloadprofile_controller_test.go
@@ -38,27 +38,29 @@ var _ = Describe("WorkloadProfile Controller", func() {
 
 		typeNamespacedName := types.NamespacedName{
 			Name:      resourceName,
-			Namespace: "default", // TODO(user):Modify as needed
+			Namespace: "default",
 		}
-		workloadprofile := &tfv1.WorkloadProfile{}
 
 		BeforeEach(func() {
 			By("creating the custom resource for the Kind WorkloadProfile")
-			err := k8sClient.Get(ctx, typeNamespacedName, workloadprofile)
+			workloadProfile := &tfv1.WorkloadProfile{}
+			err := k8sClient.Get(ctx, typeNamespacedName, workloadProfile)
 			if err != nil && errors.IsNotFound(err) {
 				resource := &tfv1.WorkloadProfile{
 					ObjectMeta: metav1.ObjectMeta{
 						Name:      resourceName,
 						Namespace: "default",
 					},
-					// TODO(user): Specify other spec details if needed.
+					Spec: tfv1.WorkloadProfileSpec{
+						PoolName:   "mock",
+						IsLocalGPU: false,
+					},
 				}
 				Expect(k8sClient.Create(ctx, resource)).To(Succeed())
 			}
 		})
 
 		AfterEach(func() {
-			// TODO(user): Cleanup logic after each test, like removing the resource instance.
 			resource := &tfv1.WorkloadProfile{}
 			err := k8sClient.Get(ctx, typeNamespacedName, resource)
 			Expect(err).NotTo(HaveOccurred())
diff --git a/internal/utils/reconcile.go b/internal/utils/reconcile.go
@@ -27,36 +27,64 @@ var ErrNextLoop = errors.New("stop this loop and return the associated Result ob
 // ErrTerminateLoop is not a real error. It forces the current reconciliation loop to stop
 var ErrTerminateLoop = errors.New("stop this loop and do not requeue")
 
-func HandleFinalizer[T client.Object](ctx context.Context, obj T, r client.Client, deleteHook func(context.Context, T) (bool, error)) (bool, error) {
-	// Check if object is being deleted
-	deleted := !obj.GetDeletionTimestamp().IsZero()
-	if deleted {
-		// Object is being deleted - process finalizer
+// HandleFinalizer ensures proper finalizer management for Kubernetes resources.
+// It automatically adds the finalizer when needed, and removes it after successful cleanup.
+// Returns (shouldReturn, err):
+//   - shouldReturn: true if the caller should immediately return and wait for the next reconcile.
+//   - err: any error encountered during update or deleteHook.
+func HandleFinalizer[T client.Object](
+	ctx context.Context,
+	obj T,
+	r client.Client,
+	deleteHook func(context.Context, T) (bool, error),
+) (shouldReturn bool, err error) {
+	// If the object is being deleted, handle finalizer removal
+	if !obj.GetDeletionTimestamp().IsZero() {
 		if controllerutil.ContainsFinalizer(obj, constants.Finalizer) {
-			// Run custom deletion hook
-			canBeDeleted, err := deleteHook(ctx, obj)
+			// Run custom deletion logic before removing the finalizer
+			var canBeDeleted bool
+			canBeDeleted, err = deleteHook(ctx, obj)
 			if err != nil {
-				return false, err
+				// Error during deletion hook, requeue for next reconcile
+				shouldReturn = true
+				return shouldReturn, err
 			}
-
-			// Remove finalizer once cleanup is done
 			if canBeDeleted {
 				controllerutil.RemoveFinalizer(obj, constants.Finalizer)
-				if err := r.Update(ctx, obj); err != nil {
-					return false, err
+				err = r.Update(ctx, obj)
+				if err != nil {
+					// Failed to update object, requeue for next reconcile
+					return shouldReturn, err
 				}
+				// Finalizer removed, wait for next reconcile
+				shouldReturn = true
+				return shouldReturn, err
 			}
+			// Cleanup not ready, wait for next reconcile
+			shouldReturn = true
+			return shouldReturn, err
 		}
-	} else {
-		// Object is not being deleted - add finalizer if not present
-		if !controllerutil.ContainsFinalizer(obj, constants.Finalizer) {
-			controllerutil.AddFinalizer(obj, constants.Finalizer)
-			if err := r.Update(ctx, obj); err != nil {
-				return false, err
-			}
+		// Finalizer already removed, continue with deletion
+		shouldReturn = false
+		return shouldReturn, err
+	}
+
+	// If the object is not being deleted, ensure the finalizer is present
+	if !controllerutil.ContainsFinalizer(obj, constants.Finalizer) {
+		controllerutil.AddFinalizer(obj, constants.Finalizer)
+		err = r.Update(ctx, obj)
+		if err != nil {
+			// Failed to update object, requeue for next reconcile
+			return shouldReturn, err
 		}
+		// Finalizer added, wait for next reconcile
+		shouldReturn = true
+		return shouldReturn, err
 	}
-	return deleted, nil
+
+	// Finalizer already present, continue with business logic
+	shouldReturn = false
+	return shouldReturn, err
 }
 
 func CalculateExponentialBackoffWithJitter(retryCount int64) time.Duration {
diff --git a/internal/webhook/v1/pod_webhook_test.go b/internal/webhook/v1/pod_webhook_test.go

Original file line number	Diff line number	Diff line change
`@@ -71,7 +71,7 @@ func (r *GPUNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct`
`71`	`71`	`return ctrl.Result{}, err`
`72`	`72`	`}`
`73`	`73`
`74`		`- deleted, err := utils.HandleFinalizer(ctx, node, r.Client, func(ctx context.Context, node *tfv1.GPUNode) (bool, error) {`
	`74`	`+ shouldReturn, err := utils.HandleFinalizer(ctx, node, r.Client, func(ctx context.Context, node *tfv1.GPUNode) (bool, error) {`
`75`	`75`	`if node.Status.Phase != tfv1.TensorFusionGPUNodePhaseDestroying {`
`76`	`76`	`node.Status.Phase = tfv1.TensorFusionGPUNodePhaseDestroying`
`77`	`77`	`if err := r.Status().Update(ctx, node); err != nil {`
`@@ -121,8 +121,8 @@ func (r *GPUNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct`
`121`	`121`	`if err != nil {`
`122`	`122`	`return ctrl.Result{}, err`
`123`	`123`	`}`
`124`		`- if deleted {`
`125`		`- return ctrl.Result{RequeueAfter: constants.PendingRequeueDuration}, nil`
	`124`	`+ if shouldReturn {`
	`125`	`+ return ctrl.Result{}, nil`
`126`	`126`	`}`
`127`	`127`
`128`	`128`	`var poolName string`
Original file line number	Diff line number	Diff line change
`@@ -76,7 +76,7 @@ func (r *GPUPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct`
`76`	`76`	`}`
`77`	`77`
`78`	`78`	`// TODO: if phase is destroying, stop all existing workers and hypervisors, stop time series flow aggregations`
`79`		`- deleted, err := utils.HandleFinalizer(ctx, pool, r.Client, func(ctx context.Context, pool *tfv1.GPUPool) (bool, error) {`
	`79`	`+ shouldReturn, err := utils.HandleFinalizer(ctx, pool, r.Client, func(ctx context.Context, pool *tfv1.GPUPool) (bool, error) {`
`80`	`80`	`log.Info("TensorFusionGPUPool is being deleted", "name", pool.Name)`
`81`	`81`	`if pool.Status.Phase != tfv1.TensorFusionPoolPhaseDestroying {`
`82`	`82`	`pool.Status.Phase = tfv1.TensorFusionPoolPhaseDestroying`
`@@ -94,8 +94,10 @@ func (r *GPUPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct`
`94`	`94`	`if err != nil {`
`95`	`95`	`return ctrl.Result{}, err`
`96`	`96`	`}`
`97`		`- if deleted {`
`98`		`- return ctrl.Result{RequeueAfter: constants.PendingRequeueDuration}, nil`
	`97`	`+ if shouldReturn {`
	`98`	`+ // requeue for next loop`
	`99`	`+ // we need manually requeue cause GenerationChangedPredicate`
	`100`	`+ return ctrl.Result{Requeue: true}, nil`
`99`	`101`	`}`
`100`	`102`
`101`	`103`	`if err := r.reconcilePoolCurrentCapacityAndReadiness(ctx, pool); err != nil {`
Original file line number	Diff line number	Diff line change
`@@ -37,7 +37,7 @@ var _ = Describe("SchedulingConfigTemplate Controller", func() {`
`37`	`37`
`38`	`38`	`typeNamespacedName := types.NamespacedName{`
`39`	`39`	`Name: resourceName,`
`40`		`- Namespace: "default", // TODO(user):Modify as needed`
	`40`	`+ Namespace: "default",`
`41`	`41`	`}`
`42`	`42`	`schedulingconfigtemplate := &tfv1.SchedulingConfigTemplate{}`
`43`	`43`
`@@ -75,7 +75,6 @@ var _ = Describe("SchedulingConfigTemplate Controller", func() {`
`75`	`75`	`Client: k8sClient,`
`76`	`76`	`Scheme: k8sClient.Scheme(),`
`77`	`77`	`}`
`78`		`-`
`79`	`78`	`_, err := controllerReconciler.Reconcile(ctx, reconcile.Request{`
`80`	`79`	`NamespacedName: typeNamespacedName,`
`81`	`80`	`})`
Original file line number	Diff line number	Diff line change
`@@ -93,7 +93,7 @@ func (r *TensorFusionClusterReconciler) Reconcile(ctx context.Context, req ctrl.`
`93`	`93`	`}`
`94`	`94`	`originalStatus := tfc.Status.DeepCopy()`
`95`	`95`
`96`		`- deleted, err := utils.HandleFinalizer(ctx, tfc, r.Client, func(context context.Context, tfc *tfv1.TensorFusionCluster) (bool, error) {`
	`96`	`+ shouldReturn, err := utils.HandleFinalizer(ctx, tfc, r.Client, func(context context.Context, tfc *tfv1.TensorFusionCluster) (bool, error) {`
`97`	`97`	`log.Info("TensorFusionCluster is being deleted", "name", tfc.Name)`
`98`	`98`	`if tfc.Status.Phase != tfv1.TensorFusionClusterDestroying {`
`99`	`99`	`tfc.Status.Phase = tfv1.TensorFusionClusterDestroying`
`@@ -110,8 +110,10 @@ func (r *TensorFusionClusterReconciler) Reconcile(ctx context.Context, req ctrl.`
`110`	`110`	`if err != nil {`
`111`	`111`	`return ctrl.Result{}, err`
`112`	`112`	`}`
`113`		`- if deleted {`
`114`		`- return ctrl.Result{RequeueAfter: constants.PendingRequeueDuration}, nil`
	`113`	`+ if shouldReturn {`
	`114`	`+ // requeue for next loop`
	`115`	`+ // we need manually requeue cause GenerationChangedPredicate`
	`116`	`+ return ctrl.Result{Requeue: true}, nil`
`115`	`117`	`}`
`116`	`118`
`117`	`119`	`if tfc.Status.Phase == "" \|\| tfc.Status.Phase == constants.PhaseUnknown {`