fix: stop reconcilation when gpunode or gpupool is deleted (#214)

* fix: stop reconcilation when gpunode or gpupool is deleted

* fix: stop reconcilation when cluster is deleted and linter issue

Author: knave

internal/controller/gpunode_controller.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"strings"
 	"time"
 
 	tfv1 "github.com/NexusGPU/tensor-fusion/api/v1"
@@ -124,18 +123,11 @@ func (r *GPUNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 	if err != nil {
 		return ctrl.Result{}, err
 	}
-	if shouldReturn {
+	if shouldReturn || !node.DeletionTimestamp.IsZero() {
 		return ctrl.Result{}, nil
 	}
 
-	var poolName string
-	for labelKey := range node.Labels {
-		after, ok := strings.CutPrefix(labelKey, constants.GPUNodePoolIdentifierLabelPrefix)
-		if ok {
-			poolName = after
-			break
-		}
-	}
+	poolName := utils.ExtractPoolNameFromNodeLabel(node)
 	if poolName == "" {
 		log.Error(nil, "failed to get pool name", "node", node.Name)
 		return ctrl.Result{}, nil

internal/controller/gpupool_controller.go

@@ -98,7 +98,7 @@ func (r *GPUPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 	if err != nil {
 		return ctrl.Result{}, err
 	}
-	if shouldReturn {
+	if shouldReturn || !pool.DeletionTimestamp.IsZero() {
 		// requeue for next loop
 		// we need manually requeue cause GenerationChangedPredicate
 		return ctrl.Result{Requeue: true}, nil

internal/controller/tensorfusioncluster_controller.go

@@ -110,7 +110,7 @@ func (r *TensorFusionClusterReconciler) Reconcile(ctx context.Context, req ctrl.
 	if err != nil {
 		return ctrl.Result{}, err
 	}
-	if shouldReturn {
+	if shouldReturn || !tfc.DeletionTimestamp.IsZero() {
 		// requeue for next loop
 		// we need manually requeue cause GenerationChangedPredicate
 		return ctrl.Result{Requeue: true}, nil

internal/utils/reconcile.go

@@ -9,12 +9,14 @@ import (
 	"math"
 	"math/rand/v2"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
 	constants "github.com/NexusGPU/tensor-fusion/internal/constants"
 	"k8s.io/apimachinery/pkg/types"
 
+	tfv1 "github.com/NexusGPU/tensor-fusion/api/v1"
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -184,3 +186,15 @@ func IsPodConditionTrue(conditions []corev1.PodCondition, conditionType corev1.P
 func IsPodTerminated(pod *corev1.Pod) bool {
 	return pod.Status.Phase == corev1.PodFailed || pod.Status.Phase == corev1.PodSucceeded
 }
+
+func ExtractPoolNameFromNodeLabel(node *tfv1.GPUNode) string {
+	var poolName string
+	for labelKey := range node.Labels {
+		after, ok := strings.CutPrefix(labelKey, constants.GPUNodePoolIdentifierLabelPrefix)
+		if ok {
+			poolName = after
+			break
+		}
+	}
+	return poolName
+}