feat: add ClusterStagedUpdateRun controller and handle deletion (#951)

add ClusterStagedUpdateRun controller and handle deletion

Author: jwtty

apis/placement/v1alpha1/common.go

@@ -14,4 +14,23 @@ const (
 
 	// ResourceOverrideSnapshotKind is the kind of the ResourceOverrideSnapshotKind.
 	ResourceOverrideSnapshotKind = "ResourceOverrideSnapshot"
+
+	// ClusterStagedUpdateRunFinalizer is used by the ClusterStagedUpdateRun controller to make sure that the ClusterStagedUpdateRun
+	// object is not deleted until all its dependent resources are deleted.
+	ClusterStagedUpdateRunFinalizer = fleetPrefix + "stagedupdaterun-finalizer"
+
+	// TargetUpdateRunLabel indicates the target update run on a staged run related object.
+	TargetUpdateRunLabel = fleetPrefix + "targetupdaterun"
+
+	// UpdateRunDeleteStageName is the name of delete stage in the staged update run.
+	UpdateRunDeleteStageName = fleetPrefix + "deleteStage"
+
+	// IsLatestUpdateRunApprovalLabel indicates if the approval is the latest approval on a staged run.
+	IsLatestUpdateRunApprovalLabel = fleetPrefix + "isLatestUpdateRunApproval"
+
+	// TargetUpdatingStageNameLabel indicates the updating stage name on a staged run related object.
+	TargetUpdatingStageNameLabel = fleetPrefix + "targetUpdatingStage"
+
+	// ApprovalTaskNameFmt is the format of the approval task name.
+	ApprovalTaskNameFmt = "%s-%s"
 )

apis/placement/v1beta1/commons.go

@@ -6,18 +6,25 @@ Licensed under the MIT license.
 package v1beta1
 
 const (
-	ClusterResourcePlacementKind        = "ClusterResourcePlacement"
-	ClusterResourcePlacementResource    = "clusterresourceplacements"
-	ClusterResourceBindingKind          = "ClusterResourceBinding"
-	ClusterResourceSnapshotKind         = "ClusterResourceSnapshot"
+	// ClusterResourcePlacementKind represents the kind of ClusterResourcePlacement.
+	ClusterResourcePlacementKind = "ClusterResourcePlacement"
+	// ClusterResourcePlacementResource represents the resource name for ClusterResourcePlacement.
+	ClusterResourcePlacementResource = "clusterresourceplacements"
+	// ClusterResourceBindingKind represents the kind of ClusterResourceBinding.
+	ClusterResourceBindingKind = "ClusterResourceBinding"
+	// ClusterResourceSnapshotKind represents the kind of ClusterResourceSnapshot.
+	ClusterResourceSnapshotKind = "ClusterResourceSnapshot"
+	// ClusterSchedulingPolicySnapshotKind represents the kind of ClusterSchedulingPolicySnapshot.
 	ClusterSchedulingPolicySnapshotKind = "ClusterSchedulingPolicySnapshot"
-	WorkKind                            = "Work"
-	AppliedWorkKind                     = "AppliedWork"
+	// WorkKind represents the kind of Work.
+	WorkKind = "Work"
+	// AppliedWorkKind represents the kind of AppliedWork.
+	AppliedWorkKind = "AppliedWork"
 )
 
 const (
+	// fleetPrefix is the prefix used for official fleet labels/annotations.
 	// Unprefixed labels/annotations are reserved for end-users
-	// we will add a kubernetes-fleet.io to designate these labels/annotations as official fleet labels/annotations.
 	// See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#label-selector-and-annotation-conventions
 	fleetPrefix = "kubernetes-fleet.io/"
 
@@ -29,25 +36,25 @@ const (
 	// cluster.
 	WorkFinalizer = fleetPrefix + "work-cleanup"
 
-	// CRPTrackingLabel is the label that points to the cluster resource policy that creates a resource binding.
+	// CRPTrackingLabel points to the cluster resource placement that creates this resource binding.
 	CRPTrackingLabel = fleetPrefix + "parent-CRP"
 
-	// IsLatestSnapshotLabel tells if the snapshot is the latest one.
+	// IsLatestSnapshotLabel indicates if the snapshot is the latest one.
 	IsLatestSnapshotLabel = fleetPrefix + "is-latest-snapshot"
 
-	// FleetResourceLabelKey is that label that indicates the resource is a fleet resource.
+	// FleetResourceLabelKey indicates that the resource is a fleet resource.
 	FleetResourceLabelKey = fleetPrefix + "is-fleet-resource"
 
-	// FirstWorkNameFmt is the format of the name of the work generated with first resource snapshot .
+	// FirstWorkNameFmt is the format of the name of the work generated with the first resource snapshot.
 	// The name of the first work is {crpName}-work.
 	FirstWorkNameFmt = "%s-work"
 
-	// WorkNameWithSubindexFmt is the format of the name of a work generated with resource snapshot with subindex.
+	// WorkNameWithSubindexFmt is the format of the name of a work generated with a resource snapshot with a subindex.
 	// The name of the first work is {crpName}-{subindex}.
 	WorkNameWithSubindexFmt = "%s-%d"
 
-	// WorkNameWithConfigEnvelopeFmt is the format of the name of a work generated with config envelop.
-	// The format is {workPrefix}-configMap-uuid
+	// WorkNameWithConfigEnvelopeFmt is the format of the name of a work generated with a config envelope.
+	// The format is {workPrefix}-configMap-uuid.
 	WorkNameWithConfigEnvelopeFmt = "%s-configmap-%s"
 
 	// ParentClusterResourceOverrideSnapshotHashAnnotation is the annotation to work that contains the hash of the parent cluster resource override snapshot list.
@@ -65,25 +72,23 @@ const (
 	// ParentBindingLabel is the label applied to work that contains the name of the binding that generates the work.
 	ParentBindingLabel = fleetPrefix + "parent-resource-binding"
 
-	// CRPGenerationAnnotation is the annotation that indicates the generation of the CRP from
-	// which an object is derived or last updated.
+	// CRPGenerationAnnotation indicates the generation of the CRP from which an object is derived or last updated.
 	CRPGenerationAnnotation = fleetPrefix + "CRP-generation"
 
-	// EnvelopeConfigMapAnnotation is the annotation that indicates the configmap is an envelope configmap that contains resources
-	// we need to apply to the member cluster instead of the configMap itself.
+	// EnvelopeConfigMapAnnotation indicates the configmap is an envelope configmap containing resources we need to apply to the member cluster instead of the configMap itself.
 	EnvelopeConfigMapAnnotation = fleetPrefix + "envelope-configmap"
 
-	// EnvelopeTypeLabel is the label that marks the work object as generated from an envelope object.
+	// EnvelopeTypeLabel marks the work object as generated from an envelope object.
 	// The value of the annotation is the type of the envelope object.
 	EnvelopeTypeLabel = fleetPrefix + "envelope-work"
 
-	// EnvelopeNamespaceLabel is the label that contains the namespace of the envelope object that the work is generated from.
+	// EnvelopeNamespaceLabel contains the namespace of the envelope object that the work is generated from.
 	EnvelopeNamespaceLabel = fleetPrefix + "envelope-namespace"
 
-	// EnvelopeNameLabel is the label that contains the name of the envelope object that the work is generated from.
+	// EnvelopeNameLabel contains the name of the envelope object that the work is generated from.
 	EnvelopeNameLabel = fleetPrefix + "envelope-name"
 
-	// PreviousBindingStateAnnotation is the annotation that records the previous state of a binding.
+	// PreviousBindingStateAnnotation records the previous state of a binding.
 	// This is used to remember if an "unscheduled" binding was moved from a "bound" state or a "scheduled" state.
 	PreviousBindingStateAnnotation = fleetPrefix + "previous-binding-state"
 )

pkg/controllers/updaterun/controller.go

@@ -0,0 +1,143 @@
+/*
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+*/
+
+// Package updaterun features a controller to reconcile the clusterStagedUpdateRun objects.
+package updaterun
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go.goms.io/fleet/pkg/utils"
+	"go.goms.io/fleet/pkg/utils/controller"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+	runtime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	placementv1alpha1 "go.goms.io/fleet/apis/placement/v1alpha1"
+)
+
+// Reconciler reconciles a ClusterStagedUpdateRun object.
+type Reconciler struct {
+	client.Client
+	recorder record.EventRecorder
+}
+
+func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtime.Result, error) {
+	startTime := time.Now()
+	klog.V(2).InfoS("ClusterStagedUpdateRun reconciliation starts", "clusterStagedUpdateRun", req.NamespacedName)
+	defer func() {
+		latency := time.Since(startTime).Milliseconds()
+		klog.V(2).InfoS("ClusterStagedUpdateRun reconciliation ends", "clusterStagedUpdateRun", req.NamespacedName, "latency", latency)
+	}()
+
+	var updateRun placementv1alpha1.ClusterStagedUpdateRun
+	if err := r.Client.Get(ctx, req.NamespacedName, &updateRun); err != nil {
+		klog.ErrorS(err, "Failed to get clusterStagedUpdateRun object", "clusterStagedUpdateRun", req.Name)
+		return runtime.Result{}, client.IgnoreNotFound(err)
+	}
+	runObjRef := klog.KObj(&updateRun)
+
+	// Handle the deletion of the clusterStagedUpdateRun.
+	if !updateRun.DeletionTimestamp.IsZero() {
+		klog.V(2).InfoS("The clusterStagedUpdateRun is being deleted", "clusterStagedUpdateRun", runObjRef)
+		deleted, waitTime, err := r.handleDelete(ctx, updateRun.DeepCopy())
+		if err != nil {
+			return runtime.Result{}, err
+		}
+		if deleted {
+			return runtime.Result{}, nil
+		}
+		return runtime.Result{RequeueAfter: waitTime}, nil
+	}
+
+	// Add the finalizer to the clusterStagedUpdateRun.
+	if err := r.ensureFinalizer(ctx, &updateRun); err != nil {
+		klog.ErrorS(err, "Failed to add the finalizer to the clusterStagedUpdateRun", "clusterStagedUpdateRun", runObjRef)
+		return runtime.Result{}, err
+	}
+
+	// TODO(wantjian): reconcile the clusterStagedUpdateRun.
+	return runtime.Result{}, nil
+}
+
+// handleDelete handles the deletion of the clusterStagedUpdateRun object.
+// We delete all the dependent resources, including approvalRequest objects, of the clusterStagedUpdateRun object.
+func (r *Reconciler) handleDelete(ctx context.Context, updateRun *placementv1alpha1.ClusterStagedUpdateRun) (bool, time.Duration, error) {
+	runObjRef := klog.KObj(updateRun)
+	// delete all the associated approvalRequests.
+	approvalRequest := &placementv1alpha1.ClusterApprovalRequest{}
+	if err := r.Client.DeleteAllOf(ctx, approvalRequest, client.MatchingLabels{placementv1alpha1.TargetUpdateRunLabel: updateRun.GetName()}); err != nil {
+		klog.ErrorS(err, "Failed to delete all associated approvalRequests", "clusterStagedUpdateRun", runObjRef)
+		return false, 0, controller.NewAPIServerError(false, err)
+	}
+	klog.V(2).InfoS("Deleted all approvalRequests associated with the clusterStagedUpdateRun", "clusterStagedUpdateRun", runObjRef)
+	controllerutil.RemoveFinalizer(updateRun, placementv1alpha1.ClusterStagedUpdateRunFinalizer)
+	if err := r.Client.Update(ctx, updateRun); err != nil {
+		klog.ErrorS(err, "Failed to remove updateRun finalizer", "clusterStagedUpdateRun", runObjRef)
+		return false, 0, controller.NewUpdateIgnoreConflictError(err)
+	}
+	return true, 0, nil
+}
+
+// ensureFinalizer makes sure that the ClusterStagedUpdateRun CR has a finalizer on it.
+func (r *Reconciler) ensureFinalizer(ctx context.Context, updateRun *placementv1alpha1.ClusterStagedUpdateRun) error {
+	if controllerutil.ContainsFinalizer(updateRun, placementv1alpha1.ClusterStagedUpdateRunFinalizer) {
+		return nil
+	}
+	klog.InfoS("Added the staged update run finalizer", "stagedUpdateRun", klog.KObj(updateRun))
+	controllerutil.AddFinalizer(updateRun, placementv1alpha1.ClusterStagedUpdateRunFinalizer)
+	return r.Update(ctx, updateRun, client.FieldOwner(utils.UpdateRunControllerFieldManagerName))
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *Reconciler) SetupWithManager(mgr runtime.Manager) error {
+	r.recorder = mgr.GetEventRecorderFor("clusterresource-stagedupdaterun-controller")
+	return runtime.NewControllerManagedBy(mgr).
+		Named("clusterresource-stagedupdaterun-controller").
+		For(&placementv1alpha1.ClusterStagedUpdateRun{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
+		Watches(&placementv1alpha1.ClusterApprovalRequest{}, &handler.Funcs{
+			// We only care about when an approval request is approved.
+			UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) {
+				klog.V(2).InfoS("Handling a clusterApprovalRequest update event", "clusterApprovalRequest", klog.KObj(e.ObjectNew))
+				handleClusterApprovalRequest(e.ObjectNew, q)
+			},
+			GenericFunc: func(ctx context.Context, e event.GenericEvent, q workqueue.RateLimitingInterface) {
+				klog.V(2).InfoS("Handling a clusterApprovalRequest generic event", "clusterApprovalRequest", klog.KObj(e.Object))
+				handleClusterApprovalRequest(e.Object, q)
+			},
+		}).Complete(r)
+}
+
+// handleClusterApprovalRequest finds the ClusterStagedUpdateRun creating the ClusterApprovalRequest,
+// and enqueues it to the ClusterStagedUpdateRun controller queue.
+func handleClusterApprovalRequest(obj client.Object, q workqueue.RateLimitingInterface) {
+	approvalRequest, ok := obj.(*placementv1alpha1.ClusterApprovalRequest)
+	if !ok {
+		klog.V(2).ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("cannot cast runtime object to ClusterApprovalRequest")),
+			"Invalid object type", "object", klog.KObj(obj))
+		return
+	}
+	updateRun := approvalRequest.Spec.TargetUpdateRun
+	if len(updateRun) == 0 {
+		klog.V(2).ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("TargetUpdateRun field in ClusterApprovalRequest is empty")),
+			"Invalid clusterApprovalRequest", "clusterApprovalRequest", klog.KObj(approvalRequest))
+		return
+	}
+	// enqueue to the updaterun controller queue.
+	q.Add(reconcile.Request{
+		NamespacedName: types.NamespacedName{Name: updateRun},
+	})
+}