Do not watch guest.net / multi priority queues

This patch further restricts the async watcher from monitoring
guest.net due to the same churn in VKS nodes that caused guest.ipStack
to be so busy. This is okay since the summary.guest will still trigger
a reconcile when a VM gets its primary IP.

Additionally, this patch also introduces multiple priority queues for a
VM's lifecycle. When a VM is going to be reconciled, it will now fall into
one of the following buckets:

- priorityLow                 int = handler.LowPriority // 0
- priorityCreating            int = 100
- priorityPowerStateChange    int = 90
- priorityWaitingForIP        int = 90
- priorityDeleting            int = 80
- priorityWaitingForDiskPromo int = 70

Thus, it no longer matters if async signal is sending thousands of VMs a minute,
because VMs that are being created, deleted, waiting on a power state change,
waiting for an IP, or waiting for disk promotion will all be moved to the head
of the line.

Finally, the VM watcher has been updated to cache VM properties for an hour
and will not send async signal updates if a VM's properties have not changed.
The local property cache is flushed once an hour or when the pod restarts.

Author: akutz

.golangci.yml

@@ -177,6 +177,8 @@ linters:
           pkg: github.com/vmware-tanzu/vm-operator/pkg/exit
         - alias: pkglog
           pkg: github.com/vmware-tanzu/vm-operator/pkg/log
+        - alias: pkgnil
+          pkg: github.com/vmware-tanzu/vm-operator/pkg/util/nil
         - alias: ctxop
           pkg: github.com/vmware-tanzu/vm-operator/pkg/context/operation
         - alias: pkgmgr

controllers/infra/zone/zone_controller_test.go

@@ -14,12 +14,10 @@ import (
 
 	"github.com/vmware/govmomi/object"
 	vimtypes "github.com/vmware/govmomi/vim25/types"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager"
 
-	vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha5"
 	"github.com/vmware-tanzu/vm-operator/controllers/infra/zone"
 	topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1"
 	pkgcfg "github.com/vmware-tanzu/vm-operator/pkg/config"
@@ -187,15 +185,11 @@ var _ = Describe(
 					})
 					Specify("a reconcile request should be enqueued for vm", func() {
 						chanSource := cource.FromContext(ctx, "VirtualMachine")
+
 						var e event.GenericEvent
-						Eventually(chanSource).Should(Receive(&e, Equal(event.GenericEvent{
-							Object: &vmopv1.VirtualMachine{
-								ObjectMeta: metav1.ObjectMeta{
-									Namespace: vcSimCtx.NSInfo.Namespace,
-									Name:      vmName,
-								},
-							},
-						})))
+						Eventually(chanSource).Should(Receive(&e))
+						Expect(e.Object.GetNamespace()).To(Equal(vcSimCtx.NSInfo.Namespace))
+						Expect(e.Object.GetName()).To(Equal(vmName))
 					})
 
 					When("a single zone with the vm's folder is removed", func() {
@@ -241,14 +235,9 @@ var _ = Describe(
 							Expect(t.Wait(ctx)).To(Succeed())
 
 							var e event.GenericEvent
-							Eventually(chanSource).Should(Receive(&e, Equal(event.GenericEvent{
-								Object: &vmopv1.VirtualMachine{
-									ObjectMeta: metav1.ObjectMeta{
-										Namespace: vcSimCtx.NSInfo.Namespace,
-										Name:      vmName,
-									},
-								},
-							})))
+							Eventually(chanSource).Should(Receive(&e))
+							Expect(e.Object.GetNamespace()).To(Equal(vcSimCtx.NSInfo.Namespace))
+							Expect(e.Object.GetName()).To(Equal(vmName))
 						})
 					})
 

controllers/virtualmachine/virtualmachine/virtualmachine_controller.go

@@ -26,7 +26,7 @@ import (
 	vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha5"
 	byokv1 "github.com/vmware-tanzu/vm-operator/external/byok/api/v1alpha1"
 	cnsv1alpha1 "github.com/vmware-tanzu/vm-operator/external/vsphere-csi-driver/api/v1alpha1"
-	"github.com/vmware-tanzu/vm-operator/pkg/conditions"
+	pkgcond "github.com/vmware-tanzu/vm-operator/pkg/conditions"
 	pkgcfg "github.com/vmware-tanzu/vm-operator/pkg/config"
 	pkgconst "github.com/vmware-tanzu/vm-operator/pkg/constants"
 	pkgctx "github.com/vmware-tanzu/vm-operator/pkg/context"
@@ -84,13 +84,21 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err
 		proberManager)
 
 	builder := ctrl.NewControllerManagedBy(mgr).
-		For(controlledType).
+		Named(strings.ToLower(controlledTypeName)).
 		WithOptions(controller.Options{
 			MaxConcurrentReconciles: ctx.MaxConcurrentReconciles,
 			SkipNameValidation:      SkipNameValidation,
 			LogConstructor:          pkglog.ControllerLogConstructor(controllerNameShort, controlledType, mgr.GetScheme()),
 		})
 
+	// Watch VirtualMachines.
+	builder = builder.Watches(
+		controlledType,
+		&kubeutil.EnqueueRequestForObject{
+			Logger:      ctrl.Log.WithName("vmqueue"),
+			GetPriority: kubeutil.GetVirtualMachineReconcilePriority,
+		})
+
 	builder = builder.Watches(&vmopv1.VirtualMachineClass{},
 		handler.EnqueueRequestsFromMapFunc(classToVMMapperFn(ctx, r.Client)))
 
@@ -105,7 +113,10 @@ func AddToManager(ctx *pkgctx.ControllerManagerContext, mgr manager.Manager) err
 	if pkgcfg.FromContext(ctx).AsyncSignalEnabled {
 		builder = builder.WatchesRawSource(source.Channel(
 			cource.FromContextWithBuffer(ctx, "VirtualMachine", 100),
-			&handler.EnqueueRequestForObject{}))
+			&kubeutil.EnqueueRequestForObject{
+				Logger:      ctrl.Log.WithName("asyncvmqueue"),
+				GetPriority: kubeutil.GetVirtualMachineReconcilePriority,
+			}))
 	}
 
 	if pkgcfg.FromContext(ctx).Features.FastDeploy {
@@ -392,7 +403,7 @@ func requeueDelay(
 	// Create VMs on the provider. Do not queue immediately to avoid exponential
 	// backoff.
 	if ignoredCreateErr(err) ||
-		!conditions.IsTrue(ctx.VM, vmopv1.VirtualMachineConditionCreated) {
+		!pkgcond.IsTrue(ctx.VM, vmopv1.VirtualMachineConditionCreated) {
 
 		return pkgcfg.FromContext(ctx).CreateVMRequeueDelay
 	}
@@ -661,7 +672,7 @@ func (r *Reconciler) isVMICacheReady(ctx *pkgctx.VirtualMachineContext) bool {
 	}
 
 	// Assert the image hardware is ready.
-	if !conditions.IsTrue(
+	if !pkgcond.IsTrue(
 		vmic,
 		vmopv1.VirtualMachineImageCacheConditionHardwareReady) {
 
@@ -695,7 +706,7 @@ func (r *Reconciler) isVMICacheReady(ctx *pkgctx.VirtualMachineContext) bool {
 	}
 
 	// Assert the cached disks are ready.
-	if locStatus == nil || !conditions.IsTrue(
+	if locStatus == nil || !pkgcond.IsTrue(
 		locStatus,
 		vmopv1.ReadyConditionType) {
 

controllers/virtualmachinepublishrequest/virtualmachinepublishrequest_controller.go

@@ -45,8 +45,8 @@ import (
 	"github.com/vmware-tanzu/vm-operator/pkg/patch"
 	"github.com/vmware-tanzu/vm-operator/pkg/providers"
 	"github.com/vmware-tanzu/vm-operator/pkg/record"
-	pkgutil "github.com/vmware-tanzu/vm-operator/pkg/util"
 	kubeutil "github.com/vmware-tanzu/vm-operator/pkg/util/kube"
+	pkgnil "github.com/vmware-tanzu/vm-operator/pkg/util/nil"
 )
 
 const (
@@ -500,7 +500,7 @@ func (r *Reconciler) checkIsTargetValid(ctx *pkgctx.VirtualMachinePublishRequest
 		return fmt.Errorf("failed to get item %q from library: %w", targetItemName, err)
 	}
 
-	if !pkgutil.IsNil(item) {
+	if !pkgnil.IsNil(item) {
 		objKey := client.ObjectKey{Name: vmPubReq.Spec.Target.Location.Name, Namespace: vmPubReq.Namespace}
 		ctx.Logger.Info("target item already exists in the content library",
 			"library", objKey, "itemName", targetItemName)
@@ -992,7 +992,7 @@ func (r *Reconciler) findCorrelatedItemIDByName(ctx *pkgctx.VirtualMachinePublis
 		return "", fmt.Errorf("failed to get item %q from library: %w", targetItemName, err)
 	}
 
-	if !pkgutil.IsNil(item) {
+	if !pkgnil.IsNil(item) {
 		// Item already exists in the content library, check if it is created from
 		// this VirtualMachinePublishRequest from its description.
 		// If VC forgets the task, or the task hadn't proceeded far enough to be submitted to VC