Add ownership relationship for node, optimize, add migration origin/dest

notandy · notandy · commit 4c4321cde11f · 2025-02-25T15:53:56.000-05:00
diff --git a/api/v1alpha1/migration_types.go b/api/v1alpha1/migration_types.go
@@ -30,7 +30,8 @@ type MigrationSpec struct {
 
 // MigrationStatus defines the observed state of Migration.
 type MigrationStatus struct {
-	Host                 string      `json:"host"`
+	Origin               string      `json:"origin"`
+	Destination          string      `json:"destination"`
 	Type                 string      `json:"type,omitempty"`
 	Started              metav1.Time `json:"started,omitempty"`
 	ErrMsg               string      `json:"errMsg,omitempty"`
@@ -62,7 +63,8 @@ type MigrationStatus struct {
 
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
-// +kubebuilder:printcolumn:name="Host",type=string,JSONPath=`.status.host`
+// +kubebuilder:printcolumn:name="Origin",type=string,JSONPath=`.status.origin`
+// +kubebuilder:printcolumn:name="Destination",type=string,JSONPath=`.status.destination`
 // +kubebuilder:printcolumn:name="Type",type=string,JSONPath=`.status.type`
 // +kubebuilder:printcolumn:name="Operation",type=string,JSONPath=`.status.operation`
 // +kubebuilder:printcolumn:name="Started",type=date,JSONPath=`.status.started`
diff --git a/charts/kvm-node-agent/crds/migration-crd.yaml b/charts/kvm-node-agent/crds/migration-crd.yaml
@@ -14,8 +14,11 @@ spec:
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
-    - jsonPath: .status.host
-      name: Host
+    - jsonPath: .status.origin
+      name: Origin
+      type: string
+    - jsonPath: .status.destination
+      name: Destination
       type: string
     - jsonPath: .status.type
       name: Type
@@ -83,6 +86,8 @@ spec:
                 type: string
               dataTotal:
                 type: string
+              destination:
+                type: string
               diskBps:
                 type: string
               diskProcessed:
@@ -95,8 +100,6 @@ spec:
                 type: string
               errMsg:
                 type: string
-              host:
-                type: string
               memBps:
                 type: string
               memConstant:
@@ -125,6 +128,8 @@ spec:
                 type: string
               operation:
                 type: string
+              origin:
+                type: string
               setupTime:
                 type: string
               started:
@@ -137,7 +142,8 @@ spec:
               type:
                 type: string
             required:
-            - host
+            - destination
+            - origin
             type: object
         type: object
     served: true
diff --git a/cmd/main.go b/cmd/main.go
@@ -21,23 +21,29 @@ import (
 	"context"
 	"crypto/tls"
 	"flag"
+	"fmt"
 	"os"
 
 	logger "sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/cobaltcode-dev/kvm-node-agent/internal/emulator"
 	"github.com/cobaltcode-dev/kvm-node-agent/internal/libvirt"
+	"github.com/cobaltcode-dev/kvm-node-agent/internal/sys"
 	"github.com/cobaltcode-dev/kvm-node-agent/internal/systemd"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
 	certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
@@ -143,6 +149,19 @@ func main() {
 		// if you are doing or is intended to do any operation such as perform cleanups
 		// after the manager stops then its usage might be unsafe.
 		// LeaderElectionReleaseOnCancel: true,
+
+		// Cache options allow to subscribe to events from Kubernetes objects and to read
+		// those objects more efficiently by avoiding to call out to the API.
+		Cache: cache.Options{
+			ByObject: map[client.Object]cache.ByObject{
+				&corev1.Node{}: {
+					Field: fields.ParseSelectorOrDie(fmt.Sprintf("metadata.name=%s", sys.Hostname)),
+				},
+				&kvmv1alpha1.Hypervisor{}: {
+					Field: fields.ParseSelectorOrDie(fmt.Sprintf("metadata.name=%s", sys.Hostname)),
+				},
+			},
+		},
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
diff --git a/config/crd/bases/kvm.cloud.sap_migrations.yaml b/config/crd/bases/kvm.cloud.sap_migrations.yaml
@@ -15,8 +15,11 @@ spec:
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
-    - jsonPath: .status.host
-      name: Host
+    - jsonPath: .status.origin
+      name: Origin
+      type: string
+    - jsonPath: .status.destination
+      name: Destination
       type: string
     - jsonPath: .status.type
       name: Type
@@ -84,6 +87,8 @@ spec:
                 type: string
               dataTotal:
                 type: string
+              destination:
+                type: string
               diskBps:
                 type: string
               diskProcessed:
@@ -96,8 +101,6 @@ spec:
                 type: string
               errMsg:
                 type: string
-              host:
-                type: string
               memBps:
                 type: string
               memConstant:
@@ -126,6 +129,8 @@ spec:
                 type: string
               operation:
                 type: string
+              origin:
+                type: string
               setupTime:
                 type: string
               started:
@@ -138,7 +143,8 @@ spec:
               type:
                 type: string
             required:
-            - host
+            - destination
+            - origin
             type: object
         type: object
     served: true
diff --git a/internal/controller/hypervisor_controller.go b/internal/controller/hypervisor_controller.go
@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	logger "sigs.k8s.io/controller-runtime/pkg/log"
@@ -68,8 +69,7 @@ func (r *HypervisorReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 
 	// only reconcile the node I am running on
 	if req.Name != sys.Hostname {
-		// only reconcile the node I am running on
-		return ctrl.Result{}, nil
+		panic(fmt.Sprintf("reconciling hypervisor %s, but I am running on %s", req.Name, sys.Hostname))
 	}
 	log.Info("Reconcile", "name", req.Name, "namespace", req.Namespace)
 
@@ -79,6 +79,13 @@ func (r *HypervisorReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
+	// ====================================================================================================
+	// Hypervisor Metal node name
+	// ====================================================================================================
+	if name, ok := hypervisor.Labels[LabelMetalNodeName]; ok {
+		hypervisor.Status.Node = types.NodeName(name)
+	}
+
 	if hypervisor.Spec.EvacuateOnReboot != r.evacuateOnReboot {
 		if hypervisor.Spec.EvacuateOnReboot {
 			e := &evacuation.EvictionController{Client: r.Client}
diff --git a/internal/controller/node_controller.go b/internal/controller/node_controller.go
@@ -21,12 +21,13 @@ import (
 	"context"
 	"fmt"
 
-	v1 "k8s.io/api/core/v1"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	logger "sigs.k8s.io/controller-runtime/pkg/log"
 
 	kvmv1alpha1 "github.com/cobaltcode-dev/kvm-node-agent/api/v1alpha1"
@@ -46,93 +47,68 @@ const LabelMetalNodeName = "kubernetes.metal.cloud.sap/name"
 
 // +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=nodes/status,verbs=get
-// +kubebuilder:rbac:groups=kvm.cloud.sap,resources=hypervisors,verbs=get;list;watch;create;update;patch;delete
-
-func (r *NodeReconciler) getNode(ctx context.Context) (*v1.Node, error) {
-	// Fetch the Node we're current running on
-	var nodes v1.NodeList
-	err := r.List(ctx, &nodes, client.MatchingLabels{v1.LabelHostname: sys.Hostname})
-	if client.IgnoreNotFound(err) != nil {
-		return nil, fmt.Errorf("failed fetching nodes: %w", err)
-	}
-
-	switch len(nodes.Items) {
-	case 0:
-		return nil, nil
-	case 1:
-		return &nodes.Items[0], nil
-	default:
-		return nil, fmt.Errorf("found more than one node with label %s=%s", v1.LabelHostname, sys.Hostname)
-	}
-
-}
+// +kubebuilder:rbac:groups=kvm.cloud.sap,resources=hypervisors,verbs=get;list;watch;create;delete
 
 func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := logger.FromContext(ctx, "controller", "node")
 
 	if req.Name != sys.Hostname {
-		// only reconcile the node I am running on
-		return ctrl.Result{}, nil
+		panic(fmt.Sprintf("reconciling node %s, but I am running on %s", req.Name, sys.Hostname))
 	}
 
-	namespace := req.Namespace
-	if namespace == "" {
-		namespace = sys.Namespace
+	node := &corev1.Node{}
+	if err := r.Get(ctx, req.NamespacedName, node); client.IgnoreNotFound(err) != nil {
+		// ignore not found errors, could be deleted
+		return ctrl.Result{}, err
 	}
 
-	node, err := r.getNode(ctx)
-	if err != nil {
-		return ctrl.Result{}, fmt.Errorf("failed fetching node: %w", err)
+	metalNodeName := sys.Hostname
+	if name, ok := node.Labels[LabelMetalNodeName]; ok {
+		metalNodeName = name
 	}
-	if node == nil {
-		return ctrl.Result{}, nil
+
+	hypervisor := &kvmv1alpha1.Hypervisor{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      node.Name,
+			Namespace: sys.Namespace,
+			Labels: map[string]string{
+				corev1.LabelHostname: sys.Hostname,
+				LabelMetalNodeName:   metalNodeName,
+			},
+		},
+		Spec: kvmv1alpha1.HypervisorSpec{
+			Reboot:                       r.Reboot,
+			EvacuateOnReboot:             r.EvacuateOnReboot,
+			CreateCertManagerCertificate: r.CreateCertManagerCertificate,
+		},
 	}
-	// Todo: check I am really an hypervisor?
 
 	// Ensure corresponding hypervisor exists
 	log.Info("Reconcile", "name", req.Name, "namespace", req.Namespace)
-	var hypervisors kvmv1alpha1.HypervisorList
-	if err = r.List(ctx, &hypervisors, client.MatchingLabels{v1.LabelHostname: sys.Hostname}); err != nil {
-		return ctrl.Result{}, fmt.Errorf("failed fetching hypervisors: %w", err)
-	}
-
-	if len(hypervisors.Items) == 0 {
-		// create hypervisor
-		if err = r.Create(ctx, &kvmv1alpha1.Hypervisor{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      node.Name,
-				Namespace: namespace,
-				Labels:    map[string]string{v1.LabelHostname: sys.Hostname},
-			},
-			Spec: kvmv1alpha1.HypervisorSpec{
-				Reboot:                       r.Reboot,
-				EvacuateOnReboot:             r.EvacuateOnReboot,
-				CreateCertManagerCertificate: r.CreateCertManagerCertificate,
-			},
-		}); err != nil {
-			return ctrl.Result{}, fmt.Errorf("failed creating hypervisor: %w", err)
+	if err := r.Get(ctx, client.ObjectKeyFromObject(hypervisor), hypervisor); err != nil {
+		if k8serrors.IsNotFound(err) {
+			// attach ownerReference for cascading deletion
+			if err = controllerutil.SetControllerReference(node, hypervisor, r.Scheme); err != nil {
+				return ctrl.Result{}, fmt.Errorf("failed setting controller reference: %w", err)
+			}
+
+			log.Info("Creating new hypervisor", "name", node.Name)
+			if err = r.Create(ctx, hypervisor); err != nil {
+				return ctrl.Result{}, err
+			}
+
+			// Requeue to update status
+			return ctrl.Result{}, nil
 		}
-		return ctrl.Result{}, nil
+
+		return ctrl.Result{}, err
 	}
 
 	if node.ObjectMeta.DeletionTimestamp != nil {
 		// node is being deleted, cleanup hypervisor
-		if err = r.Delete(ctx, &hypervisors.Items[0]); client.IgnoreNotFound(err) != nil {
+		if err := r.Delete(ctx, hypervisor); client.IgnoreNotFound(err) != nil {
 			return ctrl.Result{}, fmt.Errorf("failed cleanup up hypervisor: %w", err)
 		}
-		return ctrl.Result{}, nil
-	}
-
-	hv := hypervisors.Items[0].DeepCopy()
-	hv.Status.Node = types.NodeName(node.Name)
-	if name, ok := node.Labels[LabelMetalNodeName]; ok {
-		hv.Status.Node = types.NodeName(name)
-	}
-
-	// Update Status
-	err = r.Status().Patch(ctx, hv, client.MergeFrom(&hypervisors.Items[0]))
-	if err != nil {
-		return ctrl.Result{}, fmt.Errorf("failed patching hypervisor status: %w", err)
 	}
 
 	return ctrl.Result{}, nil
@@ -141,6 +117,6 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 // SetupWithManager sets up the controller with the Manager.
 func (r *NodeReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
-		For(&v1.Node{}).
+		For(&corev1.Node{}).
 		Complete(r)
 }
diff --git a/internal/libvirt/libvirt_events.go b/internal/libvirt/libvirt_events.go
@@ -239,7 +239,7 @@ func (l *LibVirt) startMigrationWatch(ctx context.Context, domain libvirt.Domain
 	}
 	patched := original.DeepCopy()
 	patched.Status.Started = metav1.Now()
-	patched.Status.Host = sys.Hostname
+	patched.Status.Origin = sys.NodeLabelName
 	if err := l.client.Status().Patch(ctx, patched, client.MergeFrom(&original)); err != nil {
 		return fmt.Errorf("failed to patch migration status time: %w", err)
 	}
@@ -332,7 +332,7 @@ func (l *LibVirt) populateDomainJobInfo(domain libvirt.Domain, migration *kvmv1a
 		flags = libvirt.DomainJobStatsCompleted
 	}
 
-	migration.Status.Host = sys.Hostname
+	migration.Status.Destination = sys.NodeLabelName
 	rType, params, err := l.virt.DomainGetJobStats(domain, flags)
 	if err != nil {
 		return err
diff --git a/internal/sys/hostname.go b/internal/sys/hostname.go
@@ -24,6 +24,7 @@ import (
 )
 
 var Hostname string
+var NodeLabelName string
 
 func init() {
 	var err error
@@ -33,4 +34,8 @@ func init() {
 			log.Log.Error(err, "failed fetching hostname")
 		}
 	}
+
+	if NodeLabelName = os.Getenv("NODE_LABEL"); NodeLabelName == "" {
+		NodeLabelName = Hostname
+	}
 }

Original file line number	Diff line number	Diff line change
`@@ -239,7 +239,7 @@ func (l *LibVirt) startMigrationWatch(ctx context.Context, domain libvirt.Domain`
`239`	`239`	`}`
`240`	`240`	`patched := original.DeepCopy()`
`241`	`241`	`patched.Status.Started = metav1.Now()`
`242`		`- patched.Status.Host = sys.Hostname`
	`242`	`+ patched.Status.Origin = sys.NodeLabelName`
`243`	`243`	`if err := l.client.Status().Patch(ctx, patched, client.MergeFrom(&original)); err != nil {`
`244`	`244`	`return fmt.Errorf("failed to patch migration status time: %w", err)`
`245`	`245`	`}`
`@@ -332,7 +332,7 @@ func (l LibVirt) populateDomainJobInfo(domain libvirt.Domain, migration kvmv1a`
`332`	`332`	`flags = libvirt.DomainJobStatsCompleted`
`333`	`333`	`}`
`334`	`334`
`335`		`- migration.Status.Host = sys.Hostname`
	`335`	`+ migration.Status.Destination = sys.NodeLabelName`
`336`	`336`	`rType, params, err := l.virt.DomainGetJobStats(domain, flags)`
`337`	`337`	`if err != nil {`
`338`	`338`	`return err`
Original file line number	Diff line number	Diff line change
`@@ -24,6 +24,7 @@ import (`
`24`	`24`	`)`
`25`	`25`
`26`	`26`	`var Hostname string`
	`27`	`+var NodeLabelName string`
`27`	`28`
`28`	`29`	`func init() {`
`29`	`30`	`var err error`
`@@ -33,4 +34,8 @@ func init() {`
`33`	`34`	`log.Log.Error(err, "failed fetching hostname")`
`34`	`35`	`}`
`35`	`36`	`}`
	`37`	`+`
	`38`	`+ if NodeLabelName = os.Getenv("NODE_LABEL"); NodeLabelName == "" {`
	`39`	`+ NodeLabelName = Hostname`
	`40`	`+ }`
`36`	`41`	`}`