HypervisorMaintenance: Trigger eviction based on maintenance flag

When maintenance is set, it will trigger an eviction via the respective
CRD and reflect the result back to the hypervisor status.

Author: fwiesel

api/v1/hypervisor_types.go

@@ -35,6 +35,7 @@ const (
 	ConditionReasonReadyReady = "ready"
 	// or not
 	ConditionReasonReadyMaintenance = "maintenance"
+	ConditionReasonReadyEvicted     = "evicted"
 )
 
 // HypervisorSpec defines the desired state of Hypervisor

internal/controller/hypervisor_maintenance_controller.go

@@ -24,11 +24,13 @@ import (
 	"context"
 	"fmt"
 
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	logger "sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/gophercloud/gophercloud/v2"
@@ -39,7 +41,7 @@ import (
 )
 
 const (
-	HypervisorMaintenanceControllerName = "HypervisorMaintenanceController"
+	HypervisorMaintenanceControllerName = "HypervisorMaintenance"
 )
 
 type HypervisorMaintenanceController struct {
@@ -50,6 +52,7 @@ type HypervisorMaintenanceController struct {
 
 // +kubebuilder:rbac:groups=kvm.cloud.sap,resources=hypervisors,verbs=get;list;watch
 // +kubebuilder:rbac:groups=kvm.cloud.sap,resources=hypervisors/status,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=kvm.cloud.sap,resources=evictions,verbs=get;list;watch;create;update;patch;delete
 
 func (hec *HypervisorMaintenanceController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	hv := &kvmv1.Hypervisor{}
@@ -69,36 +72,40 @@ func (hec *HypervisorMaintenanceController) Reconcile(ctx context.Context, req c
 	}
 
 	log := logger.FromContext(ctx).
-		WithName("HypervisorService")
+		WithName(HypervisorMaintenanceControllerName)
 	ctx = logger.IntoContext(ctx, log)
 
-	changed, err := hec.reconcileComputeService(ctx, hv)
-	if err != nil {
-		return ctrl.Result{}, err
-	}
+	changed := false
+	serviceId := hv.Status.ServiceID
 
-	if changed {
-		return ctrl.Result{}, hec.Status().Update(ctx, hv)
-	} else {
-		return ctrl.Result{}, nil
+	eviction := &kvmv1.Eviction{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: hv.Name,
+		},
 	}
-}
-
-func (hec *HypervisorMaintenanceController) reconcileComputeService(ctx context.Context, hv *kvmv1.Hypervisor) (bool, error) {
-	log := logger.FromContext(ctx)
-	serviceId := hv.Status.ServiceID
 
 	switch hv.Spec.Maintenance {
-	case "": // Enable the compute service (in case we haven't done so already)
+	case "": // Cleanup
+		// Update the eviction & status, but only if we have created it
+		if hv.Status.Evicted || meta.RemoveStatusCondition(&hv.Status.Conditions, kvmv1.ConditionTypeEvicting) {
+			if err := k8sclient.IgnoreNotFound(hec.Delete(ctx, eviction)); err != nil {
+				return ctrl.Result{}, err
+			}
+			hv.Status.Evicted = false
+			changed = true
+		}
+
+		// Enable the compute service (in case we haven't done so already)
 		if !meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
 			Type:    kvmv1.ConditionTypeHypervisorDisabled,
 			Status:  metav1.ConditionFalse,
 			Message: "Hypervisor is enabled",
 			Reason:  kvmv1.ConditionReasonSucceeded,
 		}) {
 			// Spec matches status
-			return false, nil
+			break
 		}
+		changed = true
 
 		meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
 			Type:    kvmv1.ConditionTypeReady,
@@ -112,18 +119,22 @@ func (hec *HypervisorMaintenanceController) reconcileComputeService(ctx context.
 		log.Info("Enabling hypervisor", "id", serviceId)
 		_, err := services.Update(ctx, hec.computeClient, serviceId, enableService).Extract()
 		if err != nil {
-			return false, fmt.Errorf("failed to enable hypervisor due to %w", err)
+			return ctrl.Result{}, fmt.Errorf("failed to enable hypervisor due to %w", err)
 		}
 	case "manual", "auto", "ha": // Disable the compute service
+		// Also in case of HA, as it doesn't hurt to disable it twice, and this
+		// allows us to enable the service again, when the maintenance field is
+		// cleared in the case above.
 		if !meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
 			Type:    kvmv1.ConditionTypeHypervisorDisabled,
 			Status:  metav1.ConditionTrue,
 			Message: "Hypervisor is disabled",
 			Reason:  kvmv1.ConditionReasonSucceeded,
 		}) {
 			// Spec matches status
-			return false, nil
+			break
 		}
+		changed = true
 
 		meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
 			Type:    kvmv1.ConditionTypeReady,
@@ -140,11 +151,94 @@ func (hec *HypervisorMaintenanceController) reconcileComputeService(ctx context.
 		log.Info("Disabling hypervisor", "id", serviceId)
 		_, err := services.Update(ctx, hec.computeClient, serviceId, enableService).Extract()
 		if err != nil {
-			return false, fmt.Errorf("failed to disable hypervisor due to %w", err)
+			return ctrl.Result{}, fmt.Errorf("failed to disable hypervisor due to %w", err)
 		}
 	}
 
-	return true, nil
+	switch hv.Spec.Maintenance {
+	case "manual", "auto": // In case of "ha", the host gets emptied from the HA service
+		if cond := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeEvicting); cond != nil {
+			if cond.Reason == kvmv1.ConditionReasonSucceeded {
+				// We are done here, no need to look at the eviction any more
+				return ctrl.Result{}, nil
+			}
+		}
+		status, err := hec.ensureEviction(ctx, eviction, hv)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+		var reason, message string
+		if status == metav1.ConditionFalse {
+			message = "Evicted"
+			reason = kvmv1.ConditionReasonSucceeded
+			if !hv.Status.Evicted {
+				changed = true
+				hv.Status.Evicted = true
+			}
+		} else {
+			message = "Evicting"
+			reason = kvmv1.ConditionReasonRunning
+			if hv.Status.Evicted {
+				changed = true
+				hv.Status.Evicted = false
+			}
+		}
+
+		if meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
+			Type:    kvmv1.ConditionTypeEvicting,
+			Status:  status,
+			Reason:  reason,
+			Message: message,
+		}) {
+			changed = true
+		}
+
+		if meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
+			Type:    kvmv1.ConditionTypeReady,
+			Status:  metav1.ConditionFalse,
+			Reason:  kvmv1.ConditionReasonReadyEvicted,
+			Message: "Hypervisor is disabled and evicted",
+		}) {
+			changed = true
+		}
+	}
+
+	if changed {
+		return ctrl.Result{}, hec.Status().Update(ctx, hv)
+	} else {
+		return ctrl.Result{}, nil
+	}
+}
+
+func (hec *HypervisorMaintenanceController) ensureEviction(ctx context.Context, eviction *kvmv1.Eviction, hypervisor *kvmv1.Hypervisor) (metav1.ConditionStatus, error) {
+	log := logger.FromContext(ctx)
+	if err := hec.Get(ctx, k8sclient.ObjectKeyFromObject(eviction), eviction); err != nil {
+		if !k8serrors.IsNotFound(err) {
+			return metav1.ConditionUnknown, fmt.Errorf("failed to get eviction due to %w", err)
+		}
+		if err := controllerutil.SetControllerReference(hypervisor, eviction, hec.Scheme); err != nil {
+			return metav1.ConditionUnknown, err
+		}
+		log.Info("Creating new eviction", "name", eviction.Name)
+		eviction.Spec = kvmv1.EvictionSpec{
+			Hypervisor: hypervisor.Name,
+			Reason:     "openstack-hypervisor-operator maintenance",
+		}
+
+		// This also transports the label-selector, if set
+		transportLabels(&eviction.ObjectMeta, hypervisor)
+
+		if err = hec.Create(ctx, eviction); err != nil {
+			return metav1.ConditionUnknown, fmt.Errorf("failed to create eviction due to %w", err)
+		}
+	}
+
+	// check if we are still evicting (defaulting to yes)
+	if meta.IsStatusConditionFalse(eviction.Status.Conditions, kvmv1.ConditionTypeEvicting) {
+		return metav1.ConditionFalse, nil
+	} else {
+		return metav1.ConditionTrue, nil
+	}
 }
 
 // SetupWithManager sets up the controller with the Manager.
@@ -161,5 +255,6 @@ func (hec *HypervisorMaintenanceController) SetupWithManager(mgr ctrl.Manager) e
 	return ctrl.NewControllerManagedBy(mgr).
 		Named(HypervisorMaintenanceControllerName).
 		For(&kvmv1.Hypervisor{}).
+		Owns(&kvmv1.Eviction{}). // trigger Reconcile whenever an Own-ed eviction is created/updated/deleted
 		Complete(hec)
 }