Reset deployment status when provisioned BMH hash changes

This would happen when a faulty node has been replaced by
deleting the BMH, openstack-baremetal-operator would provision
a new node from the available BMHs. We also need to tell the
user that they have to create a new deployment for the node
to be deployed.

Note: Cleaning up the deleted nodes from the services is still a
manual process.

jira: https://issues.redhat.com/browse/OSPRH-13948
Signed-off-by: rabi <[email protected]>

Author: rabi

apis/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml

@@ -937,6 +937,8 @@ spec:
                 type: object
               ctlplaneSearchDomain:
                 type: string
+              deployedBmhHash:
+                type: string
               deployedConfigHash:
                 type: string
               deployedVersion:

apis/dataplane/v1beta1/openstackdataplanenodeset_types.go

@@ -154,6 +154,9 @@ type OpenStackDataPlaneNodeSetStatus struct {
 
 	// DeployedVersion
 	DeployedVersion string `json:"deployedVersion,omitempty"`
+
+	//DeployedBmhHash - Hash of BMHs deployed
+	DeployedBmhHash string `json:"deployedBmhHash,omitempty"`
 }
 
 // +kubebuilder:object:root=true

bindata/crds/crds.yaml

@@ -18127,6 +18127,8 @@ spec:
                 type: object
               ctlplaneSearchDomain:
                 type: string
+              deployedBmhHash:
+                type: string
               deployedConfigHash:
                 type: string
               deployedVersion:

config/crd/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml

@@ -937,6 +937,8 @@ spec:
                 type: object
               ctlplaneSearchDomain:
                 type: string
+              deployedBmhHash:
+                type: string
               deployedConfigHash:
                 type: string
               deployedVersion:

controllers/dataplane/openstackdataplanenodeset_controller.go

@@ -371,22 +371,22 @@ func (r *OpenStackDataPlaneNodeSetReconciler) Reconcile(ctx context.Context, req
 		return ctrl.Result{}, err
 	}
 	containerImages := dataplaneutil.GetContainerImages(version)
-
+	provResult := deployment.ProvisionResult{}
 	// Reconcile BaremetalSet if required
 	if !instance.Spec.PreProvisioned {
 		// Reset the NodeSetBareMetalProvisionReadyCondition to unknown
 		instance.Status.Conditions.MarkUnknown(dataplanev1.NodeSetBareMetalProvisionReadyCondition,
 			condition.InitReason, condition.InitReason)
 
-		isReady, err := deployment.DeployBaremetalSet(ctx, helper, instance,
+		provResult, err := deployment.DeployBaremetalSet(ctx, helper, instance,
 			allIPSets, dnsDetails.ServerAddresses, containerImages)
-		if err != nil || !isReady {
+		if err != nil || !provResult.IsProvisioned {
 			return ctrl.Result{}, err
 		}
 	}
 
 	isDeploymentReady, isDeploymentRunning, isDeploymentFailed, failedDeployment, err := checkDeployment(
-		ctx, helper, instance, generationChanged)
+		ctx, helper, instance, generationChanged, provResult.BmhRefHash)
 	if !isDeploymentFailed && err != nil {
 		instance.Status.Conditions.MarkFalse(
 			condition.DeploymentReadyCondition,
@@ -430,6 +430,7 @@ func (r *OpenStackDataPlaneNodeSetReconciler) Reconcile(ctx context.Context, req
 		Log.Info("Set NodeSet DeploymentReadyCondition true")
 		instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition,
 			condition.DeploymentReadyMessage)
+		instance.Status.DeployedBmhHash = provResult.BmhRefHash
 	} else if isDeploymentRunning {
 		Log.Info("Deployment still running...", "instance", instance)
 		Log.Info("Set NodeSet DeploymentReadyCondition false")
@@ -465,7 +466,7 @@ func (r *OpenStackDataPlaneNodeSetReconciler) Reconcile(ctx context.Context, req
 
 func checkDeployment(ctx context.Context, helper *helper.Helper,
 	instance *dataplanev1.OpenStackDataPlaneNodeSet,
-	generationChanged bool,
+	generationChanged bool, bmhRefHash string,
 ) (bool, bool, bool, string, error) {
 	// Get all completed deployments
 	var failedDeployment string
@@ -524,8 +525,9 @@ func checkDeployment(ctx context.Context, helper *helper.Helper,
 				// generation metadata has changed (i.e generation metatdata won't change when
 				// fields are removed from the CRD during an update that would not require a new
 				// deployment to run) asssume nodeset has changed requiring new deployment.
-				if deployment.Status.NodeSetHashes[instance.Name] != instance.Status.ConfigHash &&
-					generationChanged {
+				if (deployment.Status.NodeSetHashes[instance.Name] != instance.Status.ConfigHash &&
+					generationChanged) ||
+					(!instance.Spec.PreProvisioned && instance.Status.DeployedBmhHash != bmhRefHash) {
 					continue
 				}
 				isDeploymentReady = true
@@ -794,6 +796,5 @@ func (r *OpenStackDataPlaneNodeSetReconciler) GetSpecConfigHash(instance *datapl
 	if err != nil {
 		return "", err
 	}
-
 	return configHash, nil
 }

pkg/dataplane/baremetal.go

@@ -29,19 +29,25 @@ import (
 	condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/labels"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/util"
 	utils "github.com/openstack-k8s-operators/lib-common/modules/common/util"
 	baremetalv1 "github.com/openstack-k8s-operators/openstack-baremetal-operator/api/v1beta1"
 	openstackv1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1"
 	dataplanev1 "github.com/openstack-k8s-operators/openstack-operator/apis/dataplane/v1beta1"
 )
 
+type ProvisionResult struct {
+	IsProvisioned bool
+	BmhRefHash    string
+}
+
 // DeployBaremetalSet Deploy OpenStackBaremetalSet
 func DeployBaremetalSet(
 	ctx context.Context, helper *helper.Helper, instance *dataplanev1.OpenStackDataPlaneNodeSet,
 	ipSets map[string]infranetworkv1.IPSet,
 	dnsAddresses []string,
 	containerImages openstackv1.ContainerImages,
-) (bool, error) {
+) (ProvisionResult, error) {
 	baremetalSet := &baremetalv1.OpenStackBaremetalSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      instance.Name,
@@ -109,7 +115,7 @@ func DeployBaremetalSet(
 			condition.ErrorReason, condition.SeverityError,
 			dataplanev1.NodeSetBaremetalProvisionErrorMessage,
 			err.Error())
-		return false, err
+		return ProvisionResult{}, err
 	}
 
 	// Check if baremetalSet is ready
@@ -119,10 +125,25 @@ func DeployBaremetalSet(
 			dataplanev1.NodeSetBareMetalProvisionReadyCondition,
 			condition.RequestedReason, condition.SeverityInfo,
 			dataplanev1.NodeSetBaremetalProvisionReadyWaitingMessage)
-		return false, nil
+		return ProvisionResult{}, nil
+	}
+
+	bmhRefHash, err := getBMHRefHash(baremetalSet)
+	if err != nil {
+		return ProvisionResult{}, err
 	}
+
 	instance.Status.Conditions.MarkTrue(
 		dataplanev1.NodeSetBareMetalProvisionReadyCondition,
 		dataplanev1.NodeSetBaremetalProvisionReadyMessage)
-	return true, nil
+	return ProvisionResult{IsProvisioned: true, BmhRefHash: bmhRefHash}, nil
+}
+
+func getBMHRefHash(bmSet *baremetalv1.OpenStackBaremetalSet) (string, error) {
+	bmhRefHash, err := util.ObjectHash(bmSet.Status.BaremetalHosts)
+	if err != nil {
+		return "", err
+	}
+	return bmhRefHash, nil
+
 }