Fix unexpected freeze during scale-down issue

Author: hardikdr

pkg/controller/deployment.go

@@ -451,7 +451,7 @@ func (dc *controller) reconcileClusterMachineDeployment(key string) error {
 	// If MachineDeployment is frozen and no deletion timestamp, don't process it
 	if deployment.Labels["freeze"] == "True" && deployment.DeletionTimestamp == nil {
 		glog.V(3).Infof("MachineDeployment %q is frozen, and hence not processeing", deployment.Name)
-		return nil
+		//return nil
 	}
 
 	// Validate MachineDeployment

pkg/controller/machine_safety.go

@@ -44,7 +44,7 @@ const (
 	LastReplicaUpdate = "safety.machine.sapcloud.io/lastreplicaupdate"
 )
 
-// SafetyCheck controller is used to protect the controller from orphan VMs being created
+// SafetyCheck controller is used to protect the controller from orphan or excess VMs being created
 func (c *controller) reconcileClusterMachineSafety(key string) error {
 	var wg sync.WaitGroup
 
@@ -125,7 +125,7 @@ func (c *controller) checkAndFreezeORUnfreezeMachineSets(wg *sync.WaitGroup) {
 		if machineSet.Labels["freeze"] != "True" &&
 			fullyLabeledReplicasCount >= higherThreshold {
 			message := fmt.Sprintf(
-				"The number of machines backing MachineSet: %s is %d > %d which is the Max-ScaleUp-Limit",
+				"The number of machines backing MachineSet: %s is %d >= %d which is the Max-ScaleUp-Limit",
 				machineSet.Name,
 				fullyLabeledReplicasCount,
 				higherThreshold,

pkg/controller/machineset.go

@@ -341,7 +341,12 @@ func (c *controller) manageReplicas(allMachines []*v1alpha1.Machine, machineSet
 
 	diff := len(activeMachines) - int(machineSet.Spec.Replicas)
 	if diff < 0 {
-		//glog.Info("Start Create:", diff)
+		// If MachineSet is frozen and no deletion timestamp, don't process it
+		if machineSet.Labels["freeze"] == "True" && machineSet.DeletionTimestamp == nil {
+			glog.V(2).Infof("MachineSet %q is frozen, and hence not processing", machineSet.Name)
+			return nil
+		}
+
 		diff *= -1
 		if diff > BurstReplicas {
 			diff = BurstReplicas
@@ -444,12 +449,6 @@ func (c *controller) reconcileClusterMachineSet(key string) error {
 		return err
 	}
 
-	// If MachineSet is frozen and no deletion timestamp, don't process it
-	if machineSet.Labels["freeze"] == "True" && machineSet.DeletionTimestamp == nil {
-		glog.V(3).Infof("MachineSet %q is frozen, and hence not processeing", machineSet.Name)
-		return nil
-	}
-
 	// Validate MachineSet
 	internalMachineSet := &machine.MachineSet{}
 	err = c.internalExternalScheme.Convert(machineSet, internalMachineSet, nil)