Prevent infinite autoscaling (#11244)

* Prevent infinite autoscaling

* Update server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java

Co-authored-by: Suresh Kumar Anaparti <[email protected]>

Author: sudo87

engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java

@@ -35,4 +35,6 @@ public interface AutoScaleVmGroupVmMapDao extends GenericDao<AutoScaleVmGroupVmM
     public boolean removeByVm(long vmId);
 
     public boolean removeByGroup(long vmGroupId);
+
+    int getErroredInstanceCount(long vmGroupId);
 }

engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImpl.java

@@ -115,4 +115,13 @@ public boolean removeByGroup(long vmGroupId) {
         sc.setParameters("vmGroupId", vmGroupId);
         return remove(sc) >= 0;
     }
+
+    @Override
+    public int getErroredInstanceCount(long vmGroupId) {
+        SearchCriteria<Integer> sc = CountBy.create();
+        sc.setParameters("vmGroupId", vmGroupId);
+        sc.setJoinParameters("vmSearch", "states", State.Error);
+        final List<Integer> results = customSearch(sc, null);
+        return results.get(0);
+    }
 }

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -47,6 +47,7 @@
 
 
 import com.cloud.hypervisor.HypervisorGuru;
+import com.cloud.network.as.AutoScaleManager;
 import com.cloud.user.AccountManagerImpl;
 import com.cloud.utils.crypt.DBEncryptionUtil;
 import com.cloud.host.HostTagVO;
@@ -570,6 +571,7 @@ protected void populateConfigValuesForValidationSet() {
         configValuesForValidation.add(UserDataManager.VM_USERDATA_MAX_LENGTH_STRING);
         configValuesForValidation.add(UnmanagedVMsManager.RemoteKvmInstanceDisksCopyTimeout.key());
         configValuesForValidation.add(UnmanagedVMsManager.ConvertVmwareInstanceToKvmTimeout.key());
+        configValuesForValidation.add(AutoScaleManager.AutoScaleErroredInstanceThreshold.key());
     }
 
     private void weightBasedParametersForValidation() {

server/src/main/java/com/cloud/network/as/AutoScaleManager.java

@@ -38,6 +38,12 @@ public interface AutoScaleManager extends AutoScaleService {
             "The Number of worker threads to scan the autoscale vm groups.",
             false);
 
+    ConfigKey<Integer> AutoScaleErroredInstanceThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Integer.class,
+            "autoscale.errored.instance.threshold",
+            "10",
+            "The number of Error Instances allowed in autoscale vm groups for scale up.",
+            true);
+
     void checkAutoScaleUser(Long autoscaleUserId, long accountId);
 
     boolean deleteAutoScaleVmGroupsByAccount(Long accountId);

server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java

@@ -1716,6 +1716,11 @@ private boolean checkConditionUp(AutoScaleVmGroupVO asGroup, Integer numVm) {
             s_logger.warn("number of VM will greater than the maximum in this group if scaling up, so do nothing more");
             return false;
         }
+        int erroredInstanceCount = autoScaleVmGroupVmMapDao.getErroredInstanceCount(asGroup.getId());
+        if (erroredInstanceCount > AutoScaleManager.AutoScaleErroredInstanceThreshold.value()) {
+            s_logger.warn("Number of Errored Instances are greater than the threshold in this group for scaling up, so do nothing more");
+            return false;
+        }
         return true;
     }
 
@@ -2202,7 +2207,8 @@ public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {
                 AutoScaleStatsInterval,
                 AutoScaleStatsCleanupDelay,
-                AutoScaleStatsWorker
+                AutoScaleStatsWorker,
+                AutoScaleErroredInstanceThreshold
         };
     }