Prevent infinite autoscaling (apache#11244)

* Prevent infinite autoscaling

* Update server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java

Co-authored-by: Suresh Kumar Anaparti <[email protected]>

Author: 2 people

engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java

@@ -37,4 +37,6 @@ public interface AutoScaleVmGroupVmMapDao extends GenericDao<AutoScaleVmGroupVmM
     public boolean removeByGroup(long vmGroupId);
 
     int expungeByVmList(List<Long> vmIds, Long batchSize);
+    
+    int getErroredInstanceCount(long vmGroupId);
 }

engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImpl.java

@@ -127,4 +127,12 @@ public int expungeByVmList(List<Long> vmIds, Long batchSize) {
         sc.setParameters("vmIds", vmIds.toArray());
         return batchExpunge(sc, batchSize);
     }
+    
+    public int getErroredInstanceCount(long vmGroupId) {
+        SearchCriteria<Integer> sc = CountBy.create();
+        sc.setParameters("vmGroupId", vmGroupId);
+        sc.setJoinParameters("vmSearch", "states", State.Error);
+        final List<Integer> results = customSearch(sc, null);
+        return results.get(0);
+    }
 }

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -51,6 +51,7 @@
 import javax.naming.ConfigurationException;
 
 import com.cloud.hypervisor.HypervisorGuru;
+import com.cloud.network.as.AutoScaleManager;
 import com.cloud.user.AccountManagerImpl;
 import com.cloud.utils.crypt.DBEncryptionUtil;
 import com.cloud.host.HostTagVO;
@@ -619,6 +620,7 @@ protected void populateConfigValuesForValidationSet() {
         configValuesForValidation.add(VMLeaseManager.InstanceLeaseSchedulerInterval.key());
         configValuesForValidation.add(VMLeaseManager.InstanceLeaseExpiryEventSchedulerInterval.key());
         configValuesForValidation.add(VMLeaseManager.InstanceLeaseExpiryEventDaysBefore.key());
+        configValuesForValidation.add(AutoScaleManager.AutoScaleErroredInstanceThreshold.key());
     }
 
     protected void weightBasedParametersForValidation() {

server/src/main/java/com/cloud/network/as/AutoScaleManager.java

@@ -39,6 +39,12 @@ public interface AutoScaleManager extends AutoScaleService {
             "The Number of worker threads to scan the autoscale vm groups.",
             false);
 
+    ConfigKey<Integer> AutoScaleErroredInstanceThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Integer.class,
+            "autoscale.errored.instance.threshold",
+            "10",
+            "The number of Error Instances allowed in autoscale vm groups for scale up.",
+            true);
+
     void checkAutoScaleUser(Long autoscaleUserId, long accountId);
 
     boolean deleteAutoScaleVmGroupsByAccount(Account account);

server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java

@@ -1722,6 +1722,11 @@ private boolean checkConditionUp(AutoScaleVmGroupVO asGroup, Integer numVm) {
             logger.warn("number of VM will greater than the maximum in this group if scaling up, so do nothing more");
             return false;
         }
+        int erroredInstanceCount = autoScaleVmGroupVmMapDao.getErroredInstanceCount(asGroup.getId());
+        if (erroredInstanceCount > AutoScaleManager.AutoScaleErroredInstanceThreshold.value()) {
+            s_logger.warn("Number of Errored Instances are greater than the threshold in this group for scaling up, so do nothing more");
+            return false;
+        }
         return true;
     }
 
@@ -2202,7 +2207,8 @@ public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {
                 AutoScaleStatsInterval,
                 AutoScaleStatsCleanupDelay,
-                AutoScaleStatsWorker
+                AutoScaleStatsWorker,
+                AutoScaleErroredInstanceThreshold
         };
     }