server: prevent duplicate HA works and alerts

Fixes #10367

Signed-off-by: Abhishek Kumar <[email protected]>

Author: shwstppr

server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java

@@ -265,6 +265,7 @@ public void scheduleRestartForVmsOnHost(final HostVO host, boolean investigate)
         s_logger.warn("Scheduling restart for VMs on host " + host.getId() + "-" + host.getName());
 
         final List<VMInstanceVO> vms = _instanceDao.listByHostId(host.getId());
+        final List<HaWorkVO> pendingHaWorks = _haDao.listPendingHAWorkForHost(host.getId());
         final DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId());
 
         // send an email alert that the host is down
@@ -294,6 +295,10 @@ public void scheduleRestartForVmsOnHost(final HostVO host, boolean investigate)
                 "Host [" + hostDesc + "] is down." + ((sb != null) ? sb.toString() : ""));
 
         for (VMInstanceVO vm : reorderedVMList) {
+            VMInstanceVO finalVm = vm;
+            if (pendingHaWorks.stream().anyMatch(h -> h.getInstanceId() == finalVm.getId())) {
+                continue;
+            }
             ServiceOfferingVO vmOffering = _serviceOfferingDao.findById(vm.getServiceOfferingId());
             if (_itMgr.isRootVolumeOnLocalStorage(vm.getId())) {
                 if (s_logger.isDebugEnabled()){

server/src/main/java/com/cloud/ha/dao/HighAvailabilityDao.java

@@ -85,4 +85,6 @@ public interface HighAvailabilityDao extends GenericDao<HaWorkVO, Long> {
     List<HaWorkVO> listPendingHaWorkForVm(long vmId);
 
     List<HaWorkVO> listPendingMigrationsForVm(long vmId);
+
+    List<HaWorkVO> listPendingHAWorkForHost(long hostId);
 }

server/src/main/java/com/cloud/ha/dao/HighAvailabilityDaoImpl.java

@@ -260,4 +260,15 @@ public int releaseWorkItems(long nodeId) {
 
         return update(vo, sc);
     }
+
+    @Override
+    public List<HaWorkVO> listPendingHAWorkForHost(long hostId) {
+        SearchBuilder<HaWorkVO> sb = createSearchBuilder();
+        sb.and("hostId", sb.entity().getHostId(), Op.EQ);
+        sb.and("type", sb.entity().getWorkType(), Op.EQ);
+        SearchCriteria<HaWorkVO> sc = sb.create();
+        sc.setParameters("hostId", hostId);
+        sc.setParameters("type", WorkType.HA);
+        return listBy(sc);
+    }
 }