direct agents transfer and pending jobs timer task fixes

Author: sureshanaparti

engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java

@@ -1411,7 +1411,8 @@ public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long
                     ManagementServerHostVO msHost = msHosts.get(_mshostCounter % msHosts.size());
                     _mshostCounter++;
 
-                    if (rebalanceAgent(host.getId(), Event.StartAgentRebalance, fromMsId, msHost.getMsid())) {
+                    _hostTransferDao.startAgentTransfering(host.getId(), fromMsId, msHost.getMsid());
+                    if (!rebalanceAgent(host.getId(), Event.StartAgentRebalance, fromMsId, msHost.getMsid())) {
                         agentTransferFailedCount++;
                     } else {
                         updateLastManagementServer(host.getId(), fromMsId);
@@ -1477,6 +1478,7 @@ public void onManagementServerMaintenance() {
         logger.debug("Management server maintenance enabled");
         s_transferExecutor.shutdownNow();
         cleanupTransferMap(_nodeId);
+        _agentLbHappened = false;
         super.onManagementServerMaintenance();
     }
 

plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImpl.java

@@ -84,7 +84,7 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen
     private long maintenanceStartTime = 0;
     private String lbAlgorithm;
 
-    private Timer timer = new Timer();
+    private Timer timer;
     private TimerTask pendingJobsTask;
 
     protected ManagementServerMaintenanceManagerImpl() {
@@ -261,10 +261,8 @@ public void cancelMaintenance() {
     }
 
     private void waitForPendingJobs() {
-        if (this.pendingJobsTask != null) {
-            this.pendingJobsTask.cancel();
-            this.pendingJobsTask = null;
-        }
+        cancelWaitForPendingJobs();
+        this.timer = new Timer();
         this.pendingJobsTask = new CheckPendingJobsTask(this);
         long pendingJobsCheckDelayInMs = 1000L; // 1 sec
         long pendingJobsCheckPeriodInMs = 3L * 1000; // every 3 secs, check more frequently for pending jobs
@@ -276,6 +274,10 @@ private void cancelWaitForPendingJobs() {
             this.pendingJobsTask.cancel();
             this.pendingJobsTask = null;
         }
+        if (this.timer != null) {
+            this.timer.cancel();
+            this.timer = null;
+        }
     }
 
     @Override
@@ -534,13 +536,14 @@ public void run() {
 
                 // No more pending jobs. Good to terminate
                 if (managementServerMaintenanceManager.isShutdownTriggered()) {
-                    logger.info("Shutting down now");
+                    logger.info("MS is Shutting Down Now");
                     // update state to down ?
                     System.exit(0);
                 }
                 if (managementServerMaintenanceManager.isPreparingForMaintenance()) {
                     ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId());
                     if (totalAgents == 0) {
+                        logger.info("MS is in Maintenance Mode");
                         msHostDao.updateState(msHost.getId(), State.Maintenance);
                         managementServerMaintenanceManager.onMaintenance();
                         this.cancel();
@@ -569,7 +572,7 @@ public void run() {
                         return;
                     }
                 } else if (managementServerMaintenanceManager.isPreparingForShutdown()) {
-                    logger.info("Ready to shutdown");
+                    logger.info("MS is Ready To Shutdown");
                     ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId());
                     msHostDao.updateState(msHost.getId(), State.ReadyToShutDown);
                     this.cancel();

server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java

@@ -94,7 +94,7 @@ public List<String> getManagementServerList() {
 
     @Override
     public List<String> getManagementServerList(final Long hostId, final Long dcId, final List<Long> orderedHostIdList) {
-        return  getManagementServerList(hostId, dcId, orderedHostIdList, null);
+        return getManagementServerList(hostId, dcId, orderedHostIdList, null);
     }
 
     @Override
@@ -342,7 +342,7 @@ public boolean migrateAgents(String fromMsUuid, long fromMsId, String lbAlgorith
                 Long lbCheckInterval = 0L;
                 if (lbAlgorithmChanged) {
                     // send new MS list when there is change in lb algorithm
-                    msList = getManagementServerList(host.getId(), dcId, orderedHostIdList);
+                    msList = getManagementServerList(host.getId(), dcId, orderedHostIdList, lbAlgorithm);
                     lbCheckInterval = getLBPreferredHostCheckInterval(host.getClusterId());
                 }