fixes & code improvements

Author: sureshanaparti

agent/src/main/java/com/cloud/agent/Agent.java

@@ -526,13 +526,14 @@ public Task create(final Task.Type type, final Link link, final byte[] data) {
     }
 
     protected void reconnect(final Link link) {
-        reconnect(link, null);
+        reconnect(link, null, false);
     }
 
-    protected void reconnect(final Link link, String host) {
-        if (!_reconnectAllowed) {
+    protected void reconnect(final Link link, String host, boolean forTransfer) {
+        if (!(forTransfer || _reconnectAllowed)) {
             return;
         }
+
         synchronized (this) {
             if (_startup != null) {
                 _startup.cancel();
@@ -885,7 +886,7 @@ private void migrateAgentConnection(List<String> avoidMsList) {
         _reconnectAllowed = true;
         _shell.resetHostCounter();
         _shell.setConnectionTransfer(true);
-        reconnect(_link, preferredHost);
+        reconnect(_link, preferredHost, true);
     }
 
     public void processResponse(final Response response, final Link link) {

api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java

@@ -83,11 +83,11 @@ public class AsyncJobResponse extends BaseResponse {
     @Param(description = "the unique ID of the instance/entity object related to the job")
     private String jobInstanceId;
 
-    @SerializedName("managementserverid")
+    @SerializedName(ApiConstants.MANAGEMENT_SERVER_ID)
     @Param(description = "the msid of the management server on which the job is running", since = "4.19")
     private String managementServerId;
 
-    @SerializedName("managementservername")
+    @SerializedName(ApiConstants.MANAGEMENT_SERVER_NAME)
     @Param(description = "the management server name of the host", since = "4.21.0")
     private String managementServerName;
 

api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java

@@ -172,7 +172,7 @@ public class HostForMigrationResponse extends BaseResponse {
     @Param(description = "the date and time the host was last pinged")
     private Date lastPinged;
 
-    @SerializedName("managementserverid")
+    @SerializedName(ApiConstants.MANAGEMENT_SERVER_ID)
     @Param(description = "the management server ID of the host")
     private Long managementServerId;
 

api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java

@@ -190,11 +190,11 @@ public class HostResponse extends BaseResponseWithAnnotations {
     @Param(description = "the virtual machine id for host type ConsoleProxy and SecondaryStorageVM", since = "4.21.0")
     private String virtualMachineId;
 
-    @SerializedName("managementserverid")
+    @SerializedName(ApiConstants.MANAGEMENT_SERVER_ID)
     @Param(description = "the management server ID of the host")
     private String managementServerId;
 
-    @SerializedName("managementservername")
+    @SerializedName(ApiConstants.MANAGEMENT_SERVER_NAME)
     @Param(description = "the management server name of the host", since = "4.21.0")
     private String managementServerName;
 

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.agent.manager;
 
+import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.nio.channels.ClosedChannelException;
@@ -307,12 +308,39 @@ public void unregisterForHostEvents(final int id) {
 
     @Override
     public void onManagementServerMaintenance() {
+        logger.debug("Management server maintenance enabled");
         _monitorExecutor.shutdownNow();
+        if (_connection != null) {
+            _connection.stop();
+
+            try {
+                _connection.cleanUp();
+            } catch (final IOException e) {
+                logger.warn("Fail to clean up old connection", e);
+            }
+        }
+        _connectExecutor.shutdownNow();
     }
 
     @Override
     public void onManagementServerCancelMaintenance() {
+        logger.debug("Management server maintenance disabled");
+        if (_connectExecutor.isShutdown()) {
+            _connectExecutor = new ThreadPoolExecutor(100, 500, 60l, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("AgentConnectTaskPool"));
+            _connectExecutor.allowCoreThreadTimeOut(true);
+        }
+
+        startDirectlyConnectedHosts(true);
+        if (_connection != null) {
+            try {
+                _connection.start();
+            } catch (final NioConnectionException e) {
+                logger.error("Error when connecting to the NioServer!", e);
+            }
+        }
+
         if (_monitorExecutor.isShutdown()) {
+            _monitorExecutor = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("AgentMonitor"));
             _monitorExecutor.scheduleWithFixedDelay(new MonitorTask(), mgmtServiceConf.getPingInterval(), mgmtServiceConf.getPingInterval(), TimeUnit.SECONDS);
         }
     }
@@ -650,7 +678,7 @@ public boolean start() {
             return true;
         }
 
-        startDirectlyConnectedHosts();
+        startDirectlyConnectedHosts(false);
 
         if (_connection != null) {
             try {
@@ -665,10 +693,10 @@ public boolean start() {
         return true;
     }
 
-    public void startDirectlyConnectedHosts() {
+    public void startDirectlyConnectedHosts(final boolean forRebalance) {
         final List<HostVO> hosts = _resourceMgr.findDirectlyConnectedHosts();
         for (final HostVO host : hosts) {
-            loadDirectlyConnectedHost(host, false);
+            loadDirectlyConnectedHost(host, forRebalance);
         }
     }
 

engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java

@@ -47,7 +47,6 @@
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.ha.dao.HAConfigDao;
-import org.apache.cloudstack.maintenance.ManagementServerMaintenanceListener;
 import org.apache.cloudstack.maintenance.ManagementServerMaintenanceManager;
 import org.apache.cloudstack.maintenance.command.BaseShutdownManagementServerHostCommand;
 import org.apache.cloudstack.maintenance.command.CancelMaintenanceManagementServerHostCommand;
@@ -108,8 +107,8 @@
 import com.cloud.utils.nio.Task;
 import com.google.gson.Gson;
 
-public class ClusteredAgentManagerImpl extends AgentManagerImpl implements ClusterManagerListener, ManagementServerMaintenanceListener, ClusteredAgentRebalanceService {
-    private static final ScheduledExecutorService s_transferExecutor = Executors.newScheduledThreadPool(2, new NamedThreadFactory("Cluster-AgentRebalancingExecutor"));
+public class ClusteredAgentManagerImpl extends AgentManagerImpl implements ClusterManagerListener, ClusteredAgentRebalanceService {
+    private static ScheduledExecutorService s_transferExecutor = Executors.newScheduledThreadPool(2, new NamedThreadFactory("Cluster-AgentRebalancingExecutor"));
     private final long rebalanceTimeOut = 300000; // 5 mins - after this time remove the agent from the transfer list
 
     public final static long STARTUP_DELAY = 5000;
@@ -172,8 +171,6 @@ public boolean configure(final String name, final Map<String, Object> xmlParams)
         _clusterMgr.registerListener(this);
         _clusterMgr.registerDispatcher(new ClusterDispatcher());
 
-        managementServerMaintenanceManager.registerListener(this);
-
         _gson = GsonHelper.getGson();
 
         return super.configure(name, xmlParams);
@@ -595,7 +592,7 @@ public boolean stop() {
     }
 
     @Override
-    public void startDirectlyConnectedHosts() {
+    public void startDirectlyConnectedHosts(final boolean forRebalance) {
         // override and let it be dummy for purpose, we will scan and load direct agents periodically.
         // We may also pickup agents that have been left over from other crashed management server
     }
@@ -1477,15 +1474,20 @@ private void updateLastManagementServer(long hostId, long msId) {
 
     @Override
     public void onManagementServerMaintenance() {
+        logger.debug("Management server maintenance enabled");
         s_transferExecutor.shutdownNow();
         cleanupTransferMap(_nodeId);
+        super.onManagementServerMaintenance();
     }
 
     @Override
     public void onManagementServerCancelMaintenance() {
+        logger.debug("Management server maintenance disabled");
+        super.onManagementServerCancelMaintenance();
         if (isAgentRebalanceEnabled()) {
             cleanupTransferMap(_nodeId);
             if (s_transferExecutor.isShutdown()) {
+                s_transferExecutor = Executors.newScheduledThreadPool(2, new NamedThreadFactory("Cluster-AgentRebalancingExecutor"));
                 s_transferExecutor.scheduleAtFixedRate(getAgentRebalanceScanTask(), 60000, 60000, TimeUnit.MILLISECONDS);
                 s_transferExecutor.scheduleAtFixedRate(getTransferScanTask(), 60000, ClusteredAgentRebalanceService.DEFAULT_TRANSFER_CHECK_INTERVAL, TimeUnit.MILLISECONDS);
             }

plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImpl.java

@@ -194,14 +194,17 @@ public void triggerShutdown() {
     }
 
     private void prepareForShutdown(boolean postTrigger) {
-        if (this.preparingForMaintenance) {
-            throw new CloudRuntimeException("Maintenance has already been initiated, cancel maintenance and try again");
-        }
+        if (!postTrigger) {
+            if (this.preparingForMaintenance) {
+                throw new CloudRuntimeException("Maintenance has already been initiated, cancel maintenance and try again");
+            }
 
-        // Ensure we don't throw an error if triggering a shutdown after just preparing for it
-        if (!postTrigger && this.preparingForShutdown) {
-            throw new CloudRuntimeException("Shutdown has already been triggered");
+            // Ensure we don't throw an error if triggering a shutdown after just preparing for it
+            if (this.preparingForShutdown) {
+                throw new CloudRuntimeException("Shutdown has already been triggered");
+            }
         }
+
         this.preparingForShutdown = true;
         jobManager.disableAsyncJobs();
         waitForPendingJobs();
@@ -220,6 +223,7 @@ public void cancelShutdown() {
 
         this.preparingForShutdown = false;
         this.shutdownTriggered = false;
+        resetPreparingForMaintenance();
         jobManager.enableAsyncJobs();
         cancelWaitForPendingJobs();
     }
@@ -246,8 +250,14 @@ public void cancelMaintenance() {
             throw new CloudRuntimeException("Maintenance has not been initiated");
         }
         resetPreparingForMaintenance();
+        this.preparingForShutdown = false;
+        this.shutdownTriggered = false;
         jobManager.enableAsyncJobs();
         cancelWaitForPendingJobs();
+        ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId());
+        if (msHost != null && State.Maintenance.equals(msHost.getState())) {
+            onCancelMaintenance();
+        }
     }
 
     private void waitForPendingJobs() {
@@ -257,7 +267,7 @@ private void waitForPendingJobs() {
         }
         this.pendingJobsTask = new CheckPendingJobsTask(this);
         long pendingJobsCheckDelayInMs = 1000L; // 1 sec
-        long pendingJobsCheckPeriodInMs = 15L * 1000; // every 15 secs
+        long pendingJobsCheckPeriodInMs = 3L * 1000; // every 3 secs, check more frequently for pending jobs
         timer.scheduleAtFixedRate(pendingJobsTask, pendingJobsCheckDelayInMs, pendingJobsCheckPeriodInMs);
     }
 
@@ -429,6 +439,8 @@ public ManagementServerMaintenanceResponse cancelMaintenance(CancelMaintenanceCm
     @Override
     public void cancelPreparingForMaintenance(ManagementServerHostVO msHost) {
         resetPreparingForMaintenance();
+        this.preparingForShutdown = false;
+        this.shutdownTriggered = false;
         jobManager.enableAsyncJobs();
         if (msHost == null) {
             msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId());
@@ -530,6 +542,7 @@ public void run() {
                     ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId());
                     if (totalAgents == 0) {
                         msHostDao.updateState(msHost.getId(), State.Maintenance);
+                        managementServerMaintenanceManager.onMaintenance();
                         this.cancel();
                         return;
                     }

server/src/main/java/com/cloud/api/ApiServer.java

@@ -475,7 +475,6 @@ public boolean start() {
                 s_apiNameCmdClassMap.put(apiName, apiCmdList);
             }
             apiCmdList.add(cmdClass);
-
         }
 
         setEncodeApiResponse(EncodeApiResponse.value());

ui/public/locales/en.json

@@ -3411,8 +3411,8 @@
 "message.setup.physical.network.during.zone.creation.basic": "When adding a basic zone, you can set up one physical Network, which corresponds to a NIC on the hypervisor. The Network carries several types of traffic.<br/><br/>You may also <strong>add</strong> other traffic types onto the physical Network.",
 "message.shared.network.offering.warning": "Domain admins and regular Users can only create shared Networks from Network offering with the setting specifyvlan=false. Please contact an administrator to create a Network offering if this list is empty.",
 "message.shared.network.unsupported.for.nsx": "Shared networks aren't supported for NSX enabled zones",
-"message.shutdown.triggered": "A shutdown has been triggered. CloudStack will not accept new jobs",
-"message.maintenance.initiated": "A maintenance has been initiated. Management Server will not accept new jobs",
+"message.shutdown.triggered": "Shutdown has been triggered. This Management Server will not accept new jobs",
+"message.maintenance.initiated": "Maintenance has been initiated. This Management Server will not accept new jobs",
 "message.snapshot.additional.zones": "Snapshots will always be created in its native zone - %x, here you can select additional zone(s) where it will be copied to at creation time",
 "message.sourcenatip.change.warning": "WARNING: Changing the sourcenat IP address of the network will cause connectivity downtime for the Instances with NICs in the Network.",
 "message.sourcenatip.change.inhibited": "Changing the sourcenat to this IP of the Network to this address is inhibited as firewall rules are defined for it. This can include port forwarding or load balancing rules.\n - If this is an Isolated Network, please use updateNetwork/click the edit button.\n - If this is a VPC, first clear all other rules for this address.",

utils/src/main/java/com/cloud/utils/nio/NioConnection.java

@@ -68,6 +68,7 @@ public abstract class NioConnection implements Callable<Boolean> {
     protected boolean _isRunning;
     protected boolean _isStartup;
     protected int _port;
+    protected int _workers;
     protected List<ChangeRequest> _todos;
     protected HandlerFactory _factory;
     protected String _name;
@@ -80,6 +81,7 @@ public NioConnection(final String name, final int port, final int workers, final
         _isRunning = false;
         _selector = null;
         _port = port;
+        _workers = workers;
         _factory = factory;
         _executor = new ThreadPoolExecutor(workers, 5 * workers, 1, TimeUnit.DAYS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory(name + "-Handler"));
         _sslHandshakeExecutor = Executors.newCachedThreadPool(new NamedThreadFactory(name + "-SSLHandshakeHandler"));
@@ -106,6 +108,9 @@ public void start() throws NioConnectionException {
         }
         _isStartup = true;
 
+        if (_executor.isShutdown()) {
+            _executor = new ThreadPoolExecutor(_workers, 5 * _workers, 1, TimeUnit.DAYS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory(_name + "-Handler"));
+        }
         _threadExecutor = Executors.newSingleThreadExecutor(new NamedThreadFactory(this._name + "-NioConnectionHandler"));
         _isRunning = true;
         _futureTask = _threadExecutor.submit(this);