From 53441a1c8cad3fdb2dc71a84cc4feaeaa2c49905 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 7 Mar 2025 15:14:23 +0530 Subject: [PATCH 1/3] Updated agent connections monitor thread timeunit to secs, in sync with the Wait config --- .../main/java/com/cloud/agent/manager/AgentManagerImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 592d45678057..818a3c5a23de 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -726,9 +726,9 @@ public boolean start() { _monitorExecutor.scheduleWithFixedDelay(new MonitorTask(), mgmtServiceConf.getPingInterval(), mgmtServiceConf.getPingInterval(), TimeUnit.SECONDS); - final int cleanupTime = Wait.value(); - newAgentConnectionsMonitor.scheduleAtFixedRate(new AgentNewConnectionsMonitorTask(), cleanupTime, - cleanupTime, TimeUnit.MINUTES); + final int cleanupTimeInSecs = Wait.value(); + newAgentConnectionsMonitor.scheduleAtFixedRate(new AgentNewConnectionsMonitorTask(), cleanupTimeInSecs, + cleanupTimeInSecs, TimeUnit.SECONDS); return true; } From 448361b1b567f65235b2eb4713dccca74a1ef81f Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Mon, 17 Mar 2025 12:01:52 +0530 Subject: [PATCH 2/3] Fix cutoff time in agent connections monitor thread Co-authored-by: abhishek.mrt22@gmail.com --- .../cloud/agent/manager/AgentManagerImpl.java | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 818a3c5a23de..1504af702bfe 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -51,8 +51,8 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; import org.apache.cloudstack.utils.identity.ManagementServerNode; -import org.apache.commons.collections.MapUtils; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.ThreadContext; @@ -1859,25 +1859,19 @@ protected void runInContext() { logger.trace("Agent New Connections Monitor is started."); final int cleanupTime = Wait.value(); Set> entrySet = newAgentConnections.entrySet(); - long cutOff = System.currentTimeMillis() - (cleanupTime * 60 * 1000L); - if (logger.isDebugEnabled()) { - List expiredConnections = newAgentConnections.entrySet() - .stream() - .filter(e -> e.getValue() <= cutOff) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - logger.debug(String.format("Currently %d active new connections, of which %d have expired - %s", - entrySet.size(), - expiredConnections.size(), - StringUtils.join(expiredConnections))); - } - for (Map.Entry entry : entrySet) { - if (entry.getValue() <= cutOff) { - if (logger.isTraceEnabled()) { - logger.trace(String.format("Cleaning up new agent connection for %s", entry.getKey())); - } - newAgentConnections.remove(entry.getKey()); - } + long cutOff = System.currentTimeMillis() - (cleanupTime * 1000L); + List expiredConnections = newAgentConnections.entrySet() + .stream() + .filter(e -> e.getValue() <= cutOff) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + logger.debug("Currently {} active new connections, of which {} have expired - {}", + entrySet.size(), + expiredConnections.size(), + StringUtils.join(expiredConnections)); + for (String connection : expiredConnections) { + logger.trace("Cleaning up new agent connection for {}", connection); + newAgentConnections.remove(connection); } } } From 745961b1bd001ef216d355bb91433d5292e53aa7 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Mon, 28 Apr 2025 16:34:19 +0530 Subject: [PATCH 3/3] New config (non-dynamic) to monitor and cleanup new agent connections, instead of using Wait which is dynamic --- .../src/main/java/com/cloud/agent/AgentManager.java | 4 ++-- .../com/cloud/agent/manager/AgentManagerImpl.java | 13 ++++++++----- .../com/cloud/vm/VirtualMachineManagerImpl.java | 2 +- .../java/com/cloud/utils/nio/NioConnection.java | 1 - 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java index 81525ca13f1b..b29eb38395fa 100644 --- a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java +++ b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java @@ -37,7 +37,7 @@ * AgentManager manages hosts. It directly coordinates between the DAOs and the connections it manages. */ public interface AgentManager { - static final ConfigKey Wait = new ConfigKey("Advanced", Integer.class, "wait", "1800", "Time in seconds to wait for control commands to return", + ConfigKey Wait = new ConfigKey("Advanced", Integer.class, "wait", "1800", "Time in seconds to wait for control commands to return", true); ConfigKey EnableKVMAutoEnableDisable = new ConfigKey<>(Boolean.class, "enable.kvm.host.auto.enable.disable", @@ -54,7 +54,7 @@ public interface AgentManager { "This timeout overrides the wait global config. This holds a comma separated key value pairs containing timeout (in seconds) for specific commands. " + "For example: DhcpEntryCommand=600, SavePasswordCommand=300, VmDataCommand=300", false); - public enum TapAgentsAction { + enum TapAgentsAction { Add, Del, Contains, } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 1504af702bfe..1abb78b32e9c 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -210,6 +210,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl "Number of maximum concurrent new connections server allows for remote agents. " + "If set to zero (default value) then no limit will be enforced on concurrent new connections", false); + protected final ConfigKey RemoteAgentNewConnectionsMonitorInterval = new ConfigKey<>("Advanced", Integer.class, "agent.connections.monitor.interval", "1800", + "Time in seconds to monitor the new agent connections and cleanup the expired connections.", false); protected final ConfigKey AlertWait = new ConfigKey("Advanced", Integer.class, "alert.wait", "1800", "Seconds to wait before alerting on a disconnected agent", true); protected final ConfigKey DirectAgentLoadSize = new ConfigKey("Advanced", Integer.class, "direct.agent.load.size", "16", @@ -726,9 +728,9 @@ public boolean start() { _monitorExecutor.scheduleWithFixedDelay(new MonitorTask(), mgmtServiceConf.getPingInterval(), mgmtServiceConf.getPingInterval(), TimeUnit.SECONDS); - final int cleanupTimeInSecs = Wait.value(); - newAgentConnectionsMonitor.scheduleAtFixedRate(new AgentNewConnectionsMonitorTask(), cleanupTimeInSecs, - cleanupTimeInSecs, TimeUnit.SECONDS); + final int agentConnectionsMonitorTimeInSecs = RemoteAgentNewConnectionsMonitorInterval.value(); + newAgentConnectionsMonitor.scheduleAtFixedRate(new AgentNewConnectionsMonitorTask(), agentConnectionsMonitorTimeInSecs, + agentConnectionsMonitorTimeInSecs, TimeUnit.SECONDS); return true; } @@ -1857,7 +1859,7 @@ protected class AgentNewConnectionsMonitorTask extends ManagedContextRunnable { @Override protected void runInContext() { logger.trace("Agent New Connections Monitor is started."); - final int cleanupTime = Wait.value(); + final int cleanupTime = RemoteAgentNewConnectionsMonitorInterval.value(); Set> entrySet = newAgentConnections.entrySet(); long cutOff = System.currentTimeMillis() - (cleanupTime * 1000L); List expiredConnections = newAgentConnections.entrySet() @@ -1952,7 +1954,8 @@ public String getConfigComponentName() { public ConfigKey[] getConfigKeys() { return new ConfigKey[] { CheckTxnBeforeSending, Workers, Port, Wait, AlertWait, DirectAgentLoadSize, DirectAgentPoolSize, DirectAgentThreadCap, EnableKVMAutoEnableDisable, ReadyCommandWait, - GranularWaitTimeForCommands, RemoteAgentSslHandshakeTimeout, RemoteAgentMaxConcurrentNewConnections }; + GranularWaitTimeForCommands, RemoteAgentSslHandshakeTimeout, RemoteAgentMaxConcurrentNewConnections, + RemoteAgentNewConnectionsMonitorInterval }; } protected class SetHostParamsListener implements Listener { diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 472fe148a5d7..6b39becfbb34 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -427,7 +427,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac static final ConfigKey VmOpCleanupInterval = new ConfigKey("Advanced", Long.class, "vm.op.cleanup.interval", "86400", "Interval to run the thread that cleans up the vm operations (in seconds)", false); static final ConfigKey VmOpCleanupWait = new ConfigKey("Advanced", Long.class, "vm.op.cleanup.wait", "3600", - "Time (in seconds) to wait before cleanuping up any vm work items", true); + "Time (in seconds) to wait before cleaning up any vm work items", true); static final ConfigKey VmOpCancelInterval = new ConfigKey("Advanced", Long.class, "vm.op.cancel.interval", "3600", "Time (in seconds) to wait before cancelling a operation", false); static final ConfigKey VmDestroyForcestop = new ConfigKey("Advanced", Boolean.class, "vm.destroy.forcestop", "false", diff --git a/utils/src/main/java/com/cloud/utils/nio/NioConnection.java b/utils/src/main/java/com/cloud/utils/nio/NioConnection.java index 2a157b9c0019..d2e2e8c89567 100644 --- a/utils/src/main/java/com/cloud/utils/nio/NioConnection.java +++ b/utils/src/main/java/com/cloud/utils/nio/NioConnection.java @@ -219,7 +219,6 @@ protected boolean rejectConnectionIfBusy(final SocketChannel socketChannel) thro return true; } - protected void accept(final SelectionKey key) throws IOException { final ServerSocketChannel serverSocketChannel = (ServerSocketChannel)key.channel(); final SocketChannel socketChannel = serverSocketChannel.accept();