add cmd queue retry feature toggle and provide get running status for platform queue

Author: Yang Guo

docker/app-cc.properties

@@ -28,6 +28,7 @@ mq.host = amqp://localhost:5672
 mq.management.host = http://localhost:15672
 
 #### cmd queue settings ###
+queue.cmd.retry.enable = false
 queue.cmd.rabbit.enable = false
 queue.cmd.rabbit.name = flow-cmd-queue-default
 queue.cmd.idle_agent.timeout = 30

platform-control-center/src/main/java/com/flow/platform/cc/config/QueueConfig.java

@@ -42,6 +42,8 @@ public class QueueConfig {
 
     public final static int CMD_QUEUE_DEFAULT_PRIORITY = 10;
 
+    public final static String PROP_CMD_QUEUE_RETRY = "queue.cmd.retry.enable";
+
     private final static Logger LOGGER = new Logger(QueueConfig.class);
 
     /**
@@ -70,14 +72,26 @@ public class QueueConfig {
     @Value("${queue.cmd.rabbit.enable}")
     private Boolean cmdQueueRabbitEnable;
 
+    /**
+     * Enable cmd queue retry instead of pause/resume logic
+     */
+    @Value("${queue.cmd.retry.enable}")
+    private Boolean cmdQueueRetryEnable;
+
+    /**
+     * AppConfig task executor
+     */
     @Autowired
-    private ThreadPoolTaskExecutor taskExecutor; // from AppConfig
+    private ThreadPoolTaskExecutor taskExecutor;
 
     @PostConstruct
     public void init() {
         LOGGER.trace("Host: %s", host);
         LOGGER.trace("Management Host: %s", mgrHost);
+
         LOGGER.trace("Cmd queue name: %s", cmdQueueName);
+        LOGGER.trace("Cmd RabbitMQ enabled: %s", cmdQueueRabbitEnable);
+        LOGGER.trace("Cmd queue retry enabled: %s", cmdQueueRetryEnable);
     }
 
     @Bean

platform-control-center/src/main/java/com/flow/platform/cc/consumer/AgentResourceEventHandler.java

@@ -16,13 +16,15 @@
 
 package com.flow.platform.cc.consumer;
 
+import com.flow.platform.cc.config.QueueConfig;
 import com.flow.platform.cc.event.AgentResourceEvent;
 import com.flow.platform.cc.event.AgentResourceEvent.Category;
 import com.flow.platform.cc.service.ZoneService;
 import com.flow.platform.core.queue.PlatformQueue;
 import com.flow.platform.util.Logger;
 import org.springframework.amqp.core.Message;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.ApplicationListener;
 import org.springframework.stereotype.Component;
 
@@ -40,6 +42,9 @@ public class AgentResourceEventHandler implements ApplicationListener<AgentResou
     @Autowired
     private PlatformQueue<Message> cmdQueue;
 
+    @Value("${queue.cmd.retry.enable}")
+    private Boolean cmdQueueRetryEnable;
+
     @Override
     public void onApplicationEvent(AgentResourceEvent event) {
         String zone = event.getZone();
@@ -48,6 +53,12 @@ public void onApplicationEvent(AgentResourceEvent event) {
         // cleanup agent from zone
         zoneService.keepIdleAgentTask();
 
+        // do not control cmd queue since enable retry
+        Boolean isRetry = Boolean.parseBoolean(System.getProperty(QueueConfig.PROP_CMD_QUEUE_RETRY, "false"));
+        if (cmdQueueRetryEnable || isRetry) {
+            return;
+        }
+
         if (event.getCategory() == Category.FULL) {
             cmdQueue.pause();
             LOGGER.trace("Pause cmd queue since no agent resources");

platform-control-center/src/main/java/com/flow/platform/cc/service/AgentServiceImpl.java

@@ -18,6 +18,8 @@
 
 import com.flow.platform.cc.config.TaskConfig;
 import com.flow.platform.cc.dao.AgentDao;
+import com.flow.platform.cc.event.AgentResourceEvent;
+import com.flow.platform.cc.event.AgentResourceEvent.Category;
 import com.flow.platform.cc.exception.AgentErr;
 import com.flow.platform.core.exception.IllegalParameterException;
 import com.flow.platform.core.exception.IllegalStatusException;
@@ -145,6 +147,11 @@ public void saveWithStatus(Agent agent, AgentStatus status) {
         if (statusIsChanged) {
             this.webhookCallback(agent);
         }
+
+        // boardcast AgentResourceEvent for release
+        if (agent.getStatus() == AgentStatus.IDLE) {
+            this.dispatchEvent(new AgentResourceEvent(this, agent.getZone(), Category.RELEASED));
+        }
     }
 
     @Override

platform-control-center/src/main/java/com/flow/platform/cc/service/CmdServiceImpl.java

@@ -305,11 +305,6 @@ private void updateAgentStatusFromCmd(Cmd cmd) {
 
         Agent agent = agentService.find(agentPath);
         agentService.saveWithStatus(agent, isAgentBusy ? AgentStatus.BUSY : AgentStatus.IDLE);
-
-        // boardcast AgentResourceEvent for release
-        if (agent.getStatus() == AgentStatus.IDLE) {
-            this.dispatchEvent(new AgentResourceEvent(this, agent.getZone(), Category.RELEASED));
-        }
     }
 
     /**

platform-control-center/src/main/resources/app-default.properties

@@ -28,6 +28,7 @@ mq.host = amqp://localhost:5672
 mq.management.host = http://localhost:15672
 
 #### cmd queue settings ###
+queue.cmd.retry.enable = false
 queue.cmd.rabbit.enable = false
 queue.cmd.rabbit.name = flow-cmd-queue-default
 queue.cmd.idle_agent.timeout = 30

platform-control-center/src/test/java/com/flow/platform/cc/test/consumer/CmdQueueConsumerTest.java

@@ -23,6 +23,7 @@
 import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
 import static com.github.tomakehurst.wiremock.client.WireMock.verify;
 
+import com.flow.platform.cc.config.QueueConfig;
 import com.flow.platform.cc.domain.CmdStatusItem;
 import com.flow.platform.cc.service.AgentService;
 import com.flow.platform.cc.service.CmdService;
@@ -47,6 +48,7 @@
 import org.junit.Test;
 import org.junit.runners.MethodSorters;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.dao.CannotAcquireLockException;
 
 /**
@@ -69,8 +71,13 @@ public class CmdQueueConsumerTest extends TestBase {
     @Autowired
     private CmdService cmdService;
 
+    @Value("${queue.cmd.retry.enable}")
+    private Boolean cmdQueueRetryEnable;
+
     @Before
     public void before() throws Throwable {
+        System.setProperty(QueueConfig.PROP_CMD_QUEUE_RETRY, "true");
+
         zoneService.createZone(new Zone(ZONE, "mock-cloud-provider"));
 
         // ensure zookeeper node is created for zone
@@ -205,5 +212,6 @@ public void should_stop_queued_cmd() throws Throwable {
     @After
     public void deleteZone() {
         deleteNodeWithChildren(ZKHelper.buildPath(ZONE, null));
+        System.setProperty(QueueConfig.PROP_CMD_QUEUE_RETRY, "false");
     }
 }

platform-control-center/src/test/java/com/flow/platform/cc/test/controller/CmdControllerTest.java

@@ -84,7 +84,7 @@ public void should_list_cmd_types() throws Throwable {
         String raw = result.getResponse().getContentAsString();
         CmdType[] types = gsonConfig.fromJson(raw, CmdType[].class);
         Assert.assertNotNull(types);
-        Assert.assertTrue(types.length == 6);
+        Assert.assertTrue(types.length == 7);
     }
 
     @Test

platform-control-center/src/test/java/com/flow/platform/cc/test/service/CmdDispatchServiceTest.java

@@ -25,6 +25,8 @@
 import com.flow.platform.cc.service.CmdService;
 import com.flow.platform.cc.test.TestBase;
 import com.flow.platform.cc.util.ZKHelper;
+import com.flow.platform.core.exception.FlowException;
+import com.flow.platform.core.queue.PlatformQueue;
 import com.flow.platform.domain.Agent;
 import com.flow.platform.domain.AgentPath;
 import com.flow.platform.domain.AgentStatus;
@@ -41,6 +43,7 @@
 import org.junit.FixMethodOrder;
 import org.junit.Test;
 import org.junit.runners.MethodSorters;
+import org.springframework.amqp.core.Message;
 import org.springframework.beans.factory.annotation.Autowired;
 
 /**
@@ -61,6 +64,9 @@ public class CmdDispatchServiceTest extends TestBase {
     @Autowired
     private List<Zone> defaultZones;
 
+    @Autowired
+    private PlatformQueue<Message> cmdQueue;
+
     private AgentPath agentPath;
 
     private Agent target;
@@ -91,8 +97,30 @@ public void toCreateSession() throws Throwable {
     }
 
     @Test
-    public void should_broadcast_agent_resource_event_if_no_available_agent() throws  Throwable {
+    public void should_broadcast_agent_resource_event_if_no_available_agent() throws Throwable {
         // given: make no available agent resource
+        Assert.assertEquals(true, cmdQueue.isRunning());
+
+        // when: send create session cmd
+        Cmd cmdToCreateSession = cmdService.create(new CmdInfo(agentPath, CmdType.CREATE_SESSION, null));
+
+        try {
+            cmdDispatchService.dispatch(cmdToCreateSession.getId(), false);
+        } catch (FlowException ignore) {
+
+        }
+
+        // then: queue should be pause since no available
+        Assert.assertEquals(false, cmdQueue.isRunning());
+
+        // when: send delete session to release agent
+        CmdInfo cmd = new CmdInfo(agentPath, CmdType.DELETE_SESSION, null);
+        cmd.setSessionId(target.getSessionId());
+        Cmd cmdToDeleteSession = cmdService.create(cmd);
+        cmdDispatchService.dispatch(cmdToDeleteSession.getId(), false);
+
+        // then: queue should be resumed since agent resource released
+        Assert.assertEquals(true, cmdQueue.isRunning());
     }
 
     @Test

platform-control-center/src/test/resources/app-test.properties

@@ -35,6 +35,7 @@ mq.host = amqp://localhost:5672
 mq.management.host = http://localhost:15672
 
 #### cmd queue settings ###
+queue.cmd.retry.enable = false
 queue.cmd.rabbit.enable = false
 queue.cmd.rabbit.name = flow-cmd-queue-default
 queue.cmd.idle_agent.timeout = 0