Merge pull request #107 from FlowCI/feature/cc/optimize_cmd_queue

yang.guo · web-flow · commit 78bdf139b7bf · 2017-10-15T03:03:27.000-05:00
Feature/cc/optimize cmd queue
diff --git a/docker/app-cc.properties b/docker/app-cc.properties
@@ -28,6 +28,7 @@ mq.host = amqp://localhost:5672
 mq.management.host = http://localhost:15672
 
 #### cmd queue settings ###
+queue.cmd.retry.enable = false
 queue.cmd.rabbit.enable = false
 queue.cmd.rabbit.name = flow-cmd-queue-default
 queue.cmd.idle_agent.timeout = 30
diff --git a/platform-control-center/src/main/java/com/flow/platform/cc/config/QueueConfig.java b/platform-control-center/src/main/java/com/flow/platform/cc/config/QueueConfig.java
@@ -42,6 +42,8 @@ public class QueueConfig {
 
     public final static int CMD_QUEUE_DEFAULT_PRIORITY = 10;
 
+    public final static String PROP_CMD_QUEUE_RETRY = "queue.cmd.retry.enable";
+
     private final static Logger LOGGER = new Logger(QueueConfig.class);
 
     /**
@@ -70,14 +72,26 @@ public class QueueConfig {
     @Value("${queue.cmd.rabbit.enable}")
     private Boolean cmdQueueRabbitEnable;
 
+    /**
+     * Enable cmd queue retry instead of pause/resume logic
+     */
+    @Value("${queue.cmd.retry.enable}")
+    private Boolean cmdQueueRetryEnable;
+
+    /**
+     * AppConfig task executor
+     */
     @Autowired
-    private ThreadPoolTaskExecutor taskExecutor; // from AppConfig
+    private ThreadPoolTaskExecutor taskExecutor;
 
     @PostConstruct
     public void init() {
         LOGGER.trace("Host: %s", host);
         LOGGER.trace("Management Host: %s", mgrHost);
+
         LOGGER.trace("Cmd queue name: %s", cmdQueueName);
+        LOGGER.trace("Cmd RabbitMQ enabled: %s", cmdQueueRabbitEnable);
+        LOGGER.trace("Cmd queue retry enabled: %s", cmdQueueRetryEnable);
     }
 
     @Bean
diff --git a/platform-control-center/src/main/java/com/flow/platform/cc/consumer/AgentResourceEventHandler.java b/platform-control-center/src/main/java/com/flow/platform/cc/consumer/AgentResourceEventHandler.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2017 flow.ci
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.flow.platform.cc.consumer;
+
+import com.flow.platform.cc.config.QueueConfig;
+import com.flow.platform.cc.event.AgentResourceEvent;
+import com.flow.platform.cc.event.AgentResourceEvent.Category;
+import com.flow.platform.cc.service.ZoneService;
+import com.flow.platform.core.queue.PlatformQueue;
+import com.flow.platform.util.Logger;
+import org.springframework.amqp.core.Message;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.ApplicationListener;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author yang
+ */
+@Component
+public class AgentResourceEventHandler implements ApplicationListener<AgentResourceEvent> {
+
+    private final static Logger LOGGER = new Logger(AgentResourceEventHandler.class);
+
+    @Autowired
+    private ZoneService zoneService;
+
+    @Autowired
+    private PlatformQueue<Message> cmdQueue;
+
+    @Value("${queue.cmd.retry.enable}")
+    private Boolean cmdQueueRetryEnable;
+
+    @Override
+    public void onApplicationEvent(AgentResourceEvent event) {
+        String zone = event.getZone();
+        LOGGER.trace("AgentResourceEvent received for zone '%s' with '%s'", zone, event.getCategory());
+
+        // cleanup agent from zone
+        zoneService.keepIdleAgentTask();
+
+        // do not control cmd queue since enable retry
+        Boolean isRetry = Boolean.parseBoolean(System.getProperty(QueueConfig.PROP_CMD_QUEUE_RETRY, "false"));
+        if (cmdQueueRetryEnable || isRetry) {
+            return;
+        }
+
+        if (event.getCategory() == Category.FULL) {
+            cmdQueue.pause();
+            LOGGER.trace("Pause cmd queue since no agent resources");
+            return;
+        }
+
+        if (event.getCategory() == Category.RELEASED) {
+            cmdQueue.resume();
+            LOGGER.trace("Resume cmd queue since has agent resource released");
+        }
+    }
+}
diff --git a/platform-control-center/src/main/java/com/flow/platform/cc/consumer/NoAvailbleResourceEventHandler.java b/platform-control-center/src/main/java/com/flow/platform/cc/consumer/NoAvailbleResourceEventHandler.java
diff --git a/platform-control-center/src/main/java/com/flow/platform/cc/event/AgentResourceEvent.java b/platform-control-center/src/main/java/com/flow/platform/cc/event/AgentResourceEvent.java
@@ -25,16 +25,32 @@
  *
  * @author yang
  */
-public class NoAvailableResourceEvent extends ApplicationEvent {
+public class AgentResourceEvent extends ApplicationEvent {
 
-    private String zone;
+    public enum Category {
+        RELEASED,
 
-    public NoAvailableResourceEvent(Object source, String zone) {
+        /**
+         * All agent resource occupied
+         */
+        FULL
+    }
+
+    private final String zone;
+
+    private final Category category;
+
+    public AgentResourceEvent(Object source, String zone, Category category) {
         super(source);
         this.zone = zone;
+        this.category = category;
     }
 
     public String getZone() {
         return zone;
     }
+
+    public Category getCategory() {
+        return category;
+    }
 }
diff --git a/platform-control-center/src/main/java/com/flow/platform/cc/service/AgentServiceImpl.java b/platform-control-center/src/main/java/com/flow/platform/cc/service/AgentServiceImpl.java
@@ -18,6 +18,8 @@
 
 import com.flow.platform.cc.config.TaskConfig;
 import com.flow.platform.cc.dao.AgentDao;
+import com.flow.platform.cc.event.AgentResourceEvent;
+import com.flow.platform.cc.event.AgentResourceEvent.Category;
 import com.flow.platform.cc.exception.AgentErr;
 import com.flow.platform.core.exception.IllegalParameterException;
 import com.flow.platform.core.exception.IllegalStatusException;
@@ -145,6 +147,11 @@ public void saveWithStatus(Agent agent, AgentStatus status) {
         if (statusIsChanged) {
             this.webhookCallback(agent);
         }
+
+        // boardcast AgentResourceEvent for release
+        if (agent.getStatus() == AgentStatus.IDLE) {
+            this.dispatchEvent(new AgentResourceEvent(this, agent.getZone(), Category.RELEASED));
+        }
     }
 
     @Override
diff --git a/platform-control-center/src/main/java/com/flow/platform/cc/service/CmdDispatchServiceImpl.java b/platform-control-center/src/main/java/com/flow/platform/cc/service/CmdDispatchServiceImpl.java
@@ -18,7 +18,8 @@
 
 import com.flow.platform.cc.config.TaskConfig;
 import com.flow.platform.cc.domain.CmdStatusItem;
-import com.flow.platform.cc.event.NoAvailableResourceEvent;
+import com.flow.platform.cc.event.AgentResourceEvent;
+import com.flow.platform.cc.event.AgentResourceEvent.Category;
 import com.flow.platform.cc.exception.AgentErr;
 import com.flow.platform.cc.exception.AgentErr.NotAvailableException;
 import com.flow.platform.cc.util.ZKHelper;
@@ -130,11 +131,9 @@ public Cmd dispatch(String cmdId, boolean reset) {
             CmdStatusItem statusItem = new CmdStatusItem(cmd.getId(), CmdStatus.REJECTED, null, false, true);
             cmdService.updateStatus(statusItem, false);
 
-            // broadcast NoAvailableResourceEvent with zone name
-            String zone = cmd.getAgentPath().getZone();
-
+            // broadcast AgentResourceEvent with zone name
             if (e instanceof NotAvailableException) {
-                this.dispatchEvent(new NoAvailableResourceEvent(this, zone));
+                this.dispatchEvent(new AgentResourceEvent(this, cmd.getAgentPath().getZone(), Category.FULL));
             }
 
             throw e;
diff --git a/platform-control-center/src/main/java/com/flow/platform/cc/service/CmdServiceImpl.java b/platform-control-center/src/main/java/com/flow/platform/cc/service/CmdServiceImpl.java
@@ -29,6 +29,8 @@
 import com.flow.platform.cc.dao.CmdResultDao;
 import com.flow.platform.cc.domain.CmdQueueItem;
 import com.flow.platform.cc.domain.CmdStatusItem;
+import com.flow.platform.cc.event.AgentResourceEvent;
+import com.flow.platform.cc.event.AgentResourceEvent.Category;
 import com.flow.platform.cc.exception.AgentErr;
 import com.flow.platform.core.queue.PlatformQueue;
 import com.flow.platform.core.service.WebhookServiceImplBase;
diff --git a/platform-control-center/src/main/java/com/flow/platform/cc/service/ZoneServiceImpl.java b/platform-control-center/src/main/java/com/flow/platform/cc/service/ZoneServiceImpl.java
@@ -17,7 +17,6 @@
 package com.flow.platform.cc.service;
 
 import com.flow.platform.cc.config.TaskConfig;
-import com.flow.platform.cc.event.NoAvailableResourceEvent;
 import com.flow.platform.cc.util.ZKHelper;
 import com.flow.platform.cloud.InstanceManager;
 import com.flow.platform.core.context.ContextEvent;
@@ -45,8 +44,6 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Isolation;
-import org.springframework.transaction.annotation.Transactional;
 
 /**
  * @author gy@fir.im
diff --git a/platform-control-center/src/main/resources/app-default.properties b/platform-control-center/src/main/resources/app-default.properties
@@ -28,6 +28,7 @@ mq.host = amqp://localhost:5672
 mq.management.host = http://localhost:15672
 
 #### cmd queue settings ###
+queue.cmd.retry.enable = false
 queue.cmd.rabbit.enable = false
 queue.cmd.rabbit.name = flow-cmd-queue-default
 queue.cmd.idle_agent.timeout = 30
diff --git a/platform-control-center/src/test/java/com/flow/platform/cc/test/consumer/CmdQueueConsumerTest.java b/platform-control-center/src/test/java/com/flow/platform/cc/test/consumer/CmdQueueConsumerTest.java
@@ -23,6 +23,7 @@
 import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
 import static com.github.tomakehurst.wiremock.client.WireMock.verify;
 
+import com.flow.platform.cc.config.QueueConfig;
 import com.flow.platform.cc.domain.CmdStatusItem;
 import com.flow.platform.cc.service.AgentService;
 import com.flow.platform.cc.service.CmdService;
@@ -47,6 +48,7 @@
 import org.junit.Test;
 import org.junit.runners.MethodSorters;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.dao.CannotAcquireLockException;
 
 /**
@@ -69,8 +71,13 @@ public class CmdQueueConsumerTest extends TestBase {
     @Autowired
     private CmdService cmdService;
 
+    @Value("${queue.cmd.retry.enable}")
+    private Boolean cmdQueueRetryEnable;
+
     @Before
     public void before() throws Throwable {
+        System.setProperty(QueueConfig.PROP_CMD_QUEUE_RETRY, "true");
+
         zoneService.createZone(new Zone(ZONE, "mock-cloud-provider"));
 
         // ensure zookeeper node is created for zone
@@ -205,5 +212,6 @@ public void should_stop_queued_cmd() throws Throwable {
     @After
     public void deleteZone() {
         deleteNodeWithChildren(ZKHelper.buildPath(ZONE, null));
+        System.setProperty(QueueConfig.PROP_CMD_QUEUE_RETRY, "false");
     }
 }
diff --git a/platform-control-center/src/test/java/com/flow/platform/cc/test/controller/CmdControllerTest.java b/platform-control-center/src/test/java/com/flow/platform/cc/test/controller/CmdControllerTest.java
@@ -84,7 +84,7 @@ public void should_list_cmd_types() throws Throwable {
         String raw = result.getResponse().getContentAsString();
         CmdType[] types = gsonConfig.fromJson(raw, CmdType[].class);
         Assert.assertNotNull(types);
-        Assert.assertTrue(types.length == 6);
+        Assert.assertTrue(types.length == 7);
     }
 
     @Test
diff --git a/platform-control-center/src/test/java/com/flow/platform/cc/test/service/CmdDispatchServiceTest.java b/platform-control-center/src/test/java/com/flow/platform/cc/test/service/CmdDispatchServiceTest.java
@@ -25,6 +25,8 @@
 import com.flow.platform.cc.service.CmdService;
 import com.flow.platform.cc.test.TestBase;
 import com.flow.platform.cc.util.ZKHelper;
+import com.flow.platform.core.exception.FlowException;
+import com.flow.platform.core.queue.PlatformQueue;
 import com.flow.platform.domain.Agent;
 import com.flow.platform.domain.AgentPath;
 import com.flow.platform.domain.AgentStatus;
@@ -41,6 +43,7 @@
 import org.junit.FixMethodOrder;
 import org.junit.Test;
 import org.junit.runners.MethodSorters;
+import org.springframework.amqp.core.Message;
 import org.springframework.beans.factory.annotation.Autowired;
 
 /**
@@ -61,6 +64,9 @@ public class CmdDispatchServiceTest extends TestBase {
     @Autowired
     private List<Zone> defaultZones;
 
+    @Autowired
+    private PlatformQueue<Message> cmdQueue;
+
     private AgentPath agentPath;
 
     private Agent target;
@@ -90,6 +96,33 @@ public void toCreateSession() throws Throwable {
         Assert.assertEquals(cmd.getSessionId(), target.getSessionId());
     }
 
+    @Test
+    public void should_broadcast_agent_resource_event_if_no_available_agent() throws Throwable {
+        // given: make no available agent resource
+        Assert.assertEquals(true, cmdQueue.isRunning());
+
+        // when: send create session cmd
+        Cmd cmdToCreateSession = cmdService.create(new CmdInfo(agentPath, CmdType.CREATE_SESSION, null));
+
+        try {
+            cmdDispatchService.dispatch(cmdToCreateSession.getId(), false);
+        } catch (FlowException ignore) {
+
+        }
+
+        // then: queue should be pause since no available
+        Assert.assertEquals(false, cmdQueue.isRunning());
+
+        // when: send delete session to release agent
+        CmdInfo cmd = new CmdInfo(agentPath, CmdType.DELETE_SESSION, null);
+        cmd.setSessionId(target.getSessionId());
+        Cmd cmdToDeleteSession = cmdService.create(cmd);
+        cmdDispatchService.dispatch(cmdToDeleteSession.getId(), false);
+
+        // then: queue should be resumed since agent resource released
+        Assert.assertEquals(true, cmdQueue.isRunning());
+    }
+
     @Test
     public void should_raise_exception_if_create_session_again() throws Throwable {
         Cmd cmdToFail = cmdService.create(new CmdInfo(agentPath, CmdType.CREATE_SESSION, null));
diff --git a/platform-control-center/src/test/resources/app-test.properties b/platform-control-center/src/test/resources/app-test.properties
@@ -35,6 +35,7 @@ mq.host = amqp://localhost:5672
 mq.management.host = http://localhost:15672
 
 #### cmd queue settings ###
+queue.cmd.retry.enable = false
 queue.cmd.rabbit.enable = false
 queue.cmd.rabbit.name = flow-cmd-queue-default
 queue.cmd.idle_agent.timeout = 0
diff --git a/platform-core/src/main/java/com/flow/platform/core/queue/InMemoryQueue.java b/platform-core/src/main/java/com/flow/platform/core/queue/InMemoryQueue.java
diff --git a/platform-core/src/main/java/com/flow/platform/core/queue/PlatformQueue.java b/platform-core/src/main/java/com/flow/platform/core/queue/PlatformQueue.java
diff --git a/platform-core/src/main/java/com/flow/platform/core/queue/RabbitQueue.java b/platform-core/src/main/java/com/flow/platform/core/queue/RabbitQueue.java
diff --git a/platform-core/src/main/java/com/flow/platform/core/service/WebhookServiceImplBase.java b/platform-core/src/main/java/com/flow/platform/core/service/WebhookServiceImplBase.java
diff --git a/platform-core/src/test/java/com/flow/platform/core/test/sysinfo/PlatformQueueTest.java b/platform-core/src/test/java/com/flow/platform/core/test/sysinfo/PlatformQueueTest.java
