Merge branch 'develop' of github.com:FlowCI/flow-platform into feature/api/branches_cache

Author: yunheli

.gitignore

@@ -1,6 +1,7 @@
 .idea/
 *.iml
 .DS_Store
+deploy.sh
 
 node_modules/
 apidoc/

docker/app-api.properties

@@ -18,6 +18,8 @@ api.workspace = ${HOME}/flow-api-data
 ### domain ###
 domain = http://localhost:8080
 
+domain.web = http://localhost:3000
+
 ### platform setting ###
 platform.url = http://localhost:8080/
 platform.zone = default

docker/app-cc.properties

@@ -25,10 +25,11 @@ zone.default.default_cmd_timeout = 600
 
 ### rabbitmq config ###
 mq.host = amqp://localhost:5672
-mq.queue.cmd.name = flow-cmd-queue-default
 mq.management.host = http://localhost:15672
 
 #### cmd queue settings ###
+queue.cmd.rabbit.enable = false
+queue.cmd.rabbit.name = flow-cmd-queue-default
 queue.cmd.idle_agent.timeout = 30
 queue.cmd.idle_agent.period = 5
 

platform-agent/src/main/java/com/flow/platform/agent/AgentManager.java

@@ -21,7 +21,6 @@
 import com.flow.platform.util.Logger;
 import com.flow.platform.util.zk.*;
 
-import java.io.Closeable;
 import java.io.IOException;
 import java.util.LinkedList;
 import java.util.List;
@@ -41,7 +40,9 @@ public class AgentManager implements Runnable, TreeCacheListener, AutoCloseable
 
     // Zk root path /flow-agents/{zone}/{name}
     private final static Object STATUS_LOCKER = new Object();
-    private final static int ZK_RECONNECT_TIME = 5;
+
+    private final static int ZK_RECONNECT_TIME = 1;
+    private final static int ZK_RETRY_PERIOD = 500;
 
     private String zkHost;
     private int zkTimeout;
@@ -59,7 +60,7 @@ public AgentManager(String zkHost, int zkTimeout, String zone, String name) thro
         this.zkHost = zkHost;
         this.zkTimeout = zkTimeout;
 
-        this.zkClient = new ZKClient(zkHost);
+        this.zkClient = new ZKClient(zkHost, ZK_RETRY_PERIOD, ZK_RECONNECT_TIME);
         this.zone = zone;
         this.name = name;
         this.zonePath = ZKPaths.makePath(Config.ZK_ROOT, this.zone);
@@ -97,7 +98,7 @@ public void run() {
             try {
                 STATUS_LOCKER.wait();
             } catch (InterruptedException e) {
-                e.printStackTrace();
+                LOGGER.warn("InterrupatdException : " + e.getMessage());
             }
         }
     }
@@ -106,13 +107,18 @@ public void run() {
     public void childEvent(CuratorFramework client, TreeCacheEvent event) throws Exception {
         ChildData eventData = event.getData();
 
+        if (event.getType() == Type.CONNECTION_LOST) {
+            LOGGER.traceMarker("ZK-Event", "========= Connection lost from zk server =========");
+            return;
+        }
+
         if (event.getType() == Type.INITIALIZED) {
-            LOGGER.trace("========= Connected to zookeeper server =========");
+            LOGGER.traceMarker("ZK-Event", "========= Connected to zk server =========");
             return;
         }
 
         if (event.getType() == Type.NODE_ADDED) {
-            LOGGER.trace("========= Node been created: %s =========", eventData.getPath());
+            LOGGER.traceMarker("ZK-Event", "========= Node been created: %s =========", eventData.getPath());
             return;
         }
 
@@ -176,6 +182,6 @@ private String registerZkNodeAndWatch() {
     }
 
     private void removeZkNode() {
-        zkClient.delete(nodePath, false);
+        zkClient.deleteWithoutGuaranteed(nodePath, false);
     }
 }

platform-agent/src/main/java/com/flow/platform/agent/App.java

@@ -43,23 +43,26 @@ public static void main(String args[]) {
             token = args[1];
         }
 
-        LOGGER.trace("========= Run agent =========");
+        LOGGER.trace("========= Flow Agent Started =========");
+        LOGGER.trace("=== Server: " + baseUrl);
+        LOGGER.trace("=== Token:  " + token);
+
         Runtime.getRuntime().addShutdownHook(new ShutdownHook());
 
         try {
-            LOGGER.trace("========= Init config =========");
+            LOGGER.trace("=== Start to load configuration");
 
             Config.AGENT_SETTINGS = Config.loadAgentConfig(baseUrl, token);
-            LOGGER.trace(" -- Settings: %s", Config.agentSettings());
+            LOGGER.trace("====== Settings: %s", Config.agentSettings());
 
             Config.ZK_URL = Config.AGENT_SETTINGS.getZookeeperUrl();
-            LOGGER.trace(" -- Zookeeper host: %s", Config.zkUrl());
+            LOGGER.trace("====== Zookeeper host: %s", Config.zkUrl());
 
             Config.ZONE = Config.AGENT_SETTINGS.getAgentPath().getZone();
-            LOGGER.trace(" -- Working zone: %s", Config.zone());
+            LOGGER.trace("====== Working zone: %s", Config.zone());
 
             Config.NAME = Config.AGENT_SETTINGS.getAgentPath().getName();
-            LOGGER.trace(" -- Agent agent: %s", Config.name());
+            LOGGER.trace("====== Agent agent: %s", Config.name());
 
             LOGGER.trace("========= Config initialized =========");
         } catch (Throwable e) {

platform-agent/src/main/java/com/flow/platform/agent/CmdManager.java

@@ -18,7 +18,6 @@
 
 import com.flow.platform.cmd.CmdExecutor;
 import com.flow.platform.cmd.Log;
-import com.flow.platform.cmd.Log.Category;
 import com.flow.platform.cmd.Log.Type;
 import com.flow.platform.cmd.ProcListener;
 import com.flow.platform.domain.Cmd;
@@ -204,10 +203,10 @@ public void run() {
         if (cmd.getType() == CmdType.SYSTEM_INFO) {
             LogEventHandler logListener = new LogEventHandler(cmd);
             Log log = new Log(Type.STDERR, collectionAgentInfo());
-            log.setCategory(Category.SYSTEM_INFO);
             logListener.onLog(log);
             logListener.onFinish();
             cmd.setStatus(CmdStatus.EXECUTED);
+            return;
         }
 
         // kill current running proc

platform-agent/src/main/java/com/flow/platform/agent/Config.java

@@ -18,6 +18,8 @@
 
 import com.flow.platform.domain.AgentSettings;
 import com.flow.platform.domain.Jsonable;
+import com.flow.platform.util.ExceptionUtil;
+import com.flow.platform.util.Logger;
 import com.flow.platform.util.http.HttpClient;
 import com.flow.platform.util.http.HttpResponse;
 import com.flow.platform.util.zk.ZKClient;
@@ -34,6 +36,8 @@
  */
 public class Config {
 
+    private final static Logger LOGGER = new Logger(Config.class);
+
     public final static String ZK_ROOT = "flow-agents";
 
     /* Config properties by using -Dxxx.xxx = xxx as JVM parameter */
@@ -79,6 +83,8 @@ public static String getProperty(String name) {
                 properties = new Properties();
                 properties.load(fileInputStream);
             } catch (Throwable e) {
+                LOGGER.warn("get property from application.properties error %s",
+                    ExceptionUtil.findRootCause(e).getMessage());
             }
         }
 

platform-agent/src/main/java/com/flow/platform/agent/LogEventHandler.java

@@ -122,7 +122,7 @@ public void onFinish() {
 
     public String websocketLogFormat(Log log) {
         return String
-            .format("%s#%s#%s#%s#%s#%s", log.getCategory(), log.getNumber(), cmd.getZoneName(), cmd.getAgentName(),
+            .format("%s#%s#%s#%s#%s#%s", cmd.getType(), log.getNumber(), cmd.getZoneName(), cmd.getAgentName(),
                 cmd.getId(),
                 log.getContent());
     }

platform-agent/src/main/java/com/flow/platform/agent/ReportManager.java

@@ -115,7 +115,7 @@ public boolean cmdLogUploadSync(final String cmdId, final Path path) {
             .retry(5)
             .bodyAsString();
 
-        if (response.hasSuccess()) {
+        if (!response.hasSuccess()) {
             LOGGER.warn("Fail to upload zipped cmd log to : %s ", url);
             return false;
         }

platform-agent/src/test/java/com/flow/platform/agent/test/LogEventHandlerTest.java

@@ -20,7 +20,6 @@
 import com.flow.platform.agent.Config;
 import com.flow.platform.agent.LogEventHandler;
 import com.flow.platform.cmd.Log;
-import com.flow.platform.cmd.Log.Category;
 import com.flow.platform.domain.Cmd;
 import com.flow.platform.domain.CmdType;
 import com.github.tomakehurst.wiremock.junit.WireMockRule;
@@ -59,7 +58,7 @@ public void should_get_correct_format_websocket() throws Throwable {
 
         // then:
         String expect = String
-            .format("%s#%s#%s#%s#%s#%s", Category.DEFAULT, null, cmd.getZoneName(), cmd.getAgentName(), cmd.getId(),
+            .format("%s#%s#%s#%s#%s#%s", CmdType.RUN_SHELL, null, cmd.getZoneName(), cmd.getAgentName(), cmd.getId(),
                 mockLogContent);
         Assert.assertEquals(expect, socketIoData);
     }