Merge branch 'develop' of github.com:FlowCI/flow-platform into develop

yunheli · yunheli · commit ed02c9b8c898 · 2017-10-20T09:45:53.000+08:00
diff --git a/platform-api/src/main/java/com/flow/platform/api/controller/GitWebHookController.java b/platform-api/src/main/java/com/flow/platform/api/controller/GitWebHookController.java
@@ -16,6 +16,7 @@
 
 package com.flow.platform.api.controller;
 
+import com.flow.platform.api.config.AppConfig;
 import com.flow.platform.api.domain.envs.FlowEnvs;
 import com.flow.platform.api.domain.envs.FlowEnvs.YmlStatusValue;
 import com.flow.platform.api.domain.node.Flow;
@@ -65,11 +66,11 @@ public class GitWebHookController extends NodeController {
     @PostMapping(path = "/{root}")
     public void onEventReceived(@RequestHeader HttpHeaders headers, HttpServletRequest request) {
         final String path = currentNodePath.get();
-
         Map<String, String> headerAsMap = headers.toSingleValueMap();
+
         String body;
         try {
-            request.setCharacterEncoding("utf8");
+            request.setCharacterEncoding(AppConfig.DEFAULT_CHARSET.name());
             body = CharStreams.toString(request.getReader());
         } catch (IOException e) {
             throw new IllegalStatusException("Cannot read raw body");
diff --git a/platform-api/src/main/java/com/flow/platform/api/controller/JobController.java b/platform-api/src/main/java/com/flow/platform/api/controller/JobController.java
@@ -25,6 +25,7 @@
 import com.flow.platform.api.service.job.JobSearchService;
 import com.flow.platform.api.service.node.YmlService;
 import com.flow.platform.api.util.I18nUtil;
+import com.flow.platform.core.exception.NotFoundException;
 import com.flow.platform.util.Logger;
 import com.flow.platform.util.StringUtil;
 import com.flow.platform.util.git.model.GitEventType;
@@ -174,7 +175,12 @@ public Job show(@PathVariable Integer buildNumber) {
     @GetMapping(path = "/{root}/{buildNumber}/yml")
     public String yml(@PathVariable Integer buildNumber) {
         String path = currentNodePath.get();
-        return jobService.findYml(path, buildNumber);
+        try {
+            return jobService.findYml(path, buildNumber);
+        } catch (NotFoundException ignore) {
+            // ignore job node not found exception since maybe job node created when yml loading
+            return StringUtil.EMPTY;
+        }
     }
 
     /**
diff --git a/platform-api/src/main/java/com/flow/platform/api/domain/envs/AgentEnvs.java b/platform-api/src/main/java/com/flow/platform/api/domain/envs/AgentEnvs.java
@@ -21,7 +21,5 @@
  */
 public enum AgentEnvs implements EnvKey {
 
-    FLOW_AGENT_WORKSPACE,
-
-    FLOW_AGENT_OUTPUT_ENV_PREFIX
+    FLOW_AGENT_WORKSPACE
 }
diff --git a/platform-api/src/main/java/com/flow/platform/api/domain/envs/FlowEnvs.java b/platform-api/src/main/java/com/flow/platform/api/domain/envs/FlowEnvs.java
@@ -36,7 +36,13 @@ public enum FlowEnvs implements EnvKey {
     /**
      * For yml error message while loading yml from git
      */
-    FLOW_YML_ERROR_MSG;
+    FLOW_YML_ERROR_MSG,
+
+
+    /**
+     * Defined env variable output prefix
+     */
+    FLOW_ENV_OUTPUT_PREFIX;
 
     public enum StatusValue implements EnvValue {
 
diff --git a/platform-api/src/main/java/com/flow/platform/api/service/job/CmdServiceImpl.java b/platform-api/src/main/java/com/flow/platform/api/service/job/CmdServiceImpl.java
@@ -17,6 +17,7 @@
 package com.flow.platform.api.service.job;
 
 import com.flow.platform.api.domain.envs.AgentEnvs;
+import com.flow.platform.api.domain.envs.FlowEnvs;
 import com.flow.platform.api.domain.job.Job;
 import com.flow.platform.api.domain.node.Node;
 import com.flow.platform.api.util.PlatformURL;
@@ -102,7 +103,7 @@ public CmdInfo runShell(Job job, Node node, String cmdId) {
         CmdInfo cmdInfo = new CmdInfo(zone, null, CmdType.RUN_SHELL, node.getScript());
         cmdInfo.setInputs(node.getEnvs());
         cmdInfo.setWebhook(buildCmdWebhook(job));
-        cmdInfo.setOutputEnvFilter(job.getEnv(AgentEnvs.FLOW_AGENT_OUTPUT_ENV_PREFIX, "FLOW_OUTPUT"));
+        cmdInfo.setOutputEnvFilter(job.getEnv(FlowEnvs.FLOW_ENV_OUTPUT_PREFIX, "FLOW_OUTPUT"));
         cmdInfo.setSessionId(job.getSessionId());
         cmdInfo.setExtra(node.getPath()); // use cmd.extra to keep node path info
         cmdInfo.setCustomizedId(cmdId);
diff --git a/platform-api/src/main/java/com/flow/platform/api/service/job/JobServiceImpl.java b/platform-api/src/main/java/com/flow/platform/api/service/job/JobServiceImpl.java
@@ -296,6 +296,10 @@ private void run(Node node, Job job) {
         // pass job env to node
         EnvUtil.merge(job.getEnvs(), node.getEnvs(), false);
 
+        // pass root node output to current node
+        NodeResult rootResult = nodeResultService.find(tree.root().getPath(), job.getId());
+        EnvUtil.merge(rootResult.getOutputs(), node.getEnvs(), false);
+
         // to run node with customized cmd id
         try {
             NodeResult nodeResult = nodeResultService.find(node.getPath(), job.getId());
@@ -450,6 +454,10 @@ public void createJobNodesAndCreateSession(Job job, String yml) {
         //create yml snapshot for job
         jobNodeService.save(job, yml);
 
+        // set root node env from yml to job env
+        Node root = jobNodeService.get(job).root();
+        EnvUtil.merge(root.getEnvs(), job.getEnvs(), true);
+
         // init for node result and set to job object
         List<NodeResult> resultList = nodeResultService.create(job);
         NodeResult rootResult = resultList.remove(resultList.size() - 1);
diff --git a/platform-api/src/main/java/com/flow/platform/api/service/node/YmlServiceImpl.java b/platform-api/src/main/java/com/flow/platform/api/service/node/YmlServiceImpl.java
@@ -49,6 +49,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.task.TaskRejectedException;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.stereotype.Service;
 
@@ -163,7 +164,7 @@ public Node loadYmlContent(final Node root, final Consumer<Yml> onSuccess, final
 
             // async to load yml file
             executor.execute(new UpdateNodeYmlTask(root, nodeService, gitService, onSuccess, onError));
-        } catch (ExecutionException e) {
+        } catch (ExecutionException | TaskRejectedException e) {
             LOGGER.warn("Fail to get task executor for node: " + root.getPath());
             nodeService.updateYmlState(root, YmlStatusValue.ERROR, e.getMessage());
         }
diff --git a/platform-api/src/main/java/com/flow/platform/api/service/user/UserServiceImpl.java b/platform-api/src/main/java/com/flow/platform/api/service/user/UserServiceImpl.java
@@ -173,6 +173,9 @@ public User register(User user, List<String> roles, boolean isSendEmail, List<St
 
     @Override
     public void delete(List<String> emailList) {
+        if (emailList.contains(currentUser().getEmail())){
+            throw new IllegalParameterException("params emails include yourself email, not delete");
+        }
         // un-assign user from role and flow
         List<User> users = userDao.list(emailList);
         for (User user : users) {
diff --git a/platform-api/src/test/java/com/flow/platform/api/test/controller/CmdWebhookControllerTest.java b/platform-api/src/test/java/com/flow/platform/api/test/controller/CmdWebhookControllerTest.java
@@ -20,21 +20,21 @@
 import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
-import com.flow.platform.api.domain.job.JobStatus;
-import com.flow.platform.api.domain.node.Flow;
 import com.flow.platform.api.domain.job.Job;
-import com.flow.platform.api.domain.node.Node;
+import com.flow.platform.api.domain.job.JobStatus;
 import com.flow.platform.api.domain.job.NodeResult;
 import com.flow.platform.api.domain.job.NodeStatus;
+import com.flow.platform.api.domain.node.Flow;
+import com.flow.platform.api.domain.node.Node;
 import com.flow.platform.api.domain.node.Step;
 import com.flow.platform.api.test.TestBase;
+import com.flow.platform.api.util.EnvUtil;
 import com.flow.platform.domain.Cmd;
 import com.flow.platform.domain.CmdResult;
 import com.flow.platform.domain.CmdStatus;
 import com.flow.platform.domain.CmdType;
 import com.flow.platform.util.git.model.GitEventType;
 import com.flow.platform.util.http.HttpURL;
-import java.util.List;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -103,6 +103,7 @@ public void should_callback_session_success() throws Throwable {
 
         CmdResult cmdResult = new CmdResult(0);
         cmdResult.setDuration(100L);
+        cmdResult.setOutput(EnvUtil.build("OUTPUT_ENV", "hello"));
         cmd.setCmdResult(cmdResult);
 
         performMockHttpRequest(cmd, job);
@@ -112,9 +113,18 @@ public void should_callback_session_success() throws Throwable {
         job = reload(job);
         Assert.assertEquals(JobStatus.RUNNING, job.getStatus());
 
+        // check step 1 node result
         resultForStep1 = nodeResultService.find(step1.getPath(), job.getId());
         Assert.assertEquals(NodeStatus.SUCCESS, resultForStep1.getStatus());
         Assert.assertEquals(0, resultForStep1.getExitCode().intValue());
+        Assert.assertEquals(1, resultForStep1.getOutputs().size());
+        Assert.assertEquals("hello", resultForStep1.getOutputs().get("OUTPUT_ENV"));
+
+        // check root node result
+        NodeResult rootNodeResult = nodeResultService.find(job.getNodePath(), job.getId());
+        Assert.assertEquals(2, rootNodeResult.getOutputs().size());
+        Assert.assertNotNull(rootNodeResult.getOutputs().get("FLOW_JOB_LOG_PATH"));
+        Assert.assertEquals("hello", rootNodeResult.getOutputs().get("OUTPUT_ENV"));
 
         resultForRoot = nodeResultService.find(job.getNodePath(), job.getId());
         Assert.assertEquals(NodeStatus.RUNNING, resultForRoot.getStatus());
diff --git a/platform-api/src/test/java/com/flow/platform/api/test/service/JobServiceTest.java b/platform-api/src/test/java/com/flow/platform/api/test/service/JobServiceTest.java
@@ -93,6 +93,8 @@ public void should_job_failure_since_cannot_create_session() throws Throwable {
     public void should_create_node_success() throws IOException {
         Node rootForFlow = createRootFlow("flow1", "demo_flow2.yaml");
         Job job = createMockJob(rootForFlow.getPath());
+        Assert.assertNotNull(job.getEnv("FLOW_WORKSPACE"));
+        Assert.assertNotNull(job.getEnv("FLOW_VERSION"));
 
         Step step1 = (Step) nodeService.find("flow1/step1");
         Step step2 = (Step) nodeService.find("flow1/step2");
@@ -229,6 +231,8 @@ public void should_stop_success() throws IOException {
     public void should_job_time_out_and_reject_callback() throws IOException, InterruptedException {
         Node rootForFlow = createRootFlow("flow1", "demo_flow2.yaml");
         Job job = jobService.createJob(rootForFlow.getPath(), GitEventType.TAG, null, mockUser);
+        Assert.assertNotNull(job.getEnv("FLOW_WORKSPACE"));
+        Assert.assertNotNull(job.getEnv("FLOW_VERSION"));
 
         build_relation(rootForFlow, job);
 
diff --git a/platform-cmd-runner/src/main/java/com/flow/platform/cmd/CmdExecutor.java b/platform-cmd-runner/src/main/java/com/flow/platform/cmd/CmdExecutor.java
@@ -18,12 +18,14 @@
 
 import com.flow.platform.domain.CmdResult;
 import com.flow.platform.util.DateUtil;
+import com.flow.platform.util.SystemUtil;
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 
 import java.io.*;
 import java.lang.reflect.Field;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.ZonedDateTime;
 import java.util.*;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -153,17 +155,17 @@ public CmdExecutor(final ProcListener procListener,
 
         // check and init working dir
         if (workingDir != null) {
-            File dir = new File(workingDir);
+            Path dir = SystemUtil.replacePathWithEnv(workingDir);
 
-            if (!dir.exists()) {
+            if (!Files.exists(dir)) {
                 try {
-                    Files.createDirectories(dir.toPath());
+                    Files.createDirectories(dir);
                 } catch (IOException e) {
                     throw new IllegalArgumentException("Unable to create working dir: " + dir);
                 }
             }
 
-            this.pBuilder.directory(dir);
+            this.pBuilder.directory(dir.toFile());
         }
 
         // init inputs env
diff --git a/platform-util/src/main/java/com/flow/platform/util/SystemUtil.java b/platform-util/src/main/java/com/flow/platform/util/SystemUtil.java
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.flow.platform.api.util;
+package com.flow.platform.util;
 
 import java.io.File;
 import java.nio.file.Path;
@@ -26,6 +26,10 @@
  */
 public class SystemUtil {
 
+    private final static char ENV_VAR_START_CHAR = '$';
+    private final static char ENV_VAR_LEFT_BRACKET = '{';
+    private final static char ENV_VAR_RIGHT_BRACKET = '}';
+
     /**
      * Parse path with ${xxx} variable to absolute path
      *
@@ -38,19 +42,38 @@ public static Path replacePathWithEnv(String pathWithEnv) {
 
         for (String pathItem : paths) {
             int index = pathItem.indexOf("$", 0);
-            int leftBracket = pathItem.indexOf("{", 0);
-            int rightBracket = pathItem.indexOf("}", 0);
 
-            if (index < 0 || leftBracket < 0 || rightBracket < 0) {
+            if (index < 0) {
                 path = Paths.get(path.toString(), pathItem);
                 continue;
             }
 
-            String envName = pathItem.substring(leftBracket + 1, rightBracket);
-            String envValue = System.getenv(envName);
-            path = Paths.get(path.toString(), envValue);
+            path = Paths.get(path.toString(), parseEnv(pathItem));
         }
 
         return path;
     }
+
+    /**
+     * Parse ${xx} variable to exact value
+     */
+    public static String parseEnv(String env) {
+        if (env == null) {
+            throw new IllegalArgumentException();
+        }
+
+        if (env.charAt(0) != ENV_VAR_START_CHAR) {
+            throw new IllegalArgumentException();
+        }
+
+        boolean hasBracket = env.charAt(1) == ENV_VAR_LEFT_BRACKET;
+        env = env.substring(1);
+
+        if (!hasBracket) {
+            return System.getenv(env);
+        }
+
+        env = env.substring(1, env.length() - 1);
+        return System.getenv(env);
+    }
 }
diff --git a/platform-util/src/test/java/com/flow/platform/util/test/SystemUtilTest.java b/platform-util/src/test/java/com/flow/platform/util/test/SystemUtilTest.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package com.flow.platform.api.test.util;
+package com.flow.platform.util.test;
 
-import com.flow.platform.api.util.SystemUtil;
+import com.flow.platform.util.SystemUtil;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import org.junit.Assert;
@@ -27,6 +27,18 @@
  */
 public class SystemUtilTest {
 
+    @Test
+    public void should_parse_linux_env() throws Throwable {
+        String home = SystemUtil.parseEnv("${HOME}");
+        Assert.assertEquals(System.getenv("HOME"), home);
+
+        home = SystemUtil.parseEnv("$HOME");
+        Assert.assertEquals(System.getenv("HOME"), home);
+
+        home = SystemUtil.parseEnv("${HOME");
+        Assert.assertNull(home);
+    }
+
     @Test
     public void should_replace_path_with_env() throws Throwable {
         // single placeholder

Original file line number	Diff line number	Diff line change
`@@ -21,7 +21,5 @@`
`21`	`21`	`*/`
`22`	`22`	`public enum AgentEnvs implements EnvKey {`
`23`	`23`
`24`		`- FLOW_AGENT_WORKSPACE,`
`25`		`-`
`26`		`- FLOW_AGENT_OUTPUT_ENV_PREFIX`
	`24`	`+ FLOW_AGENT_WORKSPACE`
`27`	`25`	`}`