Merge pull request #203 from FlowCI/develop

Develop

Author: yang.guo

config/app-api.properties

@@ -14,6 +14,8 @@ hibernate.hbm2ddl.auto = validate
 
 ### api settings ###
 api.workspace = ${HOME}/flow-api-data
+api.git.cache = ${HOME}/flow-api-data/git-cache
+api.git.workspace = ${HOME}/flow-api-data/git-repos
 api.zone.default = default
 
 ### expiration duration of token, it's in millisecond ###
@@ -29,6 +31,9 @@ system.email = [email protected]
 system.username = admin
 system.password = 123456
 
+## plugin repos url
+plugins.repository = https://raw.githubusercontent.com/flowci/plugins/master/repository.json
+
 task.job.toggle.execution_timeout = true
 ## expired in 1800 seconds for create session
 task.job.toggle.execution_create_session_duration = 1800

config/app-cc.properties

@@ -12,6 +12,9 @@ jdbc.connection.timeout = 20000
 hibernate.show_sql = false
 hibernate.hbm2ddl.auto = validate
 
+### control center configuration
+cc.workspace = ${HOME}/flow-api/cc
+
 ### zookeeper config ###
 zk.server.embedded = true
 zk.host = 127.0.0.1:2181

platform-agent/src/main/java/com/flow/platform/agent/LogEventHandler.java

@@ -205,7 +205,7 @@ private void writeZipStream(final ZipOutputStream stream, final String log) {
         // write to zip output stream
         try {
             stream.write(log.getBytes());
-            stream.write("\n".getBytes());
+            stream.write(Cmd.NEW_LINE.getBytes());
         } catch (IOException e) {
             LOGGER.warn("Log cannot write : " + log);
         }

platform-agent/src/test/java/com/flow/platform/agent/test/CmdManagerTest.java

@@ -23,6 +23,7 @@
 import com.flow.platform.domain.CmdResult;
 import com.flow.platform.domain.CmdStatus;
 import com.flow.platform.domain.CmdType;
+import com.google.common.collect.Lists;
 import java.io.IOException;
 import java.util.Map;
 import java.util.UUID;
@@ -119,19 +120,19 @@ public void onException(CmdResult result) {
         String content = String.format("source %s", resourcePath);
 
         Cmd cmd1 = new Cmd("zone1", "agent1", CmdType.RUN_SHELL, content);
-        cmd1.setOutputEnvFilter("FLOW_AGENT");
+        cmd1.setOutputEnvFilter(Lists.newArrayList("FLOW_AGENT"));
         cmd1.setId(UUID.randomUUID().toString());
 
         Cmd cmd2 = new Cmd("zone1", "agent1", CmdType.RUN_SHELL, content);
-        cmd2.setOutputEnvFilter("FLOW_AGENT");
+        cmd2.setOutputEnvFilter(Lists.newArrayList("FLOW_AGENT"));
         cmd2.setId(UUID.randomUUID().toString());
 
         Cmd cmd3 = new Cmd("zone1", "agent1", CmdType.RUN_SHELL, content);
-        cmd3.setOutputEnvFilter("FLOW_AGENT");
+        cmd3.setOutputEnvFilter(Lists.newArrayList("FLOW_AGENT"));
         cmd3.setId(UUID.randomUUID().toString());
 
         Cmd cmd4 = new Cmd("zone1", "agent1", CmdType.RUN_SHELL, content);
-        cmd4.setOutputEnvFilter("FLOW_AGENT");
+        cmd4.setOutputEnvFilter(Lists.newArrayList("FLOW_AGENT"));
         cmd4.setId(UUID.randomUUID().toString());
 
         // when: execute four command by thread

platform-api/pom.xml

@@ -107,6 +107,12 @@
       <version>${project.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>com.flow.platform</groupId>
+      <artifactId>platform-plugin</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+
     <dependency>
       <groupId>com.flow.platform</groupId>
       <artifactId>platform-util-resource</artifactId>
@@ -159,6 +165,11 @@
       <artifactId>quartz-jobs</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>org.eclipse.jgit</groupId>
+      <artifactId>org.eclipse.jgit.http.server</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
@@ -182,5 +193,6 @@
       <artifactId>jjwt</artifactId>
       <version>0.7.0</version>
     </dependency>
+
   </dependencies>
 </project>

platform-api/src/main/java/com/flow/platform/api/AppInit.java

@@ -21,18 +21,20 @@
 import com.flow.platform.api.resource.PropertyResourceLoader;
 import com.flow.platform.util.resource.AppResourceLoader;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import javax.servlet.ServletContext;
 import javax.servlet.ServletException;
-import javax.servlet.ServletRegistration;
+import javax.servlet.ServletRegistration.Dynamic;
+import org.eclipse.jgit.http.server.GitServlet;
 import org.springframework.core.io.support.ResourcePropertySource;
 import org.springframework.web.WebApplicationInitializer;
 import org.springframework.web.context.support.AnnotationConfigWebApplicationContext;
 import org.springframework.web.servlet.DispatcherServlet;
 
 /**
  * Created by gyfirim on 14/07/2017.
- *
- * @Copyright fir.im
  */
 public class AppInit implements WebApplicationInitializer {
 
@@ -43,9 +45,9 @@ public void onStartup(ServletContext servletContext) throws ServletException {
         applicationContext.register(WebConfig.class, WebSocketConfig.class);
         applicationContext.setServletContext(servletContext);
 
-        // Add the servlet mapping manually and make it initialize automatically
+        // Add the mvcServlet mapping manually and make it initialize automatically
         DispatcherServlet dispatcherServlet = new DispatcherServlet(applicationContext);
-        ServletRegistration.Dynamic servlet = servletContext.addServlet("mvc-dispatcher", dispatcherServlet);
+        Dynamic mvcServlet = servletContext.addServlet("mvc-dispatcher", dispatcherServlet);
 
         // set app property resource
         try {
@@ -55,11 +57,32 @@ public void onStartup(ServletContext servletContext) throws ServletException {
                 .getPropertySources()
                 .addFirst(new ResourcePropertySource(propertyLoader.find()));
         } catch (IOException e) {
-            throw new RuntimeException(e);
+            throw new ServletException(e.getMessage());
+        }
+
+        mvcServlet.addMapping("/");
+        mvcServlet.setAsyncSupported(true);
+        mvcServlet.setLoadOnStartup(2);
+
+        try {
+            Path gitWorkspace = Paths.get(applicationContext.getEnvironment().getProperty("api.git.workspace"));
+            initGitServlet(gitWorkspace, servletContext);
+        } catch (IOException e) {
+            throw new ServletException(e.getMessage());
+        }
+    }
+
+    private void initGitServlet(Path gitWorkspace, ServletContext servletContext) throws IOException {
+        if (!Files.exists(gitWorkspace)) {
+            Files.createDirectories(gitWorkspace);
         }
 
-        servlet.addMapping("/");
-        servlet.setAsyncSupported(true);
-        servlet.setLoadOnStartup(1);
+        // add git servlet mapping
+        Dynamic gitServlet = servletContext.addServlet("git-servlet", new GitServlet());
+        gitServlet.addMapping("/git/*");
+        gitServlet.setInitParameter("base-path", gitWorkspace.toString());
+        gitServlet.setInitParameter("export-all", "true");
+        gitServlet.setAsyncSupported(true);
+        gitServlet.setLoadOnStartup(1);
     }
 }

platform-api/src/main/java/com/flow/platform/api/config/AppConfig.java

@@ -16,15 +16,12 @@
 
 package com.flow.platform.api.config;
 
-import com.flow.platform.api.domain.CmdCallbackQueueItem;
 import com.flow.platform.api.domain.user.User;
 import com.flow.platform.api.util.PlatformURL;
 import com.flow.platform.core.config.AppConfigBase;
 import com.flow.platform.core.config.DatabaseConfig;
-import com.flow.platform.core.queue.InMemoryQueue;
-import com.flow.platform.core.queue.PlatformQueue;
-import com.flow.platform.core.queue.PriorityMessage;
 import com.flow.platform.core.util.ThreadUtil;
+import com.flow.platform.plugin.PluginConfig;
 import com.flow.platform.util.Logger;
 import java.io.IOException;
 import java.nio.charset.Charset;
@@ -47,7 +44,7 @@
  * @author yang
  */
 @Configuration
-@Import({SchedulerConfig.class, CachingConfig.class, DatabaseConfig.class})
+@Import({SchedulerConfig.class, CachingConfig.class, DatabaseConfig.class, QueueConfig.class, PluginConfig.class})
 public class AppConfig extends AppConfigBase {
 
     public final static String NAME = "API";
@@ -78,6 +75,12 @@ public class AppConfig extends AppConfigBase {
     @Value("${api.workspace}")
     private String workspace;
 
+    @Value("${api.git.cache}")
+    private String gitCloneCache;
+
+    @Value("${api.git.workspace}")
+    private String gitWorkspace;
+
     @Value("${domain.cc}")
     private String ccDomain;
 
@@ -98,14 +101,17 @@ public User superUser() {
     @Bean
     public Path workspace() {
         try {
-            Path dir = Files.createDirectories(Paths.get(workspace));
-            LOGGER.trace("flow.ci working dir been created : %s", dir);
-            return dir;
+            return Files.createDirectories(Paths.get(workspace));
         } catch (IOException e) {
             throw new RuntimeException("Fail to create flow.ci api working dir", e);
         }
     }
 
+    @Bean
+    public Path gitWorkspace() {
+        return Paths.get(gitWorkspace);
+    }
+
     @Bean(name = "applicationEventMulticaster")
     public ApplicationEventMulticaster simpleApplicationEventMulticaster() {
         multicasterExecutor.initialize();
@@ -130,14 +136,6 @@ public ThreadPoolTaskExecutor taskExecutor() {
         return executor;
     }
 
-    /**
-     * Queue to process cmd callback task
-     */
-    @Bean
-    public PlatformQueue<PriorityMessage> cmdCallbackQueue() {
-        return new InMemoryQueue<>(executor, 50, "CmdCallbackQueue");
-    }
-
     @Override
     protected String getName() {
         return NAME;

platform-api/src/main/java/com/flow/platform/api/config/QueueConfig.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2017 flow.ci
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.flow.platform.api.config;
+
+import com.flow.platform.api.service.SyncService;
+import com.flow.platform.core.queue.MemoryQueue;
+import com.flow.platform.core.queue.PriorityMessage;
+import com.flow.platform.queue.PlatformQueue;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Configurable;
+import org.springframework.context.annotation.Bean;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+
+/**
+ * @author yang
+ */
+@Configurable
+public class QueueConfig {
+
+    @Autowired
+    private ThreadPoolTaskExecutor taskExecutor;
+
+    /**
+     * Queue to process cmd callback task
+     */
+    @Bean
+    public PlatformQueue<PriorityMessage> cmdCallbackQueue() {
+        return new MemoryQueue(taskExecutor, 50, "CmdCallbackQueue");
+    }
+
+    @Bean
+    public SyncService.QueueCreator syncQueueCreator() {
+        return name -> new MemoryQueue(taskExecutor, 50, name);
+    }
+}

platform-api/src/main/java/com/flow/platform/api/config/WebConfig.java

@@ -42,10 +42,8 @@
 import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
 import org.springframework.http.MediaType;
 import org.springframework.http.converter.HttpMessageConverter;
-import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.security.web.util.matcher.AntPathRequestMatcher;
 import org.springframework.security.web.util.matcher.RequestMatcher;
-import org.springframework.web.context.request.WebRequestInterceptor;
 import org.springframework.web.multipart.MultipartResolver;
 import org.springframework.web.multipart.commons.CommonsMultipartResolver;
 import org.springframework.web.servlet.config.annotation.CorsRegistry;
@@ -65,6 +63,8 @@
     "com.flow.platform.api.context",
     "com.flow.platform.api.util",
     "com.flow.platform.api.consumer",
+    "com.flow.platform.plugin.service",
+    "com.flow.platform.plugin.consumer",
     "com.flow.platform.api.initializers"})
 @Import({AppConfig.class})
 public class WebConfig extends WebMvcConfigurerAdapter {

platform-api/src/main/java/com/flow/platform/api/consumer/CmdCallbackQueueConsumer.java

@@ -20,10 +20,10 @@
 import com.flow.platform.api.service.job.JobService;
 import com.flow.platform.core.exception.FlowException;
 import com.flow.platform.core.exception.NotFoundException;
-import com.flow.platform.core.queue.PlatformQueue;
 import com.flow.platform.core.queue.PriorityMessage;
-import com.flow.platform.core.queue.QueueListener;
 import com.flow.platform.core.util.ThreadUtil;
+import com.flow.platform.queue.PlatformQueue;
+import com.flow.platform.queue.QueueListener;
 import com.flow.platform.util.Logger;
 import javax.annotation.PostConstruct;
 import org.springframework.beans.factory.annotation.Autowired;