[CYB-193][UI] Landing page for pipelines. (#81)

Author: vpavlenko-cv

flink-cyber/cyber-jobs/pom.xml

@@ -91,6 +91,12 @@
             <version>${project.parent.version}</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>com.cloudera.cyber</groupId>
+            <artifactId>cyber-worker-service</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>provided</scope>
+        </dependency>
     </dependencies>
 
     <build>

flink-cyber/cyber-jobs/src/main/assemblies/cloudera.xml

@@ -87,6 +87,13 @@
 			<fileMode>0644</fileMode>
 		</file>
 
+		<file>
+			<source>../cyber-services/cyber-worker-service/target/cyber-worker-service-${project.version}.jar</source>
+			<outputDirectory>jobs/</outputDirectory>
+			<destName>cyber-worker-service-${cybersec.full.version}.jar</destName>
+			<fileMode>0644</fileMode>
+		</file>
+
 		<file>
 			<source>../flink-commands/scoring-commands/target/scoring-commands-${project.version}.jar</source>
 			<outputDirectory>tools/</outputDirectory>

flink-cyber/cyber-services/cyber-service-common/pom.xml

@@ -48,5 +48,10 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-autoconfigure</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-core</artifactId>
+            <scope>provided</scope>
+        </dependency>
     </dependencies>
 </project>

flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestBody.java

@@ -5,14 +5,18 @@
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
+import java.util.List;
+
 @Data
 @Builder
 @NoArgsConstructor
 @AllArgsConstructor
 public class RequestBody {
     private String clusterServiceId;
     private String jobIdHex;
-    private String pipelineDir;
+    private String pipelineName;
     private String branch;
+    private String profileName;
+    private List<String> jobs;
     private byte[] payload;
 }

flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/request/RequestType.java

@@ -1,5 +1,5 @@
 package com.cloudera.service.common.request;
 
 public enum RequestType {
-    GET_ALL_CLUSTERS_SERVICE_REQUEST, GET_CLUSTER_SERVICE_REQUEST, START_JOB_REQUEST, RESTART_JOB_REQUEST, STOP_JOB_REQUEST, GET_JOB_CONFIG_REQUEST, UPDATE_JOB_CONFIG_REQUEST
+    GET_ALL_CLUSTERS_SERVICE_REQUEST, GET_CLUSTER_SERVICE_REQUEST, START_JOB_REQUEST, RESTART_JOB_REQUEST, STOP_JOB_REQUEST, GET_JOB_CONFIG_REQUEST, CREATE_EMPTY_PIPELINE, START_ARCHIVE_PIPELINE, UPDATE_JOB_CONFIG_REQUEST
 }

flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/Job.java

@@ -31,6 +31,8 @@ public class Job {
 
     private JobType jobType;
 
+    private String user;
+
     @AllArgsConstructor
     @Getter
     public enum JobType {
@@ -46,7 +48,6 @@ public enum JobType {
 
         public String[] getScript(Job job) {
             switch (this) {
-                case GENERATOR:
                 case PROFILE:
                 case PARSER:
                     return new String[]{scriptName, job.getJobBranch(), job.getJobPipeline(), job.getJobName()};

flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/Pipeline.java

@@ -0,0 +1,18 @@
+package com.cloudera.service.common.response;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class Pipeline {
+    String id;
+    String name;
+    String clusterName;
+    String date;
+    String userName;
+}

flink-cyber/cyber-services/cyber-service-common/src/main/java/com/cloudera/service/common/response/ResponseType.java

@@ -1,5 +1,5 @@
 package com.cloudera.service.common.response;
 
 public enum ResponseType {
-    GET_ALL_CLUSTERS_SERVICE_RESPONSE, GET_CLUSTER_SERVICE_RESPONSE, START_JOB_RESPONSE, RESTART_JOB_RESPONSE, STOP_JOB_RESPONSE, GET_JOB_CONFIG_RESPONSE, UPDATE_JOB_CONFIG_RESPONSE, ERROR_RESPONSE
+    GET_ALL_CLUSTERS_SERVICE_RESPONSE, GET_CLUSTER_SERVICE_RESPONSE, START_JOB_RESPONSE, RESTART_JOB_RESPONSE, STOP_JOB_RESPONSE, GET_JOB_CONFIG_RESPONSE, UPDATE_JOB_CONFIG_RESPONSE, CREATE_EMPTY_PIPELINE_RESPONSE, START_ARCHIVE_PIPELINE_RESPONSE, ERROR_RESPONSE
 }

flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/configuration/AppWorkerConfig.java

@@ -0,0 +1,18 @@
+package com.cloudera.cyber.restcli.configuration;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties(prefix = "cluster")
+@Getter
+@Setter
+public class AppWorkerConfig {
+    private String name;
+    private String id;
+    private String status;
+    private String version;
+    private String pipelineDir;
+}

flink-cyber/cyber-services/cyber-worker-service/src/main/java/com/cloudera/cyber/restcli/controller/KafkaListenerController.java

@@ -1,6 +1,8 @@
 package com.cloudera.cyber.restcli.controller;
 
+import com.cloudera.cyber.restcli.configuration.AppWorkerConfig;
 import com.cloudera.cyber.restcli.service.JobService;
+import com.cloudera.cyber.restcli.service.FilePipelineService;
 import com.cloudera.service.common.Utils;
 import com.cloudera.service.common.request.RequestBody;
 import com.cloudera.service.common.request.RequestType;
@@ -29,18 +31,11 @@
 @RequiredArgsConstructor
 @Slf4j
 public class KafkaListenerController {
-
-    @Value("${cluster.name}")
-    private String clusterName;
-    @Value("${cluster.id}")
-    private String clusterId;
-    @Value("${cluster.status}")
-    private String clusterStatus;
-    @Value("${cluster.version}")
-    private String clusterVersion;
-
     private final JobService jobService;
+    private final FilePipelineService pipelineService;
+    private final AppWorkerConfig config;
 
+    //TODO:  Rewrite to Spring events. Probably split the events into separate types, such as cluster event, job event, pipeline event, etc.
     @KafkaListener(topics = "#{kafkaProperties.getRequestTopic()}", containerFactory = "kafkaListenerContainerFactory")
     @SendTo({"#{kafkaProperties.getReplyTopic()}"})
     public Message<ResponseBody> handleMessage(RequestBody requestBody, @Header(KafkaHeaders.RECEIVED_MESSAGE_KEY) String key, @Header(KafkaHeaders.REPLY_TOPIC) byte[] replyTo,
@@ -85,6 +80,26 @@ public Message<ResponseBody> handleMessage(RequestBody requestBody, @Header(Kafk
                 } catch (IOException e) {
                     return handleErrorResponse(e, replyTo, correlationId);
                 }
+            case CREATE_EMPTY_PIPELINE:
+                try {
+                    pipelineService.createEmptyPipeline(requestBody.getPipelineName(), requestBody.getBranch());
+                    final ResponseBody responseBody = ResponseBody.builder().build();
+                    return buildResponseMessage(responseBody, ResponseType.CREATE_EMPTY_PIPELINE_RESPONSE, replyTo, correlationId);
+                } catch (Exception e) {
+                    return handleErrorResponse(e, replyTo, correlationId);
+                }
+            case START_ARCHIVE_PIPELINE:
+                try {
+                    pipelineService.extractPipeline(requestBody.getPayload(), requestBody.getPipelineName(), requestBody.getBranch());
+                    pipelineService.startPipelineJob(requestBody.getPipelineName(), requestBody.getBranch(), requestBody.getProfileName(), requestBody.getJobs());
+                    final ResponseBody responseBody = ResponseBody.builder().build();
+                    return buildResponseMessage(responseBody, ResponseType.START_ARCHIVE_PIPELINE_RESPONSE, replyTo, correlationId);
+                } catch (Exception e) {
+                    log.error("Exception while processing the Start All request {}", e.getMessage());
+                    return handleErrorResponse(e, replyTo, correlationId);
+
+                }
+
         }
         return null;
     }
@@ -95,10 +110,10 @@ private Message<ResponseBody> getResponseBodyMessage(byte[] replyTo, byte[] corr
             ResponseBody responseBody = ResponseBody.builder()
                     .jobs(jobs)
                     .clusterMeta(ClusterMeta.builder()
-                            .name(clusterName)
-                            .clusterId(clusterId)
-                            .clusterStatus(clusterStatus)
-                            .version(clusterVersion)
+                            .name(config.getName())
+                            .clusterId(config.getId())
+                            .clusterStatus(config.getStatus())
+                            .version(config.getVersion())
                             .build())
                     .build();
             return buildResponseMessage(responseBody, responseType, replyTo, correlationId);