parse v2 yaml configuration

Author: gy2006

pom.xml

@@ -16,7 +16,7 @@
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
 
-    <spring-boot.version>3.3.4</spring-boot.version>
+    <spring-boot.version>3.3.7</spring-boot.version>
     <instancio-junit.version>2.9.0</instancio-junit.version>
     <testcontainers.version>1.20.4</testcontainers.version>
     <springdoc-openapi.version>2.5.0</springdoc-openapi.version>
@@ -124,6 +124,16 @@
       <artifactId>micrometer-registry-prometheus</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-yaml</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>com.fasterxml.jackson.datatype</groupId>
+      <artifactId>jackson-datatype-jsr310</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-starter-test</artifactId>

src/main/java/com/flowci/user/package-info.java

@@ -0,0 +1 @@
+package com.flowci.user;

src/main/java/com/flowci/yaml/business/ParseYamlV2.java

@@ -0,0 +1,7 @@
+package com.flowci.yaml.business;
+
+import com.flowci.yaml.model.FlowV2;
+
+public interface ParseYamlV2 {
+    FlowV2 invoke(String yaml);
+}

src/main/java/com/flowci/yaml/business/impl/ParseYamlV2Impl.java

@@ -0,0 +1,96 @@
+package com.flowci.yaml.business.impl;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.flowci.common.validator.ValidName;
+import com.flowci.yaml.business.ParseYamlV2;
+import com.flowci.yaml.exception.InvalidYamlException;
+import com.flowci.yaml.model.FlowV2;
+import com.flowci.yaml.model.StepV2;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
+import java.util.HashSet;
+import java.util.List;
+
+import static java.lang.String.format;
+import static org.springframework.util.CollectionUtils.isEmpty;
+import static org.springframework.util.StringUtils.hasText;
+
+@Slf4j
+@Component
+public class ParseYamlV2Impl implements ParseYamlV2 {
+
+    private static final ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
+    private static final ValidName.NameValidator nameValidator = new ValidName.NameValidator();
+
+    static {
+        objectMapper.findAndRegisterModules();
+    }
+
+    @Override
+    public FlowV2 invoke(String yaml) {
+        try {
+            var flowV2 = objectMapper.readValue(yaml, FlowV2.class);
+            for (var step : flowV2.getSteps()) {
+                step.setParent(flowV2);
+            }
+            return validateYaml(flowV2);
+        } catch (JsonProcessingException e) {
+            log.error("invalid YAML configuration", e);
+            throw new InvalidYamlException("invalid YAML configuration");
+        }
+    }
+
+    private FlowV2 validateYaml(FlowV2 flowV2) {
+        var steps = flowV2.getSteps();
+        validateSteps(steps);
+
+        return flowV2;
+    }
+
+    private void validateSteps(List<StepV2> steps) {
+        if (CollectionUtils.isEmpty(steps)) {
+            throw new InvalidYamlException("at least one step is required");
+        }
+
+        var stepNameSet = new HashSet<>(steps.size());
+        for (var step : steps) {
+            if (!stepNameSet.add(step.getName())) {
+                throw new InvalidYamlException(format("step name '%s' already exists", step.getName()));
+            }
+        }
+
+        for (var step : steps) {
+            if (!nameValidator.isValid(step.getName(), null)) {
+                throw new InvalidYamlException(format("step name '%s' is invalid", step.getName()));
+            }
+
+            if (!isEmpty(step.getDependsOn())) {
+                for (var dependsOn : step.getDependsOn()) {
+                    if (!stepNameSet.contains(dependsOn)) {
+                        throw new InvalidYamlException(format("depends on '%s' is not found", dependsOn));
+                    }
+                }
+            }
+
+            validateCommands(step);
+        }
+    }
+
+    private void validateCommands(StepV2 step) {
+        var commands = step.getCommands();
+
+        if (CollectionUtils.isEmpty(commands)) {
+            throw new InvalidYamlException(format("at least one command under step '%s' is required", step.getName()));
+        }
+
+        for (var command : commands) {
+            if (!hasText(command.getBash()) || !hasText(command.getPwsh())) {
+                throw new InvalidYamlException("bash or powershell is required");
+            }
+        }
+    }
+}

src/main/java/com/flowci/yaml/exception/InvalidYamlException.java

@@ -0,0 +1,9 @@
+package com.flowci.yaml.exception;
+
+import com.flowci.common.exception.BusinessException;
+
+public class InvalidYamlException extends BusinessException {
+    public InvalidYamlException(String message) {
+        super(message);
+    }
+}

src/main/java/com/flowci/yaml/model/BaseV2.java

@@ -0,0 +1,32 @@
+package com.flowci.yaml.model;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+@Getter
+@Setter
+public abstract class BaseV2 {
+
+    private static final Integer DEFAULT_TIMEOUT = 1800;
+
+    protected Map<String, String> variables = new LinkedHashMap<>();
+
+    protected Integer timeout = DEFAULT_TIMEOUT; // timeout in seconds
+
+    /**
+     * Groovy script
+     */
+    protected String condition;
+
+    protected DockerV2 docker;
+
+    protected List<DockerV2> dockers;
+
+    public String getCondition() {
+        return condition.trim();
+    }
+}

src/main/java/com/flowci/yaml/model/CommandV2.java

@@ -0,0 +1,26 @@
+package com.flowci.yaml.model;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.util.StringUtils;
+
+import static org.springframework.util.StringUtils.hasText;
+
+@Getter
+@Setter
+public class CommandV2 {
+
+    private String name; // optional
+
+    private String bash; // bash script
+
+    private String pwsh; // powershell script
+
+    public String getBash() {
+        return hasText(bash) ? bash.trim() : null;
+    }
+
+    public String getPwsh() {
+        return hasText(pwsh) ? pwsh.trim() : null;
+    }
+}

src/main/java/com/flowci/yaml/model/DockerV2.java

@@ -0,0 +1,38 @@
+package com.flowci.yaml.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.Map;
+import java.util.List;
+
+@Getter
+@Setter
+public class DockerV2 {
+
+    private String image;
+
+    private String auth; // auth secret for private docker registry
+
+    private String name;
+
+    private String network;
+
+    private List<String> ports;
+
+    private List<String> entrypoint;
+
+    private List<String> command;
+
+    private Map<String, String> environment;
+
+    @JsonProperty("is_runtime")
+    private Boolean isRuntime;
+
+    @JsonProperty("stop_on_finish")
+    private Boolean stopOnFinish;
+
+    @JsonProperty("delete_on_finish")
+    private Boolean deleteOnFinish;
+}

src/main/java/com/flowci/yaml/model/FlowV2.java

@@ -0,0 +1,18 @@
+package com.flowci.yaml.model;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.List;
+
+@Setter
+@Getter
+public class FlowV2 extends BaseV2 {
+
+    /**
+     * List of agent tags
+     */
+    private List<String> agents;
+
+    private List<StepV2> steps;
+}

src/main/java/com/flowci/yaml/model/StepV2.java

@@ -0,0 +1,75 @@
+package com.flowci.yaml.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.flowci.common.model.Variables;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+import java.util.List;
+
+@Data
+@EqualsAndHashCode(of = "name", callSuper = false)
+public class StepV2 extends BaseV2 {
+
+    private static final Boolean DEFAULT_ALLOW_FAILURE = false;
+
+    private String name;
+
+    // dependency steps name
+    @JsonProperty("depends_on")
+    private List<String> dependsOn;
+
+    private String plugin;
+
+    private Integer retry; // num of retry
+
+    @JsonProperty("allow_failure")
+    private Boolean allowFailure = DEFAULT_ALLOW_FAILURE;
+
+    private List<CommandV2> commands;
+
+    private List<String> output;
+
+    private List<String> secrets;
+
+    private List<String> configs;
+
+    // ref to parent flow
+    @JsonIgnore
+    private FlowV2 parent;
+
+    @Override
+    public DockerV2 getDocker() {
+        if (this.docker != null) {
+            return this.docker;
+        }
+
+        if (this.parent.docker != null) {
+            return this.parent.docker;
+        }
+
+        return null;
+    }
+
+    @Override
+    public List<DockerV2> getDockers() {
+        if (this.dockers != null) {
+            return this.dockers;
+        }
+
+        if (this.parent.dockers != null) {
+            return this.parent.dockers;
+        }
+
+        return null;
+    }
+
+    // merge variables from current step and parent flow
+    @Override
+    public Variables getVariables() {
+        var variables = new Variables(parent.getVariables());
+        variables.putAll(this.variables);
+        return variables;
+    }
+}