Merge pull request #34 from common-workflow-language/step-parsing-draft3

Step parsing draft-3

Author: MarkRobbo

src/main/java/org/commonwl/viewer/domain/CWLCollection.java

@@ -40,8 +40,11 @@ public class CWLCollection {
     private GitHubService githubService;
     private GithubDetails githubInfo;
 
-    private List<JsonNode> cwlDocs = new ArrayList<>();
-    private int mainWorkflowIndex = -1;
+    // Maps of ID to associated JSON
+    private Map<String, JsonNode> cwlDocs = new HashMap<>();
+
+    // The main workflow
+    private String mainWorkflowKey;
 
     /**
      * Creates a new collection of CWL files from a Github repository
@@ -99,7 +102,7 @@ private void addDocs(List<RepositoryContents> repoContents) throws IOException {
                         JsonNode cwlFile = mapper.valueToTree(reader.load(fileContent));
 
                         // Add document to those being considered
-                        addDoc(cwlFile);
+                        addDoc(cwlFile, repoContent.getName());
                     }
                 }
 
@@ -110,17 +113,18 @@ private void addDocs(List<RepositoryContents> repoContents) throws IOException {
     /**
      * Adds a document to the group of those being parsed
      * @param newDoc The document to be added
+     * @param fileName The name of the file this document has come from
      */
-    private void addDoc(JsonNode newDoc) {
+    private void addDoc(JsonNode newDoc, String fileName) {
         // Make sure that this document is only one object and not multiple under a $graph directive
         if (newDoc.has("$graph")) {
             // Add each of the sub documents
             for (JsonNode jsonNode : newDoc.get("$graph")) {
-                cwlDocs.add(jsonNode);
+                cwlDocs.put(extractID(jsonNode), jsonNode);
             }
         } else {
-            // Otherwise just add the document itself
-            cwlDocs.add(newDoc);
+            // Otherwise just add the document itself with ID of document name
+            cwlDocs.put(fileName, newDoc);
         }
     }
 
@@ -130,9 +134,9 @@ private void addDoc(JsonNode newDoc) {
     private void findMainWorkflow() {
         // Find the first workflow we come across
         // TODO: Consider relationship between run: parameters to better discover this
-        for (int i=0; i < cwlDocs.size(); i++) {
-            if (cwlDocs.get(i).get("class").asText().equals("Workflow")) {
-                mainWorkflowIndex = 0;
+        for (Map.Entry<String, JsonNode> doc : cwlDocs.entrySet()) {
+            if (doc.getValue().get("class").asText().equals("Workflow")) {
+                mainWorkflowKey = doc.getKey();
                 return;
             }
         }
@@ -143,57 +147,95 @@ private void findMainWorkflow() {
      * @return A Workflow object representing the main workflow amongst the files added
      */
     public Workflow getWorkflow() {
-        if (mainWorkflowIndex < 0) {
+        // Get the main workflow
+        if (mainWorkflowKey == null) {
             findMainWorkflow();
 
             // If it is still less than 0 there is no workflow to be found
-            if (mainWorkflowIndex < 0) {
+            if (mainWorkflowKey == null) {
                 return null;
             }
         }
-        JsonNode mainWorkflow = cwlDocs.get(mainWorkflowIndex);
-        return new Workflow(extractLabel(mainWorkflow), extractDoc(mainWorkflow),
-                            getInputs(mainWorkflow), getOutputs(mainWorkflow));
+        JsonNode mainWorkflow = cwlDocs.get(mainWorkflowKey);
+
+        // Use ID/filename for label if there is no defined one
+        String label = extractLabel(mainWorkflow);
+        if (label == null) {
+            label = mainWorkflowKey;
+        }
+
+        return new Workflow(label, extractDoc(mainWorkflow), getInputs(mainWorkflow),
+                            getOutputs(mainWorkflow), getSteps(mainWorkflow));
+    }
+
+    /**
+     * Get the steps for a particular document
+     * @param cwlDoc The document to get steps for
+     * @return A map of step IDs and details related to them
+     */
+    private Map<String, CWLStep> getSteps(JsonNode cwlDoc) {
+        if (cwlDoc != null && cwlDoc.has("steps")) {
+            Map<String, CWLStep> returnMap = new HashMap<>();
+
+            JsonNode steps = cwlDoc.get("steps");
+            if (steps.getClass() == ArrayNode.class) {
+                // Explicit ID and other fields within each input list
+                for (JsonNode step : steps) {
+                    CWLStep stepObject = new CWLStep(extractLabel(step), extractDoc(step),
+                            extractTypes(step), getInputs(step), getOutputs(step));
+                    returnMap.put(extractID(step), stepObject);
+                }
+            } else if (steps.getClass() == ObjectNode.class) {
+                // ID is the key of each object
+                Iterator<Map.Entry<String, JsonNode>> iterator = steps.fields();
+                while (iterator.hasNext()) {
+                    Map.Entry<String, JsonNode> stepNode = iterator.next();
+                    JsonNode stepJson = stepNode.getValue();
+                    CWLStep stepObject = new CWLStep(extractLabel(stepJson), extractDoc(stepJson),
+                            extractTypes(stepJson), getInputs(stepJson), getOutputs(stepJson));
+                    returnMap.put(stepNode.getKey(), stepObject);
+                }
+            }
+
+            return returnMap;
+        }
+        return null;
     }
 
     /**
-     * Get a list of the inputs for a particular document
+     * Get a the inputs for a particular document
      * @param cwlDoc The document to get inputs for
      * @return A map of input IDs and details related to them
      */
     private Map<String, CWLElement> getInputs(JsonNode cwlDoc) {
-        if (cwlDoc != null) {
-            if (cwlDoc.has("inputs")) {
-                return getInputsOutputs(cwlDoc.get("inputs"));
-            }
+        if (cwlDoc != null && cwlDoc.has("inputs")) {
+            return getInputsOutputs(cwlDoc.get("inputs"));
         }
         return null;
     }
 
     /**
-     * Get a list of the outputs for a particular document
+     * Get the outputs for a particular document
      * @param cwlDoc The document to get outputs for
      * @return A map of output IDs and details related to them
      */
     private Map<String, CWLElement> getOutputs(JsonNode cwlDoc) {
-        if (cwlDoc != null) {
-            if (cwlDoc.has("outputs")) {
-                return getInputsOutputs(cwlDoc.get("outputs"));
-            }
+        if (cwlDoc != null && cwlDoc.has("outputs")) {
+            return getInputsOutputs(cwlDoc.get("outputs"));
         }
         return null;
     }
 
     /**
-     * Get a list of inputs or outputs from an inputs or outputs node
+     * Get inputs or outputs from an inputs or outputs node
      * @param inputsOutputs The inputs or outputs node
      * @return A map of input IDs and details related to them
      */
     private Map<String, CWLElement> getInputsOutputs(JsonNode inputsOutputs) {
         Map<String, CWLElement> returnMap = new HashMap<>();
 
         if (inputsOutputs.getClass() == ArrayNode.class) {
-            // Explicit ID and other fields within each input list
+            // Explicit ID and other fields within each ilist
             for (JsonNode inputOutput : inputsOutputs) {
                 String id = inputOutput.get("id").asText();
                 returnMap.put(id, getDetails(inputOutput));
@@ -203,8 +245,7 @@ private Map<String, CWLElement> getInputsOutputs(JsonNode inputsOutputs) {
             Iterator<Map.Entry<String, JsonNode>> iterator = inputsOutputs.fields();
             while (iterator.hasNext()) {
                 Map.Entry<String, JsonNode> inputOutputNode = iterator.next();
-                String outputID = inputOutputNode.getKey();
-                returnMap.put(outputID, getDetails(inputOutputNode.getValue()));
+                returnMap.put(inputOutputNode.getKey(), getDetails(inputOutputNode.getValue()));
             }
         }
 
@@ -224,13 +265,10 @@ private CWLElement getDetails(JsonNode inputOutput) {
             if (inputOutput.getClass() == TextNode.class) {
                 details.setType(inputOutput.asText());
             } else {
-                if (extractLabel(inputOutput) != null) {
-                    details.setLabel(extractLabel(inputOutput));
-                }
-
-                if (extractDoc(inputOutput) != null) {
-                    details.setDoc(extractDoc(inputOutput));
-                }
+                details.setLabel(extractLabel(inputOutput));
+                details.setDoc(extractDoc(inputOutput));
+                extractOutputSource(inputOutput).forEach(details::addSourceID);
+                details.setDefaultVal(extractDefault(inputOutput));
 
                 // Type is only for inputs
                 if (inputOutput.has("type")) {
@@ -243,20 +281,103 @@ private CWLElement getDetails(JsonNode inputOutput) {
         return null;
     }
 
+    /**
+     * Extract the id from a node
+     * @param node The node to have the id extracted from
+     * @return The string for the id of the node
+     */
+    private String extractID(JsonNode node) {
+        if (node != null && node.has("id")) {
+            return node.get("id").asText();
+        }
+        return null;
+    }
+
     /**
      * Extract the label from a node
      * @param node The node to have the label extracted from
      * @return The string for the label of the node
      */
     private String extractLabel(JsonNode node) {
+        if (node != null && node.has("label")) {
+            return node.get("label").asText();
+        }
+        return null;
+    }
+
+    /**
+     * Extract the default value from a node
+     * @param node The node to have the label extracted from
+     * @return The string for the default value of the node
+     */
+    private String extractDefault(JsonNode node) {
+        if (node != null && node.has("default")) {
+            return node.get("default").asText();
+        }
+        return null;
+    }
+
+    /**
+     * Extract the source or outputSource from a node
+     * @param node The node to have the sources extracted from
+     * @return A list of strings for the sources
+     */
+    private List<String> extractOutputSource(JsonNode node) {
         if (node != null) {
-            if (node.has("label")) {
-                return node.get("label").asText();
+            List<String> sources = new ArrayList<String>();
+            JsonNode sourceNode = null;
+
+            // outputSource and source treated the same
+            if (node.has("outputSource")) {
+                sourceNode = node.get("outputSource");
+            } else if (node.has("source")) {
+                sourceNode = node.get("source");
             }
+
+            if (sourceNode != null) {
+                // Single source
+                if (sourceNode.getClass() == TextNode.class) {
+                    sources.add(stepIDFromSource(sourceNode.asText()));
+                }
+                // Can be an array of multiple sources
+                if (sourceNode.getClass() == ArrayNode.class) {
+                    for (JsonNode source : sourceNode) {
+                        sources.add(stepIDFromSource(source.asText()));
+                    }
+                }
+            }
+
+            return sources;
         }
         return null;
     }
 
+    /**
+     * Gets just the step ID from source of format 'stepID/outputID'
+     * @param source The source
+     * @return The step ID
+     */
+    private String stepIDFromSource(String source) {
+        if (source != null) {
+            // Strip leading # if it exists
+            if (source.charAt(0) == '#') {
+                source = source.substring(1);
+            }
+
+            // Get segment before / (step ID)
+            int slashSplit = source.indexOf("/");
+            if (slashSplit != -1) {
+                source = source.substring(0, slashSplit);
+            } else {
+                int dotSplit = source.indexOf(".");
+                if (dotSplit != -1) {
+                    source = "#" + source.substring(0, dotSplit);
+                }
+            }
+        }
+        return source;
+    }
+
     /**
      * Extract the doc or description from a node
      * @param node The node to have the doc/description extracted from
@@ -330,5 +451,4 @@ private String extractTypes(JsonNode typeNode) {
         }
         return null;
     }
-
 }

src/main/java/org/commonwl/viewer/domain/CWLElement.java

@@ -19,14 +19,23 @@
 
 package org.commonwl.viewer.domain;
 
+import java.util.ArrayList;
+import java.util.List;
+
 /**
- * Represents the input or output of a workflow/tool
+ * Represents the input/output of a workflow/tool
  */
 public class CWLElement {
 
     private String label;
     private String doc;
     private String type;
+    private List<String> sourceID;
+    private String defaultVal;
+
+    public CWLElement() {
+        this.sourceID = new ArrayList<String>();
+    }
 
     public String getLabel() {
         return label;
@@ -52,4 +61,21 @@ public void setType(String type) {
         this.type = type;
     }
 
+    public List<String> getSourceIDs() {
+        return sourceID;
+    }
+
+    public void addSourceID(String sourceID) {
+        if (sourceID != null) {
+            this.sourceID.add(sourceID);
+        }
+    }
+
+    public String getDefaultVal() {
+        return defaultVal;
+    }
+
+    public void setDefaultVal(String defaultVal) {
+        this.defaultVal = defaultVal;
+    }
 }