Merge pull request #151 from common-workflow-language/packed-workflows

Packed Workflow Support

Author: stain

src/main/java/org/commonwl/view/cwl/CWLProcess.java

@@ -23,21 +23,18 @@
  * Enum for possible CWL processes
  */
 public enum CWLProcess {
-    WORKFLOW,
-    COMMANDLINETOOL,
-    EXPRESSIONTOOL;
+    WORKFLOW("Workflow"),
+    COMMANDLINETOOL("CommandLineTool"),
+    EXPRESSIONTOOL("ExpressionTool");
+
+    private final String name;
+
+    CWLProcess(String name) {
+        this.name = name;
+    }
 
     @Override
     public String toString() {
-        switch (this) {
-            case WORKFLOW:
-                return "Workflow";
-            case COMMANDLINETOOL:
-                return "CommandLineTool";
-            case EXPRESSIONTOOL:
-                return "ExpressionTool";
-            default:
-                return super.toString();
-        }
+        return name;
     }
 }

src/main/java/org/commonwl/view/cwl/CWLService.java

@@ -19,18 +19,11 @@
 
 package org.commonwl.view.cwl;
 
-import static org.apache.commons.io.FileUtils.readFileToString;
-
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.jena.iri.IRI;
@@ -46,6 +39,7 @@
 import org.commonwl.view.graphviz.ModelDotWriter;
 import org.commonwl.view.graphviz.RDFDotWriter;
 import org.commonwl.view.workflow.Workflow;
+import org.commonwl.view.workflow.WorkflowNotFoundException;
 import org.commonwl.view.workflow.WorkflowOverview;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -54,11 +48,13 @@
 import org.springframework.stereotype.Service;
 import org.yaml.snakeyaml.Yaml;
 
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.fasterxml.jackson.databind.node.TextNode;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.*;
+
+import static org.apache.commons.io.FileUtils.readFileToString;
 
 /**
  * Provides CWL parsing for workflows to gather an overview
@@ -114,12 +110,58 @@ public CWLService(RDFService rdfService,
         this.singleFileSizeLimit = singleFileSizeLimit;
     }
 
+    /**
+     * Gets whether a file is packed using schema salad
+     * @param workflowFile The file to be parsed
+     * @return Whether the file is packed
+     */
+    public boolean isPacked(File workflowFile) throws IOException {
+        if (workflowFile.length() > singleFileSizeLimit) {
+            return false;
+        }
+        String fileContent = readFileToString(workflowFile);
+        return fileContent.contains("$graph");
+    }
+
+    /**
+     * Gets a list of workflows from a packed CWL file
+     * @param packedFile The packed CWL file
+     * @return The list of workflow overviews
+     */
+    public List<WorkflowOverview> getWorkflowOverviewsFromPacked(File packedFile) throws IOException {
+        if (packedFile.length() <= singleFileSizeLimit) {
+            List<WorkflowOverview> overviews = new ArrayList<>();
+
+            JsonNode packedJson = yamlStringToJson(readFileToString(packedFile));
+
+            if (packedJson.has(DOC_GRAPH)) {
+                for (JsonNode jsonNode : packedJson.get(DOC_GRAPH)) {
+                    if (extractProcess(jsonNode) == CWLProcess.WORKFLOW) {
+                        WorkflowOverview overview = new WorkflowOverview(jsonNode.get(ID).asText(),
+                                extractLabel(jsonNode), extractDoc(jsonNode));
+                        overviews.add(overview);
+                    }
+                }
+            } else {
+                throw new IOException("The file given was not recognised as a packed CWL file");
+            }
+
+            return overviews;
+
+        } else {
+            throw new IOException("File '" + packedFile.getName() +  "' is over singleFileSizeLimit - " +
+                    FileUtils.byteCountToDisplaySize(packedFile.length()) + "/" +
+                    FileUtils.byteCountToDisplaySize(singleFileSizeLimit));
+        }
+    }
+
     /**
      * Gets the Workflow object from internal parsing
      * @param workflowFile The workflow file to be parsed
+     * @param packedWorkflowId The ID of the workflow object if the file is packed
      * @return The constructed workflow object
      */
-    public Workflow parseWorkflowNative(File workflowFile) throws IOException {
+    public Workflow parseWorkflowNative(File workflowFile, String packedWorkflowId) throws IOException {
 
         // Check file size limit before parsing
         long fileSizeBytes = workflowFile.length();
@@ -128,16 +170,30 @@ public Workflow parseWorkflowNative(File workflowFile) throws IOException {
             // Parse file as yaml
             JsonNode cwlFile = yamlStringToJson(readFileToString(workflowFile));
 
-            // If the CWL file is packed there can be multiple workflows in a file
-            Map<String, JsonNode> packedFiles = new HashMap<>();
-            if (cwlFile.has(DOC_GRAPH)) {
-                // Packed CWL, find the first subelement which is a workflow and take it
-                for (JsonNode jsonNode : cwlFile.get(DOC_GRAPH)) {
-                    packedFiles.put(jsonNode.get(ID).asText(), jsonNode);
-                    if (extractProcess(jsonNode) == CWLProcess.WORKFLOW) {
-                        cwlFile = jsonNode;
+            // Check packed workflow occurs
+            if (packedWorkflowId != null) {
+                boolean found = false;
+                if (cwlFile.has(DOC_GRAPH)) {
+                    for (JsonNode jsonNode : cwlFile.get(DOC_GRAPH)) {
+                        if (extractProcess(jsonNode) == CWLProcess.WORKFLOW) {
+                            String currentId = jsonNode.get(ID).asText();
+                            if (currentId.startsWith("#")) {
+                                currentId = currentId.substring(1);
+                            }
+                            if (currentId.equals(packedWorkflowId)) {
+                                cwlFile = jsonNode;
+                                found = true;
+                                break;
+                            }
+                        }
                     }
                 }
+                if (!found) throw new WorkflowNotFoundException();
+            } else {
+                // Check the current json node is a workflow
+                if (extractProcess(cwlFile) != CWLProcess.WORKFLOW) {
+                    throw new WorkflowNotFoundException();
+                }
             }
 
             // Use filename for label if there is no defined one
@@ -150,10 +206,6 @@ public Workflow parseWorkflowNative(File workflowFile) throws IOException {
             Workflow workflowModel = new Workflow(label, extractDoc(cwlFile), getInputs(cwlFile),
                     getOutputs(cwlFile), getSteps(cwlFile), null);
 
-            if (packedFiles.size() > 0) {
-                workflowModel.setPackedWorkflowID(cwlFile.get(ID).asText());
-            }
-
             workflowModel.setCwltoolVersion(cwlTool.getVersion());
 
             // Generate DOT graph
@@ -186,7 +238,7 @@ public Workflow parseWorkflowWithCwltool(Workflow basicModel,
                                              File workflowFile) throws CWLValidationException {
         GitDetails gitDetails = basicModel.getRetrievedFrom();
         String latestCommit = basicModel.getLastCommit();
-        String packedWorkflowID = basicModel.getPackedWorkflowID();
+        String packedWorkflowID = gitDetails.getPackedId();
 
         // Get paths to workflow
         String url = gitDetails.getUrl(latestCommit).replace("https://", "");
@@ -405,13 +457,19 @@ public WorkflowOverview getWorkflowOverview(File file) throws IOException {
             JsonNode cwlFile = yamlStringToJson(readFileToString(file));
 
             // If the CWL file is packed there can be multiple workflows in a file
+            int packedCount = 0;
             if (cwlFile.has(DOC_GRAPH)) {
                 // Packed CWL, find the first subelement which is a workflow and take it
                 for (JsonNode jsonNode : cwlFile.get(DOC_GRAPH)) {
                     if (extractProcess(jsonNode) == CWLProcess.WORKFLOW) {
                         cwlFile = jsonNode;
+                        packedCount++;
                     }
                 }
+                if (packedCount > 1) {
+                    return new WorkflowOverview("/" + file.getName(), "Packed file",
+                            "contains " + packedCount + " workflows");
+                }
             }
 
             // Can only make an overview if this is a workflow
@@ -423,7 +481,7 @@ public WorkflowOverview getWorkflowOverview(File file) throws IOException {
                 }
 
                 // Return the constructed overview
-                return new WorkflowOverview(file.getName(), label, extractDoc(cwlFile));
+                return new WorkflowOverview("/" + file.getName(), label, extractDoc(cwlFile));
             } else {
                 // Return null if not a workflow file
                 return null;

src/main/java/org/commonwl/view/cwl/CWLToolRunner.java

@@ -19,6 +19,7 @@
 
 package org.commonwl.view.cwl;
 
+import org.apache.jena.query.QueryException;
 import org.commonwl.view.researchobject.ROBundleFactory;
 import org.commonwl.view.workflow.QueuedWorkflow;
 import org.commonwl.view.workflow.QueuedWorkflowRepository;
@@ -79,7 +80,6 @@ public void createWorkflowFromQueued(QueuedWorkflow queuedWorkflow, File workflo
             newWorkflow.setRetrievedFrom(tempWorkflow.getRetrievedFrom());
             newWorkflow.setRetrievedOn(new Date());
             newWorkflow.setLastCommit(tempWorkflow.getLastCommit());
-            newWorkflow.setPackedWorkflowID(tempWorkflow.getPackedWorkflowID());
             newWorkflow.setCwltoolVersion(cwlToolVersion);
 
             workflowRepository.save(newWorkflow);
@@ -89,19 +89,22 @@ public void createWorkflowFromQueued(QueuedWorkflow queuedWorkflow, File workflo
 
             // Mark success on queue
             queuedWorkflow.setCwltoolStatus(CWLToolStatus.SUCCESS);
-            queuedWorkflowRepository.save(queuedWorkflow);
 
+        } catch (QueryException ex) {
+            logger.error("Jena query exception ", ex);
+            queuedWorkflow.setCwltoolStatus(CWLToolStatus.ERROR);
+            queuedWorkflow.setMessage("An error occurred when executing a query on the SPARQL store");
         } catch (CWLValidationException ex) {
             logger.error(ex.getMessage(), ex);
             queuedWorkflow.setCwltoolStatus(CWLToolStatus.ERROR);
             queuedWorkflow.setMessage(ex.getMessage());
-            queuedWorkflowRepository.save(queuedWorkflow);
         } catch (Exception ex) {
             logger.error("Unexpected error", ex);
             queuedWorkflow.setCwltoolStatus(CWLToolStatus.ERROR);
             queuedWorkflow.setMessage("Whoops! Cwltool ran successfully, but an unexpected " +
                     "error occurred in CWLViewer!\n" +
                     "Help us by reporting it on Gitter or a Github issue\n");
+        } finally {
             queuedWorkflowRepository.save(queuedWorkflow);
         }
 

src/main/java/org/commonwl/view/git/GitDetails.java

@@ -34,6 +34,7 @@ public class GitDetails implements Serializable {
     private String repoUrl;
     private String branch;
     private String path;
+    private String packedId;
 
     public GitDetails(String repoUrl, String branch, String path) {
         this.repoUrl = repoUrl;
@@ -67,6 +68,14 @@ public void setBranch(String branch) {
         this.branch = branch;
     }
 
+    public String getPackedId() {
+        return packedId;
+    }
+
+    public void setPackedId(String packedId) {
+        this.packedId = packedId;
+    }
+
     public String getPath() {
         return path;
     }
@@ -139,13 +148,14 @@ public String getUrl() {
      * @return The URL
      */
     public String getInternalUrl() {
+        String packedPart = packedId == null ? "" : "%23" + packedId;
         String pathPart = path.equals("/") ? "" : "/" + path;
         switch (getType()) {
             case GITHUB:
             case GITLAB:
-                return "/workflows/" + normaliseUrl(repoUrl).replace(".git", "") + "/blob/" + branch + pathPart;
+                return "/workflows/" + normaliseUrl(repoUrl).replace(".git", "") + "/blob/" + branch + pathPart + packedPart;
             default:
-                return "/workflows/" + normaliseUrl(repoUrl) + "/" + branch + pathPart;
+                return "/workflows/" + normaliseUrl(repoUrl) + "/" + branch + pathPart + packedPart;
         }
     }