Support directory listing when given a directory URL

Author: MarkRobbo

src/main/java/org/commonwl/view/cwl/CWLService.java

@@ -39,6 +39,7 @@
 import org.commonwl.view.graphviz.ModelDotWriter;
 import org.commonwl.view.graphviz.RDFDotWriter;
 import org.commonwl.view.workflow.Workflow;
+import org.commonwl.view.workflow.WorkflowOverview;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -428,6 +429,55 @@ public Workflow parseWorkflowWithCwltool(Workflow basicModel,
 
     }
 
+    /**
+     * Get an overview of a workflow
+     * @param file A file, potentially a workflow
+     * @return A constructed WorkflowOverview of the workflow
+     * @throws IOException Any API errors which may have occurred
+     */
+    public WorkflowOverview getWorkflowOverview(File file) throws IOException {
+
+        // Get the content of this file from Github
+        long fileSizeBytes = file.length();
+
+        // Check file size limit before parsing
+        if (fileSizeBytes <= singleFileSizeLimit) {
+
+            // Parse file as yaml
+            JsonNode cwlFile = yamlStringToJson(readFileToString(file));
+
+            // If the CWL file is packed there can be multiple workflows in a file
+            if (cwlFile.has(DOC_GRAPH)) {
+                // Packed CWL, find the first subelement which is a workflow and take it
+                for (JsonNode jsonNode : cwlFile.get(DOC_GRAPH)) {
+                    if (extractProcess(jsonNode) == CWLProcess.WORKFLOW) {
+                        cwlFile = jsonNode;
+                    }
+                }
+            }
+
+            // Can only make an overview if this is a workflow
+            if (extractProcess(cwlFile) == CWLProcess.WORKFLOW) {
+                // Use filename for label if there is no defined one
+                String label = extractLabel(cwlFile);
+                if (label == null) {
+                    label = file.getName();
+                }
+
+                // Return the constructed overview
+                return new WorkflowOverview(file.getName(), label, extractDoc(cwlFile));
+            } else {
+                // Return null if not a workflow file
+                return null;
+            }
+        } else {
+            throw new IOException("File '" + file.getName() +  "' is over singleFileSizeLimit - " +
+                    FileUtils.byteCountToDisplaySize(fileSizeBytes) + "/" +
+                    FileUtils.byteCountToDisplaySize(singleFileSizeLimit));
+        }
+
+    }
+
     /**
      * Set the format for an input or output, handling ontologies
      * @param inputOutput The input or output CWL Element

src/main/java/org/commonwl/view/git/GitDetails.java

@@ -139,12 +139,13 @@ public String getUrl() {
      * @return The URL
      */
     public String getInternalUrl() {
+        String pathPart = path.equals("/") ? "" : "/" + path;
         switch (getType()) {
             case GITHUB:
             case GITLAB:
-                return "/workflows/" + normaliseUrl(repoUrl).replace(".git", "") + "/blob/" + branch + "/" + path;
+                return "/workflows/" + normaliseUrl(repoUrl).replace(".git", "") + "/blob/" + branch + pathPart;
             default:
-                return "/workflows/" + normaliseUrl(repoUrl) + "/" + branch + "/" + path;
+                return "/workflows/" + normaliseUrl(repoUrl) + "/" + branch + pathPart;
         }
     }
 

src/main/java/org/commonwl/view/workflow/WorkflowController.java

@@ -24,6 +24,7 @@
 import org.commonwl.view.git.GitDetails;
 import org.commonwl.view.graphviz.GraphVizService;
 import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.api.errors.TransportException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -45,6 +46,7 @@
 import javax.validation.Valid;
 import java.io.File;
 import java.io.IOException;
+import java.util.List;
 
 @Controller
 public class WorkflowController {
@@ -119,7 +121,14 @@ public ModelAndView createWorkflow(@Valid WorkflowForm workflowForm, BindingResu
             Workflow workflow = workflowService.getWorkflow(gitInfo);
             if (workflow == null) {
                 try {
-                    workflow = workflowService.createQueuedWorkflow(gitInfo).getTempRepresentation();
+                    if (gitInfo.getPath().endsWith(".cwl")) {
+                        workflow = workflowService.createQueuedWorkflow(gitInfo).getTempRepresentation();
+                    } else {
+                        return new ModelAndView("redirect:" + gitInfo.getInternalUrl());
+                    }
+                } catch (TransportException ex) {
+                    bindingResult.rejectValue("url", "git.sshError");
+                    return new ModelAndView("index");
                 } catch (GitAPIException ex) {
                     bindingResult.rejectValue("url", "git.retrievalError");
                     logger.error("Git API Error", ex);
@@ -449,10 +458,24 @@ private ModelAndView getWorkflow(GitDetails gitDetails, RedirectAttributes redir
                 workflowFormValidator.validateAndParse(workflowForm, errors);
                 if (!errors.hasErrors()) {
                     try {
-                        queued = workflowService.createQueuedWorkflow(gitDetails);
+                        if (gitDetails.getPath().endsWith(".cwl")) {
+                            queued = workflowService.createQueuedWorkflow(gitDetails);
+                        } else {
+                            List<WorkflowOverview> workflowOverviews = workflowService.getWorkflowsFromDirectory(gitDetails);
+                            if (workflowOverviews.size() > 1) {
+                                return new ModelAndView("selectworkflow", "workflowOverviews", workflowOverviews)
+                                        .addObject("gitDetails", gitDetails);
+                            } else if (workflowOverviews.size() == 1) {
+                                return new ModelAndView("redirect:" + gitDetails.getInternalUrl() +
+                                        "/" + workflowOverviews.get(0).getFileName());
+                            } else {
+                                errors.rejectValue("url", "url.noWorkflowsInDirectory", "No workflow files were found in the given directory");
+                            }
+                        }
+                    } catch (TransportException ex) {
+                        errors.rejectValue("url", "git.sshError", "SSH URLs are not supported, please provide a HTTPS URL for the repository or submodules");
                     } catch (GitAPIException ex) {
                         errors.rejectValue("url", "git.retrievalError", "The workflow could not be retrieved from the Git repository using the details given");
-                        logger.error("Git API Error", ex);
                     } catch (WorkflowNotFoundException ex) {
                         errors.rejectValue("url", "git.pathTraversal", "The path given did not resolve to a location within the repository");
                     } catch (IOException ex) {

src/main/java/org/commonwl/view/workflow/WorkflowFormValidator.java

@@ -41,11 +41,19 @@ public class WorkflowFormValidator {
     private static final String GITHUB_CWL_REGEX = "^https?:\\/\\/github\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:tree|blob)\\/([^/]+)(?:\\/(.+\\.cwl))$";
     private static final Pattern githubCwlPattern = Pattern.compile(GITHUB_CWL_REGEX);
 
+    // URL validation for directories on Github
+    private static final String GITHUB_DIR_REGEX = "^https?:\\/\\/github\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:(?:tree|blob)\\/([^/]+)\\/?(.*)?)?$";
+    private static final Pattern githubDirPattern = Pattern.compile(GITHUB_DIR_REGEX);
+
     // URL validation for cwl files on Gitlab
     private static final String GITLAB_CWL_REGEX = "^https?:\\/\\/gitlab\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:tree|blob)\\/([^/]+)(?:\\/(.+\\.cwl))$";
     private static final Pattern gitlabCwlPattern = Pattern.compile(GITLAB_CWL_REGEX);
 
-    // Git URL validation
+    // URL validation for directories on Gitlab
+    private static final String GITLAB_DIR_REGEX = "^https?:\\/\\/gitlab\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:(?:tree|blob)\\/([^/]+)\\/?(.*)?)?$";
+    private static final Pattern gitlabDirPattern = Pattern.compile(GITLAB_DIR_REGEX);
+
+    // Generic Git URL validation
     private static final String GIT_REPO_REGEX = "^((git|ssh|http(s)?)|(git@[\\w\\.]+))(:(//)?)([\\w\\.@\\:/\\-~]+)(\\.git)(/)?$";
     private static final Pattern gitRepoPattern = Pattern.compile(GIT_REPO_REGEX);
 
@@ -74,9 +82,22 @@ public GitDetails validateAndParse(WorkflowForm form, Errors e) {
                 return new GitDetails(repoUrl, m.group(3), m.group(4));
             }
 
+            // Github Dir URL
+            m = githubDirPattern.matcher(form.getUrl());
+            if (m.find()) {
+                String repoUrl = "https://github.com/" + m.group(1) + "/" + m.group(2) + ".git";
+                return new GitDetails(repoUrl, m.group(3), m.group(4));
+            }
+
+            // Gitlab Dir URL
+            m = gitlabDirPattern.matcher(form.getUrl());
+            if (m.find()) {
+                String repoUrl = "https://gitlab.com/" + m.group(1) + "/" + m.group(2) + ".git";
+                return new GitDetails(repoUrl, m.group(3), m.group(4));
+            }
+
             // General Git details if didn't match the above
             ValidationUtils.rejectIfEmptyOrWhitespace(e, "branch", "branch.emptyOrWhitespace");
-            ValidationUtils.rejectIfEmptyOrWhitespace(e, "path", "path.emptyOrWhitespace");
             if (!e.hasErrors()) {
                 m = gitRepoPattern.matcher(form.getUrl());
                 if (m.find()) {

src/main/java/org/commonwl/view/workflow/WorkflowOverview.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.commonwl.view.workflow;
+
+/**
+ * Gives an overview of a workflow
+ */
+public class WorkflowOverview {
+
+    private final String fileName;
+    private final String label;
+    private final String doc;
+
+    public WorkflowOverview(String fileName, String label, String doc) {
+        this.fileName = fileName;
+        this.label = label;
+        this.doc = doc;
+    }
+
+    public String getFileName() {
+        return fileName;
+    }
+
+    public String getLabel() {
+        return label;
+    }
+
+    public String getDoc() {
+        return doc;
+    }
+
+}

src/main/java/org/commonwl/view/workflow/WorkflowService.java

@@ -42,8 +42,10 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
+import java.util.List;
 
 @Service
 public class WorkflowService {
@@ -183,6 +185,61 @@ public Workflow getWorkflow(GitDetails gitInfo) {
         return workflow;
     }
 
+    /**
+     * Get a list of workflows from a directory
+     * @param gitInfo The Git directory information
+     * @return The list of workflow overviews
+     */
+    public List<WorkflowOverview> getWorkflowsFromDirectory(GitDetails gitInfo) throws IOException, GitAPIException {
+        List<WorkflowOverview> workflowsInDir = new ArrayList<>();
+        try {
+            Git repo = null;
+            while (repo == null) {
+                try {
+                    boolean safeToAccess = gitSemaphore.acquire(gitInfo.getRepoUrl());
+                    repo = gitService.getRepository(gitInfo, safeToAccess);
+                } catch (RefNotFoundException ex) {
+                    // Attempt slashes in branch fix
+                    GitDetails correctedForSlash = transferPathToBranch(gitInfo);
+                    if (correctedForSlash != null) {
+                        gitSemaphore.release(gitInfo.getRepoUrl());
+                        gitInfo = correctedForSlash;
+                    } else {
+                        throw ex;
+                    }
+                }
+            }
+
+            Path localPath = repo.getRepository().getWorkTree().toPath();
+            Path pathToDirectory = localPath.resolve(gitInfo.getPath()).normalize().toAbsolutePath();
+            if (pathToDirectory.toString().equals("/")) {
+                pathToDirectory = localPath;
+            } else if (!pathToDirectory.startsWith(localPath.normalize().toAbsolutePath())) {
+                // Prevent path traversal attacks
+                throw new WorkflowNotFoundException();
+            }
+
+            File directory = new File(pathToDirectory.toString());
+            if (directory.exists() && directory.isDirectory()) {
+                for (final File file : directory.listFiles()) {
+                    int eIndex = file.getName().lastIndexOf('.') + 1;
+                    if (eIndex > 0) {
+                        String extension = file.getName().substring(eIndex);
+                        if (extension.equals("cwl")) {
+                            WorkflowOverview overview = cwlService.getWorkflowOverview(file);
+                            if (overview != null) {
+                                workflowsInDir.add(overview);
+                            }
+                        }
+                    }
+                }
+            }
+        } finally {
+            gitSemaphore.release(gitInfo.getRepoUrl());
+        }
+        return workflowsInDir;
+    }
+
     /**
      * Get the RO bundle for a Workflow, triggering re-download if it does not exist
      * @param gitDetails The origin details of the workflow
@@ -224,7 +281,6 @@ public QueuedWorkflow createQueuedWorkflow(GitDetails gitInfo)
         QueuedWorkflow queuedWorkflow;
 
         try {
-            // Clone repository to temporary folder
             Git repo = null;
             while (repo == null) {
                 try {

src/main/resources/messages.properties

@@ -6,3 +6,4 @@ git.retrievalError = The workflow could not be retrieved from the Git repository
 git.pathTraversal = The path given did not resolve to a location within the repository
 
 url.parsingError = An unexpected error occurred when parsing the workflow
+url.noWorkflowsInDirectory = No workflow files were found in the given directory