Merge remote-tracking branch 'origin/master' into api-fixes

Author: MarkRobbo

src/main/java/org/commonwl/view/cwl/CWLService.java

@@ -39,6 +39,7 @@
 import org.commonwl.view.graphviz.ModelDotWriter;
 import org.commonwl.view.graphviz.RDFDotWriter;
 import org.commonwl.view.workflow.Workflow;
+import org.commonwl.view.workflow.WorkflowOverview;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -428,6 +429,55 @@ public Workflow parseWorkflowWithCwltool(Workflow basicModel,
 
     }
 
+    /**
+     * Get an overview of a workflow
+     * @param file A file, potentially a workflow
+     * @return A constructed WorkflowOverview of the workflow
+     * @throws IOException Any API errors which may have occurred
+     */
+    public WorkflowOverview getWorkflowOverview(File file) throws IOException {
+
+        // Get the content of this file from Github
+        long fileSizeBytes = file.length();
+
+        // Check file size limit before parsing
+        if (fileSizeBytes <= singleFileSizeLimit) {
+
+            // Parse file as yaml
+            JsonNode cwlFile = yamlStringToJson(readFileToString(file));
+
+            // If the CWL file is packed there can be multiple workflows in a file
+            if (cwlFile.has(DOC_GRAPH)) {
+                // Packed CWL, find the first subelement which is a workflow and take it
+                for (JsonNode jsonNode : cwlFile.get(DOC_GRAPH)) {
+                    if (extractProcess(jsonNode) == CWLProcess.WORKFLOW) {
+                        cwlFile = jsonNode;
+                    }
+                }
+            }
+
+            // Can only make an overview if this is a workflow
+            if (extractProcess(cwlFile) == CWLProcess.WORKFLOW) {
+                // Use filename for label if there is no defined one
+                String label = extractLabel(cwlFile);
+                if (label == null) {
+                    label = file.getName();
+                }
+
+                // Return the constructed overview
+                return new WorkflowOverview(file.getName(), label, extractDoc(cwlFile));
+            } else {
+                // Return null if not a workflow file
+                return null;
+            }
+        } else {
+            throw new IOException("File '" + file.getName() +  "' is over singleFileSizeLimit - " +
+                    FileUtils.byteCountToDisplaySize(fileSizeBytes) + "/" +
+                    FileUtils.byteCountToDisplaySize(singleFileSizeLimit));
+        }
+
+    }
+
     /**
      * Set the format for an input or output, handling ontologies
      * @param inputOutput The input or output CWL Element

src/main/java/org/commonwl/view/git/GitDetails.java

@@ -139,12 +139,13 @@ public String getUrl() {
      * @return The URL
      */
     public String getInternalUrl() {
+        String pathPart = path.equals("/") ? "" : "/" + path;
         switch (getType()) {
             case GITHUB:
             case GITLAB:
-                return "/workflows/" + normaliseUrl(repoUrl).replace(".git", "") + "/blob/" + branch + "/" + path;
+                return "/workflows/" + normaliseUrl(repoUrl).replace(".git", "") + "/blob/" + branch + pathPart;
             default:
-                return "/workflows/" + normaliseUrl(repoUrl) + "/" + branch + "/" + path;
+                return "/workflows/" + normaliseUrl(repoUrl) + "/" + branch + pathPart;
         }
     }
 

src/main/java/org/commonwl/view/workflow/WorkflowController.java

@@ -24,6 +24,7 @@
 import org.commonwl.view.git.GitDetails;
 import org.commonwl.view.graphviz.GraphVizService;
 import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.api.errors.TransportException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -45,6 +46,7 @@
 import javax.validation.Valid;
 import java.io.File;
 import java.io.IOException;
+import java.util.List;
 
 @Controller
 public class WorkflowController {
@@ -119,7 +121,14 @@ public ModelAndView createWorkflow(@Valid WorkflowForm workflowForm, BindingResu
             Workflow workflow = workflowService.getWorkflow(gitInfo);
             if (workflow == null) {
                 try {
-                    workflow = workflowService.createQueuedWorkflow(gitInfo).getTempRepresentation();
+                    if (gitInfo.getPath().endsWith(".cwl")) {
+                        workflow = workflowService.createQueuedWorkflow(gitInfo).getTempRepresentation();
+                    } else {
+                        return new ModelAndView("redirect:" + gitInfo.getInternalUrl());
+                    }
+                } catch (TransportException ex) {
+                    bindingResult.rejectValue("url", "git.sshError");
+                    return new ModelAndView("index");
                 } catch (GitAPIException ex) {
                     bindingResult.rejectValue("url", "git.retrievalError");
                     logger.error("Git API Error", ex);
@@ -449,10 +458,24 @@ private ModelAndView getWorkflow(GitDetails gitDetails, RedirectAttributes redir
                 workflowFormValidator.validateAndParse(workflowForm, errors);
                 if (!errors.hasErrors()) {
                     try {
-                        queued = workflowService.createQueuedWorkflow(gitDetails);
+                        if (gitDetails.getPath().endsWith(".cwl")) {
+                            queued = workflowService.createQueuedWorkflow(gitDetails);
+                        } else {
+                            List<WorkflowOverview> workflowOverviews = workflowService.getWorkflowsFromDirectory(gitDetails);
+                            if (workflowOverviews.size() > 1) {
+                                return new ModelAndView("selectworkflow", "workflowOverviews", workflowOverviews)
+                                        .addObject("gitDetails", gitDetails);
+                            } else if (workflowOverviews.size() == 1) {
+                                return new ModelAndView("redirect:" + gitDetails.getInternalUrl() +
+                                        "/" + workflowOverviews.get(0).getFileName());
+                            } else {
+                                errors.rejectValue("url", "url.noWorkflowsInDirectory", "No workflow files were found in the given directory");
+                            }
+                        }
+                    } catch (TransportException ex) {
+                        errors.rejectValue("url", "git.sshError", "SSH URLs are not supported, please provide a HTTPS URL for the repository or submodules");
                     } catch (GitAPIException ex) {
                         errors.rejectValue("url", "git.retrievalError", "The workflow could not be retrieved from the Git repository using the details given");
-                        logger.error("Git API Error", ex);
                     } catch (WorkflowNotFoundException ex) {
                         errors.rejectValue("url", "git.pathTraversal", "The path given did not resolve to a location within the repository");
                     } catch (IOException ex) {

src/main/java/org/commonwl/view/workflow/WorkflowFormValidator.java

@@ -41,11 +41,19 @@ public class WorkflowFormValidator {
     private static final String GITHUB_CWL_REGEX = "^https?:\\/\\/github\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:tree|blob)\\/([^/]+)(?:\\/(.+\\.cwl))$";
     private static final Pattern githubCwlPattern = Pattern.compile(GITHUB_CWL_REGEX);
 
+    // URL validation for directories on Github
+    private static final String GITHUB_DIR_REGEX = "^https?:\\/\\/github\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:(?:tree|blob)\\/([^/]+)\\/?(.*)?)?$";
+    private static final Pattern githubDirPattern = Pattern.compile(GITHUB_DIR_REGEX);
+
     // URL validation for cwl files on Gitlab
     private static final String GITLAB_CWL_REGEX = "^https?:\\/\\/gitlab\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:tree|blob)\\/([^/]+)(?:\\/(.+\\.cwl))$";
     private static final Pattern gitlabCwlPattern = Pattern.compile(GITLAB_CWL_REGEX);
 
-    // Git URL validation
+    // URL validation for directories on Gitlab
+    private static final String GITLAB_DIR_REGEX = "^https?:\\/\\/gitlab\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:(?:tree|blob)\\/([^/]+)\\/?(.*)?)?$";
+    private static final Pattern gitlabDirPattern = Pattern.compile(GITLAB_DIR_REGEX);
+
+    // Generic Git URL validation
     private static final String GIT_REPO_REGEX = "^((git|ssh|http(s)?)|(git@[\\w\\.]+))(:(//)?)([\\w\\.@\\:/\\-~]+)(\\.git)(/)?$";
     private static final Pattern gitRepoPattern = Pattern.compile(GIT_REPO_REGEX);
 
@@ -74,9 +82,22 @@ public GitDetails validateAndParse(WorkflowForm form, Errors e) {
                 return new GitDetails(repoUrl, m.group(3), m.group(4));
             }
 
+            // Github Dir URL
+            m = githubDirPattern.matcher(form.getUrl());
+            if (m.find()) {
+                String repoUrl = "https://github.com/" + m.group(1) + "/" + m.group(2) + ".git";
+                return new GitDetails(repoUrl, m.group(3), m.group(4));
+            }
+
+            // Gitlab Dir URL
+            m = gitlabDirPattern.matcher(form.getUrl());
+            if (m.find()) {
+                String repoUrl = "https://gitlab.com/" + m.group(1) + "/" + m.group(2) + ".git";
+                return new GitDetails(repoUrl, m.group(3), m.group(4));
+            }
+
             // General Git details if didn't match the above
             ValidationUtils.rejectIfEmptyOrWhitespace(e, "branch", "branch.emptyOrWhitespace");
-            ValidationUtils.rejectIfEmptyOrWhitespace(e, "path", "path.emptyOrWhitespace");
             if (!e.hasErrors()) {
                 m = gitRepoPattern.matcher(form.getUrl());
                 if (m.find()) {

src/main/java/org/commonwl/view/workflow/WorkflowOverview.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.commonwl.view.workflow;
+
+/**
+ * Gives an overview of a workflow
+ */
+public class WorkflowOverview {
+
+    private final String fileName;
+    private final String label;
+    private final String doc;
+
+    public WorkflowOverview(String fileName, String label, String doc) {
+        this.fileName = fileName;
+        this.label = label;
+        this.doc = doc;
+    }
+
+    public String getFileName() {
+        return fileName;
+    }
+
+    public String getLabel() {
+        return label;
+    }
+
+    public String getDoc() {
+        return doc;
+    }
+
+}

src/main/java/org/commonwl/view/workflow/WorkflowService.java

@@ -42,8 +42,10 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
+import java.util.List;
 
 @Service
 public class WorkflowService {
@@ -183,6 +185,61 @@ public Workflow getWorkflow(GitDetails gitInfo) {
         return workflow;
     }
 
+    /**
+     * Get a list of workflows from a directory
+     * @param gitInfo The Git directory information
+     * @return The list of workflow overviews
+     */
+    public List<WorkflowOverview> getWorkflowsFromDirectory(GitDetails gitInfo) throws IOException, GitAPIException {
+        List<WorkflowOverview> workflowsInDir = new ArrayList<>();
+        try {
+            Git repo = null;
+            while (repo == null) {
+                try {
+                    boolean safeToAccess = gitSemaphore.acquire(gitInfo.getRepoUrl());
+                    repo = gitService.getRepository(gitInfo, safeToAccess);
+                } catch (RefNotFoundException ex) {
+                    // Attempt slashes in branch fix
+                    GitDetails correctedForSlash = transferPathToBranch(gitInfo);
+                    if (correctedForSlash != null) {
+                        gitSemaphore.release(gitInfo.getRepoUrl());
+                        gitInfo = correctedForSlash;
+                    } else {
+                        throw ex;
+                    }
+                }
+            }
+
+            Path localPath = repo.getRepository().getWorkTree().toPath();
+            Path pathToDirectory = localPath.resolve(gitInfo.getPath()).normalize().toAbsolutePath();
+            if (pathToDirectory.toString().equals("/")) {
+                pathToDirectory = localPath;
+            } else if (!pathToDirectory.startsWith(localPath.normalize().toAbsolutePath())) {
+                // Prevent path traversal attacks
+                throw new WorkflowNotFoundException();
+            }
+
+            File directory = new File(pathToDirectory.toString());
+            if (directory.exists() && directory.isDirectory()) {
+                for (final File file : directory.listFiles()) {
+                    int eIndex = file.getName().lastIndexOf('.') + 1;
+                    if (eIndex > 0) {
+                        String extension = file.getName().substring(eIndex);
+                        if (extension.equals("cwl")) {
+                            WorkflowOverview overview = cwlService.getWorkflowOverview(file);
+                            if (overview != null) {
+                                workflowsInDir.add(overview);
+                            }
+                        }
+                    }
+                }
+            }
+        } finally {
+            gitSemaphore.release(gitInfo.getRepoUrl());
+        }
+        return workflowsInDir;
+    }
+
     /**
      * Get the RO bundle for a Workflow, triggering re-download if it does not exist
      * @param gitDetails The origin details of the workflow
@@ -224,7 +281,6 @@ public QueuedWorkflow createQueuedWorkflow(GitDetails gitInfo)
         QueuedWorkflow queuedWorkflow;
 
         try {
-            // Clone repository to temporary folder
             Git repo = null;
             while (repo == null) {
                 try {

src/main/resources/messages.properties

@@ -4,5 +4,7 @@ path.emptyOrWhitespace = You must provide a path to the workflow within the repo
 
 git.retrievalError = The workflow could not be retrieved from the Git repository using the details given
 git.pathTraversal = The path given did not resolve to a location within the repository
+git.sshError = SSH URLs are not supported, please provide a HTTPS URL for the repository or submodules
 
 url.parsingError = An unexpected error occurred when parsing the workflow
+url.noWorkflowsInDirectory = No workflow files were found in the given directory

src/main/resources/static/js/main.js

@@ -23,7 +23,7 @@
 requirejs.config({
     baseUrl: '/bower_components',
     paths: {
-        'jquery': 'jquery/dist/jquery.min',
+        'jquery': 'jquery/dist/jquery.min'
     }
 });
 
@@ -43,7 +43,7 @@ require(['jquery'],
  */
 require(['jquery'],
     function ($) {
-        var generalPattern = "\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:tree|blob)\\/([^/]+)(?:\\/(.+\\.cwl))$";
+        var generalPattern = "\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:(?:tree|blob)\\/([^/]+)\\/?(.*)?)?$";
         var githubPattern = new RegExp("^https?:\\/\\/github\\.com" + generalPattern);
         var gitlabPattern = new RegExp("^https?:\\/\\/gitlab\\.com" + generalPattern);
         var gitPattern = new RegExp("^((git|ssh|http(s)?)|(git@[\\w\\.]+))(:(//)?)([\\w\\.@\\:/\\-~]+)(\\.git)(/)?$");
@@ -76,24 +76,25 @@ require(['jquery'],
          * Validate form before submit
          */
         $('#add').submit(function() {
-            var pathPattern = new RegExp("^\\/?([^\\/]*\\/)*[^\\/]+\\.cwl$");
             var input = $("#url").val();
             if (gitPattern.test(input)) {
-                var success = true;
-                if (!$("#branch").val()) {
-                    addWarning("branch", "You must provide a branch name for the workflow");
-                    success = false;
-                }
-                if (!$("#path").val()) {
-                    addWarning("path", "You must provide a path to the workflow");
-                    success = false;
-                } else if (!pathPattern.test($("#path").val())) {
-                    addWarning("path", "Must be a valid path from the root to a .cwl workflow");
-                    success = false;
+                var success = false;
+                if (input.startsWith("ssh") || input.startsWith("git@")) {
+                    addWarning("url", "SSH is not supported as a protocol, please provide a HTTPS URL to clone");
+                } else {
+                    success = true;
+                    if (!$("#branch").val()) {
+                        addWarning("branch", "You must provide a branch name for the workflow");
+                        success = false;
+                    }
+                    if (!$("#path").val()) {
+                        addWarning("path", "You must provide a path to the workflow or a directory of workflows");
+                        success = false;
+                    }
                 }
                 return success;
             } else if (!githubPattern.test(input) && !gitlabPattern.test(input)) {
-                addWarning("url", "Must be a URL to a workflow on Gitlab or Github, or a Git repository URL");
+                addWarning("url", "Must be a URL to a workflow or directory of workflows on Gitlab or Github, or a Git repository URL");
                 return false;
             }
         });

src/main/resources/templates/about.html

@@ -187,6 +187,11 @@ <h2>Research Objects</h2>
                 (<a href="https://github.com/common-workflow-language/cwlviewer/issues/103">see Github issue</a>)</p>
             <p>We recommend that you keep all files in the containing folder for current use of CWL Viewer</p>
 
+            <h2>SSH Cloning</h2>
+            <p>SSH URLs are not able to be cloned or used as submodules due to the need for SSH keys to be set up.</p>
+            <p>We do not plan SSH support due to the impact on reproducibility from this being made a required
+            step to download the workflow.</p>
+
             <h2>Others</h2>
             <p>Other limitations or unimplemented features can be viewed
                 <a href="https://github.com/common-workflow-language/cwlviewer">on the Github issues page</a></p>