Change to only support direct links to workflows and spider content

Author: Mark Robinson

…nwl/viewer/domain/cwl/CWLCollection.java …commonwl/viewer/services/CWLService.javasrc/main/java/org/commonwl/viewer/domain/cwl/CWLCollection.java renamed to src/main/java/org/commonwl/viewer/services/CWLService.java src/main/java/org/commonwl/viewer/domain/cwl/CWLCollection.java renamed to src/main/java/org/commonwl/viewer/services/CWLService.java

@@ -56,8 +56,8 @@ public class GitHubService {
     private final CommitService commitService;
 
     // URL validation for directory links
-    private final String GITHUB_DIR_REGEX = "^https?:\\/\\/github\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:(?:tree|blob)\\/([^/]+)(?:\\/(.*))?)?$";
-    private final Pattern githubDirPattern = Pattern.compile(GITHUB_DIR_REGEX);
+    private final String GITHUB_CWL_REGEX = "^https?:\\/\\/github\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:tree|blob)\\/([^/]+)(?:\\/(.+\\.cwl))$";
+    private final Pattern githubCwlPattern = Pattern.compile(GITHUB_CWL_REGEX);
 
     @Autowired
     public GitHubService(@Value("${githubAPI.authentication}") String authSetting,
@@ -75,12 +75,12 @@ public GitHubService(@Value("${githubAPI.authentication}") String authSetting,
     }
 
     /**
-     * Extract the details of a Github directory URL using a regular expression
-     * @param url The Github directory URL
+     * Extract the details of a Github cwl file URL using a regular expression
+     * @param url The Github URL to a cwl file
      * @return A list with the groups of the regex match, [owner, repo, branch, path]
      */
-    public GithubDetails detailsFromDirURL(String url) {
-        Matcher m = githubDirPattern.matcher(url);
+    public GithubDetails detailsFromCwlURL(String url) {
+        Matcher m = githubCwlPattern.matcher(url);
         if (m.find()) {
             return new GithubDetails(m.group(1), m.group(2), m.group(3), m.group(4));
         }
@@ -120,20 +120,28 @@ public String downloadFile(GithubDetails githubInfo, String sha) throws IOExcept
     }
 
     /**
-     * Gets the latest commit sha hash from a particular branch
+     * Download a single file from a repository
+     * @param githubInfo The information to access the repository
+     * @return A string with the contents of the file
+     * @throws IOException Any API errors which may have occurred
+     */
+    public String downloadFile(GithubDetails githubInfo) throws IOException {
+        return downloadFile(githubInfo, githubInfo.getBranch());
+    }
+
+    /**
+     * Gets the latest commit sha hash for a file on a branch
      * If hash is given as a branch, this will return the same hash if valid
      * @param githubInfo The information to access the repository
-     * @return A sha hash for the latest commit
+     * @return A sha hash for the latest commit on a file
      * @throws IOException Any API errors which may have occurred
      */
     public String getCommitSha(GithubDetails githubInfo) throws IOException {
         RepositoryId repo = new RepositoryId(githubInfo.getOwner(), githubInfo.getRepoName());
-        RepositoryCommit currentCommit = commitService.getCommit(repo, githubInfo.getBranch());
-        return currentCommit.getSha();
+        return commitService.getCommits(repo, githubInfo.getBranch(), githubInfo.getPath()).get(0).getSha();
     }
 
     /**
-     *
      * Get the contributors to a specific file by their commits
      * @param githubInfo The information to access the repository
      * @return A list of unique contributors

src/main/java/org/commonwl/viewer/services/GitHubService.java

@@ -21,7 +21,6 @@
 
 import org.commonwl.viewer.domain.GithubDetails;
 import org.commonwl.viewer.domain.WorkflowForm;
-import org.eclipse.egit.github.core.RepositoryContents;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -30,7 +29,6 @@
 import org.springframework.validation.ValidationUtils;
 
 import java.io.IOException;
-import java.util.List;
 
 /**
  * Runs validation on the workflow form from the main page
@@ -51,7 +49,7 @@ public WorkflowFormValidator(GitHubService githubService) {
     }
 
     /**
-     * Validates a WorkflowForm to ensure the URL is not empty and directory contains cwl files
+     * Validates a WorkflowForm to ensure the URL is not empty and links to a cwl file
      * @param form The given WorkflowForm
      * @param e Any errors from validation
      */
@@ -60,30 +58,23 @@ public GithubDetails validateAndParse(WorkflowForm form, Errors e) {
 
         // Only continue if not null and isn't just whitespace
         if (!e.hasErrors()) {
-            GithubDetails githubInfo = githubService.detailsFromDirURL(form.getGithubURL());
+            GithubDetails githubInfo = githubService.detailsFromCwlURL(form.getGithubURL());
 
             // If the URL is valid and details could be extracted
             if (githubInfo != null) {
 
                 // Check the repository exists and get content to ensure that branch/path exist
                 try {
-                    // Return the Github information if it is valid
-                    List<RepositoryContents> repoContents = githubService.getContents(githubInfo);
-                    if (containsCWL(repoContents, githubInfo)) {
+                    // Downloads the workflow file to check for existence
+                    if (githubService.downloadFile(githubInfo) != null) {
                         return githubInfo;
-                    } else {
-                        // The URL does not contain any .cwl files
-                        logger.error("No .cwl files found at Github URL");
-                        e.rejectValue("githubURL", "githubURL.missingWorkflow", "No .cwl files were found in the specified Github directory");
                     }
                 } catch (IOException ex) {
-                    // Given repository/branch/path does not exist or API error occured
-                    logger.error("Github API Error", ex);
-                    e.rejectValue("githubURL", "githubURL.apiError", "API Error - does the specified Github directory exist?");
+                    logger.error("Given URL " + form.getGithubURL() + " was not a .cwl file");
+                    e.rejectValue("githubURL", "githubURL.missingWorkflow", "Workflow was not found at the given Github URL");
                 }
             } else {
-                // The Github URL is not valid
-                logger.error("The Github URL is not valid");
+                logger.error("The Github URL " + form.getGithubURL() + " is not valid");
                 e.rejectValue("githubURL", "githubURL.invalid");
             }
         } else {
@@ -93,34 +84,4 @@ public GithubDetails validateAndParse(WorkflowForm form, Errors e) {
         // Errors will stop this being used anyway
         return null;
     }
-
-    /**
-     * Recursively check for CWL files within a Github repository
-     * @param repoContents A list of contents within a path in the repository
-     * @param githubDetails The details of the repository
-     * @return Whether the path contains CWL files
-     * @throws IOException Any API errors which may have occurred
-     */
-    private boolean containsCWL(List<RepositoryContents> repoContents, GithubDetails githubDetails) throws IOException {
-        // Check that there is at least 1 .cwl file
-        for (RepositoryContents repoContent : repoContents) {
-            if (repoContent.getType().equals(GitHubService.TYPE_DIR)) {
-                GithubDetails githubSubdir = new GithubDetails(githubDetails.getOwner(),
-                        githubDetails.getRepoName(), githubDetails.getBranch(), repoContent.getPath());
-                List<RepositoryContents> subdirectory = githubService.getContents(githubSubdir);
-                if (containsCWL(subdirectory, githubSubdir)) {
-                    return true;
-                }
-            } else if (repoContent.getType().equals(GitHubService.TYPE_FILE)) {
-                int eIndex = repoContent.getName().lastIndexOf('.') + 1;
-                if (eIndex > 0) {
-                    String extension = repoContent.getName().substring(eIndex);
-                    if (extension.equals("cwl")) {
-                        return true;
-                    }
-                }
-            }
-        }
-        return false;
-    }
 }

src/main/java/org/commonwl/viewer/services/WorkflowFormValidator.java

@@ -37,15 +37,6 @@ public interface WorkflowRepository extends PagingAndSortingRepository<Workflow,
      */
     Workflow findByRetrievedFrom(GithubDetails retrievedFrom);
 
-    /**
-     * Finds a workflow model in the database based on which repository it is from
-     * @param owner The owner of the repository
-     * @param repoName The name of the repository
-     * @param branch The branch or commit ID of the repository
-     * @return The workflow model
-     */
-    Workflow findByRetrievedFromOwnerAndRetrievedFromRepoNameAndRetrievedFromBranch(String owner, String repoName, String branch);
-
     /**
      * Paged request to get all workflows
      * @param pageable The details of the page to be retrieved

src/main/java/org/commonwl/viewer/services/WorkflowRepository.java

@@ -21,7 +21,6 @@
 
 import org.commonwl.viewer.domain.GithubDetails;
 import org.commonwl.viewer.domain.Workflow;
-import org.commonwl.viewer.domain.cwl.CWLCollection;
 import org.commonwl.viewer.web.ROBundleNotFoundException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -41,28 +40,25 @@ public class WorkflowService {
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
 
     private final GitHubService githubService;
+    private final CWLService cwlService;
     private final WorkflowRepository workflowRepository;
     private final ROBundleFactory ROBundleFactory;
     private final GraphVizService graphVizService;
     private final int cacheDays;
-    private final int singleFileSizeLimit;
-    private final int totalFileSizeLimit;
 
     @Autowired
     public WorkflowService(GitHubService githubService,
+                           CWLService cwlService,
                            WorkflowRepository workflowRepository,
                            ROBundleFactory ROBundleFactory,
                            GraphVizService graphVizService,
-                           @Value("${cacheDays}") int cacheDays,
-                           @Value("${singleFileSizeLimit}") int singleFileSizeLimit,
-                           @Value("${totalFileSizeLimit}") int totalFileSizeLimit) {
+                           @Value("${cacheDays}") int cacheDays) {
         this.githubService = githubService;
+        this.cwlService = cwlService;
         this.workflowRepository = workflowRepository;
         this.ROBundleFactory = ROBundleFactory;
         this.graphVizService = graphVizService;
         this.cacheDays = cacheDays;
-        this.singleFileSizeLimit = singleFileSizeLimit;
-        this.totalFileSizeLimit = totalFileSizeLimit;
     }
 
     /**
@@ -145,7 +141,7 @@ public File getROBundle(String id) throws ROBundleNotFoundException {
     }
 
     /**
-     * Builds a new workflow from cwl files fetched from Github
+     * Builds a new workflow from Github
      * @param githubInfo Github information for the workflow
      * @return The constructed model for the Workflow
      */
@@ -154,12 +150,9 @@ public Workflow createWorkflow(GithubDetails githubInfo) {
             // Get the sha hash from a branch reference
             String latestCommit = githubService.getCommitSha(githubInfo);
 
-            // Set up CWL utility to collect the documents
-            CWLCollection cwlFiles = new CWLCollection(githubService, githubInfo, latestCommit,
-                    singleFileSizeLimit, totalFileSizeLimit);
+            // Get the workflow model using the cwl service
+            Workflow workflowModel = cwlService.parseWorkflow(githubInfo, latestCommit);
 
-            // Get the workflow model
-            Workflow workflowModel = cwlFiles.getWorkflow();
             if (workflowModel != null) {
                 // Set origin details
                 workflowModel.setRetrievedOn(new Date());
@@ -256,14 +249,4 @@ private boolean cacheExpired(Workflow workflow) {
             return false;
         }
     }
-
-    /**
-     * Check for the repository already being parsed
-     * @param details The details of the repository on Github
-     * @return Whether the workflows from the repository have already been added
-     */
-    private boolean repoAlreadyParsed(GithubDetails details) {
-        return (workflowRepository.findByRetrievedFromOwnerAndRetrievedFromRepoNameAndRetrievedFromBranch(
-                details.getOwner(), details.getRepoName(), details.getBranch()) != null);
-    }
 }

src/main/java/org/commonwl/viewer/services/WorkflowService.java

@@ -115,13 +115,9 @@ public ModelAndView newWorkflowFromGithubURL(@Valid WorkflowForm workflowForm, B
 
             // Redirect to the workflow
             GithubDetails githubDetails = workflow.getRetrievedFrom();
-            String path = githubDetails.getPath();
-            if (path == null) {
-                path = "";
-            }
             return new ModelAndView("redirect:/workflows/github.com/" + githubDetails.getOwner()
                     + "/" + githubDetails.getRepoName() + "/tree/" + githubDetails.getBranch()
-                    + "/" + path);
+                    + "/" + githubDetails.getPath());
         }
     }
 
@@ -183,8 +179,6 @@ public ModelAndView getWorkflowByGithubDetails(@Value("${applicationURL}") Strin
     public FileSystemResource downloadROBundle(@PathVariable("workflowID") String workflowID,
                                                HttpServletResponse response) {
         File bundleDownload = workflowService.getROBundle(workflowID);
-
-        logger.info("Serving download for workflow " + workflowID + " [" + bundleDownload.toString() + "]");
         response.setHeader("Content-Disposition", "attachment; filename=bundle.zip;");
         return new FileSystemResource(bundleDownload);
     }

src/main/java/org/commonwl/viewer/web/WorkflowController.java

@@ -1,6 +1,5 @@
 # Errors from validation of a github URL
 githubURL.emptyOrWhitespace = URL may not be empty
-githubURL.invalid = You must enter a valid Github directory URL
-githubURL.apiError = API Error - does the specified Github directory exist?
-githubURL.missingWorkflow = No .cwl files were found in the specified Github directory
+githubURL.invalid = You must enter a valid Github URL to a .cwl file
+githubURL.missingWorkflow = Workflow was not found at the given Github URL
 githubURL.parsingError = The workflow could not be parsed from the given URL

src/main/resources/messages.properties

@@ -35,7 +35,7 @@
                 <div class="col-md-12" role="main" id="main">
                     <h1>Common Workflow Language Viewer</h1>
                     <p>This tool visualises and lists the details of a workflow with its inputs, outputs and steps and
-                        packages the files into a downloadable
+                        packages the files involved into a downloadable
                         <a href="https://researchobject.github.io/specifications/bundle/" rel="noopener" target="_blank">
                             Research Object Bundle</a>
                         (zip file with metadata in a manifest), allowing it to be easily viewed and shared.</p>
@@ -44,7 +44,7 @@ <h1>Common Workflow Language Viewer</h1>
                     </div>
 
                     <h2>Workflow from Github URL</h2>
-                    <p>Provide a link to the directory of files making up your workflow here</p>
+                    <p>Provide a Github link to the workflow here</p>
                     <form id="add" action="#" th:action="@{/}" th:object="${workflowForm}" method="POST">
                         <div class="alert alert-danger" th:if="${errors} != null and ${errors.hasErrors()}">
                             <div th:each="error : ${errors.getAllErrors()}">
@@ -56,10 +56,10 @@ <h2>Workflow from Github URL</h2>
                         </div>
                         <div class="alert alert-grey">
                             <strong>Don't know what to view?</strong> Try these from <i>common-workflow-language/workflows</i>:
-                            <a class="example" href="https://github.com/common-workflow-language/workflows/tree/master/workflows/compile">compile</a>,
-                            <a class="example" href="https://github.com/common-workflow-language/workflows/tree/master/workflows/make-to-cwl">make-to-cwl</a>,
-                            <a class="example" href="https://github.com/common-workflow-language/workflows/tree/master/workflows/lobSTR">lobSTR</a>,
-                            <a class="example" href="https://github.com/common-workflow-language/workflows/tree/master/workflows/scidap">scidap</a>
+                            <a class="example" href="https://github.com/common-workflow-language/workflows/tree/master/workflows/compile/compile1.cwl">compile</a>,
+                            <a class="example" href="https://github.com/common-workflow-language/workflows/tree/master/workflows/make-to-cwl/dna.cwl">make-to-cwl</a>,
+                            <a class="example" href="https://github.com/common-workflow-language/workflows/tree/master/workflows/lobSTR/lobSTR-workflow.cwl">lobSTR</a>,
+                            <a class="example" href="https://github.com/common-workflow-language/workflows/tree/master/workflows/scidap/bam-genomecov-bigwig-rna-dutp.cwl">scidap</a>
                             or <a href="/workflows">explore the collection</a>
                         </div>
                         <div class="input-group">