Refactoring to enable URL manipulation to switch workflows

Also move various pieces of application logic out of controller layer

Author: Mark Robinson

src/main/java/org/commonwl/viewer/domain/cwl/CWLCollection.java

@@ -197,9 +197,6 @@ private void addDoc(JsonNode newDoc, String fileName) {
      * @return The file name/key of the workflow
      */
     private String findMainWorkflow() {
-        // TODO: make this path dependant so it doesn't get messed up by duplicate filenames or graphs
-        // Currently this strategy fails gracefully and returns the first workflow in the case of a graph
-
         // Store the in degree of each workflow
         Map<String, Integer> inDegrees = new HashMap<String, Integer>();
         for (Map.Entry<String, JsonNode> doc : cwlDocs.entrySet()) {
@@ -301,8 +298,6 @@ private void fillStepRunTypes(Workflow workflow) {
                     if (cwlDocs.containsKey(filePath.toString())) {
                         JsonNode runDoc = cwlDocs.get(filePath.toString());
                         step.setRunType(extractProcess(runDoc));
-                    }  else {
-                        step.setRun(null);
                     }
                 }
             }

src/main/java/org/commonwl/viewer/services/GitHubService.java

@@ -26,7 +26,6 @@
 import org.eclipse.egit.github.core.client.GitHubClient;
 import org.eclipse.egit.github.core.service.CommitService;
 import org.eclipse.egit.github.core.service.ContentsService;
-import org.eclipse.egit.github.core.service.UserService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
@@ -54,7 +53,6 @@ public class GitHubService {
 
     // Github API services
     private final ContentsService contentsService;
-    private final UserService userService;
     private final CommitService commitService;
 
     // URL validation for directory links
@@ -73,7 +71,6 @@ public GitHubService(@Value("${githubAPI.authentication}") String authSetting,
             client.setOAuth2Token(token);
         }
         this.contentsService = new ContentsService(client);
-        this.userService = new UserService(client);
         this.commitService = new CommitService(client);
     }
 
@@ -101,16 +98,6 @@ public List<RepositoryContents> getContents(GithubDetails githubInfo) throws IOE
                 githubInfo.getPath(), githubInfo.getBranch());
     }
 
-    /**
-     * Get the details of a user from the Github API
-     * @param username The username of the user we want to get data about
-     * @return A user object containing the API information
-     * @throws IOException Any API errors which may have occurred
-     */
-    public User getUser(String username) throws IOException {
-        return userService.getUser(username);
-    }
-
     /**
      * Download a single file from a Github repository
      * @param githubInfo The information to access the repository

src/main/java/org/commonwl/viewer/services/WorkflowFormValidator.java

@@ -20,15 +20,14 @@
 package org.commonwl.viewer.services;
 
 import org.commonwl.viewer.domain.GithubDetails;
-import org.eclipse.egit.github.core.RepositoryContents;
 import org.commonwl.viewer.domain.WorkflowForm;
+import org.eclipse.egit.github.core.RepositoryContents;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.springframework.validation.Errors;
 import org.springframework.validation.ValidationUtils;
-import org.springframework.validation.Validator;
 
 import java.io.IOException;
 import java.util.List;
@@ -87,12 +86,12 @@ public GithubDetails validateAndParse(WorkflowForm form, Errors e) {
                     } else {
                         // The URL does not contain any .cwl files
                         logger.error("No .cwl files found at Github URL");
-                        e.rejectValue("githubURL", "githubURL.missingWorkflow");
+                        e.rejectValue("githubURL", "githubURL.missingWorkflow", "No .cwl files were found in the specified Github directory");
                     }
                 } catch (IOException ex) {
                     // Given repository/branch/path does not exist or API error occured
-                    logger.error(ex.getMessage());
-                    e.rejectValue("githubURL", "githubURL.apiError");
+                    logger.error("Github API Error", ex);
+                    e.rejectValue("githubURL", "githubURL.apiError", "API Error - does the specified Github directory exist?");
                 }
             } else {
                 // The Github URL is not valid

src/main/java/org/commonwl/viewer/services/WorkflowService.java

@@ -22,10 +22,13 @@
 import org.commonwl.viewer.domain.GithubDetails;
 import org.commonwl.viewer.domain.Workflow;
 import org.commonwl.viewer.domain.cwl.CWLCollection;
+import org.commonwl.viewer.web.ROBundleNotFoundException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
 import org.springframework.stereotype.Service;
 
 import java.io.File;
@@ -62,13 +65,91 @@ public WorkflowService(GitHubService githubService,
         this.totalFileSizeLimit = totalFileSizeLimit;
     }
 
+    /**
+     * Gets a page of all workflows from the database
+     * @param pageable The details of the page to be requested
+     * @return The resulting page of the workflow entries
+     */
+    public Page<Workflow> getPageOfWorkflows(Pageable pageable) {
+        return workflowRepository.findAllByOrderByRetrievedOnDesc(pageable);
+    }
+
+    /**
+     * Get a workflow from the database by its ID
+     * @param id The ID of the workflow
+     * @return The model for the workflow
+     */
+    public Workflow getWorkflow(String id) {
+        return workflowRepository.findOne(id);
+    }
+
+    /**
+     * Get a workflow from the database, refreshing it if cache has expired
+     * @param githubInfo Github information for the workflow
+     * @return The workflow model associated with githubInfo
+     */
+    public Workflow getWorkflow(GithubDetails githubInfo) {
+        // Check database for existing workflow
+        Workflow workflow = workflowRepository.findByRetrievedFrom(githubInfo);
+
+        // Create a new workflow if we do not have one already
+        if (workflow != null) {
+            // Delete the existing workflow if the cache has expired
+            if (cacheExpired(workflow)) {
+                // Update by trying to add a new workflow
+                Workflow newWorkflow = createWorkflow(workflow.getRetrievedFrom());
+
+                // Only replace workflow if it could be successfully parsed
+                if (newWorkflow == null) {
+                    logger.error("Could not parse updated workflow " + workflow.getID());
+                } else {
+                    // Delete the existing workflow
+                    removeWorkflow(workflow);
+
+                    // Save new workflow
+                    workflowRepository.save(newWorkflow);
+                    workflow = newWorkflow;
+                }
+            }
+        }
+
+        return workflow;
+    }
+
+    /**
+     * Get the RO bundle for a Workflow, triggering re-download if it does not exist
+     * @param id The ID of the workflow
+     * @return The file containing the RO bundle
+     * @throws ROBundleNotFoundException If the RO bundle was not found
+     */
+    public File getROBundle(String id) throws ROBundleNotFoundException {
+        // Get workflow from database
+        Workflow workflow = getWorkflow(id);
+
+        // If workflow does not exist or the bundle doesn't yet
+        if (workflow == null || workflow.getRoBundle() == null) {
+            throw new ROBundleNotFoundException();
+        }
+
+        // 404 error with retry if the file on disk does not exist
+        File bundleDownload = new File(workflow.getRoBundle());
+        if (!bundleDownload.exists()) {
+            // Clear current RO bundle link and create a new one (async)
+            workflow.setRoBundle(null);
+            workflowRepository.save(workflow);
+            generateROBundle(workflow);
+            throw new ROBundleNotFoundException();
+        }
+
+        return bundleDownload;
+    }
+
     /**
      * Builds a new workflow from cwl files fetched from Github
      * @param githubInfo Github information for the workflow
      * @return The constructed model for the Workflow
      */
-    public Workflow newWorkflowFromGithub(GithubDetails githubInfo) {
-
+    public Workflow createWorkflow(GithubDetails githubInfo) {
         try {
             // Get the sha hash from a branch reference
             String latestCommit = githubService.getCommitSha(githubInfo);
@@ -89,9 +170,11 @@ public Workflow newWorkflowFromGithub(GithubDetails githubInfo) {
                 // This is Async so cannot just call constructor, needs intermediate as per Spring framework
                 generateROBundle(workflowModel);
 
+                // Save to database
+                workflowRepository.save(workflowModel);
+
                 // Return this model to be displayed
                 return workflowModel;
-
             } else {
                 logger.error("No workflow could be found");
             }
@@ -106,7 +189,7 @@ public Workflow newWorkflowFromGithub(GithubDetails githubInfo) {
      * Generates the RO bundle for a Workflow and adds it to the model
      * @param workflow The workflow model to create a Research Object for
      */
-    public void generateROBundle(Workflow workflow) {
+    private void generateROBundle(Workflow workflow) {
         try {
             ROBundleFactory.workflowROFromGithub(githubService, workflow.getRetrievedFrom(), workflow.getLastCommit());
         } catch (Exception ex) {
@@ -118,7 +201,7 @@ public void generateROBundle(Workflow workflow) {
      * Removes a workflow and its research object bundle
      * @param workflow The workflow to be deleted
      */
-    public void removeWorkflow(Workflow workflow) {
+    private void removeWorkflow(Workflow workflow) {
         // Delete the Research Object Bundle from disk
         File roBundle = new File(workflow.getRoBundle());
         if (roBundle.delete()) {
@@ -139,7 +222,7 @@ public void removeWorkflow(Workflow workflow) {
      * @param workflow The cached workflow model
      * @return Whether or not there are new commits
      */
-    public boolean cacheExpired(Workflow workflow) {
+    private boolean cacheExpired(Workflow workflow) {
         try {
             // Calculate expiration
             Calendar expireCal = Calendar.getInstance();

src/main/java/org/commonwl/viewer/web/ROBundleNotFoundException.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.commonwl.viewer.web;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+/**
+ * Exception thrown when a Research Object file does not exist
+ */
+@ResponseStatus(value = HttpStatus.NOT_FOUND)
+public class ROBundleNotFoundException extends RuntimeException {
+}