Merge pull request #27 from common-workflow-language/workflow-pages

Add persistent workflow pages using MongoDB

Author: MarkRobbo

src/main/java/org/commonwl/viewer/domain/GithubDetails.java

@@ -19,10 +19,12 @@
 
 package org.commonwl.viewer.domain;
 
+import java.io.Serializable;
+
 /**
  * Represents all the parameters necessary to access a file/directory in Github
  */
-public class GithubDetails {
+public class GithubDetails implements Serializable {
 
     private String owner;
     private String repoName;

src/main/java/org/commonwl/viewer/domain/ROBundle.java

@@ -29,6 +29,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -146,11 +147,15 @@ private void addFiles(List<RepositoryContents> repoContents, Path path) throws I
 
     /**
      * Save the Research Object bundle to disk
+     * @param directory The directory in which the RO will be saved
+     * @return The path to the research object
+     * @throws IOException Any errors in saving
      */
-    public void saveToTempFile() throws IOException {
+    public Path saveToFile(Path directory) throws IOException {
         // Save the Research Object Bundle
-        Path zip = Files.createTempFile("bundle", ".zip");
-        Bundles.closeAndSaveBundle(bundle, zip);
+        Path bundleLocation = Files.createTempFile(directory, "bundle", ".zip");
+        Bundles.closeAndSaveBundle(bundle, bundleLocation);
+        return bundleLocation;
     }
 
 }

src/main/java/org/commonwl/viewer/domain/Workflow.java

@@ -23,6 +23,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.data.annotation.Id;
+import org.springframework.data.mongodb.core.index.Indexed;
+import org.springframework.data.mongodb.core.mapping.Document;
 import org.springframework.format.annotation.DateTimeFormat;
 
 import java.io.IOException;
@@ -31,24 +33,27 @@
 import java.util.Date;
 import java.util.Map;
 
-import static java.util.Calendar.DATE;
-
 /**
  * Representation of a workflow
  */
+@Document
 public class Workflow {
 
     static private final Logger logger = LoggerFactory.getLogger(Workflow.class);
 
     // ID for database
     @Id
-    private String id;
+    public String id;
 
     // Metadata
-    @DateTimeFormat(pattern="MMM d yyyy 'at' hh:mm")
-    private Date retrievedOn;
+    @Indexed(unique = true)
     private GithubDetails retrievedFrom;
-    private Path roBundle;
+    @DateTimeFormat(pattern="MMM d yyyy 'at' hh:mm z")
+    private Date retrievedOn;
+
+    // A String which represents the path to a RO bundle
+    // Path types cannot be stored using Spring Data, unfortunately
+    private String roBundle;
 
     // Contents of the workflow
     private String label;
@@ -76,17 +81,10 @@ public Workflow(String label, String doc, Map<String, CWLElement> inputs, Map<St
         }
     }
 
-    @Override
-    public String toString() {
-        return "Workflow{" +
-                "id='" + id + '\'' +
-                ", retrievedFrom=" + retrievedFrom +
-                ", roBundle=" + roBundle +
-                ", label='" + label + '\'' +
-                ", doc='" + doc + '\'' +
-                ", inputs=" + inputs +
-                ", outputs=" + outputs +
-                '}';
+    public String getID() { return id; }
+
+    public void setId(String id) {
+        this.id = id;
     }
 
     public String getLabel() {
@@ -121,11 +119,11 @@ public void setOutputs(Map<String, CWLElement> outputs) {
         this.outputs = outputs;
     }
 
-    public Path getRoBundle() {
+    public String getRoBundle() {
         return roBundle;
     }
 
-    public void setRoBundle(Path roBundle) {
+    public void setRoBundle(String roBundle) {
         this.roBundle = roBundle;
     }
 

src/main/java/org/commonwl/viewer/services/ROBundleFactory.java

@@ -21,14 +21,17 @@
 
 import org.commonwl.viewer.domain.GithubDetails;
 import org.commonwl.viewer.domain.ROBundle;
+import org.commonwl.viewer.domain.Workflow;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.scheduling.annotation.EnableAsync;
 import org.springframework.stereotype.Component;
 
 import java.io.IOException;
+import java.nio.file.Path;
 
 /**
  * Class for the purpose of a Spring Framework Async method
@@ -42,25 +45,61 @@ public class ROBundleFactory {
 
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
 
-    @Value("${applicationName}")
-    private String applicationName;
-    @Value("${applicationURL}")
-    private String applicationURL;
+    private final String applicationName;
+    private final String applicationURL;
+    private final Path storageLocation;
+    private final WorkflowRepository workflowRepository;
+
+    @Autowired
+    public ROBundleFactory(@Value("${applicationName}") String applicationName,
+                           @Value("${applicationURL}") String applicationURL,
+                           @Value("${storageLocation}") Path storageLocation,
+                           WorkflowRepository workflowRepository) {
+        this.applicationName = applicationName;
+        this.applicationURL = applicationURL;
+        this.storageLocation = storageLocation;
+        this.workflowRepository = workflowRepository;
+    }
 
     /**
      * Creates a new Workflow Research Object Bundle from a Github URL
      * and saves it to a file
      * @param githubService The service for Github API functionality
      * @param githubInfo Details of the Github repository
-     * @throws IOException Any API errors which may have occured
+     * @throws IOException Any API errors which may have occurred
      */
     @Async
-    void workflowROFromGithub(GitHubService githubService, GithubDetails githubInfo) throws IOException {
-        // TODO: Add the bundle link to the page when it is finished being created
+    void workflowROFromGithub(GitHubService githubService, GithubDetails githubInfo)
+            throws IOException, InterruptedException {
         logger.info("Creating Research Object Bundle");
+
+        // Create a new Research Object Bundle with Github contents
         ROBundle bundle = new ROBundle(githubService, githubInfo,
                 applicationName, applicationURL);
-        bundle.saveToTempFile();
-        logger.info("Successfully saved Research Object Bundle");
+
+        // Save the bundle to the storage location in properties
+        Path bundleLocation = bundle.saveToFile(storageLocation);
+
+        // Add the path to the bundle to the bundle
+        Workflow workflow = workflowRepository.findByRetrievedFrom(githubInfo);
+
+        // Chance that this thread could be created before workflow model is saved
+        int attempts = 5;
+        while (attempts > 0 && workflow == null) {
+            // Delay this thread by 0.5s and try again until success or too many attempts
+            Thread.sleep(1000L);
+            workflow = workflowRepository.findByRetrievedFrom(githubInfo);
+            attempts--;
+        }
+
+        if (workflow == null) {
+            // If workflow is still null we can't find the workflow model
+            logger.error("Workflow model could not be found when adding RO Bundle path");
+        } else {
+            // Add RO Bundle to associated workflow model
+            workflow.setRoBundle(bundleLocation.toString());
+            workflowRepository.save(workflow);
+            logger.info("Finished saving Research Object Bundle");
+        }
     }
 }

src/main/java/org/commonwl/viewer/services/WorkflowFactory.java

@@ -50,47 +50,34 @@ public WorkflowFactory(GitHubService githubService,
 
     /**
      * Builds a new workflow from cwl files fetched from Github
-     * @param githubURL Github directory URL to get the files from
+     * @param githubInfo Github information for the workflow
      * @return The constructed model for the Workflow
      */
-    public Workflow workflowFromGithub(String githubURL) {
+    public Workflow workflowFromGithub(GithubDetails githubInfo) {
 
-        GithubDetails githubInfo = githubService.detailsFromDirURL(githubURL);
+        try {
+            // Set up CWL utility to collect the documents
+            CWLCollection cwlFiles = new CWLCollection(githubService, githubInfo);
 
-        // If the URL is valid and details could be extracted
-        if (githubInfo != null) {
+            // Get the workflow model
+            Workflow workflowModel = cwlFiles.getWorkflow();
+            if (workflowModel != null) {
+                // Set origin details
+                workflowModel.setRetrievedOn(new Date());
+                workflowModel.setRetrievedFrom(githubInfo);
 
-            try {
-                // Set up CWL utility to collect the documents
-                CWLCollection cwlFiles = new CWLCollection(githubService, githubInfo);
+                // Create a new research object bundle from Github details
+                // This is Async so cannot just call constructor, needs intermediate as per Spring framework
+                ROBundleFactory.workflowROFromGithub(githubService, githubInfo);
 
-                // Get the workflow model
-                Workflow workflowModel = cwlFiles.getWorkflow();
-                if (workflowModel != null) {
-                    // Set origin details
-                    workflowModel.setRetrievedOn(new Date());
-                    workflowModel.setRetrievedFrom(githubInfo);
+                // Return this model to be displayed
+                return workflowModel;
 
-                    // Save to the MongoDB database
-                    logger.info("Storing workflow in DB");
-                    workflowRepository.save(workflowModel);
-
-                    // Create a new research object bundle from Github details
-                    // This is Async so cannot just call constructor, needs intermediate as per Spring framework
-                    ROBundleFactory.workflowROFromGithub(githubService, githubInfo);
-
-                    // Return this model to be displayed
-                    return workflowModel;
-
-                } else {
-                    logger.error("No workflow could be found");
-                }
-            } catch (IOException ex) {
-                logger.error(ex.getMessage());
+            } else {
+                logger.error("No workflow could be found");
             }
-        } else {
-            // TODO: Refactor validator so this redundant step can be removed?
-            logger.error("Error should never happen, already passed URL validation");
+        } catch (Exception ex) {
+            logger.error(ex.getMessage());
         }
 
         return null;

src/main/java/org/commonwl/viewer/services/WorkflowFormValidator.java

@@ -37,7 +37,7 @@
  * Runs validation on the workflow form from the main page
  */
 @Component
-public class WorkflowFormValidator implements Validator {
+public class WorkflowFormValidator {
 
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
 
@@ -51,26 +51,16 @@ public WorkflowFormValidator(GitHubService githubService) {
         this.githubService = githubService;
     }
 
-    /**
-     * Types of class the this validator supports, WorkflowForm
-     * @param theClass The class which is being validated
-     * @return Whether the class can be validated using this validator
-     */
-    public boolean supports(Class theClass) {
-        return WorkflowForm.class.equals(theClass);
-    }
-
     /**
      * Validates a WorkflowForm to ensure the URL is not empty and directory contains cwl files
-     * @param obj The given WorkFlowForm
+     * @param form The given WorkflowForm
      * @param e Any errors from validation
      */
-    public void validate(Object obj, Errors e) {
+    public GithubDetails validateAndParse(WorkflowForm form, Errors e) {
         ValidationUtils.rejectIfEmptyOrWhitespace(e, "githubURL", "githubURL.emptyOrWhitespace");
 
         // Only continue if not null and isn't just whitespace
         if (!e.hasErrors()) {
-            WorkflowForm form = (WorkflowForm) obj;
             GithubDetails githubInfo = githubService.detailsFromDirURL(form.getGithubURL());
 
             // If the URL is valid and details could be extracted
@@ -91,7 +81,10 @@ public void validate(Object obj, Errors e) {
                             }
                         }
                     }
-                    if (!foundCWL) {
+                    if (foundCWL) {
+                        // Return the Github information
+                        return githubInfo;
+                    } else {
                         // The URL does not contain any .cwl files
                         logger.error("No .cwl files found at Github URL");
                         e.rejectValue("githubURL", "githubURL.missingWorkflow");
@@ -109,5 +102,8 @@ public void validate(Object obj, Errors e) {
         } else {
             logger.error("Github URL is empty");
         }
+
+        // Errors will stop this being used anyway
+        return null;
     }
 }