Merge branch 'master' into highlighting

# Conflicts:
#	src/main/resources/static/css/main.css
#	src/main/resources/static/js/workflow.js

Author: Mark Robinson

Dockerfile

@@ -1,6 +1,8 @@
 FROM maven:3.3-jdk-8-alpine
 MAINTAINER Stian Soiland-Reyes <[email protected]>
 
+RUN apk add --update graphviz && rm -rf /var/cache/apk/*
+
 RUN mkdir /usr/share/maven/ref/repository
 
 RUN mkdir -p /usr/src/app

pom.xml

@@ -24,6 +24,13 @@
 		<java.version>1.8</java.version>
 	</properties>
 
+    <repositories>
+        <repository>
+            <id>jitpack.io</id>
+            <url>https://jitpack.io</url>
+        </repository>
+    </repositories>
+
 	<dependencies>
 		<dependency>
 			<groupId>org.springframework.boot</groupId>
@@ -57,6 +64,11 @@
 			<artifactId>snakeyaml</artifactId>
 			<version>1.7</version>
 		</dependency>
+        <dependency>
+            <groupId>com.github.jabbalaci</groupId>
+            <artifactId>graphviz-java-api</artifactId>
+            <version>f9bf94896776de6e98b5004e819b63fb4c15b15d</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.taverna.language</groupId>
             <artifactId>taverna-robundle</artifactId>

src/main/java/org/commonwl/viewer/domain/CWLCollection.java

@@ -39,6 +39,7 @@ public class CWLCollection {
 
     private GitHubService githubService;
     private GithubDetails githubInfo;
+    private String commitSha;
 
     // Maps of ID to associated JSON
     private Map<String, JsonNode> cwlDocs = new HashMap<>();
@@ -78,11 +79,14 @@ public class CWLCollection {
      * Creates a new collection of CWL files from a Github repository
      * @param githubService Service to provide the Github API functionality
      * @param githubInfo The information necessary to access the Github directory associated with the RO
+     * @param commitSha The sha hash of the latest commit used for caching
      * @throws IOException Any API errors which may have occurred
      */
-    public CWLCollection(GitHubService githubService, GithubDetails githubInfo) throws IOException {
+    public CWLCollection(GitHubService githubService, GithubDetails githubInfo,
+                         String commitSha) throws IOException {
         this.githubInfo = githubInfo;
         this.githubService = githubService;
+        this.commitSha = commitSha;
 
         // Add any CWL files from the Github repo to this collection
         List<RepositoryContents> repoContents = githubService.getContents(githubInfo);
@@ -122,7 +126,7 @@ private void addDocs(List<RepositoryContents> repoContents) throws IOException {
                         // Get the content of this file from Github
                         GithubDetails githubFile = new GithubDetails(githubInfo.getOwner(),
                                 githubInfo.getRepoName(), githubInfo.getBranch(), repoContent.getPath());
-                        String fileContent = githubService.downloadFile(githubFile);
+                        String fileContent = githubService.downloadFile(githubFile, commitSha);
 
                         // Parse yaml to JsonNode
                         Yaml reader = new Yaml();

src/main/java/org/commonwl/viewer/domain/GithubDetails.java

@@ -28,6 +28,7 @@ public class GithubDetails implements Serializable {
 
     private String owner;
     private String repoName;
+    private String sha;
     private String branch;
     private String path;
 

src/main/java/org/commonwl/viewer/domain/ROBundle.java

@@ -29,7 +29,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -49,20 +48,22 @@ public class ROBundle {
 
     private Bundle bundle;
     private GithubDetails githubInfo;
+    private String commitSha;
 
     /**
      * Creates a new research object bundle for a workflow from a Github repository
      * @param githubInfo The information necessary to access the Github directory associated with the RO
      * @throws IOException Any API errors which may have occurred
      */
-    public ROBundle(GitHubService githubService, GithubDetails githubInfo,
+    public ROBundle(GitHubService githubService, GithubDetails githubInfo, String commitSha,
                     String appName, String appURL) throws IOException {
         // TODO: Add back file size checking on individual files as well as whole bundle
 
         // Create a new RO bundle
         this.bundle = Bundles.createBundle();
         this.githubInfo = githubInfo;
         this.githubService = githubService;
+        this.commitSha = commitSha;
 
         Manifest manifest = bundle.getManifest();
 
@@ -135,7 +136,7 @@ private void addFiles(List<RepositoryContents> repoContents, Path path) throws I
                 // Get the content of this file from Github
                 GithubDetails githubFile = new GithubDetails(githubInfo.getOwner(),
                         githubInfo.getRepoName(), githubInfo.getBranch(), repoContent.getPath());
-                String fileContent = githubService.downloadFile(githubFile);
+                String fileContent = githubService.downloadFile(githubFile, commitSha);
 
                 // Save file to research object bundle
                 Path newFilePort = path.resolve(repoContent.getName());

src/main/java/org/commonwl/viewer/domain/Workflow.java

@@ -29,7 +29,6 @@
 
 import java.io.IOException;
 import java.io.StringWriter;
-import java.nio.file.Path;
 import java.util.Date;
 import java.util.Map;
 
@@ -51,6 +50,10 @@ public class Workflow {
     @DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss z")
     private Date retrievedOn;
 
+    // The last commit from the branch at the time of fetching
+    // Used for caching purposes
+    private String lastCommit;
+
     // A String which represents the path to a RO bundle
     // Path types cannot be stored using Spring Data, unfortunately
     private String roBundle;
@@ -161,4 +164,12 @@ public String getDotGraph() {
     public void setDotGraph(String dotGraph) {
         this.dotGraph = dotGraph;
     }
+
+    public String getLastCommit() {
+        return lastCommit;
+    }
+
+    public void setLastCommit(String lastCommit) {
+        this.lastCommit = lastCommit;
+    }
 }

src/main/java/org/commonwl/viewer/services/GitHubService.java

@@ -21,10 +21,9 @@
 
 import org.apache.commons.io.IOUtils;
 import org.commonwl.viewer.domain.GithubDetails;
-import org.eclipse.egit.github.core.RepositoryContents;
-import org.eclipse.egit.github.core.RepositoryId;
-import org.eclipse.egit.github.core.User;
+import org.eclipse.egit.github.core.*;
 import org.eclipse.egit.github.core.client.GitHubClient;
+import org.eclipse.egit.github.core.service.CommitService;
 import org.eclipse.egit.github.core.service.ContentsService;
 import org.eclipse.egit.github.core.service.UserService;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -47,6 +46,7 @@ public class GitHubService {
     // Github API services
     private final ContentsService contentsService;
     private final UserService userService;
+    private final CommitService commitService;
 
     // URL validation for directory links
     private final String GITHUB_DIR_REGEX = "^https:\\/\\/github\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:tree\\/([^/]+)\\/(.*))?$";
@@ -62,6 +62,7 @@ public GitHubService(@Value("${githubAPI.authentication}") boolean authEnabled,
         }
         this.contentsService = new ContentsService(client);
         this.userService = new UserService(client);
+        this.commitService = new CommitService(client);
     }
 
     /**
@@ -81,7 +82,7 @@ public GithubDetails detailsFromDirURL(String url) {
      * Get contents of a Github path from the API
      * @param githubInfo The information to access the repository
      * @return A list of details for the file(s) or false if there is an API error
-     * @throws IOException Any API errors which may have occured
+     * @throws IOException Any API errors which may have occurred
      */
     public List<RepositoryContents> getContents(GithubDetails githubInfo) throws IOException {
         return contentsService.getContents(new RepositoryId(githubInfo.getOwner(), githubInfo.getRepoName()),
@@ -92,7 +93,7 @@ public List<RepositoryContents> getContents(GithubDetails githubInfo) throws IOE
      * Get the details of a user from the Github API
      * @param username The username of the user we want to get data about
      * @return A user object containing the API information
-     * @throws IOException Any API errors which may have occured
+     * @throws IOException Any API errors which may have occurred
      */
     public User getUser(String username) throws IOException {
         return userService.getUser(username);
@@ -101,21 +102,34 @@ public User getUser(String username) throws IOException {
     /**
      * Download a single file from a Github repository
      * @param githubInfo The information to access the repository
+     * @param sha The commit ID to download a specific version of a file
      * @return A string with the contents of the file
-     * @throws IOException Any API errors which may have occured
+     * @throws IOException Any API errors which may have occurred
      */
-    public String downloadFile(GithubDetails githubInfo) throws IOException {
+    public String downloadFile(GithubDetails githubInfo, String sha) throws IOException {
         // Download the file and return the contents
         // rawgit.com used to download individual files from git with the correct media type
         String url = String.format("https://cdn.rawgit.com/%s/%s/%s/%s", githubInfo.getOwner(),
-                githubInfo.getRepoName(), githubInfo.getBranch(), githubInfo.getPath());
+                githubInfo.getRepoName(), sha, githubInfo.getPath());
         URL downloadURL = new URL(url);
         InputStream download = downloadURL.openStream();
         try {
             return IOUtils.toString(download);
         } finally {
             IOUtils.closeQuietly(download);
         }
+    }
 
+    /**
+     * Gets the latest commit sha hash from a particular branch
+     * If hash is given as a branch, this will return the same hash if valid
+     * @param githubInfo The information to access the repository
+     * @return A sha hash for the latest commit
+     * @throws IOException Any API errors which may have occurred
+     */
+    public String getCommitSha(GithubDetails githubInfo) throws IOException {
+        RepositoryId repo = new RepositoryId(githubInfo.getOwner(), githubInfo.getRepoName());
+        RepositoryCommit currentCommit = commitService.getCommit(repo, githubInfo.getBranch());
+        return currentCommit.getSha();
     }
 }

src/main/java/org/commonwl/viewer/services/ROBundleFactory.java

@@ -53,11 +53,11 @@ public class ROBundleFactory {
     @Autowired
     public ROBundleFactory(@Value("${applicationName}") String applicationName,
                            @Value("${applicationURL}") String applicationURL,
-                           @Value("${storageLocation}") Path storageLocation,
+                           @Value("${graphvizStorage}") Path graphvizStorage,
                            WorkflowRepository workflowRepository) {
         this.applicationName = applicationName;
         this.applicationURL = applicationURL;
-        this.storageLocation = storageLocation;
+        this.storageLocation = graphvizStorage;
         this.workflowRepository = workflowRepository;
     }
 
@@ -69,12 +69,12 @@ public ROBundleFactory(@Value("${applicationName}") String applicationName,
      * @throws IOException Any API errors which may have occurred
      */
     @Async
-    void workflowROFromGithub(GitHubService githubService, GithubDetails githubInfo)
+    void workflowROFromGithub(GitHubService githubService, GithubDetails githubInfo, String commitSha)
             throws IOException, InterruptedException {
         logger.info("Creating Research Object Bundle");
 
         // Create a new Research Object Bundle with Github contents
-        ROBundle bundle = new ROBundle(githubService, githubInfo,
+        ROBundle bundle = new ROBundle(githubService, githubInfo, commitSha,
                 applicationName, applicationURL);
 
         // Save the bundle to the storage location in properties

src/main/java/org/commonwl/viewer/services/WorkflowFactory.java renamed to src/main/java/org/commonwl/viewer/services/WorkflowService.java

@@ -25,46 +25,59 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
+import java.io.File;
+import java.util.Calendar;
 import java.util.Date;
 
 @Service
-public class WorkflowFactory {
+public class WorkflowService {
 
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
 
     private final GitHubService githubService;
+    private final WorkflowRepository workflowRepository;
     private final ROBundleFactory ROBundleFactory;
+    private final int cacheDays;
 
     @Autowired
-    public WorkflowFactory(GitHubService githubService,
-                           ROBundleFactory ROBundleFactory) {
+    public WorkflowService(GitHubService githubService,
+                           WorkflowRepository workflowRepository,
+                           ROBundleFactory ROBundleFactory,
+                           @Value("${cacheDays}") int cacheDays) {
         this.githubService = githubService;
+        this.workflowRepository = workflowRepository;
         this.ROBundleFactory = ROBundleFactory;
+        this.cacheDays = cacheDays;
     }
 
     /**
      * Builds a new workflow from cwl files fetched from Github
      * @param githubInfo Github information for the workflow
      * @return The constructed model for the Workflow
      */
-    public Workflow workflowFromGithub(GithubDetails githubInfo) {
+    public Workflow newWorkflowFromGithub(GithubDetails githubInfo) {
 
         try {
+            // Get the sha hash from a branch reference
+            String latestCommit = githubService.getCommitSha(githubInfo);
+
             // Set up CWL utility to collect the documents
-            CWLCollection cwlFiles = new CWLCollection(githubService, githubInfo);
+            CWLCollection cwlFiles = new CWLCollection(githubService, githubInfo, latestCommit);
 
             // Get the workflow model
             Workflow workflowModel = cwlFiles.getWorkflow();
             if (workflowModel != null) {
                 // Set origin details
                 workflowModel.setRetrievedOn(new Date());
                 workflowModel.setRetrievedFrom(githubInfo);
+                workflowModel.setLastCommit(latestCommit);
 
                 // Create a new research object bundle from Github details
                 // This is Async so cannot just call constructor, needs intermediate as per Spring framework
-                ROBundleFactory.workflowROFromGithub(githubService, githubInfo);
+                ROBundleFactory.workflowROFromGithub(githubService, githubInfo, latestCommit);
 
                 // Return this model to be displayed
                 return workflowModel;
@@ -79,6 +92,65 @@ public Workflow workflowFromGithub(GithubDetails githubInfo) {
         return null;
     }
 
+    /**
+     * Removes a workflow and its research object bundle
+     * @param workflow The workflow to be deleted
+     */
+    public void removeWorkflow(Workflow workflow) {
+        // Delete the Research Object Bundle from disk
+        File roBundle = new File(workflow.getRoBundle());
+        if (roBundle.delete()) {
+            logger.debug("Deleted Research Object Bundle");
+        } else {
+            logger.debug("Failed to delete Research Object Bundle");
+        }
+
+        // TODO: Delete cached graphviz images when serverside graphviz is merged
+
+        // Remove the workflow from the database
+        workflowRepository.delete(workflow);
+    }
+
+    /**
+     * Check for cache expiration based on time and commit sha
+     * @param workflow The cached workflow model
+     * @return Whether or not there are new commits
+     */
+    public boolean cacheExpired(Workflow workflow) {
+        try {
+            // Calculate expiration
+            Calendar expireCal = Calendar.getInstance();
+            expireCal.setTime(workflow.getRetrievedOn());
+            expireCal.add(Calendar.DATE, cacheDays);
+            Date expirationDate = expireCal.getTime();
+
+            // Check cached retrievedOn date
+            if (expirationDate.before(new Date())) {
+                // Cache expiry time has elapsed
+                // Check current head of the branch with the cached head
+                logger.debug("Time has expired for caching, checking commits...");
+                String currentHead = githubService.getCommitSha(workflow.getRetrievedFrom());
+                logger.debug("Current: " + workflow.getLastCommit() + ", HEAD: " + currentHead);
+
+                // Reset date in database if there are still no changes
+                boolean expired = !workflow.getLastCommit().equals(currentHead);
+                if (!expired) {
+                    workflow.setRetrievedOn(new Date());
+                    workflowRepository.save(workflow);
+                }
+
+                // Return whether the cache has expired
+                return expired;
+            } else {
+                // Cache expiry time has not elapsed yet
+                return false;
+            }
+        } catch (Exception ex) {
+            // Default to no expiry if there was an API error
+            return false;
+        }
+    }
+
     /*
         // Check total file size
         int totalFileSize = 0;