Improve caching scheme to check for changes to just the workflow

Rather than the whole repository

Author: Mark Robinson

src/main/java/org/commonwl/viewer/github/GitHubService.java

@@ -20,8 +20,9 @@
 package org.commonwl.viewer.github;
 
 import org.apache.commons.io.IOUtils;
-import org.commonwl.viewer.researchobject.HashableAgent;
-import org.eclipse.egit.github.core.*;
+import org.eclipse.egit.github.core.RepositoryCommit;
+import org.eclipse.egit.github.core.RepositoryContents;
+import org.eclipse.egit.github.core.RepositoryId;
 import org.eclipse.egit.github.core.client.GitHubClient;
 import org.eclipse.egit.github.core.service.CommitService;
 import org.eclipse.egit.github.core.service.ContentsService;
@@ -31,12 +32,8 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
 import java.net.URL;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -142,34 +139,13 @@ public String getCommitSha(GithubDetails githubInfo) throws IOException {
     }
 
     /**
-     * Get the contributors to a specific file by their commits
+     * Get a list of commits
      * @param githubInfo The information to access the repository
      * @return A list of unique contributors
      * @throws IOException Any API errors which may have occurred
-     * @throws URISyntaxException Any error in the author's URI (should never occur)
      */
-    public Set<HashableAgent> getContributors(GithubDetails githubInfo, String sha)
-            throws IOException, URISyntaxException {
+    public List<RepositoryCommit> getCommits(GithubDetails githubInfo) throws IOException {
         RepositoryId repo = new RepositoryId(githubInfo.getOwner(), githubInfo.getRepoName());
-        Set<HashableAgent> authors = new HashSet<HashableAgent>();
-
-        for (RepositoryCommit commit : commitService.getCommits(repo, sha, githubInfo.getPath())) {
-            User author = commit.getAuthor();
-            CommitUser commitAuthor = commit.getCommit().getAuthor();
-
-            // If there is author information for this commit in some form
-            if (author != null || commitAuthor != null) {
-                // Create a new agent and add as much detail as possible
-                HashableAgent newAgent = new HashableAgent();
-                if (author != null) {
-                    newAgent.setUri(new URI(author.getHtmlUrl()));
-                }
-                if (commitAuthor != null) {
-                    newAgent.setName(commitAuthor.getName());
-                }
-                authors.add(newAgent);
-            }
-        }
-        return authors;
+        return commitService.getCommits(repo, githubInfo.getBranch(), githubInfo.getPath());
     }
 }

src/main/java/org/commonwl/viewer/researchobject/ROBundle.java

@@ -28,7 +28,10 @@
 import org.apache.taverna.robundle.manifest.PathMetadata;
 import org.commonwl.viewer.github.GitHubService;
 import org.commonwl.viewer.github.GithubDetails;
+import org.eclipse.egit.github.core.CommitUser;
+import org.eclipse.egit.github.core.RepositoryCommit;
 import org.eclipse.egit.github.core.RepositoryContents;
+import org.eclipse.egit.github.core.User;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -55,21 +58,23 @@ public class ROBundle {
 
     private Bundle bundle;
     private GithubDetails githubInfo;
-    private String commitSha;
     private Agent thisApp;
     private int singleFileSizeLimit;
-    private Set<HashableAgent> authors = new HashSet<HashableAgent>();
+    private Set<HashableAgent> authors = new HashSet<>();
 
     // Pattern for extracting version from a cwl file
     private final String CWL_VERSION_REGEX = "cwlVersion:\\s*\"?(?:cwl:)?([^\\s\"]+)\"?";
     private final Pattern cwlVersionPattern = Pattern.compile(CWL_VERSION_REGEX);
 
     /**
      * Creates a new research object bundle for a workflow from a Github repository
+     * @param githubService The service for handling Github functionality
      * @param githubInfo The information necessary to access the Github directory associated with the RO
+     * @param appName The name of the application from properties, for attribution
+     * @param appURL The URL of the application from properties, for attribution
      * @throws IOException Any API errors which may have occurred
      */
-    public ROBundle(GitHubService githubService, GithubDetails githubInfo, String commitSha,
+    public ROBundle(GitHubService githubService, GithubDetails githubInfo,
                     String appName, String appURL, int singleFileSizeLimit) throws IOException {
         // File size limits
         this.singleFileSizeLimit = singleFileSizeLimit;
@@ -78,7 +83,6 @@ public ROBundle(GitHubService githubService, GithubDetails githubInfo, String co
         this.bundle = Bundles.createBundle();
         this.githubInfo = githubInfo;
         this.githubService = githubService;
-        this.commitSha = commitSha;
 
         Manifest manifest = bundle.getManifest();
 
@@ -108,7 +112,7 @@ public ROBundle(GitHubService githubService, GithubDetails githubInfo, String co
         addFiles(repoContents, bundleFiles);
 
         // Add combined authors
-        manifest.setAuthoredBy(new ArrayList<Agent>(authors));
+        manifest.setAuthoredBy(new ArrayList<>(authors));
     }
 
     /**
@@ -140,19 +144,21 @@ private void addFiles(List<RepositoryContents> repoContents, Path path) throws I
             } else if (repoContent.getType().equals(GitHubService.TYPE_FILE)) {
 
                 try {
+                    GithubDetails githubFile = new GithubDetails(githubInfo.getOwner(),
+                            githubInfo.getRepoName(), githubInfo.getBranch(), repoContent.getPath());
+
                     // Where to store the new file in bundle
                     Path bundleFilePath = path.resolve(repoContent.getName());
 
-                    // Raw URI of the bundle
-                    GithubDetails githubFile = new GithubDetails(githubInfo.getOwner(),
-                            githubInfo.getRepoName(), githubInfo.getBranch(), repoContent.getPath());
+                    // Get commits
+                    List<RepositoryCommit> commitsOnFile = githubService.getCommits(githubFile);
+                    String commitSha = commitsOnFile.get(0).getSha();
+
                     URI rawURI = new URI("https://raw.githubusercontent.com/" + githubFile.getOwner() + "/" +
                             githubFile.getRepoName() + "/" + commitSha + "/" + githubFile.getPath());
 
-                    // Variable to store file contents
-                    String fileContent = null;
-
                     // Download or externally link if oversized
+                    String fileContent = null;
                     if (repoContent.getSize() <= singleFileSizeLimit) {
                         // Get the content of this file from Github
                         fileContent = githubService.downloadFile(githubFile, commitSha);
@@ -185,7 +191,24 @@ private void addFiles(List<RepositoryContents> repoContents, Path path) throws I
                     }
 
                     // Add authors from github commits to the file
-                    Set<HashableAgent> fileAuthors = githubService.getContributors(githubFile, commitSha);
+                    Set<HashableAgent> fileAuthors = new HashSet<>();
+                    for (RepositoryCommit commit : commitsOnFile) {
+                        User author = commit.getAuthor();
+                        CommitUser commitAuthor = commit.getCommit().getAuthor();
+
+                        // If there is author information for this commit in some form
+                        if (author != null || commitAuthor != null) {
+                            // Create a new agent and add as much detail as possible
+                            HashableAgent newAgent = new HashableAgent();
+                            if (author != null) {
+                                newAgent.setUri(new URI(author.getHtmlUrl()));
+                            }
+                            if (commitAuthor != null) {
+                                newAgent.setName(commitAuthor.getName());
+                            }
+                            fileAuthors.add(newAgent);
+                        }
+                    }
                     authors.addAll(fileAuthors);
                     aggregation.setAuthoredBy(new ArrayList<Agent>(fileAuthors));
 

src/main/java/org/commonwl/viewer/researchobject/ROBundleFactory.java

@@ -74,7 +74,7 @@ public ROBundleFactory(@Value("${applicationName}") String applicationName,
      * @throws IOException Any API errors which may have occurred
      */
     @Async
-    public void workflowROFromGithub(GitHubService githubService, GithubDetails githubInfo, String commitSha)
+    public void workflowROFromGithub(GitHubService githubService, GithubDetails githubInfo)
             throws IOException, InterruptedException {
         logger.info("Creating Research Object Bundle");
 
@@ -83,7 +83,7 @@ public void workflowROFromGithub(GitHubService githubService, GithubDetails gith
                 githubInfo.getBranch(), FilenameUtils.getPath(githubInfo.getPath()));
 
         // Create a new Research Object Bundle with Github contents
-        ROBundle bundle = new ROBundle(githubService, roDetails, commitSha,
+        ROBundle bundle = new ROBundle(githubService, roDetails,
                 applicationName, applicationURL, singleFileSizeLimit);
 
         // Save the bundle to the storage location in properties

src/main/java/org/commonwl/viewer/workflow/WorkflowService.java

@@ -187,7 +187,7 @@ public Workflow createWorkflow(GithubDetails githubInfo) {
      */
     private void generateROBundle(Workflow workflow) {
         try {
-            ROBundleFactory.workflowROFromGithub(githubService, workflow.getRetrievedFrom(), workflow.getLastCommit());
+            ROBundleFactory.workflowROFromGithub(githubService, workflow.getRetrievedFrom());
         } catch (Exception ex) {
             logger.error("Error creating RO Bundle", ex);
         }

src/main/resources/templates/workflow.html

@@ -126,7 +126,7 @@ <h4 class="modal-title" id="fullScreenGraphLabel">Workflow Graph</h4>
         <div class="col-md-12" role="main" id="main">
             <h2>Workflow: <span th:text="${workflow.label}">Workflow Name</span></h2>
             <p>
-                <a th:href="@{${workflow.retrievedFrom.getURL(workflow.lastCommit)}}" href="#" rel="noopener" target="_blank">
+                <a th:href="@{${workflow.retrievedFrom.getURL()}}" href="#" rel="noopener" target="_blank">
                     <img id="githubLogo" src="../static/img/GitHub-Mark-32px.png" th:src="@{/img/GitHub-Mark-32px.png}" width="24" height="24" />
                 </a>
                 <i>Fetched <span th:text="${{workflow.retrievedOn}}">01/12/16 at 21:00</span></i>

src/main/resources/templates/workflows.html

@@ -57,7 +57,7 @@ <h1>Explore Workflows</h1>
                                     <p><i th:text="${workflow.doc}">Description</i></p>
                                 </td>
                                 <td>
-                                    <a th:href="@{${workflow.retrievedFrom.getURL(workflow.lastCommit)}}" rel="noopener" target="_blank">
+                                    <a th:href="@{${workflow.retrievedFrom.getURL()}}" rel="noopener" target="_blank">
                                         <img id="githubLogo" src="../static/img/GitHub-Mark-32px.png" th:src="@{/img/GitHub-Mark-32px.png}" width="20" height="20" />
                                         <span th:if="${workflow.retrievedFrom.path != null}" th:text="@{${workflow.retrievedFrom.owner} + '/' + ${workflow.retrievedFrom.repoName} + '/' + ${workflow.retrievedFrom.path}}">https://github.com</span>
                                         <span th:unless="${workflow.retrievedFrom.path != null}" th:text="@{${workflow.retrievedFrom.owner} + '/' + ${workflow.retrievedFrom.repoName} + '/'}">https://github.com</span>