Refactoring and RO bundle info from Git log

Author: MarkRobbo

src/main/java/org/commonwl/view/cwl/CWLService.java

@@ -35,8 +35,8 @@
 import org.apache.jena.rdf.model.StmtIterator;
 import org.apache.jena.riot.RiotException;
 import org.commonwl.view.docker.DockerService;
-import org.commonwl.view.github.GitDetails;
-import org.commonwl.view.github.GitService;
+import org.commonwl.view.git.GitDetails;
+import org.commonwl.view.git.GitService;
 import org.commonwl.view.graphviz.ModelDotWriter;
 import org.commonwl.view.graphviz.RDFDotWriter;
 import org.commonwl.view.workflow.Workflow;
@@ -67,7 +67,7 @@ public class CWLService {
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
 
     // Autowired properties/services
-    private final GitService githubService;
+    private final GitService gitService;
     private final RDFService rdfService;
     private final CWLTool cwlTool;
     private final int singleFileSizeLimit;
@@ -98,16 +98,16 @@ public class CWLService {
 
     /**
      * Constructor for the Common Workflow Language service
-     * @param githubService A service for accessing Github functionality
+     * @param gitService A service for accessing Github functionality
      * @param cwlTool Handles cwltool integration
      * @param singleFileSizeLimit The file size limit for single files
      */
     @Autowired
-    public CWLService(GitService githubService,
+    public CWLService(GitService gitService,
                       RDFService rdfService,
                       CWLTool cwlTool,
                       @Value("${singleFileSizeLimit}") int singleFileSizeLimit) {
-        this.githubService = githubService;
+        this.gitService = gitService;
         this.rdfService = rdfService;
         this.cwlTool = cwlTool;
         this.singleFileSizeLimit = singleFileSizeLimit;
@@ -421,6 +421,7 @@ public Workflow parseWorkflowWithCwltool(GitDetails gitDetails,
             logger.error("Failed to create DOT graph for workflow: " + ex.getMessage());
         }
 
+
         return workflowModel;
 
     }

src/main/java/org/commonwl/view/github/GitDetails.java renamed to src/main/java/org/commonwl/view/git/GitDetails.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.commonwl.view.github;
+package org.commonwl.view.git;
 
 import java.io.Serializable;
 
@@ -96,7 +96,7 @@ public String getUrl(String branchOverride) {
                 return repoUrl;
             case GITHUB:
             case GITLAB:
-                return "https://" + normaliseURL(repoUrl).replace(".git", "") + "/blob/" + branchOverride + "/" + path;
+                return "https://" + normaliseUrl(repoUrl).replace(".git", "") + "/blob/" + branchOverride + "/" + path;
             default:
                 return null;
         }
@@ -117,10 +117,29 @@ public String getUrl() {
     public String getInternalUrl() {
         switch (this.type) {
             case GENERIC:
-                return "/workflows/" + normaliseURL(repoUrl) + "/" + branch + "/" + path;
+                return "/workflows/" + normaliseUrl(repoUrl) + "/" + branch + "/" + path;
             case GITHUB:
             case GITLAB:
-                return "/workflows/" + normaliseURL(repoUrl).replace(".git", "") + "/blob/" + branch + "/" + path;
+                return "/workflows/" + normaliseUrl(repoUrl).replace(".git", "") + "/blob/" + branch + "/" + path;
+            default:
+                return null;
+        }
+    }
+
+    /**
+     * Get the URL directly to the resource
+     * @return The URL
+     */
+    public String getRawUrl() {
+        switch (this.type) {
+            case GENERIC:
+                return repoUrl;
+            case GITHUB:
+                return "https://raw.githubusercontent.com/" +
+                        normaliseUrl(repoUrl).replace("github.com/", "").replace(".git", "") +
+                        "/" + branch + "/" + path;
+            case GITLAB:
+                return "https://whatever/gitlab/uses";
             default:
                 return null;
         }
@@ -131,7 +150,7 @@ public String getInternalUrl() {
      * @param url The URL to be normalised
      * @return The normalised URL
      */
-    private String normaliseURL(String url) {
+    private String normaliseUrl(String url) {
         return url.replace("http://", "")
                   .replace("https://", "")
                   .replace("ssh://", "")

src/main/java/org/commonwl/view/github/GitService.java renamed to src/main/java/org/commonwl/view/git/GitService.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.commonwl.view.github;
+package org.commonwl.view.git;
 
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.api.errors.GitAPIException;

src/main/java/org/commonwl/view/github/GitType.java renamed to src/main/java/org/commonwl/view/git/GitType.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.commonwl.view.github;
+package org.commonwl.view.git;
 
 /**
  * Differentiates supported sites for the

src/main/java/org/commonwl/view/researchobject/ROBundleFactory.java

@@ -21,7 +21,7 @@
 
 import org.apache.commons.io.FilenameUtils;
 import org.apache.taverna.robundle.Bundle;
-import org.commonwl.view.github.GitDetails;
+import org.commonwl.view.git.GitDetails;
 import org.commonwl.view.workflow.Workflow;
 import org.commonwl.view.workflow.WorkflowRepository;
 import org.slf4j.Logger;

src/main/java/org/commonwl/view/researchobject/ROBundleService.java

@@ -25,11 +25,11 @@
 import org.apache.taverna.robundle.Bundles;
 import org.apache.taverna.robundle.manifest.*;
 import org.commonwl.view.cwl.CWLTool;
-import org.commonwl.view.github.GitDetails;
-import org.commonwl.view.github.GitService;
+import org.commonwl.view.git.GitDetails;
 import org.commonwl.view.graphviz.GraphVizService;
 import org.commonwl.view.workflow.Workflow;
 import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.lib.PersonIdent;
 import org.eclipse.jgit.revwalk.RevCommit;
 import org.slf4j.Logger;
@@ -63,7 +63,6 @@ public class ROBundleService {
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
 
     // Services
-    private GitService githubService;
     private GraphVizService graphVizService;
     private CWLTool cwlTool;
 
@@ -82,22 +81,19 @@ public class ROBundleService {
      * @param appName The name of the application from properties, for attribution
      * @param appURL The URL of the application from properties, for attribution
      * @param singleFileSizeLimit The file size limit for each file in the RO bundle
-     * @param githubService The service for handling Github functionality
      * @throws URISyntaxException Error in creating URI for appURL
      */
     @Autowired
     public ROBundleService(@Value("${bundleStorage}") Path bundleStorage,
                            @Value("${applicationName}") String appName,
                            @Value("${applicationURL}") String appURL,
                            @Value("${singleFileSizeLimit}") int singleFileSizeLimit,
-                           GitService githubService,
                            GraphVizService graphVizService,
                            CWLTool cwlTool) throws URISyntaxException {
         this.bundleStorage = bundleStorage;
         this.appAgent = new Agent(appName);
         appAgent.setUri(new URI(appURL));
         this.singleFileSizeLimit = singleFileSizeLimit;
-        this.githubService = githubService;
         this.graphVizService = graphVizService;
         this.cwlTool = cwlTool;
     }
@@ -153,16 +149,13 @@ public Bundle createBundle(Workflow workflow, GitDetails gitInfo) throws IOExcep
             GitDetails wfDetails = workflow.getRetrievedFrom();
 
             // Run cwltool
-            /*
-            String url = "https://raw.githubusercontent.com/" + wfDetails.getOwner() + "/" +
-                    wfDetails.getRepoName() + "/" + wfDetails.getBranch() + "/" + wfDetails.getPath();
+            String rawUrl = wfDetails.getRawUrl();
             List<PathAnnotation> manifestAnnotations = new ArrayList<>();
             addAggregation(bundle, manifestAnnotations,
-                    "merged.cwl", cwlTool.getPackedVersion(url));
+                    "merged.cwl", cwlTool.getPackedVersion(rawUrl));
             addAggregation(bundle, manifestAnnotations,
-                    "workflow.ttl", cwlTool.getRDF(url));
+                    "workflow.ttl", cwlTool.getRDF(rawUrl));
             bundle.getManifest().setAnnotations(manifestAnnotations);
-            */
 
             // Git2prov history
             List<Path> history = new ArrayList<>();
@@ -208,8 +201,7 @@ private void addFilesToBundle(Bundle bundle, Path bundlePath,
                     Path bundleFilePath = bundlePath.resolve(file.getName());
 
                     // Get direct URL
-                    URI rawURI = new URI("https://raw.githubusercontent.com/" + githubFile.getOwner() + "/" +
-                            githubFile.getRepoName() + "/" + commitSha + "/" + githubFile.getPath());
+                    URI rawURI = new URI("http://example.com");
 
                     // Variable to store file contents and aggregation
                     String fileContent = null;
@@ -254,31 +246,38 @@ private void addFilesToBundle(Bundle bundle, Path bundlePath,
                         }
                     }
 
-                    // Add authors from github commits to the file
-                    Set<HashableAgent> fileAuthors = new HashSet<>();
-                    Iterable<RevCommit> logs = gitRepo.log()
-                            .addPath(bundlePath.toString())
-                            .call();
-                    for (RevCommit rev : logs) {
-                        PersonIdent author = rev.getAuthorIdent();
-                        PersonIdent committer = rev.getCommitterIdent();
-                        if (author != null || committer != null) {
+                    // Add authors from git commits to the file
+                    try {
+                        Set<HashableAgent> fileAuthors = new HashSet<>();
+                        Iterable<RevCommit> logs = gitRepo.log()
+                                .addPath(bundlePath.toString())
+                                .call();
+                        for (RevCommit rev : logs) {
+                            // Use author first with backup of committer
+                            PersonIdent author = rev.getAuthorIdent();
+                            if (author == null) {
+                                author = rev.getCommitterIdent();
+                            }
                             // Create a new agent and add as much detail as possible
-                            HashableAgent newAgent = new HashableAgent();
                             if (author != null) {
-                                newAgent.setName(author.getName());
-                                newAgent.setUri(new URI("mailto:" + author.getEmailAddress()));
-                            } else {
-                                newAgent.setName(committer.getName());
-                                newAgent.setUri(new URI("mailto:" + committer.getEmailAddress()));
+                                HashableAgent newAgent = new HashableAgent();
+                                String name = author.getName();
+                                if (name != null && name.length() > 0) {
+                                    newAgent.setName(author.getName());
+                                }
+                                String email = author.getEmailAddress();
+                                if (email != null && email.length() > 0) {
+                                    newAgent.setUri(new URI("mailto:" + author.getEmailAddress()));
+                                }
+                                fileAuthors.add(newAgent);
                             }
-                            fileAuthors.add(newAgent);
                         }
+                        authors.addAll(fileAuthors);
+                        aggregation.setAuthoredBy(new ArrayList<>(fileAuthors));
+                    } catch (GitAPIException ex) {
+                        logger.error("Could not get commits for file " + repoPath, ex);
                     }
 
-                    authors.addAll(fileAuthors);
-                    aggregation.setAuthoredBy(new ArrayList<>(fileAuthors));
-
                     // Set retrieved information for this file in the manifest
                     aggregation.setRetrievedFrom(rawURI);
                     aggregation.setRetrievedBy(appAgent);

src/main/java/org/commonwl/view/workflow/QueuedWorkflowRepository.java

@@ -1,6 +1,6 @@
 package org.commonwl.view.workflow;
 
-import org.commonwl.view.github.GitDetails;
+import org.commonwl.view.git.GitDetails;
 import org.springframework.data.mongodb.repository.Query;
 import org.springframework.data.repository.PagingAndSortingRepository;
 

src/main/java/org/commonwl/view/workflow/Workflow.java

@@ -23,7 +23,7 @@
 import com.fasterxml.jackson.annotation.JsonInclude;
 import org.commonwl.view.cwl.CWLElement;
 import org.commonwl.view.cwl.CWLStep;
-import org.commonwl.view.github.GitDetails;
+import org.commonwl.view.git.GitDetails;
 import org.springframework.data.annotation.Id;
 import org.springframework.data.mongodb.core.index.Indexed;
 import org.springframework.data.mongodb.core.mapping.Document;

src/main/java/org/commonwl/view/workflow/WorkflowController.java

@@ -22,8 +22,8 @@
 import org.apache.commons.lang.StringUtils;
 import org.commonwl.view.cwl.CWLToolStatus;
 import org.commonwl.view.cwl.CWLValidationException;
-import org.commonwl.view.github.GitDetails;
-import org.commonwl.view.github.GitType;
+import org.commonwl.view.git.GitDetails;
+import org.commonwl.view.git.GitType;
 import org.commonwl.view.graphviz.GraphVizService;
 import org.eclipse.jgit.api.errors.GitAPIException;
 import org.slf4j.Logger;

src/main/java/org/commonwl/view/workflow/WorkflowFormValidator.java

@@ -19,9 +19,9 @@
 
 package org.commonwl.view.workflow;
 
-import org.commonwl.view.github.GitDetails;
-import org.commonwl.view.github.GitService;
-import org.commonwl.view.github.GitType;
+import org.commonwl.view.git.GitDetails;
+import org.commonwl.view.git.GitService;
+import org.commonwl.view.git.GitType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;