Add local cloning to tmp directory

Author: MarkRobbo

pom.xml

@@ -62,11 +62,6 @@
 			<artifactId>commons-io</artifactId>
 			<version>1.3.2</version>
 		</dependency>
-		<dependency>
-			<groupId>org.eclipse.mylyn.github</groupId>
-			<artifactId>org.eclipse.egit.github.core</artifactId>
-			<version>2.1.5</version>
-		</dependency>
 		<dependency>
 			<groupId>org.yaml</groupId>
 			<artifactId>snakeyaml</artifactId>

src/main/java/org/commonwl/view/cwl/CWLService.java

@@ -35,12 +35,10 @@
 import org.apache.jena.rdf.model.StmtIterator;
 import org.apache.jena.riot.RiotException;
 import org.commonwl.view.docker.DockerService;
-import org.commonwl.view.github.GitDetails;
 import org.commonwl.view.github.GitService;
 import org.commonwl.view.graphviz.ModelDotWriter;
 import org.commonwl.view.graphviz.RDFDotWriter;
 import org.commonwl.view.workflow.Workflow;
-import org.commonwl.view.workflow.WorkflowOverview;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -178,18 +176,15 @@ public Workflow parseWorkflowNative(File workflowFile) throws IOException {
 
     /**
      * Create a workflow model from cwltool rdf output
-     * @param githubInfo The Github repository information
-     * @param latestCommit The latest commit ID
+     * @param workflowFile The workflow file to run cwltool on
      * @param packedWorkflowID The workflow ID if the file has multiple objects
      * @return The constructed workflow object
      */
-    public Workflow parseWorkflowWithCwltool(GitDetails githubInfo,
-                                             String latestCommit,
+    public Workflow parseWorkflowWithCwltool(File workflowFile,
                                              String packedWorkflowID) throws CWLValidationException {
 
         // Get RDF representation from cwltool
-        String url = String.format("https://cdn.rawgit.com/%s/%s/%s/%s", githubInfo.getOwner(),
-                githubInfo.getRepoName(), latestCommit, githubInfo.getPath());
+        String url = workflowFile.toPath().toAbsolutePath().toString();
         if (packedWorkflowID != null) {
             if (packedWorkflowID.charAt(0) != '#') {
                 url += "#";
@@ -209,7 +204,7 @@ public Workflow parseWorkflowWithCwltool(GitDetails githubInfo,
         }
 
         // Base workflow details
-        String label = FilenameUtils.getName(githubInfo.getPath());
+        String label = FilenameUtils.getName(url);
         String doc = null;
         ResultSet labelAndDoc = rdfService.getLabelAndDoc(url);
         if (labelAndDoc.hasNext()) {
@@ -425,57 +420,6 @@ public Workflow parseWorkflowWithCwltool(GitDetails githubInfo,
 
     }
 
-    /**
-     * Get an overview of a workflow
-     * @param githubInfo The details to access the workflow
-     * @return A constructed WorkflowOverview of the workflow
-     * @throws IOException Any API errors which may have occurred
-     */
-    public WorkflowOverview getWorkflowOverview(GitDetails githubInfo) throws IOException {
-
-        // Get the content of this file from Github
-        String fileContent = githubService.downloadFile(githubInfo);
-        int fileSizeBytes = fileContent.getBytes("UTF-8").length;
-
-        // Check file size limit before parsing
-        if (fileSizeBytes <= singleFileSizeLimit) {
-
-            // Parse file as yaml
-            JsonNode cwlFile = yamlStringToJson(fileContent);
-
-            // If the CWL file is packed there can be multiple workflows in a file
-            if (cwlFile.has(DOC_GRAPH)) {
-                // Packed CWL, find the first subelement which is a workflow and take it
-                for (JsonNode jsonNode : cwlFile.get(DOC_GRAPH)) {
-                    if (extractProcess(jsonNode) == CWLProcess.WORKFLOW) {
-                        cwlFile = jsonNode;
-                    }
-                }
-            }
-
-            // Can only make an overview if this is a workflow
-            if (extractProcess(cwlFile) == CWLProcess.WORKFLOW) {
-                // Use filename for label if there is no defined one
-                String path = FilenameUtils.getName(githubInfo.getPath());
-                String label = extractLabel(cwlFile);
-                if (label == null) {
-                    label = path;
-                }
-
-                // Return the constructed overview
-                return new WorkflowOverview(path, label, extractDoc(cwlFile));
-
-            } else {
-                return null;
-            }
-        } else {
-            throw new IOException("File '" + githubInfo.getPath() +  "' is over singleFileSizeLimit - " +
-                    FileUtils.byteCountToDisplaySize(fileSizeBytes) + "/" +
-                    FileUtils.byteCountToDisplaySize(singleFileSizeLimit));
-        }
-
-    }
-
     /**
      * Set the format for an input or output, handling ontologies
      * @param inputOutput The input or output CWL Element

src/main/java/org/commonwl/view/cwl/CWLToolRunner.java

@@ -32,6 +32,7 @@
 import org.springframework.scheduling.annotation.EnableAsync;
 import org.springframework.stereotype.Component;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.Date;
 
@@ -67,22 +68,21 @@ public CWLToolRunner(WorkflowRepository workflowRepository,
     }
 
     @Async
-    public void createWorkflowFromQueued(QueuedWorkflow queuedWorkflow)
+    public void createWorkflowFromQueued(QueuedWorkflow queuedWorkflow, File workflowFile)
             throws IOException, InterruptedException {
 
         Workflow tempWorkflow = queuedWorkflow.getTempRepresentation();
 
         // Parse using cwltool and replace in database
         try {
-            String commitSha = githubService.getCommitSha(tempWorkflow.getRetrievedFrom());
             Workflow newWorkflow = cwlService.parseWorkflowWithCwltool(
-                    tempWorkflow.getRetrievedFrom(), commitSha,
+                    workflowFile,
                     tempWorkflow.getPackedWorkflowID());
 
             // Success
             newWorkflow.setRetrievedFrom(tempWorkflow.getRetrievedFrom());
             newWorkflow.setRetrievedOn(new Date());
-            newWorkflow.setLastCommit(commitSha);
+            newWorkflow.setLastCommit(tempWorkflow.getLastCommit());
             newWorkflow.setCwltoolVersion(cwlToolVersion);
             workflowRepository.save(newWorkflow);
 

src/main/java/org/commonwl/view/github/GitService.java

@@ -21,45 +21,44 @@
 
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.api.errors.GitAPIException;
-import org.eclipse.jgit.lib.Ref;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
-import java.io.File;
-import java.util.Map;
+import java.nio.file.Path;
 
 import static java.util.Collections.singleton;
+import static org.apache.jena.ext.com.google.common.io.Files.createTempDir;
 
 /**
  * Handles Git related functionality
  */
 @Service
 public class GitService {
 
-    /**
-     * Gets a map of commits and their references from a remote repository
-     * @param gitDetails
-     * @return
-     * @throws GitAPIException
-     */
-    public Map<String, Ref> getCommits(GitDetails gitDetails) throws GitAPIException {
-        return Git.lsRemoteRepository()
-                .setHeads(true)
-                .setTags(true)
-                .setRemote(gitDetails.getRepoUrl())
-                .callAsMap();
+    // Location to check out git repositories into
+    private Path gitStorage;
+
+    // Whether submodules are also cloned
+    private boolean cloneSubmodules;
+
+    @Autowired
+    public GitService(@Value("${gitStorage}") Path gitStorage,
+                      @Value("${gitAPI.cloneSubmodules}") boolean cloneSubmodules) {
+        this.gitStorage = gitStorage;
+        this.cloneSubmodules = cloneSubmodules;
     }
 
     /**
      * Clone a repository into a local directory
      * @param gitDetails The details of the Git repository
-     * @param dest The destination folder for the clone
      */
-    public Git cloneRepository(GitDetails gitDetails, File dest)
+    public Git cloneRepository(GitDetails gitDetails)
             throws GitAPIException {
         return Git.cloneRepository()
-                .setCloneSubmodules(true)
+                .setCloneSubmodules(cloneSubmodules)
                 .setURI(gitDetails.getRepoUrl())
-                .setDirectory(dest)
+                .setDirectory(createTempDir())
                 .setBranchesToClone(singleton("refs/heads/" + gitDetails.getBranch()))
                 .setBranch("refs/heads/" + gitDetails.getBranch())
                 .call();

src/main/java/org/commonwl/view/researchobject/ROBundleFactory.java

@@ -57,7 +57,7 @@ public ROBundleFactory(ROBundleService roBundleService,
     }
 
     /**
-     * Creates a new Workflow Research Object Bundle from a Github URL
+     * Creates a new Workflow Research Object Bundle from Git details
      * and saves it to a file
      * @param workflow The workflow to generate a RO bundle for
      * @throws IOException Any API errors which may have occurred
@@ -69,11 +69,11 @@ public void workflowROFromGithub(Workflow workflow)
 
         // Get the whole containing folder, not just the workflow itself
         GitDetails githubInfo = workflow.getRetrievedFrom();
-        GitDetails roDetails = new GitDetails(githubInfo.getOwner(), githubInfo.getRepoName(),
-                githubInfo.getBranch(), FilenameUtils.getPath(githubInfo.getPath()));
+        GitDetails roDetails = new GitDetails(githubInfo.getRepoUrl(), githubInfo.getBranch(),
+                FilenameUtils.getPath(githubInfo.getPath()), githubInfo.getType());
 
         // Create a new Research Object Bundle with Github contents
-        Bundle bundle = roBundleService.newBundleFromGithub(workflow, roDetails);
+        Bundle bundle = roBundleService.createBundle(workflow, roDetails);
 
         // Save the bundle to the storage location in properties
         Path bundleLocation = roBundleService.saveToFile(bundle);

src/main/java/org/commonwl/view/researchobject/ROBundleService.java

@@ -19,38 +19,30 @@
 
 package org.commonwl.view.researchobject;
 
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.FilenameUtils;
 import org.apache.taverna.robundle.Bundle;
 import org.apache.taverna.robundle.Bundles;
-import org.apache.taverna.robundle.manifest.*;
+import org.apache.taverna.robundle.manifest.Agent;
+import org.apache.taverna.robundle.manifest.Manifest;
+import org.apache.taverna.robundle.manifest.PathAnnotation;
 import org.commonwl.view.cwl.CWLTool;
 import org.commonwl.view.github.GitDetails;
 import org.commonwl.view.github.GitService;
 import org.commonwl.view.graphviz.GraphVizService;
 import org.commonwl.view.workflow.Workflow;
-import org.eclipse.egit.github.core.CommitUser;
-import org.eclipse.egit.github.core.RepositoryCommit;
-import org.eclipse.egit.github.core.RepositoryContents;
-import org.eclipse.egit.github.core.User;
+import org.eclipse.jgit.api.Git;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
-import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 /**
@@ -102,16 +94,16 @@ public ROBundleService(@Value("${bundleStorage}") Path bundleStorage,
     }
 
     /**
-     * Creates a new research object bundle for a workflow from a Github repository
+     * Creates a new research object bundle for a workflow from a Git repository
      * @param workflow The workflow to create the research object for
      * @return The constructed bundle
      */
-    public Bundle newBundleFromGithub(Workflow workflow, GitDetails githubInfo) throws IOException {
+    public Bundle createBundle(Workflow workflow, GitDetails gitInfo) throws IOException {
 
         // Create a new RO bundle
         Bundle bundle = Bundles.createBundle();
         Manifest manifest = bundle.getManifest();
-
+/*
         // Simplified attribution for RO bundle
         try {
             // Tool attribution in createdBy
@@ -121,17 +113,17 @@ public Bundle newBundleFromGithub(Workflow workflow, GitDetails githubInfo) thro
             // TODO: Make this importedBy/On/From
             manifest.setRetrievedBy(appAgent);
             manifest.setRetrievedOn(manifest.getCreatedOn());
-            manifest.setRetrievedFrom(new URI(githubInfo.getUrl()));
+            manifest.setRetrievedFrom(new URI(gitInfo.getUrl()));
 
             // Make a directory in the RO bundle to store the files
             Path bundleRoot = bundle.getRoot();
             Path bundleFiles = bundleRoot.resolve("workflow");
             Files.createDirectory(bundleFiles);
 
             // Add the files from the Github repo to this workflow
-            List<RepositoryContents> repoContents = githubService.getContents(githubInfo);
             Set<HashableAgent> authors = new HashSet<>();
-            addFilesToBundle(bundle, githubInfo, repoContents, bundleFiles, authors);
+            Git gitRepo = Git.open(new File(workflow.getGitRepoPath()));
+            addFilesToBundle(bundle, gitInfo, gitRepo, bundleFiles, authors);
 
             // Add combined authors
             manifest.setAuthoredBy(new ArrayList<>(authors));
@@ -161,13 +153,13 @@ public Bundle newBundleFromGithub(Workflow workflow, GitDetails githubInfo) thro
 
             // Git2prov history
             List<Path> history = new ArrayList<>();
-            history.add(Paths.get("http://git2prov.org/git2prov?giturl=https://github.com/" + githubInfo.getOwner()
-                    + "/" + githubInfo.getRepoName() + "&serialization=PROV-JSON"));
+            history.add(Paths.get("http://git2prov.org/git2prov?giturl=" +
+                    gitInfo.getRepoUrl() + "&serialization=PROV-JSON"));
             bundle.getManifest().setHistory(history);
 
         } catch (URISyntaxException ex) {
             logger.error("Error creating URI for RO Bundle", ex);
-        }
+        }*/
 
         // Return the completed bundle
         return bundle;
@@ -177,39 +169,34 @@ public Bundle newBundleFromGithub(Workflow workflow, GitDetails githubInfo) thro
     /**
      * Add files to this bundle from a list of Github repository contents
      * @param bundle The RO bundle to add files/directories to
-     * @param githubInfo The information used to access the repository
-     * @param repoContents The contents of the Github repository
+     * @param gitInfo The information used to access the repository
+     * @param gitRepo The Git repository
      * @param path The path in the Research Object to add the files
      */
-    private void addFilesToBundle(Bundle bundle, GitDetails githubInfo,
-                                  List<RepositoryContents> repoContents, Path path,
+    private void addFilesToBundle(Bundle bundle, GitDetails gitInfo,
+                                  Git gitRepo, Path path,
                                   Set<HashableAgent> authors) throws IOException {
-
-        // Loop through repo contents and add them
-        for (RepositoryContents repoContent : repoContents) {
-
-            // Parse subdirectories if they exist
-            if (repoContent.getType().equals(GitService.TYPE_DIR)) {
+        /*File[] files = gitRepo.getRepository().getDirectory().listFiles();
+        for (File file : files) {
+            if (file.isDirectory()) {
 
                 // Get the contents of the subdirectory
-                GitDetails githubSubdir = new GitDetails(githubInfo.getOwner(),
-                        githubInfo.getRepoName(), githubInfo.getBranch(), repoContent.getPath());
+                GitDetails githubSubdir = new GitDetails(gitInfo.getOwner(),
+                        gitInfo.getRepoName(), gitInfo.getBranch(), repoContent.getPath());
                 List<RepositoryContents> subdirectory = githubService.getContents(githubSubdir);
 
                 // Create a new folder in the RO for it
                 Path subdirPath = path.resolve(repoContent.getName());
                 Files.createDirectory(subdirPath);
 
                 // Add the files in the subdirectory to this new folder
-                addFilesToBundle(bundle, githubInfo, subdirectory, subdirPath, authors);
-
-                // Otherwise this is a file so add to the bundle
-            } else if (repoContent.getType().equals(GitService.TYPE_FILE)) {
+                addFilesToBundle(bundle, gitInfo, subdirectory, subdirPath, authors);
 
+            } else {
                 try {
                     // Raw URI of the bundle
-                    GitDetails githubFile = new GitDetails(githubInfo.getOwner(),
-                            githubInfo.getRepoName(), githubInfo.getBranch(), repoContent.getPath());
+                    GitDetails githubFile = new GitDetails(gitInfo.getOwner(),
+                            gitInfo.getRepoName(), gitInfo.getBranch(), repoContent.getPath());
 
                     // Where to store the new file in bundle
                     Path bundleFilePath = path.resolve(repoContent.getName());
@@ -297,7 +284,7 @@ private void addFilesToBundle(Bundle bundle, GitDetails githubInfo,
                     logger.error("Error creating URI for RO Bundle", ex);
                 }
             }
-        }
+        }*/
     }
 
     /**