Investigation into fetching from remote repo with JGit

MarkRobbo · MarkRobbo · commit 49f3f3733d8c · 2017-08-08T13:04:52.000+01:00
As a solution to concurrency issues
diff --git a/src/main/java/org/commonwl/view/cwl/CWLService.java b/src/main/java/org/commonwl/view/cwl/CWLService.java
@@ -24,7 +24,6 @@
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.TextNode;
-import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.jena.ontology.OntModelSpec;
 import org.apache.jena.query.QuerySolution;
@@ -36,26 +35,24 @@
 import org.apache.jena.riot.RiotException;
 import org.commonwl.view.docker.DockerService;
 import org.commonwl.view.git.GitDetails;
+import org.commonwl.view.git.GitService;
 import org.commonwl.view.graphviz.ModelDotWriter;
 import org.commonwl.view.graphviz.RDFDotWriter;
 import org.commonwl.view.workflow.Workflow;
+import org.eclipse.jgit.errors.LargeObjectException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.yaml.snakeyaml.Yaml;
 
 import java.io.ByteArrayInputStream;
-import java.io.File;
 import java.io.IOException;
 import java.io.StringWriter;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.*;
 
-import static org.apache.commons.io.FileUtils.readFileToString;
-
 /**
  * Provides CWL parsing for workflows to gather an overview
  * for display and visualisation
@@ -68,7 +65,7 @@ public class CWLService {
     // Autowired properties/services
     private final RDFService rdfService;
     private final CWLTool cwlTool;
-    private final int singleFileSizeLimit;
+    private final GitService gitService;
 
     // CWL specific strings
     private final String DOC_GRAPH = "$graph";
@@ -98,30 +95,27 @@ public class CWLService {
      * Constructor for the Common Workflow Language service
      * @param rdfService A service for handling RDF queries
      * @param cwlTool Handles cwltool integration
-     * @param singleFileSizeLimit The file size limit for single files
+     * @param gitService Handles Git repository functionality
      */
     @Autowired
     public CWLService(RDFService rdfService,
                       CWLTool cwlTool,
-                      @Value("${singleFileSizeLimit}") int singleFileSizeLimit) {
+                      GitService gitService) {
         this.rdfService = rdfService;
         this.cwlTool = cwlTool;
-        this.singleFileSizeLimit = singleFileSizeLimit;
+        this.gitService = gitService;
     }
 
     /**
      * Gets the Workflow object from internal parsing
-     * @param workflowFile The workflow file to be parsed
+     * @param gitDetails The details for the workflow in the git repository
      * @return The constructed workflow object
      */
-    public Workflow parseWorkflowNative(File workflowFile) throws IOException {
-
-        // Check file size limit before parsing
-        long fileSizeBytes = workflowFile.length();
-        if (fileSizeBytes <= singleFileSizeLimit) {
-
+    public Workflow parseWorkflowNative(GitDetails gitDetails) throws IOException {
+        try {
             // Parse file as yaml
-            JsonNode cwlFile = yamlStringToJson(readFileToString(workflowFile));
+            JsonNode cwlFile = yamlStringToJson(gitService.getFile(gitService
+                    .getRepository(gitDetails).getRepository(), gitDetails.getPath()));
 
             // If the CWL file is packed there can be multiple workflows in a file
             Map<String, JsonNode> packedFiles = new HashMap<>();
@@ -138,7 +132,7 @@ public Workflow parseWorkflowNative(File workflowFile) throws IOException {
             // Use filename for label if there is no defined one
             String label = extractLabel(cwlFile);
             if (label == null) {
-                label = FilenameUtils.getName(workflowFile.getPath());
+                label = FilenameUtils.getName(gitDetails.getPath());
             }
 
             // Construct the rest of the workflow model
@@ -163,22 +157,19 @@ public Workflow parseWorkflowNative(File workflowFile) throws IOException {
 
             return workflowModel;
 
-        } else {
-            throw new IOException("File '" + workflowFile.getName() +  "' is over singleFileSizeLimit - " +
-                    FileUtils.byteCountToDisplaySize(fileSizeBytes) + "/" +
-                    FileUtils.byteCountToDisplaySize(singleFileSizeLimit));
+        } catch (LargeObjectException ex) {
+            throw new IOException("File '" + FilenameUtils.getName(gitDetails.getPath()) +
+                    "' is over singleFileSizeLimit";
         }
 
     }
 
     /**
      * Create a workflow model using cwltool rdf output
      * @param basicModel The basic workflow object created thus far
-     * @param workflowFile The workflow file to run cwltool on
      * @return The constructed workflow object
      */
-    public Workflow parseWorkflowWithCwltool(Workflow basicModel,
-                                             File workflowFile) throws CWLValidationException {
+    public Workflow parseWorkflowWithCwltool(Workflow basicModel) throws CWLValidationException {
         GitDetails gitDetails = basicModel.getRetrievedFrom();
         String latestCommit = basicModel.getLastCommit();
         String packedWorkflowID = basicModel.getPackedWorkflowID();
diff --git a/src/main/java/org/commonwl/view/cwl/CWLToolRunner.java b/src/main/java/org/commonwl/view/cwl/CWLToolRunner.java
@@ -31,7 +31,6 @@
 import org.springframework.scheduling.annotation.EnableAsync;
 import org.springframework.stereotype.Component;
 
-import java.io.File;
 import java.io.IOException;
 import java.util.Date;
 
@@ -64,16 +63,14 @@ public CWLToolRunner(WorkflowRepository workflowRepository,
     }
 
     @Async
-    public void createWorkflowFromQueued(QueuedWorkflow queuedWorkflow, File workflowFile)
+    public void createWorkflowFromQueued(QueuedWorkflow queuedWorkflow)
             throws IOException, InterruptedException {
 
         Workflow tempWorkflow = queuedWorkflow.getTempRepresentation();
 
         // Parse using cwltool and replace in database
         try {
-            Workflow newWorkflow = cwlService.parseWorkflowWithCwltool(
-                    tempWorkflow,
-                    workflowFile);
+            Workflow newWorkflow = cwlService.parseWorkflowWithCwltool(tempWorkflow);
 
             // Success
             newWorkflow.setRetrievedFrom(tempWorkflow.getRetrievedFrom());
diff --git a/src/main/java/org/commonwl/view/git/GitService.java b/src/main/java/org/commonwl/view/git/GitService.java
@@ -23,8 +23,15 @@
 import org.commonwl.view.researchobject.HashableAgent;
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectLoader;
 import org.eclipse.jgit.lib.PersonIdent;
+import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevTree;
+import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.treewalk.TreeWalk;
+import org.eclipse.jgit.treewalk.filter.PathFilter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -53,11 +60,16 @@ public class GitService {
     // Whether submodules are also cloned
     private boolean cloneSubmodules;
 
+    // File size limit for loading in bytes
+    private int singleFileSizeLimit;
+
     @Autowired
     public GitService(@Value("${gitStorage}") Path gitStorage,
-                      @Value("${gitAPI.cloneSubmodules}") boolean cloneSubmodules) {
+                      @Value("${gitAPI.cloneSubmodules}") boolean cloneSubmodules,
+                      @Value("${singleFileSizeLimit}") int singleFileSizeLimit) {
         this.gitStorage = gitStorage;
         this.cloneSubmodules = cloneSubmodules;
+        this.singleFileSizeLimit = singleFileSizeLimit;
     }
 
     /**
@@ -108,6 +120,46 @@ public Git getRepository(GitDetails gitDetails)
         return repo;
     }
 
+    /**
+     * Get the contents of a file from the Git repository
+     * @param repository The Git repository
+     * @param refOrCommitId The branch name or commit ID as a string
+     * @return The contents of the file as a string
+     * @throws IOException Any errors in retrieving the file
+     */
+    public String getFile(Repository repository, String refOrCommitId) throws IOException {
+        String content;
+
+        // Get the ObjectID from the ref or commit ID string
+        if (!ObjectId.isId(refOrCommitId)) {
+            refOrCommitId = "refs/remotes/origin/" + refOrCommitId;
+        }
+        ObjectId commitId = repository.resolve(refOrCommitId);
+
+        // Walk over commits using defined filtering
+        try (RevWalk revWalk = new RevWalk(repository)) {
+            RevCommit commit = revWalk.parseCommit(commitId);
+            RevTree tree = commit.getTree();
+
+            // Try to find specific file in the repository
+            try (TreeWalk treeWalk = new TreeWalk(repository)) {
+                treeWalk.addTree(tree);
+                treeWalk.setRecursive(true);
+                treeWalk.setFilter(PathFilter.create("README.md"));
+                if (!treeWalk.next()) {
+                    throw new IllegalStateException("Did not find expected file");
+                }
+
+                ObjectId objectId = treeWalk.getObjectId(0);
+                ObjectLoader loader = repository.open(objectId);
+                content = new String(loader.getCachedBytes(singleFileSizeLimit));
+            }
+            revWalk.dispose();
+        }
+
+        return content;
+    }
+
     /**
      * Gets the commit ID of the HEAD for the given repository
      * @param repo The Git repository
diff --git a/src/main/java/org/commonwl/view/workflow/WorkflowService.java b/src/main/java/org/commonwl/view/workflow/WorkflowService.java
@@ -236,14 +236,7 @@ public QueuedWorkflow createQueuedWorkflow(GitDetails gitInfo)
         File localPath = repo.getRepository().getWorkTree();
         String latestCommit = gitService.getCurrentCommitID(repo);
 
-        Path pathToWorkflowFile = localPath.toPath().resolve(gitInfo.getPath()).normalize().toAbsolutePath();
-        // Prevent path traversal attacks
-        if (!pathToWorkflowFile.startsWith(localPath.toPath().normalize().toAbsolutePath())) {
-            throw new WorkflowNotFoundException();
-        }
-
-        File workflowFile = new File(pathToWorkflowFile.toString());
-        Workflow basicModel = cwlService.parseWorkflowNative(workflowFile);
+        Workflow basicModel = cwlService.parseWorkflowNative(gitInfo);
 
         // Set origin details
         basicModel.setRetrievedOn(new Date());
@@ -258,7 +251,7 @@ public QueuedWorkflow createQueuedWorkflow(GitDetails gitInfo)
         // ASYNC OPERATIONS
         // Parse with cwltool and update model
         try {
-            cwlToolRunner.createWorkflowFromQueued(queuedWorkflow, workflowFile);
+            cwlToolRunner.createWorkflowFromQueued(queuedWorkflow);
         } catch (Exception e) {
             logger.error("Could not update workflow with cwltool", e);
         }
diff --git a/src/test/java/org/commonwl/view/git/GitServiceTest.java b/src/test/java/org/commonwl/view/git/GitServiceTest.java
@@ -0,0 +1,23 @@
+package org.commonwl.view.git;
+
+import org.eclipse.jgit.api.Git;
+import org.junit.Test;
+
+import java.io.File;
+
+/**
+ * Created by mark on 08/08/17.
+ */
+public class GitServiceTest {
+
+    @Test
+    public void name() throws Exception {
+        GitService gitService = new GitService(new File("/tmp").toPath(), true);
+
+        Git test = Git.open(new File("/home/mark/workflows/.git"));
+        String content = gitService.getFile(test.getRepository(), "aab378267a528c67f634e421be40ab6f19f3b078");
+
+        System.out.println(content);
+    }
+
+}
