Revert "Investigation into fetching from remote repo with JGit"

This reverts commit 49f3f37.

Author: MarkRobbo

src/main/java/org/commonwl/view/cwl/CWLService.java

@@ -24,6 +24,7 @@
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.TextNode;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.jena.ontology.OntModelSpec;
 import org.apache.jena.query.QuerySolution;
@@ -35,24 +36,26 @@
 import org.apache.jena.riot.RiotException;
 import org.commonwl.view.docker.DockerService;
 import org.commonwl.view.git.GitDetails;
-import org.commonwl.view.git.GitService;
 import org.commonwl.view.graphviz.ModelDotWriter;
 import org.commonwl.view.graphviz.RDFDotWriter;
 import org.commonwl.view.workflow.Workflow;
-import org.eclipse.jgit.errors.LargeObjectException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.yaml.snakeyaml.Yaml;
 
 import java.io.ByteArrayInputStream;
+import java.io.File;
 import java.io.IOException;
 import java.io.StringWriter;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.*;
 
+import static org.apache.commons.io.FileUtils.readFileToString;
+
 /**
  * Provides CWL parsing for workflows to gather an overview
  * for display and visualisation
@@ -65,7 +68,7 @@ public class CWLService {
     // Autowired properties/services
     private final RDFService rdfService;
     private final CWLTool cwlTool;
-    private final GitService gitService;
+    private final int singleFileSizeLimit;
 
     // CWL specific strings
     private final String DOC_GRAPH = "$graph";
@@ -95,27 +98,30 @@ public class CWLService {
      * Constructor for the Common Workflow Language service
      * @param rdfService A service for handling RDF queries
      * @param cwlTool Handles cwltool integration
-     * @param gitService Handles Git repository functionality
+     * @param singleFileSizeLimit The file size limit for single files
      */
     @Autowired
     public CWLService(RDFService rdfService,
                       CWLTool cwlTool,
-                      GitService gitService) {
+                      @Value("${singleFileSizeLimit}") int singleFileSizeLimit) {
         this.rdfService = rdfService;
         this.cwlTool = cwlTool;
-        this.gitService = gitService;
+        this.singleFileSizeLimit = singleFileSizeLimit;
     }
 
     /**
      * Gets the Workflow object from internal parsing
-     * @param gitDetails The details for the workflow in the git repository
+     * @param workflowFile The workflow file to be parsed
      * @return The constructed workflow object
      */
-    public Workflow parseWorkflowNative(GitDetails gitDetails) throws IOException {
-        try {
+    public Workflow parseWorkflowNative(File workflowFile) throws IOException {
+
+        // Check file size limit before parsing
+        long fileSizeBytes = workflowFile.length();
+        if (fileSizeBytes <= singleFileSizeLimit) {
+
             // Parse file as yaml
-            JsonNode cwlFile = yamlStringToJson(gitService.getFile(gitService
-                    .getRepository(gitDetails).getRepository(), gitDetails.getPath()));
+            JsonNode cwlFile = yamlStringToJson(readFileToString(workflowFile));
 
             // If the CWL file is packed there can be multiple workflows in a file
             Map<String, JsonNode> packedFiles = new HashMap<>();
@@ -132,7 +138,7 @@ public Workflow parseWorkflowNative(GitDetails gitDetails) throws IOException {
             // Use filename for label if there is no defined one
             String label = extractLabel(cwlFile);
             if (label == null) {
-                label = FilenameUtils.getName(gitDetails.getPath());
+                label = FilenameUtils.getName(workflowFile.getPath());
             }
 
             // Construct the rest of the workflow model
@@ -157,19 +163,22 @@ public Workflow parseWorkflowNative(GitDetails gitDetails) throws IOException {
 
             return workflowModel;
 
-        } catch (LargeObjectException ex) {
-            throw new IOException("File '" + FilenameUtils.getName(gitDetails.getPath()) +
-                    "' is over singleFileSizeLimit";
+        } else {
+            throw new IOException("File '" + workflowFile.getName() +  "' is over singleFileSizeLimit - " +
+                    FileUtils.byteCountToDisplaySize(fileSizeBytes) + "/" +
+                    FileUtils.byteCountToDisplaySize(singleFileSizeLimit));
         }
 
     }
 
     /**
      * Create a workflow model using cwltool rdf output
      * @param basicModel The basic workflow object created thus far
+     * @param workflowFile The workflow file to run cwltool on
      * @return The constructed workflow object
      */
-    public Workflow parseWorkflowWithCwltool(Workflow basicModel) throws CWLValidationException {
+    public Workflow parseWorkflowWithCwltool(Workflow basicModel,
+                                             File workflowFile) throws CWLValidationException {
         GitDetails gitDetails = basicModel.getRetrievedFrom();
         String latestCommit = basicModel.getLastCommit();
         String packedWorkflowID = basicModel.getPackedWorkflowID();

src/main/java/org/commonwl/view/cwl/CWLToolRunner.java

@@ -31,6 +31,7 @@
 import org.springframework.scheduling.annotation.EnableAsync;
 import org.springframework.stereotype.Component;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.Date;
 
@@ -63,14 +64,16 @@ public CWLToolRunner(WorkflowRepository workflowRepository,
     }
 
     @Async
-    public void createWorkflowFromQueued(QueuedWorkflow queuedWorkflow)
+    public void createWorkflowFromQueued(QueuedWorkflow queuedWorkflow, File workflowFile)
             throws IOException, InterruptedException {
 
         Workflow tempWorkflow = queuedWorkflow.getTempRepresentation();
 
         // Parse using cwltool and replace in database
         try {
-            Workflow newWorkflow = cwlService.parseWorkflowWithCwltool(tempWorkflow);
+            Workflow newWorkflow = cwlService.parseWorkflowWithCwltool(
+                    tempWorkflow,
+                    workflowFile);
 
             // Success
             newWorkflow.setRetrievedFrom(tempWorkflow.getRetrievedFrom());

src/main/java/org/commonwl/view/git/GitService.java

@@ -23,15 +23,8 @@
 import org.commonwl.view.researchobject.HashableAgent;
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.api.errors.GitAPIException;
-import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.ObjectLoader;
 import org.eclipse.jgit.lib.PersonIdent;
-import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.revwalk.RevCommit;
-import org.eclipse.jgit.revwalk.RevTree;
-import org.eclipse.jgit.revwalk.RevWalk;
-import org.eclipse.jgit.treewalk.TreeWalk;
-import org.eclipse.jgit.treewalk.filter.PathFilter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -60,16 +53,11 @@ public class GitService {
     // Whether submodules are also cloned
     private boolean cloneSubmodules;
 
-    // File size limit for loading in bytes
-    private int singleFileSizeLimit;
-
     @Autowired
     public GitService(@Value("${gitStorage}") Path gitStorage,
-                      @Value("${gitAPI.cloneSubmodules}") boolean cloneSubmodules,
-                      @Value("${singleFileSizeLimit}") int singleFileSizeLimit) {
+                      @Value("${gitAPI.cloneSubmodules}") boolean cloneSubmodules) {
         this.gitStorage = gitStorage;
         this.cloneSubmodules = cloneSubmodules;
-        this.singleFileSizeLimit = singleFileSizeLimit;
     }
 
     /**
@@ -120,46 +108,6 @@ public Git getRepository(GitDetails gitDetails)
         return repo;
     }
 
-    /**
-     * Get the contents of a file from the Git repository
-     * @param repository The Git repository
-     * @param refOrCommitId The branch name or commit ID as a string
-     * @return The contents of the file as a string
-     * @throws IOException Any errors in retrieving the file
-     */
-    public String getFile(Repository repository, String refOrCommitId) throws IOException {
-        String content;
-
-        // Get the ObjectID from the ref or commit ID string
-        if (!ObjectId.isId(refOrCommitId)) {
-            refOrCommitId = "refs/remotes/origin/" + refOrCommitId;
-        }
-        ObjectId commitId = repository.resolve(refOrCommitId);
-
-        // Walk over commits using defined filtering
-        try (RevWalk revWalk = new RevWalk(repository)) {
-            RevCommit commit = revWalk.parseCommit(commitId);
-            RevTree tree = commit.getTree();
-
-            // Try to find specific file in the repository
-            try (TreeWalk treeWalk = new TreeWalk(repository)) {
-                treeWalk.addTree(tree);
-                treeWalk.setRecursive(true);
-                treeWalk.setFilter(PathFilter.create("README.md"));
-                if (!treeWalk.next()) {
-                    throw new IllegalStateException("Did not find expected file");
-                }
-
-                ObjectId objectId = treeWalk.getObjectId(0);
-                ObjectLoader loader = repository.open(objectId);
-                content = new String(loader.getCachedBytes(singleFileSizeLimit));
-            }
-            revWalk.dispose();
-        }
-
-        return content;
-    }
-
     /**
      * Gets the commit ID of the HEAD for the given repository
      * @param repo The Git repository

src/main/java/org/commonwl/view/workflow/WorkflowService.java

@@ -236,7 +236,14 @@ public QueuedWorkflow createQueuedWorkflow(GitDetails gitInfo)
         File localPath = repo.getRepository().getWorkTree();
         String latestCommit = gitService.getCurrentCommitID(repo);
 
-        Workflow basicModel = cwlService.parseWorkflowNative(gitInfo);
+        Path pathToWorkflowFile = localPath.toPath().resolve(gitInfo.getPath()).normalize().toAbsolutePath();
+        // Prevent path traversal attacks
+        if (!pathToWorkflowFile.startsWith(localPath.toPath().normalize().toAbsolutePath())) {
+            throw new WorkflowNotFoundException();
+        }
+
+        File workflowFile = new File(pathToWorkflowFile.toString());
+        Workflow basicModel = cwlService.parseWorkflowNative(workflowFile);
 
         // Set origin details
         basicModel.setRetrievedOn(new Date());
@@ -251,7 +258,7 @@ public QueuedWorkflow createQueuedWorkflow(GitDetails gitInfo)
         // ASYNC OPERATIONS
         // Parse with cwltool and update model
         try {
-            cwlToolRunner.createWorkflowFromQueued(queuedWorkflow);
+            cwlToolRunner.createWorkflowFromQueued(queuedWorkflow, workflowFile);
         } catch (Exception e) {
             logger.error("Could not update workflow with cwltool", e);
         }