Add total file size limit to the content of the repository at that path

Closes #13

Author: Mark Robinson

src/main/java/org/commonwl/viewer/domain/CWLCollection.java

@@ -27,8 +27,8 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.commonwl.viewer.services.DockerService;
-import org.eclipse.egit.github.core.RepositoryContents;
 import org.commonwl.viewer.services.GitHubService;
+import org.eclipse.egit.github.core.RepositoryContents;
 import org.yaml.snakeyaml.Yaml;
 
 import java.io.IOException;
@@ -44,6 +44,8 @@ public class CWLCollection {
     private GithubDetails githubInfo;
     private String commitSha;
 
+    private int totalFileSize;
+    private int totalFileSizeLimit;
     private int singleFileSizeLimit;
 
     // Maps of ID to associated JSON
@@ -93,11 +95,13 @@ public class CWLCollection {
      * @throws IOException Any API errors which may have occurred
      */
     public CWLCollection(GitHubService githubService, GithubDetails githubInfo,
-                         String commitSha, int singleFileSizeLimit) throws IOException {
+                         String commitSha, int singleFileSizeLimit, int totalFileSizeLimit) throws IOException {
         this.githubInfo = githubInfo;
         this.githubService = githubService;
         this.commitSha = commitSha;
         this.singleFileSizeLimit = singleFileSizeLimit;
+        this.totalFileSizeLimit = totalFileSizeLimit;
+        this.totalFileSize = 0;
 
         // Add any CWL files from the Github repo to this collection
         List<RepositoryContents> repoContents = githubService.getContents(githubInfo);
@@ -123,37 +127,41 @@ private void addDocs(List<RepositoryContents> repoContents) throws IOException {
                 // Add the files in the subdirectory to this new folder
                 addDocs(subdirectory);
 
-                // Otherwise this is a file so add to the bundle
             } else if (repoContent.getType().equals(FILE)) {
-
-                // Get the file extension
-                int eIndex = repoContent.getName().lastIndexOf('.') + 1;
-                if (eIndex > 0) {
-                    String extension = repoContent.getName().substring(eIndex);
-
-                    // If this is a cwl file which needs to be parsed
-                    if (extension.equals(CWL_EXTENSION)) {
-                        if (repoContent.getSize() <= singleFileSizeLimit) {
-                            // Get the content of this file from Github
-                            GithubDetails githubFile = new GithubDetails(githubInfo.getOwner(),
-                                    githubInfo.getRepoName(), githubInfo.getBranch(), repoContent.getPath());
-                            String fileContent = githubService.downloadFile(githubFile, commitSha);
-
-                            // Parse yaml to JsonNode
-                            Yaml reader = new Yaml();
-                            ObjectMapper mapper = new ObjectMapper();
-                            JsonNode cwlFile = mapper.valueToTree(reader.load(fileContent));
-
-                            // Add document to those being considered
-                            addDoc(cwlFile, repoContent.getName());
-                        } else {
-                            throw new IOException("File '" + repoContent.getName() +  "' is over singleFileSizeLimit - " +
-                                    FileUtils.byteCountToDisplaySize(repoContent.getSize()) + "/" +
-                                    FileUtils.byteCountToDisplaySize(singleFileSizeLimit));
+                // Keep track of total file size for limit
+                totalFileSize += repoContent.getSize();
+                if (totalFileSize <= totalFileSizeLimit) {
+                    // Get the file extension
+                    int eIndex = repoContent.getName().lastIndexOf('.') + 1;
+                    if (eIndex > 0) {
+                        String extension = repoContent.getName().substring(eIndex);
+
+                        // If this is a cwl file which needs to be parsed
+                        if (extension.equals(CWL_EXTENSION)) {
+                            if (repoContent.getSize() <= singleFileSizeLimit) {
+                                // Get the content of this file from Github
+                                GithubDetails githubFile = new GithubDetails(githubInfo.getOwner(),
+                                        githubInfo.getRepoName(), githubInfo.getBranch(), repoContent.getPath());
+                                String fileContent = githubService.downloadFile(githubFile, commitSha);
+
+                                // Parse yaml to JsonNode
+                                Yaml reader = new Yaml();
+                                ObjectMapper mapper = new ObjectMapper();
+                                JsonNode cwlFile = mapper.valueToTree(reader.load(fileContent));
+
+                                // Add document to those being considered
+                                addDoc(cwlFile, repoContent.getName());
+                            } else {
+                                throw new IOException("File '" + repoContent.getName() +  "' is over singleFileSizeLimit - " +
+                                        FileUtils.byteCountToDisplaySize(repoContent.getSize()) + "/" +
+                                        FileUtils.byteCountToDisplaySize(singleFileSizeLimit));
+                            }
                         }
                     }
+                } else {
+                    throw new IOException("Contents of the repository are over totalFileSizeLimit of " +
+                            FileUtils.byteCountToDisplaySize(totalFileSizeLimit));
                 }
-
             }
         }
     }

src/main/java/org/commonwl/viewer/services/WorkflowService.java

@@ -43,20 +43,23 @@ public class WorkflowService {
     private final int cacheDays;
     private final String graphvizStorage;
     private final int singleFileSizeLimit;
+    private final int totalFileSizeLimit;
 
     @Autowired
     public WorkflowService(GitHubService githubService,
                            WorkflowRepository workflowRepository,
                            ROBundleFactory ROBundleFactory,
                            @Value("${cacheDays}") int cacheDays,
                            @Value("${graphvizStorage}") String graphvizStorage,
-                           @Value("${singleFileSizeLimit}") int singleFileSizeLimit) {
+                           @Value("${singleFileSizeLimit}") int singleFileSizeLimit,
+                           @Value("${totalFileSizeLimit}") int totalFileSizeLimit) {
         this.githubService = githubService;
         this.workflowRepository = workflowRepository;
         this.ROBundleFactory = ROBundleFactory;
         this.cacheDays = cacheDays;
         this.graphvizStorage = graphvizStorage;
         this.singleFileSizeLimit = singleFileSizeLimit;
+        this.totalFileSizeLimit = totalFileSizeLimit;
     }
 
     /**
@@ -71,7 +74,8 @@ public Workflow newWorkflowFromGithub(GithubDetails githubInfo) {
             String latestCommit = githubService.getCommitSha(githubInfo);
 
             // Set up CWL utility to collect the documents
-            CWLCollection cwlFiles = new CWLCollection(githubService, githubInfo, latestCommit, singleFileSizeLimit);
+            CWLCollection cwlFiles = new CWLCollection(githubService, githubInfo, latestCommit,
+                    singleFileSizeLimit, totalFileSizeLimit);
 
             // Get the workflow model
             Workflow workflowModel = cwlFiles.getWorkflow();

src/main/resources/application.properties

@@ -16,9 +16,11 @@ graphvizStorage = /tmp
 cacheDays = 1
 
 # File size limit for individual files in bytes
+# CWL files must be lower than this, but other files in the repo may be lower and in this case will
+# be externally linked in the Research Object Bundle
 singleFileSizeLimit = 5242880
 
-# File size limit for the contents of the entire Research Object Bundle in bytes
+# File size limit for the contents of the entire repository path in bytes
 totalFileSizeLimit = 1073741824
 
 #=======================