Merge pull request #78 from common-workflow-language/file-size-limits

File size limits

Author: MarkRobbo

src/main/java/org/commonwl/viewer/domain/CWLCollection.java

@@ -24,10 +24,11 @@
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.TextNode;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.commonwl.viewer.services.DockerService;
-import org.eclipse.egit.github.core.RepositoryContents;
 import org.commonwl.viewer.services.GitHubService;
+import org.eclipse.egit.github.core.RepositoryContents;
 import org.yaml.snakeyaml.Yaml;
 
 import java.io.IOException;
@@ -43,6 +44,10 @@ public class CWLCollection {
     private GithubDetails githubInfo;
     private String commitSha;
 
+    private int totalFileSize;
+    private int totalFileSizeLimit;
+    private int singleFileSizeLimit;
+
     // Maps of ID to associated JSON
     private Map<String, JsonNode> workflows = new HashMap<>();
 
@@ -90,10 +95,13 @@ public class CWLCollection {
      * @throws IOException Any API errors which may have occurred
      */
     public CWLCollection(GitHubService githubService, GithubDetails githubInfo,
-                         String commitSha) throws IOException {
+                         String commitSha, int singleFileSizeLimit, int totalFileSizeLimit) throws IOException {
         this.githubInfo = githubInfo;
         this.githubService = githubService;
         this.commitSha = commitSha;
+        this.singleFileSizeLimit = singleFileSizeLimit;
+        this.totalFileSizeLimit = totalFileSizeLimit;
+        this.totalFileSize = 0;
 
         // Add any CWL files from the Github repo to this collection
         List<RepositoryContents> repoContents = githubService.getContents(githubInfo);
@@ -119,32 +127,46 @@ private void addDocs(List<RepositoryContents> repoContents) throws IOException {
                 // Add the files in the subdirectory to this new folder
                 addDocs(subdirectory);
 
-                // Otherwise this is a file so add to the bundle
             } else if (repoContent.getType().equals(FILE)) {
 
-                // Get the file extension
-                int eIndex = repoContent.getName().lastIndexOf('.') + 1;
-                if (eIndex > 0) {
-                    String extension = repoContent.getName().substring(eIndex);
-
-                    // If this is a cwl file which needs to be parsed
-                    if (extension.equals(CWL_EXTENSION)) {
-
-                        // Get the content of this file from Github
-                        GithubDetails githubFile = new GithubDetails(githubInfo.getOwner(),
-                                githubInfo.getRepoName(), githubInfo.getBranch(), repoContent.getPath());
-                        String fileContent = githubService.downloadFile(githubFile, commitSha);
-
-                        // Parse yaml to JsonNode
-                        Yaml reader = new Yaml();
-                        ObjectMapper mapper = new ObjectMapper();
-                        JsonNode cwlFile = mapper.valueToTree(reader.load(fileContent));
+                // Keep track of total file size for limit - only track files which
+                // will be added to the RO bundle due to being small enough
+                if (repoContent.getSize() <= singleFileSizeLimit) {
+                    totalFileSize += repoContent.getSize();
+                }
 
-                        // Add document to those being considered
-                        addDoc(cwlFile, repoContent.getName());
+                if (totalFileSize <= totalFileSizeLimit) {
+                    // Get the file extension
+                    int eIndex = repoContent.getName().lastIndexOf('.') + 1;
+                    if (eIndex > 0) {
+                        String extension = repoContent.getName().substring(eIndex);
+
+                        // If this is a cwl file which needs to be parsed
+                        if (extension.equals(CWL_EXTENSION)) {
+                            if (repoContent.getSize() <= singleFileSizeLimit) {
+                                // Get the content of this file from Github
+                                GithubDetails githubFile = new GithubDetails(githubInfo.getOwner(),
+                                        githubInfo.getRepoName(), githubInfo.getBranch(), repoContent.getPath());
+                                String fileContent = githubService.downloadFile(githubFile, commitSha);
+
+                                // Parse yaml to JsonNode
+                                Yaml reader = new Yaml();
+                                ObjectMapper mapper = new ObjectMapper();
+                                JsonNode cwlFile = mapper.valueToTree(reader.load(fileContent));
+
+                                // Add document to those being considered
+                                addDoc(cwlFile, repoContent.getName());
+                            } else {
+                                throw new IOException("File '" + repoContent.getName() +  "' is over singleFileSizeLimit - " +
+                                        FileUtils.byteCountToDisplaySize(repoContent.getSize()) + "/" +
+                                        FileUtils.byteCountToDisplaySize(singleFileSizeLimit));
+                            }
+                        }
                     }
+                } else {
+                    throw new IOException("Contents of the repository are over totalFileSizeLimit of " +
+                            FileUtils.byteCountToDisplaySize(totalFileSizeLimit));
                 }
-
             }
         }
     }

src/main/java/org/commonwl/viewer/domain/ROBundle.java

@@ -19,6 +19,7 @@
 
 package org.commonwl.viewer.domain;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.taverna.robundle.Bundle;
 import org.apache.taverna.robundle.Bundles;
@@ -30,7 +31,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -56,6 +56,7 @@ public class ROBundle {
     private GithubDetails githubInfo;
     private String commitSha;
     private Agent thisApp;
+    private int singleFileSizeLimit;
     private Set<HashableAgent> authors = new HashSet<HashableAgent>();
 
     // Pattern for extracting version from a cwl file
@@ -68,8 +69,9 @@ public class ROBundle {
      * @throws IOException Any API errors which may have occurred
      */
     public ROBundle(GitHubService githubService, GithubDetails githubInfo, String commitSha,
-                    String appName, String appURL) throws IOException {
-        // TODO: Add back file size checking on individual files as well as whole bundle
+                    String appName, String appURL, int singleFileSizeLimit) throws IOException {
+        // File size limits
+        this.singleFileSizeLimit = singleFileSizeLimit;
 
         // Create a new RO bundle
         this.bundle = Bundles.createBundle();
@@ -137,29 +139,48 @@ private void addFiles(List<RepositoryContents> repoContents, Path path) throws I
                 // Otherwise this is a file so add to the bundle
             } else if (repoContent.getType().equals("file")) {
 
-                // Get the content of this file from Github
-                GithubDetails githubFile = new GithubDetails(githubInfo.getOwner(),
-                        githubInfo.getRepoName(), githubInfo.getBranch(), repoContent.getPath());
-                String fileContent = githubService.downloadFile(githubFile, commitSha);
-
-                // Save file to research object bundle
-                Path newFilePort = path.resolve(repoContent.getName());
-                Bundles.setStringValue(newFilePort, fileContent);
+                try {
+                    // Where to store the new file in bundle
+                    Path bundleFilePath = path.resolve(repoContent.getName());
+
+                    // Raw URI of the bundle
+                    GithubDetails githubFile = new GithubDetails(githubInfo.getOwner(),
+                            githubInfo.getRepoName(), githubInfo.getBranch(), repoContent.getPath());
+                    URI rawURI = new URI("https://raw.githubusercontent.com/" + githubFile.getOwner() + "/" +
+                            githubFile.getRepoName() + "/" + commitSha + "/" + githubFile.getPath());
+
+                    // Variable to store file contents
+                    String fileContent = null;
+
+                    // Download or externally link if oversized
+                    if (repoContent.getSize() <= singleFileSizeLimit) {
+                        // Get the content of this file from Github
+                        fileContent = githubService.downloadFile(githubFile, commitSha);
+
+                        // Save file to research object bundle
+                        Bundles.setStringValue(bundleFilePath, fileContent);
+                    } else {
+                        logger.info("File " + repoContent.getName() + " is too large to download -" +
+                                FileUtils.byteCountToDisplaySize(repoContent.getSize()) + "/" +
+                                FileUtils.byteCountToDisplaySize(singleFileSizeLimit) +
+                                " + linking externally to RO bundle");
+                        bundleFilePath = Bundles.setReference(bundleFilePath, rawURI);
+                    }
 
-                // Manifest aggregation
-                PathMetadata aggregation = bundle.getManifest().getAggregation(newFilePort);
+                    // Manifest aggregation
+                    PathMetadata aggregation = bundle.getManifest().getAggregation(bundleFilePath);
 
-                try {
                     // Special handling for cwl files
                     if (FilenameUtils.getExtension(repoContent.getName()).equals("cwl")) {
                         // Correct mime type (no official standard for yaml)
                         aggregation.setMediatype("text/x-yaml");
 
                         // Add conformsTo for version extracted from regex
-                        // Lower overhead vs parsing entire file
-                        Matcher m = cwlVersionPattern.matcher(fileContent);
-                        if (m.find()) {
-                            aggregation.setConformsTo(new URI("https://w3id.org/cwl/" + m.group(1)));
+                        if (fileContent != null) {
+                            Matcher m = cwlVersionPattern.matcher(fileContent);
+                            if (m.find()) {
+                                aggregation.setConformsTo(new URI("https://w3id.org/cwl/" + m.group(1)));
+                            }
                         }
                     }
 
@@ -169,15 +190,13 @@ private void addFiles(List<RepositoryContents> repoContents, Path path) throws I
                     aggregation.setAuthoredBy(new ArrayList<Agent>(fileAuthors));
 
                     // Set retrieved information for this file in the manifest
-                    aggregation.setRetrievedFrom(new URI("https://raw.githubusercontent.com/" + githubFile.getOwner() + "/" +
-                            githubFile.getRepoName() + "/" + commitSha + "/" + githubFile.getPath()));
+                    aggregation.setRetrievedFrom(rawURI);
                     aggregation.setRetrievedBy(thisApp);
                     aggregation.setRetrievedOn(aggregation.getCreatedOn());
 
                 } catch (URISyntaxException ex) {
                     logger.error("Error creating URI for RO Bundle", ex);
                 }
-
             }
         }
     }

src/main/java/org/commonwl/viewer/services/ROBundleFactory.java

@@ -47,18 +47,21 @@ public class ROBundleFactory {
 
     private final String applicationName;
     private final String applicationURL;
+    private final int singleFileSizeLimit;
     private final Path storageLocation;
     private final WorkflowRepository workflowRepository;
 
     @Autowired
     public ROBundleFactory(@Value("${applicationName}") String applicationName,
                            @Value("${applicationURL}") String applicationURL,
                            @Value("${graphvizStorage}") Path graphvizStorage,
+                           @Value("${singleFileSizeLimit}") int singleFileSizeLimit,
                            WorkflowRepository workflowRepository) {
         this.applicationName = applicationName;
         this.applicationURL = applicationURL;
         this.storageLocation = graphvizStorage;
         this.workflowRepository = workflowRepository;
+        this.singleFileSizeLimit = singleFileSizeLimit;
     }
 
     /**
@@ -75,7 +78,7 @@ void workflowROFromGithub(GitHubService githubService, GithubDetails githubInfo,
 
         // Create a new Research Object Bundle with Github contents
         ROBundle bundle = new ROBundle(githubService, githubInfo, commitSha,
-                applicationName, applicationURL);
+                applicationName, applicationURL, singleFileSizeLimit);
 
         // Save the bundle to the storage location in properties
         Path bundleLocation = bundle.saveToFile(storageLocation);

src/main/java/org/commonwl/viewer/services/WorkflowService.java

@@ -29,7 +29,6 @@
 import org.springframework.stereotype.Service;
 
 import java.io.File;
-import java.io.IOException;
 import java.util.Calendar;
 import java.util.Date;
 
@@ -43,18 +42,24 @@ public class WorkflowService {
     private final ROBundleFactory ROBundleFactory;
     private final int cacheDays;
     private final String graphvizStorage;
+    private final int singleFileSizeLimit;
+    private final int totalFileSizeLimit;
 
     @Autowired
     public WorkflowService(GitHubService githubService,
                            WorkflowRepository workflowRepository,
                            ROBundleFactory ROBundleFactory,
                            @Value("${cacheDays}") int cacheDays,
-                           @Value("${graphvizStorage}") String graphvizStorage) {
+                           @Value("${graphvizStorage}") String graphvizStorage,
+                           @Value("${singleFileSizeLimit}") int singleFileSizeLimit,
+                           @Value("${totalFileSizeLimit}") int totalFileSizeLimit) {
         this.githubService = githubService;
         this.workflowRepository = workflowRepository;
         this.ROBundleFactory = ROBundleFactory;
         this.cacheDays = cacheDays;
         this.graphvizStorage = graphvizStorage;
+        this.singleFileSizeLimit = singleFileSizeLimit;
+        this.totalFileSizeLimit = totalFileSizeLimit;
     }
 
     /**
@@ -69,7 +74,8 @@ public Workflow newWorkflowFromGithub(GithubDetails githubInfo) {
             String latestCommit = githubService.getCommitSha(githubInfo);
 
             // Set up CWL utility to collect the documents
-            CWLCollection cwlFiles = new CWLCollection(githubService, githubInfo, latestCommit);
+            CWLCollection cwlFiles = new CWLCollection(githubService, githubInfo, latestCommit,
+                    singleFileSizeLimit, totalFileSizeLimit);
 
             // Get the workflow model
             Workflow workflowModel = cwlFiles.getWorkflow();
@@ -170,24 +176,4 @@ public boolean cacheExpired(Workflow workflow) {
             return false;
         }
     }
-
-    /*
-        // Check total file size
-        int totalFileSize = 0;
-        for (RepositoryContents repoContent : repoContents) {
-            totalFileSize += repoContent.getSize();
-        }
-        if (totalFileSize > totalFileSizeLimit) {
-            throw new IOException("Files within the Github directory can not be above "
-                                  + totalFileSizeLimit + "B in size");
-        }
-    */
-
-
-    /*
-        // Check file size before downloading
-        if (file.getSize() > singleFileSizeLimit) {
-            throw new IOException("Files within the Github directory can not be above " + singleFileSizeLimit + "B in size");
-        }
-    */
 }

src/main/resources/application.properties

@@ -16,10 +16,12 @@ graphvizStorage = /tmp
 cacheDays = 1
 
 # File size limit for individual files in bytes
-singleFileSizeLimit = 5000000
+# CWL files must be lower than this, but other files in the repo may be lower and in this case will
+# be externally linked in the Research Object Bundle
+singleFileSizeLimit = 5242880
 
-# File size limit for the contents of the entire Research Object Bundle in bytes
-totalFileSizeLimit = 10000000
+# File size limit for the contents of the research object bundle (not counting external links)
+totalFileSizeLimit = 1073741824
 
 #=======================
 # Github API settings