JS: Add a file size limit to extractor

Author: asgerf

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -222,6 +222,7 @@ public class AutoBuild {
   private boolean installDependencies = false;
   private final VirtualSourceRoot virtualSourceRoot;
   private ExtractorState state;
+  private final long maximumFileSizeInMegabytes;
 
   /** The default timeout when installing dependencies, in milliseconds. */
   public static final int INSTALL_DEPENDENCIES_DEFAULT_TIMEOUT = 10 * 60 * 1000; // 10 minutes
@@ -236,6 +237,7 @@ public AutoBuild() {
     this.defaultEncoding = getEnvVar("LGTM_INDEX_DEFAULT_ENCODING");
     this.installDependencies = Boolean.valueOf(getEnvVar("LGTM_INDEX_TYPESCRIPT_INSTALL_DEPS"));
     this.virtualSourceRoot = makeVirtualSourceRoot();
+    this.maximumFileSizeInMegabytes = EnvironmentVariables.getMegabyteCountFromPrefixedEnv("MAX_FILE_SIZE", 10);
     setupFileTypes();
     setupXmlMode();
     setupMatchers();
@@ -446,8 +448,8 @@ private boolean addPathPattern(Set<Path> patterns, Path base, String pattern) {
   }
 
   /**
-   * Returns whether the autobuilder has seen code. 
-   * This is overridden in tests. 
+   * Returns whether the autobuilder has seen code.
+   * This is overridden in tests.
    */
   protected boolean hasSeenCode() {
     return seenCode;
@@ -741,12 +743,12 @@ private CompletableFuture<?> extractSource() throws IOException {
       dependencyInstallationResult = this.preparePackagesAndDependencies(filesToExtract);
     }
     Set<Path> extractedFiles = new LinkedHashSet<>();
-    
+
     // Extract HTML files as they may contain TypeScript
     CompletableFuture<?> htmlFuture = extractFiles(
         filesToExtract, extractedFiles, extractors,
         f -> extractors.fileType(f) == FileType.HTML);
-    
+
     htmlFuture.join(); // Wait for HTML extraction to be finished.
 
     // extract TypeScript projects and files
@@ -1229,6 +1231,11 @@ private void doExtract(FileExtractor extractor, Path file, ExtractorState state)
       warn("Skipping " + file + ", which does not exist.");
       return;
     }
+    long fileSize = f.length();
+    if (fileSize > 1_000_000L * this.maximumFileSizeInMegabytes) {
+      warn("Skipping " + file + " because it is too large (" + StringUtil.printFloat(fileSize / 1_000_000.0) + " MB). The limit is " + this.maximumFileSizeInMegabytes + " MB.");
+      return;
+    }
 
     try {
       long start = logBeginProcess("Extracting " + file);