Merge pull request #20307 from asgerf/js/overlay-extract-and-discard-only

JS: Add overlay support to extractor

Author: asgerf

javascript/downgrades/76a926a00d5f3bc199c203a1437796fd7b2835ba/old.dbscheme

@@ -0,0 +1,5 @@
+description: downgrade overlay relations
+compatibility: full
+
+databaseMetadata.rel: delete
+overlayChangedFiles.rel: delete

javascript/downgrades/76a926a00d5f3bc199c203a1437796fd7b2835ba/semmlecode.javascript.dbscheme

@@ -465,10 +465,11 @@ public int run() throws IOException {
       try {
         CompletableFuture<?> sourceFuture = extractSource();
         sourceFuture.join(); // wait for source extraction to complete
-        if (hasSeenCode()) { // don't bother with the externs if no code was seen
+        if (hasSeenCode() && !isOverlayChangeMode()) { // don't bother with the externs if no code was seen or in overlay change mode
           extractExterns();
         }
         extractXml();
+        writeOverlayMetadata();
       } catch (OutOfMemoryError oom) {
         System.err.println("Out of memory while extracting the project.");
         return 137; // the CodeQL CLI will interpret this as an out-of-memory error
@@ -509,6 +510,21 @@ public int run() throws IOException {
     return 0;
   }
 
+  private void writeOverlayMetadata() {
+    String file = getEnvVar("CODEQL_EXTRACTOR_JAVASCRIPT_OVERLAY_BASE_METADATA_OUT");
+    if (file == null) {
+      // no overlay metadata file specified, so nothing to do
+      return;
+    }
+    // Write an empty string to the file as we currently have no metadata to emit.
+    // The file must be created for the database to recognized as an overlay base.
+    try {
+      Files.writeString(Paths.get(file), "", StandardCharsets.UTF_8);
+    } catch (IOException e) {
+      throw new ResourceError("Could not write overlay metadata to " + file, e);
+    }
+  }
+
   /**
    * A kind of error that can happen during extraction of JavaScript or TypeScript
    * code.
@@ -734,6 +750,17 @@ private CompletableFuture<?> extractSource() throws IOException {
     List<Path> tsconfigFiles = new ArrayList<>();
     findFilesToExtract(defaultExtractor, filesToExtract, tsconfigFiles);
 
+    OverlayChanges overlay = getOverlayChanges();
+    if (overlay != null) {
+      Set<Path> changedFiles = overlay.changes.stream()
+          .map(file -> Paths.get(file).toAbsolutePath())
+          .collect(Collectors.toSet());
+      int before = filesToExtract.size();
+      filesToExtract.retainAll(changedFiles);
+      int after = filesToExtract.size();
+      System.out.println("Overlay filter removed " + (before - after) + " out of " + before + " files from extraction.");
+    }
+
     tsconfigFiles = tsconfigFiles.stream()
          .sorted(PATH_ORDERING)
          .collect(Collectors.toList());
@@ -1338,6 +1365,18 @@ protected void extractXml() throws IOException {
     }
   }
 
+  private boolean isOverlayChangeMode() {
+    return getEnvVar("CODEQL_EXTRACTOR_JAVASCRIPT_OVERLAY_CHANGES") != null;
+  }
+
+  private OverlayChanges getOverlayChanges() throws IOException {
+    String jsonFile = getEnvVar("CODEQL_EXTRACTOR_JAVASCRIPT_OVERLAY_CHANGES");
+    if (jsonFile == null) {
+      return null;
+    }
+    return new Gson().fromJson(Files.newBufferedReader(Paths.get(jsonFile)), OverlayChanges.class);
+  }
+
   public static void main(String[] args) {
     try {
       System.exit(new AutoBuild().run());

javascript/downgrades/76a926a00d5f3bc199c203a1437796fd7b2835ba/upgrade.properties

@@ -0,0 +1,7 @@
+package com.semmle.js.extractor;
+
+import java.util.List;
+
+public class OverlayChanges {
+    public List<String> changes;
+}

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -146,3 +146,4 @@ import semmle.javascript.linters.JSLint
 import semmle.javascript.linters.Linting
 import semmle.javascript.security.dataflow.RemoteFlowSources
 import semmle.javascript.frameworks.UnderscoreDotString
+private import semmle.javascript.internal.Overlay

javascript/extractor/src/com/semmle/js/extractor/OverlayChanges.java

@@ -0,0 +1,30 @@
+private import javascript
+
+/** Holds if the database is an overlay. */
+overlay[local]
+private predicate isOverlay() { databaseMetadata("isOverlay", "true") }
+
+overlay[local]
+private string getFileFromEntity(@locatable node) {
+  exists(@location loc, @file file |
+    hasLocation(node, loc) and
+    locations_default(loc, file, _, _, _, _) and
+    files(file, result)
+  )
+}
+
+/** Holds if `file` was changed or deleted in the overlay. */
+overlay[local]
+private predicate discardFile(string file) { isOverlay() and overlayChangedFiles(file) }
+
+/** Holds if `node` is in the `file` and is part of the overlay base database. */
+overlay[local]
+private predicate discardableEntity(string file, @locatable node) {
+  not isOverlay() and file = getFileFromEntity(node)
+}
+
+/** Holds if `node` should be discarded, because it is part of the overlay base and is in a file that was also extracted as part of the overlay database. */
+overlay[discard_entity]
+private predicate discardEntity(@locatable node) {
+  exists(string file | discardableEntity(file, node) and discardFile(file))
+}

javascript/ql/lib/javascript.qll

@@ -1192,3 +1192,13 @@ configLocations(
 );
 
 @configLocatable = @config | @configName | @configValue;
+
+/*- Database metadata -*/
+databaseMetadata(
+  string metadataKey: string ref,
+  string value: string ref
+);
+
+overlayChangedFiles(
+  string path: string ref
+);

javascript/ql/lib/semmle/javascript/internal/Overlay.qll

@@ -28260,5 +28260,52 @@
 </dep>
 </dependencies>
 </relation>
+<relation>
+<name>databaseMetadata</name>
+<cardinality>1</cardinality>
+<columnsizes>
+<e>
+<k>metadataKey</k>
+<v>1</v>
+</e>
+<e>
+<k>value</k>
+<v>1</v>
+</e>
+</columnsizes>
+<dependencies>
+<dep>
+<src>metadataKey</src>
+<trg>value</trg>
+<val>
+<hist>
+<budget>12</budget>
+<bs/>
+</hist>
+</val>
+</dep>
+<dep>
+<src>value</src>
+<trg>metadataKey</trg>
+<val>
+<hist>
+<budget>12</budget>
+<bs/>
+</hist>
+</val>
+</dep>
+</dependencies>
+</relation>
+<relation>
+<name>overlayChangedFiles</name>
+<cardinality>50</cardinality>
+<columnsizes>
+<e>
+<k>path</k>
+<v>50</v>
+</e>
+</columnsizes>
+<dependencies/>
+</relation>
 </stats>
 </dbstats>