Python extractor: in overlay mode, traverse only changed files

Author: d10c

python/extractor/semmle/traverser.py

@@ -4,6 +4,7 @@
 
 import re
 import os.path
+import json
 
 from semmle.path_filters import filter_from_pattern
 from semmle.util import Extractable, PY_EXTENSIONS, isdir, islink, listdir
@@ -30,6 +31,12 @@ def __init__(self, options, modulenames, logger):
                 if not os.path.exists(p) and not options.ignore_missing_modules:
                     raise FileNotFoundError("'%s' does not exist." % p)
                 self.paths.add(p)
+        if 'CODEQL_EXTRACTOR_PYTHON_OVERLAY_CHANGES' in os.environ:
+            # During overlay extraction, only traverse the files that were changed.
+            with open(os.environ['CODEQL_EXTRACTOR_PYTHON_OVERLAY_CHANGES'], 'r', encoding='utf-8') as f:
+                data = json.load(f)
+                changed_paths = data.get('changes', [])
+                self.paths = { p for p in changed_paths if p in self.paths }
         self.exclude_paths = set([ os.path.abspath(f) for f in options.exclude_file ])
         self.exclude = exclude_filter_from_options(options)
         self.filter = filter_from_options_and_environment(options)