fix: Skip non-TU main file command objects (#380)

When the compilation database for a codebase is generated using
grailbio/bazel-compilation-database, it ends up generating lots of
entries for headers and other template files like 'def' files in LLVM.
However, those cannot actually be semantically analyzed without
a translation unit, so skip them immediately after parsing.

Author: varungandhi-src

indexer/CompilationDatabase.cc

@@ -565,7 +565,7 @@ compdb::File::openAndExitOnErrors(const StdPath &path,
 }
 
 void ResumableParser::initialize(compdb::File compdb, size_t refillCount,
-                                 bool inferResourceDir) {
+                                 ParseOptions options) {
   auto averageJobSize = compdb.sizeInBytes() / compdb.commandCount();
   // Some customers have averageJobSize = 150KiB.
   // If numWorkers == 300 (very high core count machine),
@@ -580,7 +580,14 @@ void ResumableParser::initialize(compdb::File compdb, size_t refillCount,
       rapidjson::FileReadStream(compdb.file, this->jsonStreamBuffer.data(),
                                 this->jsonStreamBuffer.size());
   this->reader.IterativeParseInit();
-  this->inferResourceDir = inferResourceDir;
+  this->options = options;
+  std::vector<std::string> extensions;
+  // Via https://stackoverflow.com/a/3223792/2682729
+  for (auto ext : {"c", "C", "cc", "cpp", "CPP", "cxx", "c++"}) {
+    extensions.emplace_back(llvm::Regex::escape(fmt::format(".{}", ext)));
+  };
+  this->fileExtensionRegex =
+      llvm::Regex(fmt::format(".+({})$", fmt::join(extensions, "|")));
 }
 
 void ResumableParser::parseMore(std::vector<compdb::CommandObject> &out,
@@ -633,16 +640,23 @@ void ResumableParser::parseMore(std::vector<compdb::CommandObject> &out,
     for (auto &cmd : this->handler->commands) {
       cmd.index = this->currentIndex;
       ++this->currentIndex;
+      if (this->options.skipNonMainFileEntries) {
+        if (!this->fileExtensionRegex.match(cmd.filePath)) {
+          ++this->stats.skippedNonTuFileExtension;
+          continue;
+        }
+      }
       if (checkFilesExist
           && !doesFileExist(cmd.filePath, cmd.workingDirectory)) {
+        ++this->stats.skippedNonExistentTuFile;
         continue;
       }
       out.emplace_back(std::move(cmd));
     }
     this->handler->commands.clear();
   }
 
-  if (this->inferResourceDir) {
+  if (this->options.inferResourceDir) {
     for (auto &cmd : out) {
       if (cmd.arguments.empty()) {
         continue;

indexer/CompilationDatabase.h

@@ -15,6 +15,8 @@
 #include "spdlog/fmt/fmt.h"
 #include "spdlog/spdlog.h"
 
+#include "llvm/Support/Regex.h"
+
 #include "indexer/Derive.h"
 #include "indexer/FileSystem.h"
 
@@ -106,16 +108,35 @@ struct ToolchainConfig {
   std::vector<std::string> extraArgs;
 };
 
+struct ParseOptions {
+  bool inferResourceDir;
+  bool skipNonMainFileEntries;
+
+  explicit ParseOptions(bool isTesting = false)
+      : inferResourceDir(!isTesting), skipNonMainFileEntries(!isTesting) {}
+
+  ParseOptions(bool inferResourceDir, bool skipNonMainFileEntries)
+      : inferResourceDir(inferResourceDir),
+        skipNonMainFileEntries(skipNonMainFileEntries) {}
+};
+
+struct ParseStats {
+  unsigned skippedNonTuFileExtension = 0;
+  unsigned skippedNonExistentTuFile = 0;
+};
+
 class ResumableParser {
   std::string jsonStreamBuffer;
   std::optional<rapidjson::FileReadStream> compDbStream;
   std::optional<CommandObjectHandler> handler;
   rapidjson::Reader reader;
   size_t currentIndex = 0;
 
-  bool inferResourceDir;
+  ParseOptions options;
   absl::flat_hash_set<std::string> emittedErrors;
 
+  llvm::Regex fileExtensionRegex;
+
   /// Mapping from compiler/wrapper path to extra information needed
   /// to tweak the command object before invoking the driver.
   ///
@@ -126,15 +147,16 @@ class ResumableParser {
   absl::flat_hash_map<std::string, ToolchainConfig> toolchainConfigMap;
 
 public:
+  ParseStats stats;
+
   ResumableParser() = default;
   ResumableParser(const ResumableParser &) = delete;
   ResumableParser &operator=(const ResumableParser &) = delete;
 
   /// If \param inferResourceDir is set, then the parser will automatically
   /// add extra '-resource-dir' '<path>' arguments to the parsed
   /// CompileCommands' CommandLine field.
-  void initialize(compdb::File compdb, size_t refillCount,
-                  bool inferResourceDir);
+  void initialize(compdb::File compdb, size_t refillCount, ParseOptions);
 
   // Parses at most refillCount elements (passed during initialization)
   // from the compilation database passed during initialization.

indexer/Driver.cc

@@ -958,6 +958,15 @@ class Driver {
                "{:.1f}s, merging: {:.1f}s, num errored TUs: {}).\n",
                numTus.first.value, total.value<secs>(), indexing.value<secs>(),
                merging.value<secs>(), numTus.second);
+    auto parseStats = this->compdbParser.stats;
+    auto totalSkipped = parseStats.skippedNonExistentTuFile
+                        + parseStats.skippedNonTuFileExtension;
+    if (totalSkipped != 0) {
+      fmt::print("Skipped: {} compilation database entries (non main file "
+                 "extension: {}, not found on disk: {}).\n",
+                 totalSkipped, parseStats.skippedNonTuFileExtension,
+                 parseStats.skippedNonExistentTuFile);
+    }
   }
 
 private:
@@ -1194,12 +1203,14 @@ class Driver {
     this->compdbCommandCount = compdbFile.commandCount();
     this->options.numWorkers =
         std::min(this->compdbCommandCount, this->numWorkers());
-    spdlog::debug("total {} compilation jobs", this->compdbCommandCount);
+    spdlog::debug("total {} command objects in compilation database",
+                  this->compdbCommandCount);
 
     // FIXME(def: resource-dir-extra): If we're passed in a resource dir
     // as an extra argument, we should not pass it here.
-    this->compdbParser.initialize(compdbFile, this->refillCount(),
-                                  !this->options.isTesting);
+    this->compdbParser.initialize(
+        compdbFile, this->refillCount(),
+        compdb::ParseOptions(this->options.isTesting));
     return FileGuard(compdbFile.file);
   }
 

indexer/Worker.cc

@@ -146,7 +146,8 @@ Worker::Worker(WorkerOptions &&options)
         compdb::ValidationOptions{.checkDirectoryPathsAreAbsolute = true});
     compdb::ResumableParser parser{};
     // See FIXME(ref: resource-dir-extra)
-    parser.initialize(compdbFile, std::numeric_limits<size_t>::max(), true);
+    parser.initialize(compdbFile, std::numeric_limits<size_t>::max(),
+                      compdb::ParseOptions{/*isTesting*/ false});
     parser.parseMore(this->compileCommands);
     std::fclose(compdbFile.file);
     break;

test/compdb/skipping-4.snapshot.yaml

@@ -0,0 +1,20 @@
+---
+- command:
+    - abc
+  filePath:        a.c
+  directory:       not_skipped
+- command:
+    - abc
+  filePath:        a.cpp
+  directory:       not_skipped
+- command:
+    - abc
+  filePath:        a.cc
+  directory:       not_skipped
+...
+---
+- command:
+    - abc
+  filePath:        a.cxx
+  directory:       not_skipped
+...

test/compdb/skipping.json

@@ -0,0 +1,47 @@
+[
+  {
+    "command": "abc",
+    "directory": "not_skipped",
+    "file": "a.c"
+  },
+  {
+    "command": "abc",
+    "directory": "not_skipped",
+    "file": "a.cpp"
+  },
+  {
+    "command": "abc",
+    "directory": "not_skipped",
+    "file": "a.cc"
+  },
+  {
+    "command": "abc",
+    "directory": "skipped",
+    "file": "foo.h"
+  },
+  {
+    "command": "abc",
+    "directory": "skipped",
+    "file": "foo.hpp"
+  },
+  {
+    "command": "abc",
+    "directory": "skipped",
+    "file": "foo.hxx"
+  },
+  {
+    "command": "abc",
+    "directory": "not_skipped",
+    "file": "a.cxx"
+  },
+  {
+    "command": "abc",
+    "directory": "skipped",
+    "file": "foo.def"
+  },
+  {
+    "command": "abc",
+    "directory": "skipped",
+    "file": "foo.S"
+  }
+]

test/test_main.cc

@@ -60,6 +60,7 @@ struct CompDbTestCase {
   std::string jsonFilename;
   size_t checkCount;
   std::vector<size_t> refillCountsToTry;
+  compdb::ParseOptions parseOptions;
 };
 
 // Use YAML instead of JSON for easy line-based diffs (a la insta in Rust-land)
@@ -158,7 +159,14 @@ TEST_CASE("COMPDB_PARSING") {
   scip_clang::compdb::ResumableParser parser;
 
   std::vector<CompDbTestCase> testCases{};
-  testCases.push_back(CompDbTestCase{"simple.json", 3, {2, 3, 4}});
+  testCases.push_back(CompDbTestCase{
+      "simple.json", 3, {2, 3, 4}, compdb::ParseOptions(/*isTesting*/ true)});
+  testCases.push_back(
+      CompDbTestCase{"skipping.json",
+                     9,
+                     {4},
+                     compdb::ParseOptions(/*inferResourceDir*/ false,
+                                          /*skipNonMainFileEntries*/ true)});
 
   auto dataDir =
       std::filesystem::current_path().append("test").append("compdb");
@@ -182,7 +190,7 @@ TEST_CASE("COMPDB_PARSING") {
 
     for (auto refillCount : testCase.refillCountsToTry) {
       compdb::ResumableParser parser{};
-      parser.initialize(compdbFile, refillCount, false);
+      parser.initialize(compdbFile, refillCount, testCase.parseOptions);
       std::vector<std::vector<compdb::CommandObject>> commandGroups;
       std::string buffer;
       llvm::raw_string_ostream outStr(buffer);