refactor: Pass parser options uniformly (#381)

Author: varungandhi-src

indexer/CompilationDatabase.cc

@@ -564,17 +564,24 @@ compdb::File::openAndExitOnErrors(const StdPath &path,
   return compdbFile;
 }
 
-void ResumableParser::initialize(compdb::File compdb, size_t refillCount,
-                                 ParseOptions options) {
+// static
+ParseOptions ParseOptions::create(size_t refillCount, bool forTesting) {
+  ENFORCE(refillCount > 0);
+  return ParseOptions{refillCount, /*inferResourceDir*/ !forTesting,
+                      /*skipNonMainFileTuEntries*/ !forTesting,
+                      /*checkFilesExist*/ !forTesting};
+}
+
+void ResumableParser::initialize(compdb::File compdb, ParseOptions options) {
   auto averageJobSize = compdb.sizeInBytes() / compdb.commandCount();
   // Some customers have averageJobSize = 150KiB.
   // If numWorkers == 300 (very high core count machine),
   // then the computed hint will be ~88MiB. The 128MiB is rounded up from 88MiB.
   // The fudge factor of 2 is to allow for oversized jobs.
-  auto bufferSize =
-      std::min(size_t(128 * 1024 * 1024), averageJobSize * 2 * refillCount);
+  auto bufferSize = std::min(size_t(128 * 1024 * 1024),
+                             averageJobSize * 2 * options.refillCount);
   std::fseek(compdb.file, 0, SEEK_SET);
-  this->handler = CommandObjectHandler(refillCount);
+  this->handler = CommandObjectHandler(options.refillCount);
   this->jsonStreamBuffer.resize(bufferSize);
   this->compDbStream =
       rapidjson::FileReadStream(compdb.file, this->jsonStreamBuffer.data(),
@@ -590,8 +597,7 @@ void ResumableParser::initialize(compdb::File compdb, size_t refillCount,
       llvm::Regex(fmt::format(".+({})$", fmt::join(extensions, "|")));
 }
 
-void ResumableParser::parseMore(std::vector<compdb::CommandObject> &out,
-                                bool checkFilesExist) {
+void ResumableParser::parseMore(std::vector<compdb::CommandObject> &out) {
   if (this->reader.IterativeParseComplete()) {
     if (this->reader.HasParseError()) {
       spdlog::error(
@@ -646,7 +652,7 @@ void ResumableParser::parseMore(std::vector<compdb::CommandObject> &out,
           continue;
         }
       }
-      if (checkFilesExist
+      if (this->options.checkFilesExist
           && !doesFileExist(cmd.filePath, cmd.workingDirectory)) {
         ++this->stats.skippedNonExistentTuFile;
         continue;

indexer/CompilationDatabase.h

@@ -109,15 +109,12 @@ struct ToolchainConfig {
 };
 
 struct ParseOptions {
+  size_t refillCount;
   bool inferResourceDir;
   bool skipNonMainFileEntries;
+  bool checkFilesExist;
 
-  explicit ParseOptions(bool isTesting = false)
-      : inferResourceDir(!isTesting), skipNonMainFileEntries(!isTesting) {}
-
-  ParseOptions(bool inferResourceDir, bool skipNonMainFileEntries)
-      : inferResourceDir(inferResourceDir),
-        skipNonMainFileEntries(skipNonMainFileEntries) {}
+  static ParseOptions create(size_t refillCount, bool forTesting = false);
 };
 
 struct ParseStats {
@@ -156,11 +153,10 @@ class ResumableParser {
   /// If \param inferResourceDir is set, then the parser will automatically
   /// add extra '-resource-dir' '<path>' arguments to the parsed
   /// CompileCommands' CommandLine field.
-  void initialize(compdb::File compdb, size_t refillCount, ParseOptions);
+  void initialize(compdb::File compdb, ParseOptions);
 
-  // Parses at most refillCount elements (passed during initialization)
-  // from the compilation database passed during initialization.
-  void parseMore(std::vector<CommandObject> &out, bool checkFilesExist = true);
+  /// Parses at most \c options.refillCount elements into \param out.
+  void parseMore(std::vector<CommandObject> &out);
 
 private:
   void tryInferResourceDir(const std::string &directoryPath,

indexer/Driver.cc

@@ -1209,8 +1209,8 @@ class Driver {
     // FIXME(def: resource-dir-extra): If we're passed in a resource dir
     // as an extra argument, we should not pass it here.
     this->compdbParser.initialize(
-        compdbFile, this->refillCount(),
-        compdb::ParseOptions(this->options.isTesting));
+        compdbFile, compdb::ParseOptions::create(this->refillCount(),
+                                                 this->options.isTesting));
     return FileGuard(compdbFile.file);
   }
 

indexer/Worker.cc

@@ -145,9 +145,9 @@ Worker::Worker(WorkerOptions &&options)
         this->options.compdbPath,
         compdb::ValidationOptions{.checkDirectoryPathsAreAbsolute = true});
     compdb::ResumableParser parser{};
-    // See FIXME(ref: resource-dir-extra)
-    parser.initialize(compdbFile, std::numeric_limits<size_t>::max(),
-                      compdb::ParseOptions{/*isTesting*/ false});
+    parser.initialize(compdbFile,
+                      compdb::ParseOptions::create(
+                          /*refillCount*/ std::numeric_limits<size_t>::max()));
     parser.parseMore(this->compileCommands);
     std::fclose(compdbFile.file);
     break;

test/test_main.cc

@@ -159,14 +159,12 @@ TEST_CASE("COMPDB_PARSING") {
   scip_clang::compdb::ResumableParser parser;
 
   std::vector<CompDbTestCase> testCases{};
-  testCases.push_back(CompDbTestCase{
-      "simple.json", 3, {2, 3, 4}, compdb::ParseOptions(/*isTesting*/ true)});
-  testCases.push_back(
-      CompDbTestCase{"skipping.json",
-                     9,
-                     {4},
-                     compdb::ParseOptions(/*inferResourceDir*/ false,
-                                          /*skipNonMainFileEntries*/ true)});
+  auto testOptions =
+      compdb::ParseOptions::create(/*refillCount*/ 1, /*forTesting*/ true);
+  testCases.push_back(CompDbTestCase{"simple.json", 3, {2, 3, 4}, testOptions});
+  auto testOptionsSkip = testOptions;
+  testOptionsSkip.skipNonMainFileEntries = true;
+  testCases.push_back(CompDbTestCase{"skipping.json", 9, {4}, testOptionsSkip});
 
   auto dataDir =
       std::filesystem::current_path().append("test").append("compdb");
@@ -190,14 +188,16 @@ TEST_CASE("COMPDB_PARSING") {
 
     for (auto refillCount : testCase.refillCountsToTry) {
       compdb::ResumableParser parser{};
-      parser.initialize(compdbFile, refillCount, testCase.parseOptions);
+      auto parseOptions = testCase.parseOptions;
+      parseOptions.refillCount = refillCount;
+      parser.initialize(compdbFile, parseOptions);
       std::vector<std::vector<compdb::CommandObject>> commandGroups;
       std::string buffer;
       llvm::raw_string_ostream outStr(buffer);
       llvm::yaml::Output yamlOut(outStr);
       while (true) {
         std::vector<compdb::CommandObject> commands;
-        parser.parseMore(commands, /*checkFilesExist*/ false);
+        parser.parseMore(commands);
         if (commands.size() == 0) {
           break;
         }