Skip non existing file

Indexer-PiperOrigin-RevId: 840240920

Author: OSS-Fuzz Team

infra/indexer/index/file_copier.cc

@@ -42,11 +42,12 @@ void PreparePath(std::string& path) {
 FileCopier::FileCopier(absl::string_view base_path,
                        absl::string_view index_path,
                        const std::vector<std::string>& extra_paths,
-                       Behavior behavior)
+                       Behavior behavior, bool skip_missing_files)
     : base_path_(base_path),
       extra_paths_(extra_paths),
       index_path_(index_path),
-      behavior_(behavior) {
+      behavior_(behavior),
+      skip_missing_files_(skip_missing_files) {
   if (behavior_ == Behavior::kNoOp) {
     return;
   }
@@ -102,10 +103,17 @@ void FileCopier::CopyIndexedFiles() {
       dst_path = std::filesystem::path(index_path_) / "relative" / indexed_path;
     }
 
-    DLOG(INFO) << "\nFrom: " << src_path << "\n  To: " << dst_path << "\n";
+    if (!std::filesystem::exists(src_path)) {
+      if (!skip_missing_files_) {
+        LOG(QFATAL) << "Source file " << src_path
+                   << " does not exist and skip_missing_files is false.";
+      } else {
+        LOG(WARNING) << "Skipping non-existent source file: " << src_path;
+        continue;
+      }
+    }
 
-    QCHECK(std::filesystem::exists(src_path))
-        << "Source file does not exist: " << src_path;
+    DLOG(INFO) << "\nFrom: " << src_path << "\n  To: " << dst_path << "\n";
 
     std::error_code error_code;
     // The destination directory may already exist, but report other errors.

infra/indexer/index/file_copier.h

@@ -32,11 +32,16 @@ namespace indexer {
 // all InMemoryIndexes for the current project.
 class FileCopier {
  public:
-  enum class Behavior { kNoOp, kFailOnExistingFiles, kOverwriteExistingFiles };
+  enum class Behavior {
+    kNoOp,
+    kFailOnExistingFiles,
+    kOverwriteExistingFiles,
+  };
 
   FileCopier(absl::string_view base_path, absl::string_view index_path,
              const std::vector<std::string>& extra_paths,
-             Behavior behavior = Behavior::kFailOnExistingFiles);
+             Behavior behavior = Behavior::kFailOnExistingFiles,
+             bool skip_missing_files = false);
   FileCopier(const FileCopier&) = delete;
 
   // Takes an absolute path. Rewrites this path into the representation it will
@@ -55,6 +60,7 @@ class FileCopier {
   std::vector<std::string> extra_paths_;
   const std::filesystem::path index_path_;
   const Behavior behavior_;
+  const bool skip_missing_files_;
 
   absl::Mutex mutex_;
   absl::flat_hash_set<std::string> indexed_files_ ABSL_GUARDED_BY(mutex_);

infra/indexer/main.cc

@@ -62,6 +62,9 @@ ABSL_FLAG(bool, delta, false,
 ABSL_FLAG(std::string, database_only, "",
           "Do not copy source files, only build the index database at the given"
           " location (--index_dir is not effective in that case)");
+ABSL_FLAG(bool, skip_missing_files, false,
+          "If set to true, missing source files will be skipped with a warning."
+          " If false, the indexer will error out on missing files.");
 
 static constexpr char kIndexDbName[] = "db.sqlite";
 static constexpr char kDeltaDbName[] = "delta.sqlite";
@@ -122,7 +125,8 @@ int main(int argc, char** argv) {
     }
   }
 
-  FileCopier file_copier(source_dir, index_dir, extra_dirs, behavior);
+  FileCopier file_copier(source_dir, index_dir, extra_dirs, behavior,
+                         absl::GetFlag(FLAGS_skip_missing_files));
 
   std::unique_ptr<MergeQueue> merge_queue = MergeQueue::Create(
       absl::GetFlag(FLAGS_merge_queues), absl::GetFlag(FLAGS_merge_queue_size));