[WIP] Implement print-deserialized-declarations flag to dump source ranges of decls read during AST deserialization.

Author: VitaNuo

clang/include/clang/Driver/Options.td

@@ -7968,6 +7968,10 @@ def print_dependency_directives_minimized_source : Flag<["-"],
   "print-dependency-directives-minimized-source">,
   HelpText<"Print the output of the dependency directives source minimizer">;
 }
+def print_deserialized_declarations : Joined<["-"],
+  "print-deserialized-declarations-path=">,
+  HelpText<"Print traces of deserialized declarations to aid debugging and minimization">,
+  MarshallingInfoString<FrontendOpts<"PrintDeserializedDeclarationsPath">>;
 
 defm emit_llvm_uselists : BoolOption<"", "emit-llvm-uselists",
   CodeGenOpts<"EmitLLVMUseLists">, DefaultFalse,

clang/include/clang/Frontend/FrontendOptions.h

@@ -530,6 +530,9 @@ class FrontendOptions {
   /// Output Path for module output file.
   std::string ModuleOutputPath;
 
+  /// Output path to print deserialized declarations.
+  std::string PrintDeserializedDeclarationsPath;
+
 public:
   FrontendOptions()
       : DisableFree(false), RelocatablePCH(false), ShowHelp(false),

clang/lib/Frontend/CompilerInvocation.cpp

@@ -2975,6 +2975,11 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
 #undef FRONTEND_OPTION_WITH_MARSHALLING
 
   Opts.ProgramAction = frontend::ParseSyntaxOnly;
+  // if (Args.hasArg(OPT_print_deserialized_declarations)) {
+  //   llvm::errs() << "FLAG SET!!!\n";
+  //   FrontendOpts.PrintDeserializedDeclarations = true;
+  // }
+
   if (const Arg *A = Args.getLastArg(OPT_Action_Group)) {
     OptSpecifier Opt = OptSpecifier(A->getOption().getID());
     std::optional<frontend::ActionKind> ProgramAction = getFrontendAction(Opt);

clang/lib/Frontend/FrontendAction.cpp

@@ -15,6 +15,7 @@
 #include "clang/Basic/FileEntry.h"
 #include "clang/Basic/LangStandard.h"
 #include "clang/Basic/Sarif.h"
+#include "clang/Basic/SourceManager.h"
 #include "clang/Basic/Stack.h"
 #include "clang/Frontend/ASTUnit.h"
 #include "clang/Frontend/CompilerInstance.h"
@@ -35,6 +36,7 @@
 #include "clang/Serialization/ASTReader.h"
 #include "clang/Serialization/GlobalModuleIndex.h"
 #include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/BuryPointer.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
@@ -49,6 +51,135 @@ LLVM_INSTANTIATE_REGISTRY(FrontendPluginRegistry)
 
 namespace {
 
+/// Dumps deserialized declarations.
+class DeserializedDeclsLineRangePrinter : public DelegatingDeserializationListener, public ASTConsumer {
+public:
+  explicit DeserializedDeclsLineRangePrinter(SourceManager &SM, std::unique_ptr<llvm::raw_fd_ostream> OS)
+      : DelegatingDeserializationListener(nullptr, false), SM(SM), OS(std::move(OS)) {}
+
+  void DeclRead(GlobalDeclID ID, const Decl *D) override {
+    if (!IsCollectingDecls) {
+      return;
+    }
+    if (!D || isa<TranslationUnitDecl>(D) || isa<LinkageSpecDecl>(D) ||
+          isa<NamespaceDecl>(D))
+      return;
+    if (auto *DC = D->getDeclContext(); !DC || !DC->isFileContext())
+      return;
+    PendingDecls.push_back(D);
+    DelegatingDeserializationListener::DeclRead(ID, D);
+  }
+
+  using Position = std::pair<unsigned, unsigned>;
+  struct RequiredRanges {
+      StringRef Filename;
+      std::vector<std::pair<Position, Position>> FromTo;
+  };
+  void HandleTranslationUnit(ASTContext &Context) override {
+    IsCollectingDecls = false;
+    std::vector<const Decl *> Decls = std::move(PendingDecls);
+    if (!PendingDecls.empty()) {
+      llvm::errs() << "Deserialized more decls while printing, total of "
+                    << PendingDecls.size() << "\n";
+      PendingDecls.clear();
+    }
+
+    // Merge ranges in each of the files. For simplicity, track lines and hope
+    // they do not break things.
+    struct FileData {
+      std::vector<std::pair<Position, Position>> FromTo;
+      std::vector<std::pair<unsigned, unsigned>> Columns;
+      OptionalFileEntryRef Ref;
+    };
+    llvm::DenseMap<const FileEntry *, FileData> FileToLines;
+    for (const Decl *D : Decls) {
+      CharSourceRange R = SM.getExpansionRange(D->getSourceRange());
+      if (!R.isValid())
+        continue;
+
+      auto *F = SM.getFileEntryForID(SM.getFileID(R.getBegin()));
+      if (F != SM.getFileEntryForID(SM.getFileID(R.getEnd())))
+        continue;
+
+      auto &Data = FileToLines[F];
+      if (!Data.Ref)
+        Data.Ref =
+            SM.getFileEntryRefForID(SM.getFileID(R.getBegin()));
+      Data.FromTo.push_back({{SM.getSpellingLineNumber(R.getBegin()), SM.getSpellingColumnNumber(R.getBegin())},
+                            {SM.getSpellingLineNumber(R.getEnd()), SM.getSpellingColumnNumber(R.getEnd())}});
+    }
+
+    std::vector<RequiredRanges> Result;
+    for (auto &[F, Data] : FileToLines) {
+      auto& FromTo = Data.FromTo;
+      assert(!FromTo.empty());
+
+      if (!Data.Ref) continue;
+
+      llvm::sort(FromTo);
+
+      std::vector<std::pair<Position, Position>> MergedLines;
+      MergedLines.push_back(FromTo.front());
+      for (auto It = FromTo.begin() + 1; It < FromTo.end(); ++It) {
+        if (MergedLines.back().second < It->first) {
+          MergedLines.push_back(*It);
+          continue;
+        }
+        if (MergedLines.back().second < It->second)
+          MergedLines.back().second = It->second;
+      }
+      Result.push_back({Data.Ref->getName(), MergedLines});
+    }
+    printJson(Result);
+  }
+
+  void printJson(const std::vector<RequiredRanges>& Result) {
+    *OS << "{\n";
+    *OS << "  \"required_ranges\": [\n";
+    for (size_t i = 0; i < Result.size(); ++i) {
+      auto &F = Result[i].Filename;
+      auto &MergedLines = Result[i].FromTo;
+      *OS << "    {\n";
+      *OS << "      \"file\": \"" << F << "\",\n";
+      *OS << "      \"range\": [\n";
+      for (size_t j = 0; j < MergedLines.size(); ++j) {
+        auto &From = MergedLines[j].first;
+        auto &To = MergedLines[j].second;
+        *OS << "        {\n";
+        *OS << "          \"from\": {\n";
+        *OS << "            \"line\": " << From.first << ",\n";
+        *OS << "            \"column\": " << From.second << "\n          },\n";
+        *OS << "          \"to\": {\n";
+        *OS << "            \"line\": " << To.first << ",\n";
+        *OS << "            \"column\": " << To.second << "\n          }\n";
+        *OS << "        }";
+        if (j < MergedLines.size() - 1) {
+          *OS << ",";
+        }
+        *OS << "\n";
+      }
+      *OS << "      ]\n    }";
+      if (i < Result.size() - 1) {
+        *OS << ",";
+      }
+      *OS << "\n";
+    }
+    *OS << "  ]\n";
+    *OS << "}\n";
+  }
+
+  ASTDeserializationListener *GetASTDeserializationListener() override {
+    return this;
+  }
+
+private:
+std::vector<const Decl *> PendingDecls;
+bool IsCollectingDecls = true;
+const SourceManager &SM;
+std::unique_ptr<llvm::raw_ostream> OS;
+};
+
+
 /// Dumps deserialized declarations.
 class DeserializedDeclsDumper : public DelegatingDeserializationListener {
 public:
@@ -121,6 +252,19 @@ FrontendAction::CreateWrappedASTConsumer(CompilerInstance &CI,
   if (!Consumer)
     return nullptr;
 
+  std::vector<std::unique_ptr<ASTConsumer>> Consumers;
+  llvm::StringRef PrintDeserializedDeclarationsPath = CI.getFrontendOpts().PrintDeserializedDeclarationsPath;
+  if (!PrintDeserializedDeclarationsPath.empty()) {
+    std::error_code ErrorCode;
+    auto* FileStream = new llvm::raw_fd_ostream(PrintDeserializedDeclarationsPath, ErrorCode, llvm::sys::fs::OF_None);
+    if (!ErrorCode) {
+      DeserializedDeclsLineRangePrinter* Printer = new DeserializedDeclsLineRangePrinter(CI.getSourceManager(), std::unique_ptr<llvm::raw_fd_ostream>(FileStream));
+      Consumers.push_back(std::unique_ptr<ASTConsumer>(Printer));
+    } else {
+      llvm::errs() << "Could not open file with path: " << PrintDeserializedDeclarationsPath << ", error: " << ErrorCode.message() << "\n";
+    }
+  }
+
   // Validate -add-plugin args.
   bool FoundAllPlugins = true;
   for (const std::string &Arg : CI.getFrontendOpts().AddPluginActions) {
@@ -139,16 +283,17 @@ FrontendAction::CreateWrappedASTConsumer(CompilerInstance &CI,
     return nullptr;
 
   // If there are no registered plugins we don't need to wrap the consumer
-  if (FrontendPluginRegistry::begin() == FrontendPluginRegistry::end())
-    return Consumer;
+  if (FrontendPluginRegistry::begin() == FrontendPluginRegistry::end()) {
+    Consumers.push_back(std::move(Consumer));
+    return std::make_unique<MultiplexConsumer>(std::move(Consumers));
+  }
 
   // If this is a code completion run, avoid invoking the plugin consumers
   if (CI.hasCodeCompletionConsumer())
     return Consumer;
 
   // Collect the list of plugins that go before the main action (in Consumers)
   // or after it (in AfterConsumers)
-  std::vector<std::unique_ptr<ASTConsumer>> Consumers;
   std::vector<std::unique_ptr<ASTConsumer>> AfterConsumers;
   for (const FrontendPluginRegistry::entry &Plugin :
        FrontendPluginRegistry::entries()) {