[clangd] Store documentation when indexing standard library

Fixes clangd/clangd#2344

Author: HighCommander4

clang-tools-extra/clangd/ClangdServer.h

@@ -466,6 +466,9 @@ class ClangdServer {
   [[nodiscard]] bool
   blockUntilIdleForTest(std::optional<double> TimeoutSeconds = 10);
 
+  // Expose the (combined) index for test use.
+  const SymbolIndex *getIndexForTest() { return Index; }
+
   /// Builds a nested representation of memory used by components.
   void profile(MemoryTree &MT) const;
 

clang-tools-extra/clangd/index/FileIndex.cpp

@@ -48,13 +48,12 @@ SlabTuple indexSymbols(ASTContext &AST, Preprocessor &PP,
                        const MainFileMacros *MacroRefsToIndex,
                        const include_cleaner::PragmaIncludes &PI,
                        bool IsIndexMainAST, llvm::StringRef Version,
-                       bool CollectMainFileRefs) {
+                       bool CollectMainFileRefs, SymbolOrigin Origin) {
   SymbolCollector::Options CollectorOpts;
   CollectorOpts.CollectIncludePath = true;
   CollectorOpts.PragmaIncludes = Π
   CollectorOpts.CountReferences = false;
-  CollectorOpts.Origin =
-      IsIndexMainAST ? SymbolOrigin::Open : SymbolOrigin::Preamble;
+  CollectorOpts.Origin = Origin;
   CollectorOpts.CollectMainFileRefs = CollectMainFileRefs;
   // We want stdlib implementation details in the index only if we've opened the
   // file in question. This does means xrefs won't work, though.
@@ -221,22 +220,24 @@ FileShardedIndex::getShard(llvm::StringRef Uri) const {
 }
 
 SlabTuple indexMainDecls(ParsedAST &AST) {
-  return indexSymbols(
-      AST.getASTContext(), AST.getPreprocessor(), AST.getLocalTopLevelDecls(),
-      &AST.getMacros(), AST.getPragmaIncludes(),
-      /*IsIndexMainAST=*/true, AST.version(), /*CollectMainFileRefs=*/true);
+  return indexSymbols(AST.getASTContext(), AST.getPreprocessor(),
+                      AST.getLocalTopLevelDecls(), &AST.getMacros(),
+                      AST.getPragmaIncludes(),
+                      /*IsIndexMainAST=*/true, AST.version(),
+                      /*CollectMainFileRefs=*/true, SymbolOrigin::Open);
 }
 
 SlabTuple indexHeaderSymbols(llvm::StringRef Version, ASTContext &AST,
                              Preprocessor &PP,
-                             const include_cleaner::PragmaIncludes &PI) {
+                             const include_cleaner::PragmaIncludes &PI,
+                             SymbolOrigin Origin) {
   std::vector<Decl *> DeclsToIndex(
       AST.getTranslationUnitDecl()->decls().begin(),
       AST.getTranslationUnitDecl()->decls().end());
   return indexSymbols(AST, PP, DeclsToIndex,
                       /*MainFileMacros=*/nullptr, PI,
                       /*IsIndexMainAST=*/false, Version,
-                      /*CollectMainFileRefs=*/false);
+                      /*CollectMainFileRefs=*/false, Origin);
 }
 
 FileSymbols::FileSymbols(IndexContents IdxContents, bool SupportContainedRefs)
@@ -463,7 +464,7 @@ void FileIndex::updatePreamble(PathRef Path, llvm::StringRef Version,
                                const include_cleaner::PragmaIncludes &PI) {
   IndexFileIn IF;
   std::tie(IF.Symbols, std::ignore, IF.Relations) =
-      indexHeaderSymbols(Version, AST, PP, PI);
+      indexHeaderSymbols(Version, AST, PP, PI, SymbolOrigin::Preamble);
   updatePreamble(std::move(IF));
 }
 

clang-tools-extra/clangd/index/FileIndex.h

@@ -164,7 +164,8 @@ SlabTuple indexMainDecls(ParsedAST &AST);
 /// included headers.
 SlabTuple indexHeaderSymbols(llvm::StringRef Version, ASTContext &AST,
                              Preprocessor &PP,
-                             const include_cleaner::PragmaIncludes &PI);
+                             const include_cleaner::PragmaIncludes &PI,
+                             SymbolOrigin Origin);
 
 /// Takes slabs coming from a TU (multiple files) and shards them per
 /// declaration location.

clang-tools-extra/clangd/index/StdLib.cpp

@@ -15,12 +15,15 @@
 #include "Compiler.h"
 #include "Config.h"
 #include "SymbolCollector.h"
+#include "clang-include-cleaner/Record.h"
+#include "index/FileIndex.h"
 #include "index/IndexAction.h"
 #include "support/Logger.h"
 #include "support/ThreadsafeFS.h"
 #include "support/Trace.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/FrontendActions.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Tooling/Inclusions/StandardLibrary.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
@@ -223,33 +226,29 @@ SymbolSlab indexStandardLibrary(llvm::StringRef HeaderSources,
     return Symbols;
   }
 
-  SymbolCollector::Options IndexOpts;
-  IndexOpts.Origin = SymbolOrigin::StdLib;
-  IndexOpts.CollectMainFileSymbols = false;
-  IndexOpts.CollectMainFileRefs = false;
-  IndexOpts.CollectMacro = true;
-  IndexOpts.StoreAllDocumentation = true;
-  // Sadly we can't use IndexOpts.FileFilter to restrict indexing scope.
-  // Files from outside the StdLibLocation may define true std symbols anyway.
-  // We end up "blessing" such headers, and can only do that by indexing
-  // everything first.
-
-  // Refs, relations, include graph in the stdlib mostly aren't useful.
-  auto Action = createStaticIndexingAction(
-      IndexOpts, [&](SymbolSlab S) { Symbols = std::move(S); }, nullptr,
-      nullptr, nullptr);
-
-  if (!Action->BeginSourceFile(*Clang, Input)) {
+  SyntaxOnlyAction Action;
+
+  if (!Action.BeginSourceFile(*Clang, Input)) {
     elog("Standard Library Index: BeginSourceFile() failed");
     return Symbols;
   }
 
-  if (llvm::Error Err = Action->Execute()) {
+  if (llvm::Error Err = Action.Execute()) {
     elog("Standard Library Index: Execute failed: {0}", std::move(Err));
     return Symbols;
   }
 
-  Action->EndSourceFile();
+  // We don't care about include graph for stdlib headers, so provide a no-op
+  // PI.
+  include_cleaner::PragmaIncludes PI;
+  auto Slabs =
+      indexHeaderSymbols("", Clang->getASTContext(), Clang->getPreprocessor(),
+                         PI, SymbolOrigin::StdLib);
+  Symbols = std::move(std::get<0>(Slabs));
+
+  // Run EndSourceFile() after indexing completes, so ensure the AST and
+  // preprocessor state is alive during indexing.
+  Action.EndSourceFile();
 
   unsigned SymbolsBeforeFilter = Symbols.size();
   Symbols = filter(std::move(Symbols), Loc);

clang-tools-extra/clangd/unittests/StdLibTests.cpp

@@ -13,6 +13,7 @@
 #include "Config.h"
 #include "SyncAPI.h"
 #include "TestFS.h"
+#include "index/Index.h"
 #include "index/StdLib.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceManager.h"
@@ -158,6 +159,42 @@ TEST(StdLibTests, EndToEnd) {
       UnorderedElementsAre(StdlibSymbol("list"), StdlibSymbol("vector")));
 }
 
+TEST(StdLibTests, StdLibDocComments) {
+  Config Cfg;
+  Cfg.Index.StandardLibrary = true;
+  WithContextValue Enabled(Config::Key, std::move(Cfg));
+
+  MockFS FS;
+  FS.Files["stdlib/vector"] = R"cpp(
+    namespace std {
+      struct vector {
+        /**doc comment*/
+        struct inner {};
+      };
+    }
+  )cpp";
+  MockCompilationDatabase CDB;
+  CDB.ExtraClangFlags.push_back("-isystem" + testPath("stdlib"));
+  ClangdServer::Options Opts = ClangdServer::optsForTest();
+  Opts.BuildDynamicSymbolIndex = true; // also used for stdlib index
+  ClangdServer Server(CDB, FS, Opts);
+
+  // Open an empty file. <vector> will not be part of the preamble index,
+  // but it will be part of the stdlib index.
+  Server.addDocument(testPath("foo.cc"), "");
+
+  // Wait for the stdlib index to be built.
+  ASSERT_TRUE(Server.blockUntilIdleForTest());
+
+  // Check that the index contains the doc comment.
+  FuzzyFindRequest Req;
+  Req.Query = "inner";
+  Req.Scopes = {"std::vector::"};
+  SymbolSlab Result = runFuzzyFind(*Server.getIndexForTest(), Req);
+  ASSERT_EQ(Result.size(), 1u);
+  EXPECT_EQ(Result.begin()->Documentation, "doc comment");
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/unittests/TestTU.cpp

@@ -12,6 +12,7 @@
 #include "Diagnostics.h"
 #include "TestFS.h"
 #include "index/FileIndex.h"
+#include "index/SymbolOrigin.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Frontend/CompilerInvocation.h"
@@ -164,7 +165,7 @@ SymbolSlab TestTU::headerSymbols() const {
   auto AST = build();
   return std::get<0>(indexHeaderSymbols(
       /*Version=*/"null", AST.getASTContext(), AST.getPreprocessor(),
-      AST.getPragmaIncludes()));
+      AST.getPragmaIncludes(), SymbolOrigin::Preamble));
 }
 
 RefSlab TestTU::headerRefs() const {