[clangd][WIP] Add doxygen parsing for Hover

Author: tcottin

clang-tools-extra/clangd/CMakeLists.txt

@@ -108,6 +108,7 @@ add_clang_library(clangDaemon STATIC
   SemanticHighlighting.cpp
   SemanticSelection.cpp
   SourceCode.cpp
+  SymbolDocumentation.cpp
   SystemIncludeExtractor.cpp
   TidyProvider.cpp
   TUScheduler.cpp

clang-tools-extra/clangd/CodeComplete.cpp

@@ -504,11 +504,11 @@ struct CodeCompletionBuilder {
         }
       };
       if (C.IndexResult) {
-        SetDoc(C.IndexResult->Documentation);
+        SetDoc(C.IndexResult->Documentation.CommentText);
       } else if (C.SemaResult) {
-        const auto DocComment = getDocComment(*ASTCtx, *C.SemaResult,
-                                              /*CommentsFromHeaders=*/false);
-        SetDoc(formatDocumentation(*SemaCCS, DocComment));
+        const auto DocComment = getDocumentation(*ASTCtx, *C.SemaResult,
+                                                 /*CommentsFromHeaders=*/false);
+        SetDoc(formatDocumentation(*SemaCCS, DocComment.CommentText));
       }
     }
     if (Completion.Deprecated) {
@@ -1106,8 +1106,9 @@ class SignatureHelpCollector final : public CodeCompleteConsumer {
       ScoredSignatures.push_back(processOverloadCandidate(
           Candidate, *CCS,
           Candidate.getFunction()
-              ? getDeclComment(S.getASTContext(), *Candidate.getFunction())
-              : ""));
+              ? getDeclDocumentation(S.getASTContext(),
+                                     *Candidate.getFunction())
+              : SymbolDocumentationOwned{}));
     }
 
     // Sema does not load the docs from the preamble, so we need to fetch extra
@@ -1122,7 +1123,7 @@ class SignatureHelpCollector final : public CodeCompleteConsumer {
       }
       Index->lookup(IndexRequest, [&](const Symbol &S) {
         if (!S.Documentation.empty())
-          FetchedDocs[S.ID] = std::string(S.Documentation);
+          FetchedDocs[S.ID] = std::string(S.Documentation.CommentText);
       });
       vlog("SigHelp: requested docs for {0} symbols from the index, got {1} "
            "symbols with non-empty docs in the response",
@@ -1231,15 +1232,17 @@ class SignatureHelpCollector final : public CodeCompleteConsumer {
 
   // FIXME(ioeric): consider moving CodeCompletionString logic here to
   // CompletionString.h.
-  ScoredSignature processOverloadCandidate(const OverloadCandidate &Candidate,
-                                           const CodeCompletionString &CCS,
-                                           llvm::StringRef DocComment) const {
+  ScoredSignature
+  processOverloadCandidate(const OverloadCandidate &Candidate,
+                           const CodeCompletionString &CCS,
+                           const SymbolDocumentationOwned &DocComment) const {
     SignatureInformation Signature;
     SignatureQualitySignals Signal;
     const char *ReturnType = nullptr;
 
     markup::Document OverloadComment;
-    parseDocumentation(formatDocumentation(CCS, DocComment), OverloadComment);
+    parseDocumentation(formatDocumentation(CCS, DocComment.CommentText),
+                       OverloadComment);
     Signature.documentation = renderDoc(OverloadComment, DocumentationFormat);
     Signal.Kind = Candidate.getKind();
 
@@ -1898,7 +1901,7 @@ class CodeCompleteFlow {
           return;
         auto &C = Output.Completions[SymbolToCompletion.at(S.ID)];
         C.Documentation.emplace();
-        parseDocumentation(S.Documentation, *C.Documentation);
+        parseDocumentation(S.Documentation.CommentText, *C.Documentation);
       });
     }
 

clang-tools-extra/clangd/CodeCompletionStrings.cpp

@@ -80,39 +80,42 @@ bool shouldPatchPlaceholder0(CodeCompletionResult::ResultKind ResultKind,
 
 } // namespace
 
-std::string getDocComment(const ASTContext &Ctx,
-                          const CodeCompletionResult &Result,
-                          bool CommentsFromHeaders) {
+SymbolDocumentationOwned getDocumentation(const ASTContext &Ctx,
+                                          const CodeCompletionResult &Result,
+                                          bool CommentsFromHeaders) {
+  // FIXME: CommentsFromHeaders seems to be unused? Is this a bug?
+
   // FIXME: clang's completion also returns documentation for RK_Pattern if they
   // contain a pattern for ObjC properties. Unfortunately, there is no API to
   // get this declaration, so we don't show documentation in that case.
   if (Result.Kind != CodeCompletionResult::RK_Declaration)
-    return "";
-  return Result.getDeclaration() ? getDeclComment(Ctx, *Result.getDeclaration())
-                                 : "";
+    return {};
+  return Result.getDeclaration()
+             ? getDeclDocumentation(Ctx, *Result.getDeclaration())
+             : SymbolDocumentationOwned{};
 }
 
-std::string getDeclComment(const ASTContext &Ctx, const NamedDecl &Decl) {
+SymbolDocumentationOwned getDeclDocumentation(const ASTContext &Ctx,
+                                              const NamedDecl &Decl) {
   if (isa<NamespaceDecl>(Decl)) {
     // Namespaces often have too many redecls for any particular redecl comment
     // to be useful. Moreover, we often confuse file headers or generated
     // comments with namespace comments. Therefore we choose to just ignore
     // the comments for namespaces.
-    return "";
+    return {};
   }
   const RawComment *RC = getCompletionComment(Ctx, &Decl);
   if (!RC)
-    return "";
+    return {};
   // Sanity check that the comment does not come from the PCH. We choose to not
   // write them into PCH, because they are racy and slow to load.
   assert(!Ctx.getSourceManager().isLoadedSourceLocation(RC->getBeginLoc()));
-  std::string Doc =
-      RC->getFormattedText(Ctx.getSourceManager(), Ctx.getDiagnostics());
-  if (!looksLikeDocComment(Doc))
-    return "";
-  // Clang requires source to be UTF-8, but doesn't enforce this in comments.
-  if (!llvm::json::isUTF8(Doc))
-    Doc = llvm::json::fixUTF8(Doc);
+
+  SymbolDocumentationOwned Doc = parseDoxygenComment(*RC, Ctx, &Decl);
+
+  if (!looksLikeDocComment(Doc.CommentText))
+    return {};
+
   return Doc;
 }
 

clang-tools-extra/clangd/CodeCompletionStrings.h

@@ -16,24 +16,25 @@
 
 #include "clang/Sema/CodeCompleteConsumer.h"
 
+#include "SymbolDocumentation.h"
+
 namespace clang {
 class ASTContext;
 
 namespace clangd {
 
-/// Gets a minimally formatted documentation comment of \p Result, with comment
-/// markers stripped. See clang::RawComment::getFormattedText() for the detailed
-/// explanation of how the comment text is transformed.
-/// Returns empty string when no comment is available.
+/// Gets the parsed doxygen documentation of \p Result.
+/// Returns an empty SymbolDocumentationOwned when no comment is available.
 /// If \p CommentsFromHeaders parameter is set, only comments from the main
 /// file will be returned. It is used to workaround crashes when parsing
 /// comments in the stale headers, coming from completion preamble.
-std::string getDocComment(const ASTContext &Ctx,
-                          const CodeCompletionResult &Result,
-                          bool CommentsFromHeaders);
+SymbolDocumentationOwned getDocumentation(const ASTContext &Ctx,
+                                          const CodeCompletionResult &Result,
+                                          bool CommentsFromHeaders);
 
-/// Similar to getDocComment, but returns the comment for a NamedDecl.
-std::string getDeclComment(const ASTContext &Ctx, const NamedDecl &D);
+/// Similar to getDocumentation, but returns the comment for a NamedDecl.
+SymbolDocumentationOwned getDeclDocumentation(const ASTContext &Ctx,
+                                              const NamedDecl &D);
 
 /// Formats the signature for an item, as a display string and snippet.
 /// e.g. for const_reference std::vector<T>::at(size_type) const, this returns:

clang-tools-extra/clangd/Hover.cpp

@@ -347,7 +347,7 @@ void enhanceFromIndex(HoverInfo &Hover, const NamedDecl &ND,
   LookupRequest Req;
   Req.IDs.insert(ID);
   Index->lookup(Req, [&](const Symbol &S) {
-    Hover.Documentation = std::string(S.Documentation);
+    Hover.Documentation = S.Documentation.toOwned();
   });
 }
 
@@ -625,10 +625,11 @@ HoverInfo getHoverContents(const NamedDecl *D, const PrintingPolicy &PP,
 
   HI.Name = printName(Ctx, *D);
   const auto *CommentD = getDeclForComment(D);
-  HI.Documentation = getDeclComment(Ctx, *CommentD);
+  HI.Documentation = getDeclDocumentation(Ctx, *CommentD);
   enhanceFromIndex(HI, *CommentD, Index);
   if (HI.Documentation.empty())
-    HI.Documentation = synthesizeDocumentation(D);
+    HI.Documentation =
+        SymbolDocumentationOwned::descriptionOnly(synthesizeDocumentation(D));
 
   HI.Kind = index::getSymbolInfo(D).Kind;
 
@@ -682,7 +683,8 @@ getPredefinedExprHoverContents(const PredefinedExpr &PE, ASTContext &Ctx,
   HoverInfo HI;
   HI.Name = PE.getIdentKindName();
   HI.Kind = index::SymbolKind::Variable;
-  HI.Documentation = "Name of the current function (predefined variable)";
+  HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+      "Name of the current function (predefined variable)");
   if (const StringLiteral *Name = PE.getFunctionName()) {
     HI.Value.emplace();
     llvm::raw_string_ostream OS(*HI.Value);
@@ -856,7 +858,7 @@ HoverInfo getDeducedTypeHoverContents(QualType QT, const syntax::Token &Tok,
 
     if (const auto *D = QT->getAsTagDecl()) {
       const auto *CommentD = getDeclForComment(D);
-      HI.Documentation = getDeclComment(ASTCtx, *CommentD);
+      HI.Documentation = getDeclDocumentation(ASTCtx, *CommentD);
       enhanceFromIndex(HI, *CommentD, Index);
     }
   }
@@ -956,7 +958,8 @@ std::optional<HoverInfo> getHoverContents(const Attr *A, ParsedAST &AST) {
     llvm::raw_string_ostream OS(HI.Definition);
     A->printPretty(OS, AST.getASTContext().getPrintingPolicy());
   }
-  HI.Documentation = Attr::getDocumentation(A->getKind()).str();
+  HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+      Attr::getDocumentation(A->getKind()).str());
   return HI;
 }
 
@@ -1455,6 +1458,10 @@ markup::Document HoverInfo::present() const {
 
   // Put a linebreak after header to increase readability.
   Output.addRuler();
+
+  if (!Documentation.Brief.empty())
+    parseDocumentation(Documentation.Brief, Output);
+
   // Print Types on their own lines to reduce chances of getting line-wrapped by
   // editor, as they might be long.
   if (ReturnType) {
@@ -1463,15 +1470,44 @@ markup::Document HoverInfo::present() const {
     // Parameters:
     // - `bool param1`
     // - `int param2 = 5`
-    Output.addParagraph().appendText("→ ").appendCode(
+    auto &P = Output.addParagraph().appendText("→ ").appendCode(
         llvm::to_string(*ReturnType));
-  }
 
+    if (!Documentation.Returns.empty())
+      P.appendText(": ").appendText(Documentation.Returns);
+  }
   if (Parameters && !Parameters->empty()) {
     Output.addParagraph().appendText("Parameters: ");
     markup::BulletList &L = Output.addBulletList();
-    for (const auto &Param : *Parameters)
-      L.addItem().addParagraph().appendCode(llvm::to_string(Param));
+
+    llvm::SmallVector<ParameterDocumentationOwned> ParamDocs =
+        Documentation.Parameters;
+
+    for (const auto &Param : *Parameters) {
+      auto &Paragraph = L.addItem().addParagraph();
+      Paragraph.appendCode(llvm::to_string(Param));
+
+      if (Param.Name.has_value()) {
+        auto ParamDoc = std::find_if(ParamDocs.begin(), ParamDocs.end(),
+                                     [Param](const auto &ParamDoc) {
+                                       return Param.Name == ParamDoc.Name;
+                                     });
+        if (ParamDoc != ParamDocs.end()) {
+          Paragraph.appendText(": ").appendText(ParamDoc->Description);
+          ParamDocs.erase(ParamDoc);
+        }
+      }
+    }
+
+    // We erased all parameters that matched, but some may still be left,
+    // usually typos. Let's also print them here.
+    for (const auto &ParamDoc : ParamDocs) {
+      L.addItem()
+          .addParagraph()
+          .appendCode(ParamDoc.Name)
+          .appendText(": ")
+          .appendText(ParamDoc.Description);
+    }
   }
 
   // Don't print Type after Parameters or ReturnType as this will just duplicate
@@ -1518,8 +1554,30 @@ markup::Document HoverInfo::present() const {
     Output.addParagraph().appendText(OS.str());
   }
 
-  if (!Documentation.empty())
-    parseDocumentation(Documentation, Output);
+  if (!Documentation.Description.empty())
+    parseDocumentation(Documentation.Description, Output);
+
+  if (!Documentation.Warnings.empty()) {
+    Output.addRuler();
+    Output.addParagraph()
+        .appendText("Warning")
+        .appendText(Documentation.Warnings.size() > 1 ? "s" : "")
+        .appendText(": ");
+    markup::BulletList &L = Output.addBulletList();
+    for (const auto &Warning : Documentation.Warnings)
+      parseDocumentation(Warning, L.addItem());
+  }
+
+  if (!Documentation.Notes.empty()) {
+    Output.addRuler();
+    Output.addParagraph()
+        .appendText("Note")
+        .appendText(Documentation.Notes.size() > 1 ? "s" : "")
+        .appendText(": ");
+    markup::BulletList &L = Output.addBulletList();
+    for (const auto &Note : Documentation.Notes)
+      parseDocumentation(Note, L.addItem());
+  }
 
   if (!Definition.empty()) {
     Output.addRuler();

clang-tools-extra/clangd/Hover.h

@@ -11,6 +11,7 @@
 
 #include "ParsedAST.h"
 #include "Protocol.h"
+#include "SymbolDocumentation.h"
 #include "support/Markup.h"
 #include "clang/Index/IndexSymbol.h"
 #include <optional>
@@ -73,7 +74,7 @@ struct HoverInfo {
   std::string Provider;
   std::optional<Range> SymRange;
   index::SymbolKind Kind = index::SymbolKind::Unknown;
-  std::string Documentation;
+  SymbolDocumentationOwned Documentation;
   /// Source code containing the definition of the symbol.
   std::string Definition;
   const char *DefinitionLanguage = "cpp";