merge main into amd-staging (llvm#1976)

Author: ronlieb

clang-tools-extra/clangd/index/FileIndex.cpp

@@ -48,13 +48,12 @@ SlabTuple indexSymbols(ASTContext &AST, Preprocessor &PP,
                        const MainFileMacros *MacroRefsToIndex,
                        const include_cleaner::PragmaIncludes &PI,
                        bool IsIndexMainAST, llvm::StringRef Version,
-                       bool CollectMainFileRefs) {
+                       bool CollectMainFileRefs, SymbolOrigin Origin) {
   SymbolCollector::Options CollectorOpts;
   CollectorOpts.CollectIncludePath = true;
   CollectorOpts.PragmaIncludes = Π
   CollectorOpts.CountReferences = false;
-  CollectorOpts.Origin =
-      IsIndexMainAST ? SymbolOrigin::Open : SymbolOrigin::Preamble;
+  CollectorOpts.Origin = Origin;
   CollectorOpts.CollectMainFileRefs = CollectMainFileRefs;
   // We want stdlib implementation details in the index only if we've opened the
   // file in question. This does means xrefs won't work, though.
@@ -221,22 +220,24 @@ FileShardedIndex::getShard(llvm::StringRef Uri) const {
 }
 
 SlabTuple indexMainDecls(ParsedAST &AST) {
-  return indexSymbols(
-      AST.getASTContext(), AST.getPreprocessor(), AST.getLocalTopLevelDecls(),
-      &AST.getMacros(), AST.getPragmaIncludes(),
-      /*IsIndexMainAST=*/true, AST.version(), /*CollectMainFileRefs=*/true);
+  return indexSymbols(AST.getASTContext(), AST.getPreprocessor(),
+                      AST.getLocalTopLevelDecls(), &AST.getMacros(),
+                      AST.getPragmaIncludes(),
+                      /*IsIndexMainAST=*/true, AST.version(),
+                      /*CollectMainFileRefs=*/true, SymbolOrigin::Open);
 }
 
 SlabTuple indexHeaderSymbols(llvm::StringRef Version, ASTContext &AST,
                              Preprocessor &PP,
-                             const include_cleaner::PragmaIncludes &PI) {
+                             const include_cleaner::PragmaIncludes &PI,
+                             SymbolOrigin Origin) {
   std::vector<Decl *> DeclsToIndex(
       AST.getTranslationUnitDecl()->decls().begin(),
       AST.getTranslationUnitDecl()->decls().end());
   return indexSymbols(AST, PP, DeclsToIndex,
                       /*MainFileMacros=*/nullptr, PI,
                       /*IsIndexMainAST=*/false, Version,
-                      /*CollectMainFileRefs=*/false);
+                      /*CollectMainFileRefs=*/false, Origin);
 }
 
 FileSymbols::FileSymbols(IndexContents IdxContents, bool SupportContainedRefs)
@@ -462,7 +463,7 @@ void FileIndex::updatePreamble(PathRef Path, llvm::StringRef Version,
                                const include_cleaner::PragmaIncludes &PI) {
   IndexFileIn IF;
   std::tie(IF.Symbols, std::ignore, IF.Relations) =
-      indexHeaderSymbols(Version, AST, PP, PI);
+      indexHeaderSymbols(Version, AST, PP, PI, SymbolOrigin::Preamble);
   updatePreamble(std::move(IF));
 }
 

clang-tools-extra/clangd/index/FileIndex.h

@@ -164,7 +164,8 @@ SlabTuple indexMainDecls(ParsedAST &AST);
 /// included headers.
 SlabTuple indexHeaderSymbols(llvm::StringRef Version, ASTContext &AST,
                              Preprocessor &PP,
-                             const include_cleaner::PragmaIncludes &PI);
+                             const include_cleaner::PragmaIncludes &PI,
+                             SymbolOrigin Origin);
 
 /// Takes slabs coming from a TU (multiple files) and shards them per
 /// declaration location.

clang-tools-extra/clangd/index/StdLib.cpp

@@ -15,12 +15,15 @@
 #include "Compiler.h"
 #include "Config.h"
 #include "SymbolCollector.h"
+#include "clang-include-cleaner/Record.h"
+#include "index/FileIndex.h"
 #include "index/IndexAction.h"
 #include "support/Logger.h"
 #include "support/ThreadsafeFS.h"
 #include "support/Trace.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/FrontendActions.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Tooling/Inclusions/StandardLibrary.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
@@ -223,33 +226,29 @@ SymbolSlab indexStandardLibrary(llvm::StringRef HeaderSources,
     return Symbols;
   }
 
-  SymbolCollector::Options IndexOpts;
-  IndexOpts.Origin = SymbolOrigin::StdLib;
-  IndexOpts.CollectMainFileSymbols = false;
-  IndexOpts.CollectMainFileRefs = false;
-  IndexOpts.CollectMacro = true;
-  IndexOpts.StoreAllDocumentation = true;
-  // Sadly we can't use IndexOpts.FileFilter to restrict indexing scope.
-  // Files from outside the StdLibLocation may define true std symbols anyway.
-  // We end up "blessing" such headers, and can only do that by indexing
-  // everything first.
-
-  // Refs, relations, include graph in the stdlib mostly aren't useful.
-  auto Action = createStaticIndexingAction(
-      IndexOpts, [&](SymbolSlab S) { Symbols = std::move(S); }, nullptr,
-      nullptr, nullptr);
-
-  if (!Action->BeginSourceFile(*Clang, Input)) {
+  SyntaxOnlyAction Action;
+
+  if (!Action.BeginSourceFile(*Clang, Input)) {
     elog("Standard Library Index: BeginSourceFile() failed");
     return Symbols;
   }
 
-  if (llvm::Error Err = Action->Execute()) {
+  if (llvm::Error Err = Action.Execute()) {
     elog("Standard Library Index: Execute failed: {0}", std::move(Err));
     return Symbols;
   }
 
-  Action->EndSourceFile();
+  // We don't care about include graph for stdlib headers, so provide a no-op
+  // PI.
+  include_cleaner::PragmaIncludes PI;
+  auto Slabs =
+      indexHeaderSymbols("", Clang->getASTContext(), Clang->getPreprocessor(),
+                         PI, SymbolOrigin::StdLib);
+  Symbols = std::move(std::get<0>(Slabs));
+
+  // Run EndSourceFile() after indexing completes, so ensure the AST and
+  // preprocessor state is alive during indexing.
+  Action.EndSourceFile();
 
   unsigned SymbolsBeforeFilter = Symbols.size();
   Symbols = filter(std::move(Symbols), Loc);

clang-tools-extra/clangd/unittests/StdLibTests.cpp

@@ -13,9 +13,11 @@
 #include "Config.h"
 #include "SyncAPI.h"
 #include "TestFS.h"
+#include "index/Index.h"
 #include "index/StdLib.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceManager.h"
+#include "llvm/Testing/Support/Error.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include <memory>
@@ -158,6 +160,52 @@ TEST(StdLibTests, EndToEnd) {
       UnorderedElementsAre(StdlibSymbol("list"), StdlibSymbol("vector")));
 }
 
+TEST(StdLibTests, StdLibDocComments) {
+  Config Cfg;
+  Cfg.Index.StandardLibrary = true;
+  WithContextValue Enabled(Config::Key, std::move(Cfg));
+
+  MockFS FS;
+  FS.Files["stdlib/vector"] = R"cpp(
+    namespace std {
+      struct vector {
+        /**doc comment*/
+        struct inner {};
+      };
+    }
+  )cpp";
+  MockCompilationDatabase CDB;
+  CDB.ExtraClangFlags.push_back("-isystem" + testPath("stdlib"));
+  ClangdServer::Options Opts = ClangdServer::optsForTest();
+  Opts.BuildDynamicSymbolIndex = true; // also used for stdlib index
+  ClangdServer Server(CDB, FS, Opts);
+
+  // Open an empty file. <vector> will not be part of the preamble index,
+  // but it will be part of the stdlib index.
+  Server.addDocument(testPath("foo.cc"), "");
+
+  // Wait for the stdlib index to be built.
+  ASSERT_TRUE(Server.blockUntilIdleForTest());
+
+  // Check that the index contains the doc comment.
+  SymbolSlab Result;
+  EXPECT_THAT_ERROR(runCustomAction(Server, testPath("foo.cc"),
+                                    [&](InputsAndAST Inputs) {
+                                      FuzzyFindRequest Req;
+                                      Req.Query = "inner";
+                                      Req.Scopes = {"std::vector::"};
+                                      SymbolSlab::Builder Builder;
+                                      Inputs.Inputs.Index->fuzzyFind(
+                                          Req, [&](const Symbol &Sym) {
+                                            Builder.insert(Sym);
+                                          });
+                                      Result = std::move(Builder).build();
+                                    }),
+                    llvm::Succeeded());
+  ASSERT_EQ(Result.size(), 1u);
+  EXPECT_EQ(Result.begin()->Documentation, "doc comment");
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/unittests/TestTU.cpp

@@ -12,6 +12,7 @@
 #include "Diagnostics.h"
 #include "TestFS.h"
 #include "index/FileIndex.h"
+#include "index/SymbolOrigin.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Frontend/CompilerInvocation.h"
@@ -164,7 +165,7 @@ SymbolSlab TestTU::headerSymbols() const {
   auto AST = build();
   return std::get<0>(indexHeaderSymbols(
       /*Version=*/"null", AST.getASTContext(), AST.getPreprocessor(),
-      AST.getPragmaIncludes()));
+      AST.getPragmaIncludes(), SymbolOrigin::Preamble));
 }
 
 RefSlab TestTU::headerRefs() const {

clang/include/clang/Basic/BuiltinsRISCVXCV.td

@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-class RISCXCVBuiltin<string prototype, string features = ""> : TargetBuiltin {
+class RISCVXCVBuiltin<string prototype, string features = ""> : TargetBuiltin {
   let Spellings = ["__builtin_riscv_cv_" # NAME];
   let Prototype = prototype;
   let Features = features;
@@ -21,21 +21,21 @@ let Attributes = [NoThrow, Const] in {
 //===----------------------------------------------------------------------===//
 // XCValu extension.
 //===----------------------------------------------------------------------===//
-def alu_slet  : RISCXCVBuiltin<"int(int, int)", "xcvalu">;
-def alu_sletu : RISCXCVBuiltin<"int(unsigned int, unsigned int)", "xcvalu">;
-def alu_exths : RISCXCVBuiltin<"int(int)", "xcvalu">;
-def alu_exthz : RISCXCVBuiltin<"unsigned int(unsigned int)", "xcvalu">;
-def alu_extbs : RISCXCVBuiltin<"int(int)", "xcvalu">;
-def alu_extbz : RISCXCVBuiltin<"unsigned int(unsigned int)", "xcvalu">;
+def alu_slet  : RISCVXCVBuiltin<"int(int, int)", "xcvalu">;
+def alu_sletu : RISCVXCVBuiltin<"int(unsigned int, unsigned int)", "xcvalu">;
+def alu_exths : RISCVXCVBuiltin<"int(int)", "xcvalu">;
+def alu_exthz : RISCVXCVBuiltin<"unsigned int(unsigned int)", "xcvalu">;
+def alu_extbs : RISCVXCVBuiltin<"int(int)", "xcvalu">;
+def alu_extbz : RISCVXCVBuiltin<"unsigned int(unsigned int)", "xcvalu">;
 
-def alu_clip   : RISCXCVBuiltin<"int(int, int)", "xcvalu">;
-def alu_clipu  : RISCXCVBuiltin<"unsigned int(unsigned int, unsigned int)", "xcvalu">;
-def alu_addN   : RISCXCVBuiltin<"int(int, int, unsigned int)", "xcvalu">;
-def alu_adduN  : RISCXCVBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int)", "xcvalu">;
-def alu_addRN  : RISCXCVBuiltin<"int(int, int, unsigned int)", "xcvalu">;
-def alu_adduRN : RISCXCVBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int)", "xcvalu">;
-def alu_subN   : RISCXCVBuiltin<"int(int, int, unsigned int)", "xcvalu">;
-def alu_subuN  : RISCXCVBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int)", "xcvalu">;
-def alu_subRN  : RISCXCVBuiltin<"int(int, int, unsigned int)", "xcvalu">;
-def alu_subuRN : RISCXCVBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int)", "xcvalu">;
+def alu_clip   : RISCVXCVBuiltin<"int(int, int)", "xcvalu">;
+def alu_clipu  : RISCVXCVBuiltin<"unsigned int(unsigned int, unsigned int)", "xcvalu">;
+def alu_addN   : RISCVXCVBuiltin<"int(int, int, unsigned int)", "xcvalu">;
+def alu_adduN  : RISCVXCVBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int)", "xcvalu">;
+def alu_addRN  : RISCVXCVBuiltin<"int(int, int, unsigned int)", "xcvalu">;
+def alu_adduRN : RISCVXCVBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int)", "xcvalu">;
+def alu_subN   : RISCVXCVBuiltin<"int(int, int, unsigned int)", "xcvalu">;
+def alu_subuN  : RISCVXCVBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int)", "xcvalu">;
+def alu_subRN  : RISCVXCVBuiltin<"int(int, int, unsigned int)", "xcvalu">;
+def alu_subuRN : RISCVXCVBuiltin<"unsigned int(unsigned int, unsigned int, unsigned int)", "xcvalu">;
 } // Attributes = [NoThrow, Const]

clang/include/clang/Interpreter/Interpreter.h

@@ -116,6 +116,9 @@ class Interpreter {
   /// Compiler instance performing the incremental compilation.
   std::unique_ptr<CompilerInstance> CI;
 
+  /// An optional compiler instance for CUDA offloading
+  std::unique_ptr<CompilerInstance> DeviceCI;
+
 protected:
   // Derived classes can use an extended interface of the Interpreter.
   Interpreter(std::unique_ptr<CompilerInstance> Instance, llvm::Error &Err,

clang/lib/AST/ByteCode/Context.cpp

@@ -134,6 +134,85 @@ bool Context::evaluateAsInitializer(State &Parent, const VarDecl *VD,
   return true;
 }
 
+template <typename ResultT>
+bool Context::evaluateStringRepr(State &Parent, const Expr *SizeExpr,
+                                 const Expr *PtrExpr, ResultT &Result) {
+  assert(Stk.empty());
+  Compiler<EvalEmitter> C(*this, *P, Parent, Stk);
+
+  // Evaluate size value.
+  APValue SizeValue;
+  if (!evaluateAsRValue(Parent, SizeExpr, SizeValue))
+    return false;
+
+  if (!SizeValue.isInt())
+    return false;
+  uint64_t Size = SizeValue.getInt().getZExtValue();
+
+  auto PtrRes = C.interpretAsPointer(PtrExpr, [&](const Pointer &Ptr) {
+    if (Size == 0) {
+      if constexpr (std::is_same_v<ResultT, APValue>)
+        Result = APValue(APValue::UninitArray{}, 0, 0);
+      return true;
+    }
+
+    if (!Ptr.isLive() || !Ptr.getFieldDesc()->isPrimitiveArray())
+      return false;
+
+    // Must be char.
+    if (Ptr.getFieldDesc()->getElemSize() != 1 /*bytes*/)
+      return false;
+
+    if (Size > Ptr.getNumElems()) {
+      Parent.FFDiag(SizeExpr, diag::note_constexpr_access_past_end) << AK_Read;
+      Size = Ptr.getNumElems();
+    }
+
+    if constexpr (std::is_same_v<ResultT, APValue>) {
+      QualType CharTy = PtrExpr->getType()->getPointeeType();
+      Result = APValue(APValue::UninitArray{}, Size, Size);
+      for (uint64_t I = 0; I != Size; ++I) {
+        if (std::optional<APValue> ElemVal =
+                Ptr.atIndex(I).toRValue(*this, CharTy))
+          Result.getArrayInitializedElt(I) = *ElemVal;
+        else
+          return false;
+      }
+    } else {
+      assert((std::is_same_v<ResultT, std::string>));
+      if (Size < Result.max_size())
+        Result.resize(Size);
+      Result.assign(reinterpret_cast<const char *>(Ptr.getRawAddress()), Size);
+    }
+
+    return true;
+  });
+
+  if (PtrRes.isInvalid()) {
+    C.cleanup();
+    Stk.clear();
+    return false;
+  }
+
+  return true;
+}
+
+bool Context::evaluateCharRange(State &Parent, const Expr *SizeExpr,
+                                const Expr *PtrExpr, APValue &Result) {
+  assert(SizeExpr);
+  assert(PtrExpr);
+
+  return evaluateStringRepr(Parent, SizeExpr, PtrExpr, Result);
+}
+
+bool Context::evaluateCharRange(State &Parent, const Expr *SizeExpr,
+                                const Expr *PtrExpr, std::string &Result) {
+  assert(SizeExpr);
+  assert(PtrExpr);
+
+  return evaluateStringRepr(Parent, SizeExpr, PtrExpr, Result);
+}
+
 const LangOptions &Context::getLangOpts() const { return Ctx.getLangOpts(); }
 
 std::optional<PrimType> Context::classify(QualType T) const {

clang/lib/AST/ByteCode/Context.h

@@ -59,6 +59,11 @@ class Context final {
   /// Evaluates a toplevel initializer.
   bool evaluateAsInitializer(State &Parent, const VarDecl *VD, APValue &Result);
 
+  bool evaluateCharRange(State &Parent, const Expr *SizeExpr,
+                         const Expr *PtrExpr, APValue &Result);
+  bool evaluateCharRange(State &Parent, const Expr *SizeExpr,
+                         const Expr *PtrExpr, std::string &Result);
+
   /// Returns the AST context.
   ASTContext &getASTContext() const { return Ctx; }
   /// Returns the language options.
@@ -120,6 +125,10 @@ class Context final {
   /// Runs a function.
   bool Run(State &Parent, const Function *Func);
 
+  template <typename ResultT>
+  bool evaluateStringRepr(State &Parent, const Expr *SizeExpr,
+                          const Expr *PtrExpr, ResultT &Result);
+
   /// Current compilation context.
   ASTContext &Ctx;
   /// Interpreter stack, shared across invocations.