Merge pull request swiftlang#30190 from hamishknight/the-beginning-of-the-pipeline-end

Requestify SourceFile parsing

Author: hamishknight

include/swift/AST/ParseRequests.h

@@ -80,6 +80,27 @@ class ParseAbstractFunctionBodyRequest :
   void cacheResult(BraceStmt *value) const;
 };
 
+/// Parse the top-level decls of a SourceFile.
+class ParseSourceFileRequest
+    : public SimpleRequest<ParseSourceFileRequest,
+                           ArrayRef<Decl *>(SourceFile *),
+                           CacheKind::SeparatelyCached> {
+public:
+  using SimpleRequest::SimpleRequest;
+
+private:
+  friend SimpleRequest;
+
+  // Evaluation.
+  ArrayRef<Decl *> evaluate(Evaluator &evaluator, SourceFile *SF) const;
+
+public:
+  // Caching.
+  bool isCached() const { return true; }
+  Optional<ArrayRef<Decl *>> getCachedResult() const;
+  void cacheResult(ArrayRef<Decl *> decls) const;
+};
+
 /// The zone number for the parser.
 #define SWIFT_TYPEID_ZONE Parse
 #define SWIFT_TYPEID_HEADER "swift/AST/ParseTypeIDZone.def"

include/swift/AST/ParseTypeIDZone.def

@@ -19,3 +19,6 @@ SWIFT_REQUEST(Parse, ParseMembersRequest,
 SWIFT_REQUEST(Parse, ParseAbstractFunctionBodyRequest,
               BraceStmt *(AbstractFunctionDecl *), SeparatelyCached,
               NoLocationInfo)
+SWIFT_REQUEST(Parse, ParseSourceFileRequest,
+              ArrayRef<Decl *>(SourceFile *), SeparatelyCached,
+              NoLocationInfo)

include/swift/AST/SourceFile.h

@@ -27,6 +27,8 @@ class PersistentParserState;
 /// before being used for anything; a full type-check is also necessary for
 /// IR generation.
 class SourceFile final : public FileUnit {
+  friend class ParseSourceFileRequest;
+
 public:
   class Impl;
   struct SourceFileSyntaxInfo;
@@ -175,8 +177,11 @@ class SourceFile final : public FileUnit {
   /// been validated.
   llvm::SetVector<ValueDecl *> UnvalidatedDeclsWithOpaqueReturnTypes;
 
-  /// The list of top-level declarations in the source file.
-  std::vector<Decl *> Decls;
+  /// The list of top-level declarations in the source file. This is \c None if
+  /// they have not yet been parsed.
+  /// FIXME: Once addTopLevelDecl/prependTopLevelDecl/truncateTopLevelDecls
+  /// have been removed, this can become an optional ArrayRef.
+  Optional<std::vector<Decl *>> Decls;
 
   using SeparatelyImportedOverlayMap =
     llvm::SmallDenseMap<ModuleDecl *, llvm::SmallPtrSet<ModuleDecl *, 1>>;
@@ -203,9 +208,12 @@ class SourceFile final : public FileUnit {
   friend Impl;
 
 public:
-  /// Appends the given declaration to the end of the top-level decls list.
+  /// Appends the given declaration to the end of the top-level decls list. Do
+  /// not add any additional uses of this function.
   void addTopLevelDecl(Decl *d) {
-    Decls.push_back(d);
+    // Force decl parsing if we haven't already.
+    (void)getTopLevelDecls();
+    Decls->push_back(d);
   }
 
   /// Prepends a declaration to the top-level decls list.
@@ -215,18 +223,29 @@ class SourceFile final : public FileUnit {
   ///
   /// See rdar://58355191
   void prependTopLevelDecl(Decl *d) {
-    Decls.insert(Decls.begin(), d);
+    // Force decl parsing if we haven't already.
+    (void)getTopLevelDecls();
+    Decls->insert(Decls->begin(), d);
   }
 
   /// Retrieves an immutable view of the list of top-level decls in this file.
-  ArrayRef<Decl *> getTopLevelDecls() const {
-    return Decls;
+  ArrayRef<Decl *> getTopLevelDecls() const;
+
+  /// Retrieves an immutable view of the top-level decls if they have already
+  /// been parsed, or \c None if they haven't. Should only be used for dumping.
+  Optional<ArrayRef<Decl *>> getCachedTopLevelDecls() const {
+    if (!Decls)
+      return None;
+    return llvm::makeArrayRef(*Decls);
   }
 
-  /// Truncates the list of top-level decls so it contains \c count elements.
+  /// Truncates the list of top-level decls so it contains \c count elements. Do
+  /// not add any additional uses of this function.
   void truncateTopLevelDecls(unsigned count) {
-    assert(count <= Decls.size() && "Can only truncate top-level decls!");
-    Decls.resize(count);
+    // Force decl parsing if we haven't already.
+    (void)getTopLevelDecls();
+    assert(count <= Decls->size() && "Can only truncate top-level decls!");
+    Decls->resize(count);
   }
 
   /// Retrieve the parsing options for the file.
@@ -287,10 +306,8 @@ class SourceFile final : public FileUnit {
   const SourceFileKind Kind;
 
   enum ASTStage_t {
-    /// Parsing is underway.
-    Parsing,
-    /// Parsing has completed.
-    Parsed,
+    /// The source file is not name bound or type checked.
+    Unprocessed,
     /// Name binding has completed.
     NameBound,
     /// Type checking has completed.
@@ -302,7 +319,7 @@ class SourceFile final : public FileUnit {
   ///
   /// Only files that have been fully processed (i.e. type-checked) will be
   /// forwarded on to IRGen.
-  ASTStage_t ASTStage = Parsing;
+  ASTStage_t ASTStage = Unprocessed;
 
   SourceFile(ModuleDecl &M, SourceFileKind K, Optional<unsigned> bufferID,
              ImplicitModuleImportKind ModImpKind, bool KeepParsedTokens = false,
@@ -456,6 +473,13 @@ class SourceFile final : public FileUnit {
   /// Retrieves the previously set delayed parser state, asserting that it
   /// exists.
   PersistentParserState *getDelayedParserState() {
+    // Force parsing of the top-level decls, which will set DelayedParserState
+    // if necessary.
+    // FIXME: Ideally the parser state should be an output of
+    // ParseSourceFileRequest, but the evaluator doesn't currently support
+    // move-only outputs for cached requests.
+    (void)getTopLevelDecls();
+
     auto *state = DelayedParserState.get();
     assert(state && "Didn't set any delayed parser state!");
     return state;

include/swift/Parse/ParseSILSupport.h

@@ -37,15 +37,6 @@ namespace swift {
     virtual bool parseSILProperty(Parser &P) = 0;
     virtual bool parseSILScope(Parser &P) = 0;
   };
-
-  /// To assist debugging parser crashes, tell us the location of the
-  /// current token.
-  class PrettyStackTraceParser : public llvm::PrettyStackTraceEntry {
-    Parser &P;
-  public:
-    explicit PrettyStackTraceParser(Parser &P) : P(P) {}
-    void print(llvm::raw_ostream &out) const override;
-  };
 } // end namespace swift
 
 #endif // SWIFT_PARSER_PARSESILSUPPORT_H

include/swift/Parse/Parser.h

@@ -114,14 +114,17 @@ class Parser {
   DiagnosticEngine &Diags;
   SourceFile &SF;
   Lexer *L;
-  SILParserTUStateBase *SIL; // Non-null when parsing a .sil file.
+  SILParserTUStateBase *SIL; // Non-null when parsing SIL decls.
   PersistentParserState *State;
   std::unique_ptr<PersistentParserState> OwnedState;
   DeclContext *CurDeclContext;
   ASTContext &Context;
   CodeCompletionCallbacks *CodeCompletion = nullptr;
   std::vector<Located<std::vector<ParamDecl*>>> AnonClosureVars;
 
+  /// Tracks parsed decls that LLDB requires to be inserted at the top-level.
+  std::vector<Decl *> ContextSwitchedTopLevelDecls;
+
   NullablePtr<llvm::MD5> CurrentTokenHash;
   void recordTokenHash(const Token Tok) {
     if (!Tok.getText().empty())
@@ -881,8 +884,8 @@ class Parser {
   /// Returns true if the parser is at the start of a SIL decl.
   bool isStartOfSILDecl();
 
-  /// Parse the top-level Swift decls into the source file.
-  void parseTopLevel();
+  /// Parse the top-level Swift decls into the provided vector.
+  void parseTopLevel(SmallVectorImpl<Decl *> &decls);
 
   /// Parse the top-level SIL decls into the SIL module.
   void parseTopLevelSIL();
@@ -1728,6 +1731,15 @@ struct ParsedDeclName {
   DeclNameRef formDeclNameRef(ASTContext &ctx) const;
 };
 
+/// To assist debugging parser crashes, tell us the location of the
+/// current token.
+class PrettyStackTraceParser : public llvm::PrettyStackTraceEntry {
+  Parser &P;
+public:
+  explicit PrettyStackTraceParser(Parser &P) : P(P) {}
+  void print(llvm::raw_ostream &out) const override;
+};
+
 /// Parse a stringified Swift declaration name,
 /// e.g. "Foo.translateBy(self:x:y:)".
 ParsedDeclName parseDeclName(StringRef name) LLVM_READONLY;

include/swift/Subsystems.h

@@ -103,13 +103,6 @@ namespace swift {
 
   /// @}
 
-  /// Parse a single buffer into the given source file.
-  ///
-  /// \param SF The file within the module being parsed.
-  ///
-  /// \param BufferID The buffer to parse from.
-  void parseIntoSourceFile(SourceFile &SF, unsigned BufferID);
-
   /// Parse a source file's SIL declarations into a given SIL module.
   void parseSourceFileSIL(SourceFile &SF, SILParserState *sil);
 

lib/AST/ASTDumper.cpp

@@ -781,13 +781,15 @@ namespace {
       PrintWithColorRAII(OS, ParenthesisColor) << '(';
       PrintWithColorRAII(OS, ASTNodeColor) << "source_file ";
       PrintWithColorRAII(OS, LocationColor) << '\"' << SF.getFilename() << '\"';
-      
-      for (Decl *D : SF.getTopLevelDecls()) {
-        if (D->isImplicit())
-          continue;
 
-        OS << '\n';
-        printRec(D);
+      if (auto decls = SF.getCachedTopLevelDecls()) {
+        for (Decl *D : *decls) {
+          if (D->isImplicit())
+            continue;
+
+          OS << '\n';
+          printRec(D);
+        }
       }
       PrintWithColorRAII(OS, ParenthesisColor) << ')';
     }

lib/AST/Module.cpp

@@ -32,6 +32,7 @@
 #include "swift/AST/ModuleLoader.h"
 #include "swift/AST/NameLookup.h"
 #include "swift/AST/ReferencedNameTracker.h"
+#include "swift/AST/ParseRequests.h"
 #include "swift/AST/PrettyStackTrace.h"
 #include "swift/AST/PrintOptions.h"
 #include "swift/AST/ProtocolConformance.h"
@@ -674,7 +675,8 @@ void ModuleDecl::getTopLevelDeclsWhereAttributesMatch(
 }
 
 void SourceFile::getTopLevelDecls(SmallVectorImpl<Decl*> &Results) const {
-  Results.append(Decls.begin(), Decls.end());
+  auto decls = getTopLevelDecls();
+  Results.append(decls.begin(), decls.end());
 }
 
 void ModuleDecl::getPrecedenceGroups(
@@ -1190,7 +1192,14 @@ void ModuleDecl::getImportedModules(SmallVectorImpl<ImportedModule> &modules,
 void
 SourceFile::getImportedModules(SmallVectorImpl<ModuleDecl::ImportedModule> &modules,
                                ModuleDecl::ImportFilter filter) const {
-  assert(ASTStage >= Parsed || Kind == SourceFileKind::SIL);
+  // FIXME: Ideally we should assert that the file has been name bound before
+  // calling this function. However unfortunately that can cause issues for
+  // overlays which can depend on a Clang submodule for the underlying framework
+  // they are overlaying, which causes us to attempt to load the overlay again.
+  // We need to find a way to ensure that an overlay dependency with the same
+  // name as the overlay always loads the underlying Clang module. We currently
+  // handle this for a direct import from the overlay, but not when it happens
+  // through other imports.
   assert(filter && "no imports requested?");
   for (auto desc : Imports) {
     ModuleDecl::ImportFilter requiredFilter;
@@ -1688,7 +1697,7 @@ void SourceFile::print(raw_ostream &OS, const PrintOptions &PO) {
 void SourceFile::print(ASTPrinter &Printer, const PrintOptions &PO) {
   std::set<DeclKind> MajorDeclKinds = {DeclKind::Class, DeclKind::Enum,
     DeclKind::Extension, DeclKind::Protocol, DeclKind::Struct};
-  for (auto decl : Decls) {
+  for (auto decl : getTopLevelDecls()) {
     if (!decl->shouldPrintInContext(PO))
       continue;
     // For a major decl, we print an empty line before it.
@@ -2024,6 +2033,13 @@ bool SourceFile::hasDelayedBodyParsing() const {
   return true;
 }
 
+ArrayRef<Decl *> SourceFile::getTopLevelDecls() const {
+  auto &ctx = getASTContext();
+  auto *mutableThis = const_cast<SourceFile *>(this);
+  return evaluateOrDefault(ctx.evaluator, ParseSourceFileRequest{mutableThis},
+                           {});
+}
+
 bool FileUnit::walk(ASTWalker &walker) {
   SmallVector<Decl *, 64> Decls;
   getTopLevelDecls(Decls);
@@ -2057,7 +2073,7 @@ bool FileUnit::walk(ASTWalker &walker) {
 bool SourceFile::walk(ASTWalker &walker) {
   llvm::SaveAndRestore<ASTWalker::ParentTy> SAR(walker.Parent,
                                                 getParentModule());
-  for (Decl *D : Decls) {
+  for (Decl *D : getTopLevelDecls()) {
 #ifndef NDEBUG
     PrettyStackTraceDecl debugStack("walking into decl", D);
 #endif

lib/Frontend/Frontend.cpp

@@ -763,11 +763,7 @@ void CompilerInstance::performSema() {
 }
 
 void CompilerInstance::performSemaUpTo(SourceFile::ASTStage_t LimitStage) {
-  // FIXME: A lot of the logic in `performParseOnly` is a stripped-down version
-  // of the logic in `performSemaUpTo`.  We should try to unify them over time.
-  if (LimitStage <= SourceFile::Parsed) {
-    return performParseOnly();
-  }
+  assert(LimitStage > SourceFile::Unprocessed);
 
   FrontendStatsTracer tracer(getStatsReporter(), "perform-sema");
 
@@ -974,7 +970,7 @@ void CompilerInstance::parseLibraryFile(
       SourceFileKind::Library, implicitImports.kind, BufferID);
   addAdditionalInitialImportsTo(NextInput, implicitImports);
 
-  parseIntoSourceFile(*NextInput, BufferID);
+  // Name binding will lazily trigger parsing of the file.
   performNameBinding(*NextInput);
 }
 
@@ -1017,11 +1013,8 @@ void CompilerInstance::parseAndTypeCheckMainFileUpTo(
   auto DidSuppressWarnings = Diags.getSuppressWarnings();
   Diags.setSuppressWarnings(DidSuppressWarnings || !mainIsPrimary);
 
-  // Parse the Swift decls into the source file.
-  parseIntoSourceFile(MainFile, MainBufferID);
-
-  // For a primary, also perform type checking if needed. Otherwise, just do
-  // name binding.
+  // For a primary, perform type checking if needed. Otherwise, just do name
+  // binding.
   if (mainIsPrimary && LimitStage >= SourceFile::TypeChecked) {
     performTypeChecking(MainFile);
   } else {
@@ -1142,17 +1135,18 @@ void CompilerInstance::performParseOnly(bool EvaluateConditionals,
         SourceFileKind::Library, SourceFile::ImplicitModuleImportKind::None,
         BufferID, parsingOpts);
 
-    parseIntoSourceFile(*NextInput, BufferID);
+    // Force the parsing of the top level decls.
+    (void)NextInput->getTopLevelDecls();
   }
 
   // Now parse the main file.
   if (MainBufferID != NO_SUCH_BUFFER) {
     SourceFile &MainFile =
         MainModule->getMainSourceFile(Invocation.getSourceFileKind());
     MainFile.SyntaxParsingCache = Invocation.getMainFileSyntaxParsingCache();
-    assert(MainBufferID == MainFile.getBufferID());
 
-    parseIntoSourceFile(MainFile, MainBufferID);
+    // Force the parsing of the top level decls.
+    (void)MainFile.getTopLevelDecls();
   }
 
   assert(Context->LoadedModules.size() == 1 &&

lib/IDE/CompletionInstance.cpp

@@ -199,6 +199,7 @@ bool CompletionInstance::performCachedOperaitonIfPossible(
   DiagnosticEngine tmpDiags(tmpSM);
   std::unique_ptr<ASTContext> tmpCtx(
       ASTContext::get(langOpts, typeckOpts, searchPathOpts, tmpSM, tmpDiags));
+  registerParseRequestFunctions(tmpCtx->evaluator);
   registerIDERequestFunctions(tmpCtx->evaluator);
   registerTypeCheckerRequestFunctions(tmpCtx->evaluator);
   registerSILGenRequestFunctions(tmpCtx->evaluator);
@@ -209,7 +210,6 @@ bool CompletionInstance::performCachedOperaitonIfPossible(
   tmpSF->enableInterfaceHash();
   // Ensure all non-function-body tokens are hashed into the interface hash
   tmpCtx->LangOpts.EnableTypeFingerprints = false;
-  parseIntoSourceFile(*tmpSF, tmpBufferID);
 
   // Couldn't find any completion token?
   auto *newState = tmpSF->getDelayedParserState();
@@ -324,8 +324,8 @@ bool CompletionInstance::performCachedOperaitonIfPossible(
     // Tell the compiler instance we've replaced the code completion file.
     CI.setCodeCompletionFile(newSF);
 
-    // Re-parse the whole file. Still re-use imported modules.
-    parseIntoSourceFile(*newSF, newBufferID);
+    // Re-process the whole file (parsing will be lazily triggered). Still
+    // re-use imported modules.
     performNameBinding(*newSF);
     bindExtensions(*newSF);