Merge pull request swiftlang#29619 from hamishknight/a-buffet-of-tokens

Parse Swift decls in one shot

Author: hamishknight

include/swift/AST/DiagnosticsParse.def

@@ -470,6 +470,8 @@ ERROR(expected_precedencegroup_relation,none,
       (StringRef))
 
 // SIL
+ERROR(expected_sil_keyword,none,
+      "expected SIL keyword", ())
 ERROR(inout_not_attribute, none,
       "@inout is no longer an attribute", ())
 ERROR(only_allowed_in_sil,none,

include/swift/Parse/Parser.h

@@ -413,7 +413,8 @@ class Parser {
          bool DelayBodyParsing = true);
   ~Parser();
 
-  bool isInSILMode() const { return SIL != nullptr; }
+  /// Returns true if the buffer being parsed is allowed to contain SIL.
+  bool isInSILMode() const;
 
   /// Calling this function to finalize libSyntax tree creation without destroying
   /// the parser instance.
@@ -430,9 +431,8 @@ class Parser {
                           PreviousLoc);
   }
 
-  ParserPosition getParserPosition(const PersistentParserState::ParserPos &Pos){
-    return ParserPosition(L->getStateForBeginningOfTokenLoc(Pos.Loc),
-                          Pos.PrevLoc);
+  ParserPosition getParserPosition(SourceLoc loc, SourceLoc previousLoc) {
+    return ParserPosition(L->getStateForBeginningOfTokenLoc(loc), previousLoc);
   }
 
   void restoreParserPosition(ParserPosition PP, bool enableDiagnostics = false) {
@@ -661,6 +661,9 @@ class Parser {
   /// Returns \c true if the parser hit the eof before finding matched '}'.
   bool skipBracedBlock();
 
+  /// Skip over SIL decls until we encounter the start of a Swift decl or eof.
+  void skipSILUntilSwiftDecl();
+
   /// If the parser is generating only a syntax tree, try loading the current
   /// node from a previously generated syntax tree.
   /// Returns \c true if the node has been loaded and inserted into the current
@@ -880,11 +883,18 @@ class Parser {
   //===--------------------------------------------------------------------===//
   // Decl Parsing
 
-  /// Return true if parser is at the start of a decl or decl-import.
-  bool isStartOfDecl();
+  /// Returns true if parser is at the start of a Swift decl or decl-import.
+  bool isStartOfSwiftDecl();
+
+  /// Returns true if the parser is at the start of a SIL decl.
+  bool isStartOfSILDecl();
 
+  /// Parse the top-level Swift decls into the source file.
   void parseTopLevel();
 
+  /// Parse the top-level SIL decls into the SIL module.
+  void parseTopLevelSIL();
+
   /// Flags that control the parsing of declarations.
   enum ParseDeclFlags {
     PD_Default              = 0,

include/swift/Parse/PersistentParserState.h

@@ -59,23 +59,12 @@ class CodeCompletionDelayedDeclState {
 /// Parser state persistent across multiple parses.
 class PersistentParserState {
 public:
-  struct ParserPos {
-    SourceLoc Loc;
-    SourceLoc PrevLoc;
-
-    bool isValid() const { return Loc.isValid(); }
-  };
-
-  bool InPoundLineEnvironment = false;
   // FIXME: When condition evaluation moves to a later phase, remove this bit
   // and adjust the client call 'performParseOnly'.
   bool PerformConditionEvaluation = true;
 private:
   swift::ScopeInfo ScopeInfo;
 
-  /// Parser sets this if it stopped parsing before the buffer ended.
-  ParserPosition MarkedPos;
-
   std::unique_ptr<CodeCompletionDelayedDeclState> CodeCompletionDelayedDeclStat;
 
   /// The local context for all top-level code.
@@ -113,19 +102,6 @@ class PersistentParserState {
   TopLevelContext &getTopLevelContext() {
     return TopLevelCode;
   }
-
-  void markParserPosition(ParserPosition Pos,
-                          bool InPoundLineEnvironment) {
-    MarkedPos = Pos;
-    this->InPoundLineEnvironment = InPoundLineEnvironment;
-  }
-
-  /// Returns the marked parser position and resets it.
-  ParserPosition takeParserPosition() {
-    ParserPosition Pos = MarkedPos;
-    MarkedPos = ParserPosition();
-    return Pos;
-  }
 };
 
 } // end namespace swift

include/swift/Subsystems.h

@@ -106,29 +106,21 @@ namespace swift {
 
   /// Parse a single buffer into the given source file.
   ///
-  /// If the source file is the main file, stop parsing after the next
-  /// stmt-brace-item with side-effects.
+  /// \param SF The file within the module being parsed.
   ///
-  /// \param SF the file within the module being parsed.
+  /// \param BufferID The buffer to parse from.
   ///
-  /// \param BufferID the buffer to parse from.
+  /// \param PersistentState If non-null the same PersistentState object can be
+  /// used to save parser state for code completion.
   ///
-  /// \param[out] Done set to \c true if end of the buffer was reached.
-  ///
-  /// \param SIL if non-null, we're parsing a SIL file.
-  ///
-  /// \param PersistentState if non-null the same PersistentState object can
-  /// be used to resume parsing or parse delayed function bodies.
-  void parseIntoSourceFile(SourceFile &SF, unsigned BufferID, bool *Done,
-                           SILParserState *SIL = nullptr,
+  /// \param DelayBodyParsing Whether parsing of type and function bodies can be
+  /// delayed.
+  void parseIntoSourceFile(SourceFile &SF, unsigned BufferID,
                            PersistentParserState *PersistentState = nullptr,
                            bool DelayBodyParsing = true);
 
-  /// Parse a single buffer into the given source file, until the full source
-  /// contents are parsed.
-  void parseIntoSourceFileFull(SourceFile &SF, unsigned BufferID,
-                               PersistentParserState *PersistentState = nullptr,
-                               bool DelayBodyParsing = true);
+  /// Parse a source file's SIL declarations into a given SIL module.
+  void parseSourceFileSIL(SourceFile &SF, SILParserState *sil);
 
   /// Finish the code completion.
   void performCodeCompletionSecondPass(PersistentParserState &PersistentState,

lib/Frontend/Frontend.cpp

@@ -913,14 +913,8 @@ void CompilerInstance::parseLibraryFile(
   auto DidSuppressWarnings = Diags.getSuppressWarnings();
   Diags.setSuppressWarnings(DidSuppressWarnings || !IsPrimary);
 
-  bool Done;
-  do {
-    // Parser may stop at some erroneous constructions like #else, #endif
-    // or '}' in some cases, continue parsing until we are done
-    parseIntoSourceFile(*NextInput, BufferID, &Done, nullptr,
-                        PersistentState.get(),
-                        /*DelayedBodyParsing=*/!IsPrimary);
-  } while (!Done);
+  parseIntoSourceFile(*NextInput, BufferID, PersistentState.get(),
+                      /*DelayedBodyParsing=*/!IsPrimary);
 
   Diags.setSuppressWarnings(DidSuppressWarnings);
 
@@ -953,6 +947,7 @@ bool CompilerInstance::parsePartialModulesAndLibraryFiles(
 
 void CompilerInstance::parseAndTypeCheckMainFileUpTo(
     SourceFile::ASTStage_t LimitStage) {
+  assert(LimitStage >= SourceFile::NameBound);
   FrontendStatsTracer tracer(Context->Stats,
                              "parse-and-typecheck-main-file");
   bool mainIsPrimary =
@@ -965,45 +960,23 @@ void CompilerInstance::parseAndTypeCheckMainFileUpTo(
   auto DidSuppressWarnings = Diags.getSuppressWarnings();
   Diags.setSuppressWarnings(DidSuppressWarnings || !mainIsPrimary);
 
-  SILParserState SILContext(TheSILModule.get());
-  unsigned CurTUElem = 0;
-  bool Done;
-  do {
-    // Pump the parser multiple times if necessary.  It will return early
-    // after parsing any top level code in a main module, or in SIL mode when
-    // there are chunks of swift decls (e.g. imports and types) interspersed
-    // with 'sil' definitions.
-    parseIntoSourceFile(MainFile, MainFile.getBufferID().getValue(), &Done,
-                        TheSILModule ? &SILContext : nullptr,
-                        PersistentState.get(),
-                        !mainIsPrimary);
-
-    // For SIL we actually have to interleave parsing and type checking
-    // because the SIL parser expects to see fully type checked declarations.
-    if (TheSILModule) {
-      if (Done || CurTUElem < MainFile.getTopLevelDecls().size()) {
-        assert(mainIsPrimary);
-        performTypeChecking(MainFile, CurTUElem);
-      }
-    }
+  // Parse the Swift decls into the source file.
+  parseIntoSourceFile(MainFile, MainBufferID, PersistentState.get(),
+                      /*delayBodyParsing*/ !mainIsPrimary);
 
-    CurTUElem = MainFile.getTopLevelDecls().size();
-  } while (!Done);
-
-  if (!TheSILModule) {
-    if (mainIsPrimary) {
-      switch (LimitStage) {
-      case SourceFile::Parsing:
-      case SourceFile::Parsed:
-        llvm_unreachable("invalid limit stage");
-      case SourceFile::NameBound:
-        performNameBinding(MainFile);
-        break;
-      case SourceFile::TypeChecked:
-        performTypeChecking(MainFile);
-        break;
-      }
-    }
+  // For a primary, also perform type checking if needed. Otherwise, just do
+  // name binding.
+  if (mainIsPrimary && LimitStage >= SourceFile::TypeChecked) {
+    performTypeChecking(MainFile);
+  } else {
+    assert(!TheSILModule && "Should perform type checking for SIL");
+    performNameBinding(MainFile);
+  }
+
+  // Parse the SIL decls if needed.
+  if (TheSILModule) {
+    SILParserState SILContext(TheSILModule.get());
+    parseSourceFileSIL(MainFile, &SILContext);
   }
 
   Diags.setSuppressWarnings(DidSuppressWarnings);
@@ -1012,12 +985,6 @@ void CompilerInstance::parseAndTypeCheckMainFileUpTo(
       Invocation.getFrontendOptions().DebuggerTestingTransform) {
     performDebuggerTestingTransform(MainFile);
   }
-
-  if (!TheSILModule) {
-    if (!mainIsPrimary) {
-      performNameBinding(MainFile);
-    }
-  }
 }
 
 static void
@@ -1109,8 +1076,8 @@ void CompilerInstance::performParseOnly(bool EvaluateConditionals,
         SourceFileKind::Library, SourceFile::ImplicitModuleImportKind::None,
         BufferID);
 
-    parseIntoSourceFileFull(*NextInput, BufferID, PersistentState.get(),
-                            shouldDelayBodies(BufferID));
+    parseIntoSourceFile(*NextInput, BufferID, PersistentState.get(),
+                        shouldDelayBodies(BufferID));
   }
 
   // Now parse the main file.
@@ -1120,8 +1087,8 @@ void CompilerInstance::performParseOnly(bool EvaluateConditionals,
     MainFile.SyntaxParsingCache = Invocation.getMainFileSyntaxParsingCache();
     assert(MainBufferID == MainFile.getBufferID());
 
-    parseIntoSourceFileFull(MainFile, MainBufferID, PersistentState.get(),
-                            shouldDelayBodies(MainBufferID));
+    parseIntoSourceFile(MainFile, MainBufferID, PersistentState.get(),
+                        shouldDelayBodies(MainBufferID));
   }
 
   assert(Context->LoadedModules.size() == 1 &&

lib/IDE/CompletionInstance.cpp

@@ -217,7 +217,7 @@ bool CompletionInstance::performCachedOperaitonIfPossible(
   newSF->enableInterfaceHash();
   // Ensure all non-function-body tokens are hashed into the interface hash
   Ctx->LangOpts.EnableTypeFingerprints = false;
-  parseIntoSourceFileFull(*newSF, tmpBufferID, &newState);
+  parseIntoSourceFile(*newSF, tmpBufferID, &newState);
   // Couldn't find any completion token?
   if (!newState.hasCodeCompletionDelayedDeclState())
     return false;

lib/IDE/REPLCodeCompletion.cpp

@@ -208,10 +208,7 @@ doCodeCompletion(SourceFile &SF, StringRef EnteredCode, unsigned *BufferID,
   const unsigned OriginalDeclCount = SF.getTopLevelDecls().size();
 
   PersistentParserState PersistentState;
-  bool Done;
-  do {
-    parseIntoSourceFile(SF, *BufferID, &Done, nullptr, &PersistentState);
-  } while (!Done);
+  parseIntoSourceFile(SF, *BufferID, &PersistentState);
   performTypeChecking(SF, OriginalDeclCount);
 
   performCodeCompletionSecondPass(PersistentState, *CompletionCallbacksFactory);

lib/Immediate/REPL.cpp

@@ -187,11 +187,7 @@ typeCheckREPLInput(ModuleDecl *MostRecentModule, StringRef Name,
     REPLInputFile.addImports(ImportsWithOptions);
   }
 
-  bool Done;
-  do {
-    parseIntoSourceFile(REPLInputFile, BufferID, &Done, nullptr,
-                        &PersistentState);
-  } while (!Done);
+  parseIntoSourceFile(REPLInputFile, BufferID, &PersistentState);
   performTypeChecking(REPLInputFile);
   return REPLModule;
 }