Parse Swift decls in one shot

Instead of interleaving typechecking and parsing
for SIL files, first parse the file for Swift
decls by skipping over any intermixed SIL decls.
Then we can perform type checking, and finally SIL
parsing where we now skip over Swift decls.

This is an intermediate step to requestifying the
parsing of a source file for its Swift decls.

Author: hamishknight

include/swift/AST/DiagnosticsParse.def

@@ -470,6 +470,8 @@ ERROR(expected_precedencegroup_relation,none,
       (StringRef))
 
 // SIL
+ERROR(expected_sil_keyword,none,
+      "expected SIL keyword", ())
 ERROR(inout_not_attribute, none,
       "@inout is no longer an attribute", ())
 ERROR(only_allowed_in_sil,none,

include/swift/Parse/Parser.h

@@ -413,7 +413,8 @@ class Parser {
          bool DelayBodyParsing = true);
   ~Parser();
 
-  bool isInSILMode() const { return SIL != nullptr; }
+  /// Returns true if the buffer being parsed is allowed to contain SIL.
+  bool isInSILMode() const;
 
   /// Calling this function to finalize libSyntax tree creation without destroying
   /// the parser instance.
@@ -661,6 +662,9 @@ class Parser {
   /// Returns \c true if the parser hit the eof before finding matched '}'.
   bool skipBracedBlock();
 
+  /// Skip over SIL decls until we encounter the start of a Swift decl or eof.
+  void skipSILUntilSwiftDecl();
+
   /// If the parser is generating only a syntax tree, try loading the current
   /// node from a previously generated syntax tree.
   /// Returns \c true if the node has been loaded and inserted into the current
@@ -880,11 +884,18 @@ class Parser {
   //===--------------------------------------------------------------------===//
   // Decl Parsing
 
-  /// Return true if parser is at the start of a Swift decl or decl-import.
+  /// Returns true if parser is at the start of a Swift decl or decl-import.
   bool isStartOfSwiftDecl();
 
+  /// Returns true if the parser is at the start of a SIL decl.
+  bool isStartOfSILDecl();
+
+  /// Parse the top-level Swift decls into the source file.
   void parseTopLevel();
 
+  /// Parse the top-level SIL decls into the SIL module.
+  void parseTopLevelSIL();
+
   /// Flags that control the parsing of declarations.
   enum ParseDeclFlags {
     PD_Default              = 0,

include/swift/Subsystems.h

@@ -106,29 +106,21 @@ namespace swift {
 
   /// Parse a single buffer into the given source file.
   ///
-  /// If the source file is the main file, stop parsing after the next
-  /// stmt-brace-item with side-effects.
+  /// \param SF The file within the module being parsed.
   ///
-  /// \param SF the file within the module being parsed.
+  /// \param BufferID The buffer to parse from.
   ///
-  /// \param BufferID the buffer to parse from.
+  /// \param PersistentState If non-null the same PersistentState object can be
+  /// used to save parser state for code completion.
   ///
-  /// \param[out] Done set to \c true if end of the buffer was reached.
-  ///
-  /// \param SIL if non-null, we're parsing a SIL file.
-  ///
-  /// \param PersistentState if non-null the same PersistentState object can
-  /// be used to resume parsing or parse delayed function bodies.
-  void parseIntoSourceFile(SourceFile &SF, unsigned BufferID, bool *Done,
-                           SILParserState *SIL = nullptr,
+  /// \param DelayBodyParsing Whether parsing of type and function bodies can be
+  /// delayed.
+  void parseIntoSourceFile(SourceFile &SF, unsigned BufferID,
                            PersistentParserState *PersistentState = nullptr,
                            bool DelayBodyParsing = true);
 
-  /// Parse a single buffer into the given source file, until the full source
-  /// contents are parsed.
-  void parseIntoSourceFileFull(SourceFile &SF, unsigned BufferID,
-                               PersistentParserState *PersistentState = nullptr,
-                               bool DelayBodyParsing = true);
+  /// Parse a source file's SIL declarations into a given SIL module.
+  void parseSourceFileSIL(SourceFile &SF, SILParserState *sil);
 
   /// Finish the code completion.
   void performCodeCompletionSecondPass(PersistentParserState &PersistentState,

lib/Frontend/Frontend.cpp

@@ -913,14 +913,8 @@ void CompilerInstance::parseLibraryFile(
   auto DidSuppressWarnings = Diags.getSuppressWarnings();
   Diags.setSuppressWarnings(DidSuppressWarnings || !IsPrimary);
 
-  bool Done;
-  do {
-    // Parser may stop at some erroneous constructions like #else, #endif
-    // or '}' in some cases, continue parsing until we are done
-    parseIntoSourceFile(*NextInput, BufferID, &Done, nullptr,
-                        PersistentState.get(),
-                        /*DelayedBodyParsing=*/!IsPrimary);
-  } while (!Done);
+  parseIntoSourceFile(*NextInput, BufferID, PersistentState.get(),
+                      /*DelayedBodyParsing=*/!IsPrimary);
 
   Diags.setSuppressWarnings(DidSuppressWarnings);
 
@@ -953,6 +947,7 @@ bool CompilerInstance::parsePartialModulesAndLibraryFiles(
 
 void CompilerInstance::parseAndTypeCheckMainFileUpTo(
     SourceFile::ASTStage_t LimitStage) {
+  assert(LimitStage >= SourceFile::NameBound);
   FrontendStatsTracer tracer(Context->Stats,
                              "parse-and-typecheck-main-file");
   bool mainIsPrimary =
@@ -965,45 +960,23 @@ void CompilerInstance::parseAndTypeCheckMainFileUpTo(
   auto DidSuppressWarnings = Diags.getSuppressWarnings();
   Diags.setSuppressWarnings(DidSuppressWarnings || !mainIsPrimary);
 
-  SILParserState SILContext(TheSILModule.get());
-  unsigned CurTUElem = 0;
-  bool Done;
-  do {
-    // Pump the parser multiple times if necessary.  It will return early
-    // after parsing any top level code in a main module, or in SIL mode when
-    // there are chunks of swift decls (e.g. imports and types) interspersed
-    // with 'sil' definitions.
-    parseIntoSourceFile(MainFile, MainFile.getBufferID().getValue(), &Done,
-                        TheSILModule ? &SILContext : nullptr,
-                        PersistentState.get(),
-                        !mainIsPrimary);
-
-    // For SIL we actually have to interleave parsing and type checking
-    // because the SIL parser expects to see fully type checked declarations.
-    if (TheSILModule) {
-      if (Done || CurTUElem < MainFile.getTopLevelDecls().size()) {
-        assert(mainIsPrimary);
-        performTypeChecking(MainFile, CurTUElem);
-      }
-    }
+  // Parse the Swift decls into the source file.
+  parseIntoSourceFile(MainFile, MainBufferID, PersistentState.get(),
+                      /*delayBodyParsing*/ !mainIsPrimary);
 
-    CurTUElem = MainFile.getTopLevelDecls().size();
-  } while (!Done);
-
-  if (!TheSILModule) {
-    if (mainIsPrimary) {
-      switch (LimitStage) {
-      case SourceFile::Parsing:
-      case SourceFile::Parsed:
-        llvm_unreachable("invalid limit stage");
-      case SourceFile::NameBound:
-        performNameBinding(MainFile);
-        break;
-      case SourceFile::TypeChecked:
-        performTypeChecking(MainFile);
-        break;
-      }
-    }
+  // For a primary, also perform type checking if needed. Otherwise, just do
+  // name binding.
+  if (mainIsPrimary && LimitStage >= SourceFile::TypeChecked) {
+    performTypeChecking(MainFile);
+  } else {
+    assert(!TheSILModule && "Should perform type checking for SIL");
+    performNameBinding(MainFile);
+  }
+
+  // Parse the SIL decls if needed.
+  if (TheSILModule) {
+    SILParserState SILContext(TheSILModule.get());
+    parseSourceFileSIL(MainFile, &SILContext);
   }
 
   Diags.setSuppressWarnings(DidSuppressWarnings);
@@ -1012,12 +985,6 @@ void CompilerInstance::parseAndTypeCheckMainFileUpTo(
       Invocation.getFrontendOptions().DebuggerTestingTransform) {
     performDebuggerTestingTransform(MainFile);
   }
-
-  if (!TheSILModule) {
-    if (!mainIsPrimary) {
-      performNameBinding(MainFile);
-    }
-  }
 }
 
 static void
@@ -1109,8 +1076,8 @@ void CompilerInstance::performParseOnly(bool EvaluateConditionals,
         SourceFileKind::Library, SourceFile::ImplicitModuleImportKind::None,
         BufferID);
 
-    parseIntoSourceFileFull(*NextInput, BufferID, PersistentState.get(),
-                            shouldDelayBodies(BufferID));
+    parseIntoSourceFile(*NextInput, BufferID, PersistentState.get(),
+                        shouldDelayBodies(BufferID));
   }
 
   // Now parse the main file.
@@ -1120,8 +1087,8 @@ void CompilerInstance::performParseOnly(bool EvaluateConditionals,
     MainFile.SyntaxParsingCache = Invocation.getMainFileSyntaxParsingCache();
     assert(MainBufferID == MainFile.getBufferID());
 
-    parseIntoSourceFileFull(MainFile, MainBufferID, PersistentState.get(),
-                            shouldDelayBodies(MainBufferID));
+    parseIntoSourceFile(MainFile, MainBufferID, PersistentState.get(),
+                        shouldDelayBodies(MainBufferID));
   }
 
   assert(Context->LoadedModules.size() == 1 &&

lib/IDE/CompletionInstance.cpp

@@ -217,7 +217,7 @@ bool CompletionInstance::performCachedOperaitonIfPossible(
   newSF->enableInterfaceHash();
   // Ensure all non-function-body tokens are hashed into the interface hash
   Ctx->LangOpts.EnableTypeFingerprints = false;
-  parseIntoSourceFileFull(*newSF, tmpBufferID, &newState);
+  parseIntoSourceFile(*newSF, tmpBufferID, &newState);
   // Couldn't find any completion token?
   if (!newState.hasCodeCompletionDelayedDeclState())
     return false;

lib/IDE/REPLCodeCompletion.cpp

@@ -208,10 +208,7 @@ doCodeCompletion(SourceFile &SF, StringRef EnteredCode, unsigned *BufferID,
   const unsigned OriginalDeclCount = SF.getTopLevelDecls().size();
 
   PersistentParserState PersistentState;
-  bool Done;
-  do {
-    parseIntoSourceFile(SF, *BufferID, &Done, nullptr, &PersistentState);
-  } while (!Done);
+  parseIntoSourceFile(SF, *BufferID, &PersistentState);
   performTypeChecking(SF, OriginalDeclCount);
 
   performCodeCompletionSecondPass(PersistentState, *CompletionCallbacksFactory);

lib/Immediate/REPL.cpp

@@ -187,11 +187,7 @@ typeCheckREPLInput(ModuleDecl *MostRecentModule, StringRef Name,
     REPLInputFile.addImports(ImportsWithOptions);
   }
 
-  bool Done;
-  do {
-    parseIntoSourceFile(REPLInputFile, BufferID, &Done, nullptr,
-                        &PersistentState);
-  } while (!Done);
+  parseIntoSourceFile(REPLInputFile, BufferID, &PersistentState);
   performTypeChecking(REPLInputFile);
   return REPLModule;
 }