[ASTGen] Support macro expanded buffer

* Make ExportedSourceFile hold any Syntax as the root node
* Move `ExportedSourceFileRequest::evaluate()` to `ParseRequests.cpp`
* Pass  the decl context and `GeneatedSourceFileInfo::Kind` to
  `swift_ASTGen_parseSourceFile()` to customize the parsing
* Make `ExportedSourceFile` to hold an arbitrary Syntax node
* Move round-trip checking into `ExportedSourceFileRequest::evaluate()`
* Split `parseSourceFileViaASTGen` completely from C++ parsing logic
  (in `ParseSourceFileRequest::evaluate()`)
* Remove 'ParserDiagnostics' experimental feature: Now that we have
  ParserASTGen mode which includes the swift-syntax parser diagnostics.

Author: rintaro

include/swift/AST/ASTBridging.h

@@ -504,7 +504,19 @@ void BridgedDiagnostic_finish(BridgedDiagnostic cDiag);
 //===----------------------------------------------------------------------===//
 
 SWIFT_NAME("getter:BridgedDeclContext.isLocalContext(self:)")
-bool BridgedDeclContext_isLocalContext(BridgedDeclContext cDeclContext);
+BRIDGED_INLINE bool
+BridgedDeclContext_isLocalContext(BridgedDeclContext cDeclContext);
+
+SWIFT_NAME("getter:BridgedDeclContext.isTypeContext(self:)")
+BRIDGED_INLINE bool BridgedDeclContext_isTypeContext(BridgedDeclContext dc);
+
+SWIFT_NAME("getter:BridgedDeclContext.isModuleScopeContext(self:)")
+BRIDGED_INLINE bool
+BridgedDeclContext_isModuleScopeContext(BridgedDeclContext dc);
+
+SWIFT_NAME("getter:BridgedDeclContext.astContext(self:)")
+BRIDGED_INLINE BridgedASTContext
+BridgedDeclContext_getASTContext(BridgedDeclContext dc);
 
 SWIFT_NAME("BridgedPatternBindingInitializer.create(declContext:)")
 BridgedPatternBindingInitializer
@@ -1365,6 +1377,12 @@ BridgedPackExpansionExpr_createParsed(BridgedASTContext cContext,
                                       BridgedSourceLoc cRepeatLoc,
                                       BridgedExpr cPatternExpr);
 
+SWIFT_NAME("BridgedParenExpr.createParsed(_:leftParenLoc:expr:rightParenLoc:)")
+BridgedParenExpr BridgedParenExpr_createParsed(BridgedASTContext cContext,
+                                               BridgedSourceLoc cLParen,
+                                               BridgedExpr cExpr,
+                                               BridgedSourceLoc cRParen);
+
 SWIFT_NAME("BridgedPostfixUnaryExpr.createParsed(_:operator:operand:)")
 BridgedPostfixUnaryExpr
 BridgedPostfixUnaryExpr_createParsed(BridgedASTContext cContext,

include/swift/AST/ASTBridgingImpl.h

@@ -18,9 +18,10 @@
 #include "swift/AST/Decl.h"
 #include "swift/AST/Expr.h"
 #include "swift/AST/IfConfigClauseRangeInfo.h"
-#include "swift/AST/Stmt.h"
 #include "swift/AST/ProtocolConformance.h"
 #include "swift/AST/ProtocolConformanceRef.h"
+#include "swift/AST/SourceFile.h"
+#include "swift/AST/Stmt.h"
 #include "swift/Basic/Assertions.h"
 
 SWIFT_BEGIN_NULLABILITY_ANNOTATIONS
@@ -97,6 +98,26 @@ BridgedStringRef BridgedASTContext_allocateCopyString(BridgedASTContext bridged,
   return bridged.unbridged().AllocateCopy(cStr.unbridged());
 }
 
+//===----------------------------------------------------------------------===//
+// MARK: BridgedDeclContext
+//===----------------------------------------------------------------------===//
+
+bool BridgedDeclContext_isLocalContext(BridgedDeclContext dc) {
+  return dc.unbridged()->isLocalContext();
+}
+
+bool BridgedDeclContext_isTypeContext(BridgedDeclContext dc) {
+  return dc.unbridged()->isTypeContext();
+}
+
+bool BridgedDeclContext_isModuleScopeContext(BridgedDeclContext dc) {
+  return dc.unbridged()->isModuleScopeContext();
+}
+
+BridgedASTContext BridgedDeclContext_getASTContext(BridgedDeclContext dc) {
+  return dc.unbridged()->getASTContext();
+}
+
 //===----------------------------------------------------------------------===//
 // MARK: BridgedDeclObj
 //===----------------------------------------------------------------------===//

include/swift/AST/SourceFile.h

@@ -595,6 +595,8 @@ class SourceFile final : public FileUnit {
     return BufferID;
   }
 
+  const GeneratedSourceInfo *getGeneratedSourceFileInfo() const;
+
   /// For source files created to hold the source code created by expanding
   /// a macro, this is the AST node that describes the macro expansion.
   ///
@@ -641,6 +643,9 @@ class SourceFile final : public FileUnit {
   /// Otherwise, return an empty string.
   StringRef getFilename() const;
 
+  /// Retrieve the source text buffer.
+  StringRef getBuffer() const;
+
   /// Retrieve the scope that describes this source file.
   ASTScope &getScope();
 

include/swift/Basic/BasicBridging.h

@@ -432,6 +432,23 @@ class BridgedSwiftVersion {
   unsigned getMinor() const { return Minor; }
 };
 
+//===----------------------------------------------------------------------===//
+// MARK: GeneratedSourceInfo
+//===----------------------------------------------------------------------===//
+
+enum ENUM_EXTENSIBILITY_ATTR(closed) BridgedGeneratedSourceFileKind {
+#define MACRO_ROLE(Name, Description)                                          \
+  BridgedGeneratedSourceFileKind##Name##MacroExpansion,
+#include "swift/Basic/MacroRoles.def"
+#undef MACRO_ROLE
+
+  BridgedGeneratedSourceFileKindReplacedFunctionBody,
+  BridgedGeneratedSourceFileKindPrettyPrinted,
+  BridgedGeneratedSourceFileKindDefaultArgument,
+
+  BridgedGeneratedSourceFileKindNone,
+};
+
 SWIFT_END_NULLABILITY_ANNOTATIONS
 
 #ifndef PURE_BRIDGING_MODE

include/swift/Basic/Features.def

@@ -299,11 +299,6 @@ EXPERIMENTAL_FEATURE(ParserRoundTrip, false)
 /// Swift parser.
 EXPERIMENTAL_FEATURE(ParserValidation, false)
 
-/// Whether to emit diagnostics from the new parser first, and only emit
-/// diagnostics from the existing parser when there are none from the new
-/// parser.
-EXPERIMENTAL_FEATURE(ParserDiagnostics, false)
-
 /// Enables implicit some while also enabling existential `any`
 EXPERIMENTAL_FEATURE(ImplicitSome, false)
 

include/swift/Bridging/ASTGen.h

@@ -37,11 +37,11 @@ void swift_ASTGen_renderQueuedDiagnostics(
 
 // FIXME: Hack because we cannot easily get to the already-parsed source
 // file from here. Fix this egregious oversight!
-void *_Nullable swift_ASTGen_parseSourceFile(const char *_Nonnull buffer,
-                                             size_t bufferLength,
-                                             const char *_Nonnull moduleName,
-                                             const char *_Nonnull filename,
-                                             void *_Nullable ctx);
+void *_Nullable swift_ASTGen_parseSourceFile(BridgedStringRef buffer,
+                                             BridgedStringRef moduleName,
+                                             BridgedStringRef filename,
+                                             void *_Nullable declContextPtr,
+                                             BridgedGeneratedSourceFileKind);
 void swift_ASTGen_destroySourceFile(void *_Nonnull sourceFile);
 
 /// Check whether the given source file round-trips correctly. Returns 0 if
@@ -60,7 +60,7 @@ void swift_ASTGen_buildTopLevelASTNodes(
     BridgedDiagnosticEngine diagEngine, void *_Nonnull sourceFile,
     BridgedDeclContext declContext, BridgedASTContext astContext,
     BridgedLegacyParser legacyParser, void *_Nonnull outputContext,
-    void (*_Nonnull)(void *_Nonnull, void *_Nonnull));
+    void (*_Nonnull)(BridgedASTNode, void *_Nonnull));
 
 void swift_ASTGen_freeBridgedString(BridgedStringRef);
 

include/swift/Parse/Parser.h

@@ -930,12 +930,6 @@ class Parser {
   /// Each item will be a declaration, statement, or expression.
   void parseTopLevelItems(SmallVectorImpl<ASTNode> &items);
 
-  /// Parse the source file via the Swift Parser using the ASTGen library.
-  void
-  parseSourceFileViaASTGen(SmallVectorImpl<ASTNode> &items,
-                           std::optional<DiagnosticTransaction> &transaction,
-                           bool suppressDiagnostics = false);
-
   /// Parse the top-level SIL decls into the SIL module.
   /// \returns \c true if there was a parsing error.
   bool parseTopLevelSIL();

lib/AST/Bridging/DeclContextBridging.cpp

@@ -20,10 +20,6 @@ using namespace swift;
 // MARK: DeclContexts
 //===----------------------------------------------------------------------===//
 
-bool BridgedDeclContext_isLocalContext(BridgedDeclContext cDeclContext) {
-  return cDeclContext.unbridged()->isLocalContext();
-}
-
 BridgedPatternBindingInitializer
 BridgedPatternBindingInitializer_create(BridgedDeclContext cDeclContext) {
   return PatternBindingInitializer::create(cDeclContext.unbridged());

lib/AST/Bridging/ExprBridging.cpp

@@ -383,6 +383,14 @@ BridgedRegexLiteralExpr_createParsed(BridgedASTContext cContext,
                                         cRegexText.unbridged());
 }
 
+BridgedParenExpr BridgedParenExpr_createParsed(BridgedASTContext cContext,
+                                               BridgedSourceLoc cLParen,
+                                               BridgedExpr cExpr,
+                                               BridgedSourceLoc cRParen) {
+  ASTContext &context = cContext.unbridged();
+  return new (context)
+      ParenExpr(cLParen.unbridged(), cExpr.unbridged(), cRParen.unbridged());
+}
 BridgedSequenceExpr BridgedSequenceExpr_createParsed(BridgedASTContext cContext,
                                                      BridgedArrayRef exprs) {
   return SequenceExpr::create(cContext.unbridged(), exprs.unbridged<Expr *>());

lib/AST/DiagnosticBridge.cpp

@@ -114,8 +114,8 @@ void *DiagnosticBridge::getSourceFileSyntax(SourceManager &sourceMgr,
 
   auto bufferContents = sourceMgr.getEntireTextForBuffer(bufferID);
   auto sourceFile = swift_ASTGen_parseSourceFile(
-      bufferContents.data(), bufferContents.size(), "module",
-      displayName.str().c_str(), /*ctx*/ nullptr);
+      bufferContents, StringRef{"module"}, displayName,
+      /*declContextPtr=*/nullptr, BridgedGeneratedSourceFileKindNone);
 
   sourceFileSyntax[{&sourceMgr, bufferID}] = sourceFile;
   return sourceFile;