[ASTGen] Add experimental feature to use ASTGen in lieu of parsing types

Introduce a new experimental feature `ASTGenTypes` that uses ASTGen to
translate the Swift syntax tree (produced by the new Swift parser)
into C++ `TypeRepr` nodes instead of having the C++ parser create the
nodes.

The approach here is to intercept the C++ parser's `parseType`
operation to find the Swift syntax node at the given position (where
the lexer currently is) and have ASTGen translate that into the
corresponding C++ AST node. Then, we spin the lexer forward to the
token immediately following the end of the syntax node and continue
parsing.

Author: DougGregor

include/swift/Basic/Features.def

@@ -174,6 +174,10 @@ EXPERIMENTAL_FEATURE(ImplicitSome, false)
 /// corresponding syntax tree.
 EXPERIMENTAL_FEATURE(ParserASTGen, false)
 
+/// Use the syntax tree produced by the Swift (swift-syntax) parser for type
+/// parsing, using ASTGen to translate them into AST nodes.
+EXPERIMENTAL_FEATURE(ASTGenTypes, false)
+
 /// Parse using the Swift (swift-syntax) parser and use ASTGen to generate the
 /// corresponding syntax tree.
 EXPERIMENTAL_FEATURE(BuiltinMacros, false)

include/swift/Parse/Parser.h

@@ -24,6 +24,7 @@
 #include "swift/AST/LayoutConstraint.h"
 #include "swift/AST/ParseRequests.h"
 #include "swift/AST/Pattern.h"
+#include "swift/AST/SourceFile.h"
 #include "swift/AST/Stmt.h"
 #include "swift/Basic/OptionSet.h"
 #include "swift/Config.h"
@@ -1330,6 +1331,49 @@ class Parser {
   /// Get the location for a type error.
   SourceLoc getTypeErrorLoc() const;
 
+  /// Callback function used for creating a C++ AST from the syntax node at the given source location.
+  ///
+  /// The arguments to this callback are the source file to pass into ASTGen (the exported source file)
+  /// and the source location pointer to pass into ASTGen (to find the syntax node).
+  ///
+  /// The callback returns the new AST node and the ending location of the syntax node. If the AST node
+  /// is NULL, something went wrong.
+  template<typename T>
+  using ASTFromSyntaxTreeCallback = std::pair<T*, const void *>(
+      void *sourceFile, const void *sourceLoc
+  );
+
+  /// Parse by constructing a C++ AST node from the Swift syntax tree via ASTGen.
+  template<typename T>
+  ParserResult<T> parseASTFromSyntaxTree(
+      llvm::function_ref<ASTFromSyntaxTreeCallback<T>> body
+  ) {
+    if (!Context.LangOpts.hasFeature(Feature::ASTGenTypes))
+      return nullptr;
+
+    auto exportedSourceFile = SF.exportedSourceFile;
+    if (!exportedSourceFile)
+      return nullptr;
+
+    // Perform the translation.
+    auto sourceLoc = Tok.getLoc().getOpaquePointerValue();
+    T* astNode;
+    const void *endLocPtr;
+    std::tie(astNode, endLocPtr) = body(exportedSourceFile, sourceLoc);
+
+    if (!astNode) {
+      assert(false && "Could not build AST node from syntax tree");
+      return nullptr;
+    }
+
+    // Spin the lexer until we get to the ending location.
+    while (Tok.getLoc().getOpaquePointerValue() < endLocPtr &&
+           !Tok.is(tok::eof))
+      consumeToken();
+
+    return makeParserResult(astNode);
+  }
+
   //===--------------------------------------------------------------------===//
   // Type Parsing
 

lib/AST/ASTPrinter.cpp

@@ -3321,6 +3321,10 @@ static bool usesFeatureParserASTGen(Decl *decl) {
   return false;
 }
 
+static bool usesFeatureASTGenTypes(Decl *decl) {
+  return false;
+}
+
 static bool usesFeatureBuiltinMacros(Decl *decl) {
   return false;
 }

lib/ASTGen/Sources/ASTGen/Macros.swift

@@ -586,10 +586,11 @@ func expandFreestandingMacroInProcess(
 }
 
 /// Retrieve a syntax node in the given source file, with the given type.
-private func findSyntaxNodeInSourceFile<Node: SyntaxProtocol>(
+func findSyntaxNodeInSourceFile<Node: SyntaxProtocol>(
   sourceFilePtr: UnsafeRawPointer,
   sourceLocationPtr: UnsafePointer<UInt8>?,
-  type: Node.Type
+  type: Node.Type,
+  wantOutermost: Bool = false
 ) -> Node? {
   guard let sourceLocationPtr = sourceLocationPtr else {
     return nil
@@ -615,16 +616,35 @@ private func findSyntaxNodeInSourceFile<Node: SyntaxProtocol>(
   }
 
   var currentSyntax = Syntax(token)
+  var resultSyntax: Node? = nil
   while let parentSyntax = currentSyntax.parent {
     if let typedParent = parentSyntax.as(type) {
-      return typedParent
+      resultSyntax = typedParent
+      break
     }
 
     currentSyntax = parentSyntax
   }
 
-  print("unable to find node: \(token.debugDescription)")
-  return nil
+  // If we didn't find anything, complain and fail.
+  guard var resultSyntax else {
+    print("unable to find node: \(token.debugDescription)")
+    return nil
+  }
+
+  // If we want the outermost node, keep looking.
+  if wantOutermost {
+    while let parentSyntax = resultSyntax.parent {
+      guard let typedParent = parentSyntax.as(type),
+            typedParent.position == resultSyntax.position else {
+        break;
+      }
+
+      resultSyntax = typedParent
+    }
+  }
+
+  return resultSyntax
 }
 
 @_cdecl("swift_ASTGen_expandAttachedMacro")

lib/ASTGen/Sources/ASTGen/Types.swift

@@ -207,3 +207,37 @@ extension ASTGenVisitor {
     }
   }
 }
+
+@_cdecl("swift_ASTGen_buildTypeRepr")
+@usableFromInline
+func buildTypeRepr(
+  sourceFilePtr: UnsafeRawPointer,
+  typeLocPtr: UnsafePointer<UInt8>,
+  dc: UnsafeMutableRawPointer,
+  ctx: UnsafeMutableRawPointer,
+  endTypeLocPtr: UnsafeMutablePointer<UnsafePointer<UInt8>?>
+) -> UnsafeMutableRawPointer? {
+  let sourceFile = sourceFilePtr.bindMemory(
+    to: ExportedSourceFile.self, capacity: 1
+  )
+
+  // Find the type syntax node.
+  guard let typeSyntax = findSyntaxNodeInSourceFile(
+    sourceFilePtr: sourceFilePtr,
+    sourceLocationPtr: typeLocPtr,
+    type: TypeSyntax.self,
+    wantOutermost: true
+  ) else {
+    // FIXME: Produce an error
+    return nil
+  }
+
+  // Fill in the end location.
+  endTypeLocPtr.pointee = sourceFile.pointee.buffer.baseAddress!.advanced(by: typeSyntax.endPosition.utf8Offset)
+
+  // Convert the type syntax node.
+  let typeReprNode = ASTGenVisitor(ctx: ctx, base: sourceFile.pointee.buffer.baseAddress!, declContext: dc)
+    .visit(typeSyntax)
+
+  return typeReprNode.rawValue
+}

lib/Parse/ParseType.cpp

@@ -572,6 +572,14 @@ ParserResult<TypeRepr> Parser::parseTypeScalar(
                            constLoc));
 }
 
+/// Build a TypeRepr for AST node for the type at the given source location in the specified file.
+///
+/// \param sourceLoc The source location at which to start processing a type.
+/// \param endSourceLoc Will receive the source location immediately following the type.
+extern "C" TypeRepr *swift_ASTGen_buildTypeRepr(
+    void *sourceFile, const void *_Nullable sourceLoc,
+    void *declContext, void *astContext, const void *_Nullable *endSourceLoc);
+
 /// parseType
 ///   type:
 ///     type-scalar
@@ -582,6 +590,19 @@ ParserResult<TypeRepr> Parser::parseTypeScalar(
 ///
 ParserResult<TypeRepr> Parser::parseType(
     Diag<> MessageID, ParseTypeReason reason) {
+  #if SWIFT_SWIFT_PARSER
+  auto astGenResult = parseASTFromSyntaxTree<TypeRepr>(
+      [&](void *exportedSourceFile, const void *sourceLoc) {
+        const void *endLocPtr = nullptr;
+        TypeRepr *typeRepr = swift_ASTGen_buildTypeRepr(
+            exportedSourceFile, Tok.getLoc().getOpaquePointerValue(),
+            CurDeclContext, &Context, &endLocPtr);
+        return std::make_pair(typeRepr, endLocPtr);
+      });
+  if (astGenResult.isNonNull())
+    return astGenResult;
+  #endif
+
   // Parse pack expansion 'repeat T'
   if (Tok.is(tok::kw_repeat)) {
     SourceLoc repeatLoc = consumeToken(tok::kw_repeat);

test/ASTGen/verify-parse.swift

@@ -1,4 +1,5 @@
 // RUN: %target-run-simple-swift(-enable-experimental-feature SwiftParser -enable-experimental-feature ParserASTGen)
+// RUN: %target-run-simple-swift(-enable-experimental-feature ASTGenTypes)
 
 // REQUIRES: executable_test
 
@@ -25,11 +26,11 @@ func test3(y: Int) -> Int {
   return x
 }
 
-func test4(_ b: Bool) -> Int {
-  if b { 0 } else { 1 }
+func test4(_ b: [Bool]) -> Int {
+  if b.isEmpty { 0 } else { 1 }
 }
 
-func test5(_ b: Bool) -> Int {
+func test5(_ b: Swift.Bool) -> Int {
   return if b { 0 } else { 1 }
 }