Merge pull request #61660 from DougGregor/evaluate-builtin-macros

[Macros] "Expand" builtin macros for magic literals.

Author: DougGregor

include/swift/AST/SourceFile.h

@@ -316,6 +316,10 @@ class SourceFile final : public FileUnit {
   /// this source file.
   llvm::SmallVector<Located<StringRef>, 0> VirtualFilePaths;
 
+  /// The \c ExportedSourceFile instance produced by ASTGen, which includes
+  /// the SourceFileSyntax node corresponding to this source file.
+  void *exportedSourceFile = nullptr;
+
   /// Returns information about the file paths used for diagnostics and magic
   /// identifiers in this source file, including virtual filenames introduced by
   /// \c #sourceLocation(file:) declarations.

include/swift/Basic/Features.def

@@ -145,8 +145,13 @@ EXPERIMENTAL_FEATURE(ImplicitSome)
 /// corresponding syntax tree.
 EXPERIMENTAL_FEATURE(ParserASTGen)
 
+/// Enable the experimental macros feature.
 EXPERIMENTAL_FEATURE(Macros)
 
+/// Parse using the Swift (swift-syntax) parser and use ASTGen to generate the
+/// corresponding syntax tree.
+EXPERIMENTAL_FEATURE(BuiltinMacros)
+
 #undef EXPERIMENTAL_FEATURE
 #undef UPCOMING_FEATURE
 #undef SUPPRESSIBLE_LANGUAGE_FEATURE

lib/AST/ASTPrinter.cpp

@@ -3098,6 +3098,10 @@ static bool usesFeatureParserASTGen(Decl *decl) {
   return false;
 }
 
+static bool usesFeatureBuiltinMacros(Decl *decl) {
+  return false;
+}
+
 static void
 suppressingFeatureNoAsyncAvailability(PrintOptions &options,
                                       llvm::function_ref<void()> action) {

lib/ASTGen/CMakeLists.txt

@@ -4,7 +4,9 @@ if (SWIFT_SWIFT_PARSER)
     Sources/ASTGen/Decls.swift
     Sources/ASTGen/Exprs.swift
     Sources/ASTGen/Literals.swift
+    Sources/ASTGen/Macros.swift
     Sources/ASTGen/Misc.swift
+    Sources/ASTGen/SourceFile.swift
     Sources/ASTGen/Stmts.swift
   )
 
@@ -39,6 +41,8 @@ if (SWIFT_SWIFT_PARSER)
     $<TARGET_OBJECTS:SwiftSyntax::SwiftDiagnostics>
     $<TARGET_OBJECTS:SwiftSyntax::SwiftSyntax>
     $<TARGET_OBJECTS:SwiftSyntax::SwiftOperators>
+    $<TARGET_OBJECTS:SwiftSyntax::SwiftSyntaxBuilder>
+    $<TARGET_OBJECTS:SwiftSyntax::_SwiftSyntaxMacros>
     $<TARGET_OBJECTS:SwiftSyntax::SwiftCompilerSupport>
 
     swiftAST
@@ -53,6 +57,8 @@ if (SWIFT_SWIFT_PARSER)
     SwiftSyntax::SwiftDiagnostics
     SwiftSyntax::SwiftSyntax
     SwiftSyntax::SwiftOperators
+    SwiftSyntax::SwiftSyntaxBuilder
+    SwiftSyntax::_SwiftSyntaxMacros
     SwiftSyntax::SwiftCompilerSupport
     swiftAST
   )

lib/ASTGen/Package.swift

@@ -25,7 +25,8 @@ let package = Package(
             name: "swiftASTGen",
             dependencies: [
               .product(name: "SwiftSyntax", package: "swift-syntax"),
-              .product(name: "SwiftParser", package: "swift-syntax")
+              .product(name: "SwiftParser", package: "swift-syntax"),
+              .product(name: "_SwiftSyntaxMacros", package: "swift-syntax")
             ],
             path: ".",
             exclude: ["CMakeLists.txt"],

lib/ASTGen/Sources/ASTGen/ASTGen.swift

@@ -17,7 +17,8 @@ extension UnsafePointer {
   }
 }
 
-/// Little utility wrapper that lets us
+/// Little utility wrapper that lets us have some mutable state within
+/// immutable structs, and is therefore pretty evil.
 @propertyWrapper
 class Boxed<Value> {
   var wrappedValue: Value
@@ -29,7 +30,7 @@ class Boxed<Value> {
 
 struct ASTGenVisitor: SyntaxTransformVisitor {
   let ctx: UnsafeMutableRawPointer
-  let base: UnsafePointer<CChar>
+  let base: UnsafePointer<UInt8>
 
   @Boxed var declContext: UnsafeMutableRawPointer
 
@@ -68,15 +69,18 @@ struct ASTGenVisitor: SyntaxTransformVisitor {
   }
 }
 
-@_cdecl("parseTopLevelSwift")
-public func parseTopLevelSwift(
-    buffer: UnsafePointer<CChar>, dc: UnsafeMutableRawPointer,
-    ctx: UnsafeMutableRawPointer,
-    outputContext: UnsafeMutableRawPointer,
-    callback: @convention(c) (UnsafeMutableRawPointer, UnsafeMutableRawPointer) -> Void
+/// Generate AST nodes for all top-level entities in the given source file.
+@_cdecl("swift_ASTGen_buildTopLevelASTNodes")
+public func buildTopLevelASTNodes(
+  sourceFilePtr: UnsafePointer<UInt8>,
+  dc: UnsafeMutableRawPointer,
+  ctx: UnsafeMutableRawPointer,
+  outputContext: UnsafeMutableRawPointer,
+  callback: @convention(c) (UnsafeMutableRawPointer, UnsafeMutableRawPointer) -> Void
 ) {
-  let syntax = try! Parser.parse(source: String(cString: buffer))
-  ASTGenVisitor(ctx: ctx, base: buffer, declContext: dc)
-    .visit(syntax)
-    .forEach { callback($0, outputContext) }
+  sourceFilePtr.withMemoryRebound(to: ExportedSourceFile.self, capacity: 1) { sourceFile in
+    ASTGenVisitor(ctx: ctx, base: sourceFile.pointee.buffer.baseAddress!, declContext: dc)
+      .visit(sourceFile.pointee.syntax)
+      .forEach { callback($0, outputContext) }
+  }
 }

lib/ASTGen/Sources/ASTGen/Macros.swift

@@ -0,0 +1,91 @@
+import SwiftSyntax
+@_spi(Testing) import _SwiftSyntaxMacros
+
+extension SyntaxProtocol {
+  func token(at position: AbsolutePosition) -> TokenSyntax? {
+    // If the position isn't within this node at all, return early.
+    guard position >= self.position && position < self.endPosition else {
+      return nil
+    }
+
+    // If we are a token syntax, that's it!
+    if let token = Syntax(self).as(TokenSyntax.self) {
+      return token
+    }
+
+    // Otherwise, it must be one of our children.
+    return children(viewMode: .sourceAccurate).lazy.compactMap { child in
+      child.token(at: position)
+    }.first
+  }
+}
+
+@_cdecl("swift_ASTGen_evaluateMacro")
+public func evaluateMacro(
+  sourceFilePtr: UnsafePointer<UInt8>,
+  sourceLocationPtr: UnsafePointer<UInt8>?,
+  expandedSourcePointer: UnsafeMutablePointer<UnsafePointer<UInt8>?>,
+  expandedSourceLength: UnsafeMutablePointer<Int>
+) -> Int {
+  // We didn't expand anything so far.
+  expandedSourcePointer.pointee = nil
+  expandedSourceLength.pointee = 0
+
+  guard let sourceLocationPtr = sourceLocationPtr else {
+    print("NULL source location")
+    return -1
+  }
+
+  return sourceFilePtr.withMemoryRebound(
+    to: ExportedSourceFile.self, capacity: 1
+  ) { (sourceFile: UnsafePointer<ExportedSourceFile>) -> Int in
+    // Find the offset.
+    let buffer = sourceFile.pointee.buffer
+    let offset = sourceLocationPtr - buffer.baseAddress!
+    if offset < 0 || offset >= buffer.count {
+      print("source location isn't inside this buffer")
+      return -1
+    }
+
+    let sf = sourceFile.pointee.syntax
+    guard let token = sf.token(at: AbsolutePosition(utf8Offset: offset)) else {
+      print("couldn't find token at offset \(offset)")
+      return -1
+    }
+
+    guard let parentSyntax = token.parent,
+          parentSyntax.is(MacroExpansionExprSyntax.self) else {
+      print("not on a macro expansion node: \(token.recursiveDescription)")
+      return -1
+    }
+
+    let macroSystem = MacroSystem.exampleSystem
+    let converter = SourceLocationConverter(
+      file: sourceFile.pointee.fileName, tree: sf
+    )
+    let context = MacroEvaluationContext(
+      moduleName: sourceFile.pointee.moduleName,
+      sourceLocationConverter: converter
+    )
+
+    let evaluatedSyntax = parentSyntax.evaluateMacro(
+      with: macroSystem, context: context
+    ) { error in
+      /* TODO: Report errors */
+    }
+
+    var evaluatedSyntaxStr = evaluatedSyntax.withoutTrivia().description
+    evaluatedSyntaxStr.withUTF8 { utf8 in
+      let evaluatedResultPtr = UnsafeMutablePointer<UInt8>.allocate(capacity: utf8.count + 1)
+      if let baseAddress = utf8.baseAddress {
+        evaluatedResultPtr.initialize(from: baseAddress, count: utf8.count)
+      }
+      evaluatedResultPtr[utf8.count] = 0
+
+      expandedSourcePointer.pointee = UnsafePointer(evaluatedResultPtr)
+      expandedSourceLength.pointee = utf8.count + 1
+    }
+
+    return 0
+  }
+}

lib/ASTGen/Sources/ASTGen/SourceFile.swift

@@ -0,0 +1,49 @@
+import SwiftParser
+import SwiftSyntax
+
+/// Describes a source file that has been "exported" to the C++ part of the
+/// compiler, with enough information to interface with the C++ layer.
+struct ExportedSourceFile {
+  /// The underlying buffer within the C++ SourceManager, which is used
+  /// for computations of source locations.
+  let buffer: UnsafeBufferPointer<UInt8>
+
+  /// The name of the enclosing module.
+  let moduleName: String
+
+  /// The name of the source file being parsed.
+  let fileName: String
+
+  /// The syntax tree for the complete source file.
+  let syntax: SourceFileSyntax
+}
+
+/// Parses the given source file and produces a pointer to a new
+/// ExportedSourceFile instance.
+@_cdecl("swift_ASTGen_parseSourceFile")
+public func parseSourceFile(
+  buffer: UnsafePointer<UInt8>, bufferLength: Int,
+  moduleName: UnsafePointer<UInt8>, filename: UnsafePointer<UInt8>
+) -> UnsafeRawPointer {
+  let buffer = UnsafeBufferPointer(start: buffer, count: bufferLength)
+  let sourceFile = Parser.parse(source: buffer)
+
+  let exportedPtr = UnsafeMutablePointer<ExportedSourceFile>.allocate(capacity: 1)
+  exportedPtr.initialize(to: .init(
+    buffer: buffer, moduleName: String(cString: moduleName),
+    fileName: String(cString: filename), syntax: sourceFile)
+  )
+
+  return UnsafeRawPointer(exportedPtr)
+}
+
+/// Deallocate a parsed source file.
+@_cdecl("swift_ASTGen_destroySourceFile")
+public func destroySourceFile(
+  sourceFilePtr: UnsafeMutablePointer<UInt8>
+) {
+  sourceFilePtr.withMemoryRebound(to: ExportedSourceFile.self, capacity: 1) { sourceFile in
+    sourceFile.deinitialize(count: 1)
+    sourceFile.deallocate()
+  }
+}

lib/Parse/CMakeLists.txt

@@ -54,6 +54,8 @@ if (SWIFT_SWIFT_PARSER)
     $<TARGET_OBJECTS:SwiftSyntax::SwiftDiagnostics>
     $<TARGET_OBJECTS:SwiftSyntax::SwiftSyntax>
     $<TARGET_OBJECTS:SwiftSyntax::SwiftOperators>
+    $<TARGET_OBJECTS:SwiftSyntax::SwiftSyntaxBuilder>
+    $<TARGET_OBJECTS:SwiftSyntax::_SwiftSyntaxMacros>
     $<TARGET_OBJECTS:SwiftSyntax::SwiftCompilerSupport>
     $<TARGET_OBJECTS:swiftASTGen>
   )
@@ -64,6 +66,8 @@ if (SWIFT_SWIFT_PARSER)
     SwiftSyntax::SwiftDiagnostics
     SwiftSyntax::SwiftSyntax
     SwiftSyntax::SwiftOperators
+    SwiftSyntax::SwiftSyntaxBuilder
+    SwiftSyntax::_SwiftSyntaxMacros
     SwiftSyntax::SwiftCompilerSupport
     swiftASTGen
   )

lib/Parse/ParseDecl.cpp

@@ -173,6 +173,22 @@ static void appendToVector(void *declPtr, void *vecPtr) {
   vec->push_back(decl);
 }
 
+/// Parse a source file.
+extern "C" void *swift_ASTGen_parseSourceFile(const char *buffer,
+                                              size_t bufferLength,
+                                              const char *moduleName,
+                                              const char *filename);
+
+/// Destroy a source file parsed with swift_ASTGen_parseSourceFile.
+extern "C" void swift_ASTGen_destroySourceFile(void *sourceFile);
+
+// Build AST nodes for the top-level entities in the syntax.
+extern "C" void swift_ASTGen_buildTopLevelASTNodes(void *sourceFile,
+                                                   void *declContext,
+                                                   void *astContext,
+                                                   void *outputContext,
+                                                   void (*)(void *, void *));
+
 /// Main entrypoint for the parser.
 ///
 /// \verbatim
@@ -182,20 +198,36 @@ static void appendToVector(void *declPtr, void *vecPtr) {
 ///     decl-sil-stage [[only in SIL mode]
 /// \endverbatim
 void Parser::parseTopLevel(SmallVectorImpl<Decl *> &decls) {
-  StringRef contents =
-      SourceMgr.extractText(SourceMgr.getRangeForBuffer(L->getBufferID()));
-
 #ifdef SWIFT_SWIFT_PARSER
-  if (Context.LangOpts.hasFeature(Feature::ParserASTGen) &&
+  if ((Context.LangOpts.hasFeature(Feature::BuiltinMacros) ||
+       Context.LangOpts.hasFeature(Feature::ParserASTGen)) &&
       !SourceMgr.hasCodeCompletionBuffer() &&
       SF.Kind != SourceFileKind::SIL) {
-    parseTopLevelSwift(contents.data(), CurDeclContext, &Context, &decls, appendToVector);
+    StringRef contents =
+        SourceMgr.extractText(SourceMgr.getRangeForBuffer(L->getBufferID()));
+
+    // Parse the source file.
+    auto exportedSourceFile = swift_ASTGen_parseSourceFile(
+        contents.begin(), contents.size(),
+        SF.getParentModule()->getName().str().str().c_str(),
+        SF.getFilename().str().c_str());
+    SF.exportedSourceFile = exportedSourceFile;
+    Context.addCleanup([exportedSourceFile] {
+      swift_ASTGen_destroySourceFile(exportedSourceFile);
+    });
 
-    // Spin the C++ parser to the end; we won't be using it.
-    while (!Tok.is(tok::eof)) {
-      consumeToken();
+    // If we want to do ASTGen, do so now.
+    if (Context.LangOpts.hasFeature(Feature::ParserASTGen)) {
+      swift_ASTGen_buildTopLevelASTNodes(
+          exportedSourceFile, CurDeclContext, &Context, &decls, appendToVector);
+
+      // Spin the C++ parser to the end; we won't be using it.
+      while (!Tok.is(tok::eof)) {
+        consumeToken();
+      }
+
+      return;
     }
-    return;
   }
 #endif