[libSyntax] Handle deferred node data in SyntaxParseActions

By now ParsedRawSyntaxNode does not have any knowledge about deferred
node data anymore, which frees up SyntaxParseActions (and, in
particular its sublass SyntaxTreeCreator) to perform optimisations to
more efficiently create and record deferred nodes.

Author: ahoppen

include/swift/Parse/ParsedRawSyntaxNode.h

@@ -16,6 +16,7 @@
 #include "swift/Basic/Debug.h"
 #include "swift/Basic/SourceLoc.h"
 #include "swift/Parse/ParsedTrivia.h"
+#include "swift/Parse/SyntaxParseActions.h"
 #include "swift/Parse/Token.h"
 #include "swift/Syntax/SyntaxKind.h"
 #include "llvm/Support/Debug.h"
@@ -24,17 +25,6 @@ namespace swift {
 
 typedef const void *OpaqueSyntaxNode;
 class SyntaxParsingContext;
-class ParsedRawSyntaxNode;
-
-struct DeferredLayoutNode {
-  MutableArrayRef<ParsedRawSyntaxNode> Children;
-};
-struct DeferredTokenNode {
-  SourceLoc TokLoc;
-  unsigned TokLength;
-  StringRef LeadingTrivia;
-  StringRef TrailingTrivia;
-};
 
 /// Represents a raw syntax node formed by the parser.
 ///
@@ -52,12 +42,7 @@ struct DeferredTokenNode {
 /// in the current parsing context.
 class ParsedRawSyntaxNode {
   friend class ParsedRawSyntaxRecorder;
-  enum class DataKind: uint8_t {
-    Null,
-    Recorded,
-    DeferredLayout,
-    DeferredToken,
-  };
+  using DataKind = RecordedOrDeferredNode::Kind;
 
   OpaqueSyntaxNode Data;
   /// The range of this node, including trivia.
@@ -83,28 +68,6 @@ class ParsedRawSyntaxNode {
         TokKind(uint16_t(TokKind)), DK(DK), IsMissing(IsMissing) {
     assert(getKind() == SynKind && "Syntax kind with too large value!");
     assert(getTokenKind() == TokKind && "Token kind with too large value!");
-    assert(DK == DataKind::Recorded);
-  }
-
-  ParsedRawSyntaxNode(const DeferredLayoutNode *Layout, CharSourceRange Range,
-                      syntax::SyntaxKind SynKind, tok TokKind, DataKind DK,
-                      bool IsMissing)
-      : Data(Layout), Range(Range), SynKind(uint16_t(SynKind)),
-        TokKind(uint16_t(TokKind)), DK(DK), IsMissing(IsMissing) {
-    assert(getKind() == SynKind && "Syntax kind with too large value!");
-    assert(getTokenKind() == TokKind && "Token kind with too large value!");
-    assert(DK == DataKind::DeferredLayout);
-  }
-
-  ParsedRawSyntaxNode(const DeferredTokenNode *Token, CharSourceRange Range,
-                      syntax::SyntaxKind SynKind, tok TokKind, DataKind DK,
-                      bool IsMissing)
-      : Data(Token), Range(Range), SynKind(uint16_t(SynKind)),
-        TokKind(uint16_t(TokKind)), DK(DK), IsMissing(IsMissing) {
-    assert(getKind() == SynKind && "Syntax kind with too large value!");
-    assert(getTokenKind() == TokKind && "Token kind with too large value!");
-    assert(getKind() == syntax::SyntaxKind::Token);
-    assert(DK == DataKind::DeferredToken);
   }
 
 #ifndef NDEBUG
@@ -137,6 +100,22 @@ class ParsedRawSyntaxNode {
     assert(ensureDataIsNotRecorded() && "recorded data is being destructed");
   }
 
+  /// Returns the type of this node (recorded, deferred layout, deferred token,
+  /// null).
+  DataKind getDataKind() const { return DK; }
+
+  /// Returns the opaque data of this node. This must be interpreted by the
+  /// \c SyntaxParseAction, which likely also needs the node type to know
+  /// what type of node the data represents.
+  OpaqueSyntaxNode getData() const { return Data; }
+
+  /// Return the opaque data of this node and reset it.
+  OpaqueSyntaxNode takeData() {
+    OpaqueSyntaxNode Data = this->getData();
+    reset();
+    return Data;
+  }
+
   syntax::SyntaxKind getKind() const { return syntax::SyntaxKind(SynKind); }
   tok getTokenKind() const { return tok(TokKind); }
 
@@ -195,15 +174,14 @@ class ParsedRawSyntaxNode {
 
   // Deferred Layout Data ====================================================//
 
-  ArrayRef<ParsedRawSyntaxNode> getDeferredChildren() const {
-    assert(DK == DataKind::DeferredLayout);
-    return static_cast<const DeferredLayoutNode *>(Data)->Children;
-  }
-
-  MutableArrayRef<ParsedRawSyntaxNode> getDeferredChildren() {
-    assert(DK == DataKind::DeferredLayout);
-    return static_cast<const DeferredLayoutNode *>(Data)->Children;
-  }
+  /// If this node is a deferred layout node, return the child at index \p
+  /// ChildIndex.
+  /// Note that this may be an expensive operation since the \c
+  /// SyntaxParseAction, which created the node (implicitly passed via the
+  /// \p SyntaxContext) needs to be consulted to retrieve the child.
+  ParsedRawSyntaxNode
+  getDeferredChild(size_t ChildIndex,
+                   const SyntaxParsingContext *SyntaxContext) const;
 
   ParsedRawSyntaxNode copyDeferred() const {
     assert(DK == DataKind::DeferredLayout ||
@@ -218,24 +196,6 @@ class ParsedRawSyntaxNode {
     return copy;
   }
 
-  // Deferred Token Data =====================================================//
-
-  CharSourceRange getDeferredTokenRange() const {
-    assert(DK == DataKind::DeferredToken);
-    auto DeferredToken = static_cast<const DeferredTokenNode *>(Data);
-    return CharSourceRange{DeferredToken->TokLoc, DeferredToken->TokLength};
-  }
-  StringRef getDeferredLeadingTrivia() const {
-    assert(DK == DataKind::DeferredToken);
-    auto DeferredToken = static_cast<const DeferredTokenNode *>(Data);
-    return DeferredToken->LeadingTrivia;
-  }
-  StringRef getDeferredTrailingTrivia() const {
-    assert(DK == DataKind::DeferredToken);
-    auto DeferredToken = static_cast<const DeferredTokenNode *>(Data);
-    return DeferredToken->TrailingTrivia;
-  }
-
   //==========================================================================//
 
   /// Dump this piece of syntax recursively for debugging or testing.

include/swift/Parse/ParsedRawSyntaxRecorder.h

@@ -41,6 +41,10 @@ namespace syntax {
 class ParsedRawSyntaxRecorder final {
   std::shared_ptr<SyntaxParseActions> SPActions;
 
+  /// Assuming that \p node is a deferred layout or token node, record it and
+  /// return the recorded node.
+  ParsedRawSyntaxNode recordDeferredNode(const ParsedRawSyntaxNode &node);
+
 public:
   explicit ParsedRawSyntaxRecorder(std::shared_ptr<SyntaxParseActions> spActions)
     : SPActions(std::move(spActions)) {}
@@ -69,6 +73,10 @@ class ParsedRawSyntaxRecorder final {
                                                      SourceLoc loc);
 
   /// Form a deferred syntax layout node.
+  /// All nodes in \p deferred nodes must be deferred. Otherwise, we'd have a
+  /// deferred layout node with recorded child nodes. Should we decide to
+  /// discard the deferred layout node, we would also need to discard its
+  /// recorded children, which cannot be done.
   ParsedRawSyntaxNode
   makeDeferred(syntax::SyntaxKind k,
                MutableArrayRef<ParsedRawSyntaxNode> deferredNodes,
@@ -85,9 +93,14 @@ class ParsedRawSyntaxRecorder final {
   ParsedRawSyntaxNode lookupNode(size_t lexerOffset, SourceLoc loc,
                                  syntax::SyntaxKind kind);
 
-  #ifndef NDEBUG
+  /// For a deferred layout node \p parent, retrieve the deferred child node
+  /// at \p ChildIndex.
+  ParsedRawSyntaxNode getDeferredChild(const ParsedRawSyntaxNode &parent,
+                                       size_t ChildIndex) const;
+
+#ifndef NDEBUG
   static void verifyElementRanges(ArrayRef<ParsedRawSyntaxNode> elements);
-  #endif
+#endif
 };
 
 } // end namespace swift

include/swift/Parse/ParsedSyntaxNodes.h.gyb

@@ -62,9 +62,13 @@ public:
   /// ${line}
 %       end
 %       if child.is_optional:
-  llvm::Optional<Parsed${child.type_name}> getDeferred${child.name}();
+  llvm::Optional<Parsed${child.type_name}> getDeferred${child.name}(
+      const SyntaxParsingContext *SyntaxContext
+  );
 %       else:
-  Parsed${child.type_name} getDeferred${child.name}();
+  Parsed${child.type_name} getDeferred${child.name}(
+      const SyntaxParsingContext *SyntaxContext
+  );
 %       end
 %     end
 

include/swift/Parse/SyntaxParseActions.h

@@ -19,10 +19,11 @@
 #define SWIFT_PARSE_SYNTAXPARSEACTIONS_H
 
 #include "swift/Basic/LLVM.h"
+#include "swift/Basic/SourceLoc.h"
+#include "llvm/Support/Allocator.h"
 
 namespace swift {
 
-class CharSourceRange;
 class ParsedTriviaPiece;
 class SourceFile;
 class SourceLoc;
@@ -35,7 +36,54 @@ enum class SyntaxKind;
 
 typedef const void *OpaqueSyntaxNode;
 
+// MARK: - Helper types
+
+/// A syntax node that can either be deferred or recorded. The actual data is
+/// opaque and needs to be interpreted by the \c SyntaxParseAction which created
+/// it.
+class RecordedOrDeferredNode {
+public:
+  enum class Kind : uint8_t {
+    Null,
+    Recorded,
+    DeferredLayout,
+    DeferredToken,
+  };
+
+private:
+  OpaqueSyntaxNode Opaque;
+  Kind NodeKind;
+
+public:
+  RecordedOrDeferredNode(OpaqueSyntaxNode Node, Kind NodeKind)
+      : Opaque(Node), NodeKind(NodeKind) {}
+
+  OpaqueSyntaxNode getOpaque() const { return Opaque; }
+  Kind getKind() const { return NodeKind; }
+};
+
+/// Data returned from \c getDeferredChild. This is enough data to construct
+/// a \c ParsedRawSyntaxNode. We don't return \c ParsedRawSyntaxNodes from
+/// \c getDeferredChild to maintain a clean dependency relationship of
+/// \c ParsedRawSyntaxNode being on a higher level than \c SyntaxParseActions.
+struct DeferredNodeInfo {
+  OpaqueSyntaxNode Data;
+  syntax::SyntaxKind SyntaxKind;
+  tok TokenKind;
+  bool IsMissing;
+  CharSourceRange Range;
+
+  DeferredNodeInfo(OpaqueSyntaxNode Data, syntax::SyntaxKind SyntaxKind,
+                   tok TokenKind, bool IsMissing, CharSourceRange Range)
+      : Data(Data), SyntaxKind(SyntaxKind), TokenKind(TokenKind),
+        IsMissing(IsMissing), Range(Range) {}
+};
+
+// MARK: - SyntaxParseActions
+
 class SyntaxParseActions {
+  llvm::BumpPtrAllocator DeferredNodeAllocator;
+
   virtual void _anchor();
 
 public:
@@ -52,9 +100,47 @@ class SyntaxParseActions {
   /// The provided \c elements are an exact layout appropriate for the syntax
   /// \c kind. Missing optional elements are represented with a null
   /// OpaqueSyntaxNode object.
-  virtual OpaqueSyntaxNode recordRawSyntax(syntax::SyntaxKind kind,
-                                           ArrayRef<OpaqueSyntaxNode> elements,
-                                           CharSourceRange range) = 0;
+  virtual OpaqueSyntaxNode
+  recordRawSyntax(syntax::SyntaxKind kind,
+                  ArrayRef<OpaqueSyntaxNode> elements) = 0;
+
+  /// Create a deferred token node that may or may not be recorded later using
+  /// \c recordDeferredToken. The \c SyntaxParseAction is responsible for
+  /// keeping the deferred token alive until the action is destructed.
+  virtual OpaqueSyntaxNode makeDeferredToken(tok tokenKind,
+                                             StringRef leadingTrivia,
+                                             StringRef trailingTrivia,
+                                             CharSourceRange range,
+                                             bool isMissing);
+
+  /// Create a deferred layout node that may or may not be recorded later using
+  /// \c recordDeferredLayout. The \c SyntaxParseAction is responsible for
+  /// keeping the deferred token alive until it is destructed.
+  virtual OpaqueSyntaxNode
+  makeDeferredLayout(syntax::SyntaxKind k, bool isMissing,
+                     const ArrayRef<RecordedOrDeferredNode> &children);
+
+  /// Record a deferred token node that was previously created using \c
+  /// makeDeferredToken. The deferred data will never be used again, so it can
+  /// be destroyed by this method. Note that not all deferred nodes will be
+  /// recorded and that pending deferred nodes need to be freed when the \c
+  /// SyntaxParseActions is destructed.
+  virtual OpaqueSyntaxNode recordDeferredToken(OpaqueSyntaxNode deferred);
+
+  /// Record a deferred layout node. See recordDeferredToken.
+  virtual OpaqueSyntaxNode recordDeferredLayout(OpaqueSyntaxNode deferred);
+
+  /// Since most data of \c ParsedRawSyntax is described as opaque data, the
+  /// \c ParsedRawSyntax node needs to reach out to the \c SyntaxParseAction,
+  /// which created it, to retrieve children.
+  /// This method assumes that \p node represents a *deferred* layout node.
+  /// This methods returns all information needed to construct a \c
+  /// ParsedRawSyntaxNode of a child node. \p node is the parent node for which
+  /// the child at position \p ChildIndex should be retrieved. Furthmore, \p
+  /// node starts at \p StartLoc.
+  virtual DeferredNodeInfo getDeferredChild(OpaqueSyntaxNode node,
+                                            size_t childIndex,
+                                            SourceLoc startLoc);
 
   /// Attempt to realize an opaque raw syntax node for a source file into a
   /// SourceFileSyntax node. This will return \c None if the parsing action

include/swift/Parse/SyntaxParsingContext.h

@@ -255,6 +255,9 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
   const SyntaxParsingContext *getRoot() const;
 
   ParsedRawSyntaxRecorder &getRecorder() { return getRootData()->Recorder; }
+  const ParsedRawSyntaxRecorder &getRecorder() const {
+    return getRootData()->Recorder;
+  }
 
   llvm::BumpPtrAllocator &getScratchAlloc() {
     return getRootData()->ScratchAlloc;

include/swift/SyntaxParse/SyntaxTreeCreator.h

@@ -64,9 +64,9 @@ class SyntaxTreeCreator: public SyntaxParseActions {
 
   OpaqueSyntaxNode recordMissingToken(tok tokenKind, SourceLoc loc) override;
 
-  OpaqueSyntaxNode recordRawSyntax(syntax::SyntaxKind kind,
-                                   ArrayRef<OpaqueSyntaxNode> elements,
-                                   CharSourceRange range) override;
+  OpaqueSyntaxNode
+  recordRawSyntax(syntax::SyntaxKind kind,
+                  ArrayRef<OpaqueSyntaxNode> elements) override;
 
   std::pair<size_t, OpaqueSyntaxNode>
   lookupNode(size_t lexerOffset, syntax::SyntaxKind kind) override;

lib/Parse/CMakeLists.txt

@@ -22,6 +22,7 @@ add_swift_host_library(swiftParse STATIC
   ParseStmt.cpp
   ParseType.cpp
   PersistentParserState.cpp
+  SyntaxParseActions.cpp
   SyntaxParsingCache.cpp
   SyntaxParsingContext.cpp)
 _swift_gyb_target_sources(swiftParse PRIVATE

lib/Parse/ParsePattern.cpp

@@ -1263,7 +1263,7 @@ ParserResult<Pattern> Parser::parseMatchingPattern(bool isExprBasic) {
 
   if (SyntaxContext->isEnabled()) {
     if (auto UPES = PatternCtx.popIf<ParsedUnresolvedPatternExprSyntax>()) {
-      PatternCtx.addSyntax(UPES->getDeferredPattern());
+      PatternCtx.addSyntax(UPES->getDeferredPattern(SyntaxContext));
     } else {
       PatternCtx.setCreateSyntax(SyntaxKind::ExpressionPattern);
     }

lib/Parse/ParseType.cpp

@@ -406,9 +406,9 @@ ParserResult<TypeRepr> Parser::parseType(Diag<> MessageID,
       if (InputNode.is<ParsedTupleTypeSyntax>()) {
         auto TupleTypeNode = std::move(InputNode).castTo<ParsedTupleTypeSyntax>();
         // Decompose TupleTypeSyntax and repack into FunctionType.
-        auto LeftParen = TupleTypeNode.getDeferredLeftParen();
-        auto Arguments = TupleTypeNode.getDeferredElements();
-        auto RightParen = TupleTypeNode.getDeferredRightParen();
+        auto LeftParen = TupleTypeNode.getDeferredLeftParen(SyntaxContext);
+        auto Arguments = TupleTypeNode.getDeferredElements(SyntaxContext);
+        auto RightParen = TupleTypeNode.getDeferredRightParen(SyntaxContext);
         Builder
           .useLeftParen(std::move(LeftParen))
           .useArguments(std::move(Arguments))

lib/Parse/ParsedRawSyntaxNode.cpp

@@ -17,6 +17,12 @@ using namespace swift;
 using namespace swift::syntax;
 using namespace llvm;
 
+ParsedRawSyntaxNode ParsedRawSyntaxNode::getDeferredChild(
+    size_t ChildIndex, const SyntaxParsingContext *SyntaxContext) const {
+  assert(isDeferredLayout());
+  return SyntaxContext->getRecorder().getDeferredChild(*this, ChildIndex);
+}
+
 void ParsedRawSyntaxNode::dump() const {
   dump(llvm::errs(), /*Indent*/ 0);
   llvm::errs() << '\n';
@@ -42,11 +48,7 @@ void ParsedRawSyntaxNode::dump(llvm::raw_ostream &OS, unsigned Indent) const {
       break;
     case DataKind::DeferredLayout:
       dumpSyntaxKind(OS, getKind());
-      OS << " [deferred]";
-      for (const auto &child : getDeferredChildren()) {
-        OS << "\n";
-        child.dump(OS, Indent + 2);
-      }
+      OS << " [deferred layout]";
       break;
     case DataKind::DeferredToken:
       dumpSyntaxKind(OS, getKind());