[Lexer] Push trivia piece lexing down to SyntaxParseActions

The SyntaxParseActions can decide how to handle the raw trivia, either
lex them into pieces or store them raw to be lexed when needed.

Author: ahoppen

include/swift/Parse/SyntaxParseActions.h

@@ -41,10 +41,9 @@ class SyntaxParseActions {
 public:
   virtual ~SyntaxParseActions() = default;
 
-  virtual OpaqueSyntaxNode recordToken(tok tokenKind,
-                                    ArrayRef<ParsedTriviaPiece> leadingTrivia,
-                                    ArrayRef<ParsedTriviaPiece> trailingTrivia,
-                                    CharSourceRange range) = 0;
+  virtual OpaqueSyntaxNode recordToken(tok tokenKind, StringRef leadingTrivia,
+                                       StringRef trailingTrivia,
+                                       CharSourceRange range) = 0;
 
   /// Record a missing token. \c loc can be invalid or an approximate location
   /// of where the token would be if not missing.

include/swift/SyntaxParse/SyntaxTreeCreator.h

@@ -56,9 +56,8 @@ class SyntaxTreeCreator: public SyntaxParseActions {
   realizeSyntaxRoot(OpaqueSyntaxNode root, const SourceFile &SF) override;
 
 private:
-  OpaqueSyntaxNode recordToken(tok tokenKind,
-                               ArrayRef<ParsedTriviaPiece> leadingTrivia,
-                               ArrayRef<ParsedTriviaPiece> trailingTrivia,
+  OpaqueSyntaxNode recordToken(tok tokenKind, StringRef leadingTrivia,
+                               StringRef trailingTrivia,
                                CharSourceRange range) override;
 
   OpaqueSyntaxNode recordMissingToken(tok tokenKind, SourceLoc loc) override;

lib/Parse/ParsedRawSyntaxRecorder.cpp

@@ -17,7 +17,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "swift/Parse/ParsedRawSyntaxRecorder.h"
-#include "swift/Parse/Lexer.h"
 #include "swift/Parse/ParsedRawSyntaxNode.h"
 #include "swift/Parse/ParsedTrivia.h"
 #include "swift/Parse/SyntaxParseActions.h"
@@ -42,10 +41,8 @@ ParsedRawSyntaxRecorder::recordToken(tok tokKind, CharSourceRange tokRange,
   unsigned length =
       leadingTrivia.size() + tokRange.getByteLength() + trailingTrivia.size();
   CharSourceRange range(offset, length);
-  auto leadingTriviaPieces = TriviaLexer::lexTrivia(leadingTrivia).Pieces;
-  auto trailingTriviaPieces = TriviaLexer::lexTrivia(trailingTrivia).Pieces;
-  OpaqueSyntaxNode n = SPActions->recordToken(tokKind, leadingTriviaPieces,
-                                              trailingTriviaPieces, range);
+  OpaqueSyntaxNode n =
+      SPActions->recordToken(tokKind, leadingTrivia, trailingTrivia, range);
   return ParsedRawSyntaxNode(SyntaxKind::Token, tokKind, range, n);
 }
 

lib/SyntaxParse/SyntaxTreeCreator.cpp

@@ -14,6 +14,7 @@
 #include "swift/Syntax/RawSyntax.h"
 #include "swift/Syntax/SyntaxVisitor.h"
 #include "swift/Syntax/Trivia.h"
+#include "swift/Parse/Lexer.h"
 #include "swift/Parse/ParsedTrivia.h"
 #include "swift/Parse/SyntaxParsingCache.h"
 #include "swift/Parse/Token.h"
@@ -109,18 +110,16 @@ SyntaxTreeCreator::realizeSyntaxRoot(OpaqueSyntaxNode rootN,
   return rootNode;
 }
 
-OpaqueSyntaxNode
-SyntaxTreeCreator::recordToken(tok tokenKind,
-                               ArrayRef<ParsedTriviaPiece> leadingTriviaPieces,
-                               ArrayRef<ParsedTriviaPiece> trailingTriviaPieces,
-                               CharSourceRange range) {
-  size_t leadingTriviaLen =
-    ParsedTriviaPiece::getTotalLength(leadingTriviaPieces);
-  size_t trailingTriviaLen =
-    ParsedTriviaPiece::getTotalLength(trailingTriviaPieces);
-  SourceLoc tokLoc = range.getStart().getAdvancedLoc(leadingTriviaLen);
-  unsigned tokLength = range.getByteLength() -
-      leadingTriviaLen - trailingTriviaLen;
+OpaqueSyntaxNode SyntaxTreeCreator::recordToken(tok tokenKind,
+                                                StringRef leadingTrivia,
+                                                StringRef trailingTrivia,
+                                                CharSourceRange range) {
+  auto leadingTriviaPieces = TriviaLexer::lexTrivia(leadingTrivia).Pieces;
+  auto trailingTriviaPieces = TriviaLexer::lexTrivia(trailingTrivia).Pieces;
+
+  SourceLoc tokLoc = range.getStart().getAdvancedLoc(leadingTrivia.size());
+  unsigned tokLength =
+      range.getByteLength() - leadingTrivia.size() - trailingTrivia.size();
   CharSourceRange tokRange = CharSourceRange{tokLoc, tokLength};
   SourceLoc leadingTriviaLoc = range.getStart();
   SourceLoc trailingTriviaLoc = tokLoc.getAdvancedLoc(tokLength);

tools/libSwiftSyntaxParser/libSwiftSyntaxParser.cpp

@@ -154,13 +154,15 @@ class CLibParseActions : public SyntaxParseActions {
     node.present = true;
   }
 
-  OpaqueSyntaxNode recordToken(tok tokenKind,
-                               ArrayRef<ParsedTriviaPiece> leadingTrivia,
-                               ArrayRef<ParsedTriviaPiece> trailingTrivia,
+  OpaqueSyntaxNode recordToken(tok tokenKind, StringRef leadingTrivia,
+                               StringRef trailingTrivia,
                                CharSourceRange range) override {
+    auto leadingTriviaPieces = TriviaLexer::lexTrivia(leadingTrivia).Pieces;
+    auto trailingTriviaPieces = TriviaLexer::lexTrivia(trailingTrivia).Pieces;
+
     SmallVector<CTriviaPiece, 8> c_leadingTrivia, c_trailingTrivia;
-    makeCTrivia(c_leadingTrivia, leadingTrivia);
-    makeCTrivia(c_trailingTrivia, trailingTrivia);
+    makeCTrivia(c_leadingTrivia, leadingTriviaPieces);
+    makeCTrivia(c_trailingTrivia, trailingTriviaPieces);
     CRawSyntaxNode node;
     makeCRawToken(node, tokenKind, c_leadingTrivia, c_trailingTrivia,
                   range);