Skip to content

Commit 6d5d8da

Browse files
ahoppenahoppen
committed
[Lexer] Push trivia piece lexing down to ParsedRawSyntaxRecorder
Next and final stop: SyntaxParseActions
1 parent 08ad703 commit 6d5d8da

File tree

5 files changed

+46
-79
lines changed

5 files changed

+46
-79
lines changed

‎include/swift/Parse/ParsedRawSyntaxNode.h

Lines changed: 17 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,10 @@ class ParsedRawSyntaxNode {
5656
CharSourceRange Range;
5757
};
5858
struct DeferredTokenNode {
59-
const ParsedTriviaPiece *TriviaPieces;
6059
SourceLoc TokLoc;
6160
unsigned TokLength;
62-
uint16_t NumLeadingTrivia;
63-
uint16_t NumTrailingTrivia;
61+
StringRef LeadingTrivia;
62+
StringRef TrailingTrivia;
6463
};
6564

6665
union {
@@ -83,21 +82,11 @@ class ParsedRawSyntaxNode {
8382
}
8483

8584
ParsedRawSyntaxNode(tok tokKind, SourceLoc tokLoc, unsigned tokLength,
86-
const ParsedTriviaPiece *triviaPieces,
87-
unsigned numLeadingTrivia,
88-
unsigned numTrailingTrivia)
89-
: DeferredToken{triviaPieces,
90-
tokLoc, tokLength,
91-
uint16_t(numLeadingTrivia),
92-
uint16_t(numTrailingTrivia)},
93-
SynKind(uint16_t(syntax::SyntaxKind::Token)),
94-
TokKind(uint16_t(tokKind)),
95-
DK(DataKind::DeferredToken) {
85+
StringRef leadingTrivia, StringRef trailingTrivia)
86+
: DeferredToken{tokLoc, tokLength, leadingTrivia, trailingTrivia},
87+
SynKind(uint16_t(syntax::SyntaxKind::Token)),
88+
TokKind(uint16_t(tokKind)), DK(DataKind::DeferredToken) {
9689
assert(getTokenKind() == tokKind && "Token kind is too large value!");
97-
assert(DeferredToken.NumLeadingTrivia == numLeadingTrivia &&
98-
"numLeadingTrivia is too large value!");
99-
assert(DeferredToken.NumTrailingTrivia == numTrailingTrivia &&
100-
"numLeadingTrivia is too large value!");
10190
}
10291
ParsedRawSyntaxNode(const ParsedRawSyntaxNode &other) = delete;
10392
ParsedRawSyntaxNode &operator=(const ParsedRawSyntaxNode &other) = delete;
@@ -280,11 +269,8 @@ class ParsedRawSyntaxNode {
280269

281270
CharSourceRange getDeferredTokenRangeWithTrivia() const {
282271
assert(DK == DataKind::DeferredToken);
283-
auto leadTriviaPieces = getDeferredLeadingTriviaPieces();
284-
auto trailTriviaPieces = getDeferredTrailingTriviaPieces();
285-
286-
auto leadTriviaLen = ParsedTriviaPiece::getTotalLength(leadTriviaPieces);
287-
auto trailTriviaLen = ParsedTriviaPiece::getTotalLength(trailTriviaPieces);
272+
auto leadTriviaLen = DeferredToken.LeadingTrivia.size();
273+
auto trailTriviaLen = DeferredToken.TrailingTrivia.size();
288274

289275
SourceLoc begin = DeferredToken.TokLoc.getAdvancedLoc(-leadTriviaLen);
290276
unsigned len = leadTriviaLen + DeferredToken.TokLength + trailTriviaLen;
@@ -295,16 +281,13 @@ class ParsedRawSyntaxNode {
295281
assert(DK == DataKind::DeferredToken);
296282
return CharSourceRange{DeferredToken.TokLoc, DeferredToken.TokLength};
297283
}
298-
ArrayRef<ParsedTriviaPiece> getDeferredLeadingTriviaPieces() const {
284+
StringRef getDeferredLeadingTrivia() const {
299285
assert(DK == DataKind::DeferredToken);
300-
return ArrayRef<ParsedTriviaPiece>(DeferredToken.TriviaPieces,
301-
DeferredToken.NumLeadingTrivia);
286+
return DeferredToken.LeadingTrivia;
302287
}
303-
ArrayRef<ParsedTriviaPiece> getDeferredTrailingTriviaPieces() const {
288+
StringRef getDeferredTrailingTrivia() const {
304289
assert(DK == DataKind::DeferredToken);
305-
return ArrayRef<ParsedTriviaPiece>(
306-
DeferredToken.TriviaPieces + DeferredToken.NumLeadingTrivia,
307-
DeferredToken.NumTrailingTrivia);
290+
return DeferredToken.TrailingTrivia;
308291
}
309292

310293
//==========================================================================//
@@ -315,14 +298,15 @@ class ParsedRawSyntaxNode {
315298
SyntaxParsingContext &ctx);
316299

317300
/// Form a deferred token node.
318-
static ParsedRawSyntaxNode makeDeferred(Token tok,
319-
const ParsedTrivia &leadingTrivia,
320-
const ParsedTrivia &trailingTrivia,
301+
static ParsedRawSyntaxNode makeDeferred(Token tok, StringRef leadingTrivia,
302+
StringRef trailingTrivia,
321303
SyntaxParsingContext &ctx);
322304

323305
/// Form a deferred missing token node.
324306
static ParsedRawSyntaxNode makeDeferredMissing(tok tokKind, SourceLoc loc) {
325-
auto raw = ParsedRawSyntaxNode(tokKind, loc, 0, nullptr, 0, 0);
307+
auto raw = ParsedRawSyntaxNode(tokKind, loc, /*tokLength=*/0,
308+
/*leadingTrivia=*/StringRef(),
309+
/*trailingTrivia=*/StringRef());
326310
raw.IsMissing = true;
327311
return raw;
328312
}

‎include/swift/Parse/ParsedRawSyntaxRecorder.h

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,12 @@ class ParsedRawSyntaxRecorder {
4444
explicit ParsedRawSyntaxRecorder(std::shared_ptr<SyntaxParseActions> spActions)
4545
: SPActions(std::move(spActions)) {}
4646

47-
ParsedRawSyntaxNode recordToken(const Token &tok,
48-
const ParsedTrivia &leadingTrivia,
49-
const ParsedTrivia &trailingTrivia);
47+
ParsedRawSyntaxNode recordToken(const Token &tok, StringRef leadingTrivia,
48+
StringRef trailingTrivia);
5049

5150
ParsedRawSyntaxNode recordToken(tok tokenKind, CharSourceRange tokenRange,
52-
ArrayRef<ParsedTriviaPiece> leadingTrivia,
53-
ArrayRef<ParsedTriviaPiece> trailingTrivia);
51+
StringRef leadingTrivia,
52+
StringRef trailingTrivia);
5453

5554
/// Record a missing token. \p loc can be invalid or an approximate location
5655
/// of where the token would be if not missing.

‎lib/Parse/ParsedRawSyntaxNode.cpp

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -52,23 +52,13 @@ ParsedRawSyntaxNode::makeDeferred(SyntaxKind k,
5252
}
5353

5454
ParsedRawSyntaxNode
55-
ParsedRawSyntaxNode::makeDeferred(Token tok,
56-
const ParsedTrivia &leadingTrivia,
57-
const ParsedTrivia &trailingTrivia,
55+
ParsedRawSyntaxNode::makeDeferred(Token tok, StringRef leadingTrivia,
56+
StringRef trailingTrivia,
5857
SyntaxParsingContext &ctx) {
5958
CharSourceRange tokRange = tok.getRange();
60-
size_t piecesCount = leadingTrivia.size() + trailingTrivia.size();
61-
ParsedTriviaPiece *piecesPtr = nullptr;
62-
if (piecesCount > 0) {
63-
piecesPtr = ctx.getScratchAlloc().Allocate<ParsedTriviaPiece>(piecesCount);
64-
std::uninitialized_copy(leadingTrivia.begin(), leadingTrivia.end(),
65-
piecesPtr);
66-
std::uninitialized_copy(trailingTrivia.begin(), trailingTrivia.end(),
67-
piecesPtr + leadingTrivia.size());
68-
}
6959
return ParsedRawSyntaxNode(tok.getKind(), tokRange.getStart(),
70-
tokRange.getByteLength(), piecesPtr,
71-
leadingTrivia.size(), trailingTrivia.size());
60+
tokRange.getByteLength(), leadingTrivia,
61+
trailingTrivia);
7262
}
7363

7464
void ParsedRawSyntaxNode::dump() const {

‎lib/Parse/ParsedRawSyntaxRecorder.cpp

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
//===----------------------------------------------------------------------===//
1818

1919
#include "swift/Parse/ParsedRawSyntaxRecorder.h"
20+
#include "swift/Parse/Lexer.h"
2021
#include "swift/Parse/ParsedRawSyntaxNode.h"
2122
#include "swift/Parse/ParsedTrivia.h"
2223
#include "swift/Parse/SyntaxParseActions.h"
@@ -27,26 +28,25 @@ using namespace swift;
2728
using namespace swift::syntax;
2829

2930
ParsedRawSyntaxNode
30-
ParsedRawSyntaxRecorder::recordToken(const Token &tok,
31-
const ParsedTrivia &leadingTrivia,
32-
const ParsedTrivia &trailingTrivia) {
33-
return recordToken(tok.getKind(), tok.getRange(),
34-
leadingTrivia.Pieces, trailingTrivia.Pieces);
31+
ParsedRawSyntaxRecorder::recordToken(const Token &tok, StringRef leadingTrivia,
32+
StringRef trailingTrivia) {
33+
return recordToken(tok.getKind(), tok.getRange(), leadingTrivia,
34+
trailingTrivia);
3535
}
3636

3737
ParsedRawSyntaxNode
3838
ParsedRawSyntaxRecorder::recordToken(tok tokKind, CharSourceRange tokRange,
39-
ArrayRef<ParsedTriviaPiece> leadingTrivia,
40-
ArrayRef<ParsedTriviaPiece> trailingTrivia) {
41-
size_t leadingTriviaLen = ParsedTriviaPiece::getTotalLength(leadingTrivia);
42-
size_t trailingTriviaLen = ParsedTriviaPiece::getTotalLength(trailingTrivia);
43-
SourceLoc offset = tokRange.getStart().getAdvancedLoc(-leadingTriviaLen);
44-
unsigned length = leadingTriviaLen + tokRange.getByteLength() +
45-
trailingTriviaLen;
46-
CharSourceRange range{offset, length};
47-
OpaqueSyntaxNode n = SPActions->recordToken(tokKind, leadingTrivia,
48-
trailingTrivia, range);
49-
return ParsedRawSyntaxNode{SyntaxKind::Token, tokKind, range, n};
39+
StringRef leadingTrivia,
40+
StringRef trailingTrivia) {
41+
SourceLoc offset = tokRange.getStart().getAdvancedLoc(-leadingTrivia.size());
42+
unsigned length =
43+
leadingTrivia.size() + tokRange.getByteLength() + trailingTrivia.size();
44+
CharSourceRange range(offset, length);
45+
auto leadingTriviaPieces = TriviaLexer::lexTrivia(leadingTrivia).Pieces;
46+
auto trailingTriviaPieces = TriviaLexer::lexTrivia(trailingTrivia).Pieces;
47+
OpaqueSyntaxNode n = SPActions->recordToken(tokKind, leadingTriviaPieces,
48+
trailingTriviaPieces, range);
49+
return ParsedRawSyntaxNode(SyntaxKind::Token, tokKind, range, n);
5050
}
5151

5252
ParsedRawSyntaxNode
@@ -67,9 +67,8 @@ getRecordedNode(ParsedRawSyntaxNode node, ParsedRawSyntaxRecorder &rec) {
6767
tok tokKind = node.getTokenKind();
6868
if (node.isMissing())
6969
return rec.recordMissingToken(tokKind, tokRange.getStart());
70-
return rec.recordToken(tokKind,tokRange,
71-
node.getDeferredLeadingTriviaPieces(),
72-
node.getDeferredTrailingTriviaPieces());
70+
return rec.recordToken(tokKind, tokRange, node.getDeferredLeadingTrivia(),
71+
node.getDeferredTrailingTrivia());
7372
}
7473

7574
ParsedRawSyntaxNode

‎lib/Parse/SyntaxParsingContext.cpp

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
#include "swift/AST/Module.h"
1919
#include "swift/AST/SourceFile.h"
2020
#include "swift/Basic/Defer.h"
21-
#include "swift/Parse/Lexer.h"
2221
#include "swift/Parse/ParsedRawSyntaxRecorder.h"
2322
#include "swift/Parse/ParsedSyntax.h"
2423
#include "swift/Parse/ParsedSyntaxRecorder.h"
@@ -209,16 +208,12 @@ void SyntaxParsingContext::addToken(Token &Tok, StringRef LeadingTrivia,
209208
if (!Enabled)
210209
return;
211210

212-
auto LeadingTriviaPieces = TriviaLexer::lexTrivia(LeadingTrivia);
213-
auto TrailingTriviaPieces = TriviaLexer::lexTrivia(TrailingTrivia);
214-
215211
ParsedRawSyntaxNode raw;
216212
if (shouldDefer()) {
217-
raw = ParsedRawSyntaxNode::makeDeferred(Tok, LeadingTriviaPieces,
218-
TrailingTriviaPieces, *this);
213+
raw = ParsedRawSyntaxNode::makeDeferred(Tok, LeadingTrivia, TrailingTrivia,
214+
*this);
219215
} else {
220-
raw = getRecorder().recordToken(Tok, LeadingTriviaPieces,
221-
TrailingTriviaPieces);
216+
raw = getRecorder().recordToken(Tok, LeadingTrivia, TrailingTrivia);
222217
}
223218
addRawSyntax(std::move(raw));
224219
}

0 commit comments

Comments
 (0)