Skip to content

Commit 8edea31

Browse files
rintarorintaro
committed
[Syntax] Abolish 'backtick' trivia
- Stop producing 'backtick' trivia for escaping identifier token. '`'s are now parts of the token text - Adjust and simplify C++ libSyntax APIs - Add 'is_deprecated' property to Trivia.py to attribute SwiftSyntax APIs rdar://problem/54810608
1 parent 19f5ac1 commit 8edea31

File tree

12 files changed

+42
-63
lines changed

12 files changed

+42
-63
lines changed

‎include/swift/Parse/ParsedRawSyntaxNode.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ class ParsedRawSyntaxNode {
141141
case DataKind::DeferredLayout:
142142
return getDeferredLayoutRange();
143143
case DataKind::DeferredToken:
144-
return getDeferredTokenRangeWithoutBackticks();
144+
return getDeferredTokenRange();
145145
default:
146146
llvm_unreachable("node not deferred");
147147
}
@@ -194,7 +194,7 @@ class ParsedRawSyntaxNode {
194194

195195
return CharSourceRange{begin, len};
196196
}
197-
CharSourceRange getDeferredTokenRangeWithoutBackticks() const {
197+
CharSourceRange getDeferredTokenRange() const {
198198
assert(DK == DataKind::DeferredToken);
199199
return CharSourceRange{DeferredToken.TokLoc, DeferredToken.TokLength};
200200
}

‎include/swift/Parse/Token.h

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -248,17 +248,6 @@ class Token {
248248
return CharSourceRange(getLoc(), getLength());
249249
}
250250

251-
CharSourceRange getRangeWithoutBackticks() const {
252-
SourceLoc TokLoc = getLoc();
253-
unsigned TokLength = getLength();
254-
if (isEscapedIdentifier()) {
255-
// Adjust to account for the backticks.
256-
TokLoc = TokLoc.getAdvancedLoc(1);
257-
TokLength -= 2;
258-
}
259-
return CharSourceRange(TokLoc, TokLength);
260-
}
261-
262251
bool hasComment() const {
263252
return CommentLength != 0;
264253
}

‎include/swift/Syntax/TokenSyntax.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,15 @@ class TokenSyntax final : public Syntax {
8080
return getRaw()->getTokenText();
8181
}
8282

83+
StringRef getIdentifierText() const {
84+
StringRef text = getText();
85+
if (text.front() == '`') {
86+
assert(text.back() == '`');
87+
return text.slice(1, text.size() - 1);
88+
}
89+
return text;
90+
}
91+
8392
static bool kindof(SyntaxKind Kind) {
8493
return isTokenKind(Kind);
8594
}

‎lib/Parse/ASTGen.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ TupleTypeRepr *ASTGen::generateTuple(TokenSyntax LParen,
164164
ElementAST.NameLoc = generate(*Name, Loc);
165165
ElementAST.Name = Name->getText() == "_"
166166
? Identifier()
167-
: Context.getIdentifier(Name->getText());
167+
: Context.getIdentifier(Name->getIdentifierText());
168168
}
169169
if (auto Colon = Element.getColon())
170170
ElementAST.ColonLoc = generate(*Colon, Loc);
@@ -173,7 +173,7 @@ TupleTypeRepr *ASTGen::generateTuple(TokenSyntax LParen,
173173
ElementAST.SecondName =
174174
SecondName->getText() == "_"
175175
? Identifier()
176-
: Context.getIdentifier(SecondName->getText());
176+
: Context.getIdentifier(SecondName->getIdentifierText());
177177
if (IsFunction) {
178178
// Form the named parameter type representation.
179179
ElementAST.UnderscoreLoc = ElementAST.NameLoc;
@@ -351,7 +351,7 @@ TypeRepr *ASTGen::generateSimpleOrMemberIdentifier(T Type, SourceLoc &Loc) {
351351
template <typename T>
352352
ComponentIdentTypeRepr *ASTGen::generateIdentifier(T Type, SourceLoc &Loc) {
353353
auto IdentifierLoc = advanceLocBegin(Loc, Type.getName());
354-
auto Identifier = Context.getIdentifier(Type.getName().getText());
354+
auto Identifier = Context.getIdentifier(Type.getName().getIdentifierText());
355355
if (auto Clause = Type.getGenericArgumentClause()) {
356356
auto Args = Clause->getArguments();
357357
if (!Args.empty()) {

‎lib/Parse/Lexer.cpp

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -289,17 +289,14 @@ void Lexer::formToken(tok Kind, const char *TokStart) {
289289
return isCommentTriviaKind(Piece.getKind());
290290
});
291291
for (auto End = LeadingTrivia.end(); Iter != End; Iter++) {
292-
if (Iter->getKind() == TriviaKind::Backtick)
293-
// Since Token::getCommentRange() doesn't take backtick into account,
294-
// we cannot include length of backtick.
295-
break;
296292
CommentLength += Iter->getLength();
297293
}
298294
}
299295

300296
StringRef TokenText { TokStart, static_cast<size_t>(CurPtr - TokStart) };
301297

302298
if (TriviaRetention == TriviaRetentionMode::WithTrivia && Kind != tok::eof) {
299+
assert(TrailingTrivia.empty() && "TrailingTrivia is empty here");
303300
lexTrivia(TrailingTrivia, /* IsForTrailingTrivia */ true);
304301
}
305302

@@ -311,10 +308,6 @@ void Lexer::formEscapedIdentifierToken(const char *TokStart) {
311308
assert(TokStart[0] == '`' && "escaped identifier starts with backtick");
312309
assert(CurPtr[-1] == '`' && "escaped identifier ends with backtick");
313310

314-
LeadingTrivia.push_back(TriviaKind::Backtick, 1);
315-
assert(TrailingTrivia.empty() && "TrailingTrivia is empty here");
316-
TrailingTrivia.push_back(TriviaKind::Backtick, 1);
317-
318311
formToken(tok::identifier, TokStart);
319312
// If this token is at ArtificialEOF, it's forced to be tok::eof. Don't mark
320313
// this as escaped-identifier in this case.

‎lib/Parse/ParseType.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1193,7 +1193,7 @@ Parser::TypeResult Parser::parseTypeTupleBody() {
11931193
auto GetNameText = [this](Optional<ParsedTokenSyntax> Name) {
11941194
return !Name ? StringRef()
11951195
: SourceMgr.extractText(
1196-
Name->getRaw().getDeferredTokenRangeWithoutBackticks(),
1196+
Name->getRaw().getDeferredTokenRange(),
11971197
L->getBufferID());
11981198
};
11991199

@@ -1208,7 +1208,7 @@ Parser::TypeResult Parser::parseTypeTupleBody() {
12081208
auto Diag = diagnose(NameLoc, diag::tuple_type_multiple_labels);
12091209
auto Name = Element.getDeferredName();
12101210
auto NameText = SourceMgr.extractText(
1211-
Name->getRaw().getDeferredTokenRangeWithoutBackticks(),
1211+
Name->getRaw().getDeferredTokenRange(),
12121212
L->getBufferID());
12131213
if (NameText == "_") {
12141214
Diag.fixItRemoveChars(NameLoc, TypeLoc);

‎lib/Parse/ParsedRawSyntaxNode.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ ParsedRawSyntaxNode::makeDeferred(Token tok,
3535
const ParsedTrivia &leadingTrivia,
3636
const ParsedTrivia &trailingTrivia,
3737
SyntaxParsingContext &ctx) {
38-
CharSourceRange tokRange = tok.getRangeWithoutBackticks();
38+
CharSourceRange tokRange = tok.getRange();
3939
size_t piecesCount = leadingTrivia.size() + trailingTrivia.size();
4040
ParsedTriviaPiece *piecesPtr = nullptr;
4141
if (piecesCount > 0) {

‎lib/Parse/ParsedRawSyntaxRecorder.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ ParsedRawSyntaxNode
3030
ParsedRawSyntaxRecorder::recordToken(const Token &tok,
3131
const ParsedTrivia &leadingTrivia,
3232
const ParsedTrivia &trailingTrivia) {
33-
return recordToken(tok.getKind(), tok.getRangeWithoutBackticks(),
33+
return recordToken(tok.getKind(), tok.getRange(),
3434
leadingTrivia.Pieces, trailingTrivia.Pieces);
3535
}
3636

@@ -63,7 +63,7 @@ getRecordedNode(const ParsedRawSyntaxNode &node, ParsedRawSyntaxRecorder &rec) {
6363
if (node.isDeferredLayout())
6464
return rec.recordRawSyntax(node.getKind(), node.getDeferredChildren());
6565
assert(node.isDeferredToken());
66-
CharSourceRange tokRange = node.getDeferredTokenRangeWithoutBackticks();
66+
CharSourceRange tokRange = node.getDeferredTokenRange();
6767
tok tokKind = node.getTokenKind();
6868
if (node.isMissing())
6969
return rec.recordMissingToken(tokKind, tokRange.getStart());

‎lib/Parse/Parser.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -319,7 +319,7 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM,
319319
/*SplitTokens=*/ArrayRef<Token>(),
320320
[&](const Token &Tok, const ParsedTrivia &LeadingTrivia,
321321
const ParsedTrivia &TrailingTrivia) {
322-
CharSourceRange TokRange = Tok.getRangeWithoutBackticks();
322+
CharSourceRange TokRange = Tok.getRange();
323323
SourceLoc LeadingTriviaLoc =
324324
TokRange.getStart().getAdvancedLoc(-LeadingTrivia.getLength());
325325
SourceLoc TrailingTriviaLoc =
@@ -328,7 +328,7 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM,
328328
LeadingTrivia.convertToSyntaxTrivia(LeadingTriviaLoc, SM, BufferID);
329329
Trivia syntaxTrailingTrivia =
330330
TrailingTrivia.convertToSyntaxTrivia(TrailingTriviaLoc, SM, BufferID);
331-
auto Text = OwnedString::makeRefCounted(Tok.getText());
331+
auto Text = OwnedString::makeRefCounted(Tok.getRawText());
332332
auto ThisToken =
333333
RawSyntax::make(Tok.getKind(), Text, syntaxLeadingTrivia.Pieces,
334334
syntaxTrailingTrivia.Pieces,

‎test/Syntax/tokens_escaped_identifier.swift

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,16 @@
22
let /*leading trivia*/ `if` = 3
33
print(/*leading trivia*/ `if` )
44

5-
// CHECK-LABEL: 2:25
5+
// CHECK-LABEL: 2:24
66
// CHECK-NEXT:(Token identifier
77
// CHECK-NEXT: (trivia blockComment /*leading trivia*/)
88
// CHECK-NEXT: (trivia space 1)
9-
// CHECK-NEXT: (trivia backtick 1)
10-
// CHECK-NEXT: (text="if")
11-
// CHECK-NEXT: (trivia backtick 1)
9+
// CHECK-NEXT: (text="`if`")
1210
// CHECK-NEXT: (trivia space 1))
1311

14-
// CHECK-LABEL: 3:27
12+
// CHECK-LABEL: 3:26
1513
// CHECK-NEXT:(Token identifier
1614
// CHECK-NEXT: (trivia blockComment /*leading trivia*/)
1715
// CHECK-NEXT: (trivia space 1)
18-
// CHECK-NEXT: (trivia backtick 1)
19-
// CHECK-NEXT: (text="if")
20-
// CHECK-NEXT: (trivia backtick 1)
16+
// CHECK-NEXT: (text="`if`")
2117
// CHECK-NEXT: (trivia space 1))

0 commit comments

Comments
 (0)