Skip to content

Commit a7641a7

Browse files
ahoppenahoppen
committed
[Lexer] Adjust tests for new delayed trivia lexing
1 parent 6911553 commit a7641a7

File tree

3 files changed

+152
-64
lines changed

3 files changed

+152
-64
lines changed

‎lib/Parse/Lexer.cpp

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2875,7 +2875,7 @@ ParsedTrivia TriviaLexer::lexTrivia(StringRef TriviaStr) {
28752875
Pieces.appendOrSquash(TriviaKind::Newline, 1);
28762876
continue;
28772877
case '\r':
2878-
if (CurPtr[0] == '\n') {
2878+
if (CurPtr < BufferEnd && CurPtr[0] == '\n') {
28792879
Pieces.appendOrSquash(TriviaKind::CarriageReturnLineFeed, 2);
28802880
++CurPtr;
28812881
continue;
@@ -2896,7 +2896,7 @@ ParsedTrivia TriviaLexer::lexTrivia(StringRef TriviaStr) {
28962896
Pieces.appendOrSquash(TriviaKind::Formfeed, 1);
28972897
continue;
28982898
case '/':
2899-
if (*CurPtr == '/') {
2899+
if (CurPtr < BufferEnd && CurPtr[0] == '/') {
29002900
// '// ...' comment.
29012901
bool isDocComment = CurPtr[1] == '/';
29022902
advanceToEndOfLine(CurPtr, BufferEnd);
@@ -2905,7 +2905,7 @@ ParsedTrivia TriviaLexer::lexTrivia(StringRef TriviaStr) {
29052905
: TriviaKind::LineComment,
29062906
Length);
29072907
continue;
2908-
} else if (*CurPtr == '*') {
2908+
} else if (CurPtr < BufferEnd && CurPtr[0] == '*') {
29092909
// '/* ... */' comment.
29102910
bool isDocComment = CurPtr[1] == '*';
29112911
skipToEndOfSlashStarComment(CurPtr, BufferEnd);
@@ -2917,7 +2917,7 @@ ParsedTrivia TriviaLexer::lexTrivia(StringRef TriviaStr) {
29172917
}
29182918
break;
29192919
case '#':
2920-
if (*CurPtr == '!') {
2920+
if (CurPtr < BufferEnd && CurPtr[0] == '!') {
29212921
// Hashbang '#!/path/to/swift'.
29222922
advanceToEndOfLine(CurPtr, BufferEnd);
29232923
size_t Length = CurPtr - TriviaStart;
@@ -2934,6 +2934,15 @@ ParsedTrivia TriviaLexer::lexTrivia(StringRef TriviaStr) {
29342934
continue;
29352935
}
29362936
break;
2937+
case '\xEF':
2938+
if ((CurPtr + 1) < BufferEnd && CurPtr[0] == '\xBB' && CurPtr[1] == '\xBF') {
2939+
// BOM marker.
2940+
CurPtr = CurPtr + 2;
2941+
size_t Length = CurPtr - TriviaStart;
2942+
Pieces.push_back(TriviaKind::GarbageText, Length);
2943+
continue;
2944+
}
2945+
break;
29372946
case 0: {
29382947
size_t Length = CurPtr - TriviaStart;
29392948
Pieces.push_back(TriviaKind::GarbageText, Length);

‎unittests/Parse/LexerTests.cpp

Lines changed: 47 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -285,24 +285,24 @@ TEST_F(LexerTest, BOMNoCommentNoTrivia) {
285285
TriviaRetentionMode::WithoutTrivia);
286286

287287
Token Tok;
288-
ParsedTrivia LeadingTrivia, TrailingTrivia;
288+
StringRef LeadingTrivia, TrailingTrivia;
289289

290290
L.lex(Tok, LeadingTrivia, TrailingTrivia);
291291
ASSERT_EQ(tok::identifier, Tok.getKind());
292292
ASSERT_EQ("aaa", Tok.getText());
293293
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 14), Tok.getLoc());
294294
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 14), Tok.getCommentRange().getStart());
295295
ASSERT_EQ(0u, Tok.getCommentRange().getByteLength());
296-
ASSERT_EQ((ParsedTrivia{{}}), LeadingTrivia);
297-
ASSERT_EQ((ParsedTrivia{{}}), TrailingTrivia);
296+
ASSERT_EQ(StringRef(), LeadingTrivia);
297+
ASSERT_EQ(StringRef(), TrailingTrivia);
298298

299299
L.lex(Tok, LeadingTrivia, TrailingTrivia);
300300
ASSERT_EQ(tok::eof, Tok.getKind());
301301
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 31), Tok.getLoc());
302302
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 31), Tok.getCommentRange().getStart());
303303
ASSERT_EQ(0u, Tok.getCommentRange().getByteLength());
304-
ASSERT_EQ((ParsedTrivia{{}}), LeadingTrivia);
305-
ASSERT_EQ((ParsedTrivia{{}}), TrailingTrivia);
304+
ASSERT_EQ(StringRef(), LeadingTrivia);
305+
ASSERT_EQ(StringRef(), TrailingTrivia);
306306
}
307307

308308
TEST_F(LexerTest, BOMTokenCommentNoTrivia) {
@@ -317,51 +317,51 @@ TEST_F(LexerTest, BOMTokenCommentNoTrivia) {
317317
TriviaRetentionMode::WithoutTrivia);
318318

319319
Token Tok;
320-
ParsedTrivia LeadingTrivia, TrailingTrivia;
320+
StringRef LeadingTrivia, TrailingTrivia;
321321

322322
L.lex(Tok, LeadingTrivia, TrailingTrivia);
323323
ASSERT_EQ(tok::comment, Tok.getKind());
324324
ASSERT_EQ("// comment\n", Tok.getText());
325325
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 3), Tok.getLoc());
326326
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 3), Tok.getCommentRange().getStart());
327327
ASSERT_EQ(0u, Tok.getCommentRange().getByteLength());
328-
ASSERT_EQ((ParsedTrivia{{}}), LeadingTrivia);
329-
ASSERT_EQ((ParsedTrivia{{}}), TrailingTrivia);
328+
ASSERT_EQ(StringRef(), LeadingTrivia);
329+
ASSERT_EQ(StringRef(), TrailingTrivia);
330330

331331
L.lex(Tok, LeadingTrivia, TrailingTrivia);
332332
ASSERT_EQ(tok::identifier, Tok.getKind());
333333
ASSERT_EQ("aaa", Tok.getText());
334334
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 14), Tok.getLoc());
335335
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 14), Tok.getCommentRange().getStart());
336336
ASSERT_EQ(0u, Tok.getCommentRange().getByteLength());
337-
ASSERT_EQ((ParsedTrivia{{}}), LeadingTrivia);
338-
ASSERT_EQ((ParsedTrivia{{}}), TrailingTrivia);
337+
ASSERT_EQ(StringRef(), LeadingTrivia);
338+
ASSERT_EQ(StringRef(), TrailingTrivia);
339339

340340
L.lex(Tok, LeadingTrivia, TrailingTrivia);
341341
ASSERT_EQ(tok::comment, Tok.getKind());
342342
ASSERT_EQ("//xx \n", Tok.getText());
343343
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 18), Tok.getLoc());
344344
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 18), Tok.getCommentRange().getStart());
345345
ASSERT_EQ(0u, Tok.getCommentRange().getByteLength());
346-
ASSERT_EQ((ParsedTrivia{{}}), LeadingTrivia);
347-
ASSERT_EQ((ParsedTrivia{{}}), TrailingTrivia);
346+
ASSERT_EQ(StringRef(), LeadingTrivia);
347+
ASSERT_EQ(StringRef(), TrailingTrivia);
348348

349349
L.lex(Tok, LeadingTrivia, TrailingTrivia);
350350
ASSERT_EQ(tok::comment, Tok.getKind());
351351
ASSERT_EQ("/* x */", Tok.getText());
352352
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 24), Tok.getLoc());
353353
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 24), Tok.getCommentRange().getStart());
354354
ASSERT_EQ(0u, Tok.getCommentRange().getByteLength());
355-
ASSERT_EQ((ParsedTrivia{{}}), LeadingTrivia);
356-
ASSERT_EQ((ParsedTrivia{{}}), TrailingTrivia);
355+
ASSERT_EQ(StringRef(), LeadingTrivia);
356+
ASSERT_EQ(StringRef(), TrailingTrivia);
357357

358358
L.lex(Tok, LeadingTrivia, TrailingTrivia);
359359
ASSERT_EQ(tok::eof, Tok.getKind());
360360
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 31), Tok.getLoc());
361361
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 31), Tok.getCommentRange().getStart());
362362
ASSERT_EQ(0u, Tok.getCommentRange().getByteLength());
363-
ASSERT_EQ((ParsedTrivia{{}}), LeadingTrivia);
364-
ASSERT_EQ((ParsedTrivia{{}}), TrailingTrivia);
363+
ASSERT_EQ(StringRef(), LeadingTrivia);
364+
ASSERT_EQ(StringRef(), TrailingTrivia);
365365
}
366366

367367
TEST_F(LexerTest, BOMAttachCommentNoTrivia) {
@@ -376,24 +376,24 @@ TEST_F(LexerTest, BOMAttachCommentNoTrivia) {
376376
TriviaRetentionMode::WithoutTrivia);
377377

378378
Token Tok;
379-
ParsedTrivia LeadingTrivia, TrailingTrivia;
379+
StringRef LeadingTrivia, TrailingTrivia;
380380

381381
L.lex(Tok, LeadingTrivia, TrailingTrivia);
382382
ASSERT_EQ(tok::identifier, Tok.getKind());
383383
ASSERT_EQ("aaa", Tok.getText());
384384
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 14), Tok.getLoc());
385385
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 3), Tok.getCommentRange().getStart());
386386
ASSERT_EQ(10u, Tok.getCommentRange().getByteLength());
387-
ASSERT_EQ((ParsedTrivia{{}}), LeadingTrivia);
388-
ASSERT_EQ((ParsedTrivia{{}}), TrailingTrivia);
387+
ASSERT_EQ(StringRef(), LeadingTrivia);
388+
ASSERT_EQ(StringRef(), TrailingTrivia);
389389

390390
L.lex(Tok, LeadingTrivia, TrailingTrivia);
391391
ASSERT_EQ(tok::eof, Tok.getKind());
392392
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 31), Tok.getLoc());
393393
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 18), Tok.getCommentRange().getStart());
394394
ASSERT_EQ(13u, Tok.getCommentRange().getByteLength());
395-
ASSERT_EQ((ParsedTrivia{{}}), LeadingTrivia);
396-
ASSERT_EQ((ParsedTrivia{{}}), TrailingTrivia);
395+
ASSERT_EQ(StringRef(), LeadingTrivia);
396+
ASSERT_EQ(StringRef(), TrailingTrivia);
397397
}
398398

399399
TEST_F(LexerTest, BOMNoCommentTrivia) {
@@ -408,34 +408,42 @@ TEST_F(LexerTest, BOMNoCommentTrivia) {
408408
TriviaRetentionMode::WithTrivia);
409409

410410
Token Tok;
411-
ParsedTrivia LeadingTrivia, TrailingTrivia;
411+
StringRef LeadingTrivia, TrailingTrivia;
412412

413413
L.lex(Tok, LeadingTrivia, TrailingTrivia);
414414
ASSERT_EQ(tok::identifier, Tok.getKind());
415415
ASSERT_EQ("aaa", Tok.getText());
416416
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 14), Tok.getLoc());
417417
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 14), Tok.getCommentRange().getStart());
418418
ASSERT_EQ(0u, Tok.getCommentRange().getByteLength());
419+
ASSERT_EQ("\xEF\xBB\xBF" "// comment" "\n", LeadingTrivia);
420+
ASSERT_EQ(" ", TrailingTrivia);
421+
422+
ParsedTrivia LeadingTriviaPieces = TriviaLexer::lexTrivia(LeadingTrivia);
423+
ParsedTrivia TrailingTriviaPieces = TriviaLexer::lexTrivia(TrailingTrivia);
419424
ASSERT_EQ((ParsedTrivia{{
420425
ParsedTriviaPiece(TriviaKind::GarbageText, strlen("\xEF\xBB\xBF")),
421426
ParsedTriviaPiece(TriviaKind::LineComment, strlen("// comment")),
422427
ParsedTriviaPiece(TriviaKind::Newline, 1)
423-
}}), LeadingTrivia);
428+
}}), LeadingTriviaPieces);
424429
ASSERT_EQ((ParsedTrivia{{
425430
ParsedTriviaPiece(TriviaKind::Space, 1)
426-
}}), TrailingTrivia);
431+
}}), TrailingTriviaPieces);
427432

428433
L.lex(Tok, LeadingTrivia, TrailingTrivia);
429434
ASSERT_EQ(tok::eof, Tok.getKind());
430435
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 31), Tok.getLoc());
431436
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 31), Tok.getCommentRange().getStart());
432437
ASSERT_EQ(0u, Tok.getCommentRange().getByteLength());
438+
ASSERT_EQ("//xx \n/* x */", LeadingTrivia);
439+
ASSERT_EQ(StringRef(), TrailingTrivia);
440+
441+
LeadingTriviaPieces = TriviaLexer::lexTrivia(LeadingTrivia);
433442
ASSERT_EQ((ParsedTrivia{{
434443
ParsedTriviaPiece(TriviaKind::LineComment, strlen("//xx ")),
435444
ParsedTriviaPiece(TriviaKind::Newline, 1),
436445
ParsedTriviaPiece(TriviaKind::BlockComment, strlen("/* x */"))
437-
}}), LeadingTrivia);
438-
ASSERT_EQ((ParsedTrivia{{}}), TrailingTrivia);
446+
}}), LeadingTriviaPieces);
439447
}
440448

441449
TEST_F(LexerTest, BOMAttachCommentTrivia) {
@@ -450,34 +458,42 @@ TEST_F(LexerTest, BOMAttachCommentTrivia) {
450458
TriviaRetentionMode::WithTrivia);
451459

452460
Token Tok;
453-
ParsedTrivia LeadingTrivia, TrailingTrivia;
461+
StringRef LeadingTrivia, TrailingTrivia;
454462

455463
L.lex(Tok, LeadingTrivia, TrailingTrivia);
456464
ASSERT_EQ(tok::identifier, Tok.getKind());
457465
ASSERT_EQ("aaa", Tok.getText());
458466
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 14), Tok.getLoc());
459467
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 3), Tok.getCommentRange().getStart());
460468
ASSERT_EQ(10u, Tok.getCommentRange().getByteLength());
469+
ASSERT_EQ("\xEF\xBB\xBF" "// comment" "\n", LeadingTrivia);
470+
ASSERT_EQ(" ", TrailingTrivia);
471+
472+
ParsedTrivia LeadingTriviaPieces = TriviaLexer::lexTrivia(LeadingTrivia);
473+
ParsedTrivia TrailingTriviaPieces = TriviaLexer::lexTrivia(TrailingTrivia);
461474
ASSERT_EQ((ParsedTrivia{{
462475
ParsedTriviaPiece(TriviaKind::GarbageText, strlen("\xEF\xBB\xBF")),
463476
ParsedTriviaPiece(TriviaKind::LineComment, strlen("// comment")),
464477
ParsedTriviaPiece(TriviaKind::Newline, 1)
465-
}}), LeadingTrivia);
478+
}}), LeadingTriviaPieces);
466479
ASSERT_EQ((ParsedTrivia{{
467480
ParsedTriviaPiece(TriviaKind::Space, 1)
468-
}}), TrailingTrivia);
481+
}}), TrailingTriviaPieces);
469482

470483
L.lex(Tok, LeadingTrivia, TrailingTrivia);
471484
ASSERT_EQ(tok::eof, Tok.getKind());
472485
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 31), Tok.getLoc());
473486
ASSERT_EQ(SourceMgr.getLocForOffset(BufferID, 18), Tok.getCommentRange().getStart());
474487
ASSERT_EQ(13u, Tok.getCommentRange().getByteLength());
488+
ASSERT_EQ("//xx \n/* x */", LeadingTrivia);
489+
ASSERT_EQ(StringRef(), TrailingTrivia);
490+
491+
LeadingTriviaPieces = TriviaLexer::lexTrivia(LeadingTrivia);
475492
ASSERT_EQ((ParsedTrivia{{
476493
ParsedTriviaPiece(TriviaKind::LineComment, strlen("//xx ")),
477494
ParsedTriviaPiece(TriviaKind::Newline, 1),
478495
ParsedTriviaPiece(TriviaKind::BlockComment, strlen("/* x */"))
479-
}}), LeadingTrivia);
480-
ASSERT_EQ((ParsedTrivia{{}}), TrailingTrivia);
496+
}}), LeadingTriviaPieces);
481497
}
482498

483499
TEST_F(LexerTest, RestoreBasic) {

0 commit comments

Comments
 (0)