Skip to content

Commit 933f416

Browse files
CyrusNajmabadiCyrusNajmabadi
committed
Add bits in tokens to mark if they contain leading/trailing trivia.
1 parent 3439029 commit 933f416

File tree

2 files changed

+93
-54
lines changed

2 files changed

+93
-54
lines changed

‎src/services/syntax/scanner.ts

Lines changed: 69 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,15 @@ module TypeScript.Scanner {
3838
// _packedFullStartAndInfo:
3939
//
4040
// 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 000x <-- has leading trivia
41-
// 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 00x0 <-- has trailing trivia
42-
// 0000 0000 0000 0000 0000 0000 0000 0000 00xx xxxx xxxx xxxx xxxx xxxx xxxx xx00 <-- full start
41+
// 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 00x0 <-- has leading comment (implies has leading trivia)
42+
// 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0x00 <-- has trailing trivia
43+
// 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 x000 <-- has trailing comment (implies has trailing trivia)
44+
// 0000 0000 0000 0000 0000 0000 0000 0000 00xx xxxx xxxx xxxx xxxx xxxx xxxx 0000 <-- full start
4345
// ^ ^ ^
4446
// | | |
4547
// Bit 64 Bit 30 Bit 1
4648
//
47-
// This gives us 28 bits for the start of the token. At 256MB That's more than enough for
49+
// This gives us 26 bits for the start of the token. At 64MB That's more than enough for
4850
// any codebase.
4951
//
5052
// _packedFullWidthAndKind:
@@ -58,31 +60,31 @@ module TypeScript.Scanner {
5860
// This gives us 23bit for width (or 8MB of width which should be enough for any codebase).
5961

6062
enum ScannerConstants {
61-
LargeTokenFullStartShift = 2,
63+
LargeTokenFullStartShift = 4,
6264
LargeTokenFullWidthShift = 7,
65+
LargeTokenLeadingTriviaBitMask = 0x01, // 00000001
66+
LargeTokenLeadingCommentBitMask = 0x02, // 00000010
67+
LargeTokenTrailingTriviaBitMask = 0x04, // 00000100
68+
LargeTokenTrailingCommentBitMask = 0x08, // 00001000
69+
LargeTokenTriviaBitMask = 0x0F, // 00001111
6370

6471
FixedWidthTokenFullStartShift = 7,
72+
FixedWidthTokenMaxFullStart = 0x7FFFFF, // 23 ones.
6573

6674
SmallTokenFullWidthShift = 7,
6775
SmallTokenFullStartShift = 12,
76+
SmallTokenMaxFullStart = 0x3FFFF, // 18 ones.
77+
SmallTokenMaxFullWidth = 0x1F, // 5 ones
78+
SmallTokenFullWidthMask = 0x1F, // 00011111
6879

6980
KindMask = 0x7F, // 01111111
7081
IsVariableWidthMask = 0x80, // 10000000
71-
72-
LargeTokenLeadingTriviaBitMask = 0x01, // 00000001
73-
LargeTokenTrailingTriviaBitMask = 0x02, // 00000010
74-
75-
SmallTokenFullWidthMask = 0x1F, // 00011111
76-
77-
FixedWidthTokenMaxFullStart = 0x7FFFFF, // 23 ones.
78-
SmallTokenMaxFullStart = 0x3FFFF, // 18 ones.
79-
SmallTokenMaxFullWidth = 0x1F, // 5 ones
8082
}
8183

8284
// Make sure our math works for packing/unpacking large fullStarts.
83-
Debug.assert(largeTokenUnpackFullStart(largeTokenPackFullStartAndInfo(1 << 28, 1, 1)) === (1 << 28));
84-
Debug.assert(largeTokenUnpackFullStart(largeTokenPackFullStartAndInfo(3 << 27, 0, 1)) === (3 << 27));
85-
Debug.assert(largeTokenUnpackFullStart(largeTokenPackFullStartAndInfo(10 << 25, 1, 0)) === (10 << 25));
85+
Debug.assert(largeTokenUnpackFullStart(largeTokenPackFullStartAndInfo(1 << 26, 3)) === (1 << 26));
86+
Debug.assert(largeTokenUnpackFullStart(largeTokenPackFullStartAndInfo(3 << 25, 1)) === (3 << 25));
87+
Debug.assert(largeTokenUnpackFullStart(largeTokenPackFullStartAndInfo(10 << 23, 2)) === (10 << 23));
8688

8789
function fixedWidthTokenPackData(fullStart: number, kind: SyntaxKind) {
8890
return (fullStart << ScannerConstants.FixedWidthTokenFullStartShift) | kind;
@@ -94,8 +96,8 @@ module TypeScript.Scanner {
9496

9597
function smallTokenPackData(fullStart: number, fullWidth: number, kind: SyntaxKind) {
9698
return (fullStart << ScannerConstants.SmallTokenFullStartShift) |
97-
(fullWidth << ScannerConstants.SmallTokenFullWidthShift) |
98-
kind;
99+
(fullWidth << ScannerConstants.SmallTokenFullWidthShift) |
100+
kind;
99101
}
100102

101103
function smallTokenUnpackFullWidth(packedData: number): SyntaxKind {
@@ -106,8 +108,8 @@ module TypeScript.Scanner {
106108
return packedData >> ScannerConstants.SmallTokenFullStartShift;
107109
}
108110

109-
function largeTokenPackFullStartAndInfo(fullStart: number, hasLeadingTriviaInfo: number, hasTrailingTriviaInfo: number): number {
110-
return (fullStart << ScannerConstants.LargeTokenFullStartShift) | hasLeadingTriviaInfo | hasTrailingTriviaInfo;
111+
function largeTokenPackFullStartAndInfo(fullStart: number, triviaInfo: number): number {
112+
return (fullStart << ScannerConstants.LargeTokenFullStartShift) | triviaInfo;
111113
}
112114

113115
function largeTokenUnpackFullWidth(packedFullWidthAndKind: number) {
@@ -118,12 +120,24 @@ module TypeScript.Scanner {
118120
return packedFullStartAndInfo >> ScannerConstants.LargeTokenFullStartShift;
119121
}
120122

121-
function largeTokenUnpackHasLeadingTriviaInfo(packed: number): number {
122-
return packed & ScannerConstants.LargeTokenLeadingTriviaBitMask;
123+
function largeTokenUnpackHasLeadingTrivia(packed: number): boolean {
124+
return (packed & ScannerConstants.LargeTokenLeadingTriviaBitMask) !== 0;
125+
}
126+
127+
function largeTokenUnpackHasTrailingTrivia(packed: number): boolean {
128+
return (packed & ScannerConstants.LargeTokenTrailingTriviaBitMask) !== 0;
129+
}
130+
131+
function largeTokenUnpackHasLeadingComment(packed: number): boolean {
132+
return (packed & ScannerConstants.LargeTokenLeadingCommentBitMask) !== 0;
133+
}
134+
135+
function largeTokenUnpackHasTrailingComment(packed: number): boolean {
136+
return (packed & ScannerConstants.LargeTokenTrailingCommentBitMask) !== 0;
123137
}
124138

125-
function largeTokenUnpackHasTrailingTriviaInfo(packed: number): number {
126-
return packed & ScannerConstants.LargeTokenTrailingTriviaBitMask;
139+
function largeTokenUnpackTriviaInfo(packed: number): number {
140+
return packed & ScannerConstants.LargeTokenTriviaBitMask;
127141
}
128142

129143
var isKeywordStartCharacter: number[] = ArrayUtilities.createArray<number>(CharacterCodes.maxAsciiCharacter, 0);
@@ -174,7 +188,7 @@ module TypeScript.Scanner {
174188

175189
var triviaScanner = createScannerInternal(ts.ScriptTarget.ES5, SimpleText.fromString(""), () => { });
176190

177-
interface IScannerToken extends ISyntaxToken {
191+
interface IScannerToken extends ISyntaxToken {
178192
}
179193

180194
function fillSizeInfo(token: IScannerToken, text: ISimpleText): void {
@@ -256,6 +270,8 @@ module TypeScript.Scanner {
256270
public fullStart(): number { return fixedWidthTokenUnpackFullStart(this._packedData); }
257271
public hasLeadingTrivia(): boolean { return false; }
258272
public hasTrailingTrivia(): boolean { return false; }
273+
public hasLeadingComment(): boolean { return false; }
274+
public hasTrailingComment(): boolean { return false; }
259275
public clone(): ISyntaxToken { return new FixedWidthTokenWithNoTrivia(this._packedData); }
260276
}
261277

@@ -271,19 +287,18 @@ module TypeScript.Scanner {
271287

272288
public setFullStart(fullStart: number): void {
273289
this._packedFullStartAndInfo = largeTokenPackFullStartAndInfo(fullStart,
274-
largeTokenUnpackHasLeadingTriviaInfo(this._packedFullStartAndInfo),
275-
largeTokenUnpackHasTrailingTriviaInfo(this._packedFullStartAndInfo));
290+
largeTokenUnpackTriviaInfo(this._packedFullStartAndInfo));
276291
}
277292

278-
private syntaxTreeText(text: ISimpleText) {
293+
private syntaxTreeText(text: ISimpleText) {
279294
var result = text || syntaxTree(this).text;
280295
Debug.assert(result);
281296
return result;
282297
}
283298

284-
public isIncrementallyUnusable(): boolean { return tokenIsIncrementallyUnusable(this); }
285-
public isKeywordConvertedToIdentifier(): boolean { return false; }
286-
public hasSkippedToken(): boolean { return false; }
299+
public isIncrementallyUnusable(): boolean { return tokenIsIncrementallyUnusable(this); }
300+
public isKeywordConvertedToIdentifier(): boolean { return false; }
301+
public hasSkippedToken(): boolean { return false; }
287302

288303
public fullText(text?: ISimpleText): string {
289304
return fullText(this, this.syntaxTreeText(text));
@@ -294,8 +309,8 @@ module TypeScript.Scanner {
294309
return cachedText !== undefined ? cachedText : SyntaxFacts.getText(this.kind());
295310
}
296311

297-
public leadingTrivia(text?: ISimpleText): ISyntaxTriviaList { return leadingTrivia(this, this.syntaxTreeText(text)); }
298-
public trailingTrivia(text?: ISimpleText): ISyntaxTriviaList { return trailingTrivia(this, this.syntaxTreeText(text)); }
312+
public leadingTrivia(text?: ISimpleText): ISyntaxTriviaList { return leadingTrivia(this, this.syntaxTreeText(text)); }
313+
public trailingTrivia(text?: ISimpleText): ISyntaxTriviaList { return trailingTrivia(this, this.syntaxTreeText(text)); }
299314

300315
public leadingTriviaWidth(text?: ISimpleText): number {
301316
return leadingTriviaWidth(this, this.syntaxTreeText(text));
@@ -308,8 +323,10 @@ module TypeScript.Scanner {
308323
public kind(): SyntaxKind { return this._packedFullWidthAndKind & ScannerConstants.KindMask; }
309324
public fullWidth(): number { return largeTokenUnpackFullWidth(this._packedFullWidthAndKind); }
310325
public fullStart(): number { return largeTokenUnpackFullStart(this._packedFullStartAndInfo); }
311-
public hasLeadingTrivia(): boolean { return largeTokenUnpackHasLeadingTriviaInfo(this._packedFullStartAndInfo) !== 0; }
312-
public hasTrailingTrivia(): boolean { return largeTokenUnpackHasTrailingTriviaInfo(this._packedFullStartAndInfo) !== 0; }
326+
public hasLeadingTrivia(): boolean { return largeTokenUnpackHasLeadingTrivia(this._packedFullStartAndInfo); }
327+
public hasTrailingTrivia(): boolean { return largeTokenUnpackHasTrailingTrivia(this._packedFullStartAndInfo); }
328+
public hasLeadingComment(): boolean { return largeTokenUnpackHasLeadingComment(this._packedFullStartAndInfo); }
329+
public hasTrailingComment(): boolean { return largeTokenUnpackHasTrailingComment(this._packedFullStartAndInfo); }
313330
public clone(): ISyntaxToken { return new LargeScannerToken(this._packedFullStartAndInfo, this._packedFullWidthAndKind, this.cachedText); }
314331
}
315332

@@ -351,8 +368,8 @@ module TypeScript.Scanner {
351368
}
352369

353370
function reset(_text: ISimpleText, _start: number, _end: number) {
354-
Debug.assert(_start <= _text.length());
355-
Debug.assert(_end <= _text.length());
371+
Debug.assert(_start <= _text.length(), "Token's start was not within the bounds of text: " + _start + " - [0, " + _text.length() + ")");
372+
Debug.assert(_end <= _text.length(), "Token's end was not within the bounds of text: " + _end + " - [0, " + _text.length() + ")");
356373

357374
if (!str || text !== _text) {
358375
text = _text;
@@ -392,7 +409,7 @@ module TypeScript.Scanner {
392409
else {
393410
// inline the packing logic for perf.
394411
var packedFullStartAndTriviaInfo = (fullStart << ScannerConstants.LargeTokenFullStartShift) |
395-
leadingTriviaInfo | (trailingTriviaInfo << 1);
412+
leadingTriviaInfo | (trailingTriviaInfo << 2);
396413

397414
var packedFullWidthAndKind = (fullWidth << ScannerConstants.LargeTokenFullWidthShift) | kind;
398415
var cachedText = isFixedWidth ? undefined : text.substr(start, end - start);
@@ -505,7 +522,8 @@ module TypeScript.Scanner {
505522
case CharacterCodes.verticalTab:
506523
case CharacterCodes.formFeed:
507524
index++;
508-
result = 1;
525+
// we have trivia
526+
result |= 1;
509527
continue;
510528

511529
case CharacterCodes.carriageReturn:
@@ -516,27 +534,31 @@ module TypeScript.Scanner {
516534
case CharacterCodes.lineFeed:
517535
index++;
518536

537+
// we have trivia
538+
result |= 1;
539+
519540
// If we're consuming leading trivia, then we will continue consuming more
520541
// trivia (including newlines) up to the first token we see. If we're
521542
// consuming trailing trivia, then we break after the first newline we see.
522543
if (isTrailing) {
523-
return 1;
544+
return result;
524545
}
525546

526-
result = 1;
527547
continue;
528548

529549
case CharacterCodes.slash:
530550
if ((index + 1) < _end) {
531551
var ch2 = str.charCodeAt(index + 1);
532552
if (ch2 === CharacterCodes.slash) {
533-
result = 1;
553+
// we have a comment, and we have trivia
554+
result |= 3;
534555
skipSingleLineCommentTrivia();
535556
continue;
536557
}
537558

538559
if (ch2 === CharacterCodes.asterisk) {
539-
result = 1;
560+
// we have a comment, and we have trivia
561+
result |= 3;
540562
skipMultiLineCommentTrivia();
541563
continue;
542564
}
@@ -547,7 +569,7 @@ module TypeScript.Scanner {
547569

548570
default:
549571
if (ch > CharacterCodes.maxAsciiCharacter && slowScanTriviaInfo(ch)) {
550-
result = 1;
572+
result |= 1;
551573
continue;
552574
}
553575

@@ -1427,7 +1449,7 @@ module TypeScript.Scanner {
14271449
var fullEnd = fullStart + token.fullWidth();
14281450
reset(text, fullStart, fullEnd);
14291451

1430-
var leadingTriviaInfo = scanTriviaInfo(/*isTrailing: */ false);
1452+
scanTriviaInfo(/*isTrailing: */ false);
14311453

14321454
var start = index;
14331455
scanSyntaxKind(isContextualToken(token));
@@ -1616,6 +1638,8 @@ module TypeScript.Scanner {
16161638
}
16171639

16181640
function resetToPosition(absolutePosition: number): void {
1641+
Debug.assert(absolutePosition <= text.length(), "Trying to set the position outside the bounds of the text!");
1642+
16191643
_absolutePosition = absolutePosition;
16201644

16211645
// First, remove any diagnostics that came after this position.

‎src/services/syntax/syntaxToken.ts

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ module TypeScript {
1717

1818
hasLeadingTrivia(): boolean;
1919
hasTrailingTrivia(): boolean;
20+
hasLeadingComment(): boolean;
21+
hasTrailingComment(): boolean;
2022

2123
hasSkippedToken(): boolean;
2224

@@ -264,7 +266,7 @@ module TypeScript.Syntax {
264266
export function realizeToken(token: ISyntaxToken, text: ISimpleText): ISyntaxToken {
265267
return new RealizedToken(token.fullStart(), token.kind(), token.isKeywordConvertedToIdentifier(), token.leadingTrivia(text), token.text(), token.trailingTrivia(text));
266268
}
267-
269+
268270
export function convertKeywordToIdentifier(token: ISyntaxToken): ISyntaxToken {
269271
return new ConvertedKeywordToken(token);
270272
}
@@ -381,11 +383,14 @@ module TypeScript.Syntax {
381383
public fullText(): string { return ""; }
382384

383385
public hasLeadingTrivia() { return false; }
384-
public leadingTriviaWidth() { return 0; }
385386
public hasTrailingTrivia() { return false; }
387+
public hasLeadingComment() { return false; }
388+
public hasTrailingComment() { return false; }
386389
public hasSkippedToken() { return false; }
387390

391+
public leadingTriviaWidth() { return 0; }
388392
public trailingTriviaWidth() { return 0; }
393+
389394
public leadingTrivia(): ISyntaxTriviaList { return Syntax.emptyTriviaList; }
390395
public trailingTrivia(): ISyntaxTriviaList { return Syntax.emptyTriviaList; }
391396
}
@@ -401,11 +406,11 @@ module TypeScript.Syntax {
401406
public _primaryExpressionBrand: any; public _memberExpressionBrand: any; public _leftHandSideExpressionBrand: any; public _postfixExpressionBrand: any; public _unaryExpressionBrand: any; public _expressionBrand: any; public _typeBrand: any;
402407

403408
constructor(fullStart: number,
404-
kind: SyntaxKind,
405-
isKeywordConvertedToIdentifier: boolean,
406-
leadingTrivia: ISyntaxTriviaList,
407-
text: string,
408-
trailingTrivia: ISyntaxTriviaList) {
409+
kind: SyntaxKind,
410+
isKeywordConvertedToIdentifier: boolean,
411+
leadingTrivia: ISyntaxTriviaList,
412+
text: string,
413+
trailingTrivia: ISyntaxTriviaList) {
409414
this._fullStart = fullStart;
410415
this._kind = kind;
411416
this._isKeywordConvertedToIdentifier = isKeywordConvertedToIdentifier;
@@ -438,8 +443,8 @@ module TypeScript.Syntax {
438443
// Realized tokens are created from the parser. They are *never* incrementally reusable.
439444
public isIncrementallyUnusable() { return true; }
440445

441-
public isKeywordConvertedToIdentifier() {
442-
return this._isKeywordConvertedToIdentifier;
446+
public isKeywordConvertedToIdentifier() {
447+
return this._isKeywordConvertedToIdentifier;
443448
}
444449

445450
public fullStart(): number { return this._fullStart; }
@@ -450,6 +455,8 @@ module TypeScript.Syntax {
450455

451456
public hasLeadingTrivia(): boolean { return this._leadingTrivia.count() > 0; }
452457
public hasTrailingTrivia(): boolean { return this._trailingTrivia.count() > 0; }
458+
public hasLeadingComment(): boolean { return this._leadingTrivia.hasComment(); }
459+
public hasTrailingComment(): boolean { return this._trailingTrivia.hasComment(); }
453460

454461
public leadingTriviaWidth(): number { return this._leadingTrivia.fullWidth(); }
455462
public trailingTriviaWidth(): number { return this._trailingTrivia.fullWidth(); }
@@ -504,6 +511,14 @@ module TypeScript.Syntax {
504511
return this.underlyingToken.hasTrailingTrivia();
505512
}
506513

514+
public hasLeadingComment(): boolean {
515+
return this.underlyingToken.hasLeadingComment();
516+
}
517+
518+
public hasTrailingComment(): boolean {
519+
return this.underlyingToken.hasTrailingComment();
520+
}
521+
507522
public hasSkippedToken(): boolean {
508523
return this.underlyingToken.hasSkippedToken();
509524
}

0 commit comments

Comments
 (0)