Skip to content

Commit 897946e

Browse files
SaphireLatticeSaphireLattice
authored
Fix locations reporting (#2093)
1 parent 46b206a commit 897946e

File tree

7 files changed

+85
-28
lines changed

7 files changed

+85
-28
lines changed

‎DMCompiler/Compiler/DM/AST/DMAST.Expression.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,13 @@ public virtual IEnumerable<DMASTExpression> Leaves() {
1515
public virtual DMASTExpression GetUnwrapped() {
1616
return this;
1717
}
18+
19+
public override string ToStringNoLocation() {
20+
var leaves = Leaves().ToList();
21+
if (leaves.Count == 0)
22+
return $"{GetType().Name}";
23+
return $"{GetType().Name}({string.Join(", ", Leaves().Select(l => l.ToString(Location)))})";
24+
}
1825
}
1926

2027
/// <summary>

‎DMCompiler/Compiler/DM/AST/DMAST.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,20 @@ namespace DMCompiler.Compiler.DM.AST;
44

55
public abstract class DMASTNode(Location location) {
66
public readonly Location Location = location;
7+
8+
public override string ToString() {
9+
return $"{ToString(null)}";
10+
}
11+
12+
public string ToString(Location? loc) {
13+
if (loc is not null && Location.SourceFile == loc.Value.SourceFile && Location.Line == loc.Value.Line)
14+
return ToStringNoLocation();
15+
return $"{ToStringNoLocation()} [{Location}]";
16+
}
17+
18+
public virtual string ToStringNoLocation() {
19+
return GetType().Name;
20+
}
721
}
822

923
public sealed class DMASTFile(Location location, DMASTBlockInner blockInner) : DMASTNode(location) {

‎DMCompiler/Compiler/DM/DMLexer.cs‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ protected override Token ParseNextToken() {
114114
token = preprocToken;
115115
}
116116
} else {
117+
var firstTokenLocation = CurrentLocation;
117118
switch (preprocToken.Type) {
118119
case TokenType.DM_Preproc_Whitespace: Advance(); token = CreateToken(TokenType.DM_Whitespace, preprocToken.Text); break;
119120
case TokenType.DM_Preproc_Punctuator_LeftParenthesis: BracketNesting++; Advance(); token = CreateToken(TokenType.DM_LeftParenthesis, preprocToken.Text); break;
@@ -125,19 +126,19 @@ protected override Token ParseNextToken() {
125126
case TokenType.DM_Preproc_Punctuator_Question:
126127
switch (Advance().Type) {
127128
case TokenType.DM_Preproc_Punctuator_Period:
128-
token = CreateToken(TokenType.DM_QuestionPeriod, "?.");
129129
Advance();
130+
token = CreateToken(TokenType.DM_QuestionPeriod, "?.", firstTokenLocation);
130131
break;
131132

132133
case TokenType.DM_Preproc_Punctuator_Colon:
133-
token = CreateToken(TokenType.DM_QuestionColon, "?:");
134134
Advance();
135+
token = CreateToken(TokenType.DM_QuestionColon, "?:", firstTokenLocation);
135136
break;
136137

137138
case TokenType.DM_Preproc_Punctuator_LeftBracket:
138-
token = CreateToken(TokenType.DM_QuestionLeftBracket, "?[");
139-
BracketNesting++;
140139
Advance();
140+
token = CreateToken(TokenType.DM_QuestionLeftBracket, "?[", firstTokenLocation);
141+
BracketNesting++;
141142
break;
142143

143144
default:
@@ -149,11 +150,10 @@ protected override Token ParseNextToken() {
149150
switch (Advance().Type) {
150151
case TokenType.DM_Preproc_Punctuator_Period:
151152
if (Advance().Type == TokenType.DM_Preproc_Punctuator_Period) {
152-
token = CreateToken(TokenType.DM_IndeterminateArgs, "...");
153-
154153
Advance();
154+
token = CreateToken(TokenType.DM_IndeterminateArgs, "...", firstTokenLocation);
155155
} else {
156-
token = CreateToken(TokenType.DM_SuperProc, "..");
156+
token = CreateToken(TokenType.DM_SuperProc, "..", firstTokenLocation);
157157
}
158158

159159
break;
@@ -231,6 +231,7 @@ protected override Token ParseNextToken() {
231231
break;
232232
}
233233
case TokenType.DM_Preproc_ConstantString: {
234+
Advance();
234235
string tokenText = preprocToken.Text;
235236
switch (preprocToken.Text[0]) {
236237
case '"':
@@ -239,21 +240,19 @@ protected override Token ParseNextToken() {
239240
case '@': token = CreateToken(TokenType.DM_RawString, tokenText, preprocToken.Value); break;
240241
default: token = CreateToken(TokenType.Error, tokenText, "Invalid string"); break;
241242
}
242-
243-
Advance();
244243
break;
245244
}
246245
case TokenType.DM_Preproc_StringBegin:
247-
token = CreateToken(TokenType.DM_StringBegin, preprocToken.Text, preprocToken.Value);
248246
Advance();
247+
token = CreateToken(TokenType.DM_StringBegin, preprocToken.Text, preprocToken.Value);
249248
break;
250249
case TokenType.DM_Preproc_StringMiddle:
251-
token = CreateToken(TokenType.DM_StringMiddle, preprocToken.Text, preprocToken.Value);
252250
Advance();
251+
token = CreateToken(TokenType.DM_StringMiddle, preprocToken.Text, preprocToken.Value);
253252
break;
254253
case TokenType.DM_Preproc_StringEnd:
255-
token = CreateToken(TokenType.DM_StringEnd, preprocToken.Text, preprocToken.Value);
256254
Advance();
255+
token = CreateToken(TokenType.DM_StringEnd, preprocToken.Text, preprocToken.Value);
257256
break;
258257
case TokenType.DM_Preproc_Identifier: {
259258
TokenTextBuilder.Clear();
@@ -267,7 +266,7 @@ protected override Token ParseNextToken() {
267266
var identifierText = TokenTextBuilder.ToString();
268267
var tokenType = Keywords.GetValueOrDefault(identifierText, TokenType.DM_Identifier);
269268

270-
token = CreateToken(tokenType, identifierText);
269+
token = CreateToken(tokenType, identifierText, firstTokenLocation);
271270
break;
272271
}
273272
case TokenType.DM_Preproc_Number: {
@@ -290,7 +289,7 @@ protected override Token ParseNextToken() {
290289

291290
break;
292291
}
293-
case TokenType.EndOfFile: token = preprocToken; Advance(); break;
292+
case TokenType.EndOfFile: Advance(); token = preprocToken; break;
294293
default: token = CreateToken(TokenType.Error, preprocToken.Text, "Invalid token"); break;
295294
}
296295
}

‎DMCompiler/Compiler/DM/DMParser.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -337,7 +337,7 @@ public DMASTFile File() {
337337
}
338338

339339
//Empty object definition
340-
Compiler.VerbosePrint($"Parsed object {CurrentPath}");
340+
Compiler.VerbosePrint($"Parsed object {CurrentPath} - empty");
341341
return new DMASTObjectDefinition(loc, CurrentPath, null);
342342
}
343343

@@ -1747,10 +1747,10 @@ private void ExpressionTo(out DMASTExpression endRange, out DMASTExpression? ste
17471747
}
17481748

17491749
private DMASTExpression? ExpressionIn() {
1750+
var loc = Current().Location; // Don't check this inside, as Check() will advance and point at next token instead
17501751
DMASTExpression? value = ExpressionAssign();
17511752

17521753
while (value != null && Check(TokenType.DM_In)) {
1753-
var loc = Current().Location;
17541754

17551755
Whitespace();
17561756
DMASTExpression? list = ExpressionAssign();
@@ -2151,7 +2151,7 @@ private void ExpressionTo(out DMASTExpression endRange, out DMASTExpression? ste
21512151
if (inner is null) {
21522152
inner = new DMASTVoid(loc);
21532153
} else {
2154-
inner = new DMASTExpressionWrapped(inner.Location, inner);
2154+
inner = new DMASTExpressionWrapped(loc, inner);
21552155
}
21562156

21572157
return inner;

‎DMCompiler/Compiler/DMPreprocessor/DMPreprocessor.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -638,7 +638,7 @@ private void HandlePragmaDirective() {
638638

639639
Token warningTypeToken = GetNextToken(true);
640640
if (warningTypeToken.Type != TokenType.DM_Preproc_Identifier) {
641-
compiler.Emit(WarningCode.BadDirective, warningNameToken.Location, $"Warnings can only be set to disabled, notice, warning, or error");
641+
compiler.Emit(WarningCode.BadDirective, warningTypeToken.Location, "Warnings can only be set to disabled, notice, warning, or error");
642642
return;
643643
}
644644
switch(warningTypeToken.Text.ToLower()) {
@@ -660,7 +660,7 @@ private void HandlePragmaDirective() {
660660
compiler.SetPragma(warningCode, ErrorLevel.Error);
661661
break;
662662
default:
663-
compiler.Emit(WarningCode.BadDirective, warningNameToken.Location, $"Warnings can only be set to disabled, notice, warning, or error");
663+
compiler.Emit(WarningCode.BadDirective, warningTypeToken.Location, "Warnings can only be set to disabled, notice, warning, or error");
664664
return;
665665
}
666666
}

‎DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ internal sealed class DMPreprocessorLexer {
2020
private readonly bool _isDMStandard;
2121
private char _current;
2222
private int _currentLine = 1, _currentColumn;
23+
private int _previousLine = 1, _previousColumn;
2324
private readonly Queue<Token> _pendingTokenQueue = new(); // TODO: Possible to remove this?
2425

2526
public DMPreprocessorLexer(DMCompiler compiler, string includeDirectory, string file, string source) {
@@ -61,6 +62,9 @@ public Token NextToken(bool ignoreWhitespace = false) {
6162

6263
char c = GetCurrent();
6364

65+
_previousLine = _currentLine;
66+
_previousColumn = _currentColumn;
67+
6468
switch (c) {
6569
case '\0':
6670
return CreateToken(TokenType.EndOfFile, c);
@@ -367,7 +371,7 @@ public Token NextToken(bool ignoreWhitespace = false) {
367371
LexString(true) :
368372
CreateToken(TokenType.DM_Preproc_Punctuator, c);
369373
case '#': {
370-
bool isConcat = (Advance() == '#');
374+
bool isConcat = Advance() == '#';
371375
if (isConcat) Advance();
372376

373377
// Whitespace after '#' is ignored
@@ -444,7 +448,7 @@ public Token NextToken(bool ignoreWhitespace = false) {
444448
}
445449

446450
Advance();
447-
return CreateToken(TokenType.Error, string.Empty, $"Unknown character: {c.ToString()}");
451+
return CreateToken(TokenType.Error, string.Empty, $"Unknown character: {c}");
448452
}
449453
}
450454
}
@@ -619,7 +623,7 @@ private bool HandleLineEnd() {
619623
goto case '\n';
620624
case '\n':
621625
_currentLine++;
622-
_currentColumn = 1;
626+
_currentColumn = 0; // Because Advance will bump this to 1 and any position reads will happen next NextToken() call
623627

624628
if (c == '\n') // This line could have ended with only \r
625629
Advance();
@@ -641,7 +645,7 @@ private char Advance() {
641645

642646
if (value == -1) {
643647
_current = '\0';
644-
} else {
648+
} else {
645649
_currentColumn++;
646650
_current = (char)value;
647651
}
@@ -656,11 +660,11 @@ private bool AtEndOfSource() {
656660

657661
[MethodImpl(MethodImplOptions.AggressiveInlining)]
658662
private Token CreateToken(TokenType type, string text, object? value = null) {
659-
return new Token(type, text, new Location(File, _currentLine, _currentColumn, _isDMStandard), value);
663+
return new Token(type, text, new Location(File, _previousLine, _previousColumn, _isDMStandard), value);
660664
}
661665

662666
[MethodImpl(MethodImplOptions.AggressiveInlining)]
663667
private Token CreateToken(TokenType type, char text, object? value = null) {
664-
return new Token(type, text.ToString(), new Location(File, _currentLine, _currentColumn, _isDMStandard), value);
668+
return new Token(type, text.ToString(), new Location(File, _previousLine, _previousColumn, _isDMStandard), value);
665669
}
666670
}

‎DMCompiler/Compiler/Lexer.cs‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,32 @@
33
namespace DMCompiler.Compiler;
44

55
internal class Lexer<TSourceType> {
6+
/// <summary>
7+
/// Location of token that'll be output by <see cref="GetCurrent"/>. If you skip through more
8+
/// </summary>
69
public Location CurrentLocation { get; protected set; }
10+
11+
/// <summary>
12+
/// Location of a previous token.
13+
/// </summary>
14+
public Location PreviousLocation { get; private set; }
15+
716
public IEnumerable<TSourceType> Source { get; }
817
public bool AtEndOfSource { get; private set; }
9-
1018
protected Queue<Token> _pendingTokenQueue = new();
1119

1220
private readonly IEnumerator<TSourceType> _sourceEnumerator;
1321
private TSourceType _current;
1422

23+
/// <summary>
24+
/// Given a stream of some type, allows to advance through it and create <see cref="Token"/> tokens
25+
/// </summary>
26+
/// <param name="sourceName">Used to build the initial Location, access through <see cref="CurrentLocation"/></param>
27+
/// <param name="source">Source of <see cref="TSourceType"/> input</param>
28+
/// <exception cref="FileNotFoundException">Thrown if <paramref name="source"/> is null</exception>
1529
protected Lexer(string sourceName, IEnumerable<TSourceType> source) {
1630
CurrentLocation = new Location(sourceName, 1, 0);
31+
PreviousLocation = CurrentLocation;
1732
Source = source;
1833
if (source == null)
1934
throw new FileNotFoundException("Source file could not be read: " + sourceName);
@@ -24,7 +39,7 @@ public Token GetNextToken() {
2439
if (_pendingTokenQueue.Count > 0)
2540
return _pendingTokenQueue.Dequeue();
2641

27-
Token nextToken = ParseNextToken();
42+
var nextToken = ParseNextToken();
2843
while (nextToken.Type == TokenType.Skip) nextToken = ParseNextToken();
2944

3045
if (_pendingTokenQueue.Count > 0) {
@@ -39,10 +54,24 @@ protected virtual Token ParseNextToken() {
3954
return CreateToken(TokenType.Unknown, GetCurrent()?.ToString() ?? string.Empty);
4055
}
4156

57+
protected Token CreateToken(TokenType type, string text, Location location, object? value = null) {
58+
var token = new Token(type, text, location, value);
59+
return token;
60+
}
61+
62+
/// <summary>
63+
/// Creates a new <see cref="Token"/> located at <see cref="PreviousLocation"/>
64+
/// </summary>
65+
/// <remarks>
66+
/// If you have used <see cref="Advance"/> more than once, the <see cref="Location"/> will be incorrect,
67+
/// and you'll need to use <see cref="CreateToken(TokenType, string, Location, object?)"/>
68+
/// with a previously recorded <see cref="CurrentLocation"/>
69+
/// </remarks>
4270
protected Token CreateToken(TokenType type, string text, object? value = null) {
43-
return new Token(type, text, CurrentLocation, value);
71+
return CreateToken(type, text, PreviousLocation, value);
4472
}
4573

74+
/// <inheritdoc cref="CreateToken(TokenType, string, object?)"/>
4675
protected Token CreateToken(TokenType type, char text, object? value = null) {
4776
return CreateToken(type, char.ToString(text), value);
4877
}
@@ -51,7 +80,10 @@ protected virtual TSourceType GetCurrent() {
5180
return _current;
5281
}
5382

83+
/// <remarks>Call before CreateToken to make sure the location is correct</remarks>
5484
protected virtual TSourceType Advance() {
85+
PreviousLocation = CurrentLocation;
86+
5587
if (_sourceEnumerator.MoveNext()) {
5688
_current = _sourceEnumerator.Current;
5789
} else {
@@ -63,6 +95,7 @@ protected virtual TSourceType Advance() {
6395
}
6496

6597
internal class TokenLexer : Lexer<Token> {
98+
/// <inheritdoc/>
6699
protected TokenLexer(string sourceName, IEnumerable<Token> source) : base(sourceName, source) {
67100
Advance();
68101
}

0 commit comments

Comments
 (0)