Skip to content

Commit 6785aed

Browse files
webbjuwebbju
authored
Merge pull request #27 from EmbarkStudios/webbju/preprocessor-scope-parser
Preprocessor scope/tree parser.
2 parents d4f0806 + 98d2074 commit 6785aed

File tree

6 files changed

+250
-50
lines changed

6 files changed

+250
-50
lines changed

‎ServerCodeExciser/PreprocessorParser.cs‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using Antlr4.Runtime;
5+
using ServerCodeExcisionCommon;
6+
7+
namespace ServerCodeExciser
8+
{
9+
public class PreprocessorParser
10+
{
11+
public static List<PreprocessorScope> Parse(BufferedTokenStream tokenStream)
12+
{
13+
var directives = tokenStream
14+
.GetTokens()
15+
.Where(t => t.Channel == UnrealAngelscriptLexer.PREPROCESSOR_CHANNEL)
16+
.Where(t => t.Type == UnrealAngelscriptLexer.Directive)
17+
.OrderBy(t => t.StartIndex)
18+
.ToList();
19+
20+
var rootScopes = new List<PreprocessorScope>();
21+
var ifStack = new Stack<PreprocessorScope>();
22+
23+
foreach (var token in directives)
24+
{
25+
switch (token.Text)
26+
{
27+
case var t when t.StartsWith("#if", StringComparison.Ordinal): // #if, #ifdef, #ifndef
28+
{
29+
var scope = CreateScope(token);
30+
if (ifStack.TryPeek(out var parent))
31+
{
32+
parent.Children.Add(scope);
33+
}
34+
else
35+
{
36+
rootScopes.Add(scope);
37+
}
38+
ifStack.Push(scope);
39+
}
40+
break;
41+
42+
case var t when t.StartsWith("#elif", StringComparison.Ordinal) || t.StartsWith("#else", StringComparison.Ordinal): // #elif, #elifdef, #elifndef, #else
43+
{
44+
var scope = CreateScope(token);
45+
if (ifStack.TryPeek(out var parent))
46+
{
47+
parent.Children.Add(scope);
48+
// adjust #if / #elif scope for closing bounds.
49+
parent.Span = new SourceSpan
50+
{
51+
Start = parent.Span.Start,
52+
StartIndex = parent.Span.StartIndex,
53+
End = new SourcePosition(token.Line, token.Column),
54+
EndIndex = token.StartIndex,
55+
};
56+
}
57+
}
58+
break;
59+
60+
case var t when t.StartsWith("#endif", StringComparison.Ordinal):
61+
{
62+
if (ifStack.TryPop(out var parent))
63+
{
64+
// close parent (#if) to the range of the entire block.
65+
parent.Span = new SourceSpan
66+
{
67+
Start = parent.Span.Start,
68+
StartIndex = parent.Span.StartIndex,
69+
End = new SourcePosition(token.Line, token.Column),
70+
EndIndex = token.StopIndex,
71+
};
72+
}
73+
}
74+
break;
75+
}
76+
}
77+
78+
return rootScopes;
79+
}
80+
81+
private static PreprocessorScope CreateScope(IToken token)
82+
{
83+
return new PreprocessorScope(
84+
token.Text,
85+
new SourceSpan(
86+
token.Line,
87+
token.Column,
88+
token.Line,
89+
token.Column,
90+
token.StartIndex,
91+
token.StopIndex
92+
)
93+
);
94+
}
95+
}
96+
}

‎ServerCodeExciser/PreprocessorScope.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
using System.Collections.Generic;
2+
using ServerCodeExcisionCommon;
3+
4+
namespace ServerCodeExciser
5+
{
6+
public sealed class PreprocessorScope
7+
{
8+
public PreprocessorScope(string directive, SourceSpan span)
9+
{
10+
Directive = directive;
11+
Span = span;
12+
}
13+
14+
public string Directive { get; set; }
15+
16+
public SourceSpan Span { get; set; }
17+
18+
public List<PreprocessorScope> Children { get; set; } = new();
19+
}
20+
}

‎ServerCodeExciser/ServerCodeExcisionProcessor.cs‎

Lines changed: 13 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -241,8 +241,12 @@ private ExcisionStats ProcessCodeFile(string fileName, string inputPath, EExcisi
241241
}
242242

243243
// Determine if there are any existing preprocessor server-code exclusions in the source file.
244-
var detectedPreprocessorServerOnlyScopes = FindPreprocessorGuards(commonTokenStream)
245-
.Where(x => x.Directive.Contains(excisionLanguage.ServerScopeStartString, StringComparison.Ordinal));
244+
var preprocessorScopes = PreprocessorParser.Parse(commonTokenStream);
245+
var detectedPreprocessorServerOnlyScopes = new List<PreprocessorScope>();
246+
FindPreprocessorScopesForSymbolRecursive(
247+
preprocessorScopes,
248+
scope => scope.Directive.Contains(excisionLanguage.ServerScopeStartString, StringComparison.Ordinal),
249+
detectedPreprocessorServerOnlyScopes);
246250

247251
// Process scopes we've evaluated must be server only.
248252
foreach (ServerOnlyScopeData currentScope in visitor.DetectedServerOnlyScopes)
@@ -253,8 +257,8 @@ private ExcisionStats ProcessCodeFile(string fileName, string inputPath, EExcisi
253257
}
254258

255259
// Skip if there's already a server-code exclusion for the scope. (We don't want have duplicate guards.)
256-
var (StartIndex, StopIndex) = TrimWhitespace(script, currentScope);
257-
if (detectedPreprocessorServerOnlyScopes.Any(x => StartIndex >= x.StartIndex && StopIndex <= x.StopIndex))
260+
var (StartIndex, EndIndex) = TrimWhitespace(script, currentScope);
261+
if (detectedPreprocessorServerOnlyScopes.Any(x => StartIndex >= x.Span.StartIndex && EndIndex <= x.Span.EndIndex))
258262
{
259263
continue; // We're inside an existing scope.
260264
}
@@ -358,57 +362,16 @@ private static (int StartIndex, int StopIndex) TrimWhitespace(string script, Ser
358362
return (startIndex, stopIndex);
359363
}
360364

361-
private static List<(string Directive, int StartIndex, int StopIndex)> FindPreprocessorGuards(BufferedTokenStream tokenStream)
365+
private static void FindPreprocessorScopesForSymbolRecursive(List<PreprocessorScope> scopes, Predicate<PreprocessorScope> predicate, List<PreprocessorScope> result)
362366
{
363-
var preprocessorDirectives = tokenStream
364-
.GetTokens()
365-
.Where(t => t.Channel == UnrealAngelscriptLexer.PREPROCESSOR_CHANNEL)
366-
.Where(t => t.Type == UnrealAngelscriptLexer.Directive)
367-
.ToList();
368-
369-
var preprocessorGuards = new List<(string Directive, int StartIndex, int StopIndex)>();
370-
var ifStack = new Stack<IToken>();
371-
372-
foreach (var token in preprocessorDirectives)
367+
foreach (var scope in scopes)
373368
{
374-
switch (token.Text)
369+
if (predicate(scope))
375370
{
376-
case var t when t.StartsWith("#if", StringComparison.Ordinal): // #if, #ifdef, #ifndef
377-
ifStack.Push(token);
378-
break;
379-
380-
case var t when t.StartsWith("#elif", StringComparison.Ordinal): // #elif, #elifdef, #elifndef
381-
{
382-
if (ifStack.TryPop(out var removed))
383-
{
384-
preprocessorGuards.Add((removed.Text, removed.StartIndex, token.StopIndex));
385-
}
386-
ifStack.Push(token);
387-
}
388-
break;
389-
390-
case var t when t.StartsWith("#else", StringComparison.Ordinal):
391-
{
392-
if (ifStack.TryPop(out var removed))
393-
{
394-
preprocessorGuards.Add((removed.Text, removed.StartIndex, token.StopIndex));
395-
}
396-
ifStack.Push(token);
397-
}
398-
break;
399-
400-
case var t when t.StartsWith("#endif", StringComparison.Ordinal):
401-
{
402-
if (ifStack.TryPop(out var removed))
403-
{
404-
preprocessorGuards.Add((removed.Text, removed.StartIndex, token.StopIndex));
405-
}
406-
}
407-
break;
371+
result.Add(scope);
408372
}
373+
FindPreprocessorScopesForSymbolRecursive(scope.Children, predicate, result);
409374
}
410-
411-
return preprocessorGuards;
412375
}
413376

414377
private bool InjectedMacroAlreadyExistsAtLocation(StringBuilder script, int index, bool lookAhead, bool ignoreWhitespace, string macro)

‎ServerCodeExciserTest/PreprocessorParserTests.cs‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
using Antlr4.Runtime;
2+
using Microsoft.VisualStudio.TestTools.UnitTesting;
3+
4+
namespace ServerCodeExciser.Tests
5+
{
6+
[TestClass]
7+
public class PreprocessorParserTests
8+
{
9+
[TestMethod]
10+
public void ConditionalBranchTest()
11+
{
12+
var script = "#ifdef WITH_SERVER\r\n" +
13+
"#elif RELEASE\r\n" +
14+
"#elif DEBUG\r\n" +
15+
"#else\r\n" +
16+
"#endif // WITH_SERVER\r\n";
17+
18+
var lexer = new UnrealAngelscriptLexer(new AntlrInputStream(script));
19+
var tokenStream = new CommonTokenStream(lexer);
20+
tokenStream.Fill();
21+
22+
var nodes = PreprocessorParser.Parse(tokenStream);
23+
Assert.HasCount(1, nodes);
24+
Assert.AreEqual("#ifdef WITH_SERVER", nodes[0].Directive);
25+
Assert.AreEqual("#elif RELEASE", nodes[0].Children[0].Directive);
26+
Assert.AreEqual("#elif DEBUG", nodes[0].Children[1].Directive);
27+
Assert.AreEqual("#else", nodes[0].Children[2].Directive);
28+
Assert.AreEqual(1, nodes[0].Span.Start.Line);
29+
Assert.AreEqual(0, nodes[0].Span.Start.Column);
30+
Assert.AreEqual(0, nodes[0].Span.StartIndex);
31+
Assert.AreEqual(5, nodes[0].Span.End.Line);
32+
Assert.AreEqual(0, nodes[0].Span.End.Column);
33+
Assert.AreEqual(script.Length - "\r\n".Length - 1, nodes[0].Span.EndIndex);
34+
}
35+
36+
[TestMethod]
37+
public void NestedTest()
38+
{
39+
var script = "#ifdef WITH_SERVER\r\n" +
40+
" #if RELEASE\r\n" +
41+
" #elif DEBUG\r\n" +
42+
" #endif // !RELEASE\r\n" +
43+
"#endif // WITH_SERVER\r\n";
44+
45+
var lexer = new UnrealAngelscriptLexer(new AntlrInputStream(script));
46+
var tokenStream = new CommonTokenStream(lexer);
47+
tokenStream.Fill();
48+
49+
var nodes = PreprocessorParser.Parse(tokenStream);
50+
Assert.HasCount(1, nodes);
51+
Assert.AreEqual("#ifdef WITH_SERVER", nodes[0].Directive);
52+
Assert.AreEqual("#if RELEASE", nodes[0].Children[0].Directive);
53+
Assert.AreEqual("#elif DEBUG", nodes[0].Children[0].Children[0].Directive);
54+
Assert.AreEqual(0, nodes[0].Span.End.Column);
55+
Assert.AreEqual(script.Length - "\r\n".Length - 1, nodes[0].Span.EndIndex);
56+
}
57+
}
58+
}

‎ServerCodeExcisionCommon/SourcePosition.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
namespace ServerCodeExcisionCommon
2+
{
3+
/// <summary>
4+
/// Represents a position in source code (line and column).
5+
/// Lines and columns are 1-based to match editor conventions.
6+
/// </summary>
7+
public readonly struct SourcePosition
8+
{
9+
public int Line { get; }
10+
11+
public int Column { get; }
12+
13+
public SourcePosition(int line, int column)
14+
{
15+
Line = line;
16+
Column = column;
17+
}
18+
19+
public override string ToString() => $"({Line}:{Column})";
20+
}
21+
}

‎ServerCodeExcisionCommon/SourceSpan.cs‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
using System;
2+
3+
namespace ServerCodeExcisionCommon
4+
{
5+
/// <summary>
6+
/// Represents a span of source code from a start position to an end position.
7+
/// </summary>
8+
public struct SourceSpan
9+
{
10+
public SourcePosition Start { get; set; }
11+
12+
public SourcePosition End { get; set; }
13+
14+
/// <summary>
15+
/// The absolute start index in the source text (0-based).
16+
/// </summary>
17+
public int StartIndex { get; set; }
18+
19+
/// <summary>
20+
/// The absolute end index in the source text (0-based, exclusive).
21+
/// </summary>
22+
public int EndIndex { get; set; }
23+
24+
public SourceSpan(SourcePosition start, SourcePosition end, int startIndex, int endIndex)
25+
{
26+
if (startIndex > endIndex)
27+
{
28+
throw new ArgumentException($"{nameof(startIndex)} is greater than {nameof(endIndex)}");
29+
}
30+
31+
Start = start;
32+
End = end;
33+
StartIndex = startIndex;
34+
EndIndex = endIndex;
35+
}
36+
37+
public SourceSpan(int startLine, int startColumn, int endLine, int endColumn, int startIndex, int endIndex)
38+
: this(new SourcePosition(startLine, startColumn), new SourcePosition(endLine, endColumn), startIndex, endIndex)
39+
{
40+
}
41+
}
42+
}

0 commit comments

Comments
 (0)