Merge pull request #17 from EmbarkStudios/webbju/fix-unit-tests

webbju · web-flow · commit 431958dac338 · 2025-12-03T15:58:32.000+01:00
Strict-mode support and unit-test fixes.
diff --git a/ServerCodeExciser/ServerCodeExciser.cs b/ServerCodeExciser/ServerCodeExciser.cs
@@ -54,9 +54,13 @@ public sealed class Settings : CommandSettings
             public bool IsDryRun { get; init; }
 
             [CommandOption("-v|--verify")]
-            [Description("Verify that all analized code does not require modifications to excise server scopes.")]
+            [Description("Verify that all analyzed code does not require modifications to excise server scopes.")]
             public bool Verify { get; init; }
 
+            [CommandOption("--strict")]
+            [Description("Ensure that all files can be analyzed without syntactic or lexicographical errors.")]
+            public bool StrictMode { get; init; }
+
             [CommandArgument(0, "[INPUT]")]
             [Description("The input folder to excise.")]
             public string InputPath { get; init; } = string.Empty;
@@ -87,6 +91,7 @@ public override int Execute([NotNull] CommandContext context, [NotNull] Settings
             parameters.ShouldOutputUntouchedFiles = settings.ShouldOutputUntouchedFiles;
             parameters.IsDryRun = settings.IsDryRun || settings.ShouldOutputUntouchedFiles || settings.Verify;
             parameters.Verify = settings.Verify;
+            parameters.StrictMode = settings.StrictMode;
             parameters.UseFunctionStats = settings.UseFunctionStats;
             parameters.DontSkip = settings.DontSkip;
             if (settings.RequiredExcisionRatio.HasValue)
diff --git a/ServerCodeExciser/ServerCodeExcisionProcessor.cs b/ServerCodeExciser/ServerCodeExcisionProcessor.cs
@@ -17,6 +17,7 @@ public class ServerCodeExcisionParameters
         public bool ShouldOutputUntouchedFiles { get; set; } = false;
         public bool IsDryRun { get; set; } = false;
         public bool Verify { get; set; } = false;
+        public bool StrictMode { get; set; } = false;
         public bool UseFunctionStats { get; set; } = false;
         public bool DontSkip { get; set; } = false;
         public float RequiredExcisionRatio { get; set; } = -1.0f;
@@ -109,8 +110,7 @@ public EExciserReturnValues ExciseServerCode(string filePattern, IServerCodeExci
                         {
                             System.Diagnostics.Debug.Assert(stats.TotalNrCharacters > 0, "Something is terribly wrong. We have excised characters, but no total characters..?");
                             var excisionRatio = (float)stats.CharactersExcised / (float)stats.TotalNrCharacters * 100.0f;
-                            Console.WriteLine("Excised {0:0.00}% of server only code in file ({1}/{2}): {3}",
-                                    excisionRatio, fileIdx + 1, allFiles.Length, fileName);
+                            Console.WriteLine("Excised {0:0.00}% of server only code in file ({1}/{2}): {3}", excisionRatio, fileIdx + 1, allFiles.Length, fileName);
                         }
                         else
                         {
@@ -120,9 +120,10 @@ public EExciserReturnValues ExciseServerCode(string filePattern, IServerCodeExci
                         globalStats.CharactersExcised += stats.CharactersExcised;
                         globalStats.TotalNrCharacters += stats.TotalNrCharacters;
                     }
-                    catch (Exception e)
+                    catch (Exception)
                     {
                         Console.WriteLine("Failed to parse ({0}/{1}): {2}", fileIdx + 1, allFiles.Length, fileName);
+                        throw;
                     }
                 }
             }
@@ -183,10 +184,17 @@ private ExcisionStats ProcessCodeFile(string fileName, string inputPath, EExcisi
             // Setup parsing and output.
             List<KeyValuePair<int, string>> serverCodeInjections = new List<KeyValuePair<int, string>>();
             var inputStream = new AntlrInputStream(script);
-            var lexer = excisionLanguage.CreateLexer(inputStream);
+            var lexer = excisionLanguage.CreateLexer<UnrealAngelscriptLexer>(inputStream);
             lexer.AddErrorListener(new ExcisionLexerErrorListener());
             var commonTokenStream = new CommonTokenStream(lexer);
-            var parser = excisionLanguage.CreateParser(commonTokenStream);
+            var parser = excisionLanguage.CreateParser<UnrealAngelscriptParser>(commonTokenStream);
+            parser.AddErrorListener(new ExcisionParserErrorListener());
+
+            if (_parameters.StrictMode)
+            {
+                parser.ErrorHandler = new BailErrorStrategy();
+            }
+
             var answerText = new StringBuilder();
             answerText.Append(script);
 
@@ -195,7 +203,7 @@ private ExcisionStats ProcessCodeFile(string fileName, string inputPath, EExcisi
             {
                 // We want to excise this entire file.
                 serverCodeInjections.Add(new KeyValuePair<int, string>(0, excisionLanguage.ServerScopeStartString + "\r\n"));
-                serverCodeInjections.Add(new KeyValuePair<int, string>(script.Length, excisionLanguage.ServerScopeEndString));
+                serverCodeInjections.Add(new KeyValuePair<int, string>(script.Length, excisionLanguage.ServerScopeEndString + "\r\n"));
                 stats.CharactersExcised += script.Length;
             }
             else if (excisionMode == EExcisionMode.AllFunctions)
@@ -221,7 +229,8 @@ private ExcisionStats ProcessCodeFile(string fileName, string inputPath, EExcisi
             // Gather all the injections we want to make
             if (visitor != null)
             {
-                visitor.VisitContext(parser.GetParseTree());
+                visitor.VisitContext(parser.script());
+
                 if (_parameters.UseFunctionStats)
                 {
                     stats.TotalNrCharacters = visitor.TotalNumberOfFunctionCharactersVisited;
@@ -232,17 +241,17 @@ private ExcisionStats ProcessCodeFile(string fileName, string inputPath, EExcisi
                 {
                     if (currentScope.StartIndex == -1
                         || currentScope.StopIndex == -1
-                        || InjectedMacroAlreadyExistsAtLocation(answerText, currentScope.StartIndex, true, excisionLanguage.ServerScopeStartString)
-                        || InjectedMacroAlreadyExistsAtLocation(answerText, currentScope.StartIndex, false, excisionLanguage.ServerScopeStartString)
-                        || InjectedMacroAlreadyExistsAtLocation(answerText, currentScope.StopIndex, false, excisionLanguage.ServerScopeEndString))
+                        || InjectedMacroAlreadyExistsAtLocation(answerText, currentScope.StartIndex, true, true, excisionLanguage.ServerScopeStartString)
+                        || InjectedMacroAlreadyExistsAtLocation(answerText, currentScope.StartIndex, false, false, excisionLanguage.ServerScopeStartString)
+                        || InjectedMacroAlreadyExistsAtLocation(answerText, currentScope.StopIndex, false, false, excisionLanguage.ServerScopeEndString))
                     {
                         continue;
                     }
 
                     // If there are already injected macros where we want to go, we should skip injecting.
                     System.Diagnostics.Debug.Assert(currentScope.StopIndex > currentScope.StartIndex, "There must be some invalid pattern here! Stop is before start!");
-                    serverCodeInjections.Add(new KeyValuePair<int, string>(currentScope.StartIndex, excisionLanguage.ServerScopeStartString));
-                    serverCodeInjections.Add(new KeyValuePair<int, string>(currentScope.StopIndex, currentScope.Opt_ElseContent + excisionLanguage.ServerScopeEndString));
+                    serverCodeInjections.Add(new KeyValuePair<int, string>(currentScope.StartIndex, "\r\n" + excisionLanguage.ServerScopeStartString));
+                    serverCodeInjections.Add(new KeyValuePair<int, string>(currentScope.StopIndex, currentScope.Opt_ElseContent + excisionLanguage.ServerScopeEndString + "\r\n"));
                     stats.CharactersExcised += currentScope.StopIndex - currentScope.StartIndex;
                 }
 
@@ -257,10 +266,10 @@ private ExcisionStats ProcessCodeFile(string fileName, string inputPath, EExcisi
                         dummyRefDataBlockString.Append("\r\n\t" + dummyVarDef);
                     }
 
-                    dummyRefDataBlockString.Append("\r\n" + excisionLanguage.ServerScopeEndString + "\r\n");
+                    dummyRefDataBlockString.Append("\r\n" + excisionLanguage.ServerScopeEndString + "\r\n\r\n");
 
                     // If there is already a block of dummy reference variables we skip adding new ones, there is no guarantee we are adding the right code.
-                    if (InjectedMacroAlreadyExistsAtLocation(answerText, dummyRefDataPair.Key, false, dummyVarScope + "\r\n"))
+                    if (InjectedMacroAlreadyExistsAtLocation(answerText, dummyRefDataPair.Key, false, true, dummyVarScope + "\r\n"))
                     {
                         continue;
                     }
@@ -314,19 +323,47 @@ private ExcisionStats ProcessCodeFile(string fileName, string inputPath, EExcisi
             return stats;
         }
 
-        private bool InjectedMacroAlreadyExistsAtLocation(StringBuilder script, int index, bool lookAhead, string macro)
+        private static bool IsWhitespace(char c)
         {
-            int startIndex = lookAhead ? index : (index - macro.Length);
-            int endIndex = lookAhead ? (index + macro.Length) : index;
+            return c == ' ' || c == '\t' || c == '\r' || c == '\n';
+        }
 
-            if (startIndex < 0 || startIndex >= script.Length
-                || endIndex < 0 || endIndex >= script.Length)
+        private bool InjectedMacroAlreadyExistsAtLocation(StringBuilder script, int index, bool lookAhead, bool ignoreWhitespace, string macro)
+        {
+            if (lookAhead)
             {
-                return false;
+                if (ignoreWhitespace)
+                {
+                    while (index < script.Length && IsWhitespace(script[index]))
+                    {
+                        index++;
+                    }
+                }
+
+                if (script.Length - index < macro.Length)
+                {
+                    return false;
+                }
+
+                return script.ToString(index, macro.Length).Equals(macro);
             }
+            else
+            {
+                if (ignoreWhitespace)
+                {
+                    while (index > 0 && IsWhitespace(script[index]))
+                    {
+                        index--;
+                    }
+                }
+
+                if (index - macro.Length < 0)
+                {
+                    return false;
+                }
 
-            string scriptSection = script.ToString(startIndex, macro.Length);
-            return scriptSection == macro;
+                return script.ToString(index - macro.Length, macro.Length).Equals(macro);
+            }
         }
     }
 }
diff --git a/ServerCodeExciserTest/ExcisionIntegrationTests.cs b/ServerCodeExciserTest/ExcisionIntegrationTests.cs
@@ -34,7 +34,7 @@ public static int Main(string[] args)
                 ref numTestCases);
 
             Console.WriteLine("----------------------------");
-            Console.WriteLine($"{numTestCases - numTestFailures} test(s) passed.");
+            Console.WriteLine($"{numTestCases} test(s) executed.");
             Console.WriteLine($"{numTestFailures} test(s) failed.");
         }
         catch (Exception e)
@@ -54,7 +54,8 @@ private static EExciserReturnValues RunExciserIntegrationTests(string fileExtens
             Directory.Delete(outputPath, true);
         }
 
-        string searchPattern = "*" + fileExtension.TrimStart('.');
+        string searchPattern = "*." + fileExtension.TrimStart('.');
+        Console.WriteLine($"Running integration tests for {searchPattern} files...");
 
         EExciserReturnValues returnCode;
         try
@@ -65,17 +66,17 @@ private static EExciserReturnValues RunExciserIntegrationTests(string fileExtens
                 ShouldOutputUntouchedFiles = true,
                 FullExcisionRegexString = @"FullExcise1/.*",
                 ExciseAllFunctionsRegexString = @"AllFunctionExcise1/.*|||AllFunctionExcise2/.*",
+                StrictMode = true,
             };
             excisionParams.InputPaths.Add(inputPath);
 
             var angelscriptServerCodeExciser = new ServerCodeExcisionProcessor(excisionParams);
             returnCode = angelscriptServerCodeExciser.ExciseServerCode(searchPattern, new UnrealAngelscriptServerCodeExcisionLanguage());
-            Console.WriteLine($"ExciseServerCode for {fileExtension} files returned: {returnCode}");
+            Console.WriteLine($"ExciseServerCode for {searchPattern} files returned: {returnCode}");
         }
-        catch (Exception e)
+        catch (Exception)
         {
-            AnsiConsole.WriteException(e);
-            return EExciserReturnValues.InternalExcisionError;
+            throw;
         }
 
         foreach (var answerFilePath in Directory.EnumerateFiles(outputPath, searchPattern, SearchOption.AllDirectories))
diff --git a/ServerCodeExciserTest/Problems/Common/DeclarationInFunctionRootScopeWithReturnValTest.common b/ServerCodeExciserTest/Problems/Common/DeclarationInFunctionRootScopeWithReturnValTest.common
@@ -50,7 +50,7 @@ class UDeclarationInFunctionRootScopeWithReturnValTest
 	{
 		ensure(System::IsServer());
 
-		float64 ThisMustBeGuarded = 0.0f;
+		float64 ThisMustBeGuarded = 0.0;
 		ThisMustBeGuarded--;
 
 		return ThisMustBeGuarded;
diff --git a/ServerCodeExciserTest/Problems/Common/FullExcise1/FullExcise1Test.common.solution b/ServerCodeExciserTest/Problems/Common/FullExcise1/FullExcise1Test.common.solution
@@ -1,4 +1,3 @@
-
 #ifdef WITH_SERVER
 class UFullExcise1Test
 {
diff --git a/ServerCodeExcisionCommon/IServerCodeExcisionLanguage.cs b/ServerCodeExcisionCommon/IServerCodeExcisionLanguage.cs
@@ -9,16 +9,23 @@ public interface IServerCodeExcisionLanguage
 		List<string> ServerOnlySymbols { get; }
 
 		string ServerPrecompilerSymbol { get; }
+
 		string ServerScopeStartString { get; }
+
 		string ServerScopeEndString { get; }
 
-		Antlr4.Runtime.Lexer CreateLexer(Antlr4.Runtime.AntlrInputStream inputStream);
-		IServerCodeParser CreateParser(Antlr4.Runtime.CommonTokenStream tokenStream);
-		
+		T CreateLexer<T>(Antlr4.Runtime.AntlrInputStream inputStream)
+			where T : Antlr4.Runtime.Lexer;
+
+		T CreateParser<T>(Antlr4.Runtime.CommonTokenStream tokenStream)
+			where T : Antlr4.Runtime.Parser;
+
 		IServerCodeVisitor CreateSimpleVisitor(string code);
+
 		IServerCodeVisitor CreateFunctionVisitor(string code);
+
 		IServerCodeVisitor CreateSymbolVisitor(string code);
-		
+
 		bool AnyServerOnlySymbolsInScript(string script);
 	}
 }
diff --git a/ServerCodeExcisionCommon/IServerCodeParser.cs b/ServerCodeExcisionCommon/IServerCodeParser.cs
diff --git a/ServerCodeExcisionCommon/ServerCodeExcisionUtils.cs b/ServerCodeExcisionCommon/ServerCodeExcisionUtils.cs
@@ -89,7 +89,10 @@ public enum EExciserReturnValues
 
 	public class ExcisionException : Exception
 	{
-		public ExcisionException(string excisionError, Exception innerException) : base(excisionError, innerException) {}
+		public ExcisionException(string excisionError, Exception innerException)
+			: base(excisionError, innerException)
+		{
+		}
 	}
 
 	public class ExcisionParserErrorListener : Antlr4.Runtime.IAntlrErrorListener<Antlr4.Runtime.IToken>
@@ -143,24 +146,6 @@ public static int FindScriptIndexForCodePoint(string script, int line, int colum
 			return (linesTraversed == line) ? (cursor + column) : -1;
 		}
 
-		public static int ShrinkServerScope(string script, int start, int end)
-		{
-			bool skip = true;
-			while (skip)
-			{
-				skip = false;
-
-				int search = script.IndexOfAny(NewLineChars, start) + 2;
-				if ((search < end) && SkippableScopeChars.Contains<char>(script.ElementAt(search)))
-				{
-					skip = true;
-					++start;
-				}
-			}
-
-			return start;
-		}
-
 		public static Type FindFirstDirectChildOfType<Type>(Antlr4.Runtime.Tree.IParseTree currentContext)
 			where Type : class
 		{
diff --git a/UnrealAngelscriptParser/Grammar/UnrealAngelscriptParser.g4 b/UnrealAngelscriptParser/Grammar/UnrealAngelscriptParser.g4
@@ -496,6 +496,8 @@ memberSpecification:
 
 memberdeclaration:
 	propertyDefinition
+	| classSpecifier
+	| enumSpecifier
 	| functionDefinition
 	| aliasDeclaration
 	| emptyDeclaration
diff --git a/UnrealAngelscriptServerCodeExcision/UnrealAngelsciptServerCodeParser.cs b/UnrealAngelscriptServerCodeExcision/UnrealAngelsciptServerCodeParser.cs
diff --git a/UnrealAngelscriptServerCodeExcision/UnrealAngelscriptServerCodeExcisionLanguage.cs b/UnrealAngelscriptServerCodeExcision/UnrealAngelscriptServerCodeExcisionLanguage.cs
@@ -1,5 +1,6 @@
-using System.Collections.Generic;
 using ServerCodeExcisionCommon;
+using System;
+using System.Collections.Generic;
 
 namespace UnrealAngelscriptServerCodeExcision
 {
@@ -10,27 +11,33 @@ public class UnrealAngelscriptServerCodeExcisionLanguage : IServerCodeExcisionLa
 			@"^System::IsServer\(\)$",
 			@"^[A-z]+\.HasAuthority\(\)$"
 		};
+
 		public List<string> ServerOnlySymbolRegexes	{ get { return _angelscriptServerOnlySymbolRegexes; } }
 
 		private List<string> _angelscriptServerOnlySymbols = new List<string>
 		{
 			"hasauthority()",
 			"server"
 		};
+
 		public List<string> ServerOnlySymbols { get { return _angelscriptServerOnlySymbols; } }
 
 		public string ServerPrecompilerSymbol { get { return "WITH_SERVER"; } }
-		public string ServerScopeStartString { get { return "\r\n#ifdef " + ServerPrecompilerSymbol; } }
-		public string ServerScopeEndString { get { return "#endif // " + ServerPrecompilerSymbol + "\r\n"; } }
 
-		public Antlr4.Runtime.Lexer CreateLexer(Antlr4.Runtime.AntlrInputStream inputStream)
-		{
-			return new UnrealAngelscriptLexer(inputStream);
+		public string ServerScopeStartString { get { return "#ifdef " + ServerPrecompilerSymbol; } }
+
+		public string ServerScopeEndString { get { return "#endif // " + ServerPrecompilerSymbol; } }
+
+		public T CreateLexer<T>(Antlr4.Runtime.AntlrInputStream inputStream)
+            where T : Antlr4.Runtime.Lexer
+        {
+			return (T)Activator.CreateInstance(typeof(T), inputStream);
 		}
 
-		public IServerCodeParser CreateParser(Antlr4.Runtime.CommonTokenStream tokenStream)
+		public T CreateParser<T>(Antlr4.Runtime.CommonTokenStream tokenStream)
+			where T : Antlr4.Runtime.Parser
 		{
-			return new UnrealAngelscriptServerCodeParser(tokenStream);
+			return (T)Activator.CreateInstance(typeof(T), tokenStream);
 		}
 
 		public IServerCodeVisitor CreateSimpleVisitor(string code)
diff --git a/UnrealAngelscriptServerCodeExcision/UnrealAngelscriptSimpleVisitor.cs b/UnrealAngelscriptServerCodeExcision/UnrealAngelscriptSimpleVisitor.cs
diff --git a/UnrealAngelscriptServerCodeExcision/UnrealAngelscriptSymbolVisitor.cs b/UnrealAngelscriptServerCodeExcision/UnrealAngelscriptSymbolVisitor.cs

Original file line number	Diff line number	Diff line change
`@@ -50,7 +50,7 @@ class UDeclarationInFunctionRootScopeWithReturnValTest`
`50`	`50`	`{`
`51`	`51`	`ensure(System::IsServer());`
`52`	`52`
`53`		`- float64 ThisMustBeGuarded = 0.0f;`
	`53`	`+ float64 ThisMustBeGuarded = 0.0;`
`54`	`54`	`ThisMustBeGuarded--;`
`55`	`55`
`56`	`56`	`return ThisMustBeGuarded;`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-`
`2`	`1`	`#ifdef WITH_SERVER`
`3`	`2`	`class UFullExcise1Test`
`4`	`3`	`{`