Fix/bug 49 (#117)

* #49 - рефакторинг на строго типизированные теории

* #49 - исправление бага с помощью lookahead и lookbehind

Author: Stepami

src/Domain/HydraScript.Domain.FrontEnd/Lexer/TokenTypesJson.cs

@@ -31,12 +31,12 @@ public static class TokenTypesJson
             },
             {
                 "tag": "NullLiteral",
-                "pattern": "null",
+                "pattern": "(?<![a-zA-Z0-9])(null)(?![a-zA-Z0-9])",
                 "priority": 4
             },
             {
                 "tag": "BooleanLiteral",
-                "pattern": "true|false",
+                "pattern": "(?<![a-zA-Z0-9])(true|false)(?![a-zA-Z0-9])",
                 "priority": 5
             },
             {
@@ -46,7 +46,7 @@ public static class TokenTypesJson
             },
             {
                 "tag": "Keyword",
-                "pattern": "let|const|function|if|else|while|break|continue|return|as|type",
+                "pattern": "(?<![a-zA-Z0-9])(let|const|function|if|else|while|break|continue|return|as|type)(?![a-zA-Z0-9])",
                 "priority": 11
             },
             {

tests/HydraScript.Tests/TestData/LexerData.cs

@@ -1,27 +1,36 @@
-using System.Collections;
+using Xunit;
 
 namespace HydraScript.Tests.TestData;
 
-public class LexerSuccessData : IEnumerable<object[]>
+public class LexerSuccessData : TheoryData<string>
 {
-    public IEnumerator<object[]> GetEnumerator()
+    public LexerSuccessData()
     {
-        yield return new object[] { "a + b - c return while do" };
-        yield return new object[] { "=> abc null true false" };
-        yield return new object[] { "{ . } , ( ) [] =?:" };
+        Add("a + b - c return while do");
+        Add("=> abc null true false");
+        Add("{ . } , ( ) [] =?:");
     }
-
-    IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
 }
 
-public class LexerFailData : IEnumerable<object[]>
+public class LexerFailData : TheoryData<string>
 {
-    public IEnumerator<object[]> GetEnumerator()
+    public LexerFailData()
     {
-        yield return new object[] { "a + v $$$" };
-        yield return new object[] { "kkk &" };
-        yield return new object[] { "|| |" };
+        Add("a + v $$$");
+        Add("kkk &");
+        Add("|| |");
     }
+}
 
-    IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+public class LexerKeywordInsideIdentData : TheoryData<string>
+{
+    public LexerKeywordInsideIdentData()
+    {
+        Add("constExpr");
+        Add("letVarConst");
+        Add("nullObj");
+        Add("trueStmt");
+        Add("falseStmt");
+        Add("returnResult");
+    }
 }

tests/HydraScript.Tests/Unit/FrontEnd/RegexLexerTests.cs

@@ -1,5 +1,6 @@
 using HydraScript.Domain.FrontEnd.Lexer;
 using HydraScript.Domain.FrontEnd.Lexer.Impl;
+using HydraScript.Domain.FrontEnd.Lexer.TokenTypes;
 using HydraScript.Infrastructure;
 using HydraScript.Tests.TestData;
 using Xunit;
@@ -44,4 +45,12 @@ public void GetTokensSkipIgnorableTypesTest()
         var tokens = _regexLexer.GetTokens(text);
         Assert.DoesNotContain(_regexLexer.Structure.FindByTag("Comment"), tokens.Select(x => x.Type));
     }
+
+    [Theory, ClassData(typeof(LexerKeywordInsideIdentData))]
+    public void GetTokens_KeywordInsideIdent_Ident(string input)
+    {
+        var tokens = _regexLexer.GetTokens(input);
+        var token = tokens.First();
+        token.Type.Should().Be(new TokenType("Ident"));
+    }
 }