lexer optimizing refactoring (#211)

Author: Stepami

src/Domain/HydraScript.Domain.FrontEnd/Lexer/ILexer.cs

@@ -4,5 +4,5 @@ public interface ILexer
 {
     public IStructure Structure { get; }
 
-    public List<Token> GetTokens(string text);
+    public IEnumerable<Token> GetTokens(string text);
 }

src/Domain/HydraScript.Domain.FrontEnd/Lexer/Impl/RegexLexer.cs

@@ -1,28 +1,16 @@
-using System.Collections;
-using System.Diagnostics.CodeAnalysis;
 using System.Text.RegularExpressions;
-using Cysharp.Text;
 
 namespace HydraScript.Domain.FrontEnd.Lexer.Impl;
 
-public class RegexLexer(IStructure structure, ITextCoordinateSystemComputer computer) : ILexer, IEnumerable<Token>
+public class RegexLexer(IStructure structure, ITextCoordinateSystemComputer computer) : ILexer
 {
-    private IReadOnlyList<int> _lines = [];
-    private string _text = "";
-
     public IStructure Structure { get; } = structure;
 
-    public List<Token> GetTokens(string text)
+    public IEnumerable<Token> GetTokens(string text)
     {
-        _text = text;
-        _lines = computer.GetLines(_text);
-
-        return this.ToList();
-    }
+        var lines = computer.GetLines(text);
 
-    public IEnumerator<Token> GetEnumerator()
-    {
-        foreach (Match match in Structure.Regex.Matches(_text))
+        foreach (Match match in Structure.Regex.Matches(text))
         {
             for (var i = 0; i < Structure.Count; i++)
             {
@@ -33,8 +21,8 @@ public IEnumerator<Token> GetEnumerator()
 
                 var value = group.Value;
                 var segment = new Segment(
-                    computer.GetCoordinates(group.Index, _lines),
-                    computer.GetCoordinates(absoluteIndex: group.Index + group.Length, _lines));
+                    computer.GetCoordinates(group.Index, lines),
+                    computer.GetCoordinates(absoluteIndex: group.Index + group.Length, lines));
                 var token = new Token(type, segment, value);
 
                 if (type.Error()) throw new LexerException(token);
@@ -45,9 +33,4 @@ public IEnumerator<Token> GetEnumerator()
 
         yield return new EndToken();
     }
-
-    [ExcludeFromCodeCoverage]
-    IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
-
-    public override string ToString() => ZString.Join<Token>('\n', this);
 }

src/Domain/HydraScript.Domain.FrontEnd/Parser/Impl/TokensStream.cs

@@ -14,17 +14,14 @@
 
 namespace HydraScript.Domain.FrontEnd.Parser.Impl;
 
-public class TopDownParser : IParser
+public class TopDownParser(ILexer lexer) : IParser
 {
-    private TokensStream _tokens = new List<Token>();
-    private readonly ILexer _lexer;
-
-    public TopDownParser(ILexer lexer) => 
-        _lexer = lexer;
+    private IEnumerator<Token> _tokens = Enumerable.Empty<Token>().GetEnumerator();
 
     public IAbstractSyntaxTree Parse(string text)
     {
-        _tokens = _lexer.GetTokens(text);
+        _tokens = lexer.GetTokens(text).GetEnumerator();
+        _tokens.MoveNext();
 
         var root = Script();
         Expect(Eop.Tag);
@@ -45,7 +42,7 @@ private Token Expect(string expectedTag, string? expectedValue = null)
     }
 
     private bool CurrentIs(string tag) =>
-        _tokens.Current.Type == _lexer.Structure.FindByTag(tag);
+        _tokens.Current.Type == lexer.Structure.FindByTag(tag);
 
     private bool CurrentIsLiteral() =>
         CurrentIs("NullLiteral") ||

src/Domain/HydraScript.Domain.FrontEnd/Parser/Impl/TopDownParser.cs

@@ -1,4 +1,5 @@
 using System.Diagnostics.CodeAnalysis;
+using Cysharp.Text;
 using HydraScript.Domain.FrontEnd.Lexer;
 using Microsoft.Extensions.DependencyInjection;
 
@@ -12,10 +13,10 @@ internal class DumpingLexer(
     [ExcludeFromCodeCoverage]
     public IStructure Structure => lexer.Structure;
 
-    public List<Token> GetTokens(string text)
+    public IEnumerable<Token> GetTokens(string text)
     {
-        var tokens = lexer.GetTokens(text);
-        dumpingService.Dump(lexer.ToString(), "tokens");
+        var tokens = lexer.GetTokens(text).ToList();
+        dumpingService.Dump(ZString.Join('\n', tokens), "tokens");
         return tokens;
     }
 }

src/Infrastructure/HydraScript.Infrastructure/Dumping/DumpingLexer.cs

@@ -20,15 +20,15 @@ public void LexerDoesNotThrowTest(string text) =>
     [Theory]
     [ClassData(typeof(LexerFailData))]
     public void LexerThrowsErrorTest(string text) =>
-        Assert.Throws<LexerException>(() => _regexLexer.GetTokens(text));
+        Assert.Throws<LexerException>(() => _regexLexer.GetTokens(text).ToList());
 
     [Fact]
     public void LexerToStringCorrectTest()
     {
         const string text = "8";
-        var tokens = _regexLexer.GetTokens(text);
-        Assert.Contains("EOP", _regexLexer.ToString());
-        Assert.Equal("IntegerLiteral (1, 1)-(1, 2): 8", tokens.First().ToString());
+        var tokens = _regexLexer.GetTokens(text).ToList();
+        Assert.Contains("EOP", tokens[^1].ToString());
+        Assert.Equal("IntegerLiteral (1, 1)-(1, 2): 8", tokens[0].ToString());
     }
 
     [Fact]
@@ -80,7 +80,7 @@ public void GetTokens_MockedRegex_ValidOutput(
         structure.Count.Returns(tokenTypes.Count);
         structure[Arg.Any<int>()].Returns(callInfo => tokenTypes[callInfo.Arg<int>()]);
 
-        var tokens = lexer.GetTokens(input.ToString());
+        var tokens = lexer.GetTokens(input.ToString()).ToList();
         for (var i = 0; i < input.Count; i++)
         {
             output.WriteLine(tokens[i].ToString());