Hacky template parser

Author: nblumhardt

src/Serilog.Expressions/Expressions/Parsing/ExpressionParser.cs

@@ -5,7 +5,7 @@ namespace Serilog.Expressions.Parsing
 {
     static class ExpressionParser
     {
-        static ExpressionTokenizer Tokenizer { get; } = new ExpressionTokenizer(false);
+        static ExpressionTokenizer Tokenizer { get; } = new ExpressionTokenizer();
 
         public static Expression Parse(string filterExpression)
         {

src/Serilog.Expressions/Expressions/Parsing/ExpressionTokenizer.cs

@@ -1,13 +1,12 @@
 using System.Collections.Generic;
+using System.Linq;
 using Superpower;
 using Superpower.Model;
 
 namespace Serilog.Expressions.Parsing
 {
     class ExpressionTokenizer : Tokenizer<ExpressionToken>
     {
-        readonly bool _templateSyntax;
-        
         static readonly ExpressionToken[] SimpleOps = new ExpressionToken[128];
 
         static readonly ExpressionKeyword[] Keywords =
@@ -44,78 +43,17 @@ static ExpressionTokenizer()
             SimpleOps['?'] = ExpressionToken.QuestionMark;
         }
 
-        public ExpressionTokenizer(bool templateSyntax)
+        public TokenList<ExpressionToken> GreedyTokenize(TextSpan textSpan)
         {
-            _templateSyntax = templateSyntax;
+            // Dropping error info off for now
+            return new TokenList<ExpressionToken>(
+                Tokenize(textSpan)
+                    .TakeWhile(r => r.HasValue)
+                    .Select(r => new Token<ExpressionToken>(r.Value, r.Location.Until(r.Remainder)))
+                    .ToArray());
         }
 
         protected override IEnumerable<Result<ExpressionToken>> Tokenize(TextSpan stringSpan)
-        {
-            return _templateSyntax ? TokenizeTemplate(stringSpan) : TokenizeExpression(stringSpan);
-        }
-
-        IEnumerable<Result<ExpressionToken>> TokenizeTemplate(TextSpan stringSpan)
-        {
-            var last = stringSpan;
-            var next = SkipLiteral(stringSpan);
-            if (!next.HasValue)
-                yield break;
-
-            do
-            {
-                yield return Result.Value(ExpressionToken.TemplateLiteral, last, next.Location);
-
-                if (next.Value == '{')
-                {
-                    next = next.Remainder.ConsumeChar();
-                    if (!next.HasValue)
-                    {
-                        yield return Result.Empty<ExpressionToken>(next.Location, "Unexpected end-of-input.");
-                        yield break;
-                    }
-
-                    if (next.Value == '{')
-                    {
-                        yield return Result.Value(ExpressionToken.TemplateLiteral, next.Location, next.Remainder);
-                    }
-                    else
-                    {
-                        var until = next.Location;
-                        
-                        // Here we'll need to contextually apply the parser...
-                        foreach (var tok in TokenizeExpression(next.Location))
-                        {
-                            yield return tok;
-                            until = tok.Remainder;
-                        }
-                        
-                        next = until.ConsumeChar();
-                        if (!next.HasValue)
-                        {
-                            yield return Result.Empty<ExpressionToken>(next.Location, "Unexpected end-of-input.");
-                            yield break;
-                        }
-                    }
-                }
-                else if (next.Value == '}')
-                {
-                    
-                }
-
-                last = next.Remainder;
-                next = SkipLiteral(next.Remainder);
-            } while (next.HasValue);
-        }
-
-        static Result<char> SkipLiteral(TextSpan stringSpan)
-        {
-            Result<char> result = stringSpan.ConsumeChar();
-            while (result.HasValue && result.Value != '{' && result.Value != '}')
-                result = result.Remainder.ConsumeChar();
-            return result;
-        }
-        
-        IEnumerable<Result<ExpressionToken>> TokenizeExpression(TextSpan stringSpan)
         {
             var next = SkipWhiteSpace(stringSpan);
             if (!next.HasValue)

src/Serilog.Expressions/Templates/Parsing/TemplateParser.cs

@@ -1,37 +1,77 @@
+using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text;
 using Serilog.Expressions.Parsing;
 using Serilog.Templates.Ast;
+using Superpower.Model;
 
 namespace Serilog.Templates.Parsing
 {
     static class TemplateParser
     {
-        static ExpressionTokenizer Tokenizer { get; } = new ExpressionTokenizer(true);
+        static ExpressionTokenizer Tokenizer { get; } = new ExpressionTokenizer();
 
         public static Template Parse(string template)
         {
-            var tokens = Tokenizer.Tokenize(template);
-
+            if (template == null) throw new ArgumentNullException(nameof(template));
+            
             var elements = new List<Template>();
 
-            var rest = tokens;
-            while (!rest.IsAtEnd)
+            var i = 0;
+            while (i < template.Length)
             {
-                var peek = rest.ConsumeToken();
-                if (peek.Value.Kind == ExpressionToken.TemplateLiteral)
+                var ch = template[i];
+
+                if (ch == '{')
                 {
-                    elements.Add(new LiteralText(peek.Value.ToStringValue()));
-                    rest = peek.Remainder;
+                    i++;
+                    if (i == template.Length)
+                        throw new ArgumentException("Character `{` must be escaped by doubling in literal text.");
+
+                    if (template[i] == '{')
+                    {
+                        elements.Add(new LiteralText("}"));
+                        i++;
+                    }
+                    else
+                    {
+                        // Not reporting line/column
+                        var tokens = Tokenizer.GreedyTokenize(new TextSpan(template, new Position(i, 0, 0), template.Length - i));
+                        // Dropping error info; may return a zero-length parse
+                        var expr = ExpressionTokenParsers.TryPartialParse(tokens);
+                        // Throw on error; no format parsing
+                        elements.Add(new FormattedExpression(expr.Value, null));
+                        if (expr.Remainder.Position == tokens.Count())
+                            i = tokens.Last().Position.Absolute + tokens.Last().Span.Length;
+                        else
+                            i = tokens.ElementAt(i).Position.Absolute;
+
+                        if (i == template.Length || template[i] != '}')
+                            throw new ArgumentException("Un-closed hole, `}` expected.");
+                        i++;
+                    }
+                }
+                else if (ch == '}')
+                {
+                    i++;
+                    if (i == template.Length || template[i] != '}')
+                        throw new ArgumentException("Character `}` must be escaped by doubling in literal text.");
+                    elements.Add(new LiteralText("}"));
+                    i++;
                 }
                 else
                 {
-                    var partial = ExpressionTokenParsers.TryPartialParse(rest);
-                    elements.Add(new FormattedExpression(partial.Value, null));
-                    rest = partial.Remainder;
+                    var literal = new StringBuilder();
+                    do
+                    {
+                        literal.Append(template[i]);
+                        i++;
+                    } while (i < template.Length && template[i] != '{' && template[i] != '}');
+                    elements.Add(new LiteralText(literal.ToString()));
                 }
             }
-
+            
             if (elements.Count == 1)
                 return elements.Single();