WIP template parsing

nblumhardt · nblumhardt · commit f9f175864a21 · 2020-09-06T14:15:05.000+10:00
diff --git a/src/Serilog.Expressions/Expressions/Parsing/ExpressionParser.cs b/src/Serilog.Expressions/Expressions/Parsing/ExpressionParser.cs
@@ -5,6 +5,8 @@ namespace Serilog.Expressions.Parsing
 {
     static class ExpressionParser
     {
+        static ExpressionTokenizer Tokenizer { get; } = new ExpressionTokenizer(false);
+        
         public static Expression Parse(string filterExpression)
         {
             if (!TryParse(filterExpression, out var root, out var error))
@@ -17,7 +19,7 @@ public static bool TryParse(string filterExpression, out Expression root, out st
         {
             if (filterExpression == null) throw new ArgumentNullException(nameof(filterExpression));
 
-            var tokenList = ExpressionTokenizer.Instance.TryTokenize(filterExpression);       
+            var tokenList = Tokenizer.TryTokenize(filterExpression);       
             if (!tokenList.HasValue)
             {
                 error = tokenList.ToString();
diff --git a/src/Serilog.Expressions/Expressions/Parsing/ExpressionToken.cs b/src/Serilog.Expressions/Expressions/Parsing/ExpressionToken.cs
@@ -5,6 +5,9 @@ namespace Serilog.Expressions.Parsing
     enum ExpressionToken
     {
         None,
+        
+        [Token(Description = "literal text")]
+        TemplateLiteral,
 
         Identifier,
 
diff --git a/src/Serilog.Expressions/Expressions/Parsing/ExpressionTokenParsers.cs b/src/Serilog.Expressions/Expressions/Parsing/ExpressionTokenParsers.cs
@@ -17,6 +17,12 @@ public static TokenListParserResult<ExpressionToken, Expression> TryParse(
             return Expr.AtEnd().TryParse(input);
         }
 
+        public static TokenListParserResult<ExpressionToken, Expression> TryPartialParse(
+            TokenList<ExpressionToken> input)
+        {
+            return Expr.TryParse(input);
+        }
+
         static readonly TokenListParser<ExpressionToken, string> Add = Token.EqualTo(ExpressionToken.Plus).Value(Operators.OpAdd);
         static readonly TokenListParser<ExpressionToken, string> Subtract = Token.EqualTo(ExpressionToken.Minus).Value(Operators.OpSubtract);
         static readonly TokenListParser<ExpressionToken, string> Multiply = Token.EqualTo(ExpressionToken.Asterisk).Value(Operators.OpMultiply);
diff --git a/src/Serilog.Expressions/Expressions/Parsing/ExpressionTokenizer.cs b/src/Serilog.Expressions/Expressions/Parsing/ExpressionTokenizer.cs
@@ -6,6 +6,8 @@ namespace Serilog.Expressions.Parsing
 {
     class ExpressionTokenizer : Tokenizer<ExpressionToken>
     {
+        readonly bool _templateSyntax;
+        
         static readonly ExpressionToken[] SimpleOps = new ExpressionToken[128];
 
         static readonly ExpressionKeyword[] Keywords =
@@ -42,7 +44,78 @@ static ExpressionTokenizer()
             SimpleOps['?'] = ExpressionToken.QuestionMark;
         }
 
+        public ExpressionTokenizer(bool templateSyntax)
+        {
+            _templateSyntax = templateSyntax;
+        }
+
         protected override IEnumerable<Result<ExpressionToken>> Tokenize(TextSpan stringSpan)
+        {
+            return _templateSyntax ? TokenizeTemplate(stringSpan) : TokenizeExpression(stringSpan);
+        }
+
+        IEnumerable<Result<ExpressionToken>> TokenizeTemplate(TextSpan stringSpan)
+        {
+            var last = stringSpan;
+            var next = SkipLiteral(stringSpan);
+            if (!next.HasValue)
+                yield break;
+
+            do
+            {
+                yield return Result.Value(ExpressionToken.TemplateLiteral, last, next.Location);
+
+                if (next.Value == '{')
+                {
+                    next = next.Remainder.ConsumeChar();
+                    if (!next.HasValue)
+                    {
+                        yield return Result.Empty<ExpressionToken>(next.Location, "Unexpected end-of-input.");
+                        yield break;
+                    }
+
+                    if (next.Value == '{')
+                    {
+                        yield return Result.Value(ExpressionToken.TemplateLiteral, next.Location, next.Remainder);
+                    }
+                    else
+                    {
+                        var until = next.Location;
+                        
+                        // Here we'll need to contextually apply the parser...
+                        foreach (var tok in TokenizeExpression(next.Location))
+                        {
+                            yield return tok;
+                            until = tok.Remainder;
+                        }
+                        
+                        next = until.ConsumeChar();
+                        if (!next.HasValue)
+                        {
+                            yield return Result.Empty<ExpressionToken>(next.Location, "Unexpected end-of-input.");
+                            yield break;
+                        }
+                    }
+                }
+                else if (next.Value == '}')
+                {
+                    
+                }
+
+                last = next.Remainder;
+                next = SkipLiteral(next.Remainder);
+            } while (next.HasValue);
+        }
+
+        static Result<char> SkipLiteral(TextSpan stringSpan)
+        {
+            Result<char> result = stringSpan.ConsumeChar();
+            while (result.HasValue && result.Value != '{' && result.Value != '}')
+                result = result.Remainder.ConsumeChar();
+            return result;
+        }
+        
+        IEnumerable<Result<ExpressionToken>> TokenizeExpression(TextSpan stringSpan)
         {
             var next = SkipWhiteSpace(stringSpan);
             if (!next.HasValue)
@@ -166,7 +239,5 @@ static bool TryGetKeyword(TextSpan span, out ExpressionToken keyword)
             keyword = ExpressionToken.None;
             return false;
         }
-        
-        public static ExpressionTokenizer Instance { get; } = new ExpressionTokenizer();
     }
 }
diff --git a/src/Serilog.Expressions/Templates/Ast/FormattedExpression.cs b/src/Serilog.Expressions/Templates/Ast/FormattedExpression.cs
@@ -0,0 +1,17 @@
+using System;
+using Serilog.Expressions.Ast;
+
+namespace Serilog.Templates.Ast
+{
+    class FormattedExpression : Template
+    {
+        public Expression Expression { get; }
+        public string Format { get; }
+
+        public FormattedExpression(Expression expression, string format)
+        {
+            Expression = expression ?? throw new ArgumentNullException(nameof(expression));
+            Format = format;
+        }
+    }
+}
diff --git a/src/Serilog.Expressions/Templates/Ast/LiteralText.cs b/src/Serilog.Expressions/Templates/Ast/LiteralText.cs
@@ -0,0 +1,14 @@
+using System;
+
+namespace Serilog.Templates.Ast
+{
+    class LiteralText : Template
+    {
+        public string Text { get; }
+
+        public LiteralText(string text)
+        {
+            Text = text ?? throw new ArgumentNullException(nameof(text));
+        }
+    }
+}
diff --git a/src/Serilog.Expressions/Templates/Ast/Template.cs b/src/Serilog.Expressions/Templates/Ast/Template.cs
@@ -0,0 +1,6 @@
+namespace Serilog.Templates.Ast
+{
+    abstract class Template
+    {
+    }
+}
diff --git a/src/Serilog.Expressions/Templates/Ast/TemplateBlock.cs b/src/Serilog.Expressions/Templates/Ast/TemplateBlock.cs
@@ -0,0 +1,14 @@
+using System;
+
+namespace Serilog.Templates.Ast
+{
+    class TemplateBlock : Template
+    {
+        public Template[] Elements { get; }
+
+        public TemplateBlock(Template[] elements)
+        {
+            Elements = elements ?? throw new ArgumentNullException(nameof(elements));
+        }
+    }
+}
diff --git a/src/Serilog.Expressions/Templates/Compilation/CompiledFormattedExpression.cs b/src/Serilog.Expressions/Templates/Compilation/CompiledFormattedExpression.cs
@@ -0,0 +1,25 @@
+using System;
+using System.IO;
+using Serilog.Events;
+using Serilog.Expressions;
+
+namespace Serilog.Templates.Compilation
+{
+    class CompiledFormattedExpression : CompiledTemplate
+    {
+        readonly CompiledExpression _expression;
+        readonly string _format;
+
+        public CompiledFormattedExpression(CompiledExpression expression, string format)
+        {
+            _expression = expression ?? throw new ArgumentNullException(nameof(expression));
+            _format = format;
+        }
+
+        public override void Evaluate(LogEvent logEvent, TextWriter output, IFormatProvider formatProvider)
+        {
+            var value = _expression(logEvent);
+            value?.Render(output, _format, formatProvider);
+        }
+    }
+}
diff --git a/src/Serilog.Expressions/Templates/Compilation/CompiledLiteralText.cs b/src/Serilog.Expressions/Templates/Compilation/CompiledLiteralText.cs
@@ -0,0 +1,21 @@
+using System;
+using System.IO;
+using Serilog.Events;
+
+namespace Serilog.Templates.Compilation
+{
+    class CompiledLiteralText : CompiledTemplate
+    {
+        readonly string _text;
+
+        public CompiledLiteralText(string text)
+        {
+            _text = text ?? throw new ArgumentNullException(nameof(text));
+        }
+
+        public override void Evaluate(LogEvent logEvent, TextWriter output, IFormatProvider formatProvider)
+        {
+            output.Write(_text);
+        }
+    }
+}
diff --git a/src/Serilog.Expressions/Templates/Compilation/CompiledTemplate.cs b/src/Serilog.Expressions/Templates/Compilation/CompiledTemplate.cs
@@ -0,0 +1,11 @@
+using System;
+using System.IO;
+using Serilog.Events;
+
+namespace Serilog.Templates.Compilation
+{
+    abstract class CompiledTemplate
+    {
+        public abstract void Evaluate(LogEvent logEvent, TextWriter output, IFormatProvider formatProvider);
+    }
+}
diff --git a/src/Serilog.Expressions/Templates/Compilation/CompiledTemplateBlock.cs b/src/Serilog.Expressions/Templates/Compilation/CompiledTemplateBlock.cs
@@ -0,0 +1,22 @@
+using System;
+using System.IO;
+using Serilog.Events;
+
+namespace Serilog.Templates.Compilation
+{
+    class CompiledTemplateBlock : CompiledTemplate
+    {
+        readonly CompiledTemplate[] _elements;
+
+        public CompiledTemplateBlock(CompiledTemplate[] elements)
+        {
+            _elements = elements ?? throw new ArgumentNullException(nameof(elements));
+        }
+        
+        public override void Evaluate(LogEvent logEvent, TextWriter output, IFormatProvider formatProvider)
+        {
+            foreach (var element in _elements)
+                element.Evaluate(logEvent, output, formatProvider);
+        }
+    }
+}
diff --git a/src/Serilog.Expressions/Templates/Compilation/TemplateCompiler.cs b/src/Serilog.Expressions/Templates/Compilation/TemplateCompiler.cs
@@ -0,0 +1,23 @@
+using System;
+using System.Linq;
+using Serilog.Expressions.Compilation;
+using Serilog.Templates.Ast;
+
+namespace Serilog.Templates.Compilation
+{
+    static class TemplateCompiler
+    {
+        public static CompiledTemplate Compile(Template template)
+        {
+            if (template == null) throw new ArgumentNullException(nameof(template));
+            return template switch
+            {
+                LiteralText text => new CompiledLiteralText(text.Text),
+                FormattedExpression expression => new CompiledFormattedExpression(
+                    ExpressionCompiler.Compile(expression.Expression), expression.Format),
+                TemplateBlock block => new CompiledTemplateBlock(block.Elements.Select(Compile).ToArray()),
+                _ => throw new NotSupportedException()
+            };
+        }
+    }
+}
diff --git a/src/Serilog.Expressions/Templates/OutputTemplate.cs b/src/Serilog.Expressions/Templates/OutputTemplate.cs
@@ -0,0 +1,37 @@
+using System;
+using System.IO;
+using Serilog.Events;
+using Serilog.Formatting;
+using Serilog.Templates.Compilation;
+using Serilog.Templates.Parsing;
+
+namespace Serilog.Templates
+{
+    /// <summary>
+    /// Formats <see cref="LogEvent"/>s into text using embedded expressions.
+    /// </summary>
+    public class OutputTemplate : ITextFormatter
+    {
+        readonly IFormatProvider _formatProvider;
+        readonly CompiledTemplate _compiled;
+
+        /// <summary>
+        /// Construct an <see cref="OutputTemplate"/>.
+        /// </summary>
+        /// <param name="template">The template text.</param>
+        /// <param name="formatProvider">Optionally, an <see cref="IFormatProvider"/> to use when formatting
+        /// embedded values.</param>
+        public OutputTemplate(string template, IFormatProvider formatProvider = null)
+        {
+            _formatProvider = formatProvider;
+            var parsed = TemplateParser.Parse(template);
+            _compiled = TemplateCompiler.Compile(parsed);
+        }
+
+        /// <inheritdoc />
+        public void Format(LogEvent logEvent, TextWriter output)
+        {
+            _compiled.Evaluate(logEvent, output, _formatProvider);
+        }
+    }
+}
diff --git a/src/Serilog.Expressions/Templates/Parsing/TemplateParser.cs b/src/Serilog.Expressions/Templates/Parsing/TemplateParser.cs
@@ -0,0 +1,41 @@
+using System.Collections.Generic;
+using System.Linq;
+using Serilog.Expressions.Parsing;
+using Serilog.Templates.Ast;
+
+namespace Serilog.Templates.Parsing
+{
+    static class TemplateParser
+    {
+        static ExpressionTokenizer Tokenizer { get; } = new ExpressionTokenizer(true);
+        
+        public static Template Parse(string template)
+        {
+            var tokens = Tokenizer.Tokenize(template);
+
+            var elements = new List<Template>();
+
+            var rest = tokens;
+            while (!rest.IsAtEnd)
+            {
+                var peek = rest.ConsumeToken();
+                if (peek.Value.Kind == ExpressionToken.TemplateLiteral)
+                {
+                    elements.Add(new LiteralText(peek.Value.ToStringValue()));
+                    rest = peek.Remainder;
+                }
+                else
+                {
+                    var partial = ExpressionTokenParsers.TryPartialParse(rest);
+                    elements.Add(new FormattedExpression(partial.Value, null));
+                    rest = partial.Remainder;
+                }
+            }
+
+            if (elements.Count == 1)
+                return elements.Single();
+            
+            return new TemplateBlock(elements.ToArray());
+        }
+    }
+}
diff --git a/test/Serilog.Expressions.Tests/Cases/template-evaluation-cases.asv b/test/Serilog.Expressions.Tests/Cases/template-evaluation-cases.asv
@@ -0,0 +1,3 @@
+// Temporarily using Serilog's default value formatting
+Hello, {'world'}!                        ⇶ Hello, "world"!
+{@Level}                                 ⇶ "Information"
diff --git a/test/Serilog.Expressions.Tests/TemplateEvaluationTests.cs b/test/Serilog.Expressions.Tests/TemplateEvaluationTests.cs

Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,8 @@ namespace Serilog.Expressions.Parsing`
`5`	`5`	`{`
`6`	`6`	`static class ExpressionParser`
`7`	`7`	`{`
	`8`	`+ static ExpressionTokenizer Tokenizer { get; } = new ExpressionTokenizer(false);`
	`9`	`+`
`8`	`10`	`public static Expression Parse(string filterExpression)`
`9`	`11`	`{`
`10`	`12`	`if (!TryParse(filterExpression, out var root, out var error))`
`@@ -17,7 +19,7 @@ public static bool TryParse(string filterExpression, out Expression root, out st`
`17`	`19`	`{`
`18`	`20`	`if (filterExpression == null) throw new ArgumentNullException(nameof(filterExpression));`
`19`	`21`
`20`		`- var tokenList = ExpressionTokenizer.Instance.TryTokenize(filterExpression);`
	`22`	`+ var tokenList = Tokenizer.TryTokenize(filterExpression);`
`21`	`23`	`if (!tokenList.HasValue)`
`22`	`24`	`{`
`23`	`25`	`error = tokenList.ToString();`
Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,9 @@ namespace Serilog.Expressions.Parsing`
`5`	`5`	`enum ExpressionToken`
`6`	`6`	`{`
`7`	`7`	`None,`
	`8`	`+`
	`9`	`+ [Token(Description = "literal text")]`
	`10`	`+ TemplateLiteral,`
`8`	`11`
`9`	`12`	`Identifier,`
`10`	`13`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,6 @@ @@
 +namespace Serilog.Templates.Ast
 +{
 +    abstract class Template
 +    {
 +    }
 +}