Merge pull request #281 from nblumhardt/switch-to-serilog-expressions

Switch seqcli ingest filtering to use Serilog.Expressions; fix filters over non-Serilog level names

Author: KodrAus

src/SeqCli/Api/ApiConstants.cs

@@ -16,7 +16,7 @@ namespace SeqCli.Api;
 
 static class ApiConstants
 {
-    public const string ClefMediatType = "application/vnd.serilog.clef";
+    public const string ClefMediaType = "application/vnd.serilog.clef";
     public const string IngestionEndpoint = "api/events/raw";
     public const string ApiKeyHeaderName = "X-Seq-ApiKey";
 }

src/SeqCli/Cli/Commands/IngestCommand.cs

@@ -23,7 +23,7 @@
 using Serilog;
 using Serilog.Core;
 using Serilog.Events;
-using Serilog.Filters.Expressions;
+using Serilog.Expressions;
 
 namespace SeqCli.Cli.Commands;
 
@@ -57,7 +57,7 @@ public IngestCommand(SeqConnectionFactory connectionFactory)
 
         Options.Add("json",
             "Read the events as JSON (the default assumes plain text)",
-            v => _json = true);
+            _ => _json = true);
 
         Options.Add("f=|filter=",
             "Filter expression to select a subset of events",
@@ -92,9 +92,12 @@ protected override async Task<int> Run()
             Func<LogEvent, bool>? filter = null;
             if (_filter != null)
             {
-                var expr = _filter.Replace("@Level", SurrogateLevelProperty.PropertyName);
-                var eval = FilterLanguage.CreateFilter(expr);
-                filter = evt => true.Equals(eval(evt));
+                // Support non-Serilog level names (`@l` can't be overridden by the name resolver). At a later date,
+                // we hope to be able to use the Serilog.Expressions AST to do this reliably (at the moment, all occurrences
+                // of @l, whether referring to the property or not, will be replaced).
+                var expr = _filter.Replace("@l", "@Level").Replace("@Level", $"coalesce(@p['{SurrogateLevelProperty.PropertyName}'],@l)");
+                var eval = SerilogExpression.Compile(expr, nameResolver: new SeqBuiltInNameResolver());
+                filter = evt => ExpressionResult.IsTrue(eval(evt));
             }
 
             var connection = _connectionFactory.Connect(_connection);

src/SeqCli/Cli/Commands/LogCommand.cs

@@ -99,7 +99,7 @@ protected override async Task<int> Run()
             payload.WriteTo(jsonWriter);
             jsonWriter.Flush();
             builder.WriteLine();
-            content = new StringContent(builder.ToString(), Encoding.UTF8, ApiConstants.ClefMediatType);
+            content = new StringContent(builder.ToString(), Encoding.UTF8, ApiConstants.ClefMediaType);
         }
 
         var connection = _connectionFactory.Connect(_connection);

src/SeqCli/Cli/Features/OutputFormatFeature.cs

@@ -19,7 +19,6 @@
 using Newtonsoft.Json.Converters;
 using Newtonsoft.Json.Linq;
 using Seq.Api.Model;
-using Seq.Api.Model.AppInstances;
 using SeqCli.Config;
 using SeqCli.Csv;
 using SeqCli.Levels;
@@ -49,7 +48,7 @@ public OutputFormatFeature(SeqCliOutputConfig outputConfig)
 
     public bool Json => _json;
 
-    bool ApplyThemeToRedirectedOutput => _noColor == false && _forceColor == true;
+    bool ApplyThemeToRedirectedOutput => _noColor == false && _forceColor;
 
     ConsoleTheme Theme
         => _noColor                     ? ConsoleTheme.None
@@ -61,13 +60,13 @@ public override void Enable(OptionSet options)
         options.Add(
             "json",
             "Print output in newline-delimited JSON (the default is plain text)",
-            v => _json = true);
+            _ => _json = true);
 
-        options.Add("no-color", "Don't colorize text output", v => _noColor = true);
+        options.Add("no-color", "Don't colorize text output", _ => _noColor = true);
 
         options.Add("force-color",
             "Force redirected output to have ANSI color (unless `--no-color` is also specified)",
-            v => _forceColor = true);
+            _ => _forceColor = true);
     }
 
     public Logger CreateOutputLogger()

src/SeqCli/Ingestion/LogShipper.cs

@@ -159,7 +159,7 @@ static async Task<bool> SendBatchAsync(
             foreach (var evt in batch)
                 Formatter.Format(evt, builder);
 
-            content = new StringContent(builder.ToString(), Encoding.UTF8, ApiConstants.ClefMediatType);
+            content = new StringContent(builder.ToString(), Encoding.UTF8, ApiConstants.ClefMediaType);
         }
 
         var request = new HttpRequestMessage(HttpMethod.Post, ApiConstants.IngestionEndpoint) { Content = content };

src/SeqCli/Ingestion/SeqBuiltInNameResolver.cs

@@ -0,0 +1,59 @@
+// Copyright © Datalust Pty Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System.Diagnostics.CodeAnalysis;
+using Serilog.Expressions;
+
+#nullable enable
+
+namespace SeqCli.Ingestion;
+
+/// <summary>
+/// Extends Serilog.Expressions with support for commonly-used Seq property names.
+/// </summary>
+/// <remarks>SeqCli ingestion filters previously used Serilog.Filters.Expressions, so this type ensures most
+/// older filters will continue to work.</remarks>
+public class SeqBuiltInNameResolver: NameResolver
+{
+    public override bool TryResolveBuiltInPropertyName(string alias, [NotNullWhen(true)] out string? target)
+    {
+        switch (alias)
+        {
+            case "@Properties":
+                target = "@p";
+                return true;
+            case "@Timestamp":
+                target = "@t";
+                return true;
+            case "@Level":
+                target = "@l";
+                return true;
+            case "@Message":
+                target = "@m";
+                return true;
+            case "@MessageTemplate":
+                target = "@mt";
+                return true;
+            case "@EventType":
+                target = "@i";
+                return true;
+            case "@Exception":
+                target = "@x";
+                return true;
+            default:
+                target = null;
+                return false;
+        }
+    }
+}

src/SeqCli/PlainText/Extraction/ExtractionPatternInterpreter.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using SeqCli.PlainText.Patterns;
 

src/SeqCli/PlainText/LogEvents/LogEventBuilder.cs

@@ -34,9 +34,11 @@ public static LogEvent FromProperties(IDictionary<string, object?> properties, s
         var messageTemplate = GetMessageTemplate(properties);          
         var props = GetLogEventProperties(properties, remainder, level);
 
+        var fallbackMappedLevel = level != null ? LevelMapping.ToSerilogLevel(level) : LogEventLevel.Information;
+
         return new LogEvent(
             timestamp,
-            LogEventLevel.Information,
+            fallbackMappedLevel,
             exception,
             messageTemplate,
             props);
@@ -56,12 +58,13 @@ static MessageTemplate GetMessageTemplate(IDictionary<string, object?> propertie
         return NoMessage;
     }
 
-    static string GetLevel(IDictionary<string, object?> properties)
+    static string? GetLevel(IDictionary<string, object?> properties)
     {
         if (properties.TryGetValue(ReifiedProperties.Level, out var l) &&
             l is TextSpan ts)
             return ts.ToStringValue();
-        return LogEventLevel.Information.ToString();
+
+        return null;
     }
 
     static Exception? TryGetException(IDictionary<string, object?> properties)
@@ -72,7 +75,7 @@ static string GetLevel(IDictionary<string, object?> properties)
         return null;
     }
 
-    static IEnumerable<LogEventProperty> GetLogEventProperties(IDictionary<string, object?> properties, string? remainder, string level)
+    static IEnumerable<LogEventProperty> GetLogEventProperties(IDictionary<string, object?> properties, string? remainder, string? level)
     {
         var payload = properties
             .Where(p => !ReifiedProperties.IsReifiedProperty(p.Key))

src/SeqCli/SeqCli.csproj

@@ -29,7 +29,7 @@
     <PackageReference Include="Destructurama.JsonNet" Version="2.0.0" />
     <PackageReference Include="newtonsoft.json" Version="13.0.2" />
     <PackageReference Include="Serilog" Version="2.12.0" />
-    <PackageReference Include="serilog.filters.expressions" Version="2.1.0" />
+    <PackageReference Include="serilog.expressions" Version="3.4.1" />
     <PackageReference Include="Serilog.Formatting.Compact" Version="1.1.0" />
     <PackageReference Include="Serilog.Formatting.Compact.Reader" Version="2.0.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />

test/SeqCli.Tests/PlainText/LogEventBuilderTests.cs

@@ -29,12 +29,12 @@ public void SuppliedValuesAreUsed()
 
         Assert.Equal("2018-02-01T13:00:00.1230000+00:00", evt.Timestamp.ToString("o"));
         Assert.Equal("Hello, world", evt.RenderMessage());
-        Assert.Equal(LogEventLevel.Information, evt.Level);
+        Assert.Equal(LogEventLevel.Warning, evt.Level);
         Assert.Equal("WRN", ((ScalarValue)evt.Properties[SurrogateLevelProperty.PropertyName]).Value);
-        Assert.Equal("EverythingFailedException", evt.Exception.ToString());
+        Assert.Equal("EverythingFailedException", evt.Exception?.ToString());
         Assert.Equal(42, ((ScalarValue)evt.Properties["Count"]).Value);
-        Assert.Equal("TP", ((ScalarValue)evt.Properties["MachineName"]).Value.ToString());
-        Assert.Equal("rem", ((ScalarValue)evt.Properties["@unmatched"]).Value.ToString());
+        Assert.Equal("TP", ((ScalarValue)evt.Properties["MachineName"]).Value!.ToString());
+        Assert.Equal("rem", ((ScalarValue)evt.Properties["@unmatched"]).Value!.ToString());
     }
 
     [Fact]