Extracted LogEventDataGenerator, StandardColumnDataGenerator and PropertiesColumnDataGenerator from MSSqlServerSinkTraits.

Author: ckadluba

sample/AppConfigDemo/Program.cs

@@ -2,7 +2,6 @@
 using System.Threading;
 using Serilog;
 using Serilog.Events;
-using Serilog.Sinks.MSSqlServer;
 using Serilog.Sinks.MSSqlServer.Sinks.MSSqlServer.Options;
 
 namespace AppConfigDemo

src/Serilog.Sinks.MSSqlServer/GlobalSuppressions.cs

@@ -7,10 +7,10 @@
 
 [assembly: SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Supplying string literals and not using resources is accepted within this project.", Scope = "namespaceanddescendants", Target = "Serilog.Sinks.MSSqlServer")]
 [assembly: SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", Justification = "Too hard to change. Accepted for now.", Scope = "namespaceanddescendants", Target = "Serilog.Sinks.MSSqlServer")]
-[assembly: SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Too hard to change. Accepted for now.", Scope = "member", Target = "~M:Serilog.Sinks.MSSqlServer.MSSqlServerSinkTraits.TryChangeType(System.Object,System.Type,System.Object@)~System.Boolean")]
-[assembly: SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Too hard to change. Accepted for now.", Scope = "member", Target = "~M:Serilog.Sinks.MSSqlServer.MSSqlServerSink.EmitBatchAsync(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})~System.Threading.Tasks.Task")]
-[assembly: SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Too hard to change. Accepted for now.", Scope = "member", Target = "~M:Serilog.Sinks.MSSqlServer.MSSqlServerSinkTraits.ConvertPropertiesToXmlStructure(System.Collections.Generic.IEnumerable{System.Collections.Generic.KeyValuePair{System.String,Serilog.Events.LogEventPropertyValue}})~System.String")]
 [assembly: SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Too hard to change. Accepted for now.", Scope = "member", Target = "~M:Serilog.Sinks.MSSqlServer.MSSqlServerSinkTraits.#ctor(System.String,System.String,Serilog.Sinks.MSSqlServer.ColumnOptions,System.IFormatProvider,System.Boolean,Serilog.Formatting.ITextFormatter,Serilog.Sinks.MSSqlServer.Platform.ISqlTableCreator)")]
+[assembly: SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Too hard to change. Accepted for now.", Scope = "member", Target = "~M:Serilog.Sinks.MSSqlServer.MSSqlServerSink.EmitBatchAsync(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})~System.Threading.Tasks.Task")]
+[assembly: SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Too hard to change. Accepted for now.", Scope = "member", Target = "~M:Serilog.Sinks.MSSqlServer.Output.StandardColumnDataGenerator.ConvertPropertiesToXmlStructure(System.Collections.Generic.IEnumerable{System.Collections.Generic.KeyValuePair{System.String,Serilog.Events.LogEventPropertyValue}})~System.String")]
+[assembly: SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Too hard to change. Accepted for now.", Scope = "member", Target = "~M:Serilog.Sinks.MSSqlServer.Output.PropertiesColumnDataGenerator.TryChangeType(System.Object,System.Type,System.Object@)~System.Boolean")]
 [assembly: SuppressMessage("Design", "CA1062:Validate arguments of public methods", Justification = "Serilog core guarantees to call Emit() with non-null logEvent parameter.", Scope = "member", Target = "~M:Serilog.Sinks.MSSqlServer.MSSqlServerAuditSink.Emit(Serilog.Events.LogEvent)")]
 [assembly: SuppressMessage("Design", "CA1062:Validate arguments of public methods", Justification = "Serilog core guarantees to call EmitBatchAsync() with non-null logEvent parameter.", Scope = "member", Target = "~M:Serilog.Sinks.MSSqlServer.MSSqlServerSink.EmitBatchAsync(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})~System.Threading.Tasks.Task")]
 [assembly: SuppressMessage("Design", "CA1034:Nested types should not be visible", Justification = "Cannot be changed on public classes for backward compatibility reasons.", Scope = "namespaceanddescendants", Target = "Serilog.Sinks.MSSqlServer")]

src/Serilog.Sinks.MSSqlServer/Sinks/MSSqlServer/MSSqlServerSinkTraits.cs

@@ -15,9 +15,6 @@
 using System;
 using System.Collections.Generic;
 using System.Data;
-using System.Globalization;
-using System.Linq;
-using System.Text;
 using Serilog.Debugging;
 using Serilog.Events;
 using Serilog.Formatting;
@@ -30,16 +27,13 @@ namespace Serilog.Sinks.MSSqlServer
     /// <summary>Contains common functionality and properties used by both MSSqlServerSinks.</summary>
     internal class MSSqlServerSinkTraits : IDisposable
     {
+        private readonly ILogEventDataGenerator _logEventDataGenerator;
         private bool _disposedValue;
 
         public string TableName { get; }
         public string SchemaName { get; }
         public ColumnOptions ColumnOptions { get; }
-        public IFormatProvider FormatProvider { get; }
-        public ITextFormatter LogEventFormatter { get; }
-        public ISet<string> AdditionalColumnPropertyNames { get; }
         public DataTable EventTable { get; }
-        public ISet<string> StandardColumnNames { get; }
 
         public MSSqlServerSinkTraits(
             ISqlConnectionFactory sqlConnectionFactory,
@@ -73,22 +67,11 @@ internal MSSqlServerSinkTraits(
             TableName = tableName;
             SchemaName = schemaName;
             ColumnOptions = columnOptions ?? new ColumnOptions();
-            FormatProvider = formatProvider;
 
-            StandardColumnNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
-            foreach (var stdCol in ColumnOptions.Store)
-            {
-                var col = ColumnOptions.GetStandardColumnOptions(stdCol);
-                StandardColumnNames.Add(col.ColumnName);
-            }
-
-            AdditionalColumnPropertyNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
-            if (ColumnOptions.AdditionalColumns != null)
-                foreach (var col in ColumnOptions.AdditionalColumns)
-                    AdditionalColumnPropertyNames.Add(col.PropertyName);
-
-            if (ColumnOptions.Store.Contains(StandardColumn.LogEvent))
-                LogEventFormatter = logEventFormatter ?? new JsonLogEventFormatter(this);
+            // TODO initialize this outside of this class
+            var standardColumnDataGenerator = new StandardColumnDataGenerator(columnOptions, formatProvider, logEventFormatter);
+            var propertiesColumnDataGenerator = new PropertiesColumnDataGenerator(columnOptions);
+            _logEventDataGenerator = new LogEventDataGenerator(columnOptions, standardColumnDataGenerator, propertiesColumnDataGenerator);
 
             EventTable = CreateDataTable();
 
@@ -105,181 +88,9 @@ internal MSSqlServerSinkTraits(
             }
         }
 
-        /// <summary>Gets a list of the column names paired with their values to emit for the specified <paramref name="logEvent"/>.</summary>
-        /// <param name="logEvent">The log event to emit.</param>
-        /// <returns>
-        /// A list of mappings between column names and values to emit to the database for the specified <paramref name="logEvent"/>.
-        /// </returns>
         public IEnumerable<KeyValuePair<string, object>> GetColumnsAndValues(LogEvent logEvent)
         {
-            foreach (var column in ColumnOptions.Store)
-            {
-                // skip Id (auto-incrementing identity)
-                if (column != StandardColumn.Id)
-                    yield return GetStandardColumnNameAndValue(column, logEvent);
-            }
-
-            if (ColumnOptions.AdditionalColumns != null)
-            {
-                foreach (var columnValuePair in ConvertPropertiesToColumn(logEvent.Properties))
-                    yield return columnValuePair;
-            }
-        }
-
-        internal KeyValuePair<string, object> GetStandardColumnNameAndValue(StandardColumn column, LogEvent logEvent)
-        {
-            switch (column)
-            {
-                case StandardColumn.Message:
-                    return new KeyValuePair<string, object>(ColumnOptions.Message.ColumnName, logEvent.RenderMessage(FormatProvider));
-                case StandardColumn.MessageTemplate:
-                    return new KeyValuePair<string, object>(ColumnOptions.MessageTemplate.ColumnName, logEvent.MessageTemplate.Text);
-                case StandardColumn.Level:
-                    return new KeyValuePair<string, object>(ColumnOptions.Level.ColumnName, ColumnOptions.Level.StoreAsEnum ? (object)logEvent.Level : logEvent.Level.ToString());
-                case StandardColumn.TimeStamp:
-                    return GetTimeStampStandardColumnNameAndValue(logEvent);
-                case StandardColumn.Exception:
-                    return new KeyValuePair<string, object>(ColumnOptions.Exception.ColumnName, logEvent.Exception?.ToString());
-                case StandardColumn.Properties:
-                    return new KeyValuePair<string, object>(ColumnOptions.Properties.ColumnName, ConvertPropertiesToXmlStructure(logEvent.Properties));
-                case StandardColumn.LogEvent:
-                    return new KeyValuePair<string, object>(ColumnOptions.LogEvent.ColumnName, RenderLogEventColumn(logEvent));
-                default:
-                    throw new ArgumentOutOfRangeException(nameof(column));
-            }
-        }
-
-        private KeyValuePair<string, object> GetTimeStampStandardColumnNameAndValue(LogEvent logEvent)
-        {
-            var dateTimeOffset = ColumnOptions.TimeStamp.ConvertToUtc ? logEvent.Timestamp.ToUniversalTime() : logEvent.Timestamp;
-
-            if (ColumnOptions.TimeStamp.DataType == SqlDbType.DateTimeOffset)
-                return new KeyValuePair<string, object>(ColumnOptions.TimeStamp.ColumnName, dateTimeOffset);
-
-            return new KeyValuePair<string, object>(ColumnOptions.TimeStamp.ColumnName, dateTimeOffset.DateTime);
-        }
-
-        private string RenderLogEventColumn(LogEvent logEvent)
-        {
-            if (ColumnOptions.LogEvent.ExcludeAdditionalProperties)
-            {
-                var filteredProperties = logEvent.Properties.Where(p => !AdditionalColumnPropertyNames.Contains(p.Key));
-                logEvent = new LogEvent(logEvent.Timestamp, logEvent.Level, logEvent.Exception, logEvent.MessageTemplate, filteredProperties.Select(x => new LogEventProperty(x.Key, x.Value)));
-            }
-
-            var sb = new StringBuilder();
-            using (var writer = new System.IO.StringWriter(sb))
-                LogEventFormatter.Format(logEvent, writer);
-            return sb.ToString();
-        }
-
-        private string ConvertPropertiesToXmlStructure(IEnumerable<KeyValuePair<string, LogEventPropertyValue>> properties)
-        {
-            var options = ColumnOptions.Properties;
-
-            if (options.ExcludeAdditionalProperties)
-                properties = properties.Where(p => !AdditionalColumnPropertyNames.Contains(p.Key));
-
-            if (options.PropertiesFilter != null)
-            {
-                try
-                {
-                    properties = properties.Where(p => options.PropertiesFilter(p.Key));
-                }
-                catch (Exception ex)
-                {
-                    SelfLog.WriteLine("Unable to filter properties to store in {0} due to following error: {1}", this, ex);
-                }
-            }
-
-            var sb = new StringBuilder();
-
-            sb.AppendFormat(CultureInfo.InvariantCulture, "<{0}>", options.RootElementName);
-
-            foreach (var property in properties)
-            {
-                var value = XmlPropertyFormatter.Simplify(property.Value, options);
-                if (options.OmitElementIfEmpty && string.IsNullOrEmpty(value))
-                {
-                    continue;
-                }
-
-                if (options.UsePropertyKeyAsElementName)
-                {
-                    sb.AppendFormat(CultureInfo.InvariantCulture, "<{0}>{1}</{0}>", XmlPropertyFormatter.GetValidElementName(property.Key), value);
-                }
-                else
-                {
-                    sb.AppendFormat(CultureInfo.InvariantCulture, "<{0} key='{1}'>{2}</{0}>", options.PropertyElementName, property.Key, value);
-                }
-            }
-
-            sb.AppendFormat(CultureInfo.InvariantCulture, "</{0}>", options.RootElementName);
-
-            return sb.ToString();
-        }
-
-        /// <summary>
-        ///     Mapping values from properties which have a corresponding data row.
-        ///     Matching is done based on Column name and property key
-        ///     Standard columns are not mapped
-        /// </summary>
-        /// <param name="properties"></param>
-        private IEnumerable<KeyValuePair<string, object>> ConvertPropertiesToColumn(IReadOnlyDictionary<string, LogEventPropertyValue> properties)
-        {
-            foreach (var property in properties)
-            {
-                var additionalColumn = ColumnOptions
-                    .AdditionalColumns
-                    .FirstOrDefault(ac => ac.PropertyName == property.Key);
-
-                if (additionalColumn == null || StandardColumnNames.Contains(property.Key))
-                    continue;
-
-                var columnName = additionalColumn.ColumnName;
-                var columnType = EventTable.Columns[columnName].DataType;
-
-                if (!(property.Value is ScalarValue scalarValue))
-                {
-                    yield return new KeyValuePair<string, object>(columnName, property.Value.ToString());
-                    continue;
-                }
-
-                if (scalarValue.Value == null && EventTable.Columns[columnName].AllowDBNull)
-                {
-                    yield return new KeyValuePair<string, object>(columnName, DBNull.Value);
-                    continue;
-                }
-
-                if (TryChangeType(scalarValue.Value, columnType, out var conversion))
-                {
-                    yield return new KeyValuePair<string, object>(columnName, conversion);
-                }
-                else
-                {
-                    yield return new KeyValuePair<string, object>(columnName, property.Value.ToString());
-                }
-            }
-        }
-
-        /// <summary>
-        ///     Try to convert the object to the given type
-        /// </summary>
-        /// <param name="obj">object</param>
-        /// <param name="type">type to convert to</param>
-        /// <param name="conversion">result of the converted value</param>        
-        private static bool TryChangeType(object obj, Type type, out object conversion)
-        {
-            conversion = null;
-            try
-            {
-                conversion = Convert.ChangeType(obj, type, CultureInfo.InvariantCulture);
-                return true;
-            }
-            catch
-            {
-                return false;
-            }
+            return _logEventDataGenerator.GetColumnsAndValues(logEvent);
         }
 
         private DataTable CreateDataTable()

src/Serilog.Sinks.MSSqlServer/Sinks/MSSqlServer/Output/ILogEventDataGenerator.cs

@@ -0,0 +1,10 @@
+using System.Collections.Generic;
+using Serilog.Events;
+
+namespace Serilog.Sinks.MSSqlServer.Output
+{
+    internal interface ILogEventDataGenerator
+    {
+        IEnumerable<KeyValuePair<string, object>> GetColumnsAndValues(LogEvent logEvent);
+    }
+}

src/Serilog.Sinks.MSSqlServer/Sinks/MSSqlServer/Output/IPropertiesColumnDataGenerator.cs

@@ -0,0 +1,10 @@
+using System.Collections.Generic;
+using Serilog.Events;
+
+namespace Serilog.Sinks.MSSqlServer.Output
+{
+    internal interface IPropertiesColumnDataGenerator
+    {
+        IEnumerable<KeyValuePair<string, object>> ConvertPropertiesToColumn(IReadOnlyDictionary<string, LogEventPropertyValue> properties);
+    }
+}

src/Serilog.Sinks.MSSqlServer/Sinks/MSSqlServer/Output/IStandardColumnDataGenerator.cs

@@ -0,0 +1,10 @@
+using System.Collections.Generic;
+using Serilog.Events;
+
+namespace Serilog.Sinks.MSSqlServer.Output
+{
+    internal interface IStandardColumnDataGenerator
+    {
+        KeyValuePair<string, object> GetStandardColumnNameAndValue(StandardColumn column, LogEvent logEvent);
+    }
+}

src/Serilog.Sinks.MSSqlServer/Sinks/MSSqlServer/Output/JsonLogEventFormatter.cs

@@ -31,17 +31,19 @@ namespace Serilog.Sinks.MSSqlServer.Output
     internal class JsonLogEventFormatter : ITextFormatter
     {
         private const string _commaDelimiter = ",";
+        private readonly ColumnOptions _columnOptions;
+        private readonly IStandardColumnDataGenerator _standardColumnsDataGenerator;
         private readonly JsonValueFormatter _valueFormatter;
-        private readonly MSSqlServerSinkTraits _traits;
 
         /// <summary>
-        /// Constructor. A reference to the parent Traits object is used so that JSON
+        /// Constructor. A reference to the parent IStandardColumnsDataGenerator object is used so that JSON
         /// can serialize Standard Column values exactly the way they would be written
         /// to discrete SQL columns.
         /// </summary>
-        public JsonLogEventFormatter(MSSqlServerSinkTraits parent)
+        public JsonLogEventFormatter(ColumnOptions columnOptions, IStandardColumnDataGenerator standardColumnsDataGenerator)
         {
-            _traits = parent;
+            _columnOptions = columnOptions ?? throw new ArgumentNullException(nameof(columnOptions));
+            _standardColumnsDataGenerator = standardColumnsDataGenerator ?? throw new ArgumentNullException(nameof(standardColumnsDataGenerator));
             _valueFormatter = new JsonValueFormatter(typeTagName: null);
         }
 
@@ -57,7 +59,7 @@ public void Format(LogEvent logEvent, TextWriter output)
 
             var precedingDelimiter = "";
 
-            if (_traits.ColumnOptions.LogEvent.ExcludeStandardColumns == false)
+            if (_columnOptions.LogEvent.ExcludeStandardColumns == false)
             {
                 // The XML Properties column has never included the Standard Columns, but prior
                 // to adding this sink-specific JSON formatter, the LogEvent JSON column relied
@@ -104,7 +106,7 @@ private void WriteStandardColumns(LogEvent logEvent, TextWriter output, ref stri
 
         private void WriteIfPresent(StandardColumn col, LogEvent logEvent, TextWriter output, ref string precedingDelimiter)
         {
-            if (!_traits.ColumnOptions.Store.Contains(col))
+            if (!_columnOptions.Store.Contains(col))
                 return;
 
             output.Write(precedingDelimiter);
@@ -116,21 +118,21 @@ private void WriteIfPresent(StandardColumn col, LogEvent logEvent, TextWriter ou
 
         private void WriteTimeStampIfPresent(LogEvent logEvent, TextWriter output, ref string precedingDelimiter)
         {
-            if (!_traits.ColumnOptions.Store.Contains(StandardColumn.TimeStamp))
+            if (!_columnOptions.Store.Contains(StandardColumn.TimeStamp))
                 return;
 
             output.Write(precedingDelimiter);
             precedingDelimiter = _commaDelimiter;
             var colData = WritePropertyName(logEvent, output, StandardColumn.TimeStamp);
-            var value = _traits.ColumnOptions.TimeStamp.DataType == SqlDbType.DateTime
+            var value = _columnOptions.TimeStamp.DataType == SqlDbType.DateTime
                 ? ((DateTime)colData.Value).ToString("o", CultureInfo.InvariantCulture)
                 : ((DateTimeOffset)colData.Value).ToString("o", CultureInfo.InvariantCulture);
             JsonValueFormatter.WriteQuotedJsonString(value, output);
         }
 
         private KeyValuePair<string, object> WritePropertyName(LogEvent le, TextWriter writer, StandardColumn col)
         {
-            var colData = _traits.GetStandardColumnNameAndValue(col, le);
+            var colData = _standardColumnsDataGenerator.GetStandardColumnNameAndValue(col, le);
             JsonValueFormatter.WriteQuotedJsonString(colData.Key, writer);
             writer.Write(":");