Rel 1.3.27.0

TimberWinR.ServiceHost/Properties/AssemblyInfo.cs

@@ -32,5 +32,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.3.26.0")]
-[assembly: AssemblyFileVersion("1.3.26.0")]
+[assembly: AssemblyVersion("1.3.27.0")]
+[assembly: AssemblyFileVersion("1.3.27.0")]

TimberWinR/Diagnostics/Diagnostics.cs

@@ -8,7 +8,7 @@
 using System.Net;
 using System.Net.Sockets;
 using System.IO;
-
+using System.Linq.Expressions;
 using Newtonsoft.Json.Linq;
 
 using NLog;
@@ -52,29 +52,48 @@ private Assembly GetAssemblyByName(string name)
         public JObject DiagnosticsOutput()
         {
             JObject json = new JObject(
-                           new JProperty("timberwinr",
-                               new JObject(
-                                   new JProperty("version", Assembly.GetEntryAssembly().GetName().Version.ToString()),
-                                   new JProperty("messages", Manager.NumMessages),
-                                   new JProperty("startedon", Manager.StartedOn),
-                                   new JProperty("configfile", Manager.JsonConfig),
-                                   new JProperty("logdir", Manager.LogfileDir),
-                                   new JProperty("logginglevel", LogManager.GlobalThreshold.ToString()),
-                                   new JProperty("inputs",
-                                       new JArray(
-                                           from i in Manager.Listeners
-                                           select new JObject(i.ToJson()))),
-                                   new JProperty("filters",
-                                       new JArray(
-                                           from f in Manager.Config.Filters
-                                           select new JObject(f.ToJson()))),
-                                   new JProperty("outputs",
-                                       new JArray(
-                                           from o in Manager.Outputs
-                                           select new JObject(o.ToJson()))))));
+                new JProperty("timberwinr",
+                    new JObject(
+                        new JProperty("version", Assembly.GetEntryAssembly().GetName().Version.ToString()),
+                        new JProperty("messages", Manager.NumMessages),
+                        new JProperty("startedon", Manager.StartedOn),
+                        new JProperty("configfile", Manager.JsonConfig),
+                        new JProperty("logdir", Manager.LogfileDir),
+                        new JProperty("logginglevel", LogManager.GlobalThreshold.ToString())
+                        )));
+            AddDiagnosis(json);
             return json;
         }
 
+        protected void AddDiagnosis(JObject wrapper)
+        {
+            wrapper.Add("inputs", GetDiagnosisByType("inputs", Manager.Listeners.ToList<IDiagnosable>()));
+            wrapper.Add("filters", GetDiagnosisByType("filters", Manager.Config.Filters.ToList<IDiagnosable>()));
+            wrapper.Add("outputs", GetDiagnosisByType("inputs", Manager.Outputs.ToList<IDiagnosable>()));
+        }
+
+        protected JObject GetDiagnosisByType(String type, List<IDiagnosable> diags)
+        {
+            JObject category = new JObject();
+            foreach(IDiagnosable diag in diags)
+            {
+                JArray array = GetTypeArray(diag.GetType().Name.ToString(), category);
+                array.Add(diag.ToJson());
+            }
+            return category;
+        }
+
+        protected JArray GetTypeArray(String name, JObject category)
+        {
+            JArray ret = (JArray)category.GetValue(name);
+            if (ret == null)
+            {
+                ret = new JArray();
+                category.Add(new JProperty(name, ret));
+            }
+            return ret;
+        }
+
         private void DiagnosticCallback(IAsyncResult result)
         {
             if (web == null)

TimberWinR/Diagnostics/IDiagnosable.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Newtonsoft.Json.Linq;
+
+namespace TimberWinR.Diagnostics
+{
+    public interface IDiagnosable
+    {
+        JObject ToJson();
+    }
+}

TimberWinR/Inputs/IISW3CInputListener.cs

@@ -135,8 +135,6 @@ record = null;
                                 rs.close();
                                 GC.Collect();
                             }
-                            if (!Stop)
-                                syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
                         }
                         catch (OperationCanceledException)
                         {
@@ -145,7 +143,18 @@ record = null;
                         catch (Exception ex)
                         {
                             LogManager.GetCurrentClassLogger().Error(ex);
-                        }                       
+                        }
+                        finally
+                        {
+                            try
+                            {
+                                if (!Stop)
+                                    syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
+                            }
+                            catch (Exception)
+                            {
+                            }
+                        }
                     }
                 }
             }

TimberWinR/Inputs/InputListener.cs

@@ -8,10 +8,11 @@
 using System.Text;
 using System.Threading;
 using NLog;
+using TimberWinR.Diagnostics;
 
 namespace TimberWinR.Inputs
 {
-    public abstract class InputListener
+    public abstract class InputListener: IDiagnosable
     {
         public CancellationToken CancelToken { get; set; }
         public event Action<JObject> OnMessageRecieved;

TimberWinR/Inputs/WindowsEvtInputListener.cs

@@ -8,6 +8,7 @@
 using LogQuery = Interop.MSUtil.LogQueryClassClass;
 using EventLogInputFormat = Interop.MSUtil.COMEventLogInputContextClassClass;
 using LogRecordSet = Interop.MSUtil.ILogRecordset;
+using System.IO;
 
 namespace TimberWinR.Inputs
 {
@@ -97,12 +98,13 @@ private void EventWatcher(object ploc)
                     // Execute the query
                     if (!CancelToken.IsCancellationRequested)
                     {
+                        var oLogQuery = new LogQuery();
                         try
-                        {                          
-                            var oLogQuery = new LogQuery();
+                        {
 
                             var qfiles = string.Format("SELECT Distinct [EventLog] FROM {0}", location);
                             var rsfiles = oLogQuery.Execute(qfiles, iFmt);
+
                             for (; !rsfiles.atEnd(); rsfiles.moveNext())
                             {
                                 var record = rsfiles.getRecord();
@@ -113,7 +115,7 @@ private void EventWatcher(object ploc)
                                         logName);
                                     var rcount = oLogQuery.Execute(qcount, iFmt);
                                     var qr = rcount.getRecord();
-                                    var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
+                                    var lrn = (Int64) qr.getValueEx("MaxRecordNumber");
                                     logFileMaxRecords[logName] = lrn;
                                 }
                             }
@@ -137,12 +139,13 @@ private void EventWatcher(object ploc)
                                         object v = record.getValue(field.Name);
                                         if (field.Name == "Data")
                                             v = ToPrintable(v.ToString());
-                                        if ((field.Name == "TimeGenerated" || field.Name == "TimeWritten") && field.DataType == typeof (DateTime))
+                                        if ((field.Name == "TimeGenerated" || field.Name == "TimeWritten") &&
+                                            field.DataType == typeof (DateTime))
                                             v = ((DateTime) v).ToUniversalTime();
                                         json.Add(new JProperty(field.Name, v));
                                     }
 
-                                    var lrn = (Int64)record.getValueEx("RecordNumber");
+                                    var lrn = (Int64) record.getValueEx("RecordNumber");
                                     logFileMaxRecords[fileName] = lrn;
 
                                     ProcessJson(json);
@@ -163,6 +166,24 @@ private void EventWatcher(object ploc)
                         {
                             LogManager.GetCurrentClassLogger().Error(ex);
                         }
+                        finally
+                        {
+                            try
+                            {
+                                oLogQuery = null;
+                                // Sleep 
+                                if (!Stop)
+                                    syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
+                            }
+                            catch (OperationCanceledException)
+                            {
+                            }
+                            catch (Exception ex1)
+                            {
+                                LogManager.GetCurrentClassLogger().Warn(ex1);
+                            }
+
+                        }
                     }
                 }
                 Finished();

TimberWinR/Outputs/OutputSender.cs

@@ -4,11 +4,12 @@
 using System.Text;
 using System.Threading;
 using Newtonsoft.Json.Linq;
+using TimberWinR.Diagnostics;
 using TimberWinR.Inputs;
 
 namespace TimberWinR.Outputs
 {
-    public abstract class OutputSender
+    public abstract class OutputSender : IDiagnosable
     {
         public CancellationToken CancelToken { get; private set; }        
         private List<InputListener> _inputs;

TimberWinR/Outputs/Redis.cs

@@ -132,6 +132,7 @@ public long SentMessages
         private long _errorCount;
         private long _redisDepth;
         private DateTime? _lastErrorTimeUTC;
+        private DateTime? _lastSentTimeUTC;
         private readonly int _maxQueueSize;
         private readonly bool _queueOverflowDiscardOldest;
         private BatchCounter _batchCounter;
@@ -180,6 +181,7 @@ public override JObject ToJson()
                         new JProperty("host", string.Join(",", _redisHosts)),
                         new JProperty("errors", _errorCount),
                         new JProperty("lastErrorTimeUTC", _lastErrorTimeUTC),
+                        new JProperty("lastSentTimeUTC", _lastSentTimeUTC),
                         new JProperty("redisQueueDepth", _redisDepth),
                         new JProperty("sentMessageCount", _sentMessages),
                         new JProperty("queuedMessageCount", _jsonQueue.Count),
@@ -317,9 +319,13 @@ private void RedisSender()
                                 _batchCounter.SampleQueueDepth(_jsonQueue.Count);
                                 // Re-compute current batch size
 
-                                LogManager.GetCurrentClassLogger().Trace("{0}: Average Queue Depth: {1}, Current Length: {2}", Thread.CurrentThread.ManagedThreadId, _batchCounter.AverageQueueDepth(), _jsonQueue.Count); 
-
-                                _currentBatchCount = _batchCounter.UpdateCurrentBatchCount(_jsonQueue.Count, _currentBatchCount);
+                                LogManager.GetCurrentClassLogger()
+                                    .Trace("{0}: Average Queue Depth: {1}, Current Length: {2}",
+                                        Thread.CurrentThread.ManagedThreadId, _batchCounter.AverageQueueDepth(),
+                                        _jsonQueue.Count);
+
+                                _currentBatchCount = _batchCounter.UpdateCurrentBatchCount(_jsonQueue.Count,
+                                    _currentBatchCount);
 
                                 messages = _jsonQueue.Take(_currentBatchCount).ToArray();
                                 _jsonQueue.RemoveRange(0, messages.Length);
@@ -340,14 +346,17 @@ private void RedisSender()
                                             {
                                                 client.StartPipe();
                                                 LogManager.GetCurrentClassLogger()
-                                                    .Debug("{0}: Sending {1} Messages to {2}", Thread.CurrentThread.ManagedThreadId, messages.Length, client.Host);
+                                                    .Debug("{0}: Sending {1} Messages to {2}",
+                                                        Thread.CurrentThread.ManagedThreadId, messages.Length,
+                                                        client.Host);
 
                                                 try
                                                 {
                                                     _redisDepth = client.RPush(_logstashIndexName, messages);
                                                     Interlocked.Add(ref _sentMessages, messages.Length);
                                                     client.EndPipe();
                                                     sentSuccessfully = true;
+                                                    _lastSentTimeUTC = DateTime.UtcNow;
                                                     if (messages.Length > 0)
                                                         _manager.IncrementMessageCount(messages.Length);
                                                 }
@@ -391,9 +400,7 @@ private void RedisSender()
                                         _jsonQueue.InsertRange(0, messages);
                                     }
                                 }
-                            }                          
-                            if (!Stop)
-                                syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
+                            }
                         }
                         catch (OperationCanceledException)
                         {
@@ -409,6 +416,17 @@ private void RedisSender()
                             Interlocked.Increment(ref _errorCount);
                             LogManager.GetCurrentClassLogger().Error(ex);
                         }
+                        finally
+                        {
+                            try
+                            {
+                                if (!Stop)
+                                    syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
+                            }
+                            catch (Exception)
+                            {
+                            }
+                        }
                     }
                 }
             }

TimberWinR/Parser.cs

@@ -15,6 +15,7 @@
 using NLog.Config;
 using TimberWinR.Outputs;
 using System.CodeDom.Compiler;
+using TimberWinR.Diagnostics;
 
 namespace TimberWinR.Parser
 {
@@ -26,7 +27,7 @@ interface IValidateSchema
     }
 
 
-    public abstract class LogstashFilter : IValidateSchema
+    public abstract class LogstashFilter : IValidateSchema, IDiagnosable
     {
         public abstract bool Apply(JObject json);
 

TimberWinR/TimberWinR.csproj

@@ -96,6 +96,7 @@
     <Compile Include="Configuration.cs" />
     <Compile Include="ConfigurationErrors.cs" />
     <Compile Include="Diagnostics\Diagnostics.cs" />
+    <Compile Include="Diagnostics\IDiagnosable.cs" />
     <Compile Include="Filters\DateFilter.cs" />
     <Compile Include="Filters\FilterBase.cs" />
     <Compile Include="Filters\GrokFilter.cs" />

appveyor.yml

@@ -1,14 +1,50 @@
-version: 1.2.{build}
+version: build_number_{build}
+configuration: Release
+skip_tags: true
+init:
+- ps: "$v = [regex]::match($env:APPVEYOR_REPO_BRANCH,'rel-(.*)').Groups[1].Value\nWrite-Host \"On branch $($env:APPVEYOR_REPO_BRANCH)\"\nIF($v) { \n   $env:VERSION_FROM_BRANCH = \"$($v).$($env:APPVEYOR_BUILD_NUMBER)\"\n} else {\n   $env:VERSION_FROM_BRANCH = \"0.0.0.$($env:APPVEYOR_BUILD_NUMBER)\"\n}\nWrite-Host \"Set version to $($env:VERSION_FROM_BRANCH)\""
+assembly_info:
+  patch: true
+  file: '**\AssemblyInfo.*'
+  assembly_version: $(VERSION_FROM_BRANCH)
+  assembly_file_version: $(VERSION_FROM_BRANCH)
+  assembly_informational_version: $(VERSION_FROM_BRANCH)
+before_build:
+- ps: >-
+    mkdir tools
 
+    NuGet.exe restore TimberWinR.sln
 build:
-  verbosity: minimal
+  verbosity: normal
+after_build:
+- ps: >-
+    cat chocolateyInstall.ps1.template|%{$_-replace "\$\{version\}",$env:VERSION_FROM_BRANCH} > tools\chocolateyInstall.ps1
 
-assembly_info:
-  patch: true
-  file: AssemblyInfo.*
-  assembly_version: "1.2.{build}"
-  assembly_file_version: "{version}"
-  assembly_informational_version: "{version}"
+    cat chocolateyUninstall.ps1.template|%{$_-replace "\$\{version\}",$env:VERSION_FROM_BRANCH} > tools\chocolateyUninstall.ps1
+
+    cat tools\chocolateyUninstall.ps1
+
+    cat timberwinr.nuspec.template|%{$_-replace "\$\{version\}",$env:VERSION_FROM_BRANCH} > timberwinr.nuspec
 
+    choco pack timberwinr.nuspec
 artifacts:
-  - path: '**\*.msi'
+- path: .\*.nupkg
+  name: NuGet
+- path: TimberWinR.ServiceHost\bin\*\*.dll
+  name: Dlls
+- path: TimberWinR.ServiceHost\bin\*\*.exe
+  name: Exes
+- path: TimberWix\bin\*\*.msi
+  name: MSI 
+deploy:
+- provider: NuGet
+  server: https://chocolatey.org/
+  skip_symbols: true
+  artifact: NuGet
+  on:
+    branch: /release/.*/
+- provider: GitHub
+  release: $(VERSION_FROM_BRANCH)
+  artifact: Dlls, Exes, MSI
+  on:
+    branch: /release/.*/