Skip to content
This repository was archived by the owner on Jun 1, 2024. It is now read-only.

Commit d52685f

Browse files
mivanomivano
committed
Merge branch 'nuget/formatter' of https://github.com/vhatsura/serilog-sinks-elasticsearch into vhatsura-nuget/formatter
2 parents 0898da1 + b4691d6 commit d52685f

30 files changed

+2012
-687
lines changed

‎CHANGES.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
== Changelog
2-
2+
7.0
3+
* DurableElasticsearchSink is rewritten to use the same base code as the sink for Serilog.Sinks.Seq. Nuget Serilog.Sinks.File is now used instead of deprecated Serilog.Sinks.RollingFile. Lots of new fintuning options for file storage is added in ElasticsearchSinkOptions. Updated Serilog.Sinks.Elasticsearch.Sample.Main with SetupLoggerWithPersistantStorage with all available options for durable mode.
4+
* Changed datatype on singleEventSizePostingLimit from int to long? with default value null. to make it possible ro reuse code from Sinks.Seq .
5+
* IndexDecider didnt worked well in buffer mode because of LogEvent was null. Added BufferIndexDecider.
6+
* Added BufferCleanPayload and an example which makes it possible to cleanup your invalid logging document if rejected from elastic because of inconsistent datatype on a field. It'seasy to miss errors in the self log now its possible to se logrows which is bad for elasticsearch in the elastic log.
7+
* Added BufferRetainedInvalidPayloadsLimitBytes A soft limit for the number of bytes to use for storing failed requests.
8+
* Added BufferFileCountLimit The maximum number of log files that will be retained.
39
6.4
410
* Render message by default (#160).
511
* Expose interface-typed options via appsettings (#162)

‎README.md‎

Lines changed: 98 additions & 43 deletions
Large diffs are not rendered by default.

‎sample/Serilog.Sinks.Elasticsearch.Sample/Program.cs‎

Lines changed: 100 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,57 @@
11
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Reflection.Metadata.Ecma335;
5+
using System.Threading;
6+
using Microsoft.Extensions.Configuration;
7+
using Newtonsoft.Json;
8+
using Newtonsoft.Json.Linq;
29
using Serilog;
10+
using Serilog.Core;
311
using Serilog.Debugging;
12+
using Serilog.Events;
413
using Serilog.Formatting.Json;
5-
using Serilog.Sinks.RollingFile;
14+
using Serilog.Sinks.File;
615
using Serilog.Sinks.SystemConsole.Themes;
716

817
namespace Serilog.Sinks.Elasticsearch.Sample
918
{
1019
class Program
1120
{
21+
private static IConfiguration Configuration { get; } = new ConfigurationBuilder()
22+
.SetBasePath(Directory.GetCurrentDirectory())
23+
.AddJsonFile("appsettings.json", true, true)
24+
.AddEnvironmentVariables()
25+
.Build();
1226
static void Main(string[] args)
1327
{
28+
29+
// Enable the selflog output
30+
SelfLog.Enable(Console.Error);
1431
Log.Logger = new LoggerConfiguration()
1532
.MinimumLevel.Debug()
1633
.WriteTo.Console(theme: SystemConsoleTheme.Literate)
17-
.WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://elastic:changeme@localhost:9200")) // for the docker-compose implementation
18-
{
19-
AutoRegisterTemplate = true,
20-
//BufferBaseFilename = "./buffer",
21-
RegisterTemplateFailure = RegisterTemplateRecovery.IndexAnyway,
22-
FailureCallback = e => Console.WriteLine("Unable to submit event " + e.MessageTemplate),
23-
EmitEventFailure = EmitEventFailureHandling.WriteToSelfLog |
24-
EmitEventFailureHandling.WriteToFailureSink |
25-
EmitEventFailureHandling.RaiseCallback,
26-
FailureSink = new RollingFileSink("./fail-{Date}.txt", new JsonFormatter(), null, null)
27-
})
28-
.CreateLogger();
34+
//not persistant
35+
.WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(Configuration.GetConnectionString("elasticsearch"))) // for the docker-compose implementation
36+
{
37+
AutoRegisterTemplate = true,
38+
//BufferBaseFilename = "./buffer",
39+
RegisterTemplateFailure = RegisterTemplateRecovery.IndexAnyway,
40+
FailureCallback = e => Console.WriteLine("Unable to submit event " + e.MessageTemplate),
41+
EmitEventFailure = EmitEventFailureHandling.WriteToSelfLog |
42+
EmitEventFailureHandling.WriteToFailureSink |
43+
EmitEventFailureHandling.RaiseCallback,
44+
FailureSink = new FileSink("./fail-{Date}.txt", new JsonFormatter(), null, null)
45+
})
46+
.CreateLogger();
2947

30-
// Enable the selflog output
31-
SelfLog.Enable(Console.Error);
48+
//SetupLoggerWithSimplePersistantStorage();
49+
//LoggingLevelSwitch levelSwitch = SetupLoggerWithPersistantStorage();
3250

51+
//Log.Debug("To high loglevel default is Information this will not be logged");
52+
//levelSwitch.MinimumLevel = LogEventLevel.Debug;
3353
Log.Information("Hello, world!");
54+
//Log.Information("To big log row bigger than SingleEventSizePostingLimit ! {a}", new string('*', 5000));
3455

3556
int a = 10, b = 0;
3657
try
@@ -47,7 +68,70 @@ static void Main(string[] args)
4768
Log.Debug("Reusing {A} by {B}", "string", true);
4869

4970
Log.CloseAndFlush();
50-
Console.Read();
71+
Console.WriteLine("Press any key to continue...");
72+
while (!Console.KeyAvailable)
73+
{
74+
Thread.Sleep(500);
75+
}
76+
}
77+
78+
private static LoggingLevelSwitch SetupLoggerWithPersistantStorage()
79+
{
80+
//persistant storage with all settings available for this mode
81+
//please note that all limit settings here is set verry low for test, default values should usually work best!
82+
var levelSwitch = new LoggingLevelSwitch();
83+
Log.Logger = new LoggerConfiguration()
84+
.MinimumLevel.Debug()
85+
.WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(Configuration.GetConnectionString("elasticsearch")))
86+
{
87+
AutoRegisterTemplate = true,
88+
BufferBaseFilename = "./buffer/logserilog",
89+
IndexFormat = "log-serilog-{0:yyyy.MM}",
90+
RegisterTemplateFailure = RegisterTemplateRecovery.FailSink,
91+
BufferCleanPayload = (failingEvent, statuscode, exception) =>
92+
{
93+
dynamic e = JObject.Parse(failingEvent);
94+
return JsonConvert.SerializeObject(new Dictionary<string, object>()
95+
{
96+
{ "@timestamp",e["@timestamp"]},
97+
{ "level",e.level},
98+
{ "message","Error: "+e.message},
99+
{ "messageTemplate",e.messageTemplate},
100+
{ "failingStatusCode", statuscode},
101+
{ "failingException", exception}
102+
});
103+
},
104+
OverwriteTemplate = true,
105+
NumberOfShards = 1,
106+
NumberOfReplicas = 1,
107+
GetTemplateContent = null,
108+
AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6,
109+
PipelineName = null,
110+
TypeName = "logevent",
111+
BufferIndexDecider = (logEvent, offset) => "log-serilog-" + (new Random().Next(0, 2)),
112+
BatchPostingLimit = 50,
113+
BufferLogShippingInterval = TimeSpan.FromSeconds(5),
114+
SingleEventSizePostingLimit = 1000,
115+
LevelSwitch = levelSwitch,
116+
BufferRetainedInvalidPayloadsLimitBytes = 2000,
117+
BufferFileSizeLimitBytes = 2000,
118+
BufferFileCountLimit = 2
119+
})
120+
.CreateLogger();
121+
return levelSwitch;
122+
}
123+
124+
private static void SetupLoggerWithSimplePersistantStorage()
125+
{
126+
//presistant
127+
Log.Logger = new LoggerConfiguration()
128+
.MinimumLevel.Debug()
129+
.WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(Configuration.GetConnectionString("elasticsearch")))
130+
{
131+
BufferBaseFilename = "./buffer/logserilogsimple",
132+
IndexFormat = "log-serilog-simple-{0:yyyy.MM}"
133+
})
134+
.CreateLogger();
51135
}
52136
}
53137
}

‎sample/Serilog.Sinks.Elasticsearch.Sample/Serilog.Sinks.Elasticsearch.Sample.csproj‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>netcoreapp1.1</TargetFramework>
5+
<TargetFramework>netcoreapp2.1</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>
9+
<PackageReference Include="Microsoft.Extensions.Configuration" Version="2.1.1" />
10+
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="2.1.1" />
11+
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.1.1" />
12+
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.1.1" />
913
<PackageReference Include="Serilog" Version="2.6.0" />
1014
<PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
1115
</ItemGroup>
@@ -14,4 +18,10 @@
1418
<ProjectReference Include="..\..\src\Serilog.Sinks.Elasticsearch\Serilog.Sinks.Elasticsearch.csproj" />
1519
</ItemGroup>
1620

21+
<ItemGroup>
22+
<None Update="appsettings.json">
23+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
24+
</None>
25+
</ItemGroup>
26+
1727
</Project>

‎sample/Serilog.Sinks.Elasticsearch.Sample/appsettings.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"ConnectionStrings": {
3+
"elasticsearch": "http://elastic:changeme@localhost:9200"
4+
}
5+
}

‎src/Serilog.Sinks.Elasticsearch/LoggerConfigurationElasticSearchExtensions.cs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
using System.ComponentModel;
2525
using Elasticsearch.Net;
2626
using Serilog.Formatting;
27+
using Serilog.Sinks.Elasticsearch.Durable;
2728

2829
namespace Serilog
2930
{
@@ -119,6 +120,7 @@ public static LoggerConfiguration Elasticsearch(
119120
/// <param name="levelSwitch">A switch allowing the pass-through minimum level to be changed at runtime.</param>
120121
/// <param name="bufferBaseFilename"><see cref="ElasticsearchSinkOptions.BufferBaseFilename"/></param>
121122
/// <param name="bufferFileSizeLimitBytes"><see cref="ElasticsearchSinkOptions.BufferFileSizeLimitBytes"/></param>
123+
/// <param name="bufferFileCountLimit"><see cref="ElasticsearchSinkOptions.BufferFileCountLimit"/></param>
122124
/// <param name="bufferLogShippingInterval"><see cref="ElasticsearchSinkOptions.BufferLogShippingInterval"/></param>
123125
/// <param name="connectionGlobalHeaders">A comma or semi column separated list of key value pairs of headers to be added to each elastic http request</param>
124126
/// <param name="connectionTimeout"><see cref="ElasticsearchSinkOptions.ConnectionTimeout"/>The connection timeout (in seconds) when sending bulk operations to elasticsearch (defaults to 5).</param>
@@ -139,7 +141,7 @@ public static LoggerConfiguration Elasticsearch(
139141
/// <param name="customFormatter">Customizes the formatter used when converting log events into ElasticSearch documents. Please note that the formatter output must be valid JSON :)</param>
140142
/// <param name="customDurableFormatter">Customizes the formatter used when converting log events into the durable sink. Please note that the formatter output must be valid JSON :)</param>
141143
/// <param name="failureSink">Sink to use when Elasticsearch is unable to accept the events. This is optionally and depends on the EmitEventFailure setting.</param>
142-
/// <param name="singleEventSizePostingLimit"><see cref="ElasticsearchSinkOptions.SingleEventSizePostingLimit"/>The maximum length of an event allowed to be posted to Elasticsearch.</param>
144+
/// <param name="singleEventSizePostingLimit"><see cref="ElasticsearchSinkOptions.SingleEventSizePostingLimit"/>The maximum length of an event allowed to be posted to Elasticsearch.default null</param>
143145
/// <returns>LoggerConfiguration object</returns>
144146
/// <exception cref="ArgumentNullException"><paramref name="nodeUris"/> is <see langword="null" />.</exception>
145147
public static LoggerConfiguration Elasticsearch(
@@ -153,7 +155,7 @@ public static LoggerConfiguration Elasticsearch(
153155
bool inlineFields = false,
154156
LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum,
155157
string bufferBaseFilename = null,
156-
long? bufferFileSizeLimitBytes = null,
158+
long? bufferFileSizeLimitBytes = null,
157159
long bufferLogShippingInterval = 5000,
158160
string connectionGlobalHeaders = null,
159161
LoggingLevelSwitch levelSwitch = null,
@@ -175,7 +177,8 @@ public static LoggerConfiguration Elasticsearch(
175177
ITextFormatter customFormatter = null,
176178
ITextFormatter customDurableFormatter = null,
177179
ILogEventSink failureSink = null,
178-
int singleEventSizePostingLimit = 0)
180+
long? singleEventSizePostingLimit = null,
181+
int? bufferFileCountLimit = null)
179182
{
180183
if (string.IsNullOrEmpty(nodeUris))
181184
throw new ArgumentNullException(nameof(nodeUris), "No Elasticsearch node(s) specified.");
@@ -220,6 +223,10 @@ public static LoggerConfiguration Elasticsearch(
220223
options.BufferFileSizeLimitBytes = bufferFileSizeLimitBytes.Value;
221224
}
222225

226+
if (bufferFileCountLimit.HasValue)
227+
{
228+
options.BufferFileCountLimit = bufferFileCountLimit.Value;
229+
}
223230
options.BufferLogShippingInterval = TimeSpan.FromMilliseconds(bufferLogShippingInterval);
224231

225232
if (!string.IsNullOrWhiteSpace(connectionGlobalHeaders))

‎src/Serilog.Sinks.Elasticsearch/Serilog.Sinks.Elasticsearch.csproj‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,23 +28,43 @@
2828
<GenerateAssemblyVersionAttribute>false</GenerateAssemblyVersionAttribute>
2929
</PropertyGroup>
3030

31+
<ItemGroup>
32+
<Compile Remove="Sinks\ElasticSearch\Durable\ElasticSearch\ElasticSearchLogShipperOld.cs" />
33+
</ItemGroup>
34+
3135
<ItemGroup>
3236
<PackageReference Include="Elasticsearch.Net" Version="6.0.0" />
3337
<PackageReference Include="Serilog" Version="2.6.0" />
38+
<PackageReference Include="Serilog.Formatting.Compact" Version="1.0.0" />
3439
<PackageReference Include="Serilog.Sinks.File" Version="4.0.0" />
3540
<PackageReference Include="Serilog.Sinks.PeriodicBatching" Version="2.1.1" />
36-
<PackageReference Include="Serilog.Sinks.RollingFile" Version="3.3.0" />
3741
</ItemGroup>
3842

3943
<ItemGroup Condition=" '$(TargetFramework)' == 'net45' ">
4044
<Reference Include="System" />
4145
<Reference Include="Microsoft.CSharp" />
4246
</ItemGroup>
4347

48+
<PropertyGroup Condition=" '$(TargetFramework)' == 'netstandard1.3' ">
49+
<NoWarn>1591;1701;1702</NoWarn>
50+
<DefineConstants>$(DefineConstants);DURABLE;THREADING_TIMER</DefineConstants>
51+
</PropertyGroup>
52+
53+
<PropertyGroup Condition=" '$(TargetFramework)' == 'net45' ">
54+
<NoWarn>1591;1701;1702</NoWarn>
55+
<DefineConstants>$(DefineConstants);DURABLE;THREADING_TIMER;HRESULTS</DefineConstants>
56+
</PropertyGroup>
57+
4458
<PropertyGroup Condition=" '$(TargetFramework)' == 'netstandard1.3' ">
59+
<NoWarn>1591;1701;1702</NoWarn>
4560
<DefineConstants>$(DefineConstants);DOTNETCORE;NO_SERIALIZATION;NO_TIMER</DefineConstants>
4661
</PropertyGroup>
4762

63+
<PropertyGroup Condition="'$(Configuration)|$(TargetFramework)|$(Platform)'=='Debug|net45|AnyCPU'">
64+
<NoWarn>1591;1701;1702</NoWarn>
65+
<WarningsAsErrors>NU1605</WarningsAsErrors>
66+
</PropertyGroup>
67+
4868
<ItemGroup>
4969
<DotNetCliToolReference Include="dotnet-version" Version="1.1.0" />
5070
</ItemGroup>

0 commit comments

Comments
 (0)