Skip to content
This repository was archived by the owner on Jun 1, 2024. It is now read-only.

Commit 0ceaf81

Browse files
mivanomivano
committed
Merge branch 'dev'
2 parents 3548e64 + 3feba63 commit 0ceaf81

File tree

48 files changed

+2201
-792
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+2201
-792
lines changed

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,3 +189,7 @@ FakesAssemblies/
189189

190190
project.lock.json
191191
.vs
192+
193+
# JetBrains Rider
194+
.idea/
195+
*.sln.iml

‎Build.ps1‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@ echo "In directory: $PSScriptRoot"
22

33
$solution = "serilog-sinks-elasticsearch.sln"
44
$test = "test\\Serilog.Sinks.Elasticsearch.Tests\\Serilog.Sinks.Elasticsearch.Tests.csproj"
5-
$projectFolder = "src\\Serilog.Sinks.Elasticsearch"
6-
$project = $projectFolder + "\\Serilog.Sinks.Elasticsearch.csproj"
5+
[string[]]$projects = @(
6+
("src\\Serilog.Sinks.Elasticsearch\\Serilog.Sinks.Elasticsearch.csproj"),
7+
("src\\Serilog.Formatting.Elasticsearch\\Serilog.Formatting.Elasticsearch.csproj")
8+
)
79

810
function Invoke-Build()
911
{
@@ -22,7 +24,10 @@ function Invoke-Build()
2224
}
2325

2426
Write-Output "Creating packages"
25-
& dotnet pack $project -c Release -o ..\..\artifacts --include-symbols --include-source /p:PackageVersion=$env:GitVersion_NuGetVersionV2
27+
foreach ($project in $projects)
28+
{
29+
& dotnet pack $project -c Release -o ..\..\artifacts --include-symbols --include-source /p:PackageVersion=$env:GitVersion_NuGetVersionV2
30+
}
2631

2732
if($LASTEXITCODE -ne 0)
2833
{

‎CHANGES.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
== Changelog
2-
2+
7.0
3+
* DurableElasticsearchSink is rewritten to use the same base code as the sink for Serilog.Sinks.Seq. Nuget Serilog.Sinks.File is now used instead of deprecated Serilog.Sinks.RollingFile. Lots of new fintuning options for file storage is added in ElasticsearchSinkOptions. Updated Serilog.Sinks.Elasticsearch.Sample.Main with SetupLoggerWithPersistantStorage with all available options for durable mode.
4+
* Changed datatype on singleEventSizePostingLimit from int to long? with default value null. to make it possible ro reuse code from Sinks.Seq .
5+
* IndexDecider didnt worked well in buffer mode because of LogEvent was null. Added BufferIndexDecider.
6+
* Added BufferCleanPayload and an example which makes it possible to cleanup your invalid logging document if rejected from elastic because of inconsistent datatype on a field. It'seasy to miss errors in the self log now its possible to se logrows which is bad for elasticsearch in the elastic log.
7+
* Added BufferRetainedInvalidPayloadsLimitBytes A soft limit for the number of bytes to use for storing failed requests.
8+
* Added BufferFileCountLimit The maximum number of log files that will be retained.
39
6.4
410
* Render message by default (#160).
511
* Expose interface-typed options via appsettings (#162)

‎README.md‎

Lines changed: 103 additions & 42 deletions
Large diffs are not rendered by default.

‎appveyor.yml‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,21 @@ build_script:
55
- ps: ./Build.ps1
66
test: off
77
artifacts:
8-
- path: artifacts/Serilog.Sinks.Elasticsearch*.nupkg
8+
- path: artifacts/Serilog.Sinks.Elasticsearch*.nupkg
9+
- path: artifacts/Serilog.Formatting.Elasticsearch*.nupkg
10+
only_commits:
11+
files:
12+
- serilog-sinks-elasticsearch.sln
13+
- src/Serilog.Sinks.Elasticsearch/
14+
- src/Serilog.Formatting.Elasticsearch/
15+
- nuget.config
16+
- Build.ps1
17+
- assets/
18+
- test/Serilog.Sinks.Elasticsearch.Tests/
919
deploy:
1020
- provider: NuGet
1121
api_key:
12-
secure: nvZ/z+pMS91b3kG4DgfES5AcmwwGoBYQxr9kp4XiJHj25SAlgdIxFx++1N0lFH2x
22+
secure: bd9z4P73oltOXudAjPehwp9iDKsPtC+HbgshOrSgoyQKr5xVK+bxJQngrDJkHdY8
1323
on:
1424
branch: /^(master|dev)$/
1525
- provider: GitHub
@@ -25,4 +35,4 @@ assembly_info:
2535
patch: false
2636
before_build:
2737
- dotnet restore serilog-sinks-elasticsearch.sln
28-
- ps: gitversion /l console /output buildserver /updateAssemblyInfo
38+
- ps: gitversion /l console /output buildserver /updateAssemblyInfo

‎global.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"sdk": {
3+
"version": "2.1.4"
4+
}
5+
}

‎sample/Serilog.Sinks.Elasticsearch.Sample/Program.cs‎

Lines changed: 99 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Reflection.Metadata.Ecma335;
5+
using System.Threading;
6+
using Microsoft.Extensions.Configuration;
7+
using Newtonsoft.Json;
8+
using Newtonsoft.Json.Linq;
29
using Serilog;
10+
using Serilog.Core;
311
using Serilog.Debugging;
12+
using Serilog.Events;
413
using Serilog.Formatting.Json;
514
using Serilog.Sinks.File;
615
using Serilog.Sinks.SystemConsole.Themes;
@@ -9,28 +18,40 @@ namespace Serilog.Sinks.Elasticsearch.Sample
918
{
1019
class Program
1120
{
21+
private static IConfiguration Configuration { get; } = new ConfigurationBuilder()
22+
.SetBasePath(Directory.GetCurrentDirectory())
23+
.AddJsonFile("appsettings.json", true, true)
24+
.AddEnvironmentVariables()
25+
.Build();
1226
static void Main(string[] args)
1327
{
28+
29+
// Enable the selflog output
30+
SelfLog.Enable(Console.Error);
1431
Log.Logger = new LoggerConfiguration()
1532
.MinimumLevel.Debug()
1633
.WriteTo.Console(theme: SystemConsoleTheme.Literate)
17-
.WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://elastic:changeme@localhost:9200")) // for the docker-compose implementation
18-
{
19-
AutoRegisterTemplate = true,
20-
//BufferBaseFilename = "./buffer",
21-
RegisterTemplateFailure = RegisterTemplateRecovery.IndexAnyway,
22-
FailureCallback = e => Console.WriteLine("Unable to submit event " + e.MessageTemplate),
23-
EmitEventFailure = EmitEventFailureHandling.WriteToSelfLog |
24-
EmitEventFailureHandling.WriteToFailureSink |
25-
EmitEventFailureHandling.RaiseCallback,
26-
FailureSink = new FileSink("./failures.txt", new JsonFormatter(), null)
27-
})
28-
.CreateLogger();
34+
//not persistant
35+
.WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(Configuration.GetConnectionString("elasticsearch"))) // for the docker-compose implementation
36+
{
37+
AutoRegisterTemplate = true,
38+
//BufferBaseFilename = "./buffer",
39+
RegisterTemplateFailure = RegisterTemplateRecovery.IndexAnyway,
40+
FailureCallback = e => Console.WriteLine("Unable to submit event " + e.MessageTemplate),
41+
EmitEventFailure = EmitEventFailureHandling.WriteToSelfLog |
42+
EmitEventFailureHandling.WriteToFailureSink |
43+
EmitEventFailureHandling.RaiseCallback,
44+
FailureSink = new FileSink("./fail-{Date}.txt", new JsonFormatter(), null, null)
45+
})
46+
.CreateLogger();
2947

30-
// Enable the selflog output
31-
SelfLog.Enable(Console.Error);
48+
//SetupLoggerWithSimplePersistantStorage();
49+
//LoggingLevelSwitch levelSwitch = SetupLoggerWithPersistantStorage();
3250

51+
//Log.Debug("To high loglevel default is Information this will not be logged");
52+
//levelSwitch.MinimumLevel = LogEventLevel.Debug;
3353
Log.Information("Hello, world!");
54+
//Log.Information("To big log row bigger than SingleEventSizePostingLimit ! {a}", new string('*', 5000));
3455

3556
int a = 10, b = 0;
3657
try
@@ -47,7 +68,70 @@ static void Main(string[] args)
4768
Log.Debug("Reusing {A} by {B}", "string", true);
4869

4970
Log.CloseAndFlush();
50-
Console.Read();
71+
Console.WriteLine("Press any key to continue...");
72+
while (!Console.KeyAvailable)
73+
{
74+
Thread.Sleep(500);
75+
}
76+
}
77+
78+
private static LoggingLevelSwitch SetupLoggerWithPersistantStorage()
79+
{
80+
//persistant storage with all settings available for this mode
81+
//please note that all limit settings here is set verry low for test, default values should usually work best!
82+
var levelSwitch = new LoggingLevelSwitch();
83+
Log.Logger = new LoggerConfiguration()
84+
.MinimumLevel.Debug()
85+
.WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(Configuration.GetConnectionString("elasticsearch")))
86+
{
87+
AutoRegisterTemplate = true,
88+
BufferBaseFilename = "./buffer/logserilog",
89+
IndexFormat = "log-serilog-{0:yyyy.MM}",
90+
RegisterTemplateFailure = RegisterTemplateRecovery.FailSink,
91+
BufferCleanPayload = (failingEvent, statuscode, exception) =>
92+
{
93+
dynamic e = JObject.Parse(failingEvent);
94+
return JsonConvert.SerializeObject(new Dictionary<string, object>()
95+
{
96+
{ "@timestamp",e["@timestamp"]},
97+
{ "level",e.level},
98+
{ "message","Error: "+e.message},
99+
{ "messageTemplate",e.messageTemplate},
100+
{ "failingStatusCode", statuscode},
101+
{ "failingException", exception}
102+
});
103+
},
104+
OverwriteTemplate = true,
105+
NumberOfShards = 1,
106+
NumberOfReplicas = 1,
107+
GetTemplateContent = null,
108+
AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv6,
109+
PipelineName = null,
110+
TypeName = "logevent",
111+
BufferIndexDecider = (logEvent, offset) => "log-serilog-" + (new Random().Next(0, 2)),
112+
BatchPostingLimit = 50,
113+
BufferLogShippingInterval = TimeSpan.FromSeconds(5),
114+
SingleEventSizePostingLimit = 1000,
115+
LevelSwitch = levelSwitch,
116+
BufferRetainedInvalidPayloadsLimitBytes = 2000,
117+
BufferFileSizeLimitBytes = 2000,
118+
BufferFileCountLimit = 2
119+
})
120+
.CreateLogger();
121+
return levelSwitch;
122+
}
123+
124+
private static void SetupLoggerWithSimplePersistantStorage()
125+
{
126+
//presistant
127+
Log.Logger = new LoggerConfiguration()
128+
.MinimumLevel.Debug()
129+
.WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(Configuration.GetConnectionString("elasticsearch")))
130+
{
131+
BufferBaseFilename = "./buffer/logserilogsimple",
132+
IndexFormat = "log-serilog-simple-{0:yyyy.MM}"
133+
})
134+
.CreateLogger();
51135
}
52136
}
53137
}

‎sample/Serilog.Sinks.Elasticsearch.Sample/Serilog.Sinks.Elasticsearch.Sample.csproj‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>netcoreapp1.1</TargetFramework>
5+
<TargetFramework>netcoreapp2.1</TargetFramework>
66
</PropertyGroup>
77

88
<ItemGroup>
9+
<PackageReference Include="Microsoft.Extensions.Configuration" Version="2.1.1" />
10+
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="2.1.1" />
11+
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.1.1" />
12+
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.1.1" />
913
<PackageReference Include="Serilog" Version="2.6.0" />
1014
<PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
1115
</ItemGroup>
@@ -14,4 +18,10 @@
1418
<ProjectReference Include="..\..\src\Serilog.Sinks.Elasticsearch\Serilog.Sinks.Elasticsearch.csproj" />
1519
</ItemGroup>
1620

21+
<ItemGroup>
22+
<None Update="appsettings.json">
23+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
24+
</None>
25+
</ItemGroup>
26+
1727
</Project>

‎sample/Serilog.Sinks.Elasticsearch.Sample/appsettings.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"ConnectionStrings": {
3+
"elasticsearch": "http://elastic:changeme@localhost:9200"
4+
}
5+
}

‎serilog-sinks-elasticsearch.sln‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Microsoft Visual Studio Solution File, Format Version 12.00
1+
Microsoft Visual Studio Solution File, Format Version 12.00
22
# Visual Studio 15
33
VisualStudioVersion = 15.0.26730.3
44
MinimumVisualStudioVersion = 10.0.40219.1
@@ -13,6 +13,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Files", "Files", "{148431F6
1313
LICENSE = LICENSE
1414
nuget.config = nuget.config
1515
README.md = README.md
16+
global.json = global.json
1617
EndProjectSection
1718
EndProject
1819
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serilog.Sinks.Elasticsearch", "src\Serilog.Sinks.Elasticsearch\Serilog.Sinks.Elasticsearch.csproj", "{EEB0D119-687E-444E-BF14-9BDAEC9BA3EF}"
@@ -21,6 +22,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serilog.Sinks.Elasticsearch
2122
EndProject
2223
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serilog.Sinks.Elasticsearch.Sample", "sample\Serilog.Sinks.Elasticsearch.Sample\Serilog.Sinks.Elasticsearch.Sample.csproj", "{253B37AB-D82E-4A5F-BA16-F1BE398818C8}"
2324
EndProject
25+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Serilog.Formatting.Elasticsearch", "src\Serilog.Formatting.Elasticsearch\Serilog.Formatting.Elasticsearch.csproj", "{0E6D34BF-322A-4803-94D1-355F6D5024BE}"
26+
EndProject
2427
Global
2528
GlobalSection(SolutionConfigurationPlatforms) = preSolution
2629
Debug|Any CPU = Debug|Any CPU
@@ -39,6 +42,10 @@ Global
3942
{253B37AB-D82E-4A5F-BA16-F1BE398818C8}.Debug|Any CPU.Build.0 = Debug|Any CPU
4043
{253B37AB-D82E-4A5F-BA16-F1BE398818C8}.Release|Any CPU.ActiveCfg = Release|Any CPU
4144
{253B37AB-D82E-4A5F-BA16-F1BE398818C8}.Release|Any CPU.Build.0 = Release|Any CPU
45+
{0E6D34BF-322A-4803-94D1-355F6D5024BE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
46+
{0E6D34BF-322A-4803-94D1-355F6D5024BE}.Debug|Any CPU.Build.0 = Debug|Any CPU
47+
{0E6D34BF-322A-4803-94D1-355F6D5024BE}.Release|Any CPU.ActiveCfg = Release|Any CPU
48+
{0E6D34BF-322A-4803-94D1-355F6D5024BE}.Release|Any CPU.Build.0 = Release|Any CPU
4249
EndGlobalSection
4350
GlobalSection(SolutionProperties) = preSolution
4451
HideSolutionNode = FALSE

0 commit comments

Comments
 (0)