Skip to content

Commit 38b9927

Browse files
KodrAusKodrAus
authored
Merge pull request #261 from datalust/dev
2022.1.632 Release
2 parents c088250 + 63d1cfd commit 38b9927

File tree

9 files changed

+369
-3
lines changed

9 files changed

+369
-3
lines changed

‎src/SeqCli/Cli/Commands/Bench/BenchCase.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Copyright Datalust Pty Ltd and Contributors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#nullable enable
16+
using System.Collections.Generic;
17+
18+
namespace SeqCli.Cli.Commands;
19+
20+
class BenchCase
21+
{
22+
public string Id = "";
23+
public string Query = "";
24+
public string SignalExpression = "";
25+
}

‎src/SeqCli/Cli/Commands/Bench/BenchCaseTimings.cs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Copyright Datalust Pty Ltd and Contributors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#nullable enable
16+
using System;
17+
using System.Collections.Generic;
18+
using System.Linq;
19+
20+
namespace SeqCli.Cli.Commands;
21+
22+
/*
23+
* Collects benchmarking elapsed time measurements and calculates statistics.
24+
*/
25+
class BenchCaseTimings
26+
{
27+
readonly List<double> _elaspseds = new() { };
28+
public double MeanElapsed => _elaspseds.Sum() / _elaspseds.Count;
29+
public double MinElapsed => _elaspseds.Min();
30+
public double MaxElapsed => _elaspseds.Max();
31+
public double StandardDeviationElapsed => StandardDeviation(_elaspseds);
32+
public double RelativeStandardDeviationElapsed => StandardDeviation(_elaspseds) / MeanElapsed;
33+
34+
public void PushElapsed(double elapsed)
35+
{
36+
_elaspseds.Add(elapsed);
37+
}
38+
39+
double StandardDeviation(IList<double> population)
40+
{
41+
if (population.Count < 2)
42+
{
43+
return 0;
44+
}
45+
var mean = population.Sum() / population.Count;
46+
return Math.Sqrt(population.Select(e => Math.Pow(e - mean, 2)).Sum() / (population.Count - 1));
47+
}
48+
}

‎src/SeqCli/Cli/Commands/Bench/BenchCases.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"cases": [
3+
{
4+
"id": "count-star",
5+
"query": "select count(*) from stream"
6+
},
7+
{
8+
"id": "starts-with",
9+
"query": "select count(*) from stream where @Message like '%abcde'"
10+
},
11+
{
12+
"id": "without-signal",
13+
"query": "select count(*) from stream where @Level = 'Warning'"
14+
},
15+
{
16+
"id": "property-match",
17+
"query": "select count(*) from stream where Action = 'ListAsync'"
18+
}
19+
]
20+
}

‎src/SeqCli/Cli/Commands/Bench/BenchCasesCollection.cs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Copyright Datalust Pty Ltd and Contributors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#nullable enable
16+
using System.Collections.Generic;
17+
18+
namespace SeqCli.Cli.Commands;
19+
20+
/*
21+
* A target type for deserialization of bench case files.
22+
*/
23+
class BenchCasesCollection
24+
{
25+
// An identifier for the particular cases file
26+
public int CasesHash = 0;
27+
public IList<BenchCase> Cases = new List<BenchCase>();
28+
}

‎src/SeqCli/Cli/Commands/Bench/BenchCommand.cs‎

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
// Copyright Datalust Pty Ltd and Contributors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#nullable enable
16+
17+
using System;
18+
using System.IO;
19+
using System.Linq;
20+
using System.Threading.Tasks;
21+
using Newtonsoft.Json;
22+
using Seq.Api.Model.Signals;
23+
using SeqCli.Cli.Features;
24+
using SeqCli.Connection;
25+
using SeqCli.Util;
26+
using Serilog;
27+
using Serilog.Context;
28+
using Serilog.Core;
29+
30+
namespace SeqCli.Cli.Commands;
31+
32+
/*
33+
* Run performance benchmark tests against a Seq server.
34+
*
35+
* Requires test cases in a JSON file matching the format of `BenchCases.json`.
36+
*
37+
* If a Seq reporting server is configured this command logs data such as:
38+
*
39+
* {
40+
"@t": "2022-11-09T01:12:06.0293545Z",
41+
"@mt": "Bench run {BenchRunId} for query {Id}. Mean {MeanElapsed:N0} ms with relative dispersion {RelativeStandardDeviationElapsed:N2}",
42+
"@m": "Bench run \"2907\" for query \"with-signal\". Mean 4 ms with relative dispersion 0.06",
43+
"@i": "bb0c84a5",
44+
"@r": [
45+
"4",
46+
"0.06"
47+
],
48+
"BenchRunId": "2907",
49+
"End": "2022-08-15T00:00:00.0000000",
50+
"Id": "with-signal",
51+
"LastResult": 606,
52+
"MaxElapsed": 4.082,
53+
"MeanElapsed": 3.6676,
54+
"MinElapsed": 3.4334,
55+
"Query": "select count(*) from stream where @Level = 'Warning'",
56+
"RelativeStandardDeviationElapsed": 0.05619408341421253,
57+
"Runs": 10,
58+
"SignalExpression": "signal-m33302",
59+
"Start": "2022-08-14T16:00:00.0000000"
60+
}
61+
*/
62+
[Command("bench", @"Measure query performance.
63+
64+
Example cases file format:
65+
66+
{
67+
""cases"": [
68+
{
69+
""id"": ""count-star"",
70+
""query"": ""select count(*) from stream"",
71+
""signalExpression"": ""signal-expression-here""]
72+
}
73+
]
74+
}
75+
")]
76+
class BenchCommand : Command
77+
{
78+
readonly SeqConnectionFactory _connectionFactory;
79+
int _runs = 3;
80+
readonly ConnectionFeature _connection;
81+
readonly DateRangeFeature _range;
82+
string _cases = "";
83+
string _reportingServerUrl = "";
84+
string _reportingServerApiKey = "";
85+
86+
public BenchCommand(SeqConnectionFactory connectionFactory)
87+
{
88+
_connectionFactory = connectionFactory;
89+
Options.Add("r|runs=", "The number of runs to execute", r =>
90+
{
91+
int.TryParse(r, out _runs);
92+
});
93+
94+
Options.Add(
95+
"c|cases=",
96+
@"A JSON file containing the set of cases to run. Defaults to a standard set of cases.",
97+
c => _cases = c);
98+
99+
_connection = Enable<ConnectionFeature>();
100+
_range = Enable<DateRangeFeature>();
101+
102+
Options.Add(
103+
"reporting-server=",
104+
"The address of a Seq server to send bench results to",
105+
s => _reportingServerUrl = s);
106+
Options.Add(
107+
"reporting-apikey=",
108+
"The API key to use when connecting to the reporting server",
109+
a => _reportingServerApiKey = a);
110+
}
111+
112+
protected override async Task<int> Run()
113+
{
114+
try
115+
{
116+
var connection = _connectionFactory.Connect(_connection);
117+
using var reportingLogger = BuildReportingLogger();
118+
var cases = ReadCases(_cases);
119+
var runId = Guid.NewGuid().ToString("N").Substring(0, 4);
120+
var start = _range.Start ?? DateTime.UtcNow.AddDays(-7);
121+
var end = _range.End;
122+
123+
foreach (var c in cases.Cases)
124+
{
125+
var timings = new BenchCaseTimings();
126+
object? lastResult = null;
127+
128+
foreach (var i in Enumerable.Range(1, _runs))
129+
{
130+
var response = await connection.Data.QueryAsync(
131+
c.Query,
132+
start,
133+
end,
134+
SignalExpressionPart.Signal(c.SignalExpression)
135+
);
136+
137+
timings.PushElapsed(response.Statistics.ElapsedMilliseconds);
138+
139+
if (response.Rows != null)
140+
{
141+
var isScalarResult = response.Rows.Length == 1 && response.Rows[0].Length == 1;
142+
if (isScalarResult && i == _runs)
143+
{
144+
lastResult = response.Rows[0][0];
145+
}
146+
}
147+
}
148+
149+
using (lastResult != null ? LogContext.PushProperty("LastResult", lastResult) : null)
150+
using (LogContext.PushProperty("MinElapsed", timings.MinElapsed))
151+
using (LogContext.PushProperty("MaxElapsed", timings.MaxElapsed))
152+
using (LogContext.PushProperty("Runs", _runs))
153+
using (LogContext.PushProperty("SignalExpression", c.SignalExpression))
154+
using (LogContext.PushProperty("Start", start))
155+
using (LogContext.PushProperty("StandardDeviationElapsed", timings.StandardDeviationElapsed))
156+
using (end != null ? LogContext.PushProperty("End", end) : null)
157+
using (LogContext.PushProperty("Query", c.Query))
158+
{
159+
reportingLogger.Information(
160+
"Bench run {Cases}/{RunId} against {Server} for query {Id}: mean {MeanElapsed:N0} ms with relative dispersion {RelativeStandardDeviationElapsed:N2}",
161+
cases.CasesHash, runId, _reportingServerUrl, c.Id, timings.MeanElapsed, timings.RelativeStandardDeviationElapsed);
162+
}
163+
}
164+
165+
return 0;
166+
}
167+
catch (Exception ex)
168+
{
169+
Log.Error(ex, "Benchmarking failed: {ErrorMessage}", ex.Message);
170+
return 1;
171+
}
172+
}
173+
174+
/// <summary>
175+
/// Build a second Serilog logger for logging benchmark results.
176+
/// </summary>
177+
Logger BuildReportingLogger()
178+
{
179+
var loggerConfiguration = new LoggerConfiguration()
180+
.Enrich.FromLogContext()
181+
.WriteTo.Console();
182+
183+
if (!string.IsNullOrWhiteSpace(_reportingServerUrl))
184+
loggerConfiguration.WriteTo.Seq(
185+
_reportingServerUrl,
186+
apiKey: string.IsNullOrWhiteSpace(_reportingServerApiKey) ? null : _reportingServerApiKey,
187+
period: TimeSpan.FromMilliseconds(1));
188+
189+
return loggerConfiguration.CreateLogger();
190+
}
191+
192+
/// <summary>
193+
/// Read and parse the bench test cases from the file supplied or else from a default file.
194+
/// </summary>
195+
static BenchCasesCollection ReadCases(string filename)
196+
{
197+
var defaultCasesPath = Content.GetPath("Cli/Commands/Bench/BenchCases.json");
198+
var casesString = File.ReadAllText(string.IsNullOrWhiteSpace(filename)
199+
? defaultCasesPath
200+
: filename);
201+
var casesFile = JsonConvert.DeserializeObject<BenchCasesCollection>(casesString)
202+
?? new BenchCasesCollection();
203+
204+
casesFile.CasesHash = casesString.GetHashCode(); // not consistent across framework versions, but that's OK
205+
206+
if (casesFile.Cases.Select(c => c.Id).Distinct().Count() != casesFile.Cases.Count)
207+
{
208+
throw new Exception($"Cases file {filename} contains a duplicate id");
209+
}
210+
211+
if (!casesFile.Cases.Any())
212+
{
213+
throw new Exception($"Cases file {filename} contains no cases");
214+
}
215+
216+
return casesFile;
217+
}
218+
}

‎src/SeqCli/Cli/Features/DateRangeFeature.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class DateRangeFeature : CommandFeature
2323
public override void Enable(OptionSet options)
2424
{
2525
options.Add("start=", "ISO 8601 date/time to query from", v => _start = v);
26-
options.Add("end=", "Date/time to query to", v => _end = v);
26+
options.Add("end=", "ISO 8601 date/time to query to", v => _end = v);
2727
}
2828

2929
public DateTime? Start => _start != null ? DateTime.Parse(_start) : (DateTime?)null;

‎src/SeqCli/SeqCli.csproj‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,25 +10,30 @@
1010
<TreatSpecificWarningsAsErrors />
1111
<PackAsTool>true</PackAsTool>
1212
<ToolCommandName>seqcli</ToolCommandName>
13+
<LangVersion>default</LangVersion>
1314
</PropertyGroup>
1415
<ItemGroup>
1516
<Content Include="..\..\asset\SeqCli.ico" Link="SeqCli.ico" />
1617
<Content Include="Attribution\*.txt;..\..\LICENSE">
1718
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
1819
</Content>
20+
<Content Include="Cli\Commands\Bench\BenchCases.json">
21+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
22+
</Content>
1923
<Content Include="Sample\Templates\*.template">
2024
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
2125
</Content>
2226
</ItemGroup>
2327
<ItemGroup>
2428
<PackageReference Include="Destructurama.JsonNet" Version="2.0.0" />
2529
<PackageReference Include="newtonsoft.json" Version="13.0.1" />
26-
<PackageReference Include="Serilog" Version="2.11.0" />
30+
<PackageReference Include="Serilog" Version="2.12.0" />
2731
<PackageReference Include="serilog.filters.expressions" Version="2.1.0" />
2832
<PackageReference Include="Serilog.Formatting.Compact" Version="1.1.0" />
2933
<PackageReference Include="Serilog.Formatting.Compact.Reader" Version="1.0.5" />
3034
<PackageReference Include="Serilog.Sinks.Console" Version="4.0.1" />
3135
<PackageReference Include="Autofac" Version="6.4.0" />
36+
<PackageReference Include="Serilog.Sinks.Seq" Version="5.2.1" />
3237
<PackageReference Include="Superpower" Version="3.0.0" />
3338
<PackageReference Include="System.Reactive" Version="5.0.0" />
3439
<PackageReference Include="System.Security.Cryptography.ProtectedData" Version="6.0.0" />

0 commit comments

Comments
 (0)