Skip to content

Commit 2a2598b

Browse files
liammclennanliammclennan
committed
Documenting seqcli bench 
1 parent 05827de commit 2a2598b

File tree

9 files changed

+371
-181
lines changed

9 files changed

+371
-181
lines changed

‎src/SeqCli/Cli/Commands/Bench/BenchCase.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Copyright 2018 Datalust Pty Ltd
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#nullable enable
16+
using System.Collections.Generic;
17+
18+
namespace SeqCli.Cli.Commands;
19+
20+
class BenchCase
21+
{
22+
public string Id = "";
23+
public string Query = "";
24+
public IList<string> Signals = new List<string>();
25+
}

‎src/SeqCli/Cli/Commands/Bench/BenchCaseTimings.cs‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
// Copyright 2018 Datalust Pty Ltd
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#nullable enable
16+
using System;
17+
using System.Collections.Generic;
18+
using System.Linq;
19+
20+
namespace SeqCli.Cli.Commands;
21+
22+
class BenchCaseTimings
23+
{
24+
List<double> _elaspseds = new() { };
25+
26+
public double MeanElapsed => _elaspseds.Sum() / _elaspseds.Count;
27+
public double MinElapsed => _elaspseds.Min();
28+
public double MaxElapsed => _elaspseds.Max();
29+
30+
public double StandardDeviationElapsed => StandardDeviation(_elaspseds);
31+
public double RelativeStandardDeviationElapsed => StandardDeviation(_elaspseds) / MeanElapsed;
32+
33+
public void PushElapsed(double elapsed)
34+
{
35+
_elaspseds.Add(elapsed);
36+
}
37+
38+
double StandardDeviation(IList<double> population)
39+
{
40+
if (population.Count < 2)
41+
{
42+
return 0;
43+
}
44+
var mean = population.Sum() / population.Count;
45+
return Math.Sqrt(population.Select(e => Math.Pow(e - mean, 2)).Sum() / (population.Count - 1));
46+
}
47+
}

‎src/SeqCli/Cli/Commands/Bench/BenchCases.json‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{
2+
"cases": [
3+
{
4+
"id": "count-star",
5+
"query": "select count(*) from stream",
6+
"signals": []
7+
},
8+
{
9+
"id": "starts-with",
10+
"query": "select count(*) from stream where @Message like '%abcde'",
11+
"signals": []
12+
},
13+
{
14+
"id": "mean-filter",
15+
"query": "select mean(RowCount) from stream where @EventType = 0x94002EBF",
16+
"signals": []
17+
},
18+
{
19+
"id": "without-signal",
20+
"query": "select count(*) from stream where @Level = 'Warning'",
21+
"signals": []
22+
},
23+
{
24+
"id": "with-signal",
25+
"query": "select count(*) from stream where @Level = 'Warning'",
26+
"signals": ["signal-m33302"]
27+
},
28+
{
29+
"id": "property-match",
30+
"query": "select count(*) from stream where Action = 'ListAsync'",
31+
"signals": []
32+
}
33+
]
34+
}

‎src/SeqCli/Cli/Commands/Bench/BenchCasesCollection.cs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// Copyright 2018 Datalust Pty Ltd
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#nullable enable
16+
using System.Collections.Generic;
17+
18+
namespace SeqCli.Cli.Commands;
19+
20+
/*
21+
* A target type for deserialization of bench case files.
22+
*/
23+
class BenchCasesCollection
24+
{
25+
public IList<BenchCase> Cases = new List<BenchCase>();
26+
}

‎src/SeqCli/Cli/Commands/Bench/BenchCommand.cs‎

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
// Copyright 2018 Datalust Pty Ltd
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#nullable enable
16+
17+
using System;
18+
using System.IO;
19+
using System.Linq;
20+
using System.Threading.Tasks;
21+
using Newtonsoft.Json;
22+
using Seq.Api.Model.Signals;
23+
using SeqCli.Cli.Features;
24+
using SeqCli.Connection;
25+
using Serilog;
26+
using Serilog.Context;
27+
using Serilog.Core;
28+
29+
namespace SeqCli.Cli.Commands;
30+
31+
/*
32+
* Run performance benchmark tests against a Seq server.
33+
*
34+
* Requires test cases in a JSON file matching the format of `BenchCases.json`.
35+
*
36+
* If a Seq reporting server is configured this command logs data such as:
37+
*
38+
* {
39+
"@t": "2022-11-09T01:12:06.0293545Z",
40+
"@mt": "Bench run {BenchRunId} for query {Id}. Mean {MeanElapsed:N0} ms with relative dispersion {RelativeStandardDeviationElapsed:N2}",
41+
"@m": "Bench run \"2907\" for query \"with-signal\". Mean 4 ms with relative dispersion 0.06",
42+
"@i": "bb0c84a5",
43+
"@r": [
44+
"4",
45+
"0.06"
46+
],
47+
"BenchRunId": "2907",
48+
"End": "2022-08-15T00:00:00.0000000",
49+
"Id": "with-signal",
50+
"LastResult": 606,
51+
"MaxElapsed": 4.082,
52+
"MeanElapsed": 3.6676,
53+
"MinElapsed": 3.4334,
54+
"Query": "select count(*) from stream where @Level = 'Warning'",
55+
"RelativeStandardDeviationElapsed": 0.05619408341421253,
56+
"Runs": 10,
57+
"Signals": [
58+
"signal-m33302"
59+
],
60+
"Start": "2022-08-14T16:00:00.0000000"
61+
}
62+
*/
63+
[Command("bench", @"Measure query performance.
64+
65+
Example cases file format:
66+
67+
{
68+
""cases"": [
69+
{
70+
""id"": ""count-star"",
71+
""query"": ""select count(*) from stream"",
72+
""signals"": [""signal-id-here""]
73+
}
74+
]
75+
}
76+
")]
77+
class BenchCommand : Command
78+
{
79+
readonly SeqConnectionFactory _connectionFactory;
80+
int _runs = 0;
81+
readonly ConnectionFeature _connection;
82+
readonly DateRangeFeature _range;
83+
string _cases = "";
84+
string _reportingServerUrl = "";
85+
string _reportingServerApiKey = "";
86+
ILogger? _reportingLogger = null;
87+
88+
const int DefaultRuns = 3;
89+
90+
public BenchCommand(SeqConnectionFactory connectionFactory)
91+
{
92+
_connectionFactory = connectionFactory;
93+
_runs = DefaultRuns;
94+
Options.Add("r|runs=", "The number of runs to execute", r =>
95+
{
96+
if (!int.TryParse(r, out _runs))
97+
{
98+
_runs = DefaultRuns;
99+
}
100+
});
101+
102+
Options.Add(
103+
"c|cases=",
104+
@"A JSON file containing the set of cases to run. Defaults to a standard set of cases.",
105+
c => _cases = c);
106+
107+
_connection = Enable<ConnectionFeature>();
108+
_range = Enable<DateRangeFeature>();
109+
110+
Options.Add(
111+
"reporting-server=",
112+
"The address of a Seq server to send bench results to",
113+
s => _reportingServerUrl = s);
114+
Options.Add(
115+
"reporting-apikey=",
116+
"The API key to use when connecting to the reporting server",
117+
a => _reportingServerApiKey = a);
118+
}
119+
120+
protected override async Task<int> Run()
121+
{
122+
try
123+
{
124+
var connection = _connectionFactory.Connect(_connection);
125+
_reportingLogger = BuildReportingLogger();
126+
127+
var cases = ReadCases(_cases);
128+
var runId = Guid.NewGuid().ToString("N").Substring(0, 4);
129+
var start = _range.Start ?? DateTime.UtcNow.AddDays(-7);
130+
var end = _range.End;
131+
132+
foreach (var c in cases.Cases)
133+
{
134+
var timings = new BenchCaseTimings();
135+
object? lastResult = null;
136+
137+
foreach (var i in Enumerable.Range(1, _runs))
138+
{
139+
var response = await connection.Data.QueryAsync(
140+
c.Query,
141+
_range.Start ?? DateTime.UtcNow.AddDays(-7),
142+
_range.End,
143+
SignalExpressionPart.FromIntersectedIds(c.Signals)
144+
);
145+
146+
timings.PushElapsed(response.Statistics.ElapsedMilliseconds);
147+
var isScalarResult = response.Rows.Length == 1 && response.Rows[0].Length == 1;
148+
149+
if (i == _runs && isScalarResult)
150+
{
151+
lastResult = response.Rows[0][0];
152+
}
153+
}
154+
155+
using (lastResult != null ? LogContext.PushProperty("LastResult", lastResult) : null)
156+
using (LogContext.PushProperty("MinElapsed", timings.MinElapsed))
157+
using (LogContext.PushProperty("MaxElapsed", timings.MaxElapsed))
158+
using (LogContext.PushProperty("Runs", _runs))
159+
using (LogContext.PushProperty("Signals", c.Signals))
160+
using (LogContext.PushProperty("Start", start))
161+
using (LogContext.PushProperty("StandardDeviationElapsed", timings.StandardDeviationElapsed))
162+
using (end != null ? LogContext.PushProperty("End", end) : null)
163+
using (LogContext.PushProperty("Query", c.Query))
164+
{
165+
_reportingLogger.Information(
166+
"Bench run {BenchRunId} for query {Id}. Mean {MeanElapsed:N0} ms with relative dispersion {RelativeStandardDeviationElapsed:N2}",
167+
runId, c.Id,
168+
timings.MeanElapsed,
169+
timings.RelativeStandardDeviationElapsed);
170+
}
171+
}
172+
173+
return 0;
174+
} catch (Exception ex)
175+
{
176+
Log.Error(ex, "Benchmarking failed: {ErrorMessage}", ex.Message);
177+
return 1;
178+
}
179+
}
180+
181+
Logger BuildReportingLogger()
182+
{
183+
return string.IsNullOrWhiteSpace(_reportingServerUrl)
184+
? new LoggerConfiguration()
185+
.Enrich.FromLogContext()
186+
.WriteTo.Console()
187+
.CreateLogger()
188+
: new LoggerConfiguration()
189+
.Enrich.FromLogContext()
190+
.WriteTo.Console()
191+
.WriteTo.Seq(
192+
_reportingServerUrl,
193+
apiKey: string.IsNullOrWhiteSpace(_reportingServerApiKey) ? null : _reportingServerApiKey,
194+
period: TimeSpan.FromMilliseconds(1))
195+
.CreateLogger();
196+
}
197+
198+
static BenchCasesCollection ReadCases(string filename)
199+
{
200+
var defaultCasesPath = Path.Combine(Path.GetDirectoryName(typeof(BenchCommand).Assembly.Location)!, "Cli/Commands/Bench/BenchCases.json");
201+
var casesString = File.ReadAllText(string.IsNullOrWhiteSpace(filename)
202+
? defaultCasesPath
203+
: filename);
204+
var casesFile = JsonConvert.DeserializeObject<BenchCasesCollection>(casesString)
205+
?? new BenchCasesCollection();
206+
207+
if (casesFile.Cases.Select(c => c.Id).Distinct().Count() != casesFile.Cases.Count)
208+
{
209+
throw new Exception($"Cases file {filename} contains a duplicate id");
210+
}
211+
212+
return casesFile;
213+
}
214+
}

‎src/SeqCli/Cli/Commands/BenchCases.json‎

Lines changed: 0 additions & 20 deletions
This file was deleted.

0 commit comments

Comments
 (0)