Skip to content

Commit 6a246ea

Browse files
Add more logging
1 parent cfbf439 commit 6a246ea

File tree

1 file changed

+22
-13
lines changed

1 file changed

+22
-13
lines changed

src/ServiceControl.Audit.Persistence.RavenDB/CustomChecks/CheckDirtyMemory.cs

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ namespace ServiceControl.Audit.Persistence.RavenDB.CustomChecks;
55
using System.Threading;
66
using System.Threading.Tasks;
77
using NServiceBus.CustomChecks;
8+
using NServiceBus.Logging;
89

910
class CheckDirtyMemory(DatabaseConfiguration databaseConfiguration) : CustomCheck("ServiceControl.Audit database", "Dirty memory trends", TimeSpan.FromMinutes(5))
1011
{
@@ -16,31 +17,38 @@ public override async Task<CheckResult> PerformCheck(CancellationToken cancellat
1617

1718
if (memoryInfo.IsHighDirty)
1819
{
19-
//log warning
20-
return CheckResult.Failed("There is a high level of dirty memory. Check the ServiceControl " +
21-
"troubleshooting guide for guidance on how to mitigate the issue.");
20+
var message = $"There is a high level of dirty memory ({memoryInfo.DirtyMemory}kb). Check the ServiceControl " +
21+
"troubleshooting guide for guidance on how to mitigate the issue.";
22+
Log.Warn(message);
23+
return CheckResult.Failed(message);
2224
}
2325

2426
lastDirtyMemoryReads.Add(memoryInfo.DirtyMemory);
2527
if (lastDirtyMemoryReads.Count > 20)
2628
{
27-
//cap the list at 20
28-
lastDirtyMemoryReads.RemoveAt(lastDirtyMemoryReads.Count - 1);
29+
//cap the list at 20 which means we're keeping about 1 hour and 40 minutes of data
30+
lastDirtyMemoryReads.RemoveAt(0);
2931
}
3032

31-
if (lastDirtyMemoryReads.Count > 3 && AnalyzeTrendUsingRegression(lastDirtyMemoryReads) == TrendDirection.Increasing) // Three means we'll be observing for 15 minutes before calculating the trend
33+
if (lastDirtyMemoryReads.Count < 3)
3234
{
33-
// log a warning and fail the check
35+
Log.Debug("Not enough dirty memory data in the series to calculate a trend.");
36+
}
37+
38+
// Three means we'll be observing for 15 minutes before calculating the trend
39+
if (lastDirtyMemoryReads.Count >= 3 && AnalyzeTrendUsingRegression(lastDirtyMemoryReads) == TrendDirection.Increasing)
40+
{
41+
var message = $"Dirty memory is increasing. Last available value is {memoryInfo.DirtyMemory}kb. " +
42+
$"Check the ServiceControl troubleshooting guide for guidance on how to mitigate the issue.";
43+
Log.Warn(message);
44+
return CheckResult.Failed(message);
3445
}
3546

3647
return CheckResult.Pass;
3748
}
3849

3950
MemoryInformationRetriever _retriever;
40-
async Task<MemoryInformationRetriever> GetMemoryRetriever()
41-
{
42-
return _retriever ??= new MemoryInformationRetriever(databaseConfiguration.ServerConfiguration.ServerUrl);
43-
}
51+
async Task<MemoryInformationRetriever> GetMemoryRetriever() => _retriever ??= new MemoryInformationRetriever(databaseConfiguration.ServerConfiguration.ServerUrl);
4452

4553
static TrendDirection AnalyzeTrendUsingRegression(List<int> values)
4654
{
@@ -87,7 +95,8 @@ enum TrendDirection
8795
{
8896
Increasing,
8997
Decreasing,
90-
Flat,
91-
Mixed
98+
Flat
9299
}
100+
101+
static readonly ILog Log = LogManager.GetLogger<CheckDirtyMemory>();
93102
}

0 commit comments

Comments
 (0)