Add the CheckDirtyMemory custom check to the primary instance

Author: mauroservienti

src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs

@@ -49,7 +49,9 @@ public void AddPersistence(IServiceCollection services)
         services.AddCustomCheck<CheckRavenDBIndexLag>();
         services.AddCustomCheck<CheckFreeDiskSpace>();
         services.AddCustomCheck<CheckMinimumStorageRequiredForIngestion>();
+        services.AddCustomCheck<CheckDirtyMemory>();
 
+        services.AddSingleton<MemoryInformationRetriever>();
         services.AddSingleton<OperationsManager>();
 
         services.AddSingleton<IArchiveMessages, MessageArchiver>();

src/ServiceControl.Persistence.RavenDb/CustomChecks/CheckDirtyMemory.cs

@@ -0,0 +1,103 @@
+namespace ServiceControl.Persistence.RavenDB.CustomChecks;
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using NServiceBus.CustomChecks;
+using NServiceBus.Logging;
+
+class CheckDirtyMemory(MemoryInformationRetriever memoryInformationRetriever) : CustomCheck("RavenDB dirty memory trends", "ServiceControl Health", TimeSpan.FromMinutes(5))
+{
+    readonly List<int> lastDirtyMemoryReads = [];
+    public override async Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
+    {
+        var (isHighDirty, dirtyMemoryKb) = await memoryInformationRetriever.GetMemoryInformation(cancellationToken);
+
+        if (isHighDirty)
+        {
+            var message = $"There is a high level of RavenDB dirty memory ({dirtyMemoryKb}kb). Check the ServiceControl " +
+                          "troubleshooting guide for guidance on how to mitigate the issue.";
+            Log.Warn(message);
+            return CheckResult.Failed(message);
+        }
+
+        lastDirtyMemoryReads.Add(dirtyMemoryKb);
+        if (lastDirtyMemoryReads.Count > 20)
+        {
+            //cap the list at 20 which means we're keeping about 1 hour and 40 minutes of data
+            lastDirtyMemoryReads.RemoveAt(0);
+        }
+
+        switch (lastDirtyMemoryReads.Count)
+        {
+            case < 3:
+                Log.Debug("Not enough RavenDB dirty memory data in the series to calculate a trend.");
+                break;
+            // TODO do we need a threshold below which the check never fails?
+            // Three means we'll be observing for 15 minutes before calculating the trend
+            case >= 3 when AnalyzeTrendUsingRegression(lastDirtyMemoryReads) == TrendDirection.Increasing:
+                {
+                    var message = $"RavenDB dirty memory is increasing. Last available value is {dirtyMemoryKb}kb. " +
+                                  $"Check the ServiceControl troubleshooting guide for guidance on how to mitigate the issue.";
+                    Log.Warn(message);
+                    return CheckResult.Failed(message);
+                }
+
+            default:
+                // NOP
+                break;
+        }
+
+        return CheckResult.Pass;
+    }
+
+    static TrendDirection AnalyzeTrendUsingRegression(List<int> values)
+    {
+        if (values is not { Count: > 1 })
+        {
+            throw new ArgumentException("Need at least two values to determine a trend");
+        }
+
+        // Calculate slope using linear regression
+        double numberOfPoints = values.Count;
+        double sumOfIndices = 0;
+        double sumOfValues = 0;
+        double sumOfIndicesMultipliedByValues = 0;
+        double sumOfIndicesSquared = 0;
+
+        for (int i = 0; i < values.Count; i++)
+        {
+            double index = i;
+            double value = values[i];
+
+            sumOfIndices += index;
+            sumOfValues += value;
+            sumOfIndicesMultipliedByValues += index * value;
+            sumOfIndicesSquared += index * index;
+        }
+
+        // Slope formula: (n*Σxy - Σx*Σy) / (n*Σx² - (Σx)²)
+        double slopeNumerator = (numberOfPoints * sumOfIndicesMultipliedByValues) - (sumOfIndices * sumOfValues);
+        double slopeDenominator = (numberOfPoints * sumOfIndicesSquared) - (sumOfIndices * sumOfIndices);
+        double slope = slopeNumerator / slopeDenominator;
+
+        // Determine trend based on slope
+        const double slopeThreshold = 0.001; // Small threshold to handle floating-point precision
+        if (Math.Abs(slope) < slopeThreshold)
+        {
+            return TrendDirection.Flat;
+        }
+
+        return slope > 0 ? TrendDirection.Increasing : TrendDirection.Decreasing;
+    }
+
+    enum TrendDirection
+    {
+        Increasing,
+        Decreasing,
+        Flat
+    }
+
+    static readonly ILog Log = LogManager.GetLogger<CheckDirtyMemory>();
+}

src/ServiceControl.Persistence.RavenDb/MemoryInformationRetriever.cs

@@ -0,0 +1,37 @@
+namespace ServiceControl.Persistence.RavenDB;
+
+using System;
+using System.Net.Http;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+class MemoryInformationRetriever(RavenPersisterSettings persisterSettings)
+{
+    // TODO what does a connection string look like? Is it only a URI or could it contain other stuff?
+    readonly HttpClient client = new() { BaseAddress = new Uri(persisterSettings.ConnectionString) };
+
+    record ResponseDto
+    {
+        public MemoryInformation MemoryInformation { get; set; }
+    }
+
+    record MemoryInformation
+    {
+        public bool IsHighDirty { get; set; }
+        public string DirtyMemory { get; set; }
+    }
+
+    public async Task<(bool IsHighDirty, int DirtyMemoryKb)> GetMemoryInformation(CancellationToken cancellationToken = default)
+    {
+        var httpResponse = await client.GetAsync("/admin/debug/memory/stats?includeThreads=false&includeMappings=false", cancellationToken);
+        var responseDto = JsonSerializer.Deserialize<ResponseDto>(await httpResponse.Content.ReadAsStringAsync(cancellationToken));
+
+        var values = responseDto.MemoryInformation.DirtyMemory.Split(' ');
+        if (!string.Equals(values[1], "KBytes", StringComparison.OrdinalIgnoreCase))
+        {
+            throw new InvalidOperationException($"Unexpected response. Was expecting memory details in KBytes, instead received: {responseDto.MemoryInformation.DirtyMemory}");
+        }
+        return (responseDto.MemoryInformation.IsHighDirty, int.Parse(values[0]));
+    }
+}

src/ServiceControl.Persistence.Tests.RavenDB/ApprovalFiles/APIApprovals.CustomCheckDetails.approved.txt

@@ -1,4 +1,5 @@
 ServiceControl Health: Error Database Index Errors
 ServiceControl Health: Error Database Index Lag
 ServiceControl Health: Message Ingestion Process
+ServiceControl Health: RavenDB dirty memory trends
 Storage space: ServiceControl database

src/ServiceControl.Persistence.Tests.RavenDB/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt

@@ -1,4 +1,5 @@
 ServiceControl Health: Error Database Index Errors
 ServiceControl Health: Error Database Index Lag
 ServiceControl Health: Message Ingestion Process
+ServiceControl Health: RavenDB dirty memory trends
 Storage space: ServiceControl database