Add an initial draft of a dirty memory custom check

Author: mauroservienti

src/ServiceControl.Audit.Persistence.RavenDB/CustomChecks/CheckDirtyMemory.cs

@@ -0,0 +1,48 @@
+namespace ServiceControl.Audit.Persistence.RavenDB.CustomChecks;
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using NServiceBus.CustomChecks;
+
+class CheckDirtyMemory(IRavenDocumentStoreProvider documentStoreProvider) : CustomCheck("ServiceControl.Audit database", "Dirty memory trends", TimeSpan.FromMinutes(5))
+{
+    readonly List<int> lastDirtyMemoryReads = [];
+    public override async Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
+    {
+        var retriever = await GetMemoryRetriever(cancellationToken);
+        var memoryInfo = await retriever.GetMemoryInformation(cancellationToken);
+
+        if (memoryInfo.IsHighDirty)
+        {
+            //log warning
+            return CheckResult.Failed("There is a high level of dirty memory. Check the ServiceControl " +
+                                      "troubleshooting guide for guidance on how to mitigate the issue.");
+        }
+
+        lastDirtyMemoryReads.Add(memoryInfo.DirtyMemory);
+        if (lastDirtyMemoryReads.Count > 20)
+        {
+            //cap the list at 20
+            lastDirtyMemoryReads.RemoveAt(lastDirtyMemoryReads.Count - 1);
+        }
+
+        // evaluate the trends
+        // if the amount of dirty memory is constantly growing log a warning and fail the check
+
+        return CheckResult.Pass;
+    }
+
+    MemoryInformationRetriever _retriever;
+    async Task<MemoryInformationRetriever> GetMemoryRetriever(CancellationToken cancellationToken = default)
+    {
+        if (_retriever == null)
+        {
+            var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken);
+            var serverUrl = documentStore.Urls[0]; //TODO is there a better way to get the RavenDB server URL?
+            _retriever = new MemoryInformationRetriever(serverUrl);
+        }
+        return _retriever;
+    }
+}

src/ServiceControl.Audit.Persistence.RavenDB/MemoryInformationRetriever.cs

@@ -0,0 +1,32 @@
+namespace ServiceControl.Audit.Persistence.RavenDB;
+
+using System;
+using System.Linq;
+using System.Net.Http;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+class MemoryInformationRetriever(string serverUrl)
+{
+    readonly HttpClient client = new() { BaseAddress = new Uri(serverUrl) };
+
+    record ResponseDto
+    {
+        public MemoryInformation MemoryInformation { get; set; }
+    }
+
+    record MemoryInformation
+    {
+        public bool IsHighDirty { get; set; }
+        public string DirtyMemory { get; set; }
+    }
+
+    public async Task<(bool IsHighDirty, int DirtyMemory)> GetMemoryInformation(CancellationToken cancellationToken = default)
+    {
+        var httpResponse = await client.GetAsync("/admin/debug/memory/stats", cancellationToken);
+        var responseDto = JsonSerializer.Deserialize<ResponseDto>(await httpResponse.Content.ReadAsStringAsync(cancellationToken));
+
+        return (responseDto.MemoryInformation.IsHighDirty, int.Parse(responseDto.MemoryInformation.DirtyMemory.Split(' ').First()));
+    }
+}