Add a RavenDB dirty memory custom check (#4868)

* Add an initial draft of a dirty memory custom check

* Add some memory analysis

* Use the database configuration to get the server URL

* Add more logging

* Add a to-do

* minor tweaks

* Because editorconfig

* Use better variable names

* Refactor the memory information retriever to check the content schema

* Register the MemoryInformationRetriever in DI

* Fix formatting

* Update the HTTP GET URL to trim the response size

* Make the custom check work in both embedded and external mode

* Custom checks approved list

* Update log statements to mention RavenDB

* Properly invert custom check id and category

* Add the CheckDirtyMemory custom check to the primary instance

* Link to the troubleshooting guidance page

* Fix casing

* Remove the trends evaluation from the dirty memory custom check

It'll be added in a separate PR

* Deep link to guidance

* Reword custom check and log warning message

Co-authored-by: Andreas Öhlund <[email protected]>

* Rename the dirty memory custom check ID

* reword comments

---------

Co-authored-by: Brandon Ording <[email protected]>
Co-authored-by: Andreas Öhlund <[email protected]>

Author: 3 people

src/ServiceControl.Audit.Persistence.RavenDB/CustomChecks/CheckDirtyMemory.cs

@@ -0,0 +1,26 @@
+namespace ServiceControl.Audit.Persistence.RavenDB.CustomChecks;
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using NServiceBus.CustomChecks;
+using NServiceBus.Logging;
+
+class CheckDirtyMemory(MemoryInformationRetriever memoryInformationRetriever) : CustomCheck("RavenDB dirty memory", "ServiceControl.Audit Health", TimeSpan.FromMinutes(5))
+{
+    public override async Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
+    {
+        var (isHighDirty, dirtyMemoryKb) = await memoryInformationRetriever.GetMemoryInformation(cancellationToken);
+
+        if (isHighDirty)
+        {
+            var message = $"There is a high level of RavenDB dirty memory ({dirtyMemoryKb}kb). See https://docs.particular.net/servicecontrol/troubleshooting#ravendb-dirty-memory for guidance on how to mitigate the issue.";
+            Log.Warn(message);
+            return CheckResult.Failed(message);
+        }
+
+        return CheckResult.Pass;
+    }
+
+    static readonly ILog Log = LogManager.GetLogger<CheckDirtyMemory>();
+}

src/ServiceControl.Audit.Persistence.RavenDB/MemoryInformationRetriever.cs

@@ -0,0 +1,41 @@
+namespace ServiceControl.Audit.Persistence.RavenDB;
+
+using System;
+using System.Net.Http;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+class MemoryInformationRetriever(DatabaseConfiguration databaseConfiguration)
+{
+    // What does a connection string look like? Is it only a URI or could it contain other stuff?
+    // The ?? operator is needed because ServerUrl is populated when running embedded and connection
+    // string when running in external mode. However, the tricky part is that when tests are run they
+    // behave like if it was external mode. If the connection string contain always only the server
+    // URL, this code is safe, otherwise it need to be adjusted to extract the server URL.
+    readonly HttpClient client = new() { BaseAddress = new Uri(databaseConfiguration.ServerConfiguration.ServerUrl ?? databaseConfiguration.ServerConfiguration.ConnectionString) };
+
+    record ResponseDto
+    {
+        public MemoryInformation MemoryInformation { get; set; }
+    }
+
+    record MemoryInformation
+    {
+        public bool IsHighDirty { get; set; }
+        public string DirtyMemory { get; set; }
+    }
+
+    public async Task<(bool IsHighDirty, int DirtyMemoryKb)> GetMemoryInformation(CancellationToken cancellationToken = default)
+    {
+        var httpResponse = await client.GetAsync("/admin/debug/memory/stats?includeThreads=false&includeMappings=false", cancellationToken);
+        var responseDto = JsonSerializer.Deserialize<ResponseDto>(await httpResponse.Content.ReadAsStringAsync(cancellationToken));
+
+        var values = responseDto.MemoryInformation.DirtyMemory.Split(' ');
+        if (!string.Equals(values[1], "KBytes", StringComparison.OrdinalIgnoreCase))
+        {
+            throw new InvalidOperationException($"Unexpected response. Was expecting memory details in KBytes, instead received: {responseDto.MemoryInformation.DirtyMemory}");
+        }
+        return (responseDto.MemoryInformation.IsHighDirty, int.Parse(values[0]));
+    }
+}

src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs

@@ -21,6 +21,7 @@ public void AddPersistence(IServiceCollection services)
         static void ConfigureLifecycle(IServiceCollection services, DatabaseConfiguration databaseConfiguration)
         {
             services.AddSingleton(databaseConfiguration);
+            services.AddSingleton<MemoryInformationRetriever>();
 
             services.AddSingleton<IRavenSessionProvider, RavenSessionProvider>();
             services.AddHostedService<RavenPersistenceLifecycleHostedService>();

src/ServiceControl.Audit.Persistence.Tests.RavenDB/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt

@@ -1,3 +1,4 @@
 ServiceControl.Audit Health: Audit Database Index Lag
 ServiceControl.Audit Health: Audit Message Ingestion Process
+ServiceControl.Audit Health: RavenDB dirty memory
 Storage space: ServiceControl.Audit database

src/ServiceControl.Persistence.RavenDB/CustomChecks/CheckDirtyMemory.cs

@@ -0,0 +1,26 @@
+namespace ServiceControl.Persistence.RavenDB.CustomChecks;
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using NServiceBus.CustomChecks;
+using NServiceBus.Logging;
+
+class CheckDirtyMemory(MemoryInformationRetriever memoryInformationRetriever) : CustomCheck("RavenDB dirty memory", "ServiceControl Health", TimeSpan.FromMinutes(5))
+{
+    public override async Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
+    {
+        var (isHighDirty, dirtyMemoryKb) = await memoryInformationRetriever.GetMemoryInformation(cancellationToken);
+
+        if (isHighDirty)
+        {
+            var message = $"There is a high level of RavenDB dirty memory ({dirtyMemoryKb}kb). See https://docs.particular.net/servicecontrol/troubleshooting#ravendb-dirty-memory for guidance on how to mitigate the issue.";
+            Log.Warn(message);
+            return CheckResult.Failed(message);
+        }
+
+        return CheckResult.Pass;
+    }
+
+    static readonly ILog Log = LogManager.GetLogger<CheckDirtyMemory>();
+}

src/ServiceControl.Persistence.RavenDB/MemoryInformationRetriever.cs

@@ -0,0 +1,40 @@
+namespace ServiceControl.Persistence.RavenDB;
+
+using System;
+using System.Net.Http;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+class MemoryInformationRetriever(RavenPersisterSettings persisterSettings)
+{
+    // What does a connection string look like? Is it only a URI or could it contain other stuff?
+    // The primary instance has only the concept of a connection string (vs the Audit instance having
+    // both a ServiceUrl and a ConnectionString). If the connection string contain always only the
+    // server URL, this code is safe, otherwise it need to be adjusted to extract the server URL. 
+    readonly HttpClient client = new() { BaseAddress = new Uri(persisterSettings.ConnectionString) };
+
+    record ResponseDto
+    {
+        public MemoryInformation MemoryInformation { get; set; }
+    }
+
+    record MemoryInformation
+    {
+        public bool IsHighDirty { get; set; }
+        public string DirtyMemory { get; set; }
+    }
+
+    public async Task<(bool IsHighDirty, int DirtyMemoryKb)> GetMemoryInformation(CancellationToken cancellationToken = default)
+    {
+        var httpResponse = await client.GetAsync("/admin/debug/memory/stats?includeThreads=false&includeMappings=false", cancellationToken);
+        var responseDto = JsonSerializer.Deserialize<ResponseDto>(await httpResponse.Content.ReadAsStringAsync(cancellationToken));
+
+        var values = responseDto.MemoryInformation.DirtyMemory.Split(' ');
+        if (!string.Equals(values[1], "KBytes", StringComparison.OrdinalIgnoreCase))
+        {
+            throw new InvalidOperationException($"Unexpected response. Was expecting memory details in KBytes, instead received: {responseDto.MemoryInformation.DirtyMemory}");
+        }
+        return (responseDto.MemoryInformation.IsHighDirty, int.Parse(values[0]));
+    }
+}

src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs

@@ -49,7 +49,9 @@ public void AddPersistence(IServiceCollection services)
         services.AddCustomCheck<CheckRavenDBIndexLag>();
         services.AddCustomCheck<CheckFreeDiskSpace>();
         services.AddCustomCheck<CheckMinimumStorageRequiredForIngestion>();
+        services.AddCustomCheck<CheckDirtyMemory>();
 
+        services.AddSingleton<MemoryInformationRetriever>();
         services.AddSingleton<OperationsManager>();
 
         services.AddSingleton<IArchiveMessages, MessageArchiver>();

src/ServiceControl.Persistence.Tests.RavenDB/ApprovalFiles/APIApprovals.CustomCheckDetails.approved.txt

@@ -1,4 +1,5 @@
 ServiceControl Health: Error Database Index Errors
 ServiceControl Health: Error Database Index Lag
 ServiceControl Health: Message Ingestion Process
+ServiceControl Health: RavenDB dirty memory
 Storage space: ServiceControl database

src/ServiceControl.Persistence.Tests.RavenDB/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt

@@ -1,4 +1,5 @@
 ServiceControl Health: Error Database Index Errors
 ServiceControl Health: Error Database Index Lag
 ServiceControl Health: Message Ingestion Process
+ServiceControl Health: RavenDB dirty memory
 Storage space: ServiceControl database