Report saga audit misconfigurations more prominently (#4523)

* Remove TODO, make warning an error

* Add custom check to report misconfiguration more prominently to user

* More to-the-point error message, including endpoint name

* Custom check approval test

* Tweak wording

---------

Co-authored-by: Brandon Ording <[email protected]>

Author: DavidBoike

src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.CustomCheckDetails.approved.txt

@@ -1,3 +1,4 @@
+Configuration: Saga Audit Configuration
 Health: ServiceControl Primary Instance
 Health: ServiceControl Remotes
 ServiceControl Health: Error Message Ingestion

src/ServiceControl/CustomChecks/InternalCustomChecks/InternalCustomChecks.cs

@@ -7,6 +7,7 @@
     using NServiceBus.CustomChecks;
     using NServiceBus.Hosting;
     using Operations;
+    using SagaAudit;
     using ServiceBus.Management.Infrastructure.Settings;
 
     static class InternalCustomChecks
@@ -16,6 +17,7 @@ public static IHostApplicationBuilder AddInternalCustomChecks(this IHostApplicat
             var services = hostBuilder.Services;
             services.AddCustomCheck<CriticalErrorCustomCheck>();
             services.AddCustomCheck<CheckRemotes>();
+            services.AddCustomCheck<SagaAuditMisconfigurationCustomCheck>();
 
             services.AddHostedService(provider => new InternalCustomChecksHostedService(
                 provider.GetServices<ICustomCheck>().ToList(),

src/ServiceControl/SagaAudit/SagaAuditMisconfigurationCustomCheck.cs

@@ -0,0 +1,48 @@
+#nullable enable
+
+namespace ServiceControl.SagaAudit;
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using NServiceBus;
+using NServiceBus.CustomChecks;
+
+class SagaAuditMisconfigurationCustomCheck() : CustomCheck("Saga Audit Configuration", "Configuration", TimeSpan.FromMinutes(5))
+{
+    static Details? lastMisconfiguredMessageDetails;
+
+    public override Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
+    {
+        var details = lastMisconfiguredMessageDetails;
+
+        if (details is null || details.Value.OccurredAt < DateTime.UtcNow.AddMinutes(-5))
+        {
+            return Task.FromResult(CheckResult.Pass);
+        }
+
+        var endpointDescription = details.Value.Endpoint == null
+            ? "without an endpoint name header"
+            : $"from endpoint '{details.Value.Endpoint}'";
+
+        var msg = $"At least one misconfigured endpoint was detected. A saga audit message {endpointDescription} was detected within the last 5 minutes.";
+        return Task.FromResult(CheckResult.Failed(msg));
+    }
+
+    public static void LogMisconfiguredMessage(IMessageHandlerContext context)
+    {
+        lastMisconfiguredMessageDetails = new Details(context);
+    }
+
+    readonly struct Details
+    {
+        public readonly DateTime OccurredAt = DateTime.UtcNow;
+        public readonly string? Endpoint;
+
+        public Details(IMessageHandlerContext context)
+        {
+            // ReplyToAddress is the only identifying header present in saga audit messages
+            _ = context.MessageHeaders.TryGetValue(Headers.ReplyToAddress, out Endpoint);
+        }
+    }
+}

src/ServiceControl/SagaAudit/SagaUpdatedHandler.cs

@@ -15,6 +15,8 @@ class SagaUpdatedHandler(IPlatformConnectionBuilder connectionBuilder)
     {
         public async Task Handle(SagaUpdatedMessage message, IMessageHandlerContext context)
         {
+            SagaAuditMisconfigurationCustomCheck.LogMisconfiguredMessage(context);
+
             if (auditQueueName is null || nextAuditQueueNameRefresh < DateTime.UtcNow)
             {
                 await RefreshAuditQueue();
@@ -25,8 +27,8 @@ public async Task Handle(SagaUpdatedMessage message, IMessageHandlerContext cont
                 throw new UnrecoverableException("Could not determine audit queue name to forward saga update message. This message can be replayed after the ServiceControl Audit remote instance is running and accessible.");
             }
 
-            // TODO Forward saga audit messages and warn in ServiceControl 5, remove in 6
-            log.WarnFormat("Configure the Saga Audit plugin to send messages to an audit instance. ServiceControl 6 will stop ingesting and forwarding Saga Audit to audit instances.");
+            var endpointName = context.MessageHeaders.TryGetValue(Headers.ReplyToAddress, out var val) ? val : "(Unknown Endpoint)";
+            log.ErrorFormat($"Received a saga audit message in the ServiceControl queue that should have been sent to the audit queue. This indicates that the endpoint '{endpointName}' using the SagaAudit plugin is misconfigured and should be changed to use the system's audit queue instead. The message has been forwarded to the audit queue, but this may not be possible in a future version of ServiceControl.");
             await context.ForwardCurrentMessageTo(auditQueueName);
         }