MMI-3346 Notification Service (#2491)

Fosol · web-flow · commit 5a7014723482 · 2025-09-12T10:39:21.000-07:00
diff --git a/libs/net/ches/ChesService.cs b/libs/net/ches/ChesService.cs
@@ -1,6 +1,7 @@
 using System.IdentityModel.Tokens.Jwt;
 using System.Security.Claims;
 using System.Text;
+using System.Text.Json;
 using System.Text.RegularExpressions;
 using System.Web;
 using Microsoft.Extensions.Logging;
@@ -182,6 +183,11 @@ private async Task<string> SendAsync(string endpoint, HttpMethod method)
                 _logger.LogError(ex, "Failed to send/receive request: {code} {url}.  Error: {error}", ex.StatusCode, url, ex.GetAllMessages());
                 throw new ChesException("Failed to send message to CHES", ex);
             }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Failed to send/receive request: {url}.  Error: {error}", url, ex.GetAllMessages());
+                throw new ChesException("Failed to send message to CHES", ex);
+            }
         }
 
         /// <summary>
diff --git a/libs/net/core/Exceptions/HttpClientRequestException.cs b/libs/net/core/Exceptions/HttpClientRequestException.cs
@@ -49,7 +49,6 @@ public HttpClientRequestException(string message, Exception innerException, Http
         public HttpClientRequestException(HttpResponseMessage response) : base($"HTTP Request failed [{response?.RequestMessage?.Method}]:[{response?.StatusCode}]:{response?.RequestMessage?.RequestUri}", null, response?.StatusCode)
         {
             this.Response = response ?? throw new ArgumentNullException(nameof(response));
-            this.Data["Body"] = response.Content.ReadAsStringAsync().Result;
         }
 
         /// <summary>
@@ -60,7 +59,6 @@ public HttpClientRequestException(HttpResponseMessage response) : base($"HTTP Re
         public HttpClientRequestException(HttpResponseMessage response, string message) : base(message, null, response?.StatusCode)
         {
             this.Response = response ?? throw new ArgumentNullException(nameof(response));
-            this.Data["Body"] = response.Content.ReadAsStringAsync().Result;
         }
 
         /// <summary>
@@ -71,7 +69,6 @@ public HttpClientRequestException(HttpResponseMessage response, string message)
         public HttpClientRequestException(HttpResponseMessage response, Exception innerException) : base($"HTTP Request failed [{response?.RequestMessage?.Method}]:[{response?.StatusCode}]:{response?.RequestMessage?.RequestUri}", innerException, response?.StatusCode)
         {
             this.Response = response ?? throw new ArgumentNullException(nameof(response));
-            this.Data["Body"] = response.Content.ReadAsStringAsync().Result;
         }
         #endregion
     }
diff --git a/libs/net/services/Helpers/ApiService.cs b/libs/net/services/Helpers/ApiService.cs
@@ -105,6 +105,20 @@ protected async Task<T> RetryRequestAsync<T>(Func<Task<T>> callbackDelegate)
             this.FailureCount = 0;
             return response;
         }
+        catch (HttpClientRequestException ex)
+        {
+            if (this.Options.RetryLimit <= ++this.FailureCount)
+            {
+                this.FailureCount = 0;
+                throw;
+            }
+
+            // Wait before retrying.
+            var response = ex.Response?.Content.ReadAsStringAsync().Result;
+            this.Logger.LogError(ex, "Retry attempt {count}.{newline}Error:{body}", this.FailureCount, Environment.NewLine, response);
+            await Task.Delay(this.Options.RetryDelayMS);
+            return await RetryRequestAsync<T>(callbackDelegate);
+        }
         catch (Exception ex)
         {
             if (this.Options.RetryLimit <= ++this.FailureCount)
@@ -114,7 +128,7 @@ protected async Task<T> RetryRequestAsync<T>(Func<Task<T>> callbackDelegate)
             }
 
             // Wait before retrying.
-            this.Logger.LogError(ex, "Retry attempt {count}.{newline}Error:{body}", this.FailureCount, Environment.NewLine, ex.Data["Body"]);
+            this.Logger.LogError(ex, "Retry attempt {count}", this.FailureCount);
             await Task.Delay(this.Options.RetryDelayMS);
             return await RetryRequestAsync<T>(callbackDelegate);
         }
@@ -138,11 +152,11 @@ public async Task<T> HandleConcurrencyAsync<T>(Func<Task<T>> callbackDelegate)
             catch (HttpClientRequestException ex)
             {
                 // If it's a concurrency error, keep trying.  Otherwise throw the error.
+                var response = ex.Response?.Content.ReadAsStringAsync().Result;
                 this.Logger.LogError(ex, "Failed to complete request.  Determining if this is a concurrency error.");
-                var data = ex.Data["Body"] as string;
-                if (!String.IsNullOrWhiteSpace(data))
+                if (!String.IsNullOrWhiteSpace(response))
                 {
-                    var json = JsonSerializer.Deserialize<API.Models.ErrorResponseModel>(data, _serializerOptions);
+                    var json = JsonSerializer.Deserialize<API.Models.ErrorResponseModel>(response, _serializerOptions);
                     if (json != null && json.Type == nameof(Microsoft.EntityFrameworkCore.DbUpdateConcurrencyException))
                     {
                         // A concurrency error can be resolved by loading the latest and reapplying the values.
diff --git a/libs/net/services/Managers/IngestManager`.cs b/libs/net/services/Managers/IngestManager`.cs
@@ -4,6 +4,7 @@
 using TNO.API.Areas.Services.Models.Ingest;
 using TNO.Ches;
 using TNO.Ches.Configuration;
+using TNO.Core.Exceptions;
 using TNO.Services.Config;
 
 namespace TNO.Services.Managers;
@@ -124,7 +125,7 @@ public override async Task RunAsync()
 
                             if (!theLatest.IsEnabled ||
                                 !theLatest.IngestSchedules.Any(d => d.Schedule?.IsEnabled == true) ||
-                                !theLatest.DataLocations.Any(d => d.Name.ToLower() == Options.DataLocation.ToLower()))
+                                !theLatest.DataLocations.Any(d => d.Name.Equals(Options.DataLocation, StringComparison.CurrentCultureIgnoreCase)))
                             {
                                 await manager.StopAsync();
                                 continue;
@@ -172,9 +173,10 @@ public override async Task RunAsync()
                         // Successful run clears any errors.
                         this.State.ResetFailures();
                     }
-                    catch (HttpRequestException ex)
+                    catch (HttpClientRequestException ex)
                     {
-                        this.Logger.LogError(ex, "Ingest [{name}] failed to run. This is failure [{failures}] out of [{maxFailures}] maximum retries. Response: {Data}", ingest.Name, manager.Ingest.FailedAttempts + 1, manager.Ingest.RetryLimit, ex.Data["Body"]);
+                        var response = ex.Response?.Content.ReadAsStringAsync().Result;
+                        this.Logger.LogError(ex, "Ingest [{name}] failed to run. This is failure [{failures}] out of [{maxFailures}] maximum retries. Response: {Data}", ingest.Name, manager.Ingest.FailedAttempts + 1, manager.Ingest.RetryLimit, response);
 
                         // Update ingest with failure.
                         await manager.RecordFailureAsync(ex);
diff --git a/services/net/extract-quotes/CoreNLP/CoreNLPService.cs b/services/net/extract-quotes/CoreNLP/CoreNLPService.cs
@@ -1,21 +1,23 @@
-using TNO.Core.Http;
+using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using TNO.API.Areas.Services.Models.Content;
+using TNO.Core.Exceptions;
+using TNO.Core.Http;
 using TNO.Services.ExtractQuotes.Config;
 using TNO.Services.ExtractQuotes.CoreNLP.models;
-using Microsoft.Extensions.Logging;
-using TNO.API.Areas.Services.Models.Content;
 
 namespace TNO.Services.NLP.ExtractQuotes;
 
-public interface ICoreNLPService {
+public interface ICoreNLPService
+{
     Task<AnnotationResponse?> PerformAnnotation(string text);
     Task<AnnotationResponse?> PerformAnnotationWithExistingQuotes(string text, IEnumerable<QuoteModel> existingQuotes);
 }
 
 /// <summary>
 /// CoreNLPService class, provides a wrapper service for the Core NLP API.
 /// </summary>
-public class CoreNLPService: ICoreNLPService
+public class CoreNLPService : ICoreNLPService
 {
     #region Properties
     /// <summary>
@@ -97,6 +99,20 @@ protected async Task<T> RetryRequestAsync<T>(Func<Task<T>> callbackDelegate)
             this.FailureCount = 0;
             return response;
         }
+        catch (HttpClientRequestException ex)
+        {
+            if (this.Options.RetryLimit <= ++this.FailureCount)
+            {
+                this.FailureCount = 0;
+                throw;
+            }
+
+            var response = ex.Response?.Content.ReadAsStringAsync().Result;
+            // Wait before retrying.
+            this.Logger.LogError(ex, "Retry attempt {count}.{newline}Error:{body}", this.FailureCount, Environment.NewLine, response);
+            await Task.Delay(this.Options.RetryDelayMS);
+            return await RetryRequestAsync<T>(callbackDelegate);
+        }
         catch (Exception ex)
         {
             if (this.Options.RetryLimit <= ++this.FailureCount)
@@ -106,7 +122,7 @@ protected async Task<T> RetryRequestAsync<T>(Func<Task<T>> callbackDelegate)
             }
 
             // Wait before retrying.
-            this.Logger.LogError(ex, "Retry attempt {count}.{newline}Error:{body}", this.FailureCount, Environment.NewLine, ex.Data["Body"]);
+            this.Logger.LogError(ex, "Retry attempt {count}.", this.FailureCount);
             await Task.Delay(this.Options.RetryDelayMS);
             return await RetryRequestAsync<T>(callbackDelegate);
         }
@@ -141,4 +157,4 @@ protected async Task<T> RetryRequestAsync<T>(Func<Task<T>> callbackDelegate)
         return await PerformAnnotation(text);
     }
     #endregion
-}
+}
diff --git a/services/net/extract-quotes/LLM/Clients/LLMClient.cs b/services/net/extract-quotes/LLM/Clients/LLMClient.cs
@@ -3,6 +3,7 @@
 using System.Text.Json;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using TNO.Core.Exceptions;
 using TNO.Core.Http;
 using TNO.Services.ExtractQuotes.Config;
 using TNO.Services.ExtractQuotes.LLM.Models;
@@ -61,7 +62,7 @@ public LLMClient(
         {
             return await callbackDelegate();
         }
-        catch (Exception ex)
+        catch (HttpClientRequestException ex)
         {
             // Use Interlocked.Increment for thread-safe increment and wrap-around
             int currentFailureCount = Interlocked.Increment(ref _failureCount);
@@ -71,9 +72,25 @@ public LLMClient(
                 throw;
             }
 
+            var response = ex.Response?.Content.ReadAsStringAsync().Result;
             // Wait before retrying
             _logger.LogError(ex, "LLM API retry attempt {count}.{newline}Error:{body}",
-                currentFailureCount, Environment.NewLine, ex.Data["Body"]);
+                currentFailureCount, Environment.NewLine, response);
+            await Task.Delay(_options.RetryDelayMS);
+            return await RetryRequestAsync<T>(callbackDelegate);
+        }
+        catch (Exception ex)
+        {
+            // Use Interlocked.Increment for thread-safe increment and wrap-around
+            int currentFailureCount = Interlocked.Increment(ref _failureCount);
+            if (_options.RetryLimit <= currentFailureCount)
+            {
+                Interlocked.Exchange(ref _failureCount, 0); // Reset the counter
+                throw;
+            }
+
+            // Wait before retrying
+            _logger.LogError(ex, "LLM API retry attempt {count}.", currentFailureCount);
             await Task.Delay(_options.RetryDelayMS);
             return await RetryRequestAsync<T>(callbackDelegate);
         }

Original file line number	Diff line number	Diff line change
`@@ -49,7 +49,6 @@ public HttpClientRequestException(string message, Exception innerException, Http`
`49`	`49`	`public HttpClientRequestException(HttpResponseMessage response) : base($"HTTP Request failed [{response?.RequestMessage?.Method}]:[{response?.StatusCode}]:{response?.RequestMessage?.RequestUri}", null, response?.StatusCode)`
`50`	`50`	`{`
`51`	`51`	`this.Response = response ?? throw new ArgumentNullException(nameof(response));`
`52`		`- this.Data["Body"] = response.Content.ReadAsStringAsync().Result;`
`53`	`52`	`}`
`54`	`53`
`55`	`54`	`/// <summary>`
`@@ -60,7 +59,6 @@ public HttpClientRequestException(HttpResponseMessage response) : base($"HTTP Re`
`60`	`59`	`public HttpClientRequestException(HttpResponseMessage response, string message) : base(message, null, response?.StatusCode)`
`61`	`60`	`{`
`62`	`61`	`this.Response = response ?? throw new ArgumentNullException(nameof(response));`
`63`		`- this.Data["Body"] = response.Content.ReadAsStringAsync().Result;`
`64`	`62`	`}`
`65`	`63`
`66`	`64`	`/// <summary>`
`@@ -71,7 +69,6 @@ public HttpClientRequestException(HttpResponseMessage response, string message)`
`71`	`69`	`public HttpClientRequestException(HttpResponseMessage response, Exception innerException) : base($"HTTP Request failed [{response?.RequestMessage?.Method}]:[{response?.StatusCode}]:{response?.RequestMessage?.RequestUri}", innerException, response?.StatusCode)`
`72`	`70`	`{`
`73`	`71`	`this.Response = response ?? throw new ArgumentNullException(nameof(response));`
`74`		`- this.Data["Body"] = response.Content.ReadAsStringAsync().Result;`
`75`	`72`	`}`
`76`	`73`	`#endregion`
`77`	`74`	`}`