Fix syndication service (bcgov#2456)

* Fix syndication service

* Remove sources

Author: Fosol

libs/net/models/Areas/Services/Models/Ingest/SourceModel.cs

@@ -1,5 +1,5 @@
-using TNO.API.Models;
 using System.Text.Json;
+using TNO.API.Models;
 
 namespace TNO.API.Areas.Services.Models.Ingest;
 
@@ -50,22 +50,22 @@ public class SourceModel : AuditColumnsModel
     public LicenseModel? License { get; set; }
 
     /// <summary>
-    /// get/set -
+    /// get/set - The default owner of this content.
     /// </summary>
     public int? OwnerId { get; set; }
 
     /// <summary>
-    /// get/set -
+    /// get/set - The media type override for ingested content.
     /// </summary>
-    public int? MediaTypeId { get; set; }//fixed
+    public int? MediaTypeId { get; set; }
 
     /// <summary>
-    /// get/set -
+    /// get/set - Whether to auto transcribe the content when ingested.
     /// </summary>
     public bool AutoTranscribe { get; set; }
 
     /// <summary>
-    /// get/set -
+    /// get/set - Whether to disable the ability for subscribers to request a transcript.
     /// </summary>
     public bool DisableTranscribe { get; set; }
 

services/net/syndication/SyndicationAction.cs

@@ -1,4 +1,6 @@
+using System.Collections.Concurrent;
 using System.Globalization;
+using System.Linq;
 using System.Net.Http.Headers;
 using System.ServiceModel.Syndication;
 using System.Text;
@@ -32,6 +34,9 @@ public class SyndicationAction : IngestAction<SyndicationOptions>
 {
     #region Variables
     private readonly IHttpRequestClient _httpClient;
+    private readonly object _lock = new object();
+    private string? _etag;
+    private readonly ConcurrentDictionary<string, int?> _sourceMediaTypeMap = new ConcurrentDictionary<string, int?>();
     #endregion
 
     #region Properties
@@ -66,6 +71,7 @@ public override async Task<ServiceActionResult> PerformActionAsync<T>(IIngestAct
 
         // This ingest has just begun running.
         await manager.UpdateIngestStateFailedAttemptsAsync(manager.Ingest.FailedAttempts);
+        await GetSourcesConfigurationAsync();
 
         var url = GetUrl(manager.Ingest);
 
@@ -143,6 +149,45 @@ private async Task ImportFeedAsync(IIngestActionManager manager, SyndicationFeed
         }
     }
 
+    /// <summary>
+    /// Make a request to the API for all the source configurations.
+    /// Store their mediaTypeId override value in memory.
+    /// </summary>
+    /// <returns></returns>
+    private async Task GetSourcesConfigurationAsync()
+    {
+        try
+        {
+            // Fetch the latest configuration values.
+            var response = String.IsNullOrWhiteSpace(_etag) ? await this.Api.GetSourcesResponseAsync() : await this.Api.GetSourcesResponseWithEtagAsync(_etag);
+            var etag = this.Api.GetResponseEtag(response);
+            lock (_lock)
+            {
+                _etag = etag;
+            }
+            var sources = await this.Api.GetResponseDataAsync<SourceModel[]>(response);
+            if (sources != null)
+            {
+                foreach (var source in sources)
+                {
+                    _sourceMediaTypeMap.AddOrUpdate(source.Code, code => source.MediaTypeId, (code, oldId) => source.MediaTypeId);
+                }
+                var current = _sourceMediaTypeMap.Keys.ToArray() ?? [];
+                var latest = sources.Select(s => s.Code).ToArray();
+                var removeSources = current.Where(code => !latest.Contains(code));
+                foreach (var source in removeSources)
+                {
+                    // Remove any sources that no longer exist.
+                    _sourceMediaTypeMap.Remove(source, out int? value);
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            this.Logger.LogError(ex, "Failed to fetch sources");
+        }
+    }
+
     /// <summary>
     /// Parse the date time value and handle a common formatting issues.
     /// </summary>
@@ -274,6 +319,19 @@ private string DetermineSource(IngestModel ingest, SyndicationItem item)
         return defaultSource;
     }
 
+    /// <summary>
+    /// Determine the media type for the specified ingest and source.
+    /// If the source has an override, use it.
+    /// </summary>
+    /// <param name="ingest"></param>
+    /// <param name="source"></param>
+    /// <returns></returns>
+    private int DetermineMediaType(IngestModel ingest, string source)
+    {
+        _sourceMediaTypeMap.TryGetValue(source, out var mediaTypeId);
+        return mediaTypeId ?? ingest.MediaTypeId;
+    }
+
     /// <summary>
     /// Create a SourceContent object that can be sent to Kafka.
     /// </summary>
@@ -287,6 +345,7 @@ private SourceContent CreateSourceContent(IngestModel ingest, SyndicationItem it
         var contentType = ingest.IngestType?.ContentType ?? throw new InvalidOperationException($"Ingest '{ingest.Name}' is missing ingest content type.");
         var publishedOn = item.PublishDate.UtcDateTime != DateTime.MinValue ? item.PublishDate.UtcDateTime : (DateTime?)null;
         var uid = Runners.BaseService.GetContentHash(source, title, publishedOn);
+        var mediaTypeId = DetermineMediaType(ingest, source);
         string? media = null;
 
         // Extract values from namespaces.  I don't know a better way to do this.
@@ -304,7 +363,7 @@ private SourceContent CreateSourceContent(IngestModel ingest, SyndicationItem it
             this.Options.DataLocation,
             source,
             contentType,
-            ingest.MediaTypeId,
+            mediaTypeId,
             uid,
             title,
             StringExtensions.ConvertTextToParagraphs(summary, @"[\r\n]+"),