mmi-3349

Author: AlessiaYChen

api/net/Areas/Helpers/ReportHelper.cs

@@ -1,5 +1,7 @@
 
 using System.Text.Json;
+using System.Globalization;
+using System.Text;
 using Microsoft.Extensions.Options;
 using TNO.Core.Exceptions;
 using TNO.Core.Extensions;
@@ -206,6 +208,8 @@ public async Task<ReportResultModel> GenerateReportAsync(
             return new ReportSectionModel(section, content, aggregations);
         });
 
+        // Apply optional title-based de-duplication per section settings.
+        ApplyTitleDeduplication(sections);
         var result = await GenerateReportAsync(model, null, sections, viewOnWebOnly, isPreview);
         result.Data = JsonDocument.Parse(JsonSerializer.Serialize(elasticResults));
         return result;
@@ -230,12 +234,86 @@ private async Task<ReportResultModel> GenerateReportAsync(
         bool viewOnWebOnly = false,
         bool isPreview = false)
     {
+        // Apply optional title-based de-duplication per section settings.
+        ApplyTitleDeduplication(sections);
         var subject = await _reportEngine.GenerateReportSubjectAsync(report, reportInstance, sections, viewOnWebOnly, isPreview);
         var body = await _reportEngine.GenerateReportBodyAsync(report, reportInstance, sections, GetLinkedReportContent, _storageOptions.GetUploadPath(), viewOnWebOnly, isPreview);
 
         return new ReportResultModel() { ReportId = report.Id, InstanceId = reportInstance?.Id, Subject = subject, Body = body };
     }
 
+    /// <summary>
+    /// When a section has Settings.RemoveDuplicateTitles enabled, remove items whose headline
+    /// duplicates another story's headline in the overall report, keeping only the earliest story.
+    /// Earliest is determined by PublishedOn, then PostedOn, then SortOrder as tie-breaker.
+    /// </summary>
+    /// <param name="sections">Sections keyed by name.</param>
+    private static void ApplyTitleDeduplication(Dictionary<string, ReportSectionModel> sections)
+    {
+        if (sections == null || sections.Count == 0) return;
+
+        // Build a map of normalized headline -> earliest content item (by PublishedOn/PostedOn/SortOrder)
+        var earliestByTitle = new Dictionary<string, ContentModel>(StringComparer.Ordinal);
+
+        DateTime MinDate = DateTime.MinValue;
+        foreach (var section in sections.Values)
+        {
+            foreach (var item in section.Content)
+            {
+                var title = (item?.Headline ?? "").Trim();
+                var key = NormalizeTitle(title);
+                if (string.IsNullOrWhiteSpace(key)) continue;
+                var ts = item.PublishedOn ?? item.PostedOn ?? MinDate.AddSeconds(item.SortOrder);
+                if (!earliestByTitle.TryGetValue(key, out var existing))
+                {
+                    earliestByTitle[key] = item;
+                }
+                else
+                {
+                    var existingTs = existing.PublishedOn ?? existing.PostedOn ?? MinDate.AddSeconds(existing.SortOrder);
+                    if (ts < existingTs) earliestByTitle[key] = item;
+                }
+            }
+        }
+
+        // Now filter sections that have RemoveDuplicateTitles enabled
+        foreach (var section in sections.Values)
+        {
+            if (!section.Settings.RemoveDuplicateTitles || section.SectionType != TNO.Entities.ReportSectionType.Content) continue;
+
+            var filtered = section.Content.Where(item =>
+            {
+                var key = NormalizeTitle(item?.Headline ?? "");
+                if (string.IsNullOrWhiteSpace(key)) return true; // nothing to dedupe
+                if (!earliestByTitle.TryGetValue(key, out var earliest)) return true; // no dup group
+                return item.Id == earliest.Id; // keep only earliest occurrence; drop others
+            }).ToArray();
+
+            section.Content = filtered;
+        }
+    }
+
+    /// <summary>
+    /// Normalize a title for duplicate comparison:
+    /// - trim
+    /// - lowercase (invariant)
+    /// - remove diacritics
+    /// - remove all non-letter/digit characters (ignore punctuation/whitespace)
+    /// </summary>
+    private static string NormalizeTitle(string title)
+    {
+        if (string.IsNullOrWhiteSpace(title)) return string.Empty;
+        var t = title.Trim().ToLowerInvariant().Normalize(NormalizationForm.FormD);
+        var sb = new StringBuilder(t.Length);
+        foreach (var ch in t)
+        {
+            var uc = CharUnicodeInfo.GetUnicodeCategory(ch);
+            if (uc == UnicodeCategory.NonSpacingMark) continue; // strip diacritics
+            if (char.IsLetterOrDigit(ch)) sb.Append(ch);
+        }
+        return sb.ToString();
+    }
+
     #region AV Overview
     /// <summary>
     /// Execute the report template to generate the subject and body.

api/net/Dockerfile

@@ -1,9 +1,9 @@
 ARG BUILD_CONFIGURATION=Release
 FROM mcr.microsoft.com/dotnet/aspnet:9.0 AS base
 
-# RUN apt-get update --fix-missing && apt-get -y upgrade
-# RUN apt -y install curl libc6-dev libgdiplus ffmpeg
-# RUN apt-get clean
+RUN apt-get update --fix-missing && apt-get -y upgrade
+RUN apt -y install curl libc6-dev libgdiplus ffmpeg
+RUN apt-get clean
 
 EXPOSE 443 8080
 

app/editor/src/features/admin/reports/components/ReportSectionContent.tsx

@@ -68,6 +68,11 @@ export const ReportSectionContent = ({ index }: IReportSectionContentProps) => {
           label="Remove Duplicate Content"
           tooltip="Remove content from this section that is in above sections"
         />
+        <FormikCheckbox
+          name={`sections.${index}.settings.removeDuplicateTitles`}
+          label="Remove duplicate stories with same title"
+          tooltip="Ignore case, trim and punctuation; keep earliest by PublishedOn/PostedOn"
+        />
         <Show visible={!!section.folderId || !!section.linkedReportId}>
           <FormikCheckbox
             name={`sections.${index}.settings.overrideExcludeHistorical`}

libs/net/models/Settings/ReportSectionSettingsModel.cs

@@ -16,6 +16,7 @@ public class ReportSectionSettingsModel
     public bool? CacheData { get; set; }
     public string Direction { get; set; } = "";
     public bool RemoveDuplicates { get; set; }
+    public bool RemoveDuplicateTitles { get; set; }
     public bool OverrideExcludeHistorical { get; set; }
     public bool? InTableOfContents { get; set; }
     public bool HideEmpty { get; set; }
@@ -44,6 +45,7 @@ public ReportSectionSettingsModel(Dictionary<string, object> settings, JsonSeria
         this.CacheData = settings.GetDictionaryJsonValue("cacheData", false, options);
         this.Direction = settings.GetDictionaryJsonValue("direction", "", options)!;
         this.RemoveDuplicates = settings.GetDictionaryJsonValue("removeDuplicates", false, options)!;
+        this.RemoveDuplicateTitles = settings.GetDictionaryJsonValue("removeDuplicateTitles", false, options)!;
         this.OverrideExcludeHistorical = settings.GetDictionaryJsonValue("overrideExcludeHistorical", false, options)!;
         this.InTableOfContents = settings.GetDictionaryJsonValue<bool?>("inTableOfContents", null, options)!;
         this.HideEmpty = settings.GetDictionaryJsonValue("hideEmpty", false, options)!;
@@ -69,6 +71,7 @@ public ReportSectionSettingsModel(JsonDocument settings, JsonSerializerOptions o
         this.CacheData = settings.GetElementValue("cacheData", false, options);
         this.Direction = settings.GetElementValue("direction", "", options)!;
         this.RemoveDuplicates = settings.GetElementValue("removeDuplicates", false, options)!;
+        this.RemoveDuplicateTitles = settings.GetElementValue("removeDuplicateTitles", false, options)!;
         this.OverrideExcludeHistorical = settings.GetElementValue("overrideExcludeHistorical", false, options)!;
         this.InTableOfContents = settings.GetElementValue<bool?>("inTableOfContents", null, options)!;
         this.HideEmpty = settings.GetElementValue("hideEmpty", false, options)!;

libs/npm/core/src/hooks/api/interfaces/IReportSectionSettingsModel.ts

@@ -8,6 +8,7 @@ export interface IReportSectionSettingsModel {
   cacheData?: boolean;
   direction: 'row' | 'column';
   removeDuplicates: boolean;
+  removeDuplicateTitles?: boolean;
   overrideExcludeHistorical: boolean;
   inTableOfContents?: boolean;
   hideEmpty: boolean;