MMI-3249 Update Syndication Service

Author: Fosol

app/editor/src/features/admin/ingests/configurations/Newspaper.tsx

@@ -28,7 +28,7 @@ export const Newspaper: React.FC = (props) => {
   const sources = lookups.sources.map((s) => new OptionItem(s.name, s.code, !s.isEnabled));
   const source = sources.find((t) => t.value === values.configuration.defaultSource);
 
-  const initialItems = () => {
+  const initialItems = React.useCallback(() => {
     const data: any[] = [];
     let id = 0;
     const keyValues = values.configuration.sources?.split('&') ?? [];
@@ -37,24 +37,27 @@ export const Newspaper: React.FC = (props) => {
     });
     data.push({ id: ++id, name: '', source: '' });
     return data;
-  };
+  }, [values.configuration.sources]);
 
   const [items, setItems] = React.useState<{ id: number; name: string; source: string }[]>(
     initialItems(),
   );
 
-  const updateItems = (updatedItems: any[]) => {
-    if (updatedItems.filter((x) => !x.name && !x.source).length === 0)
-      updatedItems.push({ id: updatedItems.length + 1, name: '', source: '' });
-    setItems(updatedItems);
-    setFieldValue(
-      'configuration.sources',
-      updatedItems
-        .filter((t) => !!t.name)
-        .map((item) => `${item.name?.trim()}=${item.source?.trim()}`)
-        .join('&'),
-    );
-  };
+  const updateItems = React.useCallback(
+    (updatedItems: any[]) => {
+      if (updatedItems.filter((x) => !x.name && !x.source).length === 0)
+        updatedItems.push({ id: updatedItems.length + 1, name: '', source: '' });
+      setItems(updatedItems);
+      setFieldValue(
+        'configuration.sources',
+        updatedItems
+          .filter((t) => !!t.name)
+          .map((item) => `${item.name?.trim()}=${item.source?.trim()}`)
+          .join('&'),
+      );
+    },
+    [setFieldValue],
+  );
 
   const handleRemove = async (id: number) => {
     const item = items.find((i) => i.id === id);
@@ -283,7 +286,7 @@ export const Newspaper: React.FC = (props) => {
           />
         </Col>
       </Row>
-      <b>Sources</b>
+      <b>Source Mapping</b>
       <FlexboxTable
         rowId="id"
         data={items}

app/editor/src/features/admin/ingests/configurations/Syndication.tsx

@@ -1,15 +1,67 @@
 import { useFormikContext } from 'formik';
 import React from 'react';
-import { FormikCheckbox, FormikSelect, FormikText, IIngestModel } from 'tno-core';
+import { FlexboxTable, FormikCheckbox, FormikSelect, FormikText, IIngestModel } from 'tno-core';
 
 import { TimeZones } from './constants';
+import { columns } from './constants/columns';
 import { ImportContent } from './ImportContent';
 import * as styled from './styled';
 
 export const Syndication: React.FC = (props) => {
   const { values, setFieldValue } = useFormikContext<IIngestModel>();
   const timeZone = TimeZones.find((t) => t.value === values.configuration.timeZone);
 
+  const initialItems = React.useCallback(() => {
+    const data: any[] = [];
+    let id = 0;
+    const keyValues = values.configuration.sources?.split('&') ?? [];
+    keyValues.forEach((x: string) => {
+      data.push({ id: ++id, name: x.split('=')[0], source: x.split('=')[1] });
+    });
+    data.push({ id: ++id, name: '', source: '' });
+    return data;
+  }, [values.configuration.sources]);
+
+  const [items, setItems] = React.useState<{ id: number; name: string; source: string }[]>(
+    initialItems(),
+  );
+
+  const updateItems = React.useCallback(
+    (updatedItems: any[]) => {
+      if (updatedItems.filter((x) => !x.name && !x.source).length === 0)
+        updatedItems.push({ id: updatedItems.length + 1, name: '', source: '' });
+      setItems(updatedItems);
+      setFieldValue(
+        'configuration.sources',
+        updatedItems
+          .filter((t) => !!t.name)
+          .map((item) => `${item.name?.trim()}=${item.source?.trim()}`)
+          .join('&'),
+      );
+    },
+    [setFieldValue],
+  );
+
+  const handleRemove = async (id: number) => {
+    const item = items.find((i) => i.id === id);
+    if (!item) return;
+
+    const updatedItems = items.filter((i) => i.id !== id);
+    updateItems(updatedItems);
+  };
+
+  const onChange = (event: any, cell: any, isSource = true) => {
+    const pointer = event.target.selectionStart;
+    window.requestAnimationFrame(() => {
+      event.target.selectionEnd = pointer;
+    });
+    const updatedItem = isSource
+      ? { ...cell.original, source: event.target.value }
+      : { ...cell.original, name: event.target.value };
+    const updatedItems = items.map((item) => (item.id === cell.original.id ? updatedItem : item));
+    updateItems(updatedItems);
+  };
+
   return (
     <styled.IngestType>
       <ImportContent />
@@ -46,6 +98,15 @@ export const Syndication: React.FC = (props) => {
           setFieldValue('configuration.fetchContent', e.currentTarget.checked);
         }}
       />
+      <b>Source Mapping</b>
+      <FlexboxTable
+        rowId="id"
+        data={items}
+        columns={columns(handleRemove, onChange)}
+        showSort={true}
+        showActive={false}
+        pagingEnabled={false}
+      />
     </styled.IngestType>
   );
 };

services/net/syndication/SyndicationAction.cs

@@ -246,6 +246,34 @@ private async Task<string> GetContentAsync(IIngestActionManager manager, Uri url
         }
     }
 
+    /// <summary>
+    /// If the ingest has source mappings configured, find a match and apply the source to the item.
+    /// </summary>
+    /// <param name="ingest"></param>
+    /// <param name="item"></param>
+    /// <returns></returns>
+    /// <exception cref="InvalidOperationException"></exception>
+    private string DetermineSource(IngestModel ingest, SyndicationItem item)
+    {
+        KeyValuePair<string, string>[] sourceValues = ingest.Configuration.GetDictionaryJsonValue<string>("sources")?.Split('&').Select(v =>
+        {
+            var keyValue = v.Split('=');
+            if (keyValue.Length > 1) return new KeyValuePair<string, string>(keyValue[0], keyValue[1]);
+            return new KeyValuePair<string, string>(keyValue[0], keyValue[0]);
+        }).ToArray() ?? [];
+
+        var defaultSource = ingest.Source?.Code ?? throw new InvalidOperationException($"Ingest '{ingest.Name}' is missing source code.");
+        var itemSource = item.SourceFeed?.Title?.Text;
+
+        // Find a match if possible
+        if (!String.IsNullOrEmpty(itemSource))
+        {
+            var sourceMatch = sourceValues.FirstOrDefault(s => s.Key.Equals(itemSource, StringComparison.InvariantCultureIgnoreCase)).Value;
+            if (!String.IsNullOrWhiteSpace(sourceMatch)) return sourceMatch;
+        }
+        return defaultSource;
+    }
+
     /// <summary>
     /// Create a SourceContent object that can be sent to Kafka.
     /// </summary>
@@ -255,7 +283,7 @@ private async Task<string> GetContentAsync(IIngestActionManager manager, Uri url
     private SourceContent CreateSourceContent(IngestModel ingest, SyndicationItem item)
     {
         var (title, summary, body) = HandleInvalidEncoding(item);
-        var source = ingest.Source?.Code ?? throw new InvalidOperationException($"Ingest '{ingest.Name}' is missing source code.");
+        var source = DetermineSource(ingest, item);
         var contentType = ingest.IngestType?.ContentType ?? throw new InvalidOperationException($"Ingest '{ingest.Name}' is missing ingest content type.");
         var publishedOn = item.PublishDate.UtcDateTime != DateTime.MinValue ? item.PublishDate.UtcDateTime : (DateTime?)null;
         var uid = Runners.BaseService.GetContentHash(source, title, publishedOn);