Preserve embeddings (fig, fm, notes and cross references)

Only translate ft text
Configure preserving / stripping embedded and style markers

Author: johnml1135

src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs

@@ -23,7 +23,9 @@ public string UpdateUsfm(
             string bookId,
             IReadOnlyList<(IReadOnlyList<ScriptureRef>, string)> rows,
             string fullName = null,
-            UpdateUsfmBehavior behavior = UpdateUsfmBehavior.PreferExisting
+            UpdateUsfmTextBehavior textBehavior = UpdateUsfmTextBehavior.PreferExisting,
+            UpdateUsfmIntraVerseMarkerBehavior embeddedBehavior = UpdateUsfmIntraVerseMarkerBehavior.Preserve,
+            UpdateUsfmIntraVerseMarkerBehavior styleBehavior = UpdateUsfmIntraVerseMarkerBehavior.Strip
         )
         {
             string fileName = _settings.GetBookFileName(bookId);
@@ -36,7 +38,13 @@ public string UpdateUsfm(
                 usfm = reader.ReadToEnd();
             }
 
-            var handler = new UpdateUsfmParserHandler(rows, fullName is null ? null : $"- {fullName}", behavior);
+            var handler = new UpdateUsfmParserHandler(
+                rows,
+                fullName is null ? null : $"- {fullName}",
+                textBehavior,
+                embeddedBehavior,
+                styleBehavior
+            );
             try
             {
                 UsfmParser.Parse(usfm, handler, _settings.Stylesheet, _settings.Versification);

src/SIL.Machine/Corpora/ScriptureRefUsfmParserHandlerBase.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using SIL.Extensions;
 using SIL.Scripture;
 
 namespace SIL.Machine.Corpora
@@ -9,7 +10,8 @@ public enum ScriptureTextType
         None,
         NonVerse,
         Verse,
-        Note
+        Embedded,
+        NoteText
     }
 
     public abstract class ScriptureRefUsfmParserHandlerBase : UsfmParserHandlerBase
@@ -19,6 +21,9 @@ public abstract class ScriptureRefUsfmParserHandlerBase : UsfmParserHandlerBase
         private readonly Stack<ScriptureTextType> _curTextType;
         private bool _duplicateVerse = false;
 
+        private bool _inEmbedded;
+        public bool InNoteText { get; private set; }
+
         protected ScriptureRefUsfmParserHandlerBase()
         {
             _curElements = new Stack<ScriptureElement>();
@@ -59,7 +64,7 @@ string pubNumber
                 // ignore duplicate verses
                 _duplicateVerse = true;
             }
-            else if (VerseRef.AreOverlappingVersesRanges(number, _curVerseRef.Verse))
+            else if (VerseRef.AreOverlappingVersesRanges(verse1: number, verse2: _curVerseRef.Verse))
             {
                 // merge overlapping verse ranges in to one range
                 VerseRef verseRef = _curVerseRef.Clone();
@@ -153,20 +158,36 @@ public override void EndSidebar(UsfmParserState state, string marker, bool close
 
         public override void StartNote(UsfmParserState state, string marker, string caller, string category)
         {
-            if (CurrentTextType != ScriptureTextType.None && !_duplicateVerse)
+            _inEmbedded = true;
+            StartEmbedded(state, marker, caller, category);
+        }
+
+        public override void EndNote(UsfmParserState state, string marker, bool closed)
+        {
+            EndNoteText(state);
+            EndEmbedded(state, marker, null, closed);
+            _inEmbedded = false;
+        }
+
+        public virtual void StartEmbedded(UsfmParserState state, string marker, string caller, string category)
+        {
+            if (_curVerseRef.IsDefault)
+                UpdateVerseRef(state.VerseRef, marker);
+
+            if (!_duplicateVerse)
             {
                 // if we hit a note in a verse paragraph and we aren't in a verse, then start a non-verse segment
                 CheckConvertVerseParaToNonVerse(state);
                 NextElement(marker);
-                StartNoteText(state);
             }
         }
 
-        public override void EndNote(UsfmParserState state, string marker, bool closed)
-        {
-            if (CurrentTextType == ScriptureTextType.Note && !_duplicateVerse)
-                EndNoteText(state);
-        }
+        public virtual void EndEmbedded(
+            UsfmParserState state,
+            string marker,
+            IReadOnlyList<UsfmAttribute> attributes,
+            bool closed
+        ) { }
 
         public override void Text(UsfmParserState state, string text)
         {
@@ -187,9 +208,37 @@ public override void StartChar(
             IReadOnlyList<UsfmAttribute> attributes
         )
         {
+            if (IsEmbeddedPart(markerWithoutPlus))
+                EndNoteText(state);
+
             // if we hit a character marker in a verse paragraph and we aren't in a verse, then start a non-verse
             // segment
             CheckConvertVerseParaToNonVerse(state);
+
+            if (IsEmbeddedCharacter(markerWithoutPlus))
+            {
+                _inEmbedded = true;
+                StartEmbedded(state, markerWithoutPlus, null, null);
+            }
+
+            if (IsNoteText(markerWithoutPlus))
+            {
+                StartNoteText(state);
+            }
+        }
+
+        public override void EndChar(
+            UsfmParserState state,
+            string marker,
+            IReadOnlyList<UsfmAttribute> attributes,
+            bool closed
+        )
+        {
+            if (IsEmbeddedCharacter(marker))
+            {
+                EndEmbedded(state, marker, attributes, closed);
+                _inEmbedded = false;
+            }
         }
 
         protected virtual void StartVerseText(UsfmParserState state, IReadOnlyList<ScriptureRef> scriptureRefs) { }
@@ -200,8 +249,25 @@ protected virtual void StartNonVerseText(UsfmParserState state, ScriptureRef scr
 
         protected virtual void EndNonVerseText(UsfmParserState state, ScriptureRef scriptureRef) { }
 
+        public virtual void StartNoteText(UsfmParserState state)
+        {
+            InNoteText = true;
+            _curTextType.Push(ScriptureTextType.NoteText);
+            StartNoteText(state, CreateNonVerseRef());
+        }
+
         protected virtual void StartNoteText(UsfmParserState state, ScriptureRef scriptureRef) { }
 
+        public virtual void EndNoteText(UsfmParserState state)
+        {
+            if (_curTextType.Count > 0 && _curTextType.Peek() == ScriptureTextType.NoteText)
+            {
+                EndNoteText(state, CreateNonVerseRef());
+                _curTextType.Pop();
+                InNoteText = false;
+            }
+        }
+
         protected virtual void EndNoteText(UsfmParserState state, ScriptureRef scriptureRef) { }
 
         private void StartVerseText(UsfmParserState state)
@@ -227,22 +293,11 @@ private void StartNonVerseText(UsfmParserState state)
 
         private void EndNonVerseText(UsfmParserState state)
         {
+            EndEmbeddedElements();
             EndNonVerseText(state, CreateNonVerseRef());
             _curTextType.Pop();
         }
 
-        private void StartNoteText(UsfmParserState state)
-        {
-            _curTextType.Push(ScriptureTextType.Note);
-            StartNoteText(state, CreateNonVerseRef());
-        }
-
-        private void EndNoteText(UsfmParserState state)
-        {
-            EndNoteText(state, CreateNonVerseRef());
-            _curTextType.Pop();
-        }
-
         private void UpdateVerseRef(VerseRef verseRef, string marker)
         {
             if (!VerseRef.AreOverlappingVersesRanges(verseRef, _curVerseRef))
@@ -270,6 +325,12 @@ private void EndParentElement()
             _curElements.Pop();
         }
 
+        private void EndEmbeddedElements()
+        {
+            if (_curElements.Count > 0 && IsEmbeddedCharacter(_curElements.Peek().Name))
+                _curElements.Pop();
+        }
+
         private IReadOnlyList<ScriptureRef> CreateVerseRefs()
         {
             return _curVerseRef.HasMultiple
@@ -300,5 +361,25 @@ private void CheckConvertVerseParaToNonVerse(UsfmParserState state)
                 StartNonVerseText(state);
             }
         }
+
+        public bool InEmbedded(string marker)
+        {
+            return _inEmbedded || IsEmbeddedCharacter(marker);
+        }
+
+        private static bool IsNoteText(string marker)
+        {
+            return marker == "ft";
+        }
+
+        public static bool IsEmbeddedPart(string marker)
+        {
+            return !(marker is null) && marker.Length > 0 && marker[0].IsOneOf('f', 'x', 'z');
+        }
+
+        private static bool IsEmbeddedCharacter(string marker)
+        {
+            return marker.IsOneOf("f", "fe", "fig", "fm", "x");
+        }
     }
 }