Port sillsdev/machine#362; fix gloss inclusion

Author: Enkidu93

machine/corpora/__init__.py

@@ -26,7 +26,7 @@
 from .paratext_project_file_handler import ParatextProjectFileHandler
 from .paratext_project_settings import ParatextProjectSettings
 from .paratext_project_settings_parser_base import ParatextProjectSettingsParserBase
-from .paratext_project_terms_parser_base import ParatextProjectTermsParserBase
+from .paratext_project_terms_parser_base import KeyTerm, ParatextProjectTermsParserBase
 from .paratext_project_text_updater_base import ParatextProjectTextUpdaterBase
 from .paratext_project_versification_error_detector import ParatextProjectVersificationErrorDetector
 from .paratext_text_corpus import ParatextTextCorpus
@@ -115,6 +115,7 @@
     "FileParatextProjectVersificationErrorDetector",
     "flatten",
     "is_scripture",
+    "KeyTerm",
     "lowercase",
     "MemoryAlignmentCollection",
     "MemoryStreamContainer",

machine/corpora/paratext_backup_terms_corpus.py

@@ -1,10 +1,9 @@
 from typing import List, Sequence, Tuple
 from zipfile import ZipFile
 
-from .key_term_row import KeyTerm
-
 from ..utils.typeshed import StrPath
 from .dictionary_text_corpus import DictionaryTextCorpus
+from .key_term_row import KeyTerm
 from .memory_text import MemoryText
 from .text_row import TextRow
 from .zip_paratext_project_settings_parser import ZipParatextProjectSettingsParser

machine/corpora/paratext_project_terms_parser_base.py

@@ -8,11 +8,8 @@
 from xml.etree import ElementTree
 
 from ..scripture.constants import ORIGINAL_VERSIFICATION
-
 from ..scripture.verse_ref import VerseRef
-
 from .key_term_row import KeyTerm
-
 from .paratext_project_file_handler import ParatextProjectFileHandler
 from .paratext_project_settings import ParatextProjectSettings
 from .paratext_project_settings_parser_base import ParatextProjectSettingsParserBase
@@ -111,14 +108,17 @@ def parse(self, term_categories: Sequence[str], use_term_glosses: bool = True) -
                     terms_glosses[id_].extend(glosses)
         if terms_glosses or terms_renderings:
             terms: List[KeyTerm] = []
-            for id, renderings_patterns in terms_renderings.items():
+            for id in sorted(set(terms_renderings.keys()).union(terms_glosses.keys())):
+                renderings_patterns = terms_renderings.get(id, [])
                 category = term_id_to_category_dict.get(id, "?")
                 domain = term_id_to_domain_dict.get(id, "?")
                 glosses = terms_glosses.get(id, [])
                 references = term_id_to_references_dict.get(id, [])
                 renderings = [r.replace("*", "") for r in renderings_patterns]
                 if len(renderings) == 0:
-                    continue
+                    if len(glosses) == 0:
+                        continue
+                    renderings = glosses
                 term = KeyTerm(
                     id=id,
                     category=category,

machine/corpora/usfm_versification_error_detector.py

@@ -4,6 +4,7 @@
 from machine.scripture import canon
 
 from ..scripture.verse_ref import ValidStatus, VerseRef, Versification
+from .paratext_project_settings import ParatextProjectSettings
 from .usfm_parser_handler import UsfmParserHandler
 from .usfm_parser_state import UsfmParserState
 
@@ -25,6 +26,7 @@ def __init__(
         expected_verse: int,
         actual_chapter: int,
         actual_verse: int,
+        project_name: str,
         verse_ref: Optional[VerseRef] = None,
     ):
         self._book_num = book_num
@@ -34,11 +36,16 @@ def __init__(
         self._actual_verse = actual_verse
         self._verse_ref = verse_ref
         self._type: UsfmVersificationErrorType
+        self._project_name = project_name
 
     @property
     def type(self) -> UsfmVersificationErrorType:
         return self._type
 
+    @property
+    def project_name(self) -> str:
+        return self._project_name
+
     def check_error(self) -> bool:
         """Returns true if there is an error"""
         if self._expected_chapter > self._actual_chapter and self._expected_verse != 0:
@@ -71,15 +78,15 @@ def map(valid_status: ValidStatus) -> UsfmVersificationErrorType:
 
     @property
     def expected_verse_ref(self) -> str:
+        if self._type == UsfmVersificationErrorType.EXTRA_VERSE:
+            return ""
         if (
             default_verse_ref := VerseRef.try_from_string(
-                f"{self._book_num} {self._expected_chapter}:{self._expected_verse}"
+                f"{canon.book_number_to_id(self._book_num)} {self._expected_chapter}:{self._expected_verse}"
             )
             is None
         ):
             return self.default_verse(self._expected_chapter, self._expected_verse)
-        if self._type == UsfmVersificationErrorType.EXTRA_VERSE:
-            return ""
         if self._type == UsfmVersificationErrorType.MISSING_VERSE_SEGMENT:
             if (
                 verse_ref_with_segment := VerseRef.try_from_string(
@@ -96,7 +103,7 @@ def expected_verse_ref(self) -> str:
                 return str(first_verse)
             elif (
                 corrected_verse_range_ref := VerseRef.try_from_string(
-                    f"{self._book_num} {self._expected_chapter}:{first_verse}-{last_verse}"
+                    f"{canon.book_number_to_id(self._book_num)} {self._expected_chapter}:{first_verse}-{last_verse}"
                 )
                 is not None
             ):
@@ -119,8 +126,9 @@ def default_verse(self, chapter: int, verse: int):
 
 
 class UsfmVersificationErrorDetector(UsfmParserHandler):
-    def __init__(self, versification: Versification):
-        self._versification = versification
+    def __init__(self, settings: ParatextProjectSettings):
+        self._project_name = settings.name
+        self._versification = settings.versification
         self._current_book = 0
         self._current_chapter = 0
         self._current_verse = VerseRef()
@@ -140,6 +148,7 @@ def end_usfm(self, state: UsfmParserState) -> None:
                 ),
                 self._current_chapter,
                 list(self._current_verse.all_verses())[-1].verse_num,
+                self._project_name,
             )
             if versification_error.check_error():
                 self._errors.append(versification_error)
@@ -159,6 +168,7 @@ def chapter(
                 self._versification.get_last_verse(self._current_book, self._current_chapter),
                 self._current_chapter,
                 list(self._current_verse.all_verses())[-1].verse_num,
+                self._project_name,
             )
             if versification_error.check_error():
                 self._errors.append(versification_error)
@@ -173,6 +183,8 @@ def verse(
                 list(self._current_verse.all_verses())[-1].verse_num,
                 self._current_chapter,
                 list(self._current_verse.all_verses())[-1].verse_num,
+                self._project_name,
+                self._current_verse,
             )
             if versification_error.check_error():
                 self._errors.append(versification_error)

tests/corpora/test_paratext_project_terms_parser.py

@@ -1,11 +1,10 @@
 from typing import Dict, List, Optional, Sequence, Tuple
 
-from ...machine.corpora.key_term_row import KeyTerm
 from testutils.memory_paratext_project_file_handler import DefaultParatextProjectSettings
 from testutils.memory_paratext_project_terms_parser import MemoryParatextProjectTermsParser
 
-from machine.corpora import ParatextProjectSettings, ParatextProjectTermsParserBase
-from machine.corpora.paratext_project_terms_parser_base import _get_glosses, _get_renderings, _strip_parens
+from machine.corpora import KeyTerm, ParatextProjectSettings, ParatextProjectTermsParserBase
+from machine.corpora.paratext_project_terms_parser_base import _get_glosses, _get_renderings_with_pattern, _strip_parens
 
 
 def test_get_key_terms_from_terms_renderings() -> None:
@@ -48,7 +47,7 @@ def test_get_key_terms_from_terms_localizations_no_term_renderings() -> None:
     assert len(terms) == 5726
 
     glosses = terms[0].renderings
-    assert str.join(" ", glosses) == "Abagtha"
+    assert str.join(" ", glosses) == "Aaron"
 
 
 def test_get_key_terms_from_terms_localizations_no_term_renderings_do_not_use_term_glosses() -> None:
@@ -97,8 +96,8 @@ def test_get_key_terms_from_terms_localizations_term_renderings_exists_prefer_lo
 
     terms_index1_glosses = terms[1].renderings
     terms_index2_glosses = terms[2].renderings
-    assert str.join(" ", terms_index1_glosses) == "Abagtha"
-    assert str.join(" ", terms_index2_glosses) == "Abi"
+    assert str.join(" ", terms_index1_glosses) == "Obadiah"
+    assert str.join(" ", terms_index2_glosses) == "Abagtha"
 
 
 def test_strip_parens() -> None:
@@ -116,11 +115,11 @@ def test_get_glosses() -> None:
 
 
 def test_get_renderings() -> None:
-    assert _get_renderings("") == []
-    assert _get_renderings("*Abba*") == ["Abba"]
-    assert _get_renderings("Abba|| ") == ["Abba"]
-    assert _get_renderings("Abba||Abbah") == ["Abba", "Abbah"]
-    assert _get_renderings("Abba (note)") == ["Abba"]
+    assert _get_renderings_with_pattern("") == []
+    assert _get_renderings_with_pattern("*Abba*") == ["*Abba*"]
+    assert _get_renderings_with_pattern("Abba|| ") == ["Abba"]
+    assert _get_renderings_with_pattern("Abba||Abbah") == ["Abba", "Abbah"]
+    assert _get_renderings_with_pattern("Abba (note)") == ["Abba"]
 
 
 class _TestEnvironment:

tests/corpora/test_usfm_versification_error_detector.py

@@ -36,7 +36,7 @@ def get_usfm_versification_errors_no_errors():
     assert len(env.get_usfm_versification_errors()) == 0
 
 
-def get_usfm_versification_errors_missing_verses():
+def get_usfm_versification_errors_missing_verse():
     env = _TestEnvironment(
         files={
             "653JNTest.SFM": r"""\id 3JN
@@ -61,6 +61,8 @@ def get_usfm_versification_errors_missing_verses():
     errors = env.get_usfm_versification_errors()
     assert len(errors) == 1
     assert errors[0].type == UsfmVersificationErrorType.MISSING_VERSE
+    assert errors[0].expected_verse_ref == "3JN 1:15"
+    assert errors[0].actual_verse_ref == "3JN 1:14"
 
 
 def get_usfm_versification_missing_chapter():
@@ -73,6 +75,8 @@ def get_usfm_versification_missing_chapter():
     errors = env.get_usfm_versification_errors()
     assert len(errors) == 1
     assert errors[0].type == UsfmVersificationErrorType.MISSING_CHAPTER
+    assert errors[0].expected_verse_ref == "3JN 1:15"
+    assert errors[0].actual_verse_ref == "3JN 0:0"
 
 
 def get_usfm_versification_errors_extra_verse():
@@ -102,6 +106,8 @@ def get_usfm_versification_errors_extra_verse():
     errors = env.get_usfm_versification_errors()
     assert len(errors) == 1
     assert errors[0].type == UsfmVersificationErrorType.EXTRA_VERSE
+    assert errors[0].expected_verse_ref == ""
+    assert errors[0].actual_verse_ref == "3JN 1:16"
 
 
 def get_usfm_versification_errors_invalid_verse():
@@ -129,6 +135,8 @@ def get_usfm_versification_errors_invalid_verse():
     errors = env.get_usfm_versification_errors()
     assert len(errors) == 1
     assert errors[0].type == UsfmVersificationErrorType.INVALID_VERSE_RANGE
+    assert errors[0].expected_verse_ref == "3JN 1:12-13"
+    assert errors[0].actual_verse_ref == "3JN 1:13-12"
 
 
 def get_usfm_versification_errors_extra_verse_segment():
@@ -158,6 +166,8 @@ def get_usfm_versification_errors_extra_verse_segment():
     errors = env.get_usfm_versification_errors()
     assert len(errors) == 1
     assert errors[0].type == UsfmVersificationErrorType.EXTRA_VERSE_SEGMENT
+    assert errors[0].expected_verse_ref == "3JN 1:14"
+    assert errors[0].actual_verse_ref == "3JN 1:14a"
 
 
 def get_usfm_versification_errors_missing_verse_segments():
@@ -186,6 +196,8 @@ def get_usfm_versification_errors_missing_verse_segments():
     errors = env.get_usfm_versification_errors()
     assert len(errors) == 1
     assert errors[0].type == UsfmVersificationErrorType.MISSING_VERSE_SEGMENT
+    assert errors[0].expected_verse_ref == "3JN 1:13a"
+    assert errors[0].actual_verse_ref == "3JN 1:13"
 
 
 def get_usfm_versification_errors_ignore_noncanonicals():
@@ -227,6 +239,8 @@ def get_usfm_versification_errors_excluded_in_custom_vrs():
     errors = env.get_usfm_versification_errors()
     assert len(errors) == 1
     assert errors[0].type == UsfmVersificationErrorType.EXTRA_VERSE
+    assert errors[0].expected_verse_ref == ""
+    assert errors[0].actual_verse_ref == "3JN 1:13"
 
 
 def get_usfm_versification_errors_multiple_books():
@@ -270,6 +284,8 @@ def get_usfm_versification_errors_multiple_books():
     errors = env.get_usfm_versification_errors()
     assert len(errors) == 1
     assert errors[0].type == UsfmVersificationErrorType.MISSING_VERSE
+    assert errors[0].expected_verse_ref == "2JN 1:13"
+    assert errors[0].actual_verse_ref == "2JN 1:12"
 
 
 def get_usfm_versification_errors_multiple_chapters():
@@ -298,6 +314,10 @@ def get_usfm_versification_errors_multiple_chapters():
     assert len(errors) == 2
     assert errors[0].type == UsfmVersificationErrorType.MISSING_VERSE
     assert errors[0].type == UsfmVersificationErrorType.EXTRA_VERSE
+    assert errors[0].expected_verse_ref == "2JN 1:13"
+    assert errors[0].actual_verse_ref == "2JN 1:12"
+    assert errors[1].expected_verse_ref == ""
+    assert errors[1].actual_verse_ref == "2JN 2:1"
 
 
 class _TestEnvironment: