Fix regex patterns for abbreviations in tokenization_utils_seperator.py

Author: AlejandroEsquivel

guardrails/utils/tokenization_utils_seperator.py

@@ -147,9 +147,9 @@ def postproc_splits(sentences, separator):
 
     # No sentence break after specific abbreviations
     abbreviations = [
-        r"e\.?g\.",
-        r"i\.?e\.",
-        r"i\.?v\.",
+        r"e\. ?g\.",
+        r"i\. ?e\.",
+        r"i\. ?v\.",
         r"vs\.",
         r"cf\.",
         r"Dr\.",
@@ -165,11 +165,12 @@ def postproc_splits(sentences, separator):
         r"Fig\.",
         r"vol\.",
         r"Vols\.",
-        r"No\.",
+        r"no\.",
         r"Nos\.",
         r"et\.",
         r"al\.",
-        r"Inc\.",
+        r"i\. ?v\.",
+        r"inc\.",
         r"Ltd\.",
         r"Co\.",
         r"Corp\.",
@@ -188,7 +189,7 @@ def postproc_splits(sentences, separator):
         r"sen\.",
         r"st\.",
         r"vs\.",
-        r"i\.?e\.",
+        r"i\. ?e\.",
     ]
     for abbr in abbreviations:
         sentences = re.sub(

guardrails/validator_base.py

@@ -75,7 +75,7 @@ def split_sentence_word_tokenizers_jl_separator(
 
     # check for potential line endings, which is what split_sentences does
     chunk_with_potential_line_endings, count = re.subn(
-        r"([?!.])\s?", rf"\1{separator}", chunk
+        r"([?!.])(?=\s|$)", rf"\1{separator}", chunk
     )
     any_potential_line_endings = count > 0
     if not is_minimum_length or not any_potential_line_endings:

tests/integration_tests/test_streaming.py

@@ -584,8 +584,7 @@ def test_fix_behavior_three_validators(mocker):
     assert (
         text
         == """"REDACTED!!, under purple!! bridges, roams,
-<LOCATION> hills, his home.
-dreams of fog, and salty air,
+<LOCATION> hills, his home.dreams of fog, and salty air,
 in his heart, he's always there."""
     )
     assert (