Skip to content

Commit f453ecc

Browse files
fix for optional white space after potential line endings ?!.
1 parent 3fb5765 commit f453ecc

File tree

1 file changed

+1
-1
lines changed

1 file changed

+1
-1
lines changed

guardrails/validator_base.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ def split_sentence_word_tokenizers_jl(chunk: str):
6969
is_minimum_length = True
7070

7171
# check for potential line endings, which is what split_sentences does
72-
chunk_with_potential_line_endings, count = re.subn(r"([?!.])\s", r"\1\n", chunk)
72+
chunk_with_potential_line_endings, count = re.subn(r"([?!.])(\s)?", r"\1\n", chunk)
7373
any_potential_line_endings = count > 0
7474
if not is_minimum_length or not any_potential_line_endings:
7575
return []

0 commit comments

Comments
 (0)