Ensure there are no zero-length sections for batch API (#1423)

* Assert there are no zero-length sections

* Correct the overlap mechanism when the text section doesn't contain any breaks

* Update textsplitter.py

Author: tonybaloney

scripts/prepdocslib/textsplitter.py

@@ -127,8 +127,10 @@ def split_page_by_max_tokens(self, page_num: int, text: str) -> Generator[SplitP
             else:
                 # Split page in half and call function again
                 # Overlap first and second halves by DEFAULT_OVERLAP_PERCENT%
-                first_half = text[: int(len(text) // (2.0 + (DEFAULT_OVERLAP_PERCENT / 100)))]
-                second_half = text[int(len(text) // (1.0 - (DEFAULT_OVERLAP_PERCENT / 100))) :]
+                middle = int(len(text) // 2)
+                overlap = int(len(text) * (DEFAULT_OVERLAP_PERCENT / 100))
+                first_half = text[: middle + overlap]
+                second_half = text[middle - overlap :]
             yield from self.split_page_by_max_tokens(page_num, first_half)
             yield from self.split_page_by_max_tokens(page_num, second_half)
 

tests/test_prepdocslib_textsplitter.py

@@ -120,6 +120,7 @@ async def test_sentencetextsplitter_multilang(test_doc, tmp_path):
         # Verify the size of the sections
         token_lengths = []
         for section in sections:
+            assert section.split_page.text != ""
             assert len(section.split_page.text) <= (text_splitter.max_section_length * 1.2)
             # Verify the number of tokens is below 500
             token_lengths.append((len(bpe.encode(section.split_page.text)), len(section.split_page.text)))