bug #1152: Fix for indexing small docs (#1155)

* Fix for indexing small docs

* fix update

* format using black

* black formatting test check

* fix for test_split_empty_pages()

* run ruff

* add test

* reduce diff size

---------

Co-authored-by: Matt Gotteiner <[email protected]>

Author: marekjakimiuk1

scripts/prepdocslib/textsplitter.py

@@ -41,7 +41,14 @@ def find_page(offset):
             return pages[num_pages - 1].page_num
 
         all_text = "".join(page.text for page in pages)
+        if len(all_text.strip()) == 0:
+            return
+
         length = len(all_text)
+        if length <= self.max_section_length:
+            yield SplitPage(page_num=find_page(0), text=all_text)
+            return
+
         start = 0
         end = length
         while start + self.section_overlap < length:

tests/test_prepdocslib_textsplitter.py

@@ -6,7 +6,7 @@
 from scripts.prepdocslib.listfilestrategy import LocalListFileStrategy
 from scripts.prepdocslib.pdfparser import LocalPdfParser
 from scripts.prepdocslib.searchmanager import Section
-from scripts.prepdocslib.textsplitter import TextSplitter
+from scripts.prepdocslib.textsplitter import Page, TextSplitter
 
 
 def test_split_empty_pages():
@@ -15,6 +15,15 @@ def test_split_empty_pages():
     assert list(t.split_pages([])) == []
 
 
+def test_split_small_pages():
+    t = TextSplitter(has_image_embeddings=False, verbose=True)
+
+    split_pages = list(t.split_pages(pages=[Page(page_num=0, offset=0, text="Not a large page")]))
+    assert len(split_pages) == 1
+    assert split_pages[0].page_num == 0
+    assert split_pages[0].text == "Not a large page"
+
+
 @pytest.mark.asyncio
 async def test_list_parse_and_split(tmp_path):
     text_splitter = TextSplitter(False, True)