Update layout analysis

Author: BenConstable9

image_processing/src/image_processing/layout_analysis.py

@@ -356,12 +356,20 @@ def create_per_page_starting_sentence(self) -> list[PerPageStartingSentenceHolde
                 + page.spans[0]["length"]
             ]
 
-            starting_sentence = page_content.split(".")[0]
+            # Remove any leading whitespace/newlines.
+            cleaned_content = page_content.lstrip()
+            # If a newline appears before a period, split on newline; otherwise, on period.
+            if "\n" in cleaned_content:
+                first_line = cleaned_content.split("\n", 1)[0]
+            elif "." in cleaned_content:
+                first_line = cleaned_content.split(".", 1)[0]
+            else:
+                first_line = cleaned_content
 
             per_page_starting_sentences.append(
                 PerPageStartingSentenceHolder(
                     page_number=page.page_number,
-                    starting_sentence=starting_sentence,
+                    starting_sentence=first_line.strip(),
                 )
             )
 

image_processing/src/image_processing/semantic_text_chunker.py

@@ -507,11 +507,16 @@ def assign_page_number_to_chunks(
             list[ChunkHolder]: The list of chunks with page numbers assigned."""
         page_number = 1
         for chunk in chunks:
-            if per_page_starting_sentences:
-                for per_page_starting_sentence in per_page_starting_sentences:
-                    if per_page_starting_sentence.starting_sentence in chunk:
-                        page_number = per_page_starting_sentence.page_number
-                        break
+            for per_page_starting_sentence in per_page_starting_sentences[
+                page_number - 1 :
+            ]:
+                if per_page_starting_sentence.starting_sentence in chunk:
+                    logging.info(
+                        "Assigning page number %i to chunk",
+                        per_page_starting_sentence.page_number,
+                    )
+                    page_number = per_page_starting_sentence.page_number
+                    break
             chunk.page_number = page_number
         return chunks
 
@@ -546,6 +551,8 @@ async def process_semantic_text_chunker(record: dict, text_chunker) -> dict:
                 for sentence in record["data"]["per_page_starting_sentences"]
             ]
 
+            logging.info(f"Per page starting sentences: {per_page_starting_sentences}")
+
             chunks = text_chunker.assign_page_number_to_chunks(
                 chunks, per_page_starting_sentences
             )