Merge with the last version

Author: pprados

test_unstructured/partition/pdf_image/test_pdf.py

@@ -1518,16 +1518,17 @@ def test_document_to_element_list_sets_category_depth_titles():
     assert elements[2].metadata.category_depth is None
     assert elements[3].metadata.category_depth == 0
 
+
 @pytest.mark.parametrize("file_mode", ["filename", "rb", "spool"])
 @pytest.mark.parametrize(
     ("strategy", "origin"),
     # fast: can't capture the "intentionally left blank page" page
     # others: will ignore the actual blank page
     [
-        (PartitionStrategy.FAST,  {"pdfminer"}),
-        (PartitionStrategy.FAST,  {"pdfminer"}),
+        (PartitionStrategy.FAST, {"pdfminer"}),
+        (PartitionStrategy.FAST, {"pdfminer"}),
         (PartitionStrategy.HI_RES, {"yolox", "pdfminer", "ocr_tesseract"}),
-        (PartitionStrategy.OCR_ONLY,  {"ocr_tesseract"}),
+        (PartitionStrategy.OCR_ONLY, {"ocr_tesseract"}),
     ],
 )
 def test_partition_pdf_with_password(
@@ -1540,28 +1541,21 @@ def test_partition_pdf_with_password(
     def _test(result):
         # validate that the result is a non-empty list of dicts
         assert len(result) == 1
-        assert result[0].text == 'File with password'
+        assert result[0].text == "File with password"
 
     if file_mode == "filename":
-        result = pdf.partition_pdf(
-            filename=filename, strategy=strategy,
-            password="password"
-        )
+        result = pdf.partition_pdf(filename=filename, strategy=strategy, password="password")
         _test(result)
     elif file_mode == "rb":
         with open(filename, "rb") as f:
-            result = pdf.partition_pdf(
-                file=f, strategy=strategy,
-                password="password"
-            )
+            result = pdf.partition_pdf(file=f, strategy=strategy, password="password")
             _test(result)
     else:
         with open(filename, "rb") as test_file:
             spooled_temp_file = SpooledTemporaryFile()
             spooled_temp_file.write(test_file.read())
             spooled_temp_file.seek(0)
             result = pdf.partition_pdf(
-                file=spooled_temp_file, strategy=strategy,
-                password="password"
+                file=spooled_temp_file, strategy=strategy, password="password"
             )
             _test(result)

unstructured/partition/image.py

@@ -32,7 +32,7 @@ def partition_image(
     starting_page_number: int = 1,
     extract_forms: bool = False,
     form_extraction_skip_tables: bool = True,
-    password:Optional[str]=None,
+    password: Optional[str] = None,
     **kwargs: Any,
 ) -> list[Element]:
     """Parses an image into a list of interpreted elements.

unstructured/partition/pdf.py

@@ -12,8 +12,7 @@
 import numpy as np
 import wrapt
 from pdfminer import psparser
-from pdfminer.layout import LTChar, LTContainer, LTImage, LTItem, LTTextBox
-from pdfminer.pdftypes import PDFObjRef
+from pdfminer.layout import LTContainer, LTImage, LTItem, LTTextBox
 from pdfminer.utils import open_filename
 from pi_heif import register_heif_opener
 from PIL import Image as PILImage
@@ -365,7 +364,7 @@ def extractable_elements(
     languages: Optional[list[str]] = None,
     metadata_last_modified: Optional[str] = None,
     starting_page_number: int = 1,
-    password:Optional[str] = None,
+    password: Optional[str] = None,
     **kwargs: Any,
 ) -> list[list[Element]]:
     if isinstance(file, bytes):
@@ -387,7 +386,7 @@ def _partition_pdf_with_pdfminer(
     languages: list[str],
     metadata_last_modified: Optional[str],
     starting_page_number: int = 1,
-    password:Optional[str] = None,
+    password: Optional[str] = None,
     **kwargs: Any,
 ) -> list[list[Element]]:
     """Partitions a PDF using PDFMiner instead of using a layoutmodel. Used for faster
@@ -446,7 +445,7 @@ def _process_pdfminer_pages(
 
     for page_number, (page, page_layout) in enumerate(
         open_pdfminer_pages_generator(fp, password=password),
-            start=starting_page_number,
+        start=starting_page_number,
     ):
         width, height = page_layout.width, page_layout.height
 
@@ -568,7 +567,7 @@ def _partition_pdf_or_image_local(
     extract_forms: bool = False,
     form_extraction_skip_tables: bool = True,
     pdf_hi_res_max_pages: Optional[int] = None,
-    password:Optional[str] = None,
+    password: Optional[str] = None,
     **kwargs: Any,
 ) -> list[Element]:
     """Partition using package installed locally"""
@@ -609,8 +608,7 @@ def _partition_pdf_or_image_local(
         )
 
         extracted_layout, layouts_links = (
-            process_file_with_pdfminer(filename=filename, dpi=pdf_image_dpi,
-                                       password=password)
+            process_file_with_pdfminer(filename=filename, dpi=pdf_image_dpi, password=password)
             if pdf_text_extractable
             else ([], [])
         )
@@ -879,8 +877,7 @@ def _partition_pdf_or_image_with_ocr(
             elements.extend(page_elements)
     else:
         for page_number, image in enumerate(
-            convert_pdf_to_images(filename, file, password=password),
-                start=starting_page_number
+            convert_pdf_to_images(filename, file, password=password), start=starting_page_number
         ):
             page_elements = _partition_pdf_or_image_with_ocr_from_image(
                 image=image,
@@ -1177,11 +1174,11 @@ def document_to_element_list(
                     word = ""
 
                 if len(word) == 0:
-                    start_index = text_len + index
-                    x1 = character.x0
-                    y2 = height - character.y0
-                    x2 = character.x1
-                    y1 = height - character.y1
+                    text_len + index
+                    character.x0
+                    height - character.y0
+                    character.x1
+                    height - character.y1
             else:
 
                 element.metadata.links = (

unstructured/partition/pdf_image/pdf_image_utils.py

@@ -58,7 +58,7 @@ def convert_pdf_to_image(
     dpi: int = 200,
     output_folder: Optional[Union[str, PurePath]] = None,
     path_only: bool = False,
-    password:Optional[str] = None,
+    password: Optional[str] = None,
 ) -> Union[List[Image.Image], List[str]]:
     """Get the image renderings of the pdf pages using pdf2image"""
 
@@ -127,7 +127,7 @@ def save_elements(
     is_image: bool = False,
     extract_image_block_to_payload: bool = False,
     output_dir_path: str | None = None,
-    password:Optional[str] = None,
+    password: Optional[str] = None,
 ):
     """
     Saves specific elements from a PDF as images either to a directory or embeds them in the
@@ -393,7 +393,7 @@ def convert_pdf_to_images(
     filename: str = "",
     file: Optional[bytes | IO[bytes]] = None,
     chunk_size: int = 10,
-    password:Optional[str] = None,
+    password: Optional[str] = None,
 ) -> Iterator[Image.Image]:
     # Convert a PDF in small chunks of pages at a time (e.g. 1-10, 11-20... and so on)
     exactly_one(filename=filename, file=file)

unstructured/partition/pdf_image/pdfminer_processing.py

@@ -51,7 +51,7 @@ def process_file_with_pdfminer(
 def process_data_with_pdfminer(
     file: Optional[Union[bytes, BinaryIO]] = None,
     dpi: int = 200,
-    password:Optional[str]=None,
+    password: Optional[str] = None,
 ) -> tuple[List[List["TextRegion"]], List[List]]:
     """Loads the image and word objects from a pdf using pdfplumber and the image renderings of the
     pdf pages using pdf2image"""
@@ -65,8 +65,9 @@ def process_data_with_pdfminer(
     layouts_links = []
     # Coefficient to rescale bounding box to be compatible with images
     coef = dpi / 72
-    for page_number, (page, page_layout) in (
-            enumerate(open_pdfminer_pages_generator(file, password=password))):
+    for page_number, (page, page_layout) in enumerate(
+        open_pdfminer_pages_generator(file, password=password)
+    ):
         width, height = page_layout.width, page_layout.height
 
         text_layout = []

unstructured/partition/pdf_image/pdfminer_utils.py

@@ -73,7 +73,7 @@ def rect_to_bbox(
 @requires_dependencies(["pikepdf", "pypdf"])
 def open_pdfminer_pages_generator(
     fp: BinaryIO,
-    password:Optional[str]=None,
+    password: Optional[str] = None,
 ):
     """Open PDF pages using PDFMiner, handling and repairing invalid dictionary constructs."""