MAINT: Move text extraction state to TextExtraction class (#3350)

---------

Co-authored-by: Stefan <[email protected]>

Author: MartinThoma

pypdf/__init__.py

@@ -11,8 +11,9 @@
 from ._doc_common import DocumentInformation
 from ._encryption import PasswordType
 from ._merger import PdfMerger
-from ._page import PageObject, Transformation, mult
+from ._page import PageObject, Transformation
 from ._reader import PdfReader
+from ._text_extraction import mult
 from ._version import __version__
 from ._writer import ObjectDeletionFlag, PdfWriter
 from .constants import ImageType

pypdf/_page.py

@@ -51,14 +51,10 @@
 
 from ._cmap import (
     build_char_map,
-    unknown_char_map,
 )
 from ._protocols import PdfCommonDocProtocol
 from ._text_extraction import (
-    OrientationNotFoundError,
     _layout_mode,
-    crlf_space_check,
-    mult,
 )
 from ._text_extraction._text_extractor import TextExtraction
 from ._utils import (
@@ -1657,7 +1653,7 @@ def _debug_for_extract(self) -> str:  # pragma: no cover
             out += "No Font\n"
         return out
 
-    def _extract_text(  # noqa: C901, PLR0915  # Will be fixed soon.
+    def _extract_text(
         self,
         obj: Any,
         pdf: Any,
@@ -1678,9 +1674,6 @@ def _extract_text(  # noqa: C901, PLR0915  # Will be fixed soon.
 
         """
         extractor = TextExtraction()
-        text: str = ""
-        output: str = ""
-        rtl_dir: bool = False  # right-to-left
         cmaps: Dict[
             str,
             Tuple[
@@ -1707,14 +1700,6 @@ def _extract_text(  # noqa: C901, PLR0915  # Will be fixed soon.
                     cmaps[f] = build_char_map(f, space_width, obj)
                 except TypeError:
                     pass
-        cmap: Tuple[
-            Union[str, Dict[int, str]], Dict[str, str], str, Optional[DictionaryObject]
-        ] = (
-            "charmap",
-            {},
-            "NotInitialized",
-            None,
-        )  # (encoding, CMAP, font resource name, font)
 
         try:
             content = (
@@ -1728,245 +1713,57 @@ def _extract_text(  # noqa: C901, PLR0915  # Will be fixed soon.
         # are strings where the byte->string encoding was unknown, so adding
         # them to the text here would be gibberish.
 
-        cm_matrix: List[float] = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
-        tm_matrix: List[float] = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
-        cm_stack = []
-
-        # Store the last modified matrices; can be an intermediate position
-        cm_prev: List[float] = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
-        tm_prev: List[float] = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
-
-        # Store the position at the beginning of building the text
-        memo_cm: List[float] = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
-        memo_tm: List[float] = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
-
-        char_scale = 1.0
-        space_scale = 1.0
-        _space_width: float = 500.0  # will be set correctly at first Tf
-        _actual_str_size: Dict[str, float] = {
-            "str_widths": 0.0, "space_width": 0.0, "str_height": 0.0
-        }  # will be set to string length calculation result
-        TL = 0.0
-        font_size = 12.0  # init just in case of
-
-        def compute_str_widths(str_widths: float) -> float:
-            return str_widths / 1000
-
-        def process_operation(operator: bytes, operands: List[Any]) -> None:
-            nonlocal cm_matrix, tm_matrix, cm_stack, cm_prev, tm_prev, memo_cm, memo_tm
-            nonlocal char_scale, space_scale, _space_width, TL, font_size, cmap
-            nonlocal orientations, rtl_dir, visitor_text, output, text, _actual_str_size
-
-            str_widths: float = 0.0
-
-            # Table 5.4 page 405
-            if operator == b"BT":  # Begin Text
-                tm_matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
-                # Flush text:
-                output += text
-                if visitor_text is not None:
-                    visitor_text(text, memo_cm, memo_tm, cmap[3], font_size)
-                text = ""
-                memo_cm = cm_matrix.copy()
-                memo_tm = tm_matrix.copy()
-                return
-            if operator == b"ET":  # End Text
-                # Flush text:
-                output += text
-                if visitor_text is not None:
-                    visitor_text(text, memo_cm, memo_tm, cmap[3], font_size)
-                text = ""
-                memo_cm = cm_matrix.copy()
-                memo_tm = tm_matrix.copy()
-
-            # Table 4.7 "Graphics state operators", page 219
-            # cm_matrix calculation is reserved for later
-            elif operator == b"q":  # Save graphics state
-                cm_stack.append(
-                    (
-                        cm_matrix,
-                        cmap,
-                        font_size,
-                        char_scale,
-                        space_scale,
-                        _space_width,
-                        TL,
-                    )
-                )
-            elif operator == b"Q":  # Restore graphics state
-                try:
-                    (
-                        cm_matrix,
-                        cmap,
-                        font_size,
-                        char_scale,
-                        space_scale,
-                        _space_width,
-                        TL,
-                    ) = cm_stack.pop()
-                except Exception:
-                    cm_matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
-            elif operator == b"cm":  # Modify current matrix
-                output += text
-                if visitor_text is not None:
-                    visitor_text(text, memo_cm, memo_tm, cmap[3], font_size)
-                text = ""
-                try:
-                    cm_matrix = mult(
-                        [float(operand) for operand in operands[:6]],
-                        cm_matrix
-                    )
-                except Exception:
-                    cm_matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
-                memo_cm = cm_matrix.copy()
-                memo_tm = tm_matrix.copy()
-
-            # Table 5.2 page 398
-            elif operator == b"Tz":  # Set horizontal text scaling
-                char_scale = float(operands[0]) / 100 if operands else 1.0
-            elif operator == b"Tw":  # Set word spacing
-                space_scale = 1.0 + float(operands[0] if operands else 0.0)
-            elif operator == b"TL":  # Set Text Leading
-                scale_x = math.sqrt(tm_matrix[0]**2 + tm_matrix[2]**2)
-                TL = float(operands[0] if operands else 0.0) * font_size * scale_x
-            elif operator == b"Tf":  # Set font size
-                if text != "":
-                    output += text  # .translate(cmap)
-                    if visitor_text is not None:
-                        visitor_text(text, memo_cm, memo_tm, cmap[3], font_size)
-                text = ""
-                memo_cm = cm_matrix.copy()
-                memo_tm = tm_matrix.copy()
-                try:
-                    # char_map_tuple: font_type,
-                    #                 float(sp_width / 2),
-                    #                 encoding,
-                    #                 map_dict,
-                    #                 font_dict (describes the font)
-                    char_map_tuple = cmaps[operands[0]]
-                    # current cmap: encoding,
-                    #               map_dict,
-                    #               font resource name (internal name, not the real font name),
-                    #               font_dict
-                    cmap = (
-                        char_map_tuple[2],
-                        char_map_tuple[3],
-                        operands[0],
-                        char_map_tuple[4],
-                    )
-                    _space_width = char_map_tuple[1]
-                except KeyError:  # font not found
-                    cmap = (
-                        unknown_char_map[2],
-                        unknown_char_map[3],
-                        f"???{operands[0]}",
-                        None,
-                    )
-                    _space_width = unknown_char_map[1]
-                try:
-                    font_size = float(operands[1])
-                except Exception:
-                    pass  # keep previous size
-            # Table 5.5 page 406
-            elif operator == b"Td":  # Move text position
-                # A special case is a translating only tm:
-                # tm = [1, 0, 0, 1, e, f]
-                # i.e. tm[4] += tx, tm[5] += ty.
-                tx, ty = float(operands[0]), float(operands[1])
-                tm_matrix[4] += tx * tm_matrix[0] + ty * tm_matrix[2]
-                tm_matrix[5] += tx * tm_matrix[1] + ty * tm_matrix[3]
-                str_widths = compute_str_widths(_actual_str_size["str_widths"])
-                _actual_str_size["str_widths"] = 0.0
-            elif operator == b"Tm":  # Set text matrix
-                tm_matrix = [float(operand) for operand in operands[:6]]
-                str_widths = compute_str_widths(_actual_str_size["str_widths"])
-                _actual_str_size["str_widths"] = 0.0
-            elif operator == b"T*":  # Move to next line
-                tm_matrix[4] -= TL * tm_matrix[2]
-                tm_matrix[5] -= TL * tm_matrix[3]
-                str_widths = compute_str_widths(_actual_str_size["str_widths"])
-                _actual_str_size["str_widths"] = 0.0
-            elif operator == b"Tj":  # Show text
-                text, rtl_dir, _actual_str_size = extractor._handle_tj(
-                    text,
-                    operands,
-                    cm_matrix,
-                    tm_matrix,
-                    cmap,
-                    orientations,
-                    font_size,
-                    rtl_dir,
-                    visitor_text,
-                    _space_width,
-                    _actual_str_size,
-                )
-            else:
-                return
-
-            if operator in {b"Td", b"Tm", b"T*", b"Tj"}:
-                try:
-                    text, output, cm_prev, tm_prev = crlf_space_check(
-                        text,
-                        (cm_prev, tm_prev),
-                        (cm_matrix, tm_matrix),
-                        (memo_cm, memo_tm),
-                        cmap,
-                        orientations,
-                        output,
-                        font_size,
-                        visitor_text,
-                        str_widths,
-                        compute_str_widths(_actual_str_size["space_width"]),
-                        _actual_str_size["str_height"]
-                    )
-                    if text == "":
-                        memo_cm = cm_matrix.copy()
-                        memo_tm = tm_matrix.copy()
-                except OrientationNotFoundError:
-                    return
+        # Initialize the extractor with the necessary parameters
+        extractor.initialize_extraction(orientations, visitor_text, cmaps)
 
         for operands, operator in content.operations:
             if visitor_operand_before is not None:
-                visitor_operand_before(operator, operands, cm_matrix, tm_matrix)
+                visitor_operand_before(operator, operands, extractor.cm_matrix, extractor.tm_matrix)
             # Multiple operators are handled here
             if operator == b"'":
-                process_operation(b"T*", [])
-                process_operation(b"Tj", operands)
+                extractor.process_operation(b"T*", [])
+                extractor.process_operation(b"Tj", operands)
             elif operator == b'"':
-                process_operation(b"Tw", [operands[0]])
-                process_operation(b"Tc", [operands[1]])
-                process_operation(b"T*", [])
-                process_operation(b"Tj", operands[2:])
+                extractor.process_operation(b"Tw", [operands[0]])
+                extractor.process_operation(b"Tc", [operands[1]])
+                extractor.process_operation(b"T*", [])
+                extractor.process_operation(b"Tj", operands[2:])
             elif operator == b"TJ":
                 # The space width may be smaller than the font width, so the width should be 95%.
-                _confirm_space_width = _space_width * 0.95
+                _confirm_space_width = extractor._space_width * 0.95
                 if operands:
                     for op in operands[0]:
                         if isinstance(op, (str, bytes)):
-                            process_operation(b"Tj", [op])
+                            extractor.process_operation(b"Tj", [op])
                         if isinstance(op, (int, float, NumberObject, FloatObject)) and (
                             abs(float(op)) >= _confirm_space_width
-                            and text
-                            and text[-1] != " "
+                            and extractor.text
+                            and extractor.text[-1] != " "
                         ):
-                            process_operation(b"Tj", [" "])
+                            extractor.process_operation(b"Tj", [" "])
             elif operator == b"TD":
-                process_operation(b"TL", [-operands[1]])
-                process_operation(b"Td", operands)
+                extractor.process_operation(b"TL", [-operands[1]])
+                extractor.process_operation(b"Td", operands)
             elif operator == b"Do":
-                output += text
+                extractor.output += extractor.text
                 if visitor_text is not None:
-                    visitor_text(text, memo_cm, memo_tm, cmap[3], font_size)
+                    visitor_text(
+                        extractor.text,
+                        extractor.memo_cm,
+                        extractor.memo_tm,
+                        extractor.cmap[3],
+                        extractor.font_size,
+                    )
                 try:
-                    if output[-1] != "\n":
-                        output += "\n"
+                    if extractor.output[-1] != "\n":
+                        extractor.output += "\n"
                         if visitor_text is not None:
                             visitor_text(
                                 "\n",
-                                memo_cm,
-                                memo_tm,
-                                cmap[3],
-                                font_size,
+                                extractor.memo_cm,
+                                extractor.memo_tm,
+                                extractor.cmap[3],
+                                extractor.font_size,
                             )
                 except IndexError:
                     pass
@@ -1981,32 +1778,38 @@ def process_operation(operator: bytes, operands: List[Any]) -> None:
                             visitor_operand_after,
                             visitor_text,
                         )
-                        output += text
+                        extractor.output += text
                         if visitor_text is not None:
                             visitor_text(
                                 text,
-                                memo_cm,
-                                memo_tm,
-                                cmap[3],
-                                font_size,
+                                extractor.memo_cm,
+                                extractor.memo_tm,
+                                extractor.cmap[3],
+                                extractor.font_size,
                             )
                 except Exception as exception:
                     logger_warning(
                         f"Impossible to decode XFormObject {operands[0]}: {exception}",
                         __name__,
                     )
                 finally:
-                    text = ""
-                    memo_cm = cm_matrix.copy()
-                    memo_tm = tm_matrix.copy()
+                    extractor.text = ""
+                    extractor.memo_cm = extractor.cm_matrix.copy()
+                    extractor.memo_tm = extractor.tm_matrix.copy()
             else:
-                process_operation(operator, operands)
+                extractor.process_operation(operator, operands)
             if visitor_operand_after is not None:
-                visitor_operand_after(operator, operands, cm_matrix, tm_matrix)
-        output += text  # just in case
-        if text != "" and visitor_text is not None:
-            visitor_text(text, memo_cm, memo_tm, cmap[3], font_size)
-        return output
+                visitor_operand_after(operator, operands, extractor.cm_matrix, extractor.tm_matrix)
+        extractor.output += extractor.text  # just in case
+        if extractor.text != "" and visitor_text is not None:
+            visitor_text(
+                extractor.text,
+                extractor.memo_cm,
+                extractor.memo_tm,
+                extractor.cmap[3],
+                extractor.font_size,
+            )
+        return extractor.output
 
     def _layout_mode_fonts(self) -> Dict[str, _layout_mode.Font]:
         """