remove cv2 preprocessing (#204)

This PR removes `cv2` preprocessing before OCR step in table
transformer.

The preprocessing in practice tend to lead to lower recall from OCR
results. For example this image:


![table-multi-row-column-cells](https://github.com/Unstructured-IO/unstructured-inference/assets/647930/baffd3bd-d34d-404f-83f2-b29f3c42fb6e)


with current code:

```python
from unstructured_inference.models.tables import UnstructuredTableTransformerModel
from PIL import Image

model = UnstructuredTableTransformerModel()
model.initialize("microsoft/table-transformer-structure-recognition")
prediction = model.predict(Image.open('table.png'))
```

produces 

![Screenshot 2023-09-08 at 9 39 07
AM](https://github.com/Unstructured-IO/unstructured-inference/assets/647930/00857b02-c75e-446c-8c34-0b6e2d42b0f7)

after removing the preprocessing (this PR):

![Screenshot 2023-09-08 at 12 15 13
PM](https://github.com/Unstructured-IO/unstructured-inference/assets/647930/2171570c-641a-46a1-9be7-4664497d7727)

Author: badGarnet

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.5.24
+
+* remove `cv2` preprocessing step before OCR step in table transformer
+
 ## 0.5.23
 
 * Add functionality to bring back embedded images in PDF

sample-docs/table-multi-row-column-cells.png

@@ -34,102 +34,6 @@ def test_load_donut_model(model_path):
     assert type(table_model.model.model.decoder) is TableTransformerDecoder
 
 
-@pytest.fixture()
-def sample_table_transcript(platform_type):
-    if platform_type == "x86_64":
-        out = (
-            '<table><thead><th colspan="6">About these Coverage Examples:</th></thead><thead><th>'
-            '</th><th colspan="5">This is not a cost estimator. Treatments shown are just examples '
-            "of how this plan might cover medical care. Your actual costs will be different "
-            "depending on the actual care you receive, the prices your providers charge, and many "
-            "other factors. Focus on the cost sharing amounts (deductibles, copayments and "
-            "coinsurance) and excluded services under the plan. Use this information to compare "
-            "the portion of costs you might pay under different health plans. Please note these "
-            'coverage examples are based on self-only coverage</th></thead><thead><th colspan="2">'
-            "Peg is Having a Baby (9 months of in-network pre-natal care and a hospital delivery)</"
-            "th><th>Managing Joe's type 2 Diabetes (a year of routine in-network care of a well- "
-            'controlled condition)</th><th></th><th colspan="2">Mia\'s Simple Fracture (in-network '
-            'emergency room visit and follow up care)</th></thead><tr><td colspan="2">The plan\'s '
-            "overall deductible $750 Specialist copayment $50  Hospital (facility) coinsurance 10"
-            "%  Other coinsurance 10%</td><td>The plan's overall deductible Specialist copayment "
-            r"Hospital (facility) coinsurance  Other coinsurance</td><td>$750 $50 10% 10%</td><td>"
-            "The plan's overall deductible Specialist copayment  Hospital (facility) coinsurance  "
-            r'Other coinsurance</td><td>$750 $50 10% 10%</td></tr><tr><td colspan="2" rowspan="2">'
-            "This EXAMPLE event includes services like: Specialist office visits (prenatal care) "
-            "Childbirth/Delivery Professional Services Childbirth/Delivery Facility Services</td><"
-            'td colspan="2" rowspan="2">This EXAMPLE event includes services like: Primary care '
-            "physician office visits (including disease education) Diagnostic tests (blood work) "
-            'Prescription drugs Durable medical equipment (glucose meter)</td><td colspan="2" '
-            'rowspan="2">This EXAMPLE event includes services like: Emergency room care (including '
-            "medical Diagnostic test (x-ray) Durable medical equipment (crutches) Rehabilitation "
-            'services (physical therapy)</td></tr><tr><td colspan="2">Diagnostic tests ('
-            "ultrasounds and blood work) Specialist visit (anesthesia)</td></tr><tr><td>Total "
-            "Example Cost</td><td>$12,700</td><td>Total Example Cost</td><td>$5,600</td><td>Total "
-            'Example Cost</td><td>$2,800</td></tr><tr><td colspan="2">In this example, Peg would '
-            'pay:</td><td>In this example, Joe would pay:</td><td colspan="3">In this example, Mia '
-            'would pay:</td></tr><tr><td>Cost Sharing</td><td></td><td colspan="2">Cost Sharing</td'
-            "><td>Cost Sharing</td><td></td></tr><tr><td>Deductibles</td><td>$750</td><td>"
-            "Deductibles</td><td>$120</td><td>Deductibles</td><td>$750</td></tr><tr><td>Copayments"
-            '</td><td>$30</td><td>Copayments</td><td>$700</td><td colspan="2" rowspan="2">'
-            'Copayments $400 Coinsurance $30</td></tr><tr><td colspan="2" rowspan="2">Coinsurance $'
-            "1,200 What isn't covered</td><td>Coinsurance</td><td>$0</td></tr><tr><td>What isn't "
-            "covered</td><td></td><td>What isn't covered</td><td></td></tr><tr><td>Limits or "
-            "exclusions</td><td>$20</td><td>Limits or exclusions</td><td>$20</td><td>Limits or "
-            "exclusions</td><td>$0</td></tr><tr><td>The total Peg would pay is</td><td>$2,000</td><"
-            'td>The total Joe would pay is</td><td>$840</td><td colspan="2">The total Mia would '
-            "pay is $1,180</td></tr><tr><td>Plan Name: NVIDIA PPO PlanPIan ID: 14603022</td><td></"
-            "td><td>The plan would be responsible for the other costs of these EXAMPLE covered "
-            "services</td><td></td><td></td><td>Page 8 of 8</td></tr></table>"
-        )
-    else:
-        out = (
-            '<table><thead><th colspan="6">About these Coverage Examples:</th></thead><thead><th>'
-            "This is not a cost   depending on the (deductibles, pay under different</th><th "
-            'colspan="5">estimator. |reatments shown are just examples of how this plan might '
-            "cover medical care. Your actual costs will be different actual care you receive, the "
-            "prices your providers charge, and many other factors. Focus on the cost sharing "
-            "amounts copayments and coinsurance) and excluded services under the plan. Use this "
-            "information to compare the portion of costs you might health plans. Please note these "
-            'coverage examples are based on self-only coverage.</th></thead><thead><th colspan="2">'
-            "Peg is Having a Baby (9 months of in-network pre-natal care and a hospital delivery)</"
-            "th><th>Managing Joe's type 2 (a year of routine in-network care controlled conaition"
-            ')</th><th>Diabetes of a well-</th><th colspan="2">Mia\'s Simple Fracture (in-network '
-            'emergency room visit and follow up   care)</th></thead><tr><td colspan="2">= The plan'
-            "'s overall deductible $750 = Specialist copayment $50 = Hospital (facility) "
-            "coinsurance 10% = Other coinsurance 10%</td><td>= The plan's overall deductible = "
-            "Specialist copayment = Hospital (facility) coinsurance = Other coinsurance</td><td>$"
-            r"750 $50 10% 10%</td><td>= The plan's overall deductible = Specialist copayment = "
-            r"Hospital (facility) coinsurance = Other coinsurance</td><td>$750 $50 10% 10%</td></tr"
-            '><tr><td colspan="2" rowspan="2">This EXAMPLE event includes services like: '
-            "specialist office visits (prenatal care) Childbirth/Delivery Professional Services "
-            'Childbirth/Delivery Facility Services</td><td colspan="2" rowspan="2">This EXAMPLE '
-            "event includes services like: Primary care physician office visits (including aisease "
-            "education) Diagnostic tests (b/ood work) Prescription drugs Durable medical equipment "
-            '(/g/ucose meter)</td><td colspan="2" rowspan="2">This EXAMPLE event includes services '
-            "like: Emergency room care (including meaical suoplies) Diagnostic test (x-ray) "
-            "Durable medical equipment (crutches) Rehabilitation services (o/hysical therapy)</td"
-            '></tr><tr><td colspan="2">Diagnostic tests (u/trasounas and blood work) specialist '
-            "visit (anesthesia)</td></tr><tr><td>Total Example Cost</td><td>| $12,700</td><td>"
-            "Total Example Cost |</td><td>$5,600</td><td>Total Example Cost</td><td>| $2,800</td></"
-            'tr><tr><td colspan="2">In this example, Peg would pay:</td><td>In this example, Joe '
-            'would pay:</td><td colspan="3">In this example, Mia would pay:</td></tr><tr><td>Cost '
-            'Sharing</td><td></td><td colspan="2">Cost Sharing</td><td>Cost Sharing</td><td></td></'
-            "tr><tr><td>Deductibles</td><td>$/50</td><td>Deductibles</td><td>$120</td><td>"
-            "Deductibles</td><td>$/50</td></tr><tr><td>Copayments</td><td>$30</td><td>Copayments</"
-            'td><td>$/00</td><td colspan="2" rowspan="2">Copayments $400 Coinsurance $30</td></tr><'
-            'tr><td colspan="2" rowspan="2">Coinsurance $1,200 What isn t covered</td><td>'
-            "Coinsurance</td><td></td></tr><tr><td>What isnt covered</td><td></td><td>What isnt "
-            "covered</td><td></td></tr><tr><td>Limits or exclusions</td><td>$20</td><td>Limits or "
-            "exclusions |</td><td>$20</td><td>Limits or exclusions</td><td></td></tr><tr><td>The "
-            "total Peg would pay is</td><td>$2,000</td><td>The total Joe would pay is</td><td>9840"
-            '</td><td colspan="2">The total Mia would pay is $1,180</td></tr><tr><td>Plan Name: '
-            "NVIDIA PPO Plan</td><td>The plan would Plan ID: 14603022</td><td>be responsible for "
-            "the other costs of these</td><td>EXAMPLE</td><td>covered services.</td><td>Page 8 of 8"
-            "</td></tr></table>"
-        )
-    return out
-
-
 @pytest.mark.parametrize(
     ("input_test", "output_test"),
     [
@@ -185,7 +89,12 @@ def test_nms(input_test, output_test):
             {
                 "label": "table spanning cell",
                 "score": 0.526617169380188,
-                "bbox": [1446.2801513671875, 1023.817138671875, 2114.3525390625, 1099.20166015625],
+                "bbox": [
+                    1446.2801513671875,
+                    1023.817138671875,
+                    2114.3525390625,
+                    1099.20166015625,
+                ],
                 "projected row header": False,
                 "header": False,
                 "row_numbers": [3, 4],
@@ -210,7 +119,12 @@ def test_nms(input_test, output_test):
             {
                 "label": "table spanning cell",
                 "score": 0.526617169380188,
-                "bbox": [1446.2801513671875, 1023.817138671875, 2114.3525390625, 1099.20166015625],
+                "bbox": [
+                    1446.2801513671875,
+                    1023.817138671875,
+                    2114.3525390625,
+                    1099.20166015625,
+                ],
                 "projected row header": False,
                 "header": False,
                 "row_numbers": [3, 4],
@@ -235,7 +149,12 @@ def test_nms(input_test, output_test):
             {
                 "label": "table spanning cell",
                 "score": 0.526617169380188,
-                "bbox": [1446.2801513671875, 1023.817138671875, 2114.3525390625, 1099.20166015625],
+                "bbox": [
+                    1446.2801513671875,
+                    1023.817138671875,
+                    2114.3525390625,
+                    1099.20166015625,
+                ],
                 "projected row header": False,
                 "header": False,
                 "row_numbers": [3, 4],
@@ -260,7 +179,12 @@ def test_nms(input_test, output_test):
             {
                 "label": "table spanning cell",
                 "score": 0.526617169380188,
-                "bbox": [1446.2801513671875, 1023.817138671875, 2114.3525390625, 1099.20166015625],
+                "bbox": [
+                    1446.2801513671875,
+                    1023.817138671875,
+                    2114.3525390625,
+                    1099.20166015625,
+                ],
                 "projected row header": False,
                 "header": False,
                 "row_numbers": [3, 4],
@@ -402,22 +326,35 @@ def test_align_rows(rows, bbox, output):
     assert postprocess.align_rows(rows, bbox) == output
 
 
+# TODO: break this test down so it doesn't account for nearly 8% of test coverage
 @pytest.mark.parametrize(
     ("model_path", "platform_type"),
     [
         ("microsoft/table-transformer-structure-recognition", "arm64"),
         ("microsoft/table-transformer-structure-recognition", "x86_64"),
     ],
 )
-def test_table_prediction(model_path, sample_table_transcript, platform_type):
+def test_table_prediction(model_path, platform_type):
     with patch("platform.machine", return_value=platform_type):
         table_model = tables.UnstructuredTableTransformerModel()
         from PIL import Image
 
         table_model.initialize(model=model_path)
-        img = Image.open("./sample-docs/example_table.jpg").convert("RGB")
+        img = Image.open("./sample-docs/table-multi-row-column-cells.png").convert("RGB")
         prediction = table_model.predict(img)
-        assert prediction.strip() == sample_table_transcript.strip()
+        # assert rows spans two rows are detected
+        assert '<table><thead><th rowspan="2">' in prediction
+        # one of the safest rows to detect should be present
+        assert (
+            "<tr>"
+            "<td>Blind</td>"
+            "<td>5</td>"
+            "<td>1</td>"
+            "<td>4</td>"
+            "<td>34.5%, n=1</td>"
+            "<td>1199 sec, n=1</td>"
+            "</tr>"
+        ) in prediction
 
 
 def test_intersect():
@@ -551,18 +488,53 @@ def test_extract_text_from_spans(spans, join_with_space, expected):
                     "column_numbers": [1, 2, 3],
                     "score": 0.9,
                 },
-                {"header": "hi", "row_numbers": [1], "column_numbers": [1], "score": 0.9},
-                {"header": "hi", "row_numbers": [1], "column_numbers": [2], "score": 0.9},
-                {"header": "hi", "row_numbers": [1], "column_numbers": [3], "score": 0.9},
+                {
+                    "header": "hi",
+                    "row_numbers": [1],
+                    "column_numbers": [1],
+                    "score": 0.9,
+                },
+                {
+                    "header": "hi",
+                    "row_numbers": [1],
+                    "column_numbers": [2],
+                    "score": 0.9,
+                },
+                {
+                    "header": "hi",
+                    "row_numbers": [1],
+                    "column_numbers": [3],
+                    "score": 0.9,
+                },
             ],
             4,
         ),
         (
             [
-                {"header": "hi", "row_numbers": [0], "column_numbers": [0], "score": 0.9},
-                {"header": "hi", "row_numbers": [1], "column_numbers": [0], "score": 0.9},
-                {"header": "hi", "row_numbers": [1, 2], "column_numbers": [0], "score": 0.9},
-                {"header": "hi", "row_numbers": [3], "column_numbers": [0], "score": 0.9},
+                {
+                    "header": "hi",
+                    "row_numbers": [0],
+                    "column_numbers": [0],
+                    "score": 0.9,
+                },
+                {
+                    "header": "hi",
+                    "row_numbers": [1],
+                    "column_numbers": [0],
+                    "score": 0.9,
+                },
+                {
+                    "header": "hi",
+                    "row_numbers": [1, 2],
+                    "column_numbers": [0],
+                    "score": 0.9,
+                },
+                {
+                    "header": "hi",
+                    "row_numbers": [3],
+                    "column_numbers": [0],
+                    "score": 0.9,
+                },
             ],
             3,
         ),

test_unstructured_inference/models/test_tables.py

@@ -1 +1 @@
-__version__ = "0.5.23"  # pragma: no cover
+__version__ = "0.5.24"  # pragma: no cover

unstructured_inference/__version__.py

@@ -7,7 +7,6 @@
 from pathlib import Path
 from typing import Optional, Union
 
-import cv2
 import numpy as np
 import pandas as pd
 import pytesseract
@@ -78,21 +77,9 @@ def get_tokens(self, x: Image):
                     "No module named 'unstructured_paddleocr', falling back to tesseract",
                 )
                 pass
-        zoom = 6
-        img = cv2.resize(
-            cv2.cvtColor(np.array(x), cv2.COLOR_RGB2BGR),
-            None,
-            fx=zoom,
-            fy=zoom,
-            interpolation=cv2.INTER_CUBIC,
-        )
-
-        kernel = np.ones((1, 1), np.uint8)
-        img = cv2.dilate(img, kernel, iterations=1)
-        img = cv2.erode(img, kernel, iterations=1)
 
         ocr_df: pd.DataFrame = pytesseract.image_to_data(
-            Image.fromarray(img),
+            x,
             output_type="data.frame",
         )
 
@@ -103,10 +90,10 @@ def get_tokens(self, x: Image):
             tokens.append(
                 {
                     "bbox": [
-                        idtx.left / zoom,
-                        idtx.top / zoom,
-                        (idtx.left + idtx.width) / zoom,
-                        (idtx.top + idtx.height) / zoom,
+                        idtx.left,
+                        idtx.top,
+                        idtx.left + idtx.width,
+                        idtx.top + idtx.height,
                     ],
                     "text": idtx.text,
                 },