feat: add xls support (#632)

cragwolfe · web-flow · commit c5d9469001e3 · 2023-05-26T01:55:32.000-07:00
Add support for older .XLS files from the partition function in unstructured.partition.auto.

Note, this should also work on the centos7 unstructured image (with the requirements/*txt updates in this PR).
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,13 @@
+## 0.6.10
+
+### Enhancements
+
+* XLS support from auto partiton
+
+### Features
+
+### Fixes
+
 ## 0.6.9
 
 ### Enhancements
diff --git a/example-docs/tests-example.xls b/example-docs/tests-example.xls
diff --git a/requirements/base.in b/requirements/base.in
@@ -14,3 +14,4 @@ python-pptx
 python-magic
 markdown
 requests
+xlrd
diff --git a/requirements/base.txt b/requirements/base.txt
@@ -19,7 +19,7 @@ certifi==2023.5.7
 cffi==1.15.1
     # via cryptography
 chardet==5.1.0
-    # via unstructured (setup.py)
+    # via -r requirements/base.in
 charset-normalizer==3.1.0
     # via
     #   pdfminer-six
@@ -123,7 +123,7 @@ tqdm==4.65.0
     #   nltk
 typer==0.9.0
     # via argilla
-typing-extensions==4.6.0
+typing-extensions==4.6.2
     # via
     #   pydantic
     #   rich
@@ -136,6 +136,8 @@ wrapt==1.14.1
     # via
     #   argilla
     #   deprecated
+xlrd==2.0.1
+    # via -r requirements/base.in
 xlsxwriter==3.1.1
     # via python-pptx
 zipp==3.15.0
diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -39,8 +39,6 @@ cffi==1.15.1
     #   argon2-cffi-bindings
 cfgv==3.3.1
     # via pre-commit
-chardet==5.1.0
-    # via -r requirements/dev.in
 click==8.1.3
     # via
     #   -c requirements/base.txt
@@ -144,7 +142,7 @@ jupyter-core==5.3.0
     #   qtconsole
 jupyter-events==0.6.3
     # via jupyter-server
-jupyter-server==2.5.0
+jupyter-server==2.6.0
     # via
     #   nbclassic
     #   notebook-shim
@@ -192,6 +190,8 @@ notebook==6.5.4
     # via jupyter
 notebook-shim==0.2.3
     # via nbclassic
+overrides==7.3.1
+    # via jupyter-server
 packaging==23.1
     # via
     #   -c requirements/base.txt
@@ -221,7 +221,7 @@ platformdirs==3.5.1
     #   virtualenv
 pre-commit==3.3.2
     # via -r requirements/dev.in
-prometheus-client==0.16.0
+prometheus-client==0.17.0
     # via
     #   jupyter-server
     #   nbclassic
@@ -346,7 +346,7 @@ traitlets==5.9.0
     #   nbformat
     #   notebook
     #   qtconsole
-typing-extensions==4.6.0
+typing-extensions==4.6.2
     # via
     #   -c requirements/base.txt
     #   -c requirements/test.txt
diff --git a/requirements/huggingface.txt b/requirements/huggingface.txt
@@ -90,7 +90,7 @@ tqdm==4.65.0
     #   transformers
 transformers==4.29.2
     # via -r requirements/huggingface.in
-typing-extensions==4.6.0
+typing-extensions==4.6.2
     # via
     #   -c requirements/base.txt
     #   huggingface-hub
diff --git a/requirements/ingest-azure.txt b/requirements/ingest-azure.txt
@@ -93,7 +93,7 @@ six==1.16.0
     #   azure-core
     #   azure-identity
     #   isodate
-typing-extensions==4.6.0
+typing-extensions==4.6.2
     # via
     #   -c requirements/base.txt
     #   azure-core
diff --git a/requirements/ingest-google-drive.txt b/requirements/ingest-google-drive.txt
@@ -17,9 +17,9 @@ charset-normalizer==3.1.0
     #   requests
 google-api-core==2.11.0
     # via google-api-python-client
-google-api-python-client==2.86.0
+google-api-python-client==2.87.0
     # via -r requirements/ingest-google-drive.in
-google-auth==2.18.1
+google-auth==2.19.0
     # via
     #   google-api-core
     #   google-api-python-client
diff --git a/requirements/ingest-s3.txt b/requirements/ingest-s3.txt
@@ -52,7 +52,7 @@ six==1.16.0
     # via
     #   -c requirements/base.txt
     #   python-dateutil
-typing-extensions==4.6.0
+typing-extensions==4.6.2
     # via
     #   -c requirements/base.txt
     #   aioitertools
diff --git a/requirements/local-inference.txt b/requirements/local-inference.txt
@@ -102,7 +102,7 @@ numpy==1.23.5
     #   transformers
 omegaconf==2.3.0
     # via effdet
-onnxruntime==1.14.1
+onnxruntime==1.15.0
     # via unstructured-inference
 opencv-python==4.7.0.72
     # via
@@ -226,7 +226,7 @@ tqdm==4.65.0
     #   transformers
 transformers==4.29.2
     # via unstructured-inference
-typing-extensions==4.6.0
+typing-extensions==4.6.2
     # via
     #   -c requirements/base.txt
     #   huggingface-hub
diff --git a/requirements/test.txt b/requirements/test.txt
@@ -22,7 +22,7 @@ click==8.1.3
     #   -c requirements/base.txt
     #   -r requirements/test.in
     #   black
-coverage[toml]==7.2.5
+coverage[toml]==7.2.6
     # via
     #   -r requirements/test.in
     #   pytest-cov
@@ -79,7 +79,7 @@ pytest==7.3.1
     # via
     #   pytest-cov
     #   pytest-mock
-pytest-cov==4.0.0
+pytest-cov==4.1.0
     # via -r requirements/test.in
 pytest-mock==3.10.0
     # via -r requirements/test.in
@@ -89,7 +89,7 @@ requests==2.31.0
     # via
     #   -c requirements/base.txt
     #   label-studio-sdk
-ruff==0.0.269
+ruff==0.0.270
     # via -r requirements/test.in
 six==1.16.0
     # via
@@ -109,7 +109,7 @@ types-tabulate==0.9.0.2
     # via -r requirements/test.in
 types-urllib3==1.26.25.13
     # via types-requests
-typing-extensions==4.6.0
+typing-extensions==4.6.2
     # via
     #   -c requirements/base.txt
     #   black
@@ -120,7 +120,7 @@ urllib3==1.26.16
     #   -c requirements/base.txt
     #   -c requirements/constraints.in
     #   requests
-vcrpy==4.2.1
+vcrpy==4.3.0
     # via -r requirements/test.in
 wrapt==1.14.1
     # via
diff --git a/test_unstructured/partition/test_auto.py b/test_unstructured/partition/test_auto.py
@@ -695,6 +695,96 @@ def test_auto_partition_xlsx_from_file(filename="example-docs/stanley-cups.xlsx"
     assert elements[0].metadata.filetype == EXPECTED_XLSX_FILETYPE
 
 
+EXPECTED_XLS_TEXT_LEN = 883
+
+EXPECTED_XLS_INITIAL_45_CLEAN_TEXT = "MA What C datatypes are 8 bits? (assume i386)"
+
+EXPECTED_XLS_TABLE = (
+    """<table border="1" class="dataframe">
+  <tbody>
+    <tr>
+      <td>MA</td>
+      <td>What C datatypes are 8 bits? (assume i386)</td>
+      <td>int</td>
+      <td></td>
+      <td>float</td>
+      <td></td>
+      <td>double</td>
+      <td></td>
+      <td>char</td>
+    </tr>
+    <tr>
+      <td>TF</td>
+      <td>Bagpipes are awesome.</td>
+      <td>true</td>
+      <td></td>
+      <td></td>
+      <td></td>
+      <td></td>
+      <td></td>
+      <td></td>
+    </tr>
+    <tr>
+      <td>ESS</td>
+      <td>How have the original Henry Hornbostel buildings """
+    """influenced campus architecture and design in the last 30 years?</td>
+      <td></td>
+      <td></td>
+      <td></td>
+      <td></td>
+      <td></td>
+      <td></td>
+      <td></td>
+    </tr>
+    <tr>
+      <td>ORD</td>
+      <td>Rank the following in their order of operation.</td>
+      <td>Parentheses</td>
+      <td>Exponents</td>
+      <td>Division</td>
+      <td>Addition</td>
+      <td></td>
+      <td></td>
+      <td></td>
+    </tr>
+    <tr>
+      <td>FIB</td>
+      <td>The student activities fee is</td>
+      <td>95</td>
+      <td>dollars for students enrolled in</td>
+      <td>19</td>
+      <td>units or more,</td>
+      <td></td>
+      <td></td>
+      <td></td>
+    </tr>
+    <tr>
+      <td>MAT</td>
+      <td>Match the lower-case greek letter with its capital form.</td>
+      <td>λ</td>
+      <td>Λ</td>
+      <td>α</td>
+      <td>γ</td>
+      <td>Γ</td>
+      <td>φ</td>
+      <td>Φ</td>
+    </tr>
+  </tbody>
+</table>"""
+)
+
+
+def test_auto_partition_xls_from_filename(filename="example-docs/tests-example.xls"):
+    elements = partition(filename=filename)
+
+    assert all(isinstance(element, Table) for element in elements)
+    assert len(elements) == 3
+
+    assert clean_extra_whitespace(elements[0].text)[:45] == EXPECTED_XLS_INITIAL_45_CLEAN_TEXT
+    assert len(elements[0].text) == EXPECTED_XLS_TEXT_LEN
+    assert elements[0].metadata.text_as_html == EXPECTED_XLS_TABLE
+
+
 @pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
 def test_auto_partition_csv_from_filename(filename="example-docs/stanley-cups.csv"):
     elements = partition(filename=filename)
diff --git a/unstructured/__version__.py b/unstructured/__version__.py
@@ -1 +1 @@
-__version__ = "0.6.9"  # pragma: no cover
+__version__ = "0.6.10"  # pragma: no cover
diff --git a/unstructured/file_utils/filetype.py b/unstructured/file_utils/filetype.py
@@ -208,6 +208,10 @@ def detect_filetype(
     if mime_type == "text/plain" and extension == ".json":
         return FileType.JSON
 
+    # NOTE(Crag): older magic lib does not differentiate between xls and doc
+    if mime_type == "application/msword" and extension == ".xls":
+        return FileType.XLS
+
     elif mime_type.endswith("xml"):
         if extension and (extension == ".html" or extension == ".htm"):
             return FileType.HTML
diff --git a/unstructured/partition/auto.py b/unstructured/partition/auto.py
@@ -197,7 +197,7 @@ def partition(
         )
     elif filetype == FileType.JSON:
         elements = partition_json(filename=filename, file=file)
-    elif filetype == FileType.XLSX:
+    elif (filetype == FileType.XLSX) or (filetype == FileType.XLS):
         elements = partition_xlsx(filename=filename, file=file)
     elif filetype == FileType.CSV:
         elements = partition_csv(filename=filename, file=file)
