build(release): bump unstructured (#183)

Related to downstream issue: #182
And upstream PR: Unstructured-IO/unstructured-inference#165

* remove test_parallel_mode_correct_result
* dropped the file_directory field from elements metadata

Author: shreyanid

CHANGELOG.md

@@ -1,5 +1,6 @@
-## 0.0.35-dev0
+## 0.0.35
 
+* Bump unstructured library to 0.9.2
 * Fix a misleading error in make docker-test
 
 ## 0.0.34

pipeline-notebooks/pipeline-general.ipynb

@@ -888,6 +888,9 @@
     "        \n",
     "        if element.metadata.last_modified:\n",
     "            elements[i].metadata.last_modified = None\n",
+    "\n",
+    "        if element.metadata.file_directory:\n",
+    "            elements[i].metadata.file_directory = None\n",
     "        \n",
     "    if response_type == \"text/csv\":\n",
     "        df = convert_to_dataframe(elements)\n",

prepline_general/api/general.py

@@ -367,6 +367,9 @@ def pipeline_api(
         if element.metadata.last_modified:
             elements[i].metadata.last_modified = None
 
+        if element.metadata.file_directory:
+            elements[i].metadata.file_directory = None
+
     if response_type == "text/csv":
         df = convert_to_dataframe(elements)
         return df.to_csv(index=False)

requirements/base.txt

@@ -17,7 +17,9 @@ attrs==23.1.0
 autoflake==2.2.0
     # via unstructured-api-tools
 beautifulsoup4==4.12.2
-    # via nbconvert
+    # via
+    #   nbconvert
+    #   unstructured
 bleach==6.0.0
     # via nbconvert
 certifi==2023.7.22
@@ -40,8 +42,10 @@ coloredlogs==15.0.1
     # via onnxruntime
 contourpy==1.1.0
     # via matplotlib
-cryptography==41.0.3
-    # via pdfminer-six
+cryptography==41.0.2
+    # via
+    #   pdfminer-six
+    #   unstructured
 cycler==0.11.0
     # via matplotlib
 defusedxml==0.7.1
@@ -101,7 +105,7 @@ jinja2==3.1.2
     #   nbconvert
     #   torch
     #   unstructured-api-tools
-joblib==1.3.1
+joblib==1.3.2
     # via nltk
 jsonschema==4.19.0
     # via nbformat
@@ -140,7 +144,7 @@ mpmath==1.3.0
     # via sympy
 msg-parser==1.2.0
     # via unstructured
-mypy==1.4.1
+mypy==1.5.0
     # via unstructured-api-tools
 mypy-extensions==1.0.0
     # via mypy
@@ -174,7 +178,7 @@ omegaconf==2.3.0
     # via effdet
 onnxruntime==1.15.1
     # via unstructured-inference
-opencv-python==4.8.0.74
+opencv-python==4.8.0.76
     # via
     #   layoutparser
     #   unstructured-inference
@@ -221,7 +225,7 @@ platformdirs==3.10.0
     # via jupyter-core
 portalocker==2.7.0
     # via iopath
-protobuf==4.23.4
+protobuf==4.24.0
     # via onnxruntime
 pycocotools==2.0.6
     # via effdet
@@ -274,15 +278,15 @@ pyyaml==6.0.1
     #   timm
     #   transformers
     #   uvicorn
-pyzmq==25.1.0
+pyzmq==25.1.1
     # via jupyter-client
 ratelimit==2.2.1
     # via -r requirements/base.in
 referencing==0.30.2
     # via
     #   jsonschema
     #   jsonschema-specifications
-regex==2023.6.3
+regex==2023.8.8
     # via
     #   nltk
     #   transformers
@@ -297,7 +301,7 @@ rpds-py==0.9.2
     # via
     #   jsonschema
     #   referencing
-safetensors==0.3.1
+safetensors==0.3.2
     # via
     #   timm
     #   transformers
@@ -340,9 +344,9 @@ torchvision==0.15.2
     #   effdet
     #   layoutparser
     #   timm
-tornado==6.3.2
+tornado==6.3.3
     # via jupyter-client
-tqdm==4.65.0
+tqdm==4.66.1
     # via
     #   huggingface-hub
     #   iopath
@@ -365,24 +369,22 @@ types-urllib3==1.26.25.14
     # via types-requests
 typing-extensions==4.7.1
     # via
-    #   annotated-types
     #   fastapi
     #   huggingface-hub
     #   iopath
     #   mypy
     #   pydantic
-    #   pydantic-core
     #   pypdf
     #   starlette
     #   torch
     #   uvicorn
 tzdata==2023.3
     # via pandas
-unstructured[local-inference]==0.9.0
+unstructured[local-inference]==0.9.2
     # via -r requirements/base.in
 unstructured-api-tools==0.10.10
     # via -r requirements/base.in
-unstructured-inference==0.5.7
+unstructured-inference==0.5.9
     # via unstructured
 urllib3==2.0.4
     # via requests

requirements/test.txt

@@ -50,6 +50,7 @@ beautifulsoup4==4.12.2
     # via
     #   -r requirements/base.txt
     #   nbconvert
+    #   unstructured
 black==23.7.0
     # via -r requirements/test.in
 bleach==6.0.0
@@ -98,15 +99,16 @@ contourpy==1.1.0
     #   matplotlib
 coverage[toml]==7.2.7
     # via pytest-cov
-cryptography==41.0.3
+cryptography==41.0.2
     # via
     #   -r requirements/base.txt
     #   pdfminer-six
+    #   unstructured
 cycler==0.11.0
     # via
     #   -r requirements/base.txt
     #   matplotlib
-debugpy==1.6.7
+debugpy==1.6.7.post1
     # via ipykernel
 decorator==5.1.1
     # via ipython
@@ -254,7 +256,7 @@ jinja2==3.1.2
     #   nbconvert
     #   torch
     #   unstructured-api-tools
-joblib==1.3.1
+joblib==1.3.2
     # via
     #   -r requirements/base.txt
     #   nltk
@@ -366,7 +368,7 @@ msg-parser==1.2.0
     # via
     #   -r requirements/base.txt
     #   unstructured
-mypy==1.4.1
+mypy==1.5.0
     # via
     #   -r requirements/base.txt
     #   -r requirements/test.in
@@ -435,7 +437,7 @@ onnxruntime==1.15.1
     # via
     #   -r requirements/base.txt
     #   unstructured-inference
-opencv-python==4.8.0.74
+opencv-python==4.8.0.76
     # via
     #   -r requirements/base.txt
     #   layoutparser
@@ -529,7 +531,7 @@ prompt-toolkit==3.0.39
     # via
     #   ipython
     #   jupyter-console
-protobuf==4.23.4
+protobuf==4.24.0
     # via
     #   -r requirements/base.txt
     #   onnxruntime
@@ -598,7 +600,6 @@ pytest-mock==3.11.1
 python-dateutil==2.8.2
     # via
     #   -r requirements/base.txt
-    #   arrow
     #   jupyter-client
     #   matplotlib
     #   pandas
@@ -641,7 +642,7 @@ pyyaml==6.0.1
     #   timm
     #   transformers
     #   uvicorn
-pyzmq==25.1.0
+pyzmq==25.1.1
     # via
     #   -r requirements/base.txt
     #   ipykernel
@@ -661,7 +662,7 @@ referencing==0.30.2
     #   jsonschema
     #   jsonschema-specifications
     #   jupyter-events
-regex==2023.6.3
+regex==2023.8.8
     # via
     #   -r requirements/base.txt
     #   nltk
@@ -687,7 +688,7 @@ rpds-py==0.9.2
     #   -r requirements/base.txt
     #   jsonschema
     #   referencing
-safetensors==0.3.1
+safetensors==0.3.2
     # via
     #   -r requirements/base.txt
     #   timm
@@ -769,7 +770,7 @@ torchvision==0.15.2
     #   effdet
     #   layoutparser
     #   timm
-tornado==6.3.2
+tornado==6.3.3
     # via
     #   -r requirements/base.txt
     #   ipykernel
@@ -778,7 +779,7 @@ tornado==6.3.2
     #   jupyterlab
     #   notebook
     #   terminado
-tqdm==4.65.0
+tqdm==4.66.1
     # via
     #   -r requirements/base.txt
     #   huggingface-hub
@@ -838,11 +839,11 @@ tzdata==2023.3
     # via
     #   -r requirements/base.txt
     #   pandas
-unstructured[local-inference]==0.9.0
+unstructured[local-inference]==0.9.2
     # via -r requirements/base.txt
 unstructured-api-tools==0.10.10
     # via -r requirements/base.txt
-unstructured-inference==0.5.7
+unstructured-inference==0.5.9
     # via
     #   -r requirements/base.txt
     #   unstructured

test_general/api/test_app.py

@@ -1,6 +1,5 @@
 from pathlib import Path
 
-import json
 import io
 import pytest
 import re
@@ -10,8 +9,6 @@
 from unstructured_api_tools.pipelines.api_conventions import get_pipeline_path
 
 from prepline_general.api.app import app
-from unstructured.partition.auto import partition
-from unstructured.staging.base import convert_to_isd
 import tempfile
 
 MAIN_API_ROUTE = get_pipeline_path("general")
@@ -400,61 +397,6 @@ def json(self):
         return self.body
 
 
-def mock_partition_file_via_api(url, **kwargs):
-    file = kwargs["files"]["files"][1]
-
-    partition_kwargs = kwargs["data"]
-
-    # Hack - the api takes `coordinates` but regular partition does not
-    del partition_kwargs["coordinates"]
-
-    elements = partition(file=file, **partition_kwargs)
-
-    response = MockResponse(200)
-    response.body = elements
-    response.text = json.dumps(convert_to_isd(elements))
-    return response
-
-
-def test_parallel_mode_correct_result(monkeypatch):
-    """
-    Validate that parallel processing mode merges the results
-    to look the same as normal mode. The api call is mocked to
-    use local partition, so this is just testing the merge logic.
-    """
-    client = TestClient(app)
-    test_file = Path("sample-docs") / "layout-parser-paper.pdf"
-
-    response = client.post(
-        MAIN_API_ROUTE,
-        files=[("files", (str(test_file), open(test_file, "rb"), "application/pdf"))],
-    )
-
-    assert response.status_code == 200
-    result_serial = response.json()
-
-    monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_ENABLED", "true")
-    monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_URL", "unused")
-    # Replace our callout with regular old partition
-    monkeypatch.setattr(
-        requests,
-        "post",
-        lambda *args, **kwargs: mock_partition_file_via_api(*args, **kwargs),
-    )
-
-    response = client.post(
-        MAIN_API_ROUTE,
-        files=[("files", (str(test_file), open(test_file, "rb"), "application/pdf"))],
-    )
-
-    assert response.status_code == 200
-    result_parallel = response.json()
-
-    for pair in zip(result_serial, result_parallel):
-        print(json.dumps(pair, indent=2))
-        assert pair[0] == pair[1]
-
-
 def test_parallel_mode_returns_errors(monkeypatch):
     """
     If we get an error sending a page to the api, bubble it up