feat: add option to parallelize pdfs (#41)

This is implementation of #17

It adds a boolean `split_pdf_page` to PartitionParameters, which if
True, causes the pdf to be split at client side to 1-page chunks, and
send to API. The returned elements are joined to a single result list.

Author: hubert-rutkowski85

.github/workflows/ci.yaml

@@ -19,22 +19,22 @@ jobs:
         python-version: ["3.9","3.10","3.11"]
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: make install-test
     - name: Run unit tests
       run: |
         pip install  .
-        make test
+        make test-unit
 
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install dependencies
       run: pip install .[dev]
     - name: Lint

.github/workflows/speakeasy_sdk_generation.yml

@@ -25,6 +25,8 @@ jobs:
       openapi_doc_auth_header: x-api-key
       openapi_docs: |
         - https://raw.githubusercontent.com/Unstructured-IO/unstructured-api/main/openapi.json
+      overlay_docs: |
+        - ../../overlay_client.yaml
       publish_python: true
       speakeasy_version: latest
     secrets:

.gitignore

@@ -7,3 +7,6 @@ __pycache__/
 
 # human-added igore files
 .ipynb_checkpoints/
+.idea/
+openapi.json
+openapi_client.json

Makefile

@@ -1,15 +1,15 @@
 PACKAGE_NAME := unstructured-python-client
 CURRENT_DIR := $(shell pwd)
 ARCH := $(shell uname -m)
+DOCKER_IMAGE ?= downloads.unstructured.io/unstructured-io/unstructured-api:latest
 
 ###########
 # Install #
 ###########
 
 .PHONY: install-test
 install-test:
-	pip install pytest
-	pip install requests_mock
+	pip install pytest requests_mock pypdf deepdiff
 
 .PHONY: install-dev
 install-dev:
@@ -25,14 +25,41 @@ install: install-test install-dev
 #################
 
 .PHONY: test
-test:
-	PYTHONPATH=. pytest \
-		_test_unstructured_client
+test: test-unit test-integration-docker
+
+.PHONY: test-unit
+test-unit:
+	PYTHONPATH=. pytest _test_unstructured_client -v -k "unit"
+
+# Assumes you have unstructured-api running on localhost:8000
+.PHONY: test-integration
+test-integration:
+	PYTHONPATH=. pytest _test_unstructured_client -v -k "integration"
+
+# Runs the unstructured-api in docker for tests
+.PHONY: test-integration-docker
+test-integration-docker:
+	-docker stop unstructured-api && docker kill unstructured-api
+	docker run --name unstructured-api -p 8000:8000 -d --rm ${DOCKER_IMAGE} --host 0.0.0.0 && \
+	curl -s -o /dev/null --retry 10 --retry-delay 5 --retry-all-errors http://localhost:8000/general/docs && \
+	PYTHONPATH=. pytest _test_unstructured_client -v -k "integration" && \
+	docker kill unstructured-api
 
 .PHONY: lint
 lint:
 	pylint --rcfile=pylintrc src
 
+#############
+# Speakeasy #
+#############
+
+.PHONY: client-generate
+client-generate:
+	wget -nv -q -O openapi.json https://raw.githubusercontent.com/Unstructured-IO/unstructured-api/main/openapi.json
+	speakeasy overlay validate -o ./overlay_client.yaml
+	speakeasy overlay apply -s ./openapi.json -o ./overlay_client.yaml > ./openapi_client.json
+	speakeasy generate sdk -s ./openapi_client.json -o ./ -l python
+
 ###########
 # Jupyter #
 ###########

README.md

@@ -23,10 +23,8 @@ pip install unstructured-client
 ```
 <!-- End SDK Installation [installation] -->
 
-/docs/models/shared/partitionparameters.md
-
 ## Usage
-Only the `files` parameter is required. See the [general partition](/docs/models/shared/partitionparameters.md) page for all available parameters. 
+Only the `files` parameter is required. 
 
 ```python
 from unstructured_client import UnstructuredClient
@@ -35,7 +33,7 @@ from unstructured_client.models.errors import SDKError
 
 s = UnstructuredClient(api_key_auth="YOUR_API_KEY")
 
-filename = "sample-docs/layout-parser-paper-fast.pdf"
+filename = "_sample_docs/layout-parser-paper-fast.pdf"
 
 with open(filename, "rb") as f:
     # Note that this currently only supports a single file
@@ -56,20 +54,26 @@ try:
     print(resp.elements[0])
 except SDKError as e:
     print(e)
-
-# {
-# 'type': 'UncategorizedText', 
-# 'element_id': 'fc550084fda1e008e07a0356894f5816', 
-# 'metadata': {
-#   'filename': 'layout-parser-paper-fast.pdf', 
-#   'filetype': 'application/pdf', 
-#   'languages': ['eng'], 
-#   'page_number': 1
-#   }
-# }
+```
+    
+Result:
+
+```json
+{
+'type': 'UncategorizedText', 
+'element_id': 'fc550084fda1e008e07a0356894f5816', 
+'metadata': {
+  'filename': 'layout-parser-paper-fast.pdf', 
+  'filetype': 'application/pdf', 
+  'languages': ['eng'], 
+  'page_number': 1
+  }
+}
 ```
 
-## Change the base URL
+### UnstructuredClient
+
+#### Change the base URL
 
 If you are self hosting the API, or developing locally, you can change the server URL when setting up the client.
 
@@ -86,6 +90,24 @@ s = unstructured_client.UnstructuredClient(
     api_key_auth=api_key,
 )
 ```
+
+### PartitionParameters
+
+See the [general partition](/docs/models/shared/partitionparameters.md) page for all available parameters. 
+
+#### Splitting PDF by pages
+
+In order to speed up processing of long PDF files, set `split_pdf_page=True`. It will cause the PDF
+to be split page-by-page at client side, before sending to API, and combining individual responses
+as single result. This will work only for PDF files, so don't set it for other filetypes.
+
+Warning: this feature causes the `parent_id` metadata generation in elements to be disabled, as that
+requires having context of multiple pages.
+
+The amount of threads that will be used for sending individual pdf pages, is controlled by
+`UNSTRUCTURED_CLIENT_SPLIT_CALL_THREADS` env var. By default it equals to 5. 
+It can't be more than 15, to avoid too high resource usage and costs.
+
 <!-- No SDK Example Usage -->
 <!-- No SDK Available Operations -->
 <!-- No Pagination -->
@@ -119,9 +141,38 @@ This SDK is in beta, and there may be breaking changes between versions without
 to a specific package version. This way, you can install the same version each time without breaking changes unless you are intentionally
 looking for the latest version.
 
+### Installation Instructions for Local Development
+
+The following instructions are intended to help you get up and running with `unstructured-python-client` locally if you are planning to contribute to the project.
+
+* Using `pyenv` to manage virtualenv's is recommended but not necessary
+   * Mac install instructions. See [here](https://github.com/Unstructured-IO/community#mac--homebrew) for more detailed instructions.
+      * `brew install pyenv-virtualenv`
+      * `pyenv install 3.10`
+   * Linux instructions are available [here](https://github.com/Unstructured-IO/community#linux).
+
+* Create a virtualenv to work in and activate it, e.g. for one named `unstructured-python-client`:
+
+  `pyenv  virtualenv 3.10 unstructured-python-client`
+  `pyenv activate unstructured-python-client`
+
+* Run `make install` and `make test`
+
 ### Contributions
 
-While we value open-source contributions to this SDK, this library is generated programmatically.
+While we value open-source contributions to this SDK, this library is generated programmatically by Speakeasy. In order to start working with this repo, you need to:
+1. Install Speakeasy client locally https://github.com/speakeasy-api/speakeasy#installation
+2. Run `speakeasy auth login`
+3. Run `make client-generate`. This allows to iterate development with python client.
+
+There are two important files used by `make client-generate`:
+1. `openapi.json` which is actually not stored here, [but fetched from unstructured-api](https://raw.githubusercontent.com/Unstructured-IO/unstructured-api/main/openapi.json), represents the API that is supported on backend.
+2. `overlay_client.yaml` is a handcrafted diff that when applied over above, produces `openapi_client.json` 
+   which is used to generate SDK.
+
+Once PR with changes is merged, Github CI will autogenerate the Speakeasy client in a new PR, using
+the `openapi.json` and `overlay_client.yaml` You will have to manually bring back the human created lines in it.
+
 Feel free to open a PR or a Github issue as a proof of concept and we'll do our best to include it in a future release!
 
 ### SDK Created by [Speakeasy](https://www.speakeasyapi.dev/docs/sdk-design/python/methodology-python)

_sample_docs/fake.doc

@@ -1,9 +1,12 @@
+import os
+import pypdf
 import pytest
+import requests
+from deepdiff import DeepDiff
 
 from unstructured_client import UnstructuredClient
 from unstructured_client.models import shared
-from unstructured_client.models.errors import SDKError
-
+from unstructured_client.models.errors import SDKError, HTTPValidationError
 
 FAKE_KEY = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 
@@ -20,7 +23,7 @@
         "unstructured-000mock.api.unstructuredapp.io/general/v0/general",
     ],
 )
-def test_clean_server_url_fixes_malformed_paid_api_url(server_url: str):
+def test_unit_clean_server_url_fixes_malformed_paid_api_url(server_url: str):
     client = UnstructuredClient(
         server_url=server_url,
         api_key_auth=FAKE_KEY,
@@ -42,20 +45,20 @@ def test_clean_server_url_fixes_malformed_paid_api_url(server_url: str):
         "http://localhost:8000/general/v0/general",
     ],
 )
-def test_clean_server_url_fixes_malformed_localhost_url(server_url: str):
+def test_unit_clean_server_url_fixes_malformed_localhost_url(server_url: str):
     client = UnstructuredClient(
         server_url=server_url,
         api_key_auth=FAKE_KEY,
     )
     assert client.general.sdk_configuration.server_url == "http://localhost:8000"
 
 
-def test_clean_server_url_returns_empty_string_given_empty_string():
+def test_unit_clean_server_url_returns_empty_string_given_empty_string():
     client = UnstructuredClient( server_url="", api_key_auth=FAKE_KEY)
     assert client.general.sdk_configuration.server_url == ""
 
 
-def test_clean_server_url_returns_None_given_no_server_url():
+def test_unit_clean_server_url_returns_None_given_no_server_url():
     client = UnstructuredClient(
         api_key_auth=FAKE_KEY,
     )
@@ -71,7 +74,7 @@ def test_clean_server_url_returns_None_given_no_server_url():
         "unstructured-000mock.api.unstructuredapp.io/general/v0/general",
     ],
 )
-def test_clean_server_url_fixes_malformed_urls_with_positional_arguments(
+def test_unit_clean_server_url_fixes_malformed_urls_with_positional_arguments(
     server_url: str,
 ):
     client = UnstructuredClient(
@@ -85,7 +88,7 @@ def test_clean_server_url_fixes_malformed_urls_with_positional_arguments(
     )
 
 
-def test_suggest_defining_url_issues_a_warning_on_a_401():
+def test_unit_suggest_defining_url_issues_a_warning_on_a_401():
     client = UnstructuredClient(
         api_key_auth=FAKE_KEY,
     )
@@ -108,3 +111,78 @@ def test_suggest_defining_url_issues_a_warning_on_a_401():
             match="If intending to use the paid API, please define `server_url` in your request.",
         ):
             client.general.partition(req)
+
+
+@pytest.mark.parametrize("call_threads", [1, 2, 5])
+@pytest.mark.parametrize(
+    "filename, expected_ok",
+    [
+        ("_sample_docs/list-item-example-1.pdf", True),       # 1 page
+        ("_sample_docs/layout-parser-paper-fast.pdf", True),  # 2 pages
+        ("_sample_docs/layout-parser-paper.pdf", True),       # 16 pages
+        ("_sample_docs/fake.doc", True),
+        ("_sample_docs/fake.doc", False),  # This will append .pdf to filename to fool first line of filetype detection, to simulate decoding error
+    ],
+)
+def test_integration_split_pdf_has_same_output_as_non_split(
+    call_threads: int,
+    filename: str,
+    expected_ok: bool,
+    caplog
+):
+    """
+    Tests that output that we get from the split-by-page pdf is the same as from non-split.
+
+    Requires unstructured-api running in bg. See Makefile for how to run it.
+    Doesn't check for raw_response as there's no clear patter for how it changes with the number of pages / call_threads.
+    """
+    try:
+        response = requests.get("http://localhost:8000/general/docs")
+        assert response.status_code == 200, "The unstructured-api is not running on localhost:8000"
+    except requests.exceptions.ConnectionError:
+        assert False, "The unstructured-api is not running on localhost:8000"
+
+    client = UnstructuredClient(
+        api_key_auth=FAKE_KEY,
+        server_url="localhost:8000"
+    )
+
+    with open(filename, "rb") as f:
+        files = shared.Files(
+            content=f.read(),
+            file_name=filename,
+        )
+
+    if not expected_ok:
+        files.file_name += ".pdf"
+
+    req = shared.PartitionParameters(
+        files=files,
+        strategy='fast',
+        languages=["eng"],
+        split_pdf_page=True,
+    )
+
+    os.environ["UNSTRUCTURED_CLIENT_SPLIT_CALL_THREADS"] = str(call_threads)
+
+    try:
+        resp_split = client.general.partition(req)
+    except (HTTPValidationError, AttributeError) as exc:
+        if not expected_ok:
+            assert "error arose when splitting by pages" in caplog.text
+            assert "File does not appear to be a valid PDF" in str(exc)
+            return
+        else:
+            pytest.exit("unexpected error", returncode=1)
+
+    req.split_pdf_page = False
+    resp_single = client.general.partition(req)
+
+    assert len(resp_split.elements) == len(resp_single.elements)
+    assert resp_split.content_type == resp_single.content_type
+    assert resp_split.status_code == resp_single.status_code
+
+    # Difference in the parent_id is expected, because parent_ids are assigned when element crosses page boundary
+    diff = DeepDiff(t1=resp_split.elements, t2=resp_single.elements,
+                    exclude_regex_paths=r"root\[\d+\]\['metadata'\]\['parent_id'\]")
+    assert len(diff) == 0