feat/remove unstructured as a required dependency to run ingest (#23)

* Refactor to support unstructured as an optional (extra) dependency

* Still generate base.txt for CI

* Downgrade version of unstructured back to 0.15.1

* Fix example script paths

* Make python version a string

* Add explicit dependency for click when using CLI

* Add cli dep to unit tests

* Add dataclasses_json to base deps

* Remove unstructured as a required dependency from v1

* Add additional dependency

* Update dependencies

* Fix setup.py file

* Isolate backoff dep

* Make requests a connector-specific dep

* Make httpx a connector-specific dep

* Make httpx a connector-specific dep

* Add tqdm to base deps

* Fix assign_and_map_hash_ids logic

* tidy

* bugfix assign_and_map_hash_ids

* Add numpy constraint <2

* Create DataSourceMetadata extension to pass for data_source_metadata

* Apply assign_and_map_hash_ids after chunking

* Add click to base deps

* Fix assign_and_map_hash_ids and add unit test arround it

* Remove reference to olf install-cli make target

* remove reference to old cli extra

* Add some more requires_dependencies annoations

* Fix embedder bug

* unit test both chunking strategies

Author: rbiseck3

.github/workflows/e2e.yml

@@ -107,6 +107,30 @@ jobs:
         docker compose version
         ./test_e2e/test-src.sh
 
+  test_src_api:
+    runs-on: ubuntu-latest-m
+    needs: [ setup ]
+    steps:
+    # actions/checkout MUST come before auth
+    - uses: 'actions/checkout@v4'
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.10"
+    - name: Get full Python version
+      id: full-python-version
+      run: echo version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") >> $GITHUB_OUTPUT
+    - name: Install limited dependencies
+      run: |
+        make install-client
+        make install-base
+    - name: Run test against remote API
+      env:
+        UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
+      run: |
+        ./test_e2e/src/against-api.sh
+    
+
 
   test_dest:
     environment: ci

.github/workflows/unit_tests.yml

@@ -94,9 +94,11 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Validate --help
+      - name: Install local deps
         run: |
           pip install .
+      - name: Validate --help
+        run: |
           unstructured-ingest --help
 
   test_ingest_unit:

.gitignore

@@ -205,3 +205,5 @@ metricsdiff.txt
 
 # analysis
 annotated/
+
+/tmp_ingest

CHANGELOG.md

@@ -1,10 +1,11 @@
-## 0.0.4-dev2
+## 0.0.4-dev3
 
 ### Enhancements
 
 * **Add Couchbase Destination Connector** Adds support for storing artifacts in Couchbase DB for Vector Search
 * **Leverage pydantic base models** All user-supplied configs are now derived from pydantic base models to leverage better type checking and add built in support for sensitive fields.
 * **Autogenerate click options from base models** Leverage th pydantic base models for all configs to autogenerate teh cli options exposed when running ingest as a CLI.
+* **Drop required Unstructured dependency** Unstructured was moved to an extra dependency to only be imported when needed for functionality such as local partitioning/chunking.
 
 ## 0.0.3
 

Makefile

@@ -14,6 +14,11 @@ pip-compile:
 install-lint:
 	pip install -r requirements/lint.txt
 
+
+.PHONY: install-client
+install-client:
+	pip install -r requirements/remote/client.txt
+
 .PHONY: install-test
 install-test:
 	pip install -r requirements/test.txt

requirements/common/base.in

@@ -1,6 +1,8 @@
 -c constraints.txt
 
 python-dateutil
-unstructured
 pandas
 pydantic
+dataclasses_json
+tqdm
+click

requirements/common/base.txt

@@ -2,166 +2,48 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile ./base.in
+#    pip-compile ./requirements/common/base.in
 #
 annotated-types==0.7.0
     # via pydantic
-anyio==3.7.1
-    # via
-    #   -c ./constraints.txt
-    #   httpx
-backoff==2.2.1
-    # via unstructured
-beautifulsoup4==4.12.3
-    # via unstructured
-certifi==2024.7.4
-    # via
-    #   -c ./constraints.txt
-    #   httpcore
-    #   httpx
-    #   requests
-    #   unstructured-client
-chardet==5.2.0
-    # via unstructured
-charset-normalizer==3.3.2
-    # via
-    #   requests
-    #   unstructured-client
 click==8.1.7
-    # via nltk
+    # via -r ./requirements/common/base.in
 dataclasses-json==0.6.7
-    # via
-    #   unstructured
-    #   unstructured-client
-deepdiff==7.0.1
-    # via unstructured-client
-emoji==2.12.1
-    # via unstructured
-exceptiongroup==1.2.2
-    # via anyio
-filetype==1.2.0
-    # via unstructured
-h11==0.14.0
-    # via httpcore
-httpcore==1.0.5
-    # via httpx
-httpx==0.27.0
-    # via unstructured-client
-idna==3.7
-    # via
-    #   anyio
-    #   httpx
-    #   requests
-    #   unstructured-client
-joblib==1.4.2
-    # via nltk
-jsonpath-python==1.0.6
-    # via unstructured-client
-langdetect==1.0.9
-    # via unstructured
-lxml==5.3.0
-    # via unstructured
+    # via -r ./requirements/common/base.in
 marshmallow==3.21.3
-    # via
-    #   dataclasses-json
-    #   unstructured-client
+    # via dataclasses-json
 mypy-extensions==1.0.0
-    # via
-    #   typing-inspect
-    #   unstructured-client
-nest-asyncio==1.6.0
-    # via unstructured-client
-nltk==3.8.1
-    # via
-    #   -c ./constraints.txt
-    #   unstructured
+    # via typing-inspect
 numpy==1.26.4
     # via
+    #   -c ./requirements/common/constraints.txt
     #   pandas
-    #   unstructured
-ordered-set==4.1.0
-    # via deepdiff
 packaging==23.2
     # via
-    #   -c ./constraints.txt
+    #   -c ./requirements/common/constraints.txt
     #   marshmallow
-    #   unstructured-client
 pandas==2.2.2
-    # via -r ./base.in
-psutil==6.0.0
-    # via unstructured
+    # via -r ./requirements/common/base.in
 pydantic==2.8.2
-    # via -r ./base.in
+    # via -r ./requirements/common/base.in
 pydantic-core==2.20.1
     # via pydantic
-pypdf==4.3.1
-    # via unstructured-client
 python-dateutil==2.9.0.post0
     # via
-    #   -r ./base.in
+    #   -r ./requirements/common/base.in
     #   pandas
-    #   unstructured-client
-python-iso639==2024.4.27
-    # via unstructured
-python-magic==0.4.27
-    # via unstructured
 pytz==2024.1
     # via pandas
-rapidfuzz==3.9.6
-    # via unstructured
-regex==2024.7.24
-    # via nltk
-requests==2.32.3
-    # via
-    #   requests-toolbelt
-    #   unstructured
-    #   unstructured-client
-requests-toolbelt==1.0.0
-    # via unstructured-client
 six==1.16.0
-    # via
-    #   langdetect
-    #   python-dateutil
-    #   unstructured-client
-sniffio==1.3.1
-    # via
-    #   anyio
-    #   httpx
-soupsieve==2.5
-    # via beautifulsoup4
-tabulate==0.9.0
-    # via unstructured
+    # via python-dateutil
 tqdm==4.66.5
-    # via
-    #   nltk
-    #   unstructured
+    # via -r ./requirements/common/base.in
 typing-extensions==4.12.2
     # via
-    #   emoji
     #   pydantic
     #   pydantic-core
-    #   pypdf
     #   typing-inspect
-    #   unstructured
-    #   unstructured-client
 typing-inspect==0.9.0
-    # via
-    #   dataclasses-json
-    #   unstructured-client
+    # via dataclasses-json
 tzdata==2024.1
     # via pandas
-unstructured==0.15.1
-    # via -r ./base.in
-unstructured-client==0.25.4
-    # via
-    #   -c ./constraints.txt
-    #   unstructured
-urllib3==1.26.19
-    # via
-    #   -c ./constraints.txt
-    #   requests
-    #   unstructured-client
-wrapt==1.16.0
-    # via
-    #   -c ./constraints.txt
-    #   unstructured

requirements/common/constraints.txt

@@ -63,3 +63,6 @@ langchain-community>=0.2.5
 importlib-metadata==7.1.0
 
 nltk==3.8.1
+
+unstructured==0.15.1
+numpy<2

requirements/connectors/airtable.in

@@ -1,4 +1,3 @@
--c ../common/constraints.txt
--c ../common/base.txt
+-r ../common/base.in
 
 pyairtable

requirements/connectors/airtable.txt

@@ -2,50 +2,70 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile ./connectors/airtable.in
+#    pip-compile ./requirements/connectors/airtable.in
 #
 annotated-types==0.7.0
-    # via
-    #   -c ./connectors/../common/base.txt
-    #   pydantic
+    # via pydantic
 certifi==2024.7.4
     # via
-    #   -c ./connectors/../common/base.txt
-    #   -c ./connectors/../common/constraints.txt
+    #   -c ./requirements/connectors/../common/constraints.txt
     #   requests
 charset-normalizer==3.3.2
-    # via
-    #   -c ./connectors/../common/base.txt
-    #   requests
+    # via requests
+click==8.1.7
+    # via -r ./requirements/connectors/../common/base.in
+dataclasses-json==0.6.7
+    # via -r ./requirements/connectors/../common/base.in
 idna==3.7
-    # via
-    #   -c ./connectors/../common/base.txt
-    #   requests
+    # via requests
 inflection==0.5.1
     # via pyairtable
+marshmallow==3.21.3
+    # via dataclasses-json
+mypy-extensions==1.0.0
+    # via typing-inspect
+numpy==1.26.4
+    # via
+    #   -c ./requirements/connectors/../common/constraints.txt
+    #   pandas
+packaging==23.2
+    # via
+    #   -c ./requirements/connectors/../common/constraints.txt
+    #   marshmallow
+pandas==2.2.2
+    # via -r ./requirements/connectors/../common/base.in
 pyairtable==2.3.3
-    # via -r ./connectors/airtable.in
+    # via -r ./requirements/connectors/airtable.in
 pydantic==2.8.2
     # via
-    #   -c ./connectors/../common/base.txt
+    #   -r ./requirements/connectors/../common/base.in
     #   pyairtable
 pydantic-core==2.20.1
+    # via pydantic
+python-dateutil==2.9.0.post0
     # via
-    #   -c ./connectors/../common/base.txt
-    #   pydantic
+    #   -r ./requirements/connectors/../common/base.in
+    #   pandas
+pytz==2024.1
+    # via pandas
 requests==2.32.3
-    # via
-    #   -c ./connectors/../common/base.txt
-    #   pyairtable
+    # via pyairtable
+six==1.16.0
+    # via python-dateutil
+tqdm==4.66.5
+    # via -r ./requirements/connectors/../common/base.in
 typing-extensions==4.12.2
     # via
-    #   -c ./connectors/../common/base.txt
     #   pyairtable
     #   pydantic
     #   pydantic-core
+    #   typing-inspect
+typing-inspect==0.9.0
+    # via dataclasses-json
+tzdata==2024.1
+    # via pandas
 urllib3==1.26.19
     # via
-    #   -c ./connectors/../common/base.txt
-    #   -c ./connectors/../common/constraints.txt
+    #   -c ./requirements/connectors/../common/constraints.txt
     #   pyairtable
     #   requests