Merge branch 'main' into pprados/fix_password

Author: pprados

.github/workflows/ingest-test-fixtures-update-pr.yml

@@ -109,6 +109,7 @@ jobs:
           sudo apt-get install -y tesseract-ocr-kor
           sudo apt-get install diffstat
           tesseract --version
+          python -m nltk.downloader punkt_tab averaged_perceptron_tagger_eng
           ./test_unstructured_ingest/test-ingest-src.sh
 
       - name: Save branch name to environment file

CHANGELOG.md

@@ -1,3 +1,54 @@
+## 0.16.19-dev2
+
+### Enhancements
+
+### Features
+
+### Fixes
+- **fix a bug where table extraction is skipped when it shouldn't**. Pages with just one table as its content or starts with a table misses table extraction. The routing logic is now fixed.
+- **Correct deprecated `ruff` invocation in `make tidy`**.  This will future-proof it or avoid surprises if someone happens to upgrade Ruff.
+- **Remove upper bound constraint on python version** in setup.py. Python3.13 is not yet officially supported, but allow users to try.
+
+## 0.16.17
+
+### Enhancements
+- **Refactoring the VoyageAI integration** to use voyageai package directly, allowing extra features.
+
+### Features
+
+### Fixes
+- **Fix a bug where `build_layout_elements_from_cor_regions` incorrectly joins texts in wrong order**.
+
+## 0.16.16
+
+### Enhancements
+
+### Features
+- **Vectorize layout (inferred, extracted, and OCR) data structure** Using `np.ndarray` to store a group of layout elements or text regions instead of using a list of objects. This improves the memory efficiency and compute speed around layout merging and deduplication.
+
+### Fixes
+- **Add auto-download for NLTK for Python Enviroment** When user import tokenize, It will  automatic download nltk data from `tokenize.py` file. Added `AUTO_DOWNLOAD_NLTK` flag in `tokenize.py` to download `NLTK_DATA`.
+- **Correctly patch pdfminer to avoid PDF repair**. The patch applied to pdfminer's parser caused it to occasionally split tokens in content streams, throwing `PDFSyntaxError`.  Repairing these PDFs sometimes failed (since they were not actually invalid) resulting in unnecessary OCR fallback.
+- **Drop usage of ndjson dependency**
+
+## 0.16.15
+### Enhancements
+
+### Features
+
+### Fixes
+- **Update `unstructured-inference`** to 0.8.6 in requirements which removed `layoutparser` dependency libs
+- **Update `pdfminer-six` to 20240706**
+
+## 0.16.14
+
+### Enhancements
+
+### Features
+
+### Fixes
+- **Fix an issue with multiple values for `infer_table_structure`** when paritioning email with image attachements the kwarg calls into `partition` to partition the image already contains `infer_table_structure`. Now `partition` function checks if the `kwarg` has `infer_table_structure` already
+
 ## 0.16.13
 
 ### Enhancements

Dockerfile

@@ -1,7 +1,7 @@
 FROM quay.io/unstructured-io/base-images:wolfi-base-latest AS base
 
 ARG PYTHON=python3.11
-ARG PIP=pip3.11
+ARG PIP="${PYTHON} -m pip"
 
 USER root
 
@@ -19,6 +19,9 @@ RUN chown -R notebook-user:notebook-user /app && \
 
 USER notebook-user
 
+# append PATH before pip install to avoid warning logs; it also avoids issues with packages that needs compilation during installation
+ENV PATH="${PATH}:/home/notebook-user/.local/bin"
+ENV TESSDATA_PREFIX=/usr/local/share/tessdata
 ENV NLTK_DATA=/home/notebook-user/nltk_data
 
 # Install Python dependencies and download required NLTK packages
@@ -28,7 +31,4 @@ RUN find requirements/ -type f -name "*.txt" -exec $PIP install --no-cache-dir -
     $PYTHON -c "from unstructured.partition.model_init import initialize; initialize()" && \
     $PYTHON -c "from unstructured_inference.models.tables import UnstructuredTableTransformerModel; model = UnstructuredTableTransformerModel(); model.initialize('microsoft/table-transformer-structure-recognition')"
 
-ENV PATH="${PATH}:/home/notebook-user/.local/bin"
-ENV TESSDATA_PREFIX=/usr/local/share/tessdata
-
 CMD ["/bin/bash"]

Makefile

@@ -254,7 +254,7 @@ tidy-shell:
 
 .PHONY: tidy-python
 tidy-python:
-	ruff . --fix-only || true
+	ruff check . --fix-only || true
 	autoflake --in-place .
 	black --line-length=100 .
 
@@ -308,7 +308,7 @@ docker-test:
 	$(DOCKER_IMAGE) \
 	bash -c "CI=$(CI) \
 	UNSTRUCTURED_INCLUDE_DEBUG_METADATA=$(UNSTRUCTURED_INCLUDE_DEBUG_METADATA) \
-	pytest $(if $(TEST_FILE),$(TEST_FILE),test_unstructured)"
+	python3 -m pytest $(if $(TEST_FILE),$(TEST_FILE),test_unstructured)"
 
 .PHONY: docker-smoke-test
 docker-smoke-test:

example-docs/pdf/single_table.pdf

@@ -1,5 +1,24 @@
 [tool.black]
 line-length = 100
+exclude = '''
+/(
+    \.venv  # Ignore virtual environment directory
+    | \.git
+    | \.hg
+    | \.mypy_cache
+    | \.tox
+    | \.nox
+    | \.env
+    | \.pytest_cache
+    | \.venv
+    | _build
+    | buck-out
+    | build
+    | dist
+    | unstructured/nlp/patterns\.py
+)/|
+unstructured/nlp/patterns\.py
+'''
 
 [tool.pyright]
 pythonPlatform = "Linux"

pyproject.toml

@@ -22,4 +22,3 @@ tqdm
 psutil
 python-oxmsg
 html5lib
-ndjson

requirements/base.in

@@ -4,7 +4,7 @@
 #
 #    pip-compile ./base.in
 #
-anyio==4.7.0
+anyio==4.8.0
     # via httpx
 backoff==2.2.1
     # via -r ./base.in
@@ -36,7 +36,7 @@ dataclasses-json==0.6.7
     #   unstructured-client
 deepdiff==8.1.1
     # via unstructured-client
-emoji==2.14.0
+emoji==2.14.1
     # via -r ./base.in
 exceptiongroup==1.2.2
     # via anyio
@@ -64,16 +64,14 @@ langdetect==1.0.9
     # via -r ./base.in
 lxml==5.3.0
     # via -r ./base.in
-marshmallow==3.23.2
+marshmallow==3.26.0
     # via
     #   dataclasses-json
     #   unstructured-client
 mypy-extensions==1.0.0
     # via
     #   typing-inspect
     #   unstructured-client
-ndjson==0.3.1
-    # via -r ./base.in
 nest-asyncio==1.6.0
     # via unstructured-client
 nltk==3.9.1
@@ -92,11 +90,11 @@ psutil==6.1.1
     # via -r ./base.in
 pycparser==2.22
     # via cffi
-pypdf==5.1.0
+pypdf==5.2.0
     # via unstructured-client
 python-dateutil==2.9.0.post0
     # via unstructured-client
-python-iso639==2024.10.22
+python-iso639==2025.1.28
     # via -r ./base.in
 python-magic==0.4.27
     # via -r ./base.in
@@ -150,5 +148,5 @@ urllib3==1.26.20
     #   unstructured-client
 webencodings==0.5.1
     # via html5lib
-wrapt==1.17.0
+wrapt==1.17.2
     # via -r ./base.in

requirements/base.txt

@@ -3,8 +3,8 @@
 # extras. Putting a dependency here will only affect dependency sets that contain them -- in other
 # words, if something does not require a constraint, it will not be installed.
 ####################################################################################################
-# (jennings): Versions greater than 5.0 create dependency conflicts with other packages
-protobuf<5.0
+# we are using v3 client https://weaviate.io/developers/weaviate/client-libraries/python/python_v3
+weaviate-client>=3.26.7,<4.0.0
 # TODO: Constriant due to multiple versions being installed during pip-compile
 grpcio>=1.65.5
 # TODO: Pinned in transformers package, remove when that gets updated (https://github.com/huggingface/transformers/blob/main/setup.py)

requirements/deps/constraints.txt

@@ -15,11 +15,11 @@ click==8.1.8
     #   pip-tools
 distlib==0.3.9
     # via virtualenv
-filelock==3.16.1
+filelock==3.17.0
     # via virtualenv
-identify==2.6.4
+identify==2.6.6
     # via pre-commit
-importlib-metadata==8.5.0
+importlib-metadata==8.6.1
     # via
     #   -c ././deps/constraints.txt
     #   build
@@ -36,7 +36,7 @@ platformdirs==4.3.6
     # via
     #   -c ./test.txt
     #   virtualenv
-pre-commit==4.0.1
+pre-commit==4.1.0
     # via -r ./dev.in
 pyproject-hooks==1.2.0
     # via
@@ -51,7 +51,7 @@ tomli==2.2.1
     #   -c ./test.txt
     #   build
     #   pip-tools
-virtualenv==20.28.1
+virtualenv==20.29.1
     # via pre-commit
 wheel==0.45.1
     # via pip-tools