build: add ruff and format (#124)

Author: qued

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.5.2-dev0
+
+* Add ruff to keep code consistent with unstructured
+
 ## 0.5.1
 
 * Add annotation for pages

Makefile

@@ -75,6 +75,7 @@ check: check-src check-tests check-version
 ## check-src:               runs linters (source only, no tests)
 .PHONY: check-src
 check-src:
+	ruff . --line-length 100 --select I,UP015,UP032,UP034,UP018,COM,C4,PT,SIM,PLR0402 --ignore PT011,PT012,SIM117
 	black --line-length 100 ${PACKAGE_NAME} --check
 	flake8 ${PACKAGE_NAME}
 	mypy ${PACKAGE_NAME} --ignore-missing-imports
@@ -101,6 +102,7 @@ check-version:
 ## tidy:                    run black
 .PHONY: tidy
 tidy:
+	ruff . --fix --line-length 100 --select I,UP015,UP032,UP034,UP018,COM,C4,PT,SIM,PLR0402 --ignore PT011,PT012,SIM117
 	black --line-length 100 ${PACKAGE_NAME}
 	black --line-length 100 test_${PACKAGE_NAME}
 

requirements/base.txt

@@ -18,24 +18,24 @@ coloredlogs==15.0.1
     # via onnxruntime
 contourpy==1.0.7
     # via matplotlib
-cryptography==40.0.2
+cryptography==41.0.1
     # via pdfminer-six
 cycler==0.11.0
     # via matplotlib
-effdet==0.3.0
+effdet==0.4.1
     # via layoutparser
 filelock==3.12.0
     # via
     #   huggingface-hub
     #   torch
     #   transformers
-flatbuffers==23.5.9
+flatbuffers==23.5.26
     # via onnxruntime
 fonttools==4.39.4
     # via matplotlib
 fsspec==2023.5.0
     # via huggingface-hub
-huggingface-hub==0.14.1
+huggingface-hub==0.15.1
     # via
     #   -r requirements/base.in
     #   timm
@@ -54,7 +54,7 @@ kiwisolver==1.4.4
     # via matplotlib
 layoutparser[layoutmodels,tesseract]==0.3.4
     # via -r requirements/base.in
-markupsafe==2.1.2
+markupsafe==2.1.3
     # via jinja2
 matplotlib==3.7.1
     # via pycocotools
@@ -76,7 +76,7 @@ numpy==1.24.3
     #   transformers
 omegaconf==2.3.0
     # via effdet
-onnxruntime==1.14.1
+onnxruntime==1.15.0
     # via -r requirements/base.in
 opencv-python==4.7.0.72
     # via
@@ -89,7 +89,7 @@ packaging==23.1
     #   onnxruntime
     #   pytesseract
     #   transformers
-pandas==2.0.1
+pandas==2.0.2
     # via layoutparser
 pdf2image==1.16.3
     # via layoutparser
@@ -107,7 +107,7 @@ pillow==9.5.0
     #   torchvision
 portalocker==2.7.0
     # via iopath
-protobuf==4.23.1
+protobuf==4.23.2
     # via onnxruntime
 pycocotools==2.0.6
     # via effdet
@@ -132,9 +132,9 @@ pyyaml==6.0
     #   omegaconf
     #   timm
     #   transformers
-regex==2023.5.5
+regex==2023.6.3
     # via transformers
-requests==2.30.0
+requests==2.31.0
     # via
     #   huggingface-hub
     #   torchvision
@@ -171,7 +171,7 @@ tqdm==4.65.0
     #   transformers
 transformers==4.29.2
     # via -r requirements/base.in
-typing-extensions==4.5.0
+typing-extensions==4.6.3
     # via
     #   huggingface-hub
     #   iopath

requirements/dev.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile requirements/dev.in
 #
-anyio==3.6.2
+anyio==3.7.0
     # via
     #   -c requirements/test.txt
     #   jupyter-server
@@ -49,9 +49,13 @@ defusedxml==0.7.1
     # via nbconvert
 entrypoints==0.4
     # via jupyter-client
+exceptiongroup==1.1.1
+    # via
+    #   -c requirements/test.txt
+    #   anyio
 executing==1.2.0
     # via stack-data
-fastjsonschema==2.16.3
+fastjsonschema==2.17.1
     # via nbformat
 idna==3.4
     # via
@@ -131,7 +135,7 @@ jupyterlab-pygments==0.2.2
     # via nbconvert
 jupyterlab-widgets==3.0.7
     # via ipywidgets
-markupsafe==2.1.2
+markupsafe==2.1.3
     # via
     #   -c requirements/base.txt
     #   jinja2
@@ -144,15 +148,15 @@ mistune==2.0.5
     # via nbconvert
 nbclassic==1.0.0
     # via notebook
-nbclient==0.7.4
+nbclient==0.8.0
     # via nbconvert
 nbconvert==7.4.0
     # via
     #   jupyter
     #   jupyter-server
     #   nbclassic
     #   notebook
-nbformat==5.8.0
+nbformat==5.9.0
     # via
     #   jupyter-server
     #   nbclassic
@@ -195,7 +199,7 @@ platformdirs==3.5.1
     # via
     #   -c requirements/test.txt
     #   jupyter-core
-prometheus-client==0.16.0
+prometheus-client==0.17.0
     # via
     #   jupyter-server
     #   nbclassic
@@ -230,7 +234,7 @@ python-dateutil==2.8.2
     # via
     #   -c requirements/base.txt
     #   jupyter-client
-pyzmq==25.0.2
+pyzmq==25.1.0
     # via
     #   ipykernel
     #   jupyter-client
@@ -300,7 +304,7 @@ traitlets==5.9.0
     #   nbformat
     #   notebook
     #   qtconsole
-typing-extensions==4.5.0
+typing-extensions==4.6.3
     # via
     #   -c requirements/base.txt
     #   -c requirements/test.txt
@@ -311,7 +315,7 @@ webencodings==0.5.1
     # via
     #   bleach
     #   tinycss2
-websocket-client==1.5.1
+websocket-client==1.5.2
     # via jupyter-server
 wheel==0.40.0
     # via pip-tools

requirements/test.in

@@ -14,3 +14,4 @@ mypy
 pytest-cov
 pdf2image>=1.16.2
 huggingface_hub>=0.11.1
+ruff

requirements/test.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile requirements/test.in
 #
-anyio==3.6.2
+anyio==3.7.0
     # via httpcore
 black==23.3.0
     # via -r requirements/test.in
@@ -22,12 +22,14 @@ click==8.1.3
     # via
     #   -r requirements/test.in
     #   black
-coverage[toml]==7.2.5
+coverage[toml]==7.2.7
     # via
     #   -r requirements/test.in
     #   pytest-cov
 exceptiongroup==1.1.1
-    # via pytest
+    # via
+    #   anyio
+    #   pytest
 filelock==3.12.0
     # via
     #   -c requirements/base.txt
@@ -44,11 +46,11 @@ fsspec==2023.5.0
     #   huggingface-hub
 h11==0.14.0
     # via httpcore
-httpcore==0.17.1
+httpcore==0.17.2
     # via httpx
-httpx==0.24.0
+httpx==0.24.1
     # via -r requirements/test.in
-huggingface-hub==0.14.1
+huggingface-hub==0.15.1
     # via
     #   -c requirements/base.txt
     #   -r requirements/test.in
@@ -96,16 +98,18 @@ pyflakes==3.0.1
     # via flake8
 pytest==7.3.1
     # via pytest-cov
-pytest-cov==4.0.0
+pytest-cov==4.1.0
     # via -r requirements/test.in
 pyyaml==6.0
     # via
     #   -c requirements/base.txt
     #   huggingface-hub
-requests==2.30.0
+requests==2.31.0
     # via
     #   -c requirements/base.txt
     #   huggingface-hub
+ruff==0.0.270
+    # via -r requirements/test.in
 sniffio==1.3.0
     # via
     #   anyio
@@ -123,7 +127,7 @@ tqdm==4.65.0
     # via
     #   -c requirements/base.txt
     #   huggingface-hub
-typing-extensions==4.5.0
+typing-extensions==4.6.3
     # via
     #   -c requirements/base.txt
     #   black

setup.py

@@ -17,9 +17,10 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-from setuptools import setup, find_packages
 from typing import List
 
+from setuptools import find_packages, setup
+
 from unstructured_inference.__version__ import __version__
 
 
@@ -36,10 +37,17 @@ def load_requirements(file_list=None):
     return requirements
 
 
+def load_text_from_file(filename: str):
+    """Retrieves text from a file."""
+    with open(filename, encoding="utf-8") as fp:
+        description = fp.read()
+    return description
+
+
 setup(
     name="unstructured_inference",
     description="A library for performing inference using trained models.",
-    long_description=open("README.md", "r", encoding="utf-8").read(),
+    long_description=load_text_from_file("README.md"),
     long_description_content_type="text/markdown",
     keywords="NLP PDF HTML CV XML parsing preprocessing",
     url="https://github.com/Unstructured-IO/unstructured-inference",
@@ -78,6 +86,6 @@ def load_requirements(file_list=None):
             'protobuf<3.21 ; platform_machine=="x86_64"',
             # NOTE(alan): Pin to get around error: undefined symbol: _dl_sym, version GLIBC_PRIVATE
             'paddlepaddle>=2.4 ; platform_machine=="x86_64"',
-        ]
+        ],
     },
 )