Feat/detectron2 mask rcnn x101 (#149)

* Update CHANGELOG
* Added alternative model for detectron2
* Added script for showing results
* Corrected index at getting output from detectron2
* Version sync
* Detectron2 with architecture mask-rcnn-x101 is now the default model
* Change default values during test for detectron2onnx
* Format changes
* Changed type of stored model_path

Signed-off-by:  Benjamin Torres <[email protected]>
Co-authored-by: Alan Bertl <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* chore(deps-dev): Bump ipython from 8.12.2 to 8.14.0 in /requirements
* chore(deps): Bump transformers from 4.29.2 to 4.30.2 in /requirements (#142)
* chore(deps): Bump transformers from 4.29.2 to 4.30.2 in /requirements
* chore(deps): Bump opencv-python from 4.7.0.72 to 4.8.0.74 in /requirements (#137)
* chore(deps): Bump opencv-python in /requirements

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Benjamin Torres <[email protected]>

Author: benjats07

CHANGELOG.md

@@ -1,6 +1,15 @@
-## 0.5.8-dev1
+## 0.5.8-dev2
 
-* Cache named models that have been lodaed
+* Add alternative architecture for detectron2
+* Updates:
+
+| Library       | From      | To       |
+|---------------|-----------|----------|
+| transformers  | 4.29.2    | 4.30.2   |
+| opencv-python | 4.7.0.72  | 4.8.0.74 |
+| ipython       | 8.12.2    | 8.14.0   |
+
+* Cache named models that have been loaded
 
 ## 0.5.7
 

requirements/dev.txt

@@ -6,12 +6,8 @@
 #
 anyio==3.7.1
     # via
-    #   -c requirements/test.txt
+    #   -c test.txt
     #   jupyter-server
-appnope==0.1.3
-    # via
-    #   ipykernel
-    #   ipython
 argon2-cffi==21.3.0
     # via jupyter-server
 argon2-cffi-bindings==21.2.0
@@ -43,7 +39,7 @@ certifi==2023.7.22
     #   requests
 cffi==1.15.1
     # via
-    #   -c requirements/base.txt
+    #   -c base.txt
     #   argon2-cffi-bindings
 charset-normalizer==3.2.0
     # via
@@ -52,7 +48,7 @@ charset-normalizer==3.2.0
     #   requests
 click==8.1.6
     # via
-    #   -c requirements/test.txt
+    #   -c test.txt
     #   pip-tools
 comm==0.1.3
     # via ipykernel
@@ -74,8 +70,8 @@ fqdn==1.5.1
     # via jsonschema
 idna==3.4
     # via
-    #   -c requirements/base.txt
-    #   -c requirements/test.txt
+    #   -c base.txt
+    #   -c test.txt
     #   anyio
     #   jsonschema
     #   requests
@@ -100,9 +96,9 @@ ipykernel==6.25.0
     #   jupyter-console
     #   jupyterlab
     #   qtconsole
-ipython==8.12.2
+ipython==8.14.0
     # via
-    #   -r requirements/dev.in
+    #   -r dev.in
     #   ipykernel
     #   ipywidgets
     #   jupyter-console
@@ -116,7 +112,7 @@ jedi==0.18.2
     # via ipython
 jinja2==3.1.2
     # via
-    #   -c requirements/base.txt
+    #   -c base.txt
     #   jupyter-server
     #   jupyterlab
     #   jupyterlab-server
@@ -179,7 +175,7 @@ jupyterlab-widgets==3.0.8
     # via ipywidgets
 markupsafe==2.1.3
     # via
-    #   -c requirements/base.txt
+    #   -c base.txt
     #   jinja2
     #   nbconvert
 matplotlib-inline==0.1.6
@@ -211,8 +207,8 @@ overrides==7.3.1
     # via jupyter-server
 packaging==23.1
     # via
-    #   -c requirements/base.txt
-    #   -c requirements/test.txt
+    #   -c base.txt
+    #   -c test.txt
     #   build
     #   ipykernel
     #   jupyter-server
@@ -235,7 +231,7 @@ pkgutil-resolve-name==1.3.10
     # via jsonschema
 platformdirs==3.9.1
     # via
-    #   -c requirements/test.txt
+    #   -c test.txt
     #   jupyter-core
 prometheus-client==0.17.1
     # via jupyter-server
@@ -253,7 +249,7 @@ pure-eval==0.2.2
     # via stack-data
 pycparser==2.21
     # via
-    #   -c requirements/base.txt
+    #   -c base.txt
     #   cffi
 pygments==2.15.1
     # via
@@ -315,14 +311,14 @@ send2trash==1.8.2
     # via jupyter-server
 six==1.16.0
     # via
-    #   -c requirements/base.txt
+    #   -c base.txt
     #   asttokens
     #   bleach
     #   python-dateutil
     #   rfc3339-validator
 sniffio==1.3.0
     # via
-    #   -c requirements/test.txt
+    #   -c test.txt
     #   anyio
 soupsieve==2.4.1
     # via beautifulsoup4

test_unstructured_inference/models/test_detectron2onnx.py

@@ -14,7 +14,7 @@ def __init__(self, *args, **kwargs):
         self.kwargs = kwargs
 
     def run(self, *args):
-        return ([(1, 2, 3, 4)], [0], [0.818], [(4, 5)])
+        return ([(1, 2, 3, 4)], [0], [(4, 5)], [0.818])
 
     def get_inputs(self):
         class input_thing:
@@ -30,7 +30,7 @@ def test_load_default_model(monkeypatch):
         "InferenceSession",
         new=MockDetectron2ONNXLayoutModel,
     ):
-        model = models.get_model("detectron2_onnx")
+        model = models.get_model("detectron2_mask_rcnn")
 
     assert isinstance(model.model, MockDetectron2ONNXLayoutModel)
 

unstructured_inference/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.8-dev1"  # pragma: no cover
+__version__ = "0.5.8-dev2"  # pragma: no cover

unstructured_inference/models/base.py

@@ -23,7 +23,7 @@
     UnstructuredYoloXModel,
 )
 
-DEFAULT_MODEL = "detectron2_onnx"
+DEFAULT_MODEL = "detectron2_mask_rcnn"
 
 models: Dict[str, UnstructuredModel] = {}
 

unstructured_inference/models/detectron2onnx.py

@@ -37,6 +37,15 @@
         label_map=DEFAULT_LABEL_MAP,
         confidence_threshold=0.8,
     ),
+    "detectron2_mask_rcnn": LazyDict(
+        model_path=LazyEvaluateInfo(
+            hf_hub_download,
+            "unstructuredio/detectron2_mask_rcnn_X_101_32x8d_FPN_3x",
+            "model.onnx",
+        ),
+        label_map=DEFAULT_LABEL_MAP,
+        confidence_threshold=0.8,
+    ),
 }
 
 
@@ -53,7 +62,12 @@ def predict(self, image: Image.Image) -> List[LayoutElement]:
 
         prepared_input = self.preprocess(image)
         try:
-            bboxes, labels, confidence_scores, _ = self.model.run(None, prepared_input)
+            result = self.model.run(None, prepared_input)
+            bboxes = result[0]
+            labels = result[1]
+            # Previous model detectron2_onnx stored confidence scores at index 2,
+            # bigger model stores it at index 3
+            confidence_scores = result[2] if "R_50" in self.model_path else result[3]
         except onnxruntime.capi.onnxruntime_pybind11_state.RuntimeException:
             logger_onnx.debug(
                 "Ignoring runtime error from onnx (likely due to encountering blank page).",
@@ -72,6 +86,7 @@ def initialize(
     ):
         """Loads the detectron2 model using the specified parameters"""
         logger.info("Loading the Detectron2 layout model ...")
+        self.model_path = str(model_path)
         self.model = onnxruntime.InferenceSession(
             model_path,
             providers=[