Expose model name in api (#148)

* Add in latest unstructed dependency as a requirement

* Add update to api code to expose the model name as an optional parameter

* Update docker to use multistage builds and initialize the chipper model

* Remove unneeded logger configs in notebook

* Fix smoketest

* regenerate api

* tidy notebooks

* Bump api tools version

* regenerate api

* fix unit test

* update response code to 400 from 403

Author: rbiseck3

CHANGELOG.md

@@ -1,5 +1,6 @@
 ## 0.0.31
 
+* Support model name as api parameter
 * Add retry parameters on fanout requests
 * Bump unstructured library to 0.8.1
 * Fix how to remove an element's coordinate information

Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:experimental
-FROM quay.io/unstructured-io/base-images:rocky8.7-3
+FROM quay.io/unstructured-io/base-images:rocky8.7-3 as base
 
 # NOTE(crag): NB_USER ARG for mybinder.org compat:
 #             https://mybinder.readthedocs.io/en/latest/tutorials/dockerfile.html
@@ -21,7 +21,7 @@ RUN mkdir ${HOME}/.ssh && chmod go-rwx ${HOME}/.ssh \
 ENV PYTHONPATH="${PYTHONPATH}:${HOME}"
 ENV PATH="/home/${NB_USER}/.local/bin:${PATH}"
 
-
+FROM base as python-deps
 # COPY requirements/dev.txt requirements-dev.txt
 COPY requirements/base.txt requirements-base.txt
 RUN python3.8 -m pip install pip==${PIP_VERSION} \
@@ -33,10 +33,13 @@ RUN python3.8 -m pip install pip==${PIP_VERSION} \
 
 USER ${NB_USER}
 
+FROM python-deps as model-deps
 RUN python3.8 -c "import nltk; nltk.download('punkt')" && \
   python3.8 -c "import nltk; nltk.download('averaged_perceptron_tagger')" && \
-  python3.8 -c "from unstructured.ingest.doc_processor.generalized import initialize; initialize()"
+   UNSTRUCTURED_HI_RES_SUPPORTED_MODEL=chipper python3.8 -c "from unstructured.ingest.doc_processor.generalized import initialize; initialize()"
+
 
+FROM model-deps as code
 COPY --chown=${NB_USER}:${NB_USER} CHANGELOG.md CHANGELOG.md
 COPY --chown=${NB_USER}:${NB_USER} logger_config.yaml logger_config.yaml
 COPY --chown=${NB_USER}:${NB_USER} prepline_${PIPELINE_PACKAGE}/ prepline_${PIPELINE_PACKAGE}/
@@ -46,6 +49,6 @@ COPY --chown=${NB_USER}:${NB_USER} pipeline-notebooks pipeline-notebooks
 ENTRYPOINT ["uvicorn", "prepline_general.api.app:app", \
   "--log-config", "logger_config.yaml", \
   "--host", "0.0.0.0"]
-# Expose a default port of 8000. Note: The EXPOSE instruction does not actually publish the port, 
-# but some tooling will inspect containers and perform work contingent on networking support declared. 
-EXPOSE 8000
+# Expose a default port of 8000. Note: The EXPOSE instruction does not actually publish the port,
+# but some tooling will inspect containers and perform work contingent on networking support declared.
+EXPOSE 8000

logger_config.yaml

@@ -44,4 +44,9 @@ loggers:
     handlers:
       - standard_handler
     propagate: no
+  unstructured_api:
+    level: DEBUG
+    handlers:
+      - standard_handler
+    propagate: no
 

pipeline-notebooks/pipeline-general.ipynb

@@ -1,7 +1,6 @@
 {
  "cells": [
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "e908195c",
    "metadata": {},
@@ -10,7 +9,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "727614ba",
    "metadata": {},
@@ -37,7 +35,6 @@
    "source": []
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "3848e558",
    "metadata": {},
@@ -46,7 +43,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "01a62fe4",
    "metadata": {},
@@ -102,7 +98,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "63e3b32b",
    "metadata": {},
@@ -222,7 +217,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "15d69b6b",
    "metadata": {},
@@ -231,7 +225,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "5c9e618c",
    "metadata": {},
@@ -324,7 +317,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "258531fe",
    "metadata": {},
@@ -369,7 +361,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "10e1d3df",
    "metadata": {},
@@ -378,7 +369,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "52943c00",
    "metadata": {},
@@ -461,7 +451,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "0f7fea99",
    "metadata": {},
@@ -528,7 +517,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "cde38923",
    "metadata": {},
@@ -730,6 +718,10 @@
    "outputs": [],
    "source": [
     "# pipeline-api\n",
+    "from unstructured_inference.models.chipper import MODEL_TYPES as CHIPPER_MODEL_TYPES\n",
+    "import logging\n",
+    "\n",
+    "logger = logging.getLogger(\"unstructured_api\")\n",
     "\n",
     "def pipeline_api(\n",
     "    file, \n",
@@ -741,9 +733,23 @@
     "    m_encoding=[],\n",
     "    m_xml_keep_tags=[],\n",
     "    m_pdf_infer_table_structure = [],\n",
+    "    m_hi_res_model_name=[],\n",
     "    file_content_type=None,\n",
     "    response_type=\"application/json\"\n",
     "):\n",
+    "    logger.debug(\n",
+    "        f\"\\npipeline_api input params:\\n\"\n",
+    "        f\"filename: {filename}\\n\"\n",
+    "        f\"m_strategy: {m_strategy}\\n\"\n",
+    "        f\"m_coordinates: {m_coordinates}\\n\"\n",
+    "        f\"m_ocr_languages: {m_ocr_languages}\\n\"\n",
+    "        f\"m_encoding: {m_encoding}\\n\"\n",
+    "        f\"m_xml_keep_tags: {m_xml_keep_tags}\\n\"\n",
+    "        f\"m_pdf_infer_table_structure: {m_pdf_infer_table_structure}\\n\"\n",
+    "        f\"m_hi_res_model_name: {m_hi_res_model_name}\\n\"\n",
+    "        f\"file_content_type: {file_content_type}\\n\"\n",
+    "        f\"response_type: {response_type}\"\n",
+    "    )\n",
     "    if filename.endswith(\".msg\"):\n",
     "        # Note(yuming): convert file type for msg files\n",
     "        # since fast api might sent the wrong one.\n",
@@ -760,6 +766,11 @@
     "    show_coordinates_str = (m_coordinates[0] if len(m_coordinates) else \"false\").lower()\n",
     "    show_coordinates = show_coordinates_str == \"true\"\n",
     "    \n",
+    "    hi_res_model_name = m_hi_res_model_name[0] if len(m_hi_res_model_name) else None\n",
+    "    \n",
+    "    if hi_res_model_name and hi_res_model_name in CHIPPER_MODEL_TYPES and show_coordinates:\n",
+    "        raise HTTPException(status_code=400, detail=f\"coordinates aren't available when using the {hi_res_model_name} model type\")\n",
+    "    \n",
     "    # Parallel mode is set by env variable\n",
     "    enable_parallel_mode = os.environ.get(\"UNSTRUCTURED_PARALLEL_MODE_ENABLED\", \"false\")\n",
     "    pdf_parallel_mode_enabled = enable_parallel_mode == \"true\"\n",
@@ -780,6 +791,18 @@
     "        pdf_infer_table_structure = False\n",
     "    \n",
     "    try:\n",
+    "        logger.debug(\n",
+    "            f\"\\npartition input data:\\n\"\n",
+    "            f\"content_type: {file_content_type}\\n\"\n",
+    "            f\"strategy: {strategy}\\n\"\n",
+    "            f\"ocr_languages: {ocr_languages}\\n\"\n",
+    "            f\"coordinates: {show_coordinates}\\n\"\n",
+    "            f\"pdf_infer_table_structure: {pdf_infer_table_structure}\\n\"\n",
+    "            f\"encoding: {encoding}\\n\"\n",
+    "            f\"model_name: {hi_res_model_name}\\n\"\n",
+    "            f\"xml_keep_tags: {xml_keep_tags}\\n\"\n",
+    "        )\n",
+    "          \n",
     "        if file_content_type == \"application/pdf\" and pdf_parallel_mode_enabled:\n",
     "            elements = partition_pdf_splits(\n",
     "                request,\n",
@@ -791,6 +814,7 @@
     "                coordinates=show_coordinates,\n",
     "                pdf_infer_table_structure=pdf_infer_table_structure,\n",
     "                encoding=encoding,\n",
+    "                model_name=hi_res_model_name\n",
     "            )\n",
     "        else:\n",
     "            elements = partition(\n",
@@ -802,6 +826,7 @@
     "                pdf_infer_table_structure=pdf_infer_table_structure,\n",
     "                encoding=encoding,\n",
     "                xml_keep_tags=xml_keep_tags,\n",
+    "                model_name=hi_res_model_name\n",
     "            )\n",
     "    except ValueError as e:\n",
     "        if 'Invalid file' in e.args[0]:\n",
@@ -968,7 +993,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "e997bff5",
    "metadata": {},

prepline_general/api/app.py

@@ -39,6 +39,12 @@ def filter(self, record: logging.LogRecord) -> bool:
         return record.getMessage().find("/healthcheck") == -1
 
 
+# Filter out /metrics noise
+class MetricsCheckFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        return record.getMessage().find("/metrics") == -1
+
+
 logging.getLogger("uvicorn.access").addFilter(HealthCheckFilter())