Chore: add support for include_page_breaks param (#153)

* add page break param to api

* test inclusion of page breaks

* add test file

* fix: include page breaks test

* rebase

* update changelog and readme

* added include page breaks param to smoke test

Author: shreyanid

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## 0.0.32-dev0
 
 * Improve logging of params to single line json
+* Add support for `include_page_breaks` parameter
 
 ## 0.0.31
 

README.md

@@ -136,6 +136,20 @@ curl -X 'POST'
  | jq -C . | less -R
 ```
 
+#### Page Breaks
+
+For supported filetypes, set the `include_page_breaks` parameter to `true` to include `PageBreak` elements in the output.
+
+```
+curl -X 'POST' 
+ 'https://api.unstructured.io/general/v0/general' \
+ -H 'accept: application/json'  \
+ -H 'Content-Type: multipart/form-data' \
+ -F 'files=@sample-docs/layout-parser-paper-fast.pdf' \
+ -F 'include_page_breaks=true' \
+ | jq -C . | less -R
+```
+
 ## Developer Quick Start
 
 * Using `pyenv` to manage virtualenv's is recommended

pipeline-notebooks/pipeline-general.ipynb

@@ -1,6 +1,7 @@
 {
  "cells": [
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "e908195c",
    "metadata": {},
@@ -9,6 +10,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "727614ba",
    "metadata": {},
@@ -35,6 +37,7 @@
    "source": []
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "3848e558",
    "metadata": {},
@@ -43,6 +46,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "01a62fe4",
    "metadata": {},
@@ -98,6 +102,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "63e3b32b",
    "metadata": {},
@@ -217,6 +222,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "15d69b6b",
    "metadata": {},
@@ -225,6 +231,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "5c9e618c",
    "metadata": {},
@@ -317,6 +324,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "258531fe",
    "metadata": {},
@@ -361,6 +369,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "10e1d3df",
    "metadata": {},
@@ -369,6 +378,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "52943c00",
    "metadata": {},
@@ -451,6 +461,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "0f7fea99",
    "metadata": {},
@@ -517,6 +528,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "cde38923",
    "metadata": {},
@@ -730,6 +742,7 @@
     "    m_strategy=[],\n",
     "    m_coordinates=[],\n",
     "    m_ocr_languages=[],\n",
+    "    m_include_page_breaks=[],\n",
     "    m_encoding=[],\n",
     "    m_xml_keep_tags=[],\n",
     "    m_pdf_infer_table_structure = [],\n",
@@ -744,6 +757,7 @@
     "        \"m_strategy\": m_strategy,\n",
     "        \"m_coordinates\": m_coordinates,\n",
     "        \"m_ocr_languages\": m_ocr_languages,\n",
+    "        \"m_include_page_breaks\": m_include_page_breaks,\n",
     "        \"m_encoding\": m_encoding,\n",
     "        \"m_xml_keep_tags\": m_xml_keep_tags,\n",
     "        \"m_pdf_infer_table_structure\": m_pdf_infer_table_structure,\n",
@@ -780,6 +794,9 @@
     "    \n",
     "    ocr_languages= ('+'.join(m_ocr_languages) if len(m_ocr_languages) else 'eng').lower()\n",
     "\n",
+    "    include_page_breaks_str = (m_include_page_breaks[0] if len(m_include_page_breaks) else \"false\").lower()\n",
+    "    include_page_breaks = include_page_breaks_str == \"true\"\n",
+    "    \n",
     "    encoding = m_encoding[0] if len(m_encoding) else None\n",
     "    \n",
     "    xml_keep_tags_str = (m_xml_keep_tags[0] if len(m_xml_keep_tags) else \"false\").lower()\n",
@@ -801,6 +818,7 @@
     "            \"ocr_languages\": ocr_languages,\n",
     "            \"coordinates\": show_coordinates,\n",
     "            \"pdf_infer_table_structure\": pdf_infer_table_structure,\n",
+    "            \"include_page_breaks\": include_page_breaks,\n",
     "            \"encoding\": encoding,\n",
     "            \"model_name\": hi_res_model_name,\n",
     "            \"xml_keep_tags\": xml_keep_tags\n",
@@ -817,6 +835,7 @@
     "                ocr_languages=ocr_languages,\n",
     "                coordinates=show_coordinates,\n",
     "                pdf_infer_table_structure=pdf_infer_table_structure,\n",
+    "                include_page_breaks=include_page_breaks,\n",
     "                encoding=encoding,\n",
     "                model_name=hi_res_model_name\n",
     "            )\n",
@@ -828,6 +847,7 @@
     "                strategy=strategy,\n",
     "                ocr_languages=ocr_languages,\n",
     "                pdf_infer_table_structure=pdf_infer_table_structure,\n",
+    "                include_page_breaks=include_page_breaks,\n",
     "                encoding=encoding,\n",
     "                xml_keep_tags=xml_keep_tags,\n",
     "                model_name=hi_res_model_name\n",
@@ -997,6 +1017,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "e997bff5",
    "metadata": {},

prepline_general/api/general.py

@@ -202,6 +202,7 @@ def pipeline_api(
     m_strategy=[],
     m_coordinates=[],
     m_ocr_languages=[],
+    m_include_page_breaks=[],
     m_encoding=[],
     m_xml_keep_tags=[],
     m_pdf_infer_table_structure=[],
@@ -218,6 +219,7 @@ def pipeline_api(
                     "m_strategy": m_strategy,
                     "m_coordinates": m_coordinates,
                     "m_ocr_languages": m_ocr_languages,
+                    "m_include_page_breaks": m_include_page_breaks,
                     "m_encoding": m_encoding,
                     "m_xml_keep_tags": m_xml_keep_tags,
                     "m_pdf_infer_table_structure": m_pdf_infer_table_structure,
@@ -258,6 +260,11 @@ def pipeline_api(
 
     ocr_languages = ("+".join(m_ocr_languages) if len(m_ocr_languages) else "eng").lower()
 
+    include_page_breaks_str = (
+        m_include_page_breaks[0] if len(m_include_page_breaks) else "false"
+    ).lower()
+    include_page_breaks = include_page_breaks_str == "true"
+
     encoding = m_encoding[0] if len(m_encoding) else None
 
     xml_keep_tags_str = (m_xml_keep_tags[0] if len(m_xml_keep_tags) else "false").lower()
@@ -281,6 +288,7 @@ def pipeline_api(
                         "ocr_languages": ocr_languages,
                         "coordinates": show_coordinates,
                         "pdf_infer_table_structure": pdf_infer_table_structure,
+                        "include_page_breaks": include_page_breaks,
                         "encoding": encoding,
                         "model_name": hi_res_model_name,
                         "xml_keep_tags": xml_keep_tags,
@@ -300,6 +308,7 @@ def pipeline_api(
                 ocr_languages=ocr_languages,
                 coordinates=show_coordinates,
                 pdf_infer_table_structure=pdf_infer_table_structure,
+                include_page_breaks=include_page_breaks,
                 encoding=encoding,
                 model_name=hi_res_model_name,
             )
@@ -311,6 +320,7 @@ def pipeline_api(
                 strategy=strategy,
                 ocr_languages=ocr_languages,
                 pdf_infer_table_structure=pdf_infer_table_structure,
+                include_page_breaks=include_page_breaks,
                 encoding=encoding,
                 xml_keep_tags=xml_keep_tags,
                 model_name=hi_res_model_name,
@@ -467,6 +477,7 @@ def pipeline_1(
     strategy: List[str] = Form(default=[]),
     coordinates: List[str] = Form(default=[]),
     ocr_languages: List[str] = Form(default=[]),
+    include_page_breaks: List[str] = Form(default=[]),
     encoding: List[str] = Form(default=[]),
     xml_keep_tags: List[str] = Form(default=[]),
     pdf_infer_table_structure: List[str] = Form(default=[]),
@@ -513,6 +524,7 @@ def response_generator(is_multipart):
                     m_strategy=strategy,
                     m_coordinates=coordinates,
                     m_ocr_languages=ocr_languages,
+                    m_include_page_breaks=include_page_breaks,
                     m_encoding=encoding,
                     m_xml_keep_tags=xml_keep_tags,
                     m_pdf_infer_table_structure=pdf_infer_table_structure,

requirements/base.txt

@@ -65,7 +65,7 @@ filetype==1.2.0
     # via unstructured
 flatbuffers==23.5.26
     # via onnxruntime
-fonttools==4.40.0
+fonttools==4.41.0
     # via matplotlib
 fsspec==2023.6.0
     # via huggingface-hub
@@ -103,7 +103,7 @@ jinja2==3.1.2
     #   unstructured-api-tools
 joblib==1.3.1
     # via nltk
-jsonschema==4.18.1
+jsonschema==4.18.3
     # via nbformat
 jsonschema-specifications==2023.6.1
     # via jsonschema
@@ -146,7 +146,7 @@ mypy-extensions==1.0.0
     # via mypy
 nbclient==0.8.0
     # via nbconvert
-nbconvert==7.6.0
+nbconvert==7.7.1
     # via unstructured-api-tools
 nbformat==5.9.1
     # via
@@ -203,7 +203,7 @@ pdfminer-six==20221105
     # via
     #   pdfplumber
     #   unstructured
-pdfplumber==0.9.0
+pdfplumber==0.10.0
     # via layoutparser
 pillow==10.0.0
     # via
@@ -217,7 +217,7 @@ pillow==10.0.0
     #   unstructured
 pkgutil-resolve-name==1.3.10
     # via jsonschema
-platformdirs==3.8.1
+platformdirs==3.9.1
     # via jupyter-core
 portalocker==2.7.0
     # via iopath
@@ -239,8 +239,10 @@ pypandoc==1.11
     # via unstructured
 pyparsing==3.0.9
     # via matplotlib
-pypdf==3.12.1
+pypdf==3.12.2
     # via -r requirements/base.in
+pypdfium2==4.18.0
+    # via pdfplumber
 pytesseract==0.3.10
     # via layoutparser
 python-dateutil==2.8.2
@@ -289,7 +291,7 @@ requests==2.31.0
     #   torchvision
     #   transformers
     #   unstructured
-rpds-py==0.8.10
+rpds-py==0.8.11
     # via
     #   jsonschema
     #   referencing
@@ -379,12 +381,10 @@ unstructured-inference==0.5.5
     # via unstructured
 urllib3==2.0.3
     # via requests
-uvicorn[standard]==0.22.0
+uvicorn[standard]==0.23.0
     # via unstructured-api-tools
 uvloop==0.17.0
     # via uvicorn
-wand==0.6.11
-    # via pdfplumber
 watchfiles==0.19.0
     # via uvicorn
 webencodings==0.5.1
@@ -397,7 +397,7 @@ xlrd==2.0.1
     # via unstructured
 xlsxwriter==3.1.2
     # via python-pptx
-zipp==3.16.0
+zipp==3.16.2
     # via
     #   importlib-metadata
     #   importlib-resources