Jj/297 max chars (#298)

Closes #297 
Adds `max_characters` param for chunking.
Refactors associated tests for chunking parameters.

Author: Coniferish

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.0.56-dev0
+* **Add `max_characters` param for chunking** This param gives users additional control to "chunk" elements into larger or smaller `CompositeElement`s
+
+
 ## 0.0.55
 
 * Bump unstructured to 0.10.26

README.md

@@ -219,7 +219,10 @@ Set the `chunking_strategy` to chunk text into larger or smaller elements. Defau
       reaches a length of n characters. Defaults to 500.
 
     `new_after_n_chars`
-      Cuts off new sections once they reach a length of "n" characters. Defaults to 1500.
+      Cuts off new sections once they reach a length of "n" characters (soft max). Defaults to 1500.
+
+    `max_characters`
+      Cuts off new sections once they reach a length of "n" characters (hard max). Defaults to 1500.
 
 ```
 curl -X 'POST' 

openapi.json

@@ -205,7 +205,13 @@
                     "new_after_n_chars": {
                         "type": "integer",
                         "title": "New after n chars",
-                        "description": "If chunking strategy is set, cut off new sections after reaching a length of n chars. Default: 1500",
+                        "description": "If chunking strategy is set, cut off new sections after reaching a length of n chars (soft max). Default: 1500",
+                        "example": 1500
+                    },
+                    "max_characters": {
+                        "type": "integer",
+                        "title": "Max Characters",
+                        "description": "If chunking strategy is set, cut off new sections after reaching a length of n chars (hard max). Default: 1500",
                         "example": 1500
                     }
                 },

prepline_general/api/app.py

@@ -11,7 +11,7 @@
 app = FastAPI(
     title="Unstructured Pipeline API",
     description="""""",
-    version="0.0.55",
+    version="0.0.56",
     docs_url="/general/docs",
     openapi_url="/general/openapi.json",
 )

prepline_general/api/general.py

@@ -269,6 +269,7 @@ def pipeline_api(
     m_multipage_sections=[],
     m_combine_under_n_chars=[],
     m_new_after_n_chars=[],
+    m_max_characters=[],
 ):
     if filename.endswith(".msg"):
         # Note(yuming): convert file type for msg files
@@ -300,6 +301,7 @@ def pipeline_api(
                         "m_multipage_sections": m_multipage_sections,
                         "m_combine_under_n_chars": m_combine_under_n_chars,
                         "new_after_n_chars": m_new_after_n_chars,
+                        "m_max_characters": m_max_characters,
                     },
                     default=str,
                 )
@@ -403,6 +405,10 @@ def pipeline_api(
         else 1500
     )
 
+    max_characters = (
+        int(m_max_characters[0]) if m_max_characters and m_max_characters[0].isdigit() else 1500
+    )
+
     try:
         logger.debug(
             "partition input data: {}".format(
@@ -423,6 +429,7 @@ def pipeline_api(
                         "multipage_sections": multipage_sections,
                         "combine_under_n_chars": combine_under_n_chars,
                         "new_after_n_chars": new_after_n_chars,
+                        "max_characters": max_characters,
                     },
                     default=str,
                 )
@@ -446,6 +453,7 @@ def pipeline_api(
             "multipage_sections": multipage_sections,
             "combine_under_n_chars": combine_under_n_chars,
             "new_after_n_chars": new_after_n_chars,
+            "max_characters": max_characters,
         }
 
         if file_content_type == "application/pdf" and pdf_parallel_mode_enabled:
@@ -621,7 +629,7 @@ def return_content_type(filename):
 
 
 @router.post("/general/v0/general")
-@router.post("/general/v0.0.55/general")
+@router.post("/general/v0.0.56/general")
 def pipeline_1(
     request: Request,
     gz_uncompressed_content_type: Optional[str] = Form(default=None),
@@ -641,6 +649,7 @@ def pipeline_1(
     multipage_sections: List[str] = Form(default=[]),
     combine_under_n_chars: List[str] = Form(default=[]),
     new_after_n_chars: List[str] = Form(default=[]),
+    max_characters: List[str] = Form(default=[]),
 ):
     if files:
         for file_index in range(len(files)):
@@ -697,6 +706,7 @@ def response_generator(is_multipart):
                     m_multipage_sections=multipage_sections,
                     m_combine_under_n_chars=combine_under_n_chars,
                     m_new_after_n_chars=new_after_n_chars,
+                    m_max_characters=max_characters,
                 )
 
                 if is_expected_response_type(media_type, type(response)):

preprocessing-pipeline-family.yaml

@@ -1,2 +1,2 @@
 name: general
-version: 0.0.55
+version: 0.0.56

test_general/api/test_app.py

@@ -489,6 +489,7 @@ def test_parallel_mode_passes_params(monkeypatch):
             "multipage_sections": False,
             "combine_under_n_chars": 501,
             "new_after_n_chars": 1501,
+            "max_characters": 1502,
         },
     )
 
@@ -511,6 +512,7 @@ def test_parallel_mode_passes_params(monkeypatch):
         multipage_sections=False,
         combine_under_n_chars=501,
         new_after_n_chars=1501,
+        max_characters=1502,
     )
 
 
@@ -649,56 +651,45 @@ def test_chunking_strategy_param():
     assert "CompositeElement" in [element.get("type") for element in response_with_chunking]
 
 
-# def test_chunking_strategy_additional_params():
-#     client = TestClient(app)
-#     test_file = Path("sample-docs") / "layout-parser-paper-fast.pdf"
-#     res = client.post(
-#         MAIN_API_ROUTE,
-#         files=[("files", (str(test_file), open(test_file, "rb")))],
-#         data={
-#             "chunking_strategy": "by_title",
-#             "multipage_sections": "False",
-#             "combine_under_n_chars": "0",
-#         },
-#     )
-#     response_from_multipage_false_combine_chars_0  = res.json()
-
-#     res = client.post(
-#         MAIN_API_ROUTE,
-#         files=[("files", (str(test_file), open(test_file, "rb")))],
-#         data={
-#             "chunking_strategy": "by_title",
-#             "multipage_sections": "True",
-#             "combine_under_n_chars": "0",
-#         },
-#     )
-#     response_from_multipage_true_combine_chars_0 = res.json()
-
-#     res = client.post(
-#         MAIN_API_ROUTE,
-#         files=[("files", (str(test_file), open(test_file, "rb")))],
-#         data={
-#             "chunking_strategy": "by_title",
-#             "multipage_sections": "True",
-#             "combine_under_n_chars": "5000",
-#             # Defining new_after_n_chars since it has to be greater than combine_under_n_chars
-#             "new_after_n_chars": "50000",
-#         },
-#     )
-#     response_multipage_true_combine_chars_5000 = res.json()
-
-#     assert (
-#         response_multipage_true_combine_chars_5000
-#         != response_from_multipage_true_combine_chars_0
-#     )
-#     assert (
-#         response_from_multipage_true_combine_chars_0
-#         != response_from_multipage_false_combine_chars_0
-#     )
-#     assert (
-#         response_multipage_true_combine_chars_5000
-#         != response_from_multipage_false_combine_chars_0
-#     )
+# Defaults:
+# multippage = True, combine_text_under_n_chars = None, new_after_n_chars = None,
+# max_characters = 500
+@pytest.mark.parametrize(
+    ("multipage_sections", "combine_under_n_chars", "new_after_n_chars", "max_characters"),
+    [
+        (False, None, None, 500),  # test multipage_sections
+        (True, 1000, None, 5000),  # test combine_under_n_chars
+        (True, None, 10, 500),  # test new_after_n_chars
+        (True, None, None, 100),  # test max__characters
+    ],
+)
+def test_chunking_strategy_additional_params(
+    multipage_sections, combine_under_n_chars, new_after_n_chars, max_characters
+):
+    client = TestClient(app)
+    test_file = Path("sample-docs") / "layout-parser-paper-fast.pdf"
+
+    arg_resp = client.post(
+        MAIN_API_ROUTE,
+        files=[("files", (str(test_file), open(test_file, "rb")))],
+        data={
+            "chunking_strategy": "by_title",
+            "multipage_sections": multipage_sections,
+            "combine_under_n_chars": combine_under_n_chars,
+            "new_after_n_chars": new_after_n_chars,
+            "max_characters": max_characters,
+        },
+    )
+    arg_resp_json = arg_resp.json()
+
+    default_resp = client.post(
+        MAIN_API_ROUTE,
+        files=[("files", (str(test_file), open(test_file, "rb")))],
+        data={"chunking_strategy": "by_title"},
+    )
+    default_resp_json = default_resp.json()
+
+    assert arg_resp_json != default_resp_json
 
 
 def test_encrypted_pdf():