Feat/dynamic page number llm whisperer (#82)

* adding dynamic page loading

* adding page separator llm-whisperer

* adding regex pattern

* removing pattern

* PR changes

* upgrade sdk version

Author: kirtimanmishrazipstack

src/unstract/sdk/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.41.1"
+__version__ = "0.42.0"
 
 
 def get_sdk_version():

src/unstract/sdk/adapters/x2text/llm_whisperer/src/constants.py

@@ -61,6 +61,7 @@ class WhispererConfig:
     HORIZONTAL_STRETCH_FACTOR = "horizontal_stretch_factor"
     PAGES_TO_EXTRACT = "pages_to_extract"
     STORE_METADATA_FOR_HIGHLIGHTING = "store_metadata_for_highlighting"
+    PAGE_SEPARATOR = "page_seperator"
 
 
 class WhisperStatus:
@@ -86,3 +87,4 @@ class WhispererDefaults:
     POLL_INTERVAL = int(os.getenv(WhispererEnv.POLL_INTERVAL, 30))
     MAX_POLLS = int(os.getenv(WhispererEnv.MAX_POLLS, 30))
     PAGES_TO_EXTRACT = ""
+    PAGE_SEPARATOR = "<<< >>>"

src/unstract/sdk/adapters/x2text/llm_whisperer/src/llm_whisperer.py

@@ -10,7 +10,10 @@
 from unstract.sdk.adapters.exceptions import ExtractorError
 from unstract.sdk.adapters.utils import AdapterUtils
 from unstract.sdk.adapters.x2text.constants import X2TextConstants
-from unstract.sdk.adapters.x2text.dto import TextExtractionMetadata, TextExtractionResult
+from unstract.sdk.adapters.x2text.dto import (
+    TextExtractionMetadata,
+    TextExtractionResult,
+)
 from unstract.sdk.adapters.x2text.llm_whisperer.src.constants import (
     HTTPMethod,
     OutputModes,
@@ -159,6 +162,10 @@ def _get_whisper_params(self, enable_highlight: bool = False) -> dict[str, Any]:
                 WhispererConfig.PAGES_TO_EXTRACT,
                 WhispererDefaults.PAGES_TO_EXTRACT,
             ),
+            WhispererConfig.PAGE_SEPARATOR: self.config.get(
+                WhispererConfig.PAGE_SEPARATOR,
+                WhispererDefaults.PAGE_SEPARATOR,
+            ),
         }
         if not params[WhispererConfig.FORCE_TEXT_PROCESSING]:
             params.update(

src/unstract/sdk/adapters/x2text/llm_whisperer/src/static/json_schema.json

@@ -83,6 +83,12 @@
       "default": "",
       "pattern": "^(\\s*\\d+-\\d+|\\s*\\d+-|\\s*\\d+|^$)(,\\d+-\\d+|,\\d+-|,\\d+)*$",
       "description": "Specify the range of pages to extract (e.g., 1-5, 7, 10-12, 50-). Leave it empty to extract all pages."
+    },
+    "page_seperator": {
+      "type": "string",
+      "title": "Page separator",
+      "default": "<<< >>>",
+      "description": "Specify a pattern to separate the pages in the document (e.g., <<< {{page_no}} >>>, <<< >>>). This pattern will be inserted at the end of every page. Omit {{page_no}} if you don't want to include the page number in the separator."
     }
   },
   "if": {