Merge branch 'main' into python-upgrade

Author: jaseemjaskp

src/unstract/sdk/index.py

@@ -162,7 +162,10 @@ def extract_text(
             usage_kwargs=usage_kwargs,
         )
         try:
-            if enable_highlight and (isinstance(x2text.x2text_instance, LLMWhisperer) or isinstance(x2text.x2text_instance, LLMWhispererV2)):
+            if enable_highlight and (
+                isinstance(x2text.x2text_instance, LLMWhisperer)
+                or isinstance(x2text.x2text_instance, LLMWhispererV2)
+            ):
                 process_response: TextExtractionResult = x2text.process(
                     input_file_path=file_path,
                     output_file_path=output_file_path,
@@ -172,7 +175,7 @@ def extract_text(
                 )
                 whisper_hash_value = process_response.extraction_metadata.whisper_hash
                 metadata = {X2TextConstants.WHISPER_HASH: whisper_hash_value}
-                if hasattr(self.tool, 'update_exec_metadata'):
+                if hasattr(self.tool, "update_exec_metadata"):
                     self.tool.update_exec_metadata(metadata)
             else:
                 process_response: TextExtractionResult = x2text.process(
@@ -202,6 +205,7 @@ def extract_text(
         return extracted_text
 
     # TODO: Reduce the number of params by some dataclass
+    # TODO: Deprecate and remove `process_text` argument
     @log_elapsed(operation="CHECK_AND_INDEX(overall)")
     @capture_metrics
     def index(

src/unstract/sdk/tool/executor.py

@@ -36,6 +36,7 @@ def execute(self, args: argparse.Namespace) -> None:
 
     def _setup_for_run(self) -> None:
         """Helps initialize tool execution for RUN command."""
+        # TODO: Review if file storage equivalent is needed or remove this
         shutil.rmtree(self.tool.get_output_dir(), ignore_errors=True)
         Path(self.tool.get_output_dir()).mkdir(parents=True, exist_ok=True)
 

src/unstract/sdk/x2txt.py

@@ -123,15 +123,12 @@ def push_usage_details(
     ) -> None:
         file_size = ToolUtils.get_file_size(input_file_path, fs)
 
-        self._x2text_instance
-
         if mime_type == MimeType.PDF:
             pdf_contents = io.BytesIO(fs.read(path=input_file_path, mode="rb"))
             with pdfplumber.open(pdf_contents) as pdf:
                 # calculate the number of pages
                 page_count = len(pdf.pages)
             if isinstance(self._x2text_instance, LLMWhisperer):
-                self._x2text_instance.config.get(WhispererConfig.PAGES_TO_EXTRACT)
                 page_count = ToolUtils.calculate_page_count(
                     self._x2text_instance.config.get(WhispererConfig.PAGES_TO_EXTRACT),
                     page_count,
@@ -144,6 +141,7 @@ def push_usage_details(
                 kwargs=self._usage_kwargs,
             )
         else:
+            # TODO: Calculate page usage for other file types (3000 words = 1 page)
             # We are allowing certain image types,and raw texts. We will consider them
             # as single page documents as there in no concept of page numbers.
             Audit().push_page_usage_data(