[FIX] Fixes for Prompt studio Indexing and tool runs (#143)

* Refactoring changed file names

* Roll version

* Update src/unstract/sdk/utils/tool_utils.py

Co-authored-by: Chandrasekharan M <[email protected]>
Signed-off-by: Gayathri <[email protected]>

* Update tests/test_fs_permanent.py

Co-authored-by: Chandrasekharan M <[email protected]>
Signed-off-by: Gayathri <[email protected]>

* Update tests/test_fs_permanent.py

Co-authored-by: Chandrasekharan M <[email protected]>
Signed-off-by: Gayathri <[email protected]>

* Update src/unstract/sdk/utils/tool_utils.py

Co-authored-by: Chandrasekharan M <[email protected]>
Signed-off-by: Gayathri <[email protected]>

* Address review comments

* Add support for passing length to mime_type

* Add recursive and fix mypy issue

* CHange test case with new behavior to return FileNotFound in read()

* Remove typing kwargs.

* Resolve mypy issues

* Resolve mypy issues

* Remove unwanted conditionals/vars

* Remove pandoc and tessaract.

* Details of provider added to error message

* fixed enum conditional matching value

* Include EnvHelper in __init__

* Rename error handler

* Upgrade version

* Expose StorageType outside

* Resolve circular dependency issue

* Resolve circular dependency issue

* Indexing fixes + clean up

* Add deprecation warnings

* Add deprecation warnings

---------

Signed-off-by: Gayathri <[email protected]>
Co-authored-by: Chandrasekharan M <[email protected]>

Author: gaya3-zipstack

src/unstract/sdk/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.54.0rc11"
+__version__ = "0.54.0rc12"
 
 
 def get_sdk_version():

src/unstract/sdk/adapters/utils.py

@@ -1,4 +1,5 @@
 import logging
+import warnings
 from pathlib import Path
 
 import filetype
@@ -76,6 +77,13 @@ def get_file_mime_type(
         Returns:
             str: MIME type of the file
         """
+        # Adding the following DeprecationWarning manually as the package "deprecated"
+        # does not support deprecation on static methods.
+        warnings.warn(
+            "`get_file_mime_type` is deprecated. "
+            "Use `FileStorage mime_type()` instead.",
+            DeprecationWarning,
+        )
         sample_contents = fs.read(path=input_file, mode="rb", length=100)
         input_file_mime = magic.from_buffer(sample_contents, mime=True)
         return input_file_mime
@@ -93,6 +101,13 @@ def guess_extention(
         Returns:
             str: File extention
         """
+        # Adding the following DeprecationWarning manually as the package "deprecated"
+        # does not support deprecation on static methods.
+        warnings.warn(
+            "`guess_extention` is deprecated. "
+            "Use `FileStorage guess_extension()` instead.",
+            DeprecationWarning,
+        )
         input_file_extention = ""
         sample_contents = fs.read(path=input_file_path, mode="rb", length=100)
         if sample_contents:

src/unstract/sdk/adapters/x2text/helper.py

@@ -73,7 +73,7 @@ def process_document(
             if not local_storage.exists(input_file_path):
                 fs.download(from_path=input_file_path, to_path=input_file_path)
             with open(input_file_path, "rb") as input_f:
-                mime_type = AdapterUtils.get_file_mime_type(input_file=input_file_path)
+                mime_type = local_storage.mime_type(input_file=input_file_path)
                 files = {"file": (input_file_path, input_f, mime_type)}
                 response = UnstructuredHelper.make_request(
                     unstructured_adapter_config=unstructured_adapter_config,

src/unstract/sdk/adapters/x2text/llama_parse/src/llama_parse.py

@@ -7,7 +7,6 @@
 from llama_parse import LlamaParse
 
 from unstract.sdk.adapters.exceptions import AdapterError
-from unstract.sdk.adapters.utils import AdapterUtils
 from unstract.sdk.adapters.x2text.dto import TextExtractionResult
 from unstract.sdk.adapters.x2text.llama_parse.src.constants import LlamaParseConfig
 from unstract.sdk.adapters.x2text.x2text_adapter import X2TextAdapter
@@ -62,9 +61,7 @@ def _call_parser(
             file_extension = pathlib.Path(input_file_path).suffix
             if not file_extension:
                 try:
-                    input_file_extension = AdapterUtils.guess_extention(
-                        input_file_path, fs
-                    )
+                    input_file_extension = fs.guess_extension(input_file_path)
                     input_file_path_copy = input_file_path
                     input_file_path = ".".join(
                         (input_file_path_copy, input_file_extension)

src/unstract/sdk/constants.py

@@ -22,8 +22,9 @@ class ToolEnv:
     DATA_DIR = "TOOL_DATA_DIR"
     EXECUTION_BY_TOOL = "EXECUTION_BY_TOOL"
     EXECUTION_DATA_DIR = "EXECUTION_DATA_DIR"
-    WORKFLOW_EXECUTION_FS_PROVIDER = "WORKFLOW_EXECUTION_FS_PROVIDER"
-    WORKFLOW_EXECUTION_FS_CREDENTIAL = "WORKFLOW_EXECUTION_FS_CREDENTIAL"
+    WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS = (
+        "WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS"
+    )
 
 
 class ConnectorKeys:

src/unstract/sdk/file_storage/constants.py

@@ -4,6 +4,7 @@
 class FileOperationParams:
     READ_ENTIRE_LENGTH = -1
     MIME_TYPE_DEFAULT_READ_LENGTH = 100
+    EXTENSION_DEFAULT_READ_LENGTH = 100
     DEFAULT_ENCODING = "utf-8"
 
 
@@ -15,7 +16,7 @@ class FileSeekPosition:
 
 class StorageType(Enum):
     PERMANENT = "permanent"
-    TEMPORARY = "temporary"
+    SHARED_TEMPORARY = "shared_temporary"
 
 
 class CredentialKeyword:

src/unstract/sdk/file_storage/env_helper.py

@@ -39,7 +39,7 @@ def get_storage(storage_type: StorageType, env_name: str) -> FileStorage:
             credentials = file_storage_creds.get(CredentialKeyword.CREDENTIALS, "{}")
             if storage_type == StorageType.PERMANENT:
                 file_storage = PermanentFileStorage(provider=provider, **credentials)
-            elif storage_type == StorageType.TEMPORARY:
+            elif storage_type == StorageType.SHARED_TEMPORARY:
                 file_storage = SharedTemporaryFileStorage(
                     provider=provider, **credentials
                 )

src/unstract/sdk/file_storage/impl.py

@@ -4,6 +4,7 @@
 from hashlib import sha256
 from typing import Any, Union
 
+import filetype
 import fsspec
 import magic
 import yaml
@@ -361,3 +362,24 @@ def yaml_load(
         with self.fs.open(path=path) as f:
             data: dict[str, Any] = yaml.safe_load(f)
             return data
+
+    @skip_local_cache
+    def guess_extension(self, path: str) -> str:
+        """Returns the extension of the file passed.
+
+        Args:
+            path (str): String holding the path
+
+        Returns:
+            str: File extension
+        """
+        file_extension = ""
+        sample_contents = self.read(
+            path=path,
+            mode="rb",
+            length=FileOperationParams.EXTENSION_DEFAULT_READ_LENGTH,
+        )
+        if sample_contents:
+            file_type = filetype.guess(sample_contents)
+            file_extension = file_type.EXTENSION
+        return file_extension

src/unstract/sdk/file_storage/interface.py

@@ -121,3 +121,7 @@ def yaml_load(
         path: str,
     ) -> dict[Any, Any]:
         pass
+
+    @abstractmethod
+    def guess_extension(self, path: str) -> str:
+        pass

src/unstract/sdk/file_storage/shared_temporary.py

@@ -17,7 +17,7 @@ def __init__(
     ):
         if provider.value not in self.SUPPORTED_FILE_STORAGE_TYPES:
             raise FileStorageError(
-                f"File storage provider is not supported in Permanent mode. "
+                f"File storage provider is not supported in Shared Temporary mode. "
                 f"Supported providers: {self.SUPPORTED_FILE_STORAGE_TYPES}"
             )
         if provider == FileStorageProvider.MINIO: