SDK & adapters merge (#73)

* SDK & adapters merge - First cut

* Refactor unstract.sdk.core.* back to unstract.sdk.*

* Pick up latest changes

* Remove LICENSE

* Refactoring adapters.py to adapter.py to resolve name conflict

* Remove dependency on adapters

* Update lock file

* Fix regex pattern

* pin llama-index-core version

---------

Signed-off-by: Gayathri <[email protected]>

Author: gaya3-zipstack

pdm.lock

@@ -11,13 +11,48 @@ dependencies = [
     "jsonschema~=4.18.2",
     "python-magic~=0.4.27",
     "python-dotenv==1.0.0",
-    # LLM Triad
-    "unstract-adapters~=0.22.0",
+    # Adapter changes
     "llama-index==0.10.38",
+    # Temporary hack to get out of llama-index issues
+    # To be removed once llama-index version is upgraded
+    "llama-index-core==0.10.56",
     "tiktoken~=0.4.0",
     "transformers==4.37.0",
-    # Error handling, remove after moving it to adapters
-    "openai~=1.21.2"
+    "llama-index-embeddings-google==0.1.5",
+    "llama-index-embeddings-azure-openai==0.1.6",
+    # Disabling Hugging Face & FastEmbed to
+    # keep the image size under check
+    #        "llama-index-embeddings-huggingface==0.2.0",
+    # Disabling fast embed due to high processing power
+    #        "llama-index-embeddings-fastembed==0.1.4",
+    "llama-index-embeddings-openai==0.1.11",
+    "llama-index-embeddings-azure-openai==0.1.6",
+    "llama-index-embeddings-ollama==0.1.2",
+    "llama-index-vector-stores-postgres==0.1.3",
+    # Including Supabase conflicts with postgres on pg-vector.
+    # Hence, commenting it out at the moment
+    #        "llama-index-vector-stores-supabase==0.1.3",
+    "llama-index-vector-stores-milvus==0.1.18",
+    "llama-index-vector-stores-weaviate==0.1.4",
+    "llama-index-vector-stores-pinecone==0.1.4",
+    "llama-index-vector-stores-qdrant==0.2.8",
+    "llama-index-llms-openai==0.1.26",
+    "llama-index-llms-palm==0.1.5",
+    "llama-index-llms-mistralai==0.1.10",
+    "llama-index-llms-anyscale==0.1.3",
+    "llama-index-llms-anthropic==0.1.11",
+    "llama-index-llms-azure-openai==0.1.5",
+    "llama-index-llms-vertex==0.1.8",
+    "llama-index-llms-replicate==0.1.3",
+    "llama-index-llms-ollama==0.1.3",
+    # OCR
+    "filetype~=1.2.0",
+    # Others
+    # For singleton classes
+    "singleton-decorator~=1.0.0",
+    # For Llama Parse X2Text
+    "llama-parse==0.4.1",
+    "httpx>=0.25.2",
 ]
 readme = "README.md"
 urls = { Homepage = "https://unstract.com", "Release notes" = "https://github.com/Zipstack/unstract-sdk/releases", Source = "https://github.com/Zipstack/unstract-sdk" }

pyproject.toml

@@ -1,4 +1,5 @@
-__version__ = "0.38.1"
+__version__ = "0.39.0"
+
 
 
 def get_sdk_version():

src/unstract/sdk/LICENSE

@@ -67,8 +67,7 @@ def get_adapter_configuration(
 
         elif response.status_code == 404:
             self.tool.stream_log(
-                f"adapter not found for: for adapter instance"
-                f"{adapter_instance_id}",
+                f"adapter not found for: for adapter instance" f"{adapter_instance_id}",
                 level=LogLevel.ERROR,
             )
             return None
@@ -127,7 +126,6 @@ def get_adapter_config(
         ] = tool_adapter.get_adapter_configuration(adapter_instance_id)
         if not adapter_metadata:
             tool.stream_error_and_exit(
-                f"Adapter not found for "
-                f"adapter instance: {adapter_instance_id}"
+                f"Adapter not found for " f"adapter instance: {adapter_instance_id}"
             )
         return adapter_metadata

src/unstract/sdk/__init__.py

@@ -0,0 +1,8 @@
+import logging
+from logging import NullHandler
+from typing import Any
+
+logging.getLogger(__name__).addHandler(NullHandler())
+
+AdapterDict = dict[str, dict[str, Any]]
+

src/unstract/sdk/adapters.py renamed to src/unstract/sdk/adapter.py

@@ -0,0 +1,86 @@
+import logging
+from typing import Any
+
+from singleton_decorator import singleton
+
+from unstract.sdk.adapters import AdapterDict
+from unstract.sdk.adapters.base import Adapter
+from unstract.sdk.adapters.constants import Common
+from unstract.sdk.adapters.embedding import adapters as embedding_adapters
+from unstract.sdk.adapters.llm import adapters as llm_adapters
+from unstract.sdk.adapters.ocr import adapters as ocr_adapters
+from unstract.sdk.adapters.vectordb import adapters as vectordb_adapters
+from unstract.sdk.adapters.x2text import adapters as x2text_adapters
+
+logger = logging.getLogger(__name__)
+
+
+# Declaring this class as a Singleton to avoid initialising
+# adapters list everytime
+@singleton
+class Adapterkit:
+    def __init__(self) -> None:
+        self._adapters: AdapterDict = (
+            embedding_adapters
+            | llm_adapters
+            | vectordb_adapters
+            | x2text_adapters
+            | ocr_adapters
+        )
+
+    @property
+    def adapters(self) -> AdapterDict:
+        return self._adapters
+
+    def get_adapter_class_by_adapter_id(self, adapter_id: str) -> Adapter:
+        if adapter_id in self._adapters:
+            adapter_class: Adapter = self._adapters[adapter_id][
+                Common.METADATA
+            ][Common.ADAPTER]
+            return adapter_class
+        else:
+            raise RuntimeError(f"Couldn't obtain adapter for {adapter_id}")
+
+    def get_adapter_by_id(
+        self, adapter_id: str, *args: Any, **kwargs: Any
+    ) -> Adapter:
+        """Instantiates and returns a adapter.
+
+        Args:
+            adapter_id (str): Identifies adapter to create
+
+        Raises:
+            RuntimeError: If the ID is invalid/adapter is missing
+
+        Returns:
+            Adapter: Concrete impl of the `Adapter` base
+        """
+        adapter_class: Adapter = self.get_adapter_class_by_adapter_id(
+            adapter_id
+        )
+        return adapter_class(*args, **kwargs)
+
+    def get_adapters_list(self) -> list[dict[str, Any]]:
+        adapters = []
+        for adapter_id, adapter_registry_metadata in self._adapters.items():
+            m: Adapter = adapter_registry_metadata[Common.METADATA][
+                Common.ADAPTER
+            ]
+            _id = m.get_id()
+            name = m.get_name()
+            adapter_type = m.get_adapter_type().name
+            json_schema = m.get_json_schema()
+            desc = m.get_description()
+            icon = m.get_icon()
+            adapters.append(
+                {
+                    "id": _id,
+                    "name": name,
+                    "class_name": m.__name__,
+                    "description": desc,
+                    "icon": icon,
+                    "adapter_type": adapter_type,
+                    "json_schema": json_schema,
+                }
+            )
+        return adapters

src/unstract/sdk/adapters/__init__.py

@@ -0,0 +1,50 @@
+import logging
+from abc import ABC, abstractmethod
+
+from unstract.sdk.adapters.enums import AdapterTypes
+
+logger = logging.getLogger(__name__)
+
+
+class Adapter(ABC):
+    def __init__(self, name: str):
+        self.name = name
+
+    @staticmethod
+    @abstractmethod
+    def get_id() -> str:
+        return ""
+
+    @staticmethod
+    @abstractmethod
+    def get_name() -> str:
+        return ""
+
+    @staticmethod
+    @abstractmethod
+    def get_description() -> str:
+        return ""
+
+    @staticmethod
+    @abstractmethod
+    def get_icon() -> str:
+        return ""
+
+    @staticmethod
+    @abstractmethod
+    def get_json_schema() -> str:
+        return ""
+
+    @staticmethod
+    @abstractmethod
+    def get_adapter_type() -> AdapterTypes:
+        return ""
+
+    @abstractmethod
+    def test_connection(self) -> bool:
+        """Override to test connection for a adapter.
+
+        Returns:
+            bool: Flag indicating if the credentials are valid or not
+        """
+        pass

src/unstract/sdk/adapters/adapterkit.py

@@ -0,0 +1,10 @@
+class Common:
+    METADATA = "metadata"
+    MODULE = "module"
+    ADAPTER = "adapter"
+    SRC_FOLDER = "src"
+    ADAPTER_METADATA = "adapter_metadata"
+    ICON = "icon"
+    ADAPTER_ID = "adapter_id"
+    ADAPTER_TYPE = "adapter_type"
+    DEFAULT_ERR_MESSAGE = "Something went wrong"

src/unstract/sdk/adapters/base.py

@@ -0,0 +1,5 @@
+from unstract.sdk.adapters import AdapterDict
+from unstract.sdk.adapters.embedding.register import EmbeddingRegistry
+
+adapters: AdapterDict = {}
+EmbeddingRegistry.register_adapters(adapters)