Refactor Data directory structure (#252)

Author: vegito22

llmstack/apps/tasks.py

@@ -2,13 +2,9 @@
 
 import weaviate
 
-from llmstack.datasources.handlers.datasource_processor import DataSourceProcessor
-from llmstack.datasources.models import (
-    DataSource,
-    DataSourceEntry,
-    DataSourceEntryStatus,
-)
-from llmstack.datasources.types import DataSourceTypeFactory
+from llmstack.data.datasource_processor import DataSourceProcessor
+from llmstack.data.models import DataSource, DataSourceEntry, DataSourceEntryStatus
+from llmstack.data.types import DataSourceTypeFactory
 
 logger = logging.getLogger(__name__)
 

llmstack/datasources/__init__.py renamed to llmstack/data/__init__.py

@@ -19,11 +19,8 @@
     delete_data_source_task,
     resync_data_entry_task,
 )
-from llmstack.datasources.handlers.datasource_processor import (
-    DataSourceEntryItem,
-    DataSourceProcessor,
-)
-from llmstack.datasources.types import (
+from llmstack.data.datasource_processor import DataSourceEntryItem, DataSourceProcessor
+from llmstack.data.types import (
     DataSourceTypeFactory,
     get_data_source_type_interface_subclasses,
 )
@@ -438,7 +435,7 @@ def add_entry_async(self, request, uid):
 
         adhoc_job = AdhocJob(
             name=f"add_entry_{datasource.uuid}",
-            callable="llmstack.datasources.tasks.process_datasource_add_entry_request",
+            callable="llmstack.data.tasks.process_datasource_add_entry_request",
             callable_args=[
                 uid,
                 request.data["entry_data"],

llmstack/datasources/admin.py renamed to llmstack/data/admin.py

@@ -3,5 +3,5 @@
 
 class DatasourcesConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
-    name = "llmstack.datasources"
+    name = "llmstack.data"
     label = "datasources"

llmstack/datasources/apis.py renamed to llmstack/data/apis.py

@@ -17,8 +17,10 @@
 from llmstack.common.blocks.base.processor import BaseInputType, ProcessorInterface
 from llmstack.common.blocks.base.schema import BaseSchema as _Schema
 from llmstack.common.blocks.data.store.vectorstore import Document
-from llmstack.datasources.models import DataSource
-from llmstack.datasources.vector_stores.types import get_vector_store_configuration
+from llmstack.data.destinations.vector_stores.types import (
+    get_vector_store_configuration,
+)
+from llmstack.data.models import DataSource
 
 logger = logging.getLogger(__name__)
 

llmstack/datasources/apps.py renamed to llmstack/data/apps.py

@@ -3,7 +3,7 @@
 import chromadb
 from llama_index.vector_stores.chroma import ChromaVectorStore
 
-from llmstack.datasources.vector_stores.base import VectorStoreConfiguration
+from llmstack.data.destinations.vector_stores.base import VectorStoreConfiguration
 
 
 class PromptlyChromaVectorStore(ChromaVectorStore):