✨ Abstract Storage SDK #1219

Author: Phinease

backend/database/attachment_db.py

@@ -165,7 +165,7 @@ def get_file_size_from_minio(object_name: str, bucket: Optional[str] = None) ->
     """
     Get file size by object name
     """
-    bucket = bucket or minio_client.default_bucket
+    bucket = bucket or minio_client.storage_config.default_bucket
     return minio_client.get_file_size(object_name, bucket)
 
 
@@ -207,7 +207,7 @@ def delete_file(object_name: str, bucket: Optional[str] = None) -> Dict[str, Any
         Dict[str, Any]: Delete result, containing success flag and error message (if any)
     """
     if not bucket:
-        bucket = minio_client.default_bucket
+        bucket = minio_client.storage_config.default_bucket
     success, result = minio_client.delete_file(object_name, bucket)
 
     response = {"success": success, "object_name": object_name}

backend/database/client.py

@@ -1,12 +1,8 @@
 import logging
-import os
 from contextlib import contextmanager
 from typing import Any, BinaryIO, Dict, List, Optional, Tuple
 
-import boto3
 import psycopg2
-from botocore.client import Config
-from botocore.exceptions import ClientError
 from sqlalchemy import create_engine
 from sqlalchemy.orm import class_mapper, sessionmaker
 
@@ -23,6 +19,8 @@
     POSTGRES_USER,
 )
 from database.db_models import TableBase
+from nexent.storage.storage_client_factory import create_storage_client_from_config, MinIOStorageConfig
+
 
 logger = logging.getLogger("database.client")
 
@@ -73,6 +71,12 @@ def clean_string_values(data: Dict[str, Any]) -> Dict[str, Any]:
 
 
 class MinioClient:
+    """
+    MinIO client wrapper using storage SDK
+    
+    This class maintains backward compatibility with the existing MinioClient interface
+    while using the new storage SDK under the hood.
+    """
     _instance: Optional['MinioClient'] = None
 
     def __new__(cls):
@@ -81,39 +85,18 @@ def __new__(cls):
         return cls._instance
 
     def __init__(self):
-        self.endpoint = MINIO_ENDPOINT
-        self.access_key = MINIO_ACCESS_KEY
-        self.secret_key = MINIO_SECRET_KEY
-        self.region = MINIO_REGION
-        self.default_bucket = MINIO_DEFAULT_BUCKET
-
-        # Initialize S3 client with proxy settings
-        self.client = boto3.client(
-            's3',
-            endpoint_url=self.endpoint,
-            aws_access_key_id=self.access_key,
-            aws_secret_access_key=self.secret_key,
-            region_name=self.region,
-            config=Config(
-                signature_version='s3v4',
-                proxies={
-                    'http': None,
-                    'https': None
-                }
-            )
+        # Determine if endpoint uses HTTPS
+        secure = MINIO_ENDPOINT.startswith('https://') if MINIO_ENDPOINT else True
+        # Initialize storage client using SDK factory
+        self.storage_config = MinIOStorageConfig(
+            endpoint=MINIO_ENDPOINT,
+            access_key=MINIO_ACCESS_KEY,
+            secret_key=MINIO_SECRET_KEY,
+            region=MINIO_REGION,
+            default_bucket=MINIO_DEFAULT_BUCKET,
+            secure=secure
         )
-
-        # Ensure default bucket exists
-        self._ensure_bucket_exists(self.default_bucket)
-
-    def _ensure_bucket_exists(self, bucket_name: str) -> None:
-        """Ensure bucket exists, create if it doesn't"""
-        try:
-            self.client.head_bucket(Bucket=bucket_name)
-        except ClientError:
-            # Bucket doesn't exist, create it
-            self.client.create_bucket(Bucket=bucket_name)
-            logger.info(f"Created bucket: {bucket_name}")
+        self._storage_client = create_storage_client_from_config(self.storage_config)
 
     def upload_file(
             self,
@@ -132,16 +115,7 @@ def upload_file(
         Returns:
             Tuple[bool, str]: (Success status, File URL or error message)
         """
-        bucket = bucket or self.default_bucket
-        if object_name is None:
-            object_name = os.path.basename(file_path)
-
-        try:
-            self.client.upload_file(file_path, bucket, object_name)
-            file_url = f"/{bucket}/{object_name}"
-            return True, file_url
-        except Exception as e:
-            return False, str(e)
+        return self._storage_client.upload_file(file_path, object_name, bucket)
 
     def upload_fileobj(self, file_obj: BinaryIO, object_name: str, bucket: Optional[str] = None) -> Tuple[bool, str]:
         """
@@ -155,13 +129,7 @@ def upload_fileobj(self, file_obj: BinaryIO, object_name: str, bucket: Optional[
         Returns:
             Tuple[bool, str]: (Success status, File URL or error message)
         """
-        bucket = bucket or self.default_bucket
-        try:
-            self.client.upload_fileobj(file_obj, bucket, object_name)
-            file_url = f"/{bucket}/{object_name}"
-            return True, file_url
-        except Exception as e:
-            return False, str(e)
+        return self._storage_client.upload_fileobj(file_obj, object_name, bucket)
 
     def download_file(self, object_name: str, file_path: str, bucket: Optional[str] = None) -> Tuple[bool, str]:
         """
@@ -175,12 +143,7 @@ def download_file(self, object_name: str, file_path: str, bucket: Optional[str]
         Returns:
             Tuple[bool, str]: (Success status, Success message or error message)
         """
-        bucket = bucket or self.default_bucket
-        try:
-            self.client.download_file(bucket, object_name, file_path)
-            return True, f"File downloaded successfully to {file_path}"
-        except Exception as e:
-            return False, str(e)
+        return self._storage_client.download_file(object_name, file_path, bucket)
 
     def get_file_url(self, object_name: str, bucket: Optional[str] = None, expires: int = 3600) -> Tuple[bool, str]:
         """
@@ -194,23 +157,20 @@ def get_file_url(self, object_name: str, bucket: Optional[str] = None, expires:
         Returns:
             Tuple[bool, str]: (Success status, Presigned URL or error message)
         """
-        bucket = bucket or self.default_bucket
-        try:
-            url = self.client.generate_presigned_url('get_object', Params={'Bucket': bucket, 'Key': object_name},
-                                                     ExpiresIn=expires)
-            return True, url
-        except Exception as e:
-            return False, str(e)
+        return self._storage_client.get_file_url(object_name, bucket, expires)
 
     def get_file_size(self, object_name: str, bucket: Optional[str] = None) -> int:
-        bucket = bucket or self.default_bucket
-        try:
-            response = self.client.head_object(Bucket=bucket, Key=object_name)
-            return int(response['ContentLength'])
-        except ClientError as e:
-            logger.error(
-                f"Get file size by objectname({object_name}) failed: {e}")
-            return 0
+        """
+        Get file size in bytes
+
+        Args:
+            object_name: Object name
+            bucket: Bucket name, if not specified use default bucket
+
+        Returns:
+            int: File size in bytes, 0 if file not found or error
+        """
+        return self._storage_client.get_file_size(object_name, bucket)
 
     def list_files(self, prefix: str = "", bucket: Optional[str] = None) -> List[dict]:
         """
@@ -223,19 +183,7 @@ def list_files(self, prefix: str = "", bucket: Optional[str] = None) -> List[dic
         Returns:
             List[dict]: List of file information
         """
-        bucket = bucket or self.default_bucket
-        try:
-            response = self.client.list_objects_v2(
-                Bucket=bucket, Prefix=prefix)
-            files = []
-            if 'Contents' in response:
-                for obj in response['Contents']:
-                    files.append(
-                        {'key': obj['Key'], 'size': obj['Size'], 'last_modified': obj['LastModified']})
-            return files
-        except Exception as e:
-            logger.error(f"Error listing files: {str(e)}")
-            return []
+        return self._storage_client.list_files(prefix, bucket)
 
     def delete_file(self, object_name: str, bucket: Optional[str] = None) -> Tuple[bool, str]:
         """
@@ -248,12 +196,7 @@ def delete_file(self, object_name: str, bucket: Optional[str] = None) -> Tuple[b
         Returns:
             Tuple[bool, str]: (Success status, Success message or error message)
         """
-        bucket = bucket or self.default_bucket
-        try:
-            self.client.delete_object(Bucket=bucket, Key=object_name)
-            return True, f"File {object_name} deleted successfully"
-        except Exception as e:
-            return False, str(e)
+        return self._storage_client.delete_file(object_name, bucket)
 
     def get_file_stream(self, object_name: str, bucket: Optional[str] = None) -> Tuple[bool, Any]:
         """
@@ -266,12 +209,7 @@ def get_file_stream(self, object_name: str, bucket: Optional[str] = None) -> Tup
         Returns:
             Tuple[bool, Any]: (Success status, File stream object or error message)
         """
-        bucket = bucket or self.default_bucket
-        try:
-            response = self.client.get_object(Bucket=bucket, Key=object_name)
-            return True, response['Body']
-        except Exception as e:
-            return False, str(e)
+        return self._storage_client.get_file_stream(object_name, bucket)
 
 
 # Create global database and MinIO client instances

sdk/nexent/__init__.py

@@ -1,7 +1,8 @@
 from .core import *
 from .data_process import *
 from .memory import *
+from .storage import *
 from .vector_database import *
 
 
-__all__ = ["core", "data_process", "memory", "vector_database"]
+__all__ = ["core", "data_process", "memory", "storage", "vector_database"]

sdk/nexent/storage/__init__.py

@@ -0,0 +1,19 @@
+"""
+Storage module for Nexent SDK
+
+Provides abstract storage interface and implementations for various storage backends.
+"""
+
+from .storage_client_base import StorageClient, StorageConfig
+from .storage_client_factory import create_storage_client_from_config
+from .minio_config import MinIOStorageConfig
+from .minio import MinIOStorageClient
+
+__all__ = [
+    "StorageClient",
+    "StorageConfig",
+    "MinIOStorageConfig",
+    "create_storage_client_from_config",
+    "MinIOStorageClient",
+]
+