Merge branch 'main' into issue-18

Author: ZoranPandovski

.github/workflows/cla.yml

@@ -0,0 +1,22 @@
+name: "MindsDB CLA Assistant"
+on:
+  issue_comment:
+    types: [created]
+  pull_request_target:
+    types: [opened,closed,synchronize]    
+jobs:
+  CLAssistant:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "CLA Assistant"
+        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
+        uses: contributor-assistant/[email protected]
+        env: 
+          GITHUB_TOKEN: ${{ secrets.CLA_TOKEN }}          
+          PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_TOKEN }}
+        with: 
+          path-to-signatures: 'assets/contributions-agreement/cla.json'
+          # Add path to the CLA here
+          path-to-document: 'https://github.com/mindsdb/mindsdb/blob/main/assets/contributions-agreement/individual-contributor.md'
+          branch: 'cla'
+          allowlist: bot*, ZoranPandovski, torrmal, Stpmax, mindsdbadmin, ea-rus, tmichaeldb, dusvyat, hamishfagg, MinuraPunchihewa, martyna-mindsdb, lucas-koontz

.github/workflows/test_on_deploy.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.8', '3.9','3.10', '3.11']
+        python-version: ['3.10']
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -28,4 +28,4 @@ jobs:
         env:
           PYTHONPATH: ./
           API_KEY: ${{ secrets.API_KEY }}
-          BASE_URL: ${{ secrets.BASE_URL }}
+          BASE_URL: 'https://mdb.ai'

README.md

@@ -149,6 +149,12 @@ client.datasources.drop('my_datasource')
 ```
 >Note: The SDK currently does not support automatically removing a data source if it is no longer connected to any mind.
 
-
 ### Other SDKs
-#### [Javascript](https://github.com/scshiv29-dev/mindsbd_sdk_js)
+- [Ruby-SDK](https://github.com/tungnt1203/minds_ruby_sdk)
+- [Dart-SDK](https://github.com/ArnavK-09/mdb_dart)
+- [Javascript](https://github.com/scshiv29-dev/mindsbd_sdk_js)
+
+#### Command Line Tools
+- [Minds CLI](https://github.com/Better-Boy/minds-cli-sdk)
+
+

assets/contributions-agreement/cla.json

@@ -1,6 +1,6 @@
 __title__ = 'minds_sdk'
 __package_name__ = 'minds'
-__version__ = '1.0.7'
+__version__ = '1.2.0'
 __description__ = 'An AI-Data Mind is an LLM with the built-in power to answer data questions for Agents'
 __email__ = '[email protected]'
 __author__ = 'MindsDB Inc'

minds/__about__.py

@@ -2,6 +2,7 @@
 from minds.rest_api import RestAPI
 
 from minds.datasources import Datasources
+from minds.knowledge_bases import KnowledgeBases
 from minds.minds import Minds
 
 
@@ -12,5 +13,6 @@ def __init__(self, api_key, base_url=None):
         self.api = RestAPI(api_key, base_url)
 
         self.datasources = Datasources(self)
+        self.knowledge_bases = KnowledgeBases(self)
 
         self.minds = Minds(self)

minds/client.py

@@ -1,7 +1,7 @@
 from typing import List, Optional, Union
 
 from pydantic import BaseModel, Field
-
+import minds.utils as utils
 import minds.exceptions as exc
 
 class DatabaseConfig(BaseModel):
@@ -16,11 +16,12 @@ class DatabaseConfig(BaseModel):
 class Datasource(DatabaseConfig):
     ...
 
+
 class Datasources:
     def __init__(self, client):
         self.api = client.api
 
-    def create(self, ds_config: DatabaseConfig, replace=False):
+    def create(self, ds_config: DatabaseConfig, update=False):
         """
         Create new datasource and return it
 
@@ -30,19 +31,18 @@ def create(self, ds_config: DatabaseConfig, replace=False):
            - description: str, description of the database. Used by mind to know what data can be got from it.
            - connection_data: dict, optional, credentials to connect to database
            - tables: list of str, optional, list of allowed tables
+        :param update: if true - to update datasourse if exists, default is false
         :return: datasource object
         """
 
         name = ds_config.name
 
-        if replace:
-            try:
-                self.get(name)
-                self.drop(name)
-            except exc.ObjectNotFound:
-                ...
+        utils.validate_datasource_name(name)
 
-        self.api.post('/datasources', data=ds_config.model_dump())
+        if update:
+            self.api.put(f'/datasources/{name}', data=ds_config.model_dump())
+        else:
+            self.api.post('/datasources', data=ds_config.model_dump())
         return self.get(name)
 
     def list(self) -> List[Datasource]:
@@ -76,11 +76,15 @@ def get(self, name: str) -> Datasource:
             raise exc.ObjectNotSupported(f'Wrong type of datasource: {name}')
         return Datasource(**data)
 
-    def drop(self, name: str):
+    def drop(self, name: str, force=False):
         """
         Drop datasource by name
 
         :param name: name of datasource
+        :param force: if True - remove from all minds, default: False
         """
+        data = None
+        if force:
+            data = {'cascade': True}
 
-        self.api.delete(f'/datasources/{name}')
+        self.api.delete(f'/datasources/{name}', data=data)

minds/datasources/datasources.py

@@ -18,3 +18,10 @@ class Unauthorized(Exception):
 class UnknownError(Exception):
     ...
 
+
+class MindNameInvalid(Exception):
+    ...
+
+
+class DatasourceNameInvalid(Exception):
+    ...

minds/exceptions.py

@@ -0,0 +1 @@
+from .knowledge_bases import *

minds/knowledge_bases/__init__.py

@@ -0,0 +1,175 @@
+from typing import Any, Dict, List, Optional, Union
+
+from pydantic import BaseModel
+
+from minds.knowledge_bases.preprocessing import PreprocessingConfig
+from minds.rest_api import RestAPI
+
+
+class VectorStoreConfig(BaseModel):
+    '''Configuration for the underlying vector store for knowledge base embeddings'''
+    engine: str
+    connection_data: Dict[str, Any]
+    table: str = 'embeddings'
+
+
+class EmbeddingConfig(BaseModel):
+    '''Configuration for embeddings to use with underlying vector store for knowledge base'''
+    provider: str
+    model: str
+    params: Optional[Dict[str, Any]] = None
+
+
+class KnowledgeBaseConfig(BaseModel):
+    '''Configuration for a knowledge base'''
+    name: str
+    description: str
+    vector_store_config: Optional[VectorStoreConfig] = None
+    embedding_config: Optional[EmbeddingConfig] = None
+    # Params to apply to retrieval pipeline.
+    params: Optional[Dict] = None
+
+
+class KnowledgeBaseDocument(BaseModel):
+    '''Represents a document that can be inserted into a knowledge base'''
+    id: Union[int, str]
+    content: str
+    metadata: Optional[Dict[str, Any]] = {}
+
+
+class KnowledgeBase:
+    def __init__(self, name, api: RestAPI):
+        self.name = name
+        self.api = api
+
+    def insert_from_select(self, query: str, preprocessing_config: PreprocessingConfig = None):
+        '''
+        Inserts select content of a connected datasource into this knowledge base
+
+        :param query: The SQL SELECT query to use to retrieve content to be inserted
+        '''
+        update_request = {
+            'query': query
+        }
+        if preprocessing_config is not None:
+            update_request['preprocessing'] = preprocessing_config.model_dump()
+        _ = self.api.put(f'/knowledge_bases/{self.name}', data=update_request)
+
+    def insert_documents(self, documents: List[KnowledgeBaseDocument], preprocessing_config: PreprocessingConfig = None):
+        '''
+        Inserts documents directly into this knowledge base
+
+        :param documents: The documents to insert
+        '''
+        update_request = {
+            'rows': [d.model_dump() for d in documents]
+        }
+        if preprocessing_config is not None:
+            update_request['preprocessing'] = preprocessing_config.model_dump()
+        _ = self.api.put(f'/knowledge_bases/{self.name}', data=update_request)
+
+    def insert_urls(self, urls: List[str], preprocessing_config: PreprocessingConfig = None):
+        '''
+        Crawls URLs & inserts the retrieved webpages into this knowledge base
+
+        :param urls: Valid URLs to crawl & insert
+        '''
+        update_request = {
+            'urls': urls
+        }
+        if preprocessing_config is not None:
+            update_request['preprocessing'] = preprocessing_config.model_dump()
+        _ = self.api.put(f'/knowledge_bases/{self.name}', data=update_request)
+
+    def insert_files(self, files: List[str], preprocessing_config: PreprocessingConfig = None):
+        '''
+        Inserts files that have already been uploaded to MindsDB into this knowledge base
+
+        :param files: Names of preuploaded files to insert
+        '''
+        update_request = {
+            'files': files
+        }
+        if preprocessing_config is not None:
+            update_request['preprocessing'] = preprocessing_config.model_dump()
+        _ = self.api.put(f'/knowledge_bases/{self.name}', data=update_request)
+
+
+class KnowledgeBases:
+    def __init__(self, client):
+        self.api = client.api
+
+    def create(self, config: KnowledgeBaseConfig) -> KnowledgeBase:
+        '''
+        Create new knowledge base and return it
+
+        :param config: knowledge base configuration, properties:
+           - name: str, name of knowledge base
+           - description: str, description of the knowledge base. Used by minds to know what data can be retrieved.
+           - vector_store_config: VectorStoreConfig, configuration for embeddings vector store.
+           - embedding_config: EmbeddingConfig, configuration for embeddings.
+        :return: knowledge base object
+        '''
+        create_request = {
+            'name': config.name,
+            'description': config.description
+        }
+        if config.vector_store_config is not None:
+            vector_store_data = {
+                'engine': config.vector_store_config.engine,
+                'connection_data': config.vector_store_config.connection_data
+            }
+            create_request['vector_store'] = vector_store_data
+        if config.embedding_config is not None:
+            embedding_data = {
+                'provider': config.embedding_config.provider,
+                'name': config.embedding_config.model
+            }
+            if config.embedding_config.params is not None:
+                embedding_data.update(config.embedding_config.params)
+            create_request['embedding_model'] = embedding_data
+        if config.params is not None:
+            create_request['params'] = config.params
+
+        _ = self.api.post('/knowledge_bases', data=create_request)
+        return self.get(config.name)
+
+    def list(self) -> List[KnowledgeBase]:
+        '''
+        Returns list of knowledge bases
+
+        :return: iterable knowledge bases
+        '''
+
+        list_knowledge_bases_response = self.api.get('/knowledge_bases')
+        knowledge_bases = list_knowledge_bases_response.json()
+
+        all_knowledge_bases = []
+        for knowledge_base in knowledge_bases:
+            all_knowledge_bases.append(KnowledgeBase(knowledge_base['name'], self.api))
+        return all_knowledge_bases
+
+    def get(self, name: str) -> KnowledgeBase:
+        '''
+        Get knowledge base by name
+
+        :param name: name of knowledge base
+        :return: knowledge base object
+        '''
+
+        knowledge_base_response = self.api.get(f'/knowledge_bases/{name}')
+        knowledge_base = knowledge_base_response.json()
+        return KnowledgeBase(knowledge_base['name'], self.api)
+
+    def drop(self, name: str, force=False):
+        '''
+        Drop knowledge base by name
+
+        :param name: name of knowledge base
+        :param force: if True - remove from all minds, default: False
+        '''
+        data = None
+        if force:
+            data = {'cascade': True}
+
+        self.api.delete(f'/knowledge_bases/{name}', data=data)