Merge pull request #238 from lvalics/main

MAJOR UPGRADE - LangChain/OpenAI libraries.

Author: lvalics

dj_backend_server/CHANGELOG.MD

@@ -1,3 +1,11 @@
+2.17.2024
+- Incorporate 'Ollama' into your example.env configuration and make sure to reflect these changes in your .env file for compatibility.
+- We've expanded the logging capabilities within settings.py by deploying logging.debug for more detailed insights, although it remains inactive when the DEBUG mode is off.
+- We've refreshed the dependencies listed in requirements.txt to ensure the latest support and functionality.
+- Updates to LangChain and OpenAI libraries, now feature the exciting addition of Ollama, currently under development. 
+- This version has a significant update that is still pending comprehensive testing; therefore, we advise against deploying it in production environments until it is fully vetted.
+- Modifications have been made to the history module to append the input message to requests sent to OpenAI. These changes are still being tested. (disabled for the moemnt)
+
 2.14.2024
 - Added example.env to streamline environment setup.
 - Implemented translation fixes to enhance application localization.

dj_backend_server/api/data_sources/codebase_handler.py

@@ -3,7 +3,7 @@
 from django.views.decorators.csrf import csrf_exempt
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from api.utils import get_embeddings
-from langchain.document_loaders import GitLoader
+from langchain_community.document_loaders import GitLoader
 from api.utils import init_vector_store
 from api.interfaces import StoreOptions
 

dj_backend_server/api/data_sources/pdf_handler.py

@@ -13,9 +13,9 @@
 from django.utils import timezone
 from django.shortcuts import get_object_or_404
 from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.document_loaders.directory import DirectoryLoader
-from langchain.document_loaders import PyPDFium2Loader
-from langchain.document_loaders import TextLoader
+from langchain_community.document_loaders.directory import DirectoryLoader
+from langchain_community.document_loaders import PyPDFium2Loader
+from langchain_community.document_loaders import TextLoader
 from api.utils import get_embeddings
 from api.utils import init_vector_store
 from api.interfaces import StoreOptions

dj_backend_server/api/data_sources/website_handler.py

@@ -1,9 +1,9 @@
 import os
 from django.http import JsonResponse
 import traceback
-from langchain.document_loaders.directory import DirectoryLoader
+from langchain_community.document_loaders.directory import DirectoryLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.document_loaders import TextLoader
+from langchain_community.document_loaders import TextLoader
 from api.utils import init_vector_store
 from api.utils.get_embeddings import get_embeddings
 from api.interfaces import StoreOptions

dj_backend_server/api/enums/embedding_type.py

@@ -6,4 +6,5 @@ class EmbeddingProvider(Enum):
     BARD = "bard"
     azure = "azure"
     llama2 = "llama2"
+    ollama = "ollama"
 

dj_backend_server/api/utils/custom_pdf_loader.py

@@ -1,7 +1,7 @@
 import io
 from langchain.docstore.base import Document
-from langchain.document_loaders.base import BaseLoader
-from langchain.document_loaders import PyPDFLoader
+from langchain_community.document_loaders.base import BaseLoader
+from langchain_community.document_loaders import PyPDFLoader
 
 class BufferLoader(BaseLoader):
     def __init__(self, filePathOrBlob):

dj_backend_server/api/utils/get_embeddings.py

@@ -1,18 +1,18 @@
-from langchain.embeddings.openai import OpenAIEmbeddings
-from api.enums import EmbeddingProvider
 import os
 from dotenv import load_dotenv
 from langchain.embeddings.base import Embeddings
-from langchain.embeddings import LlamaCppEmbeddings
+from langchain_community.embeddings import LlamaCppEmbeddings
+from langchain_openai import OpenAIEmbeddings
+from api.enums import EmbeddingProvider
 
 load_dotenv()
-import os
 
 
 def get_embedding_provider():
     """Gets the chosen embedding provider from environment variables."""
     return os.environ.get("EMBEDDING_PROVIDER")
 
+
 def get_azure_embedding():
     """Gets embeddings using the Azure embedding provider."""
     deployment = os.environ.get("AZURE_OPENAI_EMBEDDING_MODEL_NAME")
@@ -30,16 +30,18 @@ def get_azure_embedding():
         openai_api_version=openai_api_version
     )
 
+
 def get_openai_embedding():
     """Gets embeddings using the OpenAI embedding provider."""
     openai_api_key = os.environ.get("OPENAI_API_KEY")
+    return OpenAIEmbeddings(openai_api_key=openai_api_key,  chunk_size=1)
 
-    return OpenAIEmbeddings(openai_api_key=openai_api_key, chunk_size=1)
 
 def get_llama2_embedding():
     """Gets embeddings using the llama2 embedding provider."""
     return LlamaCppEmbeddings(model_path="llama-2-7b-chat.ggmlv3.q4_K_M.bin")
 
+
 def choose_embedding_provider():
     """Chooses and returns the appropriate embedding provider instance."""
     embedding_provider = get_embedding_provider()
@@ -52,7 +54,6 @@ def choose_embedding_provider():
 
     elif embedding_provider == EmbeddingProvider.llama2.value:
         return get_llama2_embedding()
-    
 
     else:
         available_providers = ", ".join([service.value for service in EmbeddingProvider])
@@ -61,6 +62,7 @@ def choose_embedding_provider():
             f"Available services: {available_providers}"
         )
 
+
 # Main function to get embeddings
 def get_embeddings() -> Embeddings:
     """Gets embeddings using the chosen embedding provider."""

dj_backend_server/api/utils/get_openai_llm.py

@@ -1,21 +1,33 @@
-from langchain.llms import AzureOpenAI, OpenAI
 import os
 from dotenv import load_dotenv
-from langchain.llms import LlamaCpp
-load_dotenv()
-from langchain import PromptTemplate, LLMChain
+import logging.config
+import traceback
+from django.utils.timezone import make_aware
+from datetime import datetime, timezone
+from uuid import uuid4
+from ollama import Client
+from openai import OpenAI
+from django.conf import settings
+from langchain_openai.chat_models import ChatOpenAI
+from langchain_community.llms import Ollama
+from langchain_community.llms import AzureOpenAI
+from langchain_community.llms import LlamaCpp
+from langchain.prompts import PromptTemplate
+from langchain.chains import LLMChain
 from langchain.callbacks.manager import CallbackManager
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
-import traceback
 from web.models.failed_jobs import FailedJob
-from datetime import datetime
-from uuid import uuid4
+
+load_dotenv()
+logging.config.dictConfig(settings.LOGGING)
+logger = logging.getLogger(__name__)
+
 
 def get_llama_llm():
     try:
         n_gpu_layers = 1  # Metal set to 1 is enough.
         n_batch = 512  # Should be between 1 and n_ctx, consider the amount of RAM of your Apple Silicon Chip.
-        
+
         # Callbacks support token-wise streaming
         callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
         llm = LlamaCpp(
@@ -28,24 +40,44 @@ def get_llama_llm():
             verbose=True,
             temperature=0.2,
         )
-        
+
         return llm
     except Exception as e:
-        failed_job = FailedJob(uuid=str(uuid4()), connection='default', queue='default', payload='get_llama_llm', exception=str(e), failed_at=datetime.now())
+
+        logger.debug(f"Exception in get_llama_llm: {e}")
+        failed_job = FailedJob(
+            uuid=str(uuid4()),
+            connection="default",
+            queue="default",
+            payload="get_llama_llm",
+            exception=str(e),
+            failed_at=make_aware(datetime.now(), timezone.utc),
+        )
         failed_job.save()
         print(f"Exception occurred: {e}")
         traceback.print_exc()
 
+
 # Azure OpenAI Language Model client
 def get_azure_openai_llm():
     """Returns AzureOpenAI instance configured from environment variables"""
     try:
-        openai_api_type = os.environ['OPENAI_API_TYPE']
-        openai_api_key = os.environ['AZURE_OPENAI_API_KEY']
-        openai_deployment_name = os.environ['AZURE_OPENAI_DEPLOYMENT_NAME']
-        openai_model_name = os.environ['AZURE_OPENAI_COMPLETION_MODEL']
-        openai_api_version = os.environ['AZURE_OPENAI_API_VERSION']
-        openai_api_base=os.environ['AZURE_OPENAI_API_BASE']
+        if settings.DEBUG:
+            openai_api_type = "openai"  # JUST FOR DEVELOPMENT
+            logging.debug(f"DEVELOPMENT Using API Type: {openai_api_type}")
+        else:
+            openai_api_type = os.environ["AZURE_OPENAI_API_TYPE"]
+
+        openai_api_key = os.environ["AZURE_OPENAI_API_KEY"]
+        openai_deployment_name = os.environ["AZURE_OPENAI_DEPLOYMENT_NAME"]
+        openai_model_name = os.environ["AZURE_OPENAI_COMPLETION_MODEL"]
+        openai_api_version = os.environ["AZURE_OPENAI_API_VERSION"]
+        openai_api_base = os.environ["AZURE_OPENAI_API_BASE"]
+        openai_api_key = os.environ["AZURE_OPENAI_API_KEY"]
+        openai_deployment_name = os.environ["AZURE_OPENAI_DEPLOYMENT_NAME"]
+        openai_model_name = os.environ["AZURE_OPENAI_COMPLETION_MODEL"]
+        openai_api_version = os.environ["AZURE_OPENAI_API_VERSION"]
+        openai_api_base = os.environ["AZURE_OPENAI_API_BASE"]
         return AzureOpenAI(
             openai_api_base=openai_api_base,
             openai_api_key=openai_api_key,
@@ -54,51 +86,127 @@ def get_azure_openai_llm():
             openai_api_type=openai_api_type,
             openai_api_version=openai_api_version,
             temperature=0,
-            batch_size=8
+            batch_size=8,
         )
     except Exception as e:
-        failed_job = FailedJob(uuid=str(uuid4()), connection='default', queue='default', payload='get_azure_openai_llm', exception=str(e), failed_at=datetime.now())
+        logger.debug(f"Exception in get_azure_openai_llm: {e}")
+        failed_job = FailedJob(
+            uuid=str(uuid4()),
+            connection="default",
+            queue="default",
+            payload="get_azure_openai_llm",
+            exception=str(e),
+            failed_at=make_aware(datetime.now(), timezone.utc),
+        )
         failed_job.save()
         print(f"Exception occurred: {e}")
         traceback.print_exc()
 
-# OpenAI Language Model client  
+
+# OpenAI Language Model client
 def get_openai_llm():
     """Returns OpenAI instance configured from environment variables"""
     try:
-        openai_api_key = os.environ['OPENAI_API_KEY']
+        openai_api_key = os.environ.get("OPENAI_API_KEY")
+        temperature = os.environ.get("OPENAI_API_TEMPERATURE")
+        model = os.environ.get("OPENAI_API_MODEL", "gpt-3.5-turbo")
 
-        return OpenAI(
-            temperature=float(os.environ.get('OPENAI_API_TEMPERATURE', '0')),
+        logging.debug(
+            f"We are in get_openai_llm: {openai_api_key} {temperature} {model}"
+        )
+        return ChatOpenAI(
+            temperature=temperature,
             openai_api_key=openai_api_key,
-            model_name=os.environ.get('OPENAI_API_MODEL', 'gpt-3.5-turbo'),
+            model=model,
         )
     except Exception as e:
-        failed_job = FailedJob(uuid=str(uuid4()), connection='default', queue='default', payload='get_openai_llm', exception=str(e), failed_at=datetime.now())
+        logger.debug(f"Exception in get_openai_llm: {e}")
+        failed_job = FailedJob(
+            uuid=str(uuid4()),
+            connection="default",
+            queue="default",
+            payload="get_openai_llm",
+            exception=str(e),
+            failed_at=make_aware(datetime.now(), timezone.utc),
+        )
+        failed_job.save()
+        print(f"Exception occurred: {e}")
+        traceback.print_exc()
+
+
+def get_ollama_llm(sanitized_question):
+    """Returns an Ollama Server instance configured from environment variables"""
+    llm = Client(host=os.environ.get("OLLAMA_URL"))
+    # Use the client to make a request
+    try:
+        if sanitized_question:
+            response = llm.chat(
+                model=os.environ.get("OLLAMA_MODEL_NAME"),
+                messages=[{"role": "user", "content": sanitized_question}],
+            )
+        else:
+            raise ValueError("Question cannot be None.")
+        if response:
+            return response
+        else:
+            raise ValueError("Invalid response from Ollama.")
+
+    except Exception as e:
+        logger.debug(f"Exception in get_ollama_llm: {e}")
+        failed_job = FailedJob(
+            uuid=str(uuid4()),
+            connection="default",
+            queue="default",
+            payload="get_openai_llm",
+            exception=str(e),
+            failed_at=make_aware(datetime.now(), timezone.utc),
+        )
         failed_job.save()
         print(f"Exception occurred: {e}")
         traceback.print_exc()
 
-        
-# recommend not caching initially, and optimizing only if you observe a clear performance benefit from caching the clients. 
-# The simplest thing that works is often best to start.
 
 def get_llm():
     """Returns LLM client instance based on OPENAI_API_TYPE"""
     try:
         clients = {
-            'azure': get_azure_openai_llm,
-            'openai': get_openai_llm,
-            'llama2': get_llama_llm
+            "azure": get_azure_openai_llm,
+            "openai": get_openai_llm,
+            "llama2": get_llama_llm,
+            "ollama": lambda: get_ollama_llm(),
         }
-        
-        api_type = os.environ.get('OPENAI_API_TYPE')
+
+        api_type = os.environ.get("OPENAI_API_TYPE", "openai")
         if api_type not in clients:
             raise ValueError(f"Invalid OPENAI_API_TYPE: {api_type}")
-        
-        return clients[api_type]()
+
+        logging.debug(f"Using LLM: {api_type}")
+
+        if api_type in clients:
+            if api_type == "ollama":
+                return clients[api_type]()
+            elif api_type != "ollama":
+                return clients[api_type]()
+        else:
+            raise ValueError(f"Invalid OPENAI_API_TYPE: {api_type}")
+
     except Exception as e:
-        failed_job = FailedJob(uuid=str(uuid4()), connection='default', queue='default', payload='get_llm', exception=str(e), failed_at=datetime.now())
-        failed_job.save()
-        print(f"Exception occurred: {e}")
-        traceback.print_exc()
+        failed_job = FailedJob(
+            uuid=str(uuid4()),
+            connection="default",
+            queue="default",
+            payload="get_llm",
+            exception=str(e),
+            failed_at=datetime.now(),
+        )
+        failed_job = FailedJob(
+            uuid=str(uuid4()),
+            connection="default",
+            queue="default",
+            payload="get_llm",
+            exception=str(e),
+            failed_at=make_aware(datetime.now(), timezone.utc),
+        )
+        failed_job.save()  # Ensure datetime is timezone-aware
+        print(f"Exception occurred in get_llm: {e}")
+        traceback.print_exc()

dj_backend_server/api/utils/init_vector_store.py

@@ -1,6 +1,6 @@
 from langchain.docstore.document import Document
 from langchain.vectorstores.qdrant import Qdrant
-from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain_openai import OpenAIEmbeddings
 from langchain.vectorstores.pinecone import Pinecone
 from qdrant_client import QdrantClient
 from qdrant_client import models
@@ -130,6 +130,18 @@ def delete_from_vector_store(namespace: str, filter_criteria: dict) -> None:
 
 
 def ensure_vector_database_exists(namespace):
+    """
+    This function ensures that a vector database exists for a given namespace. If the database does not exist, it attempts to 
+    create it. The function uses the 'STORE' environment variable to determine the type of store to use. If the store type is 
+    'QDRANT', it uses a QdrantClient to interact with the Qdrant server.
+
+    Args:
+        namespace (str): The namespace for which to ensure a vector database exists.
+
+    Raises:
+        Exception: If the function fails to ensure or create the vector database after 3 attempts, it raises an exception. 
+        It also raises an exception if any other error occurs during the process.
+    """
     store_type = StoreType[os.environ['STORE']]
     try:
         if store_type == StoreType.QDRANT: