Merge pull request #43 from AET-DevOps25/feature/setup-llm-inference

Add LLM inference for response generation

Author: edkaya

genai/app.py

@@ -1,9 +1,9 @@
 from dotenv import load_dotenv
 from waitress import serve
 from flask import Flask
-from controller.generate_controller import generate_bp
+from genai.controller.generate_controller import generate_bp
 
-from config import Config
+from genai.config import Config
 
 app = Flask(__name__)
 app.register_blueprint(generate_bp)

genai/config.py

@@ -9,11 +9,15 @@
     "Config",
     [
         "api_key_openai",
-        "waitress"
+        "waitress",
+        "api_openwebui",
+        "base_url"
     ],
 )
 
 Config = ConfigT(
     api_key_openai=environ.get("API_SECRET_OPENAI_MINE"),
     waitress=environ.get("USE_WAITRESS", "false").lower() == "true",
+    api_openwebui=environ.get("API_OPENWEBUI"),
+    base_url=environ.get("BASE_URL")
 )

genai/controller/generate_controller.py

@@ -3,12 +3,25 @@
 import logging
 from werkzeug.utils import secure_filename
 
-from rag.ingestion_pipeline import IngestionPipeline
-from vector_database.qdrant_vdb import QdrantVDB
+from genai.rag.ingestion_pipeline import IngestionPipeline
+from genai.vector_database.qdrant_vdb import QdrantVDB
+from genai.rag.llm.chat_model import ChatModel
+from genai.service.rag_service import (
+    retrieve_similar_docs,
+    prepare_prompt,
+    process_raw_messages
+    )
+
 
 # Set Logging
 logging.getLogger().setLevel(logging.INFO)
 
+# Set ChatModel
+llm = ChatModel(model_name="llama3.3:latest")
+
+# Set Vector Database
+qdrant = QdrantVDB()
+
 generate_bp = Blueprint('generate', __name__)
 
 
@@ -31,8 +44,6 @@ def upload_file():
 
     try:
         collection_name = "recipes"
-        # Initialize vector database
-        qdrant = QdrantVDB()
         # Check if the file already in the collection
         if (qdrant.client.collection_exists(collection_name)
                 and qdrant.collection_contains_file(
@@ -69,6 +80,60 @@ def upload_file():
         os.remove(file_path)
 
 
-@generate_bp.route('/api/generate', methods=['POST'])
+@generate_bp.route('/genai/generate', methods=['POST'])
 def generate():
-    return jsonify({'output': 'Hello World!'})
+    """
+    API Endpoint for generating recipe responses using retrieved context.
+
+    This endpoint processes a user query against a vector database of recipes
+    and returns an AI-generated response using both retrieved context and
+    the full conversation history provided in the request.
+
+    Request Body:
+        query (str): The user's recipe-related query
+        messages (List[Dict]): Full conversation history,
+        each with 'role' and 'content'
+            Example:
+            [
+                {"role": "USER", "content": "I have eggs and tomatoes."},
+                {"role": "ASSISTANT", "content": "You could make shakshuka."}
+            ]
+
+    Returns:
+        JSON response containing:
+            - 'response': The generated assistant reply
+    """
+    data = request.get_json()
+
+    if not data or "query" not in data or "messages" not in data:
+        return jsonify({"error": "Missing 'query' or 'messages'"}), 400
+
+    query = data["query"]
+    messages_raw = data["messages"]
+
+    try:
+        collection_name = "recipes"
+
+        if qdrant.client.collection_exists(collection_name):
+            # Get vector store
+            vector_store = qdrant.create_and_get_vector_storage(
+                collection_name
+            )
+            # turn raw message into BaseMessage type
+            messages = process_raw_messages(messages_raw)
+            retrieved_docs = retrieve_similar_docs(vector_store, query)
+            prompt = prepare_prompt(
+                llm.get_system_prompt(),
+                query,
+                retrieved_docs,
+                messages
+                )
+
+            response = llm.invoke(prompt)
+
+            return jsonify({
+                "response": response.content,
+            }), 200
+
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500

genai/rag/llm/__init__.py

@@ -0,0 +1,54 @@
+from typing import List
+from langchain_core.messages import BaseMessage, HumanMessage, AIMessage
+from langchain_core.language_models.chat_models import BaseChatModel
+from langchain_core.outputs import ChatResult, ChatGeneration
+from pydantic import Field
+
+from genai.service.openwebui_service import generate_response
+
+
+class ChatModel(BaseChatModel):
+    model_name: str = Field(default="llama3.3:latest")
+
+    def _generate(self, messages: List[BaseMessage],
+                  stop=None,
+                  **kwargs) -> ChatResult:
+        prompt = "\n".join([
+            f"User: {m.content}" if isinstance(m, HumanMessage)
+            else f"Assistant: {m.content}" if isinstance(m, AIMessage)
+            else ""
+            for m in messages
+            ])
+        response_text = generate_response(self.model_name, prompt)
+
+        return ChatResult(
+            generations=[
+                ChatGeneration(message=AIMessage(content=response_text))
+                ]
+        )
+
+    @property
+    def _llm_type(self) -> str:
+        return "recipai-custom-model"
+
+    def get_system_prompt(self) -> str:
+        """System prompt for the LLM"""
+        return """
+            You are an intelligent assistant that helps users discover
+            and generate recipes based on the ingredients they provide.
+
+            Use the contextual information provided below to tailor
+            your responses.
+
+            If relevant recipes or suggestions are found in the context,
+            prioritize those. If no relevant context is available,
+            use your own knowledge to help the user.
+
+            Context:
+            {context}
+
+            Be clear, creative, and helpful. If the user also asks
+            follow-up questions (e.g., dietary adjustments, name references,
+            meal timing), answer them precisely based on the
+            context and query.
+            """

genai/rag/llm/chat_model.py

@@ -0,0 +1,45 @@
+import requests
+
+from genai.config import Config
+
+BASE_URL = Config.base_url
+
+
+def generate_response(model_name: str, prompt: str):
+    """Making a POST request to the respective endpoint for
+    response generation by an LLM"""
+    url = f"{BASE_URL}/api/chat/completions"
+
+    headers = {
+        "Authorization": f"Bearer {Config.api_openwebui}",
+        "Content-Type": "application/json"
+    }
+
+    payload = {
+        "model": model_name,
+        "messages": [
+            {
+                "role": "user",
+                "content": prompt
+                }
+            ]
+    }
+
+    try:
+        response = requests.post(
+            url,
+            json=payload,
+            headers=headers,
+            timeout=120
+        )
+        response.raise_for_status()
+        return response.json()["choices"][0]["message"]["content"]
+
+    except requests.exceptions.HTTPError as e:
+        raise RuntimeError(
+            f"HTTP error from LLM server: {e}, {response.status_code})"
+            ) from e
+    except requests.exceptions.Timeout as e:
+        raise RuntimeError(f"Request to LLM timed out: {e}") from e
+    except requests.exceptions.RequestException as e:
+        raise RuntimeError(f"Request to LLM failed: {e}") from e

genai/service/openwebui_service.py

@@ -0,0 +1,61 @@
+from typing import List, Dict
+
+from langchain_qdrant import QdrantVectorStore
+from langchain_core.messages import BaseMessage, HumanMessage, AIMessage
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+# from genai.rag.llm.chat_model import ChatModel
+
+
+def retrieve_similar_docs(vector_store: QdrantVectorStore, user_query: str):
+    """Retrieve similar documents based on the user query"""
+    retriever = vector_store.as_retriever(search_kwargs={"k": 5})
+    retrieved_docs = retriever.invoke(user_query)
+    docs_content = "\n\n".join(doc.page_content for doc in retrieved_docs)
+    return docs_content
+
+
+def prepare_prompt(system_prompt: str,
+                   user_query: str,
+                   docs_content: str,
+                   messages: List[BaseMessage]):
+    """Prepare the prompt with prompt templates to give to LLM"""
+    prompt_template = ChatPromptTemplate([
+        "system", system_prompt,
+        MessagesPlaceholder("msgs")
+    ])
+
+    full_messages = messages + [HumanMessage(content=user_query)]
+
+    prompt = prompt_template.invoke({
+        "context": docs_content,
+        "msgs": full_messages
+    })
+
+    return prompt
+
+
+def process_raw_messages(raw_messages: List[Dict]) -> List[BaseMessage]:
+    """Turns raw messages into BaseMessages, so they can be passed into LLM"""
+    processed_messages = []
+    for msg in raw_messages:
+        role = msg.get("role")
+        content = msg.get("content")
+
+        if role.upper() == "USER":
+            processed_messages.append(HumanMessage(content=content))
+
+        elif role.upper() == "ASSISTANT":
+            processed_messages.append(AIMessage(content=content))
+
+    return processed_messages
+
+# For testing purposes
+# if __name__ == "__main__":
+#     msg = HumanMessage(content="My name is John Doe.")
+#     llm = ChatModel()
+#     prompt = prepare_prompt(llm.get_system_prompt(),
+#                             "Suggest me a basic breakfast.",
+#                             "",
+#                             [msg])
+#     response = llm.invoke(prompt)
+#     print(response.content)