Merge pull request #23 from AET-DevOps25/feature/seperate-chat-llm

Run All LLM Calls Async and Introduce Separate LLM for Gen

Author: yassinsws

.github/workflows/deploy_vm.yml

@@ -88,6 +88,9 @@ jobs:
           IMAGE_TAG: ${{ env.COMMIT_SHA }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_ACTOR: ${{ github.actor }}
+          OPEN_WEBUI_API_KEY_CHAT: ${{ secrets.OPEN_WEBUI_API_KEY_CHAT }}
+          OPEN_WEBUI_API_KEY_GEN: ${{ secrets.OPEN_WEBUI_API_KEY_GEN }}
+          LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}
         run: ansible-playbook -i inventory.ini deploy/ansible/deploy.yml
 
   deploy-k8s:

deploy/ansible/deploy.yml

@@ -6,6 +6,9 @@
   vars:
     project_dir: /home/ubuntu/app
     image_tag: "{{ lookup('env','IMAGE_TAG') }}"
+    open_webui_api_key_chat: "{{ lookup('env','OPEN_WEBUI_API_KEY_CHAT') }}"
+    open_webui_api_key_gen: "{{ lookup('env','OPEN_WEBUI_API_KEY_GEN') }}"
+    langsmith_api_key: "{{ lookup('env','LANGSMITH_API_KEY') }}"
 
   tasks:
     - name: Ensure project directory exists

docker-compose.yml.j2

@@ -52,6 +52,13 @@ services:
   genai:
     image: ghcr.io/aet-devops25/team-3/genai:{{ image_tag }}
     restart: unless-stopped
+    environment:
+      - OPEN_WEBUI_API_KEY_CHAT="{{ open_webui_api_key_chat }}"
+      - OPEN_WEBUI_API_KEY_GEN="{{ open_webui_api_key_gen }}"
+      - LANGSMITH_TRACING=true
+      - LANGSMITH_ENDPOINT="https://api.smith.langchain.com"
+      - LANGSMITH_API_KEY="{{ langsmith_api_key }}"
+      - LANGSMITH_PROJECT="studymate"
     networks:
       - proxy
     labels:

genAi/llm.py

@@ -18,10 +18,19 @@
 
 
 class StudyLLM:
-    llm = ChatOpenAI(
+    # for chat
+    chat_llm = ChatOpenAI(
         model="llama3.3:latest",
         temperature=0.5,
-        api_key=os.getenv("OPEN_WEBUI_API_KEY"),
+        api_key=os.getenv("OPEN_WEBUI_API_KEY_CHAT"),
+        base_url="https://gpu.aet.cit.tum.de/api/"
+    )
+    
+    # For summaries, quizzes, flashcards
+    generation_llm = ChatOpenAI(
+        model="llama3.3:latest",
+        temperature=0.5,
+        api_key=os.getenv("OPEN_WEBUI_API_KEY_GEN"),
         base_url="https://gpu.aet.cit.tum.de/api/"
     )
 
@@ -39,26 +48,9 @@ def __init__(self, doc_path: str):
             self.rag_helper = RAGHelper(doc_path)
         except Exception as e:
             raise ValueError(f"Error initializing RAGHelper: {e}")
-    
-    def _chain(self, output_model: BaseModel = None):
-        """
-        Construct a chain for the LLM with given configurations.
-        
-        Args:
-            OutputModel (BaseModel, optional): A Pydantic model for structured output.
-            ...
-        Returns:
-            RnnableSequence: The chain for the LLM.
-        """
-        llm = self.llm
-        
-        if output_model:
-            llm = llm.with_structured_output(output_model)
-        
-        return self.base_prompt_template | llm
 
 
-    def prompt(self, prompt: str) -> str:
+    async def prompt(self, prompt: str) -> str:
         """
         Call the LLM with a given prompt.
         
@@ -74,13 +66,16 @@ def prompt(self, prompt: str) -> str:
             )
 
         context = self.rag_helper.retrieve(prompt, top_k=5)
-        return self._chain().invoke({
+        chain = self.base_prompt_template | self.chat_llm
+        response = await chain.ainvoke({
             'context': context,
             'task':task,
             'input':prompt
-            }).content
+            })
+        
+        return response.content
 
-    def summarize(self):
+    async def summarize(self):
         """
         Summarize the given document using the LLM.
         
@@ -107,13 +102,13 @@ def summarize(self):
         )
 
         chain = load_summarize_chain(
-            self.llm,
+            self.generation_llm,
             chain_type="map_reduce",
             map_prompt=map_prompt,
             combine_prompt=combine_prompt
         )
 
-        result = chain.invoke({"input_documents": self.rag_helper.summary_chunks})
+        result = await chain.ainvoke({"input_documents": self.rag_helper.summary_chunks})
 
         return result["output_text"]
 
@@ -124,7 +119,7 @@ async def generate_flashcards(self):
         Returns:
             list: A list of flashcard objects.
         """
-        flashcard_chain = FlashcardChain(self.llm)
+        flashcard_chain = FlashcardChain(self.generation_llm)
         cards = await flashcard_chain.invoke(self.rag_helper.summary_chunks)
         return cards
 
@@ -135,7 +130,7 @@ async def generate_quiz(self):
         Returns:
             list: A quiz object.
         """
-        quiz_chain = QuizChain(self.llm)
+        quiz_chain = QuizChain(self.generation_llm)
         quiz = await quiz_chain.invoke(self.rag_helper.summary_chunks)
         return quiz
 

genAi/main.py

@@ -85,7 +85,7 @@ async def receive_prompt(data: PromptRequest):
             return {"response": f"ERROR: {error_msg}"}
 
         logger.info(f"Processing chat request for session {data.session_id}")
-        response = llm_instances[data.session_id].prompt(data.message)
+        response = await llm_instances[data.session_id].prompt(data.message)
         return {"response": response}
     except Exception as e:
         error_msg = f"Chat error for session {data.session_id}: {str(e)}"
@@ -104,7 +104,7 @@ async def generate_summary(data: SummaryRequest):
             return {"response": f"ERROR: {error_msg}"}
 
         logger.info(f"Generating summary for session {data.session_id}")
-        response = llm_instances[data.session_id].summarize()
+        response = await llm_instances[data.session_id].summarize()
         logger.info(f"Summary generated successfully for session {data.session_id}")
         return {"response": response}
     except Exception as e: