feat: deep generate v0

.github/workflows/backend-core-tests.yml

@@ -2,12 +2,11 @@ name: Run Tests with Tika Server
 
 on:
   push:
-    paths:
-      - "core/**"
-  pull_request:
-    paths:
-      - "core/**"
-  workflow_dispatch:
+#   pull_request:
+#   workflow_dispatch:
+
+env:
+  NATS_TOKEN: test
 
 jobs:
   test:
@@ -23,22 +22,55 @@ jobs:
       - name: 👀 Checkout code
         uses: actions/checkout@v2
 
+      - name: Setup apt cache
+        uses: actions/cache@v4
+        with:
+          path: /var/cache/apt/archives
+          key: ${{ runner.os }}-apt-${{ hashFiles('/etc/apt/sources.list') }}
+
+      - name: 😭 Install system dependencies
+        run: |
+          sudo apt-get update && \
+            sudo apt-get install -y libmagic-dev poppler-utils libreoffice tesseract-ocr  pandoc
+
       - name: 🔨 Install the latest version of rye
         uses: eifinger/setup-rye@v4
         with:
           enable-cache: true
           working-directory: backend
+
+      - name: 🔽 Download and Install NATS Server
+        run: |
+            curl -L https://github.com/nats-io/nats-server/releases/download/v2.10.22/nats-server-v2.10.22-linux-amd64.zip -o nats-server.zip
+            unzip nats-server.zip -d nats-server && sudo cp nats-server/nats-server-v2.10.22-linux-amd64/nats-server /usr/bin
+
+      - name: 🛠️ Set up NATS arguments
+        run: |
+          nohup nats-server \
+            --addr 0.0.0.0 \
+            --port 4222 \
+            --auth "$NATS_TOKEN" > nats.log 2>&1 &
+
+      - name: 🔍 Verify NATS Server is Running
+        run: |
+          sleep 1 # Give the server some time to start
+          if nc -zv localhost 4222; then
+            echo "✅ NATS Server is running on port 4222."
+          else
+            echo "❌ Failed to start NATS Server."
+            cat nats.log
+            exit 1
+          fi
+
       - name: 🔄 Sync dependencies
         run: |
           cd core
           UV_INDEX_STRATEGY=unsafe-first-match rye sync --no-lock
 
-      - name: Run tests
+      - name: 🚀 Run tests
         env:
           TIKA_URL: http://localhost:9998/tika
           OPENAI_API_KEY: this-is-a-test-key
         run: |
-          sudo apt-get update
-          sudo apt-get install -y libmagic-dev poppler-utils libreoffice tesseract-ocr  pandoc
           cd core
-          rye test -p quivr-core
+          rye test -p quivr_core -v

core/quivr_core/brain/brain.py

@@ -3,14 +3,25 @@
 import os
 from pathlib import Path
 from pprint import PrettyPrinter
-from typing import Any, AsyncGenerator, Callable, Dict, Self, Type, Union
+from typing import Any, AsyncGenerator, Callable, Dict, Self, Union
 from uuid import UUID, uuid4
 
 from langchain_core.documents import Document
 from langchain_core.embeddings import Embeddings
 from langchain_core.messages import AIMessage, HumanMessage
 from langchain_core.vectorstores import VectorStore
 from langchain_openai import OpenAIEmbeddings
+from quivr_core.rag.entities.prompt import PromptConfig
+from quivr_core.rag.langgraph_framework.state import AgentState
+from quivr_core.rag.langgraph_framework.base.extractors import ConfigMapping
+from quivr_core.rag.langgraph_framework.entities.filter_history_config import (
+    FilterHistoryConfig,
+)
+from quivr_core.rag.langgraph_framework.services.llm_service import LLMService
+from quivr_core.rag.langgraph_framework.services.service_container import (
+    ServiceContainer,
+)
+from quivr_core.rag.quivr_rag_langgraph import QuivrQARAGLangGraph
 from rich.console import Console
 from rich.panel import Panel
 
@@ -26,16 +37,19 @@
 from quivr_core.llm import LLMEndpoint
 from quivr_core.processor.registry import get_processor_class
 from quivr_core.rag.entities.chat import ChatHistory
-from quivr_core.rag.entities.config import RetrievalConfig
+from quivr_core.rag.entities.config import (
+    LLMEndpointConfig,
+    RetrievalConfig,
+    RunTimeContext,
+    WorkflowConfig,
+)
 from quivr_core.rag.entities.models import (
     LangchainMetadata,
     ParsedRAGChunkResponse,
     ParsedRAGResponse,
     QuivrKnowledge,
     SearchResult,
 )
-from quivr_core.rag.quivr_rag import QuivrQARAG
-from quivr_core.rag.quivr_rag_langgraph import QuivrQARAGLangGraph
 from quivr_core.storage.local_storage import LocalStorage, TransparentStorage
 from quivr_core.storage.storage_base import StorageBase
 
@@ -119,12 +133,19 @@ def __init__(
         storage: StorageBase | None = None,
         workspace_id: UUID | None = None,
         chat_id: UUID | None = None,
+        service_container: ServiceContainer | None = None,
     ):
         self.id = id
         self.name = name
         self.storage = storage
         self.workspace_id = workspace_id
         self.chat_id = chat_id
+
+        # Service container
+        self.service_container = service_container
+        if not self.service_container:
+            self.service_container = ServiceContainer(vector_store=vector_db)
+
         # Chat history
         self._chats = self._init_chats()
         self.default_chat = list(self._chats.values())[0]
@@ -500,7 +521,6 @@ async def ask_streaming(
         run_id: UUID,
         system_prompt: str | None = None,
         retrieval_config: RetrievalConfig | None = None,
-        rag_pipeline: Type[Union[QuivrQARAG, QuivrQARAGLangGraph]] | None = None,
         list_files: list[QuivrKnowledge] | None = None,
         chat_history: ChatHistory | None = None,
         **input_kwargs,
@@ -510,7 +530,6 @@ async def ask_streaming(
         Args:
             question (str): The question to ask.
             retrieval_config (RetrievalConfig | None): The retrieval configuration (see RetrievalConfig docs).
-            rag_pipeline (Type[Union[QuivrQARAG, QuivrQARAGLangGraph]] | None): The RAG pipeline to use.
         list_files (list[QuivrKnowledge] | None): The list of files to include in the RAG pipeline.
             chat_history (ChatHistory | None): The chat history to use.
         Returns:
@@ -522,17 +541,34 @@ async def ask_streaming(
             print(chunk.answer)
         ```
         """
-        llm = self.llm
-
-        # If you passed a different llm model we'll override the brain  one
-        if retrieval_config:
-            if retrieval_config.llm_config != self.llm.get_config():
-                llm = LLMEndpoint.from_config(config=retrieval_config.llm_config)
-        else:
+        # If you passed a different llm model we 'll override the brain  one
+        if not retrieval_config:
             retrieval_config = RetrievalConfig(llm_config=self.llm.get_config())
 
+        retrieval_config.runtime_context.workspace_id = self.workspace_id
+
+        workflow_config = retrieval_config.workflow_config
+        llm_config = retrieval_config.llm_config
+        assert self.service_container, "Service container is not set"
+        llm_service = self.service_container.get_service(LLMService, llm_config)
+        config_extractor = ConfigMapping(
+            {
+                PromptConfig: "prompt_config",
+                LLMEndpointConfig: "llm_config",
+                WorkflowConfig: "workflow_config",
+                FilterHistoryConfig: "filter_history_config",
+                RunTimeContext: "runtime_context",
+            }
+        )
+
         rag_instance = QuivrQARAGLangGraph(
-            retrieval_config=retrieval_config, llm=llm, vector_store=self.vector_db
+            workflow_config=workflow_config,
+            graph_state=AgentState,
+            graph_config=retrieval_config.model_dump(),
+            graph_config_schema=RetrievalConfig,
+            llm_service=llm_service,
+            config_extractor=config_extractor,
+            service_container=self.service_container,
         )
 
         chat_history = self.default_chat if chat_history is None else chat_history
@@ -552,7 +588,8 @@ async def ask_streaming(
             system_prompt=system_prompt or None,
             history=chat_history,
             list_files=list_files,
-            metadata=metadata,
+            workspace_id=self.workspace_id,
+            metadata=metadata.model_dump(),
             **input_kwargs,
         ):
             # Format output to be correct servicedf;j
@@ -571,7 +608,6 @@ async def aask(
         question: str,
         system_prompt: str | None = None,
         retrieval_config: RetrievalConfig | None = None,
-        rag_pipeline: Type[Union[QuivrQARAG, QuivrQARAGLangGraph]] | None = None,
         list_files: list[QuivrKnowledge] | None = None,
         chat_history: ChatHistory | None = None,
         **input_kwargs,
@@ -581,7 +617,6 @@ async def aask(
         Args:
             question (str): The question to ask.
             retrieval_config (RetrievalConfig | None): The retrieval configuration (see RetrievalConfig docs).
-            rag_pipeline (Type[Union[QuivrQARAG, QuivrQARAGLangGraph]] | None): The RAG pipeline to use.
             list_files (list[QuivrKnowledge] | None): The list of files to include in the RAG pipeline.
             chat_history (ChatHistory | None): The chat history to use.
         Returns:
@@ -596,7 +631,6 @@ async def aask(
             question=question,
             system_prompt=system_prompt,
             retrieval_config=retrieval_config,
-            rag_pipeline=rag_pipeline,
             list_files=list_files,
             chat_history=chat_history,
             **input_kwargs,
@@ -613,7 +647,6 @@ def ask(
         question: str,
         system_prompt: str | None = None,
         retrieval_config: RetrievalConfig | None = None,
-        rag_pipeline: Type[Union[QuivrQARAG, QuivrQARAGLangGraph]] | None = None,
         list_files: list[QuivrKnowledge] | None = None,
         chat_history: ChatHistory | None = None,
     ) -> ParsedRAGResponse:
@@ -623,7 +656,6 @@ def ask(
             question (str): The question to ask.
             system_prompt (str | None): The system prompt to use.
             retrieval_config (RetrievalConfig | None): The retrieval configuration (see RetrievalConfig docs).
-            rag_pipeline (Type[Union[QuivrQARAG, QuivrQARAGLangGraph]] | None): The RAG pipeline to use.
             list_files (list[QuivrKnowledge] | None): The list of files to include in the RAG pipeline.
             chat_history (ChatHistory | None): The chat history to use.
         Returns:
@@ -636,7 +668,6 @@ def ask(
                 question=question,
                 system_prompt=system_prompt,
                 retrieval_config=retrieval_config,
-                rag_pipeline=rag_pipeline,
                 list_files=list_files,
                 chat_history=chat_history,
             )

core/quivr_core/llm_tools/base/__init__.py

@@ -0,0 +1,15 @@
+"""
+Base classes for LLM tools with unified interface and lifecycle management.
+"""
+
+from .tool import (
+    QuivrBaseTool,
+    ToolConfig,
+    ToolResult,
+)
+
+__all__ = [
+    "QuivrBaseTool",
+    "ToolConfig",
+    "ToolResult",
+]