Add advanced RAG

Author: ks6088ts

apps/12_langgraph_agent/README.md

@@ -31,17 +31,31 @@ $ source .venv/bin/activate
 
 # Install dependencies
 $ pip install -r requirements.txt
+```
+
+### Examples
+
+#### [reflection_agent](./reflection_agent/main.py)
+
+#### [react_agent](./react_agent/main.py)
+
+#### [advanced_rag_flows](./advanced_rag_flows/main.py)
+
+```shell
+# create vector store
+python apps/12_langgraph_agent/advanced_rag_flows/ingestion.py
 
-# Run the script
-$ python apps/12_langgraph_agent/reflection_agent/main.py
+# run main.py
+python apps/12_langgraph_agent/advanced_rag_flows/main.py
 ```
 
-### Example
+![Advanced RAG Flows](../../docs/images/12_langgraph_agent_graph.png)
 
 ## References
 
 - [LangGraph](https://langchain-ai.github.io/langgraph/)
 - [Udemy > LangGraph- Develop LLM powered agents with LangGraph](https://www.udemy.com/course/langgraph)
+- [emarco177/langgaph-course](https://github.com/emarco177/langgaph-course)
 - [Prompt flow > Tracing](https://microsoft.github.io/promptflow/how-to-guides/tracing/index.html)
 - [Reflection Agents](https://blog.langchain.dev/reflection-agents/)
 - [LangChain > Reflexion](https://langchain-ai.github.io/langgraph/tutorials/reflexion/reflexion/)

apps/12_langgraph_agent/advanced_rag_flows/graph/__init__.py

@@ -0,0 +1,33 @@
+from os import getenv
+
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnableSequence
+from langchain_openai import AzureChatOpenAI
+from pydantic import BaseModel, Field
+
+
+class GradeAnswer(BaseModel):
+
+    binary_score: bool = Field(description="Answer addresses the question, 'yes' or 'no'")
+
+
+llm = AzureChatOpenAI(
+    temperature=0,
+    api_key=getenv("AZURE_OPENAI_API_KEY"),
+    api_version=getenv("AZURE_OPENAI_API_VERSION"),
+    azure_endpoint=getenv("AZURE_OPENAI_ENDPOINT"),
+    model=getenv("AZURE_OPENAI_GPT_MODEL"),
+)
+
+structured_llm_grader = llm.with_structured_output(GradeAnswer)
+
+system = """You are a grader assessing whether an answer addresses / resolves a question \n
+     Give a binary score 'yes' or 'no'. Yes' means that the answer resolves the question."""
+answer_prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", system),
+        ("human", "User question: \n\n {question} \n\n LLM generation: {generation}"),
+    ]
+)
+
+answer_grader: RunnableSequence = answer_prompt | structured_llm_grader

apps/12_langgraph_agent/advanced_rag_flows/graph/chains/__init__.py

@@ -0,0 +1,16 @@
+from os import getenv
+
+from langchain import hub
+from langchain_core.output_parsers import StrOutputParser
+from langchain_openai import AzureChatOpenAI
+
+llm = AzureChatOpenAI(
+    temperature=0,
+    api_key=getenv("AZURE_OPENAI_API_KEY"),
+    api_version=getenv("AZURE_OPENAI_API_VERSION"),
+    azure_endpoint=getenv("AZURE_OPENAI_ENDPOINT"),
+    model=getenv("AZURE_OPENAI_GPT_MODEL"),
+)
+prompt = hub.pull("rlm/rag-prompt")
+
+generation_chain = prompt | llm | StrOutputParser()

apps/12_langgraph_agent/advanced_rag_flows/graph/chains/answer_grader.py

@@ -0,0 +1,34 @@
+from os import getenv
+
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnableSequence
+from langchain_openai import AzureChatOpenAI
+from pydantic import BaseModel, Field
+
+llm = AzureChatOpenAI(
+    temperature=0,
+    api_key=getenv("AZURE_OPENAI_API_KEY"),
+    api_version=getenv("AZURE_OPENAI_API_VERSION"),
+    azure_endpoint=getenv("AZURE_OPENAI_ENDPOINT"),
+    model=getenv("AZURE_OPENAI_GPT_MODEL"),
+)
+
+
+class GradeHallucinations(BaseModel):
+    """Binary score for hallucination present in generation answer."""
+
+    binary_score: bool = Field(description="Answer is grounded in the facts, 'yes' or 'no'")
+
+
+structured_llm_grader = llm.with_structured_output(GradeHallucinations)
+
+system = """You are a grader assessing whether an LLM generation is grounded in / supported by a set of retrieved facts. \n
+     Give a binary score 'yes' or 'no'. 'Yes' means that the answer is grounded in / supported by the set of facts."""  # noqa
+hallucination_prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", system),
+        ("human", "Set of facts: \n\n {documents} \n\n LLM generation: {generation}"),
+    ]
+)
+
+hallucination_grader: RunnableSequence = hallucination_prompt | structured_llm_grader

apps/12_langgraph_agent/advanced_rag_flows/graph/chains/generation.py

@@ -0,0 +1,34 @@
+from os import getenv
+
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_openai import AzureChatOpenAI
+from pydantic import BaseModel, Field
+
+llm = AzureChatOpenAI(
+    temperature=0,
+    api_key=getenv("AZURE_OPENAI_API_KEY"),
+    api_version=getenv("AZURE_OPENAI_API_VERSION"),
+    azure_endpoint=getenv("AZURE_OPENAI_ENDPOINT"),
+    model=getenv("AZURE_OPENAI_GPT_MODEL"),
+)
+
+
+class GradeDocuments(BaseModel):
+    """Binary score for relevance check on retrieved documents."""
+
+    binary_score: str = Field(description="Documents are relevant to the question, 'yes' or 'no'")
+
+
+structured_llm_grader = llm.with_structured_output(GradeDocuments)
+
+system = """You are a grader assessing relevance of a retrieved document to a user question. \n
+    If the document contains keyword(s) or semantic meaning related to the question, grade it as relevant. \n
+    Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question."""
+grade_prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", system),
+        ("human", "Retrieved document: \n\n {document} \n\n User question: {question}"),
+    ]
+)
+
+retrieval_grader = grade_prompt | structured_llm_grader

apps/12_langgraph_agent/advanced_rag_flows/graph/chains/hallucination_grader.py

@@ -0,0 +1,37 @@
+from os import getenv
+from typing import Literal
+
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_openai import AzureChatOpenAI
+from pydantic import BaseModel, Field
+
+
+class RouteQuery(BaseModel):
+    """Route a user query to the most relevant datasource."""
+
+    datasource: Literal["vectorstore", "websearch"] = Field(
+        ...,
+        description="Given a user question choose to route it to web search or a vectorstore.",
+    )
+
+
+llm = AzureChatOpenAI(
+    temperature=0,
+    api_key=getenv("AZURE_OPENAI_API_KEY"),
+    api_version=getenv("AZURE_OPENAI_API_VERSION"),
+    azure_endpoint=getenv("AZURE_OPENAI_ENDPOINT"),
+    model=getenv("AZURE_OPENAI_GPT_MODEL"),
+)
+structured_llm_router = llm.with_structured_output(RouteQuery)
+
+system = """You are an expert at routing a user question to a vectorstore or web search.
+The vectorstore contains documents related to agents, prompt engineering, and adversarial attacks.
+Use the vectorstore for questions on these topics. For all else, use web-search."""
+route_prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", system),
+        ("human", "{question}"),
+    ]
+)
+
+question_router = route_prompt | structured_llm_router

apps/12_langgraph_agent/advanced_rag_flows/graph/chains/retrieval_grader.py

@@ -0,0 +1,74 @@
+from pprint import pprint
+
+from dotenv import load_dotenv
+from ingestion import retriever
+
+from graph.chains.generation import generation_chain
+from graph.chains.hallucination_grader import GradeHallucinations, hallucination_grader
+from graph.chains.retrieval_grader import GradeDocuments, retrieval_grader
+from graph.chains.router import RouteQuery, question_router
+
+load_dotenv()
+
+
+def test_generation_chain() -> None:
+    question = "agent memory"
+    docs = retriever.invoke(question)
+    generation = generation_chain.invoke({"context": docs, "question": question})
+    pprint(generation)
+
+
+def test_retrival_grader_answer_yes() -> None:
+    question = "agent memory"
+    docs = retriever.invoke(question)
+    doc_txt = docs[1].page_content
+
+    res: GradeDocuments = retrieval_grader.invoke({"question": question, "document": doc_txt})
+
+    assert res.binary_score == "yes"
+
+
+def test_retrival_grader_answer_no() -> None:
+    question = "agent memory"
+    docs = retriever.invoke(question)
+    doc_txt = docs[1].page_content
+
+    res: GradeDocuments = retrieval_grader.invoke({"question": "how to make pizaa", "document": doc_txt})
+
+    assert res.binary_score == "no"
+
+
+def test_hallucination_grader_answer_yes() -> None:
+    question = "agent memory"
+    docs = retriever.invoke(question)
+
+    generation = generation_chain.invoke({"context": docs, "question": question})
+    res: GradeHallucinations = hallucination_grader.invoke({"documents": docs, "generation": generation})
+    assert res.binary_score
+
+
+def test_hallucination_grader_answer_no() -> None:
+    question = "agent memory"
+    docs = retriever.invoke(question)
+
+    res: GradeHallucinations = hallucination_grader.invoke(
+        {
+            "documents": docs,
+            "generation": "In order to make pizza we need to first start with the dough",
+        }
+    )
+    assert not res.binary_score
+
+
+def test_router_to_vectorstore() -> None:
+    question = "agent memory"
+
+    res: RouteQuery = question_router.invoke({"question": question})
+    assert res.datasource == "vectorstore"
+
+
+def test_router_to_websearch() -> None:
+    question = "how to make pizza"
+
+    res: RouteQuery = question_router.invoke({"question": question})
+    assert res.datasource == "websearch"