Retrieval Evaluation

Author: zamanmiraz

evaluation/datasets/retrieval_evaluate.py

@@ -0,0 +1,52 @@
+import numpy as np
+from niw_np_rag.app.rag import RAGPipeline
+import json
+
+with open(r".\evaluation\datasets\niw_qna.json", "r", encoding="utf-8") as f:
+    dataset = json.load(f)
+
+# Initialize your RAG pipeline
+rag = RAGPipeline(
+    pdfs_path="./data/uscis_aao_pdfs",
+    vector_store_path="./data/chunks_vector_store_faiss",
+    semantic_chunking=True
+)
+
+retriever = rag.get_retriever(k=5)
+
+
+def evaluate_recall_at_k(dataset, retriever, k=5):
+    """
+    dataset: list of dicts with fields:
+        - question
+        - answer  (not required for recall)
+        - context (ground-truth context from source docs)
+    """
+
+    hits = []
+
+    for item in dataset:
+        question = item["question"]
+        ground_truth_context = item["context"]
+        # print(question, ground_truth_context)
+
+        # Retrieve top-k documents
+        retrieved_docs = retriever.invoke(question)
+
+        # Extract retrieved text
+        retrieved_texts = [doc.page_content for doc in retrieved_docs[:k]]
+
+        # Check if ground-truth context appears in retrieved docs
+        hit = any(
+            ground_truth_context in retrieved_doc
+            for retrieved_doc in retrieved_texts
+        )
+
+        hits.append(1 if hit else 0)
+
+    recall_k = np.mean(hits)
+
+    print(f"Recall@{k}: {recall_k:.4f}")
+    return recall_k
+
+evaluate_recall_at_k(dataset, retriever, k=15)

niw_np_rag/app/__pycache__/rag.cpython-310.pyc

@@ -125,7 +125,6 @@ def build_vector_store_Qdrant(self):
             )
         )
 
-        # ✅ Initialize QdrantVectorStore once
         vector_store = QdrantVectorStore(
             client=client,
             collection_name=collection_name,
@@ -137,7 +136,7 @@ def build_vector_store_Qdrant(self):
         for pdf_file in tqdm(filtered_pdf_files, desc="Processing PDFs for Qdrant"):
             try:
                 texts = self.chunk_documents(pdf_file)
-                vector_store.add_documents(texts)  # ✅ add to existing collection
+                vector_store.add_documents(texts)  
                 tqdm.write(f"[ADD] Added chunks from: {pdf_file}")
                 count += 1
             except Exception as e: