fix: Improve error handling and warnings

Author: jasperan

agentic_rag/pdf_processor.py

@@ -5,6 +5,11 @@
 from docling.document_converter import DocumentConverter
 from docling.chunking import HybridChunker
 from urllib.parse import urlparse
+import warnings
+import transformers
+
+# Suppress the token length warning
+warnings.filterwarnings('ignore', category=UserWarning, module='transformers.generation.utils')
 
 def is_url(string: str) -> bool:
     """Check if a string is a valid URL"""

agentic_rag/store.py

@@ -22,14 +22,31 @@ def __init__(self, persist_directory: str = "chroma_db"):
             metadata={"hnsw:space": "cosine"}
         )
 
+    def _sanitize_metadata(self, metadata: Dict) -> Dict:
+        """Sanitize metadata to ensure all values are valid types for ChromaDB"""
+        sanitized = {}
+        for key, value in metadata.items():
+            if isinstance(value, (str, int, float, bool)):
+                sanitized[key] = value
+            elif isinstance(value, list):
+                # Convert list to string representation
+                sanitized[key] = str(value)
+            elif value is None:
+                # Replace None with empty string
+                sanitized[key] = ""
+            else:
+                # Convert any other type to string
+                sanitized[key] = str(value)
+        return sanitized
+    
     def add_pdf_chunks(self, chunks: List[Dict[str, Any]], document_id: str):
         """Add chunks from a PDF document to the vector store"""
         if not chunks:
             return
 
         # Prepare data for ChromaDB
         texts = [chunk["text"] for chunk in chunks]
-        metadatas = [chunk["metadata"] for chunk in chunks]
+        metadatas = [self._sanitize_metadata(chunk["metadata"]) for chunk in chunks]
         ids = [f"{document_id}_{i}" for i in range(len(chunks))]
 
         # Add to collection
@@ -46,7 +63,7 @@ def add_general_knowledge(self, chunks: List[Dict[str, Any]], source_id: str):
 
         # Prepare data for ChromaDB
         texts = [chunk["text"] for chunk in chunks]
-        metadatas = [chunk["metadata"] for chunk in chunks]
+        metadatas = [self._sanitize_metadata(chunk["metadata"]) for chunk in chunks]
         ids = [f"{source_id}_{i}" for i in range(len(chunks))]
 
         # Add to collection