fix(web): handle metadata extraction failures - Add proper object to dict conversion for etadata - Add fallback for metadata extraction errors - Improve error handling and logging

Author: jasperan

agentic_rag/web_processor.py

@@ -55,7 +55,21 @@ def process_url(self, url: str) -> List[Dict[str, Any]]:
 
             # Extract text and metadata
             text = extract(downloaded, include_comments=False, include_tables=False)
-            metadata = extract_metadata(downloaded)
+            try:
+                metadata = extract_metadata(downloaded)
+                # Convert metadata to dict if it's not already
+                if not isinstance(metadata, dict):
+                    metadata = {
+                        'title': getattr(metadata, 'title', ''),
+                        'author': getattr(metadata, 'author', ''),
+                        'date': getattr(metadata, 'date', ''),
+                        'sitename': getattr(metadata, 'sitename', ''),
+                        'categories': getattr(metadata, 'categories', []),
+                        'tags': getattr(metadata, 'tags', [])
+                    }
+            except Exception as e:
+                print(f"Warning: Metadata extraction failed: {str(e)}")
+                metadata = {}
 
             if not text:
                 raise ValueError(f"No text content extracted from URL: {url}")