- Refactored the MCP Server for better usability and extensibility

- Added ListIndex (beta)
- Minor other changes

Author: vaaale

packages/fetchcraft-core/src/examples/advanced/arbitrary_object_index.py

@@ -5,7 +5,6 @@
 from typing import List, Any, Dict
 
 import openai
-from mongomock.mongo_client import MongoClient
 from pydantic_ai import Tool
 from pydantic_ai.models.openai import OpenAIChatModel
 from pydantic_ai.providers.openai import OpenAIProvider
@@ -15,7 +14,7 @@
 from fetchcraft.agents import PydanticAgent, RetrieverTool
 from fetchcraft.embeddings import OpenAIEmbeddings
 from fetchcraft.index.vector_index import VectorIndex
-from fetchcraft.node import Node, ObjectNode, DefaultObjectMapper, ObjectType
+from fetchcraft.node import Node, ObjectNode, DefaultObjectMapper, ObjectType, DocumentNode
 from fetchcraft.retriever import VectorIndexRetriever
 from fetchcraft.vector_store import QdrantVectorStore
 
@@ -25,7 +24,6 @@
 
 COLLECTION_NAME = os.getenv("COLLECTION_NAME", "fetchcraft_objects")
 
-mongo_client = MongoClient()
 client = QdrantClient(":memory:")
 
 
@@ -35,13 +33,7 @@ async def get_vector_index(index_id: str, nodes: List[Node]):
         api_key=OPENAI_API_KEY,
         base_url=OPENAI_BASE_URL
     )
-    #
-    # document_store = MongoDBDocumentStore(
-    #     client=mongo_client,
-    #     database_name=COLLECTION_NAME,
-    #     collection_name=COLLECTION_NAME
-    # )
-    #
+
     vector_store = QdrantVectorStore(
         client=client,
         collection_name=COLLECTION_NAME,
@@ -50,10 +42,9 @@ async def get_vector_index(index_id: str, nodes: List[Node]):
 
     index = VectorIndex(
         vector_store=vector_store,
-        # doc_store=document_store,
         index_id=index_id
     )
-    await index.add_nodes(DocumentNode, nodes)
+    _ids = await index.add_nodes(doc=None, nodes=nodes)
 
     return index
 

packages/fetchcraft-core/src/examples/advanced/recursive_object_retrieval.py

@@ -46,7 +46,7 @@ async def get_vector_index(index_id: str, nodes: List[Node]):
         # doc_store=document_store,
         index_id=index_id
     )
-    await index.add_nodes(DocumentNode, nodes)
+    await index.add_nodes(doc=None, nodes=nodes)
 
     return index
 

packages/fetchcraft-core/src/fetchcraft/document_store/mongodb_store.py

@@ -2,7 +2,7 @@
 MongoDB document store implementation.
 """
 
-from typing import List, Optional, Dict, Any, Type
+from typing import List, Optional, Dict, Any, Type, Union
 from pydantic import BaseModel, Field, ConfigDict
 
 from .base import DocumentStore
@@ -413,7 +413,67 @@ async def get_documents_by_doc_id(self, doc_id: str) -> List[Node]:
             documents.append(doc_class(**doc_dict))
 
         return documents
-    
+
+    async def find(
+        self,
+        query: Dict[str, Any],
+        values: List[str],
+        unique: bool = False,
+    ) -> Union[List[Any], List[Dict[str, Any]]]:
+        """
+        Find arbitrary values from a MongoDB collection.
+
+        Args:
+            query: MongoDB filter query
+            values: list of dot-path fields to return (e.g. ["metadata.source"])
+            unique: whether to return unique values only
+
+        Returns:
+            - If one value field is provided:
+                List[Any]
+            - If multiple value fields are provided:
+                List[Dict[str, Any]]
+        """
+
+        # Build projection
+        projection = {field: 1 for field in values}
+        projection["_id"] = 0
+
+        cursor = await self.collection.find(query, projection)
+
+        def extract(doc: Dict[str, Any], path: str):
+            """Safely extract dotted-path values."""
+            current = doc
+            for key in path.split("."):
+                if not isinstance(current, dict) or key not in current:
+                    return None
+                current = current[key]
+            return current
+
+        results = []
+
+        for doc in cursor:
+            if len(values) == 1:
+                results.append(extract(doc, values[0]))
+            else:
+                results.append({v: extract(doc, v) for v in values})
+
+        if unique:
+            if len(values) == 1:
+                return list({v for v in results if v is not None})
+            else:
+                # Deduplicate dicts
+                seen = set()
+                unique_results = []
+                for item in results:
+                    key = tuple(sorted(item.items()))
+                    if key not in seen:
+                        seen.add(key)
+                        unique_results.append(item)
+                return unique_results
+
+        return results
+
     async def close(self):
         """Close the MongoDB connection."""
         if self.client:

packages/fetchcraft-core/src/fetchcraft/index/__init__.py

@@ -1,8 +1,10 @@
 from fetchcraft.index.base import BaseIndex, IndexFactory
 from fetchcraft.index.vector_index import VectorIndex
+from fetchcraft.index.list_index import ListIndex
 
 __all__ = [
     "BaseIndex",
     "IndexFactory",
     "VectorIndex",
+    "ListIndex"
 ]

packages/fetchcraft-core/src/fetchcraft/index/base.py

@@ -126,6 +126,38 @@ async def _resolve_to_top_parent(
         # Max depth reached, return current node
         return current_node
 
+    async def _resolve_parent_node(
+        self,
+        doc: Node,
+        score: float
+    ) -> tuple[Node, float]:
+        resolved_node: tuple[Node, float] = (doc, score)
+        seen_parent_ids: Set[str] = set()
+
+        # Check if this is a SymNode that needs parent resolution
+        if doc.node_type == NodeType.SYMNODE and doc.parent_id:
+            # Recursively resolve to top-level parent
+            top_parent = await self._resolve_to_top_parent(doc)
+
+            if top_parent and top_parent.id not in seen_parent_ids:
+                resolved_node = (top_parent, score)
+                seen_parent_ids.add(top_parent.id)
+            elif not top_parent:
+                # Resolution failed, fall back to original node
+                if doc.id not in seen_parent_ids:
+                    resolved_node = (doc, score)
+                    seen_parent_ids.add(doc.id)
+        else:
+            # Not a SymNode
+            # Check if this document is already a parent we've seen
+            if doc.id not in seen_parent_ids:
+                resolved_node = (doc, score)
+                # If this could be a parent (Chunk), track it to avoid duplicates
+                if isinstance(doc, Chunk):
+                    seen_parent_ids.add(doc.id)
+
+        return resolved_node
+
     async def _resolve_parent_nodes(
         self,
         results: List[tuple[D, float]]
@@ -197,7 +229,7 @@ async def delete_documents(self, document_ids):
         pass
 
     @abstractmethod
-    async def as_retriever(self, top_k, resolve_parents, object_mapper: Optional[ObjectMapper] = None, **search_kwargs):
+    def as_retriever(self, top_k: int = 4, resolve_parents: bool = True, object_mapper: Optional[ObjectMapper] = None, **search_kwargs):
         """
         Create a retriever from this index.
 

packages/fetchcraft-core/src/fetchcraft/index/list_index.py

@@ -0,0 +1,38 @@
+import logging
+from typing import List, Optional, AsyncIterator, Tuple
+
+from fetchcraft.index.base import BaseIndex, D
+from fetchcraft.kv_storage import KVStorage
+from fetchcraft.kv_storage.storage import InMemoryKVStorage
+from fetchcraft.node import Node, ObjectMapper
+from fetchcraft.retriever import ListIndexRetriever
+
+logger = logging.getLogger(__name__)
+
+
+class ListIndex(BaseIndex[Node]):
+    storage: KVStorage
+
+    def __init__(self, index_id: Optional[str], storage: Optional[KVStorage] = None, **kwargs):
+        _storage = storage or InMemoryKVStorage()
+        super().__init__(index_id=index_id, storage=_storage, **kwargs)
+
+
+    async def add_nodes(self, nodes: List[D], doc: Optional[D] = None, show_progress: bool = False):
+        self.storage.insert_nodes(nodes)
+
+    async def search_by_text_iter(self, query: str, query_embedding: List[float] = None, resolve_parents: bool = True, **kwargs) -> AsyncIterator[Tuple[D, float]]:
+        for node in self.storage.get_all():
+            score = 1.0
+            if resolve_parents:
+                node, score = await self._resolve_parent_node(node, 1.0)
+            yield node, score
+
+    async def get_document(self, document_id):
+        pass
+
+    async def delete_documents(self, document_ids):
+        pass
+
+    def as_retriever(self, top_k: int = 4, resolve_parents: bool = True, object_mapper: Optional[ObjectMapper] = None, **search_kwargs):
+        return ListIndexRetriever(index=self, top_k=top_k, resolve_parents=resolve_parents, object_mapper=object_mapper, **search_kwargs)

packages/fetchcraft-core/src/fetchcraft/index/vector_index.py

@@ -99,9 +99,8 @@ async def add_nodes(self, doc: Optional[D], nodes: List[D], show_progress: bool
             :param doc: The source document
         """
         existing_docs = {}
-        if self._doc_store:
+        if self._doc_store and doc:
             for node in nodes:
-                # _docs = await self._doc_store.list_documents(filters={"metadata.source": node.metadata.get("source")})
                 _docs = await self._doc_store.list_documents(filters={"persistent_key": node.persistent_key})
                 existing_docs.update({doc.id: doc for doc in _docs})
 
@@ -116,7 +115,10 @@ async def add_nodes(self, doc: Optional[D], nodes: List[D], show_progress: bool
 
         try:
             if self._doc_store:
-                await self._doc_store.add_documents([doc] + nodes)
+                all_nodes = nodes
+                if doc:
+                    all_nodes = [doc] + nodes
+                await self._doc_store.add_documents(all_nodes)
         except Exception as e:
             logger.error(f"Error adding documents to document store: {e}")
             raise e

packages/fetchcraft-core/src/fetchcraft/kv_storage/__init__.py

@@ -0,0 +1,5 @@
+from fetchcraft.kv_storage.storage import KVStorage
+
+__all__ = [
+    "KVStorage"
+]

packages/fetchcraft-core/src/fetchcraft/kv_storage/storage.py

@@ -0,0 +1,31 @@
+from abc import abstractmethod, ABC
+from typing import *
+from pydantic import BaseModel
+
+from fetchcraft.node import Node
+
+
+class KVStorage(BaseModel, ABC):
+
+    @abstractmethod
+    def insert_nodes(self, nodes: List[Node]):
+        ...
+
+    @abstractmethod
+    def get_all(self) -> List[Node]:
+        ...
+
+
+class InMemoryKVStorage(KVStorage):
+    _storage: Dict[str, Any]
+
+    def __init__(self):
+        super().__init__()
+        self._storage = {}
+
+    def insert_nodes(self, nodes: List[Node]):
+        self._storage.update({node.id: node for node in nodes})
+
+    def get_all(self) -> List[Node]:
+        return list(self._storage.values())
+

packages/fetchcraft-core/src/fetchcraft/node.py

@@ -504,6 +504,7 @@ def from_retriever(cls, text: str, retriever: ObjectNodeMixin):
         return cls(
             text=text,
             object_type=ObjectType.VECTOR_INDEX_RETRIEVER,
+            obj=retriever,
             data=retriever.to_json(),
         )
 
@@ -534,8 +535,9 @@ class DefaultObjectMapper(ObjectMapper):
     object_map: Dict[str, Any] = {}
     factories: Dict[str, Any] = {}
 
-    def __init__(self, factories: Optional[Dict[str, Any]] = None):
+    def __init__(self, factories: Optional[Dict[str, Any]] = None, object_map: Optional[Dict[str, Any]] = None):
         super().__init__()
+        self.object_map = object_map or {}
         self.factories = factories or {}
 
     async def resolve_object_node(self, node: ObjectNode, score: float, query: str, top_k: Optional[int] = None, **kwargs) -> List[Node]: