Latest round of changes to support a vector store interface

Author: abrookins

CLAUDE.md

@@ -39,6 +39,7 @@ docker-compose down          # Stop all services
 IMPORTANT: This project uses `pre-commit`. You should run `pre-commit`
 before committing:
 ```bash
+uv run pre-commit install  # Install the hooks first
 uv run pre-commit run --all-files
 ```
 

agent_memory_server/api.py

@@ -381,13 +381,12 @@ async def search_long_term_memory(
     if not settings.long_term_memory:
         raise HTTPException(status_code=400, detail="Long-term memory is disabled")
 
-    redis = await get_redis_conn()
+    await get_redis_conn()
 
     # Extract filter objects from the payload
     filters = payload.get_filters()
 
     kwargs = {
-        "redis": redis,
         "distance_threshold": payload.distance_threshold,
         "limit": payload.limit,
         "offset": payload.offset,
@@ -397,7 +396,7 @@ async def search_long_term_memory(
     if payload.text:
         kwargs["text"] = payload.text
 
-    # Pass text, redis, and filter objects to the search function
+    # Pass text and filter objects to the search function (no redis needed for vectorstore adapter)
     return await long_term_memory.search_long_term_memories(**kwargs)
 
 

agent_memory_server/config.py

@@ -1,5 +1,5 @@
 import os
-from typing import Literal
+from typing import Any, Literal
 
 import yaml
 from dotenv import load_dotenv
@@ -9,12 +9,42 @@
 load_dotenv()
 
 
-def load_yaml_settings():
-    config_path = os.getenv("APP_CONFIG_FILE", "config.yaml")
-    if os.path.exists(config_path):
-        with open(config_path) as f:
-            return yaml.safe_load(f) or {}
-    return {}
+# Model configuration mapping
+MODEL_CONFIGS = {
+    "gpt-4o": {"provider": "openai", "embedding_dimensions": None},
+    "gpt-4o-mini": {"provider": "openai", "embedding_dimensions": None},
+    "gpt-4": {"provider": "openai", "embedding_dimensions": None},
+    "gpt-3.5-turbo": {"provider": "openai", "embedding_dimensions": None},
+    "text-embedding-3-small": {"provider": "openai", "embedding_dimensions": 1536},
+    "text-embedding-3-large": {"provider": "openai", "embedding_dimensions": 3072},
+    "text-embedding-ada-002": {"provider": "openai", "embedding_dimensions": 1536},
+    "claude-3-opus-20240229": {"provider": "anthropic", "embedding_dimensions": None},
+    "claude-3-sonnet-20240229": {"provider": "anthropic", "embedding_dimensions": None},
+    "claude-3-haiku-20240307": {"provider": "anthropic", "embedding_dimensions": None},
+    "claude-3-5-sonnet-20240620": {
+        "provider": "anthropic",
+        "embedding_dimensions": None,
+    },
+    "claude-3-5-sonnet-20241022": {
+        "provider": "anthropic",
+        "embedding_dimensions": None,
+    },
+    "claude-3-5-haiku-20241022": {
+        "provider": "anthropic",
+        "embedding_dimensions": None,
+    },
+    "claude-3-7-sonnet-20250219": {
+        "provider": "anthropic",
+        "embedding_dimensions": None,
+    },
+    "claude-3-7-sonnet-latest": {"provider": "anthropic", "embedding_dimensions": None},
+    "claude-3-5-sonnet-latest": {"provider": "anthropic", "embedding_dimensions": None},
+    "claude-3-5-haiku-latest": {"provider": "anthropic", "embedding_dimensions": None},
+    "claude-3-opus-latest": {"provider": "anthropic", "embedding_dimensions": None},
+    "o1": {"provider": "openai", "embedding_dimensions": None},
+    "o1-mini": {"provider": "openai", "embedding_dimensions": None},
+    "o3-mini": {"provider": "openai", "embedding_dimensions": None},
+}
 
 
 class Settings(BaseSettings):
@@ -28,55 +58,19 @@ class Settings(BaseSettings):
     port: int = 8000
     mcp_port: int = 9000
 
-    # Long-term memory backend configuration
-    long_term_memory_backend: str = (
-        "redis"  # redis, chroma, pinecone, weaviate, qdrant, etc.
+    # Vector store factory configuration
+    # Python dotted path to function that returns VectorStore or VectorStoreAdapter
+    # Function signature: (embeddings: Embeddings) -> Union[VectorStore, VectorStoreAdapter]
+    # Examples:
+    #   - "agent_memory_server.vectorstore_factory.create_redis_vectorstore"
+    #   - "my_module.my_vectorstore_factory"
+    #   - "my_package.adapters.create_custom_adapter"
+    vectorstore_factory: str = (
+        "agent_memory_server.vectorstore_factory.create_redis_vectorstore"
     )
 
-    # Redis backend settings (existing)
-    # redis_url already defined above
-
-    # Chroma backend settings
-    chroma_host: str = "localhost"
-    chroma_port: int = 8000
-    chroma_collection_name: str = "agent_memory"
-    chroma_persist_directory: str | None = None
-
-    # Pinecone backend settings
-    pinecone_api_key: str | None = None
-    pinecone_environment: str | None = None
-    pinecone_index_name: str = "agent-memory"
-
-    # Weaviate backend settings
-    weaviate_url: str = "http://localhost:8080"
-    weaviate_api_key: str | None = None
-    weaviate_class_name: str = "AgentMemory"
-
-    # Qdrant backend settings
-    qdrant_url: str = "http://localhost:6333"
-    qdrant_api_key: str | None = None
-    qdrant_collection_name: str = "agent_memory"
-
-    # Milvus backend settings
-    milvus_host: str = "localhost"
-    milvus_port: int = 19530
-    milvus_collection_name: str = "agent_memory"
-    milvus_user: str | None = None
-    milvus_password: str | None = None
-
-    # PostgreSQL/PGVector backend settings
-    postgres_url: str | None = None
-    postgres_table_name: str = "agent_memory"
-
-    # LanceDB backend settings
-    lancedb_uri: str = "./lancedb"
-    lancedb_table_name: str = "agent_memory"
-
-    # OpenSearch backend settings
-    opensearch_url: str = "http://localhost:9200"
-    opensearch_username: str | None = None
-    opensearch_password: str | None = None
-    opensearch_index_name: str = "agent-memory"
+    # RedisVL configuration (used by default Redis factory)
+    redisvl_index_name: str = "memory_records"
 
     # The server indexes messages in long-term memory by default. If this
     # setting is enabled, we also extract discrete memories from message text
@@ -95,10 +89,9 @@ class Settings(BaseSettings):
     ner_model: str = "dbmdz/bert-large-cased-finetuned-conll03-english"
     enable_ner: bool = True
 
-    # RedisVL Settings (kept for backwards compatibility)
+    # RedisVL Settings
     redisvl_distance_metric: str = "COSINE"
     redisvl_vector_dimensions: str = "1536"
-    redisvl_index_name: str = "memory_idx"
     redisvl_index_prefix: str = "memory_idx"
 
     # Docket settings
@@ -122,8 +115,54 @@ class Settings(BaseSettings):
     class Config:
         env_file = ".env"
         env_file_encoding = "utf-8"
+        extra = "ignore"  # Ignore extra environment variables
+
+    @property
+    def generation_model_config(self) -> dict[str, Any]:
+        """Get configuration for the generation model."""
+        return MODEL_CONFIGS.get(self.generation_model, {})
+
+    @property
+    def embedding_model_config(self) -> dict[str, Any]:
+        """Get configuration for the embedding model."""
+        return MODEL_CONFIGS.get(self.embedding_model, {})
+
+    def load_yaml_config(self, config_path: str) -> dict[str, Any]:
+        """Load configuration from YAML file."""
+        if not os.path.exists(config_path):
+            return {}
+        with open(config_path) as f:
+            return yaml.safe_load(f) or {}
+
+
+settings = Settings()
+
+
+def get_config():
+    """Get configuration from environment and settings files."""
+    config_data = {}
+
+    # If REDIS_MEMORY_CONFIG is set, load config from file
+    config_file = os.getenv("REDIS_MEMORY_CONFIG")
+    if config_file:
+        try:
+            with open(config_file) as f:
+                if config_file.endswith((".yaml", ".yml")):
+                    config_data = yaml.safe_load(f) or {}
+                else:
+                    # Assume JSON
+                    import json
+
+                    config_data = json.load(f) or {}
+        except FileNotFoundError:
+            print(f"Warning: Config file {config_file} not found")
+        except Exception as e:
+            print(f"Warning: Error loading config file {config_file}: {e}")
 
+    # Environment variables override file config
+    for key, value in os.environ.items():
+        if key.startswith("REDIS_MEMORY_"):
+            config_key = key[13:].lower()  # Remove REDIS_MEMORY_ prefix
+            config_data[config_key] = value
 
-# Load YAML config first, then let env vars override
-yaml_settings = load_yaml_settings()
-settings = Settings(**yaml_settings)
+    return config_data

agent_memory_server/filters.py

@@ -238,3 +238,7 @@ def __init__(self, **data):
 
 class EventDate(DateTimeFilter):
     field: str = "event_date"
+
+
+class MemoryHash(TagFilter):
+    field: str = "memory_hash"

agent_memory_server/long_term_memory.py

@@ -19,6 +19,7 @@
     Entities,
     EventDate,
     LastAccessed,
+    MemoryHash,
     MemoryType,
     Namespace,
     SessionId,
@@ -683,7 +684,6 @@ async def index_long_term_memories(
 
 async def search_long_term_memories(
     text: str,
-    redis: Redis | None = None,
     session_id: SessionId | None = None,
     user_id: UserId | None = None,
     namespace: Namespace | None = None,
@@ -694,6 +694,7 @@ async def search_long_term_memories(
     distance_threshold: float | None = None,
     memory_type: MemoryType | None = None,
     event_date: EventDate | None = None,
+    memory_hash: MemoryHash | None = None,
     limit: int = 10,
     offset: int = 0,
 ) -> MemoryRecordResults:
@@ -713,6 +714,7 @@ async def search_long_term_memories(
         distance_threshold: Optional similarity threshold
         memory_type: Optional memory type filter
         event_date: Optional event date filter
+        memory_hash: Optional memory hash filter
         limit: Maximum number of results
         offset: Offset for pagination
 
@@ -734,6 +736,7 @@ async def search_long_term_memories(
         entities=entities,
         memory_type=memory_type,
         event_date=event_date,
+        memory_hash=memory_hash,
         distance_threshold=distance_threshold,
         limit=limit,
         offset=offset,
@@ -793,7 +796,6 @@ async def search_memories(
         try:
             long_term_results = await search_long_term_memories(
                 text=text,
-                redis=redis,
                 session_id=session_id,
                 user_id=user_id,
                 namespace=namespace,
@@ -994,49 +996,62 @@ async def deduplicate_by_hash(
         }
     )
 
-    # Build filters for the search
-    filters = []
-    if namespace or memory.namespace:
-        ns = namespace or memory.namespace
-        filters.append(f"@namespace:{{{ns}}}")
-    if user_id or memory.user_id:
-        uid = user_id or memory.user_id
-        filters.append(f"@user_id:{{{uid}}}")
-    if session_id or memory.session_id:
-        sid = session_id or memory.session_id
-        filters.append(f"@session_id:{{{sid}}}")
-
-    filter_str = " ".join(filters) if filters else ""
+    # Use vectorstore adapter to search for memories with the same hash
+    try:
+        # Build filter objects
+        namespace_filter = None
+        if namespace or memory.namespace:
+            namespace_filter = Namespace(eq=namespace or memory.namespace)
+
+        user_id_filter = None
+        if user_id or memory.user_id:
+            user_id_filter = UserId(eq=user_id or memory.user_id)
+
+        session_id_filter = None
+        if session_id or memory.session_id:
+            session_id_filter = SessionId(eq=session_id or memory.session_id)
+
+        # Create memory hash filter
+        memory_hash_filter = MemoryHash(eq=memory_hash)
+
+        # Use vectorstore adapter to search for memories with the same hash
+        adapter = await get_vectorstore_adapter()
+
+        # Search for existing memories with the same hash
+        # Use a dummy query since we're filtering by hash, not doing semantic search
+        results = await adapter.search_memories(
+            query="",  # Empty query since we're filtering by hash
+            session_id=session_id_filter,
+            user_id=user_id_filter,
+            namespace=namespace_filter,
+            memory_hash=memory_hash_filter,
+            limit=1,  # We only need to know if one exists
+        )
 
-    # Search for existing memories with the same hash
-    index_name = Keys.search_index_name()
+        if results.memories and len(results.memories) > 0:
+            # Found existing memory with the same hash
+            logger.info(f"Found existing memory with hash {memory_hash}")
 
-    # Use FT.SEARCH to find memories with this hash
-    # TODO: Use RedisVL
-    search_query = (
-        f"FT.SEARCH {index_name} "
-        f"(@memory_hash:{{{memory_hash}}}) {filter_str} "
-        "RETURN 1 id_ "
-        "SORTBY last_accessed DESC"  # Newest first
-    )
+            # Update the last_accessed timestamp of the existing memory
+            existing_memory = results.memories[0]
+            if existing_memory.id:
+                # Use the memory key format to update last_accessed
+                existing_key = Keys.memory_key(
+                    existing_memory.id, existing_memory.namespace
+                )
+                await redis_client.hset(
+                    existing_key,
+                    "last_accessed",
+                    str(int(datetime.now(UTC).timestamp())),
+                )  # type: ignore
 
-    search_results = await redis_client.execute_command(search_query)
+                # Don't save this memory, it's a duplicate
+                return None, True
 
-    if search_results and search_results[0] > 0:
-        # Found existing memory with the same hash
-        logger.info(f"Found existing memory with hash {memory_hash}")
-
-        # Update the last_accessed timestamp of the existing memory
-        if search_results[0] >= 1:
-            existing_key = search_results[1].decode()
-            await redis_client.hset(
-                existing_key,
-                "last_accessed",
-                str(int(datetime.now(UTC).timestamp())),
-            )  # type: ignore
-
-            # Don't save this memory, it's a duplicate
-            return None, True
+    except Exception as e:
+        logger.error(f"Error searching for hash duplicates using vectorstore: {e}")
+        # If search fails, proceed with the original memory
+        pass
 
     # No duplicates found, return the original memory
     return memory, False