feat: update dim env (#186)

* feat: add user manager factory pattern and product API enhancements

- Add user manager factory pattern with SQLite and MySQL backends
- Add user manager configuration to MOSConfig
- Add product API router and configuration
- Add DingDing notification integration
- Add notification service utilities
- Update OpenAPI documentation

* fix: change user manager default

* fix: update config

* fix： remove db_name  for neublar

* fix:host

* feat:update db

* fix:test

* fix: reomve dup file

* feat: add logs

* feat: add users profile field

* fix: product and format utils and change prompt

* fix: bugs for chat

* feat: add dim for config

Author: fridayL

src/memos/api/config.py

@@ -169,14 +169,14 @@ def get_neo4j_community_config(user_id: str | None = None) -> dict[str, Any]:
             "user_name": f"memos{user_id.replace('-', '')}",
             "auto_create": True,
             "use_multi_db": False,
-            "embedding_dimension": 3072,
+            "embedding_dimension": int(os.getenv("EMBEDDING_DIMENSION", 3072)),
             "vec_config": {
                 # Pass nested config to initialize external vector DB
                 # If you use qdrant, please use Server instead of local mode.
                 "backend": "qdrant",
                 "config": {
                     "collection_name": "neo4j_vec_db",
-                    "vector_dimension": 3072,
+                    "vector_dimension": int(os.getenv("EMBEDDING_DIMENSION", 3072)),
                     "distance_metric": "cosine",
                     "host": "localhost",
                     "port": 6333,
@@ -202,7 +202,7 @@ def get_noshared_neo4j_config(user_id) -> dict[str, Any]:
             "password": os.getenv("NEO4J_PASSWORD", "12345678"),
             "auto_create": True,
             "use_multi_db": True,
-            "embedding_dimension": 3072,
+            "embedding_dimension": int(os.getenv("EMBEDDING_DIMENSION", 3072)),
         }
 
     @staticmethod
@@ -216,7 +216,7 @@ def get_neo4j_shared_config(user_id: str | None = None) -> dict[str, Any]:
             "user_name": f"memos{user_id.replace('-', '')}",
             "auto_create": True,
             "use_multi_db": False,
-            "embedding_dimension": 3072,
+            "embedding_dimension": int(os.getenv("EMBEDDING_DIMENSION", 3072)),
         }
 
     @staticmethod
@@ -230,7 +230,7 @@ def get_nebular_config(user_id: str | None = None) -> dict[str, Any]:
             "user_name": f"memos{user_id.replace('-', '')}",
             "use_multi_db": False,
             "auto_create": True,
-            "embedding_dimension": 3072,
+            "embedding_dimension": int(os.getenv("EMBEDDING_DIMENSION", 3072)),
         }
 
     @staticmethod

src/memos/mem_os/core.py

@@ -1,7 +1,6 @@
 import json
 import os
 import time
-import uuid
 
 from datetime import datetime
 from pathlib import Path
@@ -565,7 +564,8 @@ def search(
         logger.info(
             f"User {target_user_id} has access to {len(user_cube_ids)} cubes: {user_cube_ids}"
         )
-
+        if target_user_id not in self.chat_history_manager:
+            self._register_chat_history(target_user_id)
         chat_history = self.chat_history_manager[target_user_id]
 
         result: MOSSearchResult = {

src/memos/mem_os/product.py

@@ -22,6 +22,7 @@
     filter_nodes_by_tree_ids,
     remove_embedding_recursive,
     sort_children_by_memory_type,
+    split_continuous_references,
 )
 from memos.mem_scheduler.schemas.general_schemas import (
     ANSWER_LABEL,
@@ -33,6 +34,7 @@
 from memos.memories.textual.item import (
     TextualMemoryItem,
 )
+from memos.templates.mos_prompts import MEMOS_PRODUCT_BASE_PROMPT, MEMOS_PRODUCT_ENHANCE_PROMPT
 from memos.types import MessageList
 
 
@@ -360,28 +362,49 @@ def _build_system_prompt(
         """
 
         # Build base prompt
-        base_prompt = (
-            "You are a knowledgeable and helpful AI assistant with access to user memories. "
-            "When responding to user queries, you should reference relevant memories using the provided memory IDs. "
-            "Use the reference format: [1-n:memoriesID] "
-            "where refid is a sequential number starting from 1 and increments for each reference in your response, "
-            "and memoriesID is the specific memory ID provided in the available memories list. "
-            "For example: [1:abc123], [2:def456], [3:ghi789], [4:jkl101], [5:mno112] "
-            "Only reference memories that are directly relevant to the user's question. "
-            "Make your responses natural and conversational while incorporating memory references when appropriate."
-        )
-
         # Add memory context if available
         if memories_all:
             memory_context = "\n\n## Available ID Memories:\n"
             for i, memory in enumerate(memories_all, 1):
                 # Format: [memory_id]: memory_content
                 memory_id = f"{memory.id.split('-')[0]}" if hasattr(memory, "id") else f"mem_{i}"
                 memory_content = memory.memory[:500] if hasattr(memory, "memory") else str(memory)
+                memory_content = memory_content.replace("\n", " ")
                 memory_context += f"{memory_id}: {memory_content}\n"
-            return base_prompt + memory_context
+            return MEMOS_PRODUCT_BASE_PROMPT + memory_context
+
+        return MEMOS_PRODUCT_BASE_PROMPT
 
-        return base_prompt
+    def _build_enhance_system_prompt(
+        self, user_id: str, memories_all: list[TextualMemoryItem]
+    ) -> str:
+        """
+        Build enhance prompt for the user with memory references.
+        """
+        if memories_all:
+            personal_memory_context = "\n\n## Available ID and PersonalMemory Memories:\n"
+            outer_memory_context = "\n\n## Available ID and OuterMemory Memories:\n"
+            for i, memory in enumerate(memories_all, 1):
+                # Format: [memory_id]: memory_content
+                if memory.metadata.memory_type != "OuterMemory":
+                    memory_id = (
+                        f"{memory.id.split('-')[0]}" if hasattr(memory, "id") else f"mem_{i}"
+                    )
+                    memory_content = (
+                        memory.memory[:500] if hasattr(memory, "memory") else str(memory)
+                    )
+                    personal_memory_context += f"{memory_id}: {memory_content}\n"
+                else:
+                    memory_id = (
+                        f"{memory.id.split('-')[0]}" if hasattr(memory, "id") else f"mem_{i}"
+                    )
+                    memory_content = (
+                        memory.memory[:500] if hasattr(memory, "memory") else str(memory)
+                    )
+                    memory_content = memory_content.replace("\n", " ")
+                    outer_memory_context += f"{memory_id}: {memory_content}\n"
+            return MEMOS_PRODUCT_ENHANCE_PROMPT + personal_memory_context + outer_memory_context
+        return MEMOS_PRODUCT_ENHANCE_PROMPT
 
     def _process_streaming_references_complete(self, text_buffer: str) -> tuple[str, str]:
         """
@@ -406,9 +429,13 @@ def _process_streaming_references_complete(self, text_buffer: str) -> tuple[str,
             last_match = complete_matches[-1]
             end_pos = last_match.end()
 
-            # Return text up to the end of the last complete tag
+            # Get text up to the end of the last complete tag
             processed_text = text_buffer[:end_pos]
             remaining_buffer = text_buffer[end_pos:]
+
+            # Apply reference splitting to the processed text
+            processed_text = split_continuous_references(processed_text)
+
             return processed_text, remaining_buffer
 
         # Check for incomplete reference tags
@@ -427,15 +454,22 @@ def _process_streaming_references_complete(self, text_buffer: str) -> tuple[str,
                 return "", text_buffer
             else:
                 # Incomplete opening pattern, return text before it
-                return text_buffer[:opening_start], text_buffer[opening_start:]
+                processed_text = text_buffer[:opening_start]
+                # Apply reference splitting to the processed text
+                processed_text = split_continuous_references(processed_text)
+                return processed_text, text_buffer[opening_start:]
 
         # Check for partial opening pattern (starts with [ but not complete)
         if "[" in text_buffer:
             ref_start = text_buffer.find("[")
-            return text_buffer[:ref_start], text_buffer[ref_start:]
+            processed_text = text_buffer[:ref_start]
+            # Apply reference splitting to the processed text
+            processed_text = split_continuous_references(processed_text)
+            return processed_text, text_buffer[ref_start:]
 
-        # No reference tags found, return all text
-        return text_buffer, ""
+        # No reference tags found, apply reference splitting and return all text
+        processed_text = split_continuous_references(text_buffer)
+        return processed_text, ""
 
     def _extract_references_from_response(self, response: str) -> tuple[str, list[dict]]:
         """
@@ -760,9 +794,8 @@ def chat_with_references(
         if memories_result:
             memories_list = memories_result[0]["memories"]
             memories_list = self._filter_memories_by_threshold(memories_list)
-        # Build custom system prompt with relevant memories
-        system_prompt = self._build_system_prompt(memories_list, base_prompt=None)
-
+        # Build custom system prompt with relevant memories)
+        system_prompt = self._build_enhance_system_prompt(user_id, memories_list)
         # Get chat history
         if user_id not in self.chat_history_manager:
             self._register_chat_history(user_id)
@@ -775,6 +808,9 @@ def chat_with_references(
             *chat_history.chat_history,
             {"role": "user", "content": query},
         ]
+        logger.info(
+            f"user_id: {user_id}, cube_id: {cube_id}, current_system_prompt: {system_prompt}"
+        )
         yield f"data: {json.dumps({'type': 'status', 'data': '2'})}\n\n"
         # Generate response with custom prompt
         past_key_values = None

src/memos/mem_os/utils/format_utils.py

@@ -1355,3 +1355,47 @@ def clean_json_response(response: str) -> str:
         str: Clean JSON string without markdown formatting
     """
     return response.replace("```json", "").replace("```", "").strip()
+
+
+def split_continuous_references(text: str) -> str:
+    """
+    Split continuous reference tags into individual reference tags.
+
+    Converts patterns like [1:92ff35fb, 4:bfe6f044] to [1:92ff35fb] [4:bfe6f044]
+
+    Only processes text if:
+    1. '[' appears exactly once
+    2. ']' appears exactly once
+    3. Contains commas between '[' and ']'
+
+    Args:
+        text (str): Text containing reference tags
+
+    Returns:
+        str: Text with split reference tags, or original text if conditions not met
+    """
+    # Early return if text is empty
+    if not text:
+        return text
+    # Check if '[' appears exactly once
+    if text.count("[") != 1:
+        return text
+    # Check if ']' appears exactly once
+    if text.count("]") != 1:
+        return text
+    # Find positions of brackets
+    open_bracket_pos = text.find("[")
+    close_bracket_pos = text.find("]")
+
+    # Check if brackets are in correct order
+    if open_bracket_pos >= close_bracket_pos:
+        return text
+    # Extract content between brackets
+    content_between_brackets = text[open_bracket_pos + 1 : close_bracket_pos]
+    # Check if there's a comma between brackets
+    if "," not in content_between_brackets:
+        return text
+    text = text.replace(content_between_brackets, content_between_brackets.replace(", ", "]["))
+    text = text.replace(content_between_brackets, content_between_brackets.replace(",", "]["))
+
+    return text

src/memos/memories/textual/tree_text_memory/retrieve/searcher.py

@@ -215,6 +215,8 @@ def retrieve_from_internet():
 
         # Step 5: Update usage history with current timestamp
         now_time = datetime.now().isoformat()
+        if "chat_history" in info:
+            info.pop("chat_history")
         usage_record = json.dumps(
             {"time": now_time, "info": info}
         )  # `info` should be a serializable dict or string

src/memos/templates/mos_prompts.py

@@ -62,6 +62,57 @@
 4. Is well-structured and easy to understand
 5. Maintains a natural conversational tone"""
 
+MEMOS_PRODUCT_BASE_PROMPT = (
+    "You are a knowledgeable and helpful AI assistant with access to user memories. "
+    "When responding to user queries, you should reference relevant memories using the provided memory IDs. "
+    "Use the reference format: [1-n:memoriesID] "
+    "where refid is a sequential number starting from 1 and increments for each reference in your response, "
+    "and memoriesID is the specific memory ID provided in the available memories list. "
+    "For example: [1:abc123], [2:def456], [3:ghi789], [4:jkl101], [5:mno112] "
+    "Do not use connect format like [1:abc123,2:def456]"
+    "Only reference memories that are directly relevant to the user's question. "
+    "Make your responses natural and conversational while incorporating memory references when appropriate."
+)
+
+MEMOS_PRODUCT_ENHANCE_PROMPT = """
+# Memory-Enhanced AI Assistant Prompt
+
+You are a knowledgeable and helpful AI assistant with access to two types of memory sources:
+
+## Memory Types
+- **PersonalMemory**: User-specific memories and information stored from previous interactions
+- **OuterMemory**: External information retrieved from the internet and other sources
+
+## Memory Reference Guidelines
+
+### Reference Format
+When citing memories in your responses, use the following format:
+- `[refid:memoriesID]` where:
+  - `refid` is a sequential number starting from 1 and incrementing for each reference
+  - `memoriesID` is the specific memory ID from the available memories list
+
+### Reference Examples
+- Correct: `[1:abc123]`, `[2:def456]`, `[3:ghi789]`, `[4:jkl101]`, `[5:mno112]`
+- Incorrect: `[1:abc123,2:def456]` (do not use connected format)
+
+## Response Guidelines
+
+### Memory Selection
+- Intelligently choose which memories (PersonalMemory or OuterMemory) are most relevant to the user's query
+- Only reference memories that are directly relevant to the user's question
+- Prioritize the most appropriate memory type based on the context and nature of the query
+
+### Response Style
+- Make your responses natural and conversational
+- Seamlessly incorporate memory references when appropriate
+- Ensure the flow of conversation remains smooth despite memory citations
+- Balance factual accuracy with engaging dialogue
+
+## Key Principles
+- Reference only relevant memories to avoid information overload
+- Maintain conversational tone while being informative
+- Use memory references to enhance, not disrupt, the user experience
+"""
 QUERY_REWRITING_PROMPT = """
 I'm in discussion with my friend about a question, and we have already talked about something before that. Please help me analyze the logic between the question and the former dialogue, and rewrite the question we are discussing about.