Merge branch 'dev' into feat/mos-product-api-0716

Author: tangg555

examples/core_memories/general_textual_memory.py

@@ -1,7 +1,6 @@
 from memos.configs.memory import MemoryConfigFactory
 from memos.memories.factory import MemoryFactory
 
-
 config = MemoryConfigFactory(
     backend="general_text",
     config={
@@ -36,33 +35,24 @@
     {
         "memory": "I'm a RUCer, I'm happy.",
         "metadata": {
-            "type": "self-introduction",
-            "memory_time": "2025-05-26",
+            "key": "happy RUCer",
             "source": "conversation",
-            "confidence": 90.0,
-            "entities": ["RUCer"],
             "tags": ["happy"],
-            "visibility": "private",
             "updated_at": "2025-05-19T00:00:00",
         },
     },
     {
         "memory": "MemOS is awesome!",
         "metadata": {
-            "type": "fact",
-            "memory_time": "2025-05-19",
+            "key": "MemOS",
             "source": "conversation",
-            "confidence": 100.0,
-            "entities": ["MemOS"],
             "tags": ["awesome"],
-            "visibility": "public",
             "updated_at": "2025-05-19T00:00:00",
         },
     },
 ]
 example_id = "a19b6caa-5d59-42ad-8c8a-e4f7118435b4"
 
-
 print("===== Extract memories =====")
 memories = m.extract(
     [
@@ -80,7 +70,12 @@
         {
             "id": example_id,
             "memory": "User is Chinese.",
-            "metadata": {"type": "opinion"},
+            "metadata": {
+                "key": "User Nationality",
+                "source": "conversation",
+                "tags": ["Nationality"],
+                "updated_at": "2025-05-18T00:00:00",
+            },
         }
     ]
 )
@@ -103,13 +98,9 @@
         "id": example_id,
         "memory": "User is Canadian.",
         "metadata": {
-            "type": "opinion",
-            "confidence": 85,
-            "memory_time": "2025-05-24",
+            "key": "User Nationality",
             "source": "conversation",
-            "entities": ["Canadian"],
-            "tags": ["happy"],
-            "visibility": "private",
+            "tags": ["Nationality"],
             "updated_at": "2025-05-19T00:00:00",
         },
     },

poetry.lock

@@ -44,6 +44,7 @@ dependencies = [
     "sqlalchemy (>=2.0.41,<3.0.0)",  # SQL toolkit
     "scikit-learn (>=1.7.0,<2.0.0)",  # Machine learning
     "fastmcp (>=2.10.5,<3.0.0)",
+    "python-dateutil (>=2.9.0.post0,<3.0.0)",
 ]
 
 [project.urls]

pyproject.toml

@@ -58,9 +58,13 @@ def _process_chat_data(self, scene_data_info, info):
                 metadata=TreeNodeTextualMemoryMetadata(
                     user_id=info.get("user_id"),
                     session_id=info.get("session_id"),
-                    memory_type=memory_i_raw.get("memory_type", ""),
+                    memory_type=memory_i_raw.get("memory_type", "")
+                    .replace("长期记忆", "LongTermMemory")
+                    .replace("用户记忆", "UserMemory"),
                     status="activated",
-                    tags=memory_i_raw.get("tags", ""),
+                    tags=memory_i_raw.get("tags", [])
+                    if type(memory_i_raw.get("tags", [])) is list
+                    else [],
                     key=memory_i_raw.get("key", ""),
                     embedding=self.embedder.embed([memory_i_raw.get("value", "")])[0],
                     usage=[],
@@ -214,7 +218,7 @@ def _process_doc_data(self, scene_data_info, info):
                         session_id=info.get("session_id"),
                         memory_type="LongTermMemory",
                         status="activated",
-                        tags=chunk_res["tags"],
+                        tags=chunk_res["tags"] if type(chunk_res["tags"]) is list else [],
                         key=chunk_res["key"],
                         embedding=self.embedder.embed([chunk_res["value"]])[0],
                         usage=[],

src/memos/mem_reader/simple_struct.py

@@ -12,11 +12,11 @@
 from memos.log import get_logger
 from memos.memories.textual.base import BaseTextMemory
 from memos.memories.textual.item import TextualMemoryItem
+from memos.templates.mem_reader_prompts import SIMPLE_STRUCT_MEM_READER_PROMPT
 from memos.types import MessageList
 from memos.vec_dbs.factory import QdrantVecDB, VecDBFactory
 from memos.vec_dbs.item import VecDBItem
 
-
 logger = get_logger(__name__)
 
 
@@ -36,7 +36,7 @@ def __init__(self, config: GeneralTextMemoryConfig):
         stop=stop_after_attempt(3),
         retry=retry_if_exception_type(json.JSONDecodeError),
         before_sleep=lambda retry_state: logger.warning(
-            EXTRACTION_RETRY_LOG.format(
+            "Extracting memory failed due to JSON decode error: {error}, Attempt retry: {attempt_number} / {max_attempt_number}".format(
                 error=retry_state.outcome.exception(),
                 attempt_number=retry_state.attempt_number,
                 max_attempt_number=3,
@@ -52,14 +52,27 @@ def extract(self, messages: MessageList) -> list[TextualMemoryItem]:
         Returns:
             List of TextualMemoryItem objects representing the extracted memories.
         """
-        str_messages = json.dumps(messages)
-        user_query = EXTRACTION_PROMPT_PART_1 + EXTRACTION_PROMPT_PART_2.format(
-            messages=str_messages
+
+        str_messages = "\n".join(
+            [message["role"] + ":" + message["content"] for message in messages]
         )
-        response = self.extractor_llm.generate([{"role": "user", "content": user_query}])
-        raw_extracted_memories = json.loads(response)
+
+        prompt = SIMPLE_STRUCT_MEM_READER_PROMPT.replace("${conversation}", str_messages)
+        messages = [{"role": "user", "content": prompt}]
+        response_text = self.extractor_llm.generate(messages)
+        response_json = self.parse_json_result(response_text)
+
         extracted_memories = [
-            TextualMemoryItem(**memory_dict) for memory_dict in raw_extracted_memories
+            TextualMemoryItem(
+                memory=memory_dict["value"],
+                metadata={
+                    "key": memory_dict["key"],
+                    "source": "conversation",
+                    "tags": memory_dict["tags"],
+                    "updated_at": datetime.now().isoformat(),
+                },
+            )
+            for memory_dict in response_json["memory list"]
         ]
 
         return extracted_memories
@@ -206,83 +219,17 @@ def _embed_one_sentence(self, sentence: str) -> list[float]:
         """Embed a single sentence."""
         return self.embedder.embed([sentence])[0]
 
-
-EXTRACTION_PROMPT_PART_1 = f"""You are a memory extractor. Your task is to extract memories from the given messages.
-* You will receive a list of messages, each with a role (user or assistant) and content.
-* Your job is to extract memories related to the user's long-term goals, interests, and emotional states.
-* Each memory should be a dictionary with the following keys:
-    - "memory": The content of the memory (string). Rephrase the content if necessary.
-    - "metadata": A dictionary containing additional information about the memory.
-* The metadata dictionary should include:
-    - "type": The type of memory (string), e.g., "procedure", "fact", "event", "opinion", etc.
-    - "memory_time": The time the memory occurred or refers to (string). Must be in standard `YYYY-MM-DD` format. Relative expressions such as "yesterday" or "tomorrow" are not allowed.
-    - "source": The origin of the memory (string), e.g., `"conversation"`, `"retrieved"`, `"web"`, `"file"`.
-    - "confidence": A numeric score (float between 0 and 100) indicating how certain you are about the accuracy or reliability of the memory.
-    - "entities": A list of key entities (array of strings) mentioned in the memory, e.g., people, places, organizations, e.g., `["Alice", "Paris", "OpenAI"]`.
-    - "tags": A list of keywords or thematic labels (array of strings) associated with the memory for categorization or retrieval, e.g., `["travel", "health", "project-x"]`.
-    - "visibility": The accessibility scope of the memory (string), e.g., `"private"`, `"public"`, `"session"`, determining who or what contexts can access it.
-    - "updated_at": The timestamp of the last modification to the memory (string). Useful for tracking memory freshness or change history. Format: ISO 8601 or natural language.
-* Current date and time is {datetime.now().isoformat()}.
-* Only return the list of memories in JSON format.
-* Do not include any explanations
-* Do not include any extra text
-* Do not include code blocks (```json```)
-
-## Example
-
-### Input
-
-[
-    {{"role": "user", "content": "I plan to visit Paris next week."}},
-    {{"role": "assistant", "content": "Paris is a beautiful city with many attractions."}},
-    {{"role": "user", "content": "I love the Eiffel Tower."}},
-    {{"role": "assistant", "content": "The Eiffel Tower is a must-see landmark in Paris."}}
-]
-
-### Output
-
-[
-  {{
-    "memory": "The user plans to visit Paris on 05-26-2025.",
-    "metadata": {{
-      "type": "event",
-      "memory_time": "2025-05-26",
-      "source": "conversation",
-      "confidence": 90.0,
-      "entities": ["Paris"],
-      "tags": ["travel", "plans"],
-      "visibility": "private",
-      "updated_at": "2025-05-19T00:00:00"
-    }}
-  }},
-  {{
-    "memory": "The user loves the Eiffel Tower.",
-    "metadata": {{
-      "type": "opinion",
-      "memory_time": "2025-05-19",
-      "source": "conversation",
-      "confidence": 100.0,
-      "entities": ["Eiffel Tower"],
-      "tags": ["opinions", "landmarks"],
-      "visibility": "session",
-      "updated_at": "2025-05-19T00:00:00"
-    }}
-  }}
-]
-
-"""
-
-EXTRACTION_PROMPT_PART_2 = """
-## Query
-
-### Input
-
-{messages}
-
-### Output
-
-"""
-
-EXTRACTION_RETRY_LOG = """Extracting memory failed due to JSON decode error: {error},
-Attempt retry: {attempt_number} / {max_attempt_number}
-"""
+    def parse_json_result(self, response_text):
+        try:
+            json_start = response_text.find("{")
+            response_text = response_text[json_start:]
+            response_text = response_text.replace("```", "").strip()
+            if response_text[-1] != "}":
+                response_text += "}"
+            response_json = json.loads(response_text)
+            return response_json
+        except json.JSONDecodeError as e:
+            logger.warning(
+                f"Failed to parse LLM response as JSON: {e}\nRaw response:\n{response_text}"
+            )
+            return {}

src/memos/memories/textual/general.py

@@ -27,23 +27,14 @@ class TextualMemoryMetadata(BaseModel):
         default="activated",
         description="The status of the memory, e.g., 'activated', 'archived', 'deleted'.",
     )
-    type: Literal["procedure", "fact", "event", "opinion", "topic", "reasoning"] | None = Field(
-        default=None
-    )
-    memory_time: str | None = Field(
-        default=None,
-        description='The time the memory occurred or refers to. Must be in standard `YYYY-MM-DD` format. Relative expressions such as "yesterday" or "tomorrow" are not allowed.',
-    )
-    source: Literal["conversation", "retrieved", "web", "file"] | None = Field(
-        default=None, description="The origin of the memory"
-    )
+    type: str | None = Field(default=None)
+    key: str | None = Field(default=None, description="Memory key or title.")
     confidence: float | None = Field(
         default=None,
         description="A numeric score (float between 0 and 100) indicating how certain you are about the accuracy or reliability of the memory.",
     )
-    entities: list[str] | None = Field(
-        default=None,
-        description='A list of key entities mentioned in the memory, e.g., people, places, organizations, e.g., `["Alice", "Paris", "OpenAI"]`.',
+    source: Literal["conversation", "retrieved", "web", "file"] | None = Field(
+        default=None, description="The origin of the memory"
     )
     tags: list[str] | None = Field(
         default=None,
@@ -59,23 +50,6 @@ class TextualMemoryMetadata(BaseModel):
 
     model_config = ConfigDict(extra="allow")
 
-    @field_validator("memory_time")
-    @classmethod
-    def validate_memory_time(cls, v):
-        try:
-            if v:
-                datetime.strptime(v, "%Y-%m-%d")
-        except ValueError as e:
-            raise ValueError("Invalid date format. Use YYYY-MM-DD.") from e
-        return v
-
-    @field_validator("confidence")
-    @classmethod
-    def validate_confidence(cls, v):
-        if v is not None and (v < 0 or v > 100):
-            raise ValueError("Confidence must be between 0 and 100.")
-        return v
-
     def __str__(self) -> str:
         """Pretty string representation of the metadata."""
         meta = self.model_dump(exclude_none=True)
@@ -88,7 +62,6 @@ class TreeNodeTextualMemoryMetadata(TextualMemoryMetadata):
     memory_type: Literal["WorkingMemory", "LongTermMemory", "UserMemory"] = Field(
         default="WorkingMemory", description="Memory lifecycle type."
     )
-    key: str | None = Field(default=None, description="Memory key or title.")
     sources: list[str] | None = Field(
         default=None, description="Multiple origins of the memory (e.g., URLs, notes)."
     )
@@ -148,7 +121,6 @@ class TextualMemoryItem(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
-    @field_validator("id")
     @classmethod
     def validate_id(cls, v):
         try:

src/memos/memories/textual/item.py

@@ -2,7 +2,7 @@
 import re
 
 from datetime import datetime
-
+from dateutil import parser
 from memos.embedders.base import BaseEmbedder
 from memos.graph_dbs.neo4j import Neo4jGraphDB
 from memos.llms.base import BaseLLM
@@ -133,8 +133,8 @@ def _hard_update(self, memory_a: TextualMemoryItem, memory_b: TextualMemoryItem)
         """
         Hard update: compare updated_at, keep the newer one, overwrite the older one's metadata.
         """
-        time_a = datetime.fromisoformat(memory_a.metadata.updated_at)
-        time_b = datetime.fromisoformat(memory_b.metadata.updated_at)
+        time_a = parser.isoparse(memory_a.metadata.updated_at)
+        time_b = parser.isoparse(memory_b.metadata.updated_at)
 
         newer_mem = memory_a if time_a >= time_b else memory_b
         older_mem = memory_b if time_a >= time_b else memory_a

src/memos/memories/textual/tree_text_memory/organize/conflict.py

@@ -150,7 +150,7 @@
   "summary": "Tom is currently focused on managing a new project with a tight schedule. After a team meeting on June 25, 2025, he realized the original deadline of December 15 might not be feasible due to backend delays. Concerned about insufficient testing time, he welcomed Jerry’s suggestion of proposing an extension. Tom plans to raise the idea of shifting the deadline to January 5, 2026 in the next morning’s meeting. His actions reflect both stress about timelines and a proactive, team-oriented problem-solving approach."
 }
 
-Another Example in Chinese (注意: 你的输出必须和输入的user语言一致)：
+Another Example in Chinese (注意: 当user的语言为中文时，你就需要也输出中文)：
 {
   "memory list": [
     {