feat: add image type

Author: CaralHsi

src/memos/mem_reader/read_multi_modal/__init__.py

@@ -16,6 +16,7 @@
 from .assistant_parser import AssistantParser
 from .base import BaseMessageParser
 from .file_content_parser import FileContentParser
+from .image_parser import ImageParser
 from .multi_modal_parser import MultiModalParser
 from .string_parser import StringParser
 from .system_parser import SystemParser
@@ -29,6 +30,7 @@
     "AssistantParser",
     "BaseMessageParser",
     "FileContentParser",
+    "ImageParser",
     "MultiModalParser",
     "StringParser",
     "SystemParser",

src/memos/mem_reader/read_multi_modal/assistant_parser.py

@@ -227,6 +227,10 @@ def parse_fast(
         # Combine all content parts
         content = " ".join(content_parts) if content_parts else ""
 
+        # If content is empty but we have tool_calls, audio, or refusal, still create memory
+        if not content and not tool_calls and not audio and not refusal:
+            return []
+
         parts = [f"{role}: "]
         if chat_time:
             parts.append(f"[{chat_time}]: ")

src/memos/mem_reader/read_multi_modal/image_parser.py

@@ -0,0 +1,93 @@
+"""Parser for image_url content parts."""
+
+from typing import Any
+
+from memos.embedders.base import BaseEmbedder
+from memos.llms.base import BaseLLM
+from memos.log import get_logger
+from memos.memories.textual.item import SourceMessage, TextualMemoryItem
+from memos.types.openai_chat_completion_types import ChatCompletionContentPartImageParam
+
+from .base import BaseMessageParser
+
+
+logger = get_logger(__name__)
+
+
+class ImageParser(BaseMessageParser):
+    """Parser for image_url content parts."""
+
+    def __init__(self, embedder: BaseEmbedder, llm: BaseLLM | None = None):
+        """
+        Initialize ImageParser.
+
+        Args:
+            embedder: Embedder for generating embeddings
+            llm: Optional LLM for fine mode processing
+        """
+        super().__init__(embedder, llm)
+
+    def create_source(
+        self,
+        message: ChatCompletionContentPartImageParam,
+        info: dict[str, Any],
+    ) -> SourceMessage:
+        """Create SourceMessage from image_url content part."""
+        if isinstance(message, dict):
+            image_url = message.get("image_url", {})
+            if isinstance(image_url, dict):
+                url = image_url.get("url", "")
+                detail = image_url.get("detail", "auto")
+            else:
+                url = str(image_url)
+                detail = "auto"
+            return SourceMessage(
+                type="image",
+                content=f"[image_url]: {url}",
+                original_part=message,
+                url=url,
+                detail=detail,
+            )
+        return SourceMessage(type="image", content=str(message))
+
+    def rebuild_from_source(
+        self,
+        source: SourceMessage,
+    ) -> ChatCompletionContentPartImageParam:
+        """Rebuild image_url content part from SourceMessage."""
+        # Use original_part if available
+        if hasattr(source, "original_part") and source.original_part:
+            return source.original_part
+
+        # Rebuild from source fields
+        url = getattr(source, "url", "") or (source.content or "").replace("[image_url]: ", "")
+        detail = getattr(source, "detail", "auto")
+        return {
+            "type": "image_url",
+            "image_url": {
+                "url": url,
+                "detail": detail,
+            },
+        }
+
+    def parse_fast(
+        self,
+        message: ChatCompletionContentPartImageParam,
+        info: dict[str, Any],
+        **kwargs,
+    ) -> list[TextualMemoryItem]:
+        """Parse image_url in fast mode - returns empty list as images need fine mode processing."""
+        # In fast mode, images are not processed (they need vision models)
+        # They will be processed in fine mode via process_transfer
+        return []
+
+    def parse_fine(
+        self,
+        message: ChatCompletionContentPartImageParam,
+        info: dict[str, Any],
+        **kwargs,
+    ) -> list[TextualMemoryItem]:
+        """Parse image_url in fine mode - placeholder for future vision model integration."""
+        # Fine mode processing would use vision models to extract text from images
+        # For now, return empty list
+        return []

src/memos/mem_reader/read_multi_modal/multi_modal_parser.py

@@ -15,6 +15,7 @@
 from .assistant_parser import AssistantParser
 from .base import BaseMessageParser
 from .file_content_parser import FileContentParser
+from .image_parser import ImageParser
 from .string_parser import StringParser
 from .system_parser import SystemParser
 from .text_content_parser import TextContentParser
@@ -55,7 +56,7 @@ def __init__(
         self.tool_parser = ToolParser(embedder, llm)
         self.text_content_parser = TextContentParser(embedder, llm)
         self.file_content_parser = FileContentParser(embedder, llm, parser)
-        self.image_parser = None  # future
+        self.image_parser = ImageParser(embedder, llm)
         self.audio_parser = None  # future
 
         self.role_parsers = {
@@ -69,7 +70,12 @@ def __init__(
             "text": self.text_content_parser,
             "file": self.file_content_parser,
             "image": self.image_parser,
+            "image_url": self.image_parser,  # Support both "image" and "image_url"
             "audio": self.audio_parser,
+            # Custom tool formats
+            "tool_description": self.tool_parser,
+            "tool_input": self.tool_parser,
+            "tool_output": self.tool_parser,
         }
 
     def _get_parser(self, message: Any) -> BaseMessageParser | None:

src/memos/mem_reader/read_multi_modal/tool_parser.py

@@ -29,16 +29,52 @@ def __init__(self, embedder: BaseEmbedder, llm: BaseLLM | None = None):
 
     def create_source(
         self,
-        message: ChatCompletionToolMessageParam,
+        message: ChatCompletionToolMessageParam | dict[str, Any],
         info: dict[str, Any],
     ) -> SourceMessage:
-        """Create SourceMessage from tool message."""
+        """Create SourceMessage from tool message or custom tool format."""
         if not isinstance(message, dict):
             return SourceMessage(type="chat", role="tool")
 
+        # Handle custom tool formats (tool_description, tool_input, tool_output)
+        msg_type = message.get("type", "")
+        if msg_type == "tool_description":
+            name = message.get("name", "")
+            description = message.get("description", "")
+            parameters = message.get("parameters", {})
+            content = f"[tool_description] name={name}, description={description}, parameters={parameters}"
+            return SourceMessage(
+                type="tool_description",
+                content=content,
+                original_part=message,
+            )
+        elif msg_type == "tool_input":
+            call_id = message.get("call_id", "")
+            name = message.get("name", "")
+            argument = message.get("argument", {})
+            content = f"[tool_input] call_id={call_id}, name={name}, argument={argument}"
+            return SourceMessage(
+                type="tool_input",
+                content=content,
+                message_id=call_id,
+                original_part=message,
+            )
+        elif msg_type == "tool_output":
+            call_id = message.get("call_id", "")
+            name = message.get("name", "")
+            output = message.get("output", {})
+            content = f"[tool_output] call_id={call_id}, name={name}, output={output}"
+            return SourceMessage(
+                type="tool_output",
+                content=content,
+                message_id=call_id,
+                original_part=message,
+            )
+
+        # Handle standard tool message
         content = _extract_text_from_content(message.get("content", ""))
         return SourceMessage(
-            type="chat",
+            type="tool",
             role="tool",
             chat_time=message.get("chat_time"),
             message_id=message.get("message_id"),
@@ -60,10 +96,54 @@ def rebuild_from_source(
 
     def parse_fast(
         self,
-        message: ChatCompletionToolMessageParam,
+        message: ChatCompletionToolMessageParam | dict[str, Any],
         info: dict[str, Any],
         **kwargs,
     ) -> list[TextualMemoryItem]:
+        """Parse tool message in fast mode."""
+        from memos.memories.textual.item import TreeNodeTextualMemoryMetadata
+
+        from .base import _derive_key
+
+        if not isinstance(message, dict):
+            return []
+
+        # Handle custom tool formats
+        msg_type = message.get("type", "")
+        if msg_type in ("tool_description", "tool_input", "tool_output"):
+            # Create source
+            source = self.create_source(message, info)
+            content = source.content or ""
+            if not content:
+                return []
+
+            # Extract info fields
+            info_ = info.copy()
+            user_id = info_.pop("user_id", "")
+            session_id = info_.pop("session_id", "")
+
+            # Create memory item
+            memory_item = TextualMemoryItem(
+                memory=content,
+                metadata=TreeNodeTextualMemoryMetadata(
+                    user_id=user_id,
+                    session_id=session_id,
+                    memory_type="LongTermMemory",
+                    status="activated",
+                    tags=["mode:fast"],
+                    key=_derive_key(content),
+                    embedding=self.embedder.embed([content])[0],
+                    usage=[],
+                    sources=[source],
+                    background="",
+                    confidence=0.99,
+                    type="fact",
+                    info=info_,
+                ),
+            )
+            return [memory_item]
+
+        # Handle standard tool message
         return super().parse_fast(message, info, **kwargs)
 
     def parse_fine(

src/memos/mem_reader/read_multi_modal/utils.py

@@ -93,6 +93,11 @@ def coerce_scene_data(scene_data: SceneDataInput, scene_type: str) -> list[Messa
             if not items:
                 continue
 
+            # Keep string as-is (MessagesType supports str)
+            if isinstance(items, str):
+                complete_scene_data.append(items)
+                continue
+
             # ONLY add chat_time if it's a MessageList
             if not _is_message_list(items):
                 complete_scene_data.append(items)