feat: complete multi modal (#562)

* fix: multi-model memreader init error

* fix: kwargs bug

* feat: init examples for each multi-model parser

* feat: simple user_parser

* feat: add multi-model-parser example

* feat: add multi-model-parser example

* feat: update user parser: only tackle with ChatCompletionUserMessageParam message

* feat: rewrite create source and parse fast for system parser

* feat: rewrite create source and parse fast for system parser

* feat: rewrite assistant parser

* feat: add additional sources to assistant parser

* feat: add concat fast-mode memories from multi parsers

* refactor: fix name

* refactor: fix name

* refactor: fix name

* refactor: fix name

* refactor: fix name

* refactor: fix name

* feat: add fine process path-A in multi_modal_struct

* feat: add fine process path-A in multi_modal_struct

* feat: add compare simple&multimodal example

* feat: add _process_transfer_multi_modal_data in multimodal

* feat: add image type

* feat: add tool role; update string/text/tool parser

* feat: update file_content_parser and multimodal reader

* feat: default mem-reader for api is not set to multimodal reqader

Author: CaralHsi

examples/mem_reader/multimodal_struct_reader.py

@@ -426,7 +426,7 @@ def get_embedder_config() -> dict[str, Any]:
     def get_reader_config() -> dict[str, Any]:
         """Get reader configuration."""
         return {
-            "backend": os.getenv("MEM_READER_BACKEND", "simple_struct"),
+            "backend": os.getenv("MEM_READER_BACKEND", "multimodal_struct"),
             "config": {
                 "chunk_type": os.getenv("MEM_READER_CHAT_CHUNK_TYPE", "default"),
                 "chunk_length": int(os.getenv("MEM_READER_CHAT_CHUNK_TOKEN_SIZE", 1600)),

src/memos/api/config.py

@@ -16,6 +16,7 @@
 from .assistant_parser import AssistantParser
 from .base import BaseMessageParser
 from .file_content_parser import FileContentParser
+from .image_parser import ImageParser
 from .multi_modal_parser import MultiModalParser
 from .string_parser import StringParser
 from .system_parser import SystemParser
@@ -29,6 +30,7 @@
     "AssistantParser",
     "BaseMessageParser",
     "FileContentParser",
+    "ImageParser",
     "MultiModalParser",
     "StringParser",
     "SystemParser",

src/memos/mem_reader/read_multi_modal/__init__.py

@@ -227,6 +227,10 @@ def parse_fast(
         # Combine all content parts
         content = " ".join(content_parts) if content_parts else ""
 
+        # If content is empty but we have tool_calls, audio, or refusal, still create memory
+        if not content and not tool_calls and not audio and not refusal:
+            return []
+
         parts = [f"{role}: "]
         if chat_time:
             parts.append(f"[{chat_time}]: ")

src/memos/mem_reader/read_multi_modal/assistant_parser.py

@@ -5,11 +5,15 @@
 from memos.embedders.base import BaseEmbedder
 from memos.llms.base import BaseLLM
 from memos.log import get_logger
-from memos.memories.textual.item import SourceMessage, TextualMemoryItem
+from memos.memories.textual.item import (
+    SourceMessage,
+    TextualMemoryItem,
+    TreeNodeTextualMemoryMetadata,
+)
 from memos.parsers.factory import ParserFactory
 from memos.types.openai_chat_completion_types import File
 
-from .base import BaseMessageParser
+from .base import BaseMessageParser, _derive_key
 
 
 logger = get_logger(__name__)
@@ -121,7 +125,111 @@ def parse_fast(
         info: dict[str, Any],
         **kwargs,
     ) -> list[TextualMemoryItem]:
-        return []
+        """
+        Parse file content part in fast mode.
+
+        Fast mode extracts file information and creates a memory item without parsing file content.
+        Handles various file parameter scenarios:
+        - file_data: base64 encoded data, URL, or plain text content
+        - file_id: ID of an uploaded file
+        - filename: name of the file
+
+        Args:
+            message: File content part to parse (dict with "type": "file" and "file": {...})
+            info: Dictionary containing user_id and session_id
+            **kwargs: Additional parameters
+
+        Returns:
+            List of TextualMemoryItem objects
+        """
+        if not isinstance(message, dict):
+            logger.warning(f"[FileContentParser] Expected dict, got {type(message)}")
+            return []
+
+        # Extract file information
+        file_info = message.get("file", {})
+        if not isinstance(file_info, dict):
+            logger.warning(f"[FileContentParser] Expected file dict, got {type(file_info)}")
+            return []
+
+        # Extract file parameters (all are optional)
+        file_data = file_info.get("file_data", "")
+        file_id = file_info.get("file_id", "")
+        filename = file_info.get("filename", "")
+
+        # Build content string based on available information
+        content_parts = []
+
+        # Priority 1: If file_data is provided, use it (could be base64, URL, or plain text)
+        if file_data:
+            # In fast mode, we don't decode base64 or fetch URLs, just record the reference
+            if isinstance(file_data, str):
+                # Check if it looks like base64 (starts with data: or is long base64 string)
+                if file_data.startswith("data:") or (
+                    len(file_data) > 100
+                    and all(
+                        c in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
+                        for c in file_data[:100]
+                    )
+                ):
+                    content_parts.append(f"[File Data (base64/encoded): {len(file_data)} chars]")
+                # Check if it looks like a URL
+                elif file_data.startswith(("http://", "https://", "file://")):
+                    content_parts.append(f"[File URL: {file_data}]")
+                else:
+                    # TODO: split into multiple memory items
+                    content_parts.append(file_data)
+            else:
+                content_parts.append(f"[File Data: {type(file_data).__name__}]")
+
+        # Priority 2: If file_id is provided, reference it
+        if file_id:
+            content_parts.append(f"[File ID: {file_id}]")
+
+        # Priority 3: If filename is provided, include it
+        if filename:
+            content_parts.append(f"[Filename: {filename}]")
+
+        # If no content can be extracted, create a placeholder
+        if not content_parts:
+            content_parts.append("[File: unknown]")
+
+        # Combine content parts
+        content = " ".join(content_parts)
+
+        # Create source
+        source = self.create_source(message, info)
+
+        # Extract info fields
+        info_ = info.copy()
+        user_id = info_.pop("user_id", "")
+        session_id = info_.pop("session_id", "")
+
+        # For file content parts, default to LongTermMemory
+        # (since we don't have role information at this level)
+        memory_type = "LongTermMemory"
+
+        # Create memory item
+        memory_item = TextualMemoryItem(
+            memory=content,
+            metadata=TreeNodeTextualMemoryMetadata(
+                user_id=user_id,
+                session_id=session_id,
+                memory_type=memory_type,
+                status="activated",
+                tags=["mode:fast", "multimodal:file"],
+                key=_derive_key(content),
+                embedding=self.embedder.embed([content])[0],
+                usage=[],
+                sources=[source],
+                background="",
+                confidence=0.99,
+                type="fact",
+                info=info_,
+            ),
+        )
+
+        return [memory_item]
 
     def parse_fine(
         self,

src/memos/mem_reader/read_multi_modal/file_content_parser.py

@@ -0,0 +1,93 @@
+"""Parser for image_url content parts."""
+
+from typing import Any
+
+from memos.embedders.base import BaseEmbedder
+from memos.llms.base import BaseLLM
+from memos.log import get_logger
+from memos.memories.textual.item import SourceMessage, TextualMemoryItem
+from memos.types.openai_chat_completion_types import ChatCompletionContentPartImageParam
+
+from .base import BaseMessageParser
+
+
+logger = get_logger(__name__)
+
+
+class ImageParser(BaseMessageParser):
+    """Parser for image_url content parts."""
+
+    def __init__(self, embedder: BaseEmbedder, llm: BaseLLM | None = None):
+        """
+        Initialize ImageParser.
+
+        Args:
+            embedder: Embedder for generating embeddings
+            llm: Optional LLM for fine mode processing
+        """
+        super().__init__(embedder, llm)
+
+    def create_source(
+        self,
+        message: ChatCompletionContentPartImageParam,
+        info: dict[str, Any],
+    ) -> SourceMessage:
+        """Create SourceMessage from image_url content part."""
+        if isinstance(message, dict):
+            image_url = message.get("image_url", {})
+            if isinstance(image_url, dict):
+                url = image_url.get("url", "")
+                detail = image_url.get("detail", "auto")
+            else:
+                url = str(image_url)
+                detail = "auto"
+            return SourceMessage(
+                type="image",
+                content=f"[image_url]: {url}",
+                original_part=message,
+                url=url,
+                detail=detail,
+            )
+        return SourceMessage(type="image", content=str(message))
+
+    def rebuild_from_source(
+        self,
+        source: SourceMessage,
+    ) -> ChatCompletionContentPartImageParam:
+        """Rebuild image_url content part from SourceMessage."""
+        # Use original_part if available
+        if hasattr(source, "original_part") and source.original_part:
+            return source.original_part
+
+        # Rebuild from source fields
+        url = getattr(source, "url", "") or (source.content or "").replace("[image_url]: ", "")
+        detail = getattr(source, "detail", "auto")
+        return {
+            "type": "image_url",
+            "image_url": {
+                "url": url,
+                "detail": detail,
+            },
+        }
+
+    def parse_fast(
+        self,
+        message: ChatCompletionContentPartImageParam,
+        info: dict[str, Any],
+        **kwargs,
+    ) -> list[TextualMemoryItem]:
+        """Parse image_url in fast mode - returns empty list as images need fine mode processing."""
+        # In fast mode, images are not processed (they need vision models)
+        # They will be processed in fine mode via process_transfer
+        return []
+
+    def parse_fine(
+        self,
+        message: ChatCompletionContentPartImageParam,
+        info: dict[str, Any],
+        **kwargs,
+    ) -> list[TextualMemoryItem]:
+        """Parse image_url in fine mode - placeholder for future vision model integration."""
+        # Fine mode processing would use vision models to extract text from images
+        # For now, return empty list
+        return []

src/memos/mem_reader/read_multi_modal/image_parser.py

@@ -15,6 +15,7 @@
 from .assistant_parser import AssistantParser
 from .base import BaseMessageParser
 from .file_content_parser import FileContentParser
+from .image_parser import ImageParser
 from .string_parser import StringParser
 from .system_parser import SystemParser
 from .text_content_parser import TextContentParser
@@ -55,7 +56,7 @@ def __init__(
         self.tool_parser = ToolParser(embedder, llm)
         self.text_content_parser = TextContentParser(embedder, llm)
         self.file_content_parser = FileContentParser(embedder, llm, parser)
-        self.image_parser = None  # future
+        self.image_parser = ImageParser(embedder, llm)
         self.audio_parser = None  # future
 
         self.role_parsers = {
@@ -69,7 +70,12 @@ def __init__(
             "text": self.text_content_parser,
             "file": self.file_content_parser,
             "image": self.image_parser,
+            "image_url": self.image_parser,  # Support both "image" and "image_url"
             "audio": self.audio_parser,
+            # Custom tool formats
+            "tool_description": self.tool_parser,
+            "tool_input": self.tool_parser,
+            "tool_output": self.tool_parser,
         }
 
     def _get_parser(self, message: Any) -> BaseMessageParser | None: