feat: add memreader to tackle with internet (#154)

* feat: add memreader to tackle with internet

* feat: delete chunker in internet retriever and add 'info' parameter in all retrievers

* test: fix test for updated memreader

* test: fix test for updated memreader

* test: fix test for updated memreader

Author: CaralHsi

src/memos/configs/internet_retriever.py

@@ -4,9 +4,9 @@
 
 from pydantic import Field, field_validator, model_validator
 
-from memos.chunkers.factory import ChunkerConfigFactory
 from memos.configs.base import BaseConfig
 from memos.exceptions import ConfigurationError
+from memos.mem_reader.factory import MemReaderConfigFactory
 
 
 class BaseInternetRetrieverConfig(BaseConfig):
@@ -48,10 +48,10 @@ class XinyuSearchConfig(BaseInternetRetrieverConfig):
     num_per_request: int = Field(
         default=10, description="Number of results per API request (not used for Xinyu)"
     )
-    chunker: ChunkerConfigFactory = Field(
+    reader: MemReaderConfigFactory = Field(
         ...,
-        default_factory=ChunkerConfigFactory,
-        description="Chunker configuration",
+        default_factory=MemReaderConfigFactory,
+        description="Reader configuration",
     )
 
 

src/memos/mem_reader/simple_struct.py

@@ -180,8 +180,12 @@ def get_scene_data_info(self, scene_data: list, type: str) -> list[str]:
         elif type == "doc":
             for item in scene_data:
                 try:
-                    parsed_text = parser.parse(item)
-                    results.append({"file": item, "text": parsed_text})
+                    if not isinstance(item, str):
+                        parsed_text = parser.parse(item)
+                        results.append({"file": "pure_text", "text": parsed_text})
+                    else:
+                        parsed_text = item
+                        results.append({"file": item, "text": parsed_text})
                 except Exception as e:
                     print(f"Error parsing file {item}: {e!s}")
 

src/memos/memories/textual/tree_text_memory/retrieve/internet_retriever.py

@@ -127,7 +127,7 @@ def __init__(
         self.embedder = embedder
 
     def retrieve_from_internet(
-        self, query: str, top_k: int = 10, parsed_goal=None
+        self, query: str, top_k: int = 10, parsed_goal=None, info=None
     ) -> list[TextualMemoryItem]:
         """
         Retrieve information from the internet and convert to TextualMemoryItem format
@@ -136,6 +136,7 @@ def retrieve_from_internet(
             query: Search query
             top_k: Number of results to return
             parsed_goal: Parsed task goal (optional)
+            info (dict): Leave a record of memory consumption.
 
         Returns:
             List of TextualMemoryItem
@@ -157,8 +158,8 @@ def retrieve_from_internet(
             memory_content = f"Title: {title}\nSummary: {snippet}\nSource: {link}"
             # Create metadata
             metadata = TreeNodeTextualMemoryMetadata(
-                user_id=None,
-                session_id=None,
+                user_id=info.get("user_id", ""),
+                session_id=info.get("session_id", ""),
                 status="activated",
                 type="fact",  # Internet search results are usually factual information
                 memory_time=datetime.now().strftime("%Y-%m-%d"),

src/memos/memories/textual/tree_text_memory/retrieve/internet_retriever_factory.py

@@ -2,9 +2,9 @@
 
 from typing import Any, ClassVar
 
-from memos.chunkers.factory import ChunkerFactory
 from memos.configs.internet_retriever import InternetRetrieverConfigFactory
 from memos.embedders.base import BaseEmbedder
+from memos.mem_reader.factory import MemReaderFactory
 from memos.memories.textual.tree_text_memory.retrieve.internet_retriever import (
     InternetGoogleRetriever,
 )
@@ -67,7 +67,7 @@ def from_config(
                 access_key=config.api_key,  # Use api_key as access_key for xinyu
                 search_engine_id=config.search_engine_id,
                 embedder=embedder,
-                chunker=ChunkerFactory.from_config(config.chunker),
+                reader=MemReaderFactory.from_config(config.reader),
                 max_results=config.max_results,
             )
         else:

src/memos/memories/textual/tree_text_memory/retrieve/searcher.py

@@ -141,7 +141,7 @@ def retrieve_from_internet():
             if memory_type not in ["All"]:
                 return []
             internet_items = self.internet_retriever.retrieve_from_internet(
-                query=query, top_k=top_k, parsed_goal=parsed_goal
+                query=query, top_k=top_k, parsed_goal=parsed_goal, info=info
             )
 
             # Convert to the format expected by reranker

src/memos/memories/textual/tree_text_memory/retrieve/xinyusearch.py

@@ -8,10 +8,10 @@
 
 import requests
 
-from memos.chunkers.base import BaseChunker
 from memos.embedders.factory import OllamaEmbedder
 from memos.log import get_logger
-from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata
+from memos.mem_reader.base import BaseMemReader
+from memos.memories.textual.item import TextualMemoryItem
 
 
 logger = get_logger(__name__)
@@ -114,7 +114,7 @@ def __init__(
         access_key: str,
         search_engine_id: str,
         embedder: OllamaEmbedder,
-        chunker: BaseChunker,
+        reader: BaseMemReader,
         max_results: int = 20,
     ):
         """
@@ -124,13 +124,14 @@ def __init__(
             access_key: Xinyu API access key
             embedder: Embedder instance for generating embeddings
             max_results: Maximum number of results to retrieve
+            reader: MemReader Moduel to deal with internet contents
         """
         self.xinyu_api = XinyuSearchAPI(access_key, search_engine_id, max_results=max_results)
         self.embedder = embedder
-        self.chunker = chunker
+        self.reader = reader
 
     def retrieve_from_internet(
-        self, query: str, top_k: int = 10, parsed_goal=None
+        self, query: str, top_k: int = 10, parsed_goal=None, info=None
     ) -> list[TextualMemoryItem]:
         """
         Retrieve information from Xinyu search and convert to TextualMemoryItem format
@@ -139,7 +140,7 @@ def retrieve_from_internet(
             query: Search query
             top_k: Number of results to return
             parsed_goal: Parsed task goal (optional)
-
+            info (dict): Leave a record of memory consumption.
         Returns:
             List of TextualMemoryItem
         """
@@ -151,7 +152,7 @@ def retrieve_from_internet(
 
         with ThreadPoolExecutor(max_workers=8) as executor:
             futures = [
-                executor.submit(self._process_result, result, query, parsed_goal)
+                executor.submit(self._process_result, result, query, parsed_goal, info)
                 for result in search_results
             ]
             for future in as_completed(futures):
@@ -301,7 +302,7 @@ def _extract_tags(self, title: str, content: str, summary: str, parsed_goal=None
         return list(set(tags))[:15]  # Limit to 15 tags
 
     def _process_result(
-        self, result: dict, query: str, parsed_goal: str
+        self, result: dict, query: str, parsed_goal: str, info: dict
     ) -> list[TextualMemoryItem]:
         title = result.get("title", "")
         content = result.get("content", "")
@@ -318,55 +319,19 @@ def _process_result(
                 publish_time = datetime.now().strftime("%Y-%m-%d")
         else:
             publish_time = datetime.now().strftime("%Y-%m-%d")
-        source = result.get("source", "")
-        site = result.get("site", "")
-        if site:
-            site = site.split("|")[0]
 
-        qualified_chunks = self._chunk(content)
+        read_items = self.reader.get_memory([content], type="doc", info=info)
 
         memory_items = []
-        for chunk_text, chunk_emb, score in qualified_chunks:
-            memory_content = (
+        for read_item_i in read_items[0]:
+            read_item_i.memory = (
                 f"Title: {title}\nNewsTime: {publish_time}\nSummary: {summary}\n"
-                f"Content: {chunk_text}\nSource: {url}"
-            )
-            metadata = TreeNodeTextualMemoryMetadata(
-                user_id=None,
-                session_id=None,
-                status="activated",
-                type="fact",
-                source="web",
-                confidence=score,
-                entities=self._extract_entities(title, content, summary),
-                tags=self._extract_tags(title, content, summary, parsed_goal),
-                visibility="public",
-                memory_type="OuterMemory",
-                key=f"[{source}]" + title,
-                sources=[url] if url else [],
-                embedding=chunk_emb,
-                created_at=datetime.now().isoformat(),
-                usage=[],
-                background=f"Xinyu search result from {site or source}",
-            )
-            memory_items.append(
-                TextualMemoryItem(id=str(uuid.uuid4()), memory=memory_content, metadata=metadata)
+                f"Content: {read_item_i.memory}"
             )
+            read_item_i.metadata.source = "web"
+            read_item_i.metadata.memory_type = "OuterMemory"
+            read_item_i.metadata.sources = [url] if url else []
+            read_item_i.metadata.visibility = "public"
 
+            memory_items.append(read_item_i)
         return memory_items
-
-    def _chunk(self, content: str) -> list[tuple[str, list[float], float]]:
-        """
-        Use SentenceChunker to split content into chunks and embed each.
-
-        Returns:
-            List of (chunk_text, chunk_embedding, dummy_score)
-        """
-        chunks = self.chunker.chunk(content)
-        if not chunks:
-            return []
-
-        chunk_texts = [c.text for c in chunks]
-        chunk_embeddings = self.embedder.embed(chunk_texts)
-
-        return [(text, emb, 1.0) for text, emb in zip(chunk_texts, chunk_embeddings, strict=False)]

tests/mem_reader/test_simple_structure.py

@@ -117,18 +117,18 @@ def test_get_scene_data_info_with_chat(self):
         self.assertEqual(len(result), 1)
         self.assertEqual(result[0][0], "user: [3 May 2025]: I'm feeling a bit down today.")
 
-    @patch("memos.parsers.factory.ParserFactory")
+    @patch("memos.mem_reader.simple_struct.ParserFactory")
     def test_get_scene_data_info_with_doc(self, mock_parser_factory):
         """Test parsing document files."""
         parser_instance = MagicMock()
         parser_instance.parse.return_value = "Parsed document text.\n"
         mock_parser_factory.from_config.return_value = parser_instance
 
-        scene_data = ["tests/mem_reader/test.txt"]
+        scene_data = [{"fake_file_like": "should trigger parse"}]
         result = self.reader.get_scene_data_info(scene_data, type="doc")
 
         self.assertIsInstance(result, list)
-        self.assertEqual(result[0]["text"], "Parsed document text\n")
+        self.assertEqual(result[0]["text"], "Parsed document text.\n")
 
     def test_parse_json_result_success(self):
         """Test successful JSON parsing."""