Feat: sync hotfix to dev and add full text for polardb (#563)

* feat: update memos headers

* feat: headers add

* feat: update search agent

* feat: upadte mem story

* feat: update mem scehduler

* feat: update deepsearch mem code

* feat:  update deepsearch agent

* feat: update test code

* fix: remove dup config

* feat: dock search pipeline

* fix: code test

* feat: add test scripts

* feat: add test

* feat: update need_raw process

* fix: add initter

* fix: change agent search func name

* feat: update logs and defined

* feat:  update full text mem search

* feat: cp  plugin  to dev

Author: fridayL

src/memos/api/handlers/component_init.py

@@ -41,6 +41,7 @@
 from memos.memories.textual.simple_preference import SimplePreferenceTextMemory
 from memos.memories.textual.simple_tree import SimpleTreeTextMemory
 from memos.memories.textual.tree_text_memory.organize.manager import MemoryManager
+from memos.memories.textual.tree_text_memory.retrieve.retrieve_utils import FastTokenizer
 
 
 if TYPE_CHECKING:
@@ -196,6 +197,7 @@ def init_server() -> dict[str, Any]:
 
     logger.debug("Memory manager initialized")
 
+    tokenizer = FastTokenizer()
     # Initialize text memory
     text_mem = SimpleTreeTextMemory(
         llm=llm,
@@ -206,6 +208,7 @@ def init_server() -> dict[str, Any]:
         memory_manager=memory_manager,
         config=default_cube_config.text_mem.config,
         internet_retriever=internet_retriever,
+        tokenizer=tokenizer,
     )
 
     logger.debug("Text memory initialized")

src/memos/api/product_models.py

@@ -388,6 +388,12 @@ class APISearchRequest(BaseRequest):
         description="(Internal) Operation definitions for multi-cube read permissions.",
     )
 
+    # ==== Source for  plugin ====
+    source: str | None = Field(
+        None,
+        description="Source of the search query [plugin will router diff search]",
+    )
+
     @model_validator(mode="after")
     def _convert_deprecated_fields(self) -> "APISearchRequest":
         """

src/memos/graph_dbs/polardb.py

@@ -1451,6 +1451,130 @@ def get_context_chain(self, id: str, type: str = "FOLLOWS") -> list[str]:
         """Get the ordered context chain starting from a node."""
         raise NotImplementedError
 
+    @timed
+    def search_by_fulltext(
+        self,
+        query_words: list[str],
+        top_k: int = 10,
+        scope: str | None = None,
+        status: str | None = None,
+        threshold: float | None = None,
+        search_filter: dict | None = None,
+        user_name: str | None = None,
+        filter: dict | None = None,
+        knowledgebase_ids: list[str] | None = None,
+        tsvector_field: str = "properties_tsvector_zh",
+        tsquery_config: str = "jiebaqry",
+        **kwargs,
+    ) -> list[dict]:
+        """
+        Full-text search functionality using PostgreSQL's full-text search capabilities.
+
+        Args:
+            query_text: query text
+            top_k: maximum number of results to return
+            scope: memory type filter (memory_type)
+            status: status filter, defaults to "activated"
+            threshold: similarity threshold filter
+            search_filter: additional property filter conditions
+            user_name: username filter
+            knowledgebase_ids: knowledgebase ids filter
+            filter: filter conditions with 'and' or 'or' logic for search results.
+            tsvector_field: full-text index field name, defaults to properties_tsvector_zh_1
+            tsquery_config: full-text search configuration, defaults to jiebaqry (Chinese word segmentation)
+            **kwargs: other parameters (e.g. cube_name)
+
+        Returns:
+            list[dict]: result list containing id and score
+        """
+        # Build WHERE clause dynamically, same as search_by_embedding
+        where_clauses = []
+
+        if scope:
+            where_clauses.append(
+                f"ag_catalog.agtype_access_operator(properties, '\"memory_type\"'::agtype) = '\"{scope}\"'::agtype"
+            )
+        if status:
+            where_clauses.append(
+                f"ag_catalog.agtype_access_operator(properties, '\"status\"'::agtype) = '\"{status}\"'::agtype"
+            )
+        else:
+            where_clauses.append(
+                "ag_catalog.agtype_access_operator(properties, '\"status\"'::agtype) = '\"activated\"'::agtype"
+            )
+
+        # Build user_name filter with knowledgebase_ids support (OR relationship) using common method
+        user_name_conditions = self._build_user_name_and_kb_ids_conditions_sql(
+            user_name=user_name,
+            knowledgebase_ids=knowledgebase_ids,
+            default_user_name=self.config.user_name,
+        )
+
+        # Add OR condition if we have any user_name conditions
+        if user_name_conditions:
+            if len(user_name_conditions) == 1:
+                where_clauses.append(user_name_conditions[0])
+            else:
+                where_clauses.append(f"({' OR '.join(user_name_conditions)})")
+
+        # Add search_filter conditions
+        if search_filter:
+            for key, value in search_filter.items():
+                if isinstance(value, str):
+                    where_clauses.append(
+                        f"ag_catalog.agtype_access_operator(properties, '\"{key}\"'::agtype) = '\"{value}\"'::agtype"
+                    )
+                else:
+                    where_clauses.append(
+                        f"ag_catalog.agtype_access_operator(properties, '\"{key}\"'::agtype) = {value}::agtype"
+                    )
+
+        # Build filter conditions using common method
+        filter_conditions = self._build_filter_conditions_sql(filter)
+        where_clauses.extend(filter_conditions)
+        # Add fulltext search condition
+        # Convert query_text to OR query format: "word1 | word2 | word3"
+        tsquery_string = " | ".join(query_words)
+
+        where_clauses.append(f"{tsvector_field} @@ to_tsquery('{tsquery_config}', %s)")
+
+        where_clause = f"WHERE {' AND '.join(where_clauses)}" if where_clauses else ""
+
+        # Build fulltext search query
+        query = f"""
+            SELECT
+                ag_catalog.agtype_access_operator(properties, '"id"'::agtype) AS old_id,
+                agtype_object_field_text(properties, 'memory') as memory_text,
+                ts_rank({tsvector_field}, to_tsquery('{tsquery_config}', %s)) as rank
+            FROM "{self.db_name}_graph"."Memory"
+            {where_clause}
+            ORDER BY rank DESC
+            LIMIT {top_k};
+        """
+
+        params = [tsquery_string, tsquery_string]
+
+        conn = self._get_connection()
+        try:
+            with conn.cursor() as cursor:
+                cursor.execute(query, params)
+                results = cursor.fetchall()
+                output = []
+                for row in results:
+                    oldid = row[0]  # old_id
+                    rank = row[2]  # rank score
+
+                    id_val = str(oldid)
+                    score_val = float(rank)
+
+                    # Apply threshold filter if specified
+                    if threshold is None or score_val >= threshold:
+                        output.append({"id": id_val, "score": score_val})
+
+                return output[:top_k]
+        finally:
+            self._return_connection(conn)
+
     @timed
     def search_by_embedding(
         self,

src/memos/memories/textual/simple_tree.py

@@ -9,6 +9,7 @@
 from memos.memories.textual.tree import TreeTextMemory
 from memos.memories.textual.tree_text_memory.organize.manager import MemoryManager
 from memos.memories.textual.tree_text_memory.retrieve.bm25_util import EnhancedBM25
+from memos.memories.textual.tree_text_memory.retrieve.retrieve_utils import FastTokenizer
 from memos.reranker.base import BaseReranker
 
 
@@ -35,6 +36,7 @@ def __init__(
         config: TreeTextMemoryConfig,
         internet_retriever: None = None,
         is_reorganize: bool = False,
+        tokenizer: FastTokenizer | None = None,
     ):
         """Initialize memory with the given configuration."""
         self.config: TreeTextMemoryConfig = config
@@ -51,6 +53,7 @@ def __init__(
             if self.search_strategy and self.search_strategy.get("bm25", False)
             else None
         )
+        self.tokenizer = tokenizer
         self.reranker = reranker
         self.memory_manager: MemoryManager = memory_manager
         # Create internet retriever if configured

src/memos/memories/textual/tree.py

@@ -91,6 +91,7 @@ def __init__(self, config: TreeTextMemoryConfig):
             )
         else:
             logger.info("No internet retriever configured")
+        self.tokenizer = None
 
     def add(
         self,
@@ -165,6 +166,7 @@ def search(
         search_priority: dict | None = None,
         search_filter: dict | None = None,
         user_name: str | None = None,
+        **kwargs,
     ) -> list[TextualMemoryItem]:
         """Search for memories based on a query.
         User query -> TaskGoalParser -> MemoryPathResolver ->
@@ -197,6 +199,7 @@ def search(
                 internet_retriever=None,
                 search_strategy=self.search_strategy,
                 manual_close_internet=manual_close_internet,
+                tokenizer=self.tokenizer,
             )
         else:
             searcher = Searcher(
@@ -208,6 +211,7 @@ def search(
                 internet_retriever=self.internet_retriever,
                 search_strategy=self.search_strategy,
                 manual_close_internet=manual_close_internet,
+                tokenizer=self.tokenizer,
             )
         return searcher.search(
             query,
@@ -218,6 +222,7 @@ def search(
             search_filter,
             search_priority,
             user_name=user_name,
+            plugin=kwargs.get("plugin", False),
         )
 
     def get_relevant_subgraph(

src/memos/memories/textual/tree_text_memory/retrieve/advanced_searcher.py

@@ -10,6 +10,7 @@
 from memos.memories.textual.item import TextualMemoryItem, TextualMemoryMetadata
 from memos.memories.textual.tree_text_memory.retrieve.bm25_util import EnhancedBM25
 from memos.memories.textual.tree_text_memory.retrieve.retrieve_utils import (
+    FastTokenizer,
     parse_structured_output,
 )
 from memos.memories.textual.tree_text_memory.retrieve.searcher import Searcher
@@ -33,6 +34,7 @@ def __init__(
         search_strategy: dict | None = None,
         manual_close_internet: bool = True,
         process_llm: Any | None = None,
+        tokenizer: FastTokenizer | None = None,
     ):
         super().__init__(
             dispatcher_llm=dispatcher_llm,
@@ -43,6 +45,7 @@ def __init__(
             internet_retriever=internet_retriever,
             search_strategy=search_strategy,
             manual_close_internet=manual_close_internet,
+            tokenizer=tokenizer,
         )
 
         self.stage_retrieve_top = 3

src/memos/memories/textual/tree_text_memory/retrieve/recall.py

@@ -148,6 +148,25 @@ def retrieve_from_cube(
 
         return list(combined.values())
 
+    def retrieve_from_mixed(
+        self,
+        top_k: int,
+        memory_scope: str | None = None,
+        query_embedding: list[list[float]] | None = None,
+        search_filter: dict | None = None,
+        user_name: str | None = None,
+    ) -> list[TextualMemoryItem]:
+        """Retrieve from mixed and memory"""
+        vector_results = self._vector_recall(
+            query_embedding or [],
+            memory_scope,
+            top_k,
+            search_filter=search_filter,
+            user_name=user_name,
+        )  # Merge and deduplicate by ID
+        combined = {item.id: item for item in vector_results}
+        return list(combined.values())
+
     def _graph_recall(
         self, parsed_goal: ParsedTaskGoal, memory_scope: str, user_name: str | None = None, **kwargs
     ) -> list[TextualMemoryItem]:

src/memos/memories/textual/tree_text_memory/retrieve/retrieve_utils.py

@@ -4,6 +4,8 @@
 from pathlib import Path
 from typing import Any
 
+import numpy as np
+
 from memos.dependency import require_python_package
 from memos.log import get_logger
 
@@ -463,3 +465,28 @@ def format_memory_item(memory_data: Any) -> dict[str, Any]:
     memory["metadata"]["memory"] = memory["memory"]
 
     return memory
+
+
+def find_best_unrelated_subgroup(sentences: list, similarity_matrix: list, bar: float = 0.8):
+    assert len(sentences) == len(similarity_matrix)
+
+    num_sentence = len(sentences)
+    selected_sentences = []
+    selected_indices = []
+    for i in range(num_sentence):
+        can_add = True
+        for j in selected_indices:
+            if similarity_matrix[i][j] > bar:
+                can_add = False
+                break
+        if can_add:
+            selected_sentences.append(i)
+            selected_indices.append(i)
+    return selected_sentences, selected_indices
+
+
+def cosine_similarity_matrix(embeddings: list[list[float]]) -> list[list[float]]:
+    norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+    x_normalized = embeddings / norms
+    similarity_matrix = np.dot(x_normalized, x_normalized.T)
+    return similarity_matrix