fix: delete special charactors (#724)

* update reader and search strategy

* set strategy reader and search config

* fix install problem

* fix

* fix test

* turn off graph recall

* turn off graph recall

* turn off graph recall

* fix Searcher input bug

* fix Searcher

* fix Search

* fix bug

* adjust strategy reader

* adjust strategy reader

* adjust search config input

* reformat code

* re pr

* format repair

* fix time issue

* develop feedback process

* feedback handler configuration

* upgrade feedback using

* add threshold

* update prompt

* update prompt

* fix handler

* add feedback scheduler

* add handler change node update

* add handler change node update

* add handler change node update

* add handler change node update

* fix interface input

* add chunk and ratio filter

* update stopwords

* fix messages queue

* add seach_by_keywords_LIKE

* add doc filter

* add retrieve query

* add retrieve queies

* patch info filter

* add log and make embedding safety net

* add log and make embedding safety net

* deduplicate add objects

* use _add_memories_parallel

* delete Special characters

* delete Special characters

---------

Co-authored-by: 黑布林 <[email protected]>
Co-authored-by: CaralHsi <[email protected]>
Co-authored-by: chunyu li <[email protected]>

Author: 4 people

src/memos/mem_feedback/feedback.py

@@ -1,6 +1,7 @@
 import concurrent.futures
 import difflib
 import json
+import re
 
 from datetime import datetime
 from typing import TYPE_CHECKING, Any
@@ -493,7 +494,7 @@ def _info_comparison(self, memory: TextualMemoryItem, _info: dict, include_keys:
         record = []
         for key in include_keys:
             info_v = _info.get(key)
-            mem_v = memory.metadata.info.get(key, None)
+            mem_v = memory.metadata.info.get(key, None) if memory.metadata.info else None
             record.append(info_v == mem_v)
         return all(record)
 
@@ -554,7 +555,8 @@ def _get_llm_response(self, prompt: str, dsl: bool = True) -> dict:
             response_text = self.llm.generate(messages, temperature=0.3, timeout=60)
             if dsl:
                 response_text = response_text.replace("```", "").replace("json", "")
-                response_json = json.loads(response_text)
+                cleaned_text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "", response_text)
+                response_json = json.loads(cleaned_text)
             else:
                 return response_text
         except Exception as e:
@@ -620,7 +622,7 @@ def correct_item(data):
         dehallu_res = [correct_item(item) for item in operations]
         dehalluded_operations = [item for item in dehallu_res if item]
 
-        # deduplicate add objects
+        # c add objects
         add_texts = []
         llm_operations = []
         for item in dehalluded_operations:
@@ -631,6 +633,9 @@ def correct_item(data):
                 add_texts.append(item["text"])
             elif item["operation"].lower() == "update":
                 llm_operations.append(item)
+        logger.info(
+            f"[Feedback Core: deduplicate add] {len(dehalluded_operations)} ->  {len(llm_operations)} memories"
+        )
 
         # Update takes precedence over add
         has_update = any(item.get("operation").lower() == "update" for item in llm_operations)