patch:deduplicate add items (#725)

* update reader and search strategy

* set strategy reader and search config

* fix install problem

* fix

* fix test

* turn off graph recall

* turn off graph recall

* turn off graph recall

* fix Searcher input bug

* fix Searcher

* fix Search

* fix bug

* adjust strategy reader

* adjust strategy reader

* adjust search config input

* reformat code

* re pr

* format repair

* fix time issue

* develop feedback process

* feedback handler configuration

* upgrade feedback using

* add threshold

* update prompt

* update prompt

* fix handler

* add feedback scheduler

* add handler change node update

* add handler change node update

* add handler change node update

* add handler change node update

* fix interface input

* add chunk and ratio filter

* update stopwords

* fix messages queue

* add seach_by_keywords_LIKE

* add doc filter

* add retrieve query

* add retrieve queies

* patch info filter

* add log and make embedding safety net

* add log and make embedding safety net

* deduplicate add objects

* use _add_memories_parallel

* delete Special characters

* delete Special characters

* delete Special characters

* delete Special characters

---------

Co-authored-by: 黑布林 <[email protected]>
Co-authored-by: CaralHsi <[email protected]>
Co-authored-by: chunyu li <[email protected]>

Author: 4 people

src/memos/mem_feedback/feedback.py

@@ -1,4 +1,5 @@
 import concurrent.futures
+import copy
 import difflib
 import json
 import re
@@ -331,7 +332,7 @@ def semantics_feedback(
             current_memories = [
                 item for item in current_memories if self._info_comparison(item, info, include_keys)
             ]
-
+        operations = []
         if not current_memories:
             operations = [{"operation": "ADD"}]
             logger.warning(
@@ -371,6 +372,21 @@ def semantics_feedback(
 
             operations = self.standard_operations(all_operations, current_memories)
 
+        add_texts = []
+        final_operations = []
+        for item in operations:
+            if item["operation"].lower() == "add" and "text" in item and item["text"]:
+                if item["text"] in add_texts:
+                    continue
+                final_operations.append(item)
+                add_texts.append(item["text"])
+            elif item["operation"].lower() == "update":
+                final_operations.append(item)
+        logger.info(
+            f"[Feedback Core: deduplicate add] {len(operations)} ->  {len(final_operations)} memories"
+        )
+        operations = copy.deepcopy(final_operations)
+
         logger.info(f"[Feedback Core Operations]: {operations!s}")
 
         if not operations:
@@ -621,6 +637,7 @@ def correct_item(data):
 
         dehallu_res = [correct_item(item) for item in operations]
         dehalluded_operations = [item for item in dehallu_res if item]
+        logger.info(f"[Feedback Core: dehalluded_operations] {dehalluded_operations}")
 
         # c add objects
         add_texts = []