feat(eval): support memos api mode (#116)

* feat(eval): add eval dependencies

* feat(eval): add configs example

* docs(eval): update README.md

* feat(eval): remove the dependency (pydantic)

* feat(eval): add run locomo eval script

* fix(eval): delete about memos redundant search branches

* chore: fix format

* feat(eval): add openai memory on locomo - eval guide

* docs(eval): modify openai memory on locomo - eval guide

* feat(eval): add longmemeval evaluation pipeline

* chore(eval): formatter

* chore: update

* feat(eval): add configs example

* fix(eval): bugs about longmemeval

* fix(eval): search top k

* chore(eval): update

* feat(eval): support memos api mode

* feat(eval): add memobase; fix bugs about share db

Author: Duguce

.gitignore

@@ -6,13 +6,16 @@ tmp/
 **/tmp_data/
 
 # evaluation data
+*.csv
+*.jsonl
 evaluation/*tmp/
 evaluation/results
 evaluation/.env
 !evaluation/configs-example/*.json
 evaluation/configs/*
 **tree_textual_memory_locomo**
 .env
+evaluation/scripts/personamem
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

evaluation/data/personamem/.gitkeep

@@ -32,7 +32,6 @@
 except Exception as e:
     print(f"Warning: Failed to download NLTK resources: {e}")
 
-
 try:
     sentence_model_name = "Qwen/Qwen3-Embedding-0.6B"
     sentence_model = SentenceTransformer(sentence_model_name)
@@ -363,8 +362,7 @@ async def limited_task(task):
     parser.add_argument(
         "--lib",
         type=str,
-        choices=["zep", "memos", "mem0", "mem0_graph", "langmem", "openai"],
-        help="Specify the memory framework (zep or memos or mem0 or mem0_graph)",
+        choices=["zep", "memos", "mem0", "mem0_graph", "openai", "memos-api", "memobase"],
     )
     parser.add_argument(
         "--version",

evaluation/scripts/locomo/locomo_eval.py

@@ -1,7 +1,26 @@
+import os
+import sys
+import uuid
+
+
+sys.path.insert(
+    0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+)
+sys.path.insert(
+    0,
+    os.path.join(
+        os.path.dirname(
+            os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+        ),
+        "evaluation",
+        "scripts",
+    ),
+)
+
 import argparse
 import concurrent.futures
 import json
-import os
+import threading
 import time
 
 from datetime import datetime, timezone
@@ -10,7 +29,9 @@
 
 from dotenv import load_dotenv
 from mem0 import MemoryClient
+from memobase import ChatBlob
 from tqdm import tqdm
+from utils.client import memobase_client, memos_client
 from zep_cloud.client import Zep
 
 from memos.configs.mem_cube import GeneralMemCubeConfig
@@ -93,7 +114,34 @@ def get_client(frame: str, user_id: str | None = None, version: str = "default")
         return mos
 
 
-def ingest_session(client, session, frame, metadata, revised_client=None):
+def string_to_uuid(s: str, salt="memobase_client") -> str:
+    return str(uuid.uuid5(uuid.NAMESPACE_DNS, s + salt))
+
+
+def memobase_add_memory(user, message, retries=3):
+    for attempt in range(retries):
+        try:
+            _ = user.insert(ChatBlob(messages=message), sync=True)
+            return
+        except Exception as e:
+            if attempt < retries - 1:
+                time.sleep(1)
+                continue
+            else:
+                raise e
+
+
+def memobase_add_memories_for_speaker(client, speaker, messages):
+    real_uid = string_to_uuid(speaker)
+    u = client.get_or_create_user(real_uid)
+    for i in range(0, len(messages), 2):
+        batch_messages = messages[i : i + 2]
+        memobase_add_memory(u, batch_messages)
+        print(f"[{i + 1}/{len(messages)}] Added messages for {speaker} successfully.")
+    u.flush(sync=True)
+
+
+def ingest_session(client, session, frame, version, metadata, revised_client=None):
     session_date = metadata["session_date"]
     date_format = "%I:%M %p on %d %B, %Y UTC"
     date_string = datetime.strptime(session_date, date_format).replace(tzinfo=timezone.utc)
@@ -125,7 +173,7 @@ def ingest_session(client, session, frame, metadata, revised_client=None):
                 group_id=conv_id,
             )
 
-    elif frame == "memos":
+    elif frame == "memos" or frame == "memos-api":
         messages = []
         messages_reverse = []
 
@@ -149,16 +197,22 @@ def ingest_session(client, session, frame, metadata, revised_client=None):
 
         speaker_a_user_id = conv_id + "_speaker_a"
         speaker_b_user_id = conv_id + "_speaker_b"
+        if frame == "memos-api":
+            client.add(messages=messages, user_id=f"{speaker_a_user_id.replace('_', '')}{version}")
 
-        client.add(
-            messages=messages,
-            user_id=speaker_a_user_id,
-        )
+            revised_client.add(
+                messages=messages_reverse, user_id=f"{speaker_b_user_id.replace('_', '')}{version}"
+            )
+        elif frame == "memos":
+            client.add(
+                messages=messages,
+                user_id=speaker_a_user_id,
+            )
 
-        revised_client.add(
-            messages=messages_reverse,
-            user_id=speaker_b_user_id,
-        )
+            revised_client.add(
+                messages=messages_reverse,
+                user_id=speaker_b_user_id,
+            )
         print(f"Added messages for {speaker_a_user_id} and {speaker_b_user_id} successfully.")
 
     elif frame == "mem0" or frame == "mem0_graph":
@@ -217,6 +271,77 @@ def ingest_session(client, session, frame, metadata, revised_client=None):
                     version="v2",
                     enable_graph=True,
                 )
+    elif frame == "memobase":
+        print(f"Processing abc for {metadata['session_key']}")
+        messages = []
+        messages_reverse = []
+
+        for chat in tqdm(session, desc=f"{metadata['session_key']}"):
+            data = chat.get("speaker") + ": " + chat.get("text")
+
+            if chat.get("speaker") == metadata["speaker_a"]:
+                messages.append(
+                    {
+                        "role": "user",
+                        "content": chat.get("text"),
+                        "alias": metadata["speaker_a"],
+                        "created_at": iso_date,
+                    }
+                )
+                messages_reverse.append(
+                    {
+                        "role": "assistant",
+                        "content": chat.get("text"),
+                        "alias": metadata["speaker_b"],
+                        "created_at": iso_date,
+                    }
+                )
+            elif chat.get("speaker") == metadata["speaker_b"]:
+                messages.append(
+                    {
+                        "role": "assistant",
+                        "content": chat.get("text"),
+                        "alias": metadata["speaker_b"],
+                        "created_at": iso_date,
+                    }
+                )
+                messages_reverse.append(
+                    {
+                        "role": "user",
+                        "content": chat.get("text"),
+                        "alias": metadata["speaker_a"],
+                        "created_at": iso_date,
+                    }
+                )
+            else:
+                raise ValueError(
+                    f"Unknown speaker {chat.get('speaker')} in session {metadata['session_key']}"
+                )
+
+            print({"context": data, "conv_id": conv_id, "created_at": iso_date})
+
+        thread_a = threading.Thread(
+            target=memobase_add_memories_for_speaker,
+            args=(
+                client,
+                metadata["speaker_a_user_id"],
+                messages,
+            ),
+        )
+
+        thread_b = threading.Thread(
+            target=memobase_add_memories_for_speaker,
+            args=(
+                client,
+                metadata["speaker_b_user_id"],
+                messages_reverse,
+            ),
+        )
+
+        thread_a.start()
+        thread_b.start()
+        thread_a.join()
+        thread_b.join()
 
     end_time = time.time()
     elapsed_time = round(end_time - start_time, 2)
@@ -246,7 +371,19 @@ def process_user(conv_idx, frame, locomo_df, version, num_workers=1):
             speaker_b_user_id = conv_id + "_speaker_b"
             client = get_client("memos", speaker_a_user_id, version)
             revised_client = get_client("memos", speaker_b_user_id, version)
-
+        elif frame == "memos-api":
+            conv_id = "locomo_exp_user_" + str(conv_idx)
+            speaker_a_user_id = conv_id + "_speaker_a"
+            speaker_b_user_id = conv_id + "_speaker_b"
+            client = memos_client(mode="api")
+            revised_client = memos_client(mode="api")
+        elif frame == "memobase":
+            client = memobase_client()
+            conv_id = "locomo_exp_user_" + str(conv_idx)
+            speaker_a_user_id = conv_id + "_speaker_a"
+            speaker_b_user_id = conv_id + "_speaker_b"
+            client.delete_user(string_to_uuid(speaker_a_user_id))
+            client.delete_user(string_to_uuid(speaker_b_user_id))
         sessions_to_process = []
         for session_idx in range(max_session_count):
             session_key = f"session_{session_idx}"
@@ -272,7 +409,7 @@ def process_user(conv_idx, frame, locomo_df, version, num_workers=1):
         with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor:
             futures = {
                 executor.submit(
-                    ingest_session, client, session, frame, metadata, revised_client
+                    ingest_session, client, session, frame, version, metadata, revised_client
                 ): metadata["session_key"]
                 for session, metadata in sessions_to_process
             }
@@ -340,8 +477,7 @@ def main(frame, version="default", num_workers=4):
     parser.add_argument(
         "--lib",
         type=str,
-        choices=["zep", "memos", "mem0", "mem0_graph"],
-        help="Specify the memory framework (zep or memos or mem0 or mem0_graph)",
+        choices=["zep", "memos", "mem0", "mem0_graph", "memos-api", "memobase"],
     )
     parser.add_argument(
         "--version",

evaluation/scripts/locomo/locomo_ingestion.py

@@ -9,8 +9,7 @@
 parser.add_argument(
     "--lib",
     type=str,
-    choices=["zep", "memos", "mem0", "mem0_graph", "langmem", "openai"],
-    help="Specify the memory framework (zep or memos or mem0 or mem0_graph)",
+    choices=["zep", "memos", "mem0", "mem0_graph", "openai", "memos-api", "memobase"],
 )
 parser.add_argument(
     "--version",

evaluation/scripts/locomo/locomo_metric.py

@@ -124,8 +124,7 @@ async def main(frame, version="default"):
     parser.add_argument(
         "--lib",
         type=str,
-        choices=["zep", "memos", "mem0", "mem0_graph", "openai"],
-        help="Specify the memory framework (zep or memos or mem0 or mem0_graph)",
+        choices=["zep", "memos", "mem0", "mem0_graph", "openai", "memos-api", "memobase"],
     )
     parser.add_argument(
         "--version",