feat(eval): add longmemeval evaluation pipeline (#104)

* feat(eval): add eval dependencies

* feat(eval): add configs example

* docs(eval): update README.md

* feat(eval): remove the dependency (pydantic)

* feat(eval): add run locomo eval script

* fix(eval): delete about memos redundant search branches

* chore: fix format

* feat(eval): add openai memory on locomo - eval guide

* docs(eval): modify openai memory on locomo - eval guide

* feat(eval): add longmemeval evaluation pipeline

* chore(eval): formatter

* chore: update

* feat(eval): add configs example

Author: Duguce

.gitignore

@@ -6,10 +6,10 @@ tmp/
 **/tmp_data/
 
 # evaluation data
-evaluation/data/langmemeval
 evaluation/*tmp/
 evaluation/results
 evaluation/.env
+!evaluation/configs-example/*.json
 evaluation/configs/*
 **tree_textual_memory_locomo**
 .env

evaluation/configs-example/mem_cube_config.json

@@ -0,0 +1,51 @@
+{
+  "user_id": "__USER_ID__",
+  "cube_id": "__USER_ID__",
+  "text_mem": {
+    "backend": "tree_text",
+    "config": {
+      "extractor_llm": {
+        "backend": "openai",
+        "config": {
+          "model_name_or_path": "gpt-4o-mini",
+          "temperature": 0.8,
+          "max_tokens": 1024,
+          "top_p": 0.9,
+          "top_k": 50,
+          "api_key": "sk-***REDACTED***",
+          "api_base": "http://***.***.***.***:3000/v1"
+        }
+      },
+      "dispatcher_llm": {
+        "backend": "openai",
+        "config": {
+          "model_name_or_path": "gpt-4o-mini",
+          "temperature": 0.8,
+          "max_tokens": 1024,
+          "top_p": 0.9,
+          "top_k": 50,
+          "api_key": "sk-***REDACTED***",
+          "api_base": "http://***.***.***.***:3000/v1"
+        }
+      },
+      "graph_db": {
+        "backend": "neo4j",
+        "config": {
+          "uri": "bolt://***.***.***.***:7687",
+          "user": "***REDACTED***",
+          "password": "***REDACTED***",
+          "db_name": "__DB_NAME__",
+          "auto_create": true
+        }
+      },
+      "embedder": {
+        "backend": "ollama",
+        "config": {
+          "model_name_or_path": "nomic-embed-text:latest"
+        }
+      }
+    }
+  },
+  "act_mem": {},
+  "para_mem": {}
+}

evaluation/configs-example/mos_memos_config.json

@@ -0,0 +1,51 @@
+{
+  "user_id": "root",
+  "chat_model": {
+    "backend": "openai",
+    "config": {
+      "model_name_or_path": "gpt-4o-mini",
+      "api_key": "sk-***REDACTED***",
+      "api_base": "http://***.***.***.***:3000/v1",
+      "temperature": 0.1,
+      "remove_think_prefix": true,
+      "max_tokens": 4096
+    }
+  },
+  "mem_reader": {
+    "backend": "simple_struct",
+    "config": {
+      "llm": {
+        "backend": "openai",
+        "config": {
+          "model_name_or_path": "gpt-4o-mini",
+          "temperature": 0.8,
+          "max_tokens": 1024,
+          "top_p": 0.9,
+          "top_k": 50,
+          "api_key": "sk-***REDACTED***",
+          "api_base": "http://***.***.***.***:3000/v1"
+        }
+      },
+      "embedder": {
+        "backend": "ollama",
+        "config": {
+          "model_name_or_path": "nomic-embed-text:latest"
+        }
+      },
+      "chunker": {
+        "backend": "sentence",
+        "config": {
+          "tokenizer_or_token_counter": "gpt2",
+          "chunk_size": 512,
+          "chunk_overlap": 128,
+          "min_sentences_per_chunk": 1
+        }
+      }
+    }
+  },
+  "max_turns_window": 30,
+  "top_k": "__TOP_K__",
+  "enable_textual_memory": true,
+  "enable_activation_memory": false,
+  "enable_parametric_memory": false
+}

evaluation/data/longmemeval/.gitkeep

@@ -53,7 +53,7 @@ async def locomo_grader(llm_client, question: str, gold_answer: str, response: s
         """
 
     accuracy_prompt = f"""
-    Your task is to label an answer to a question as ’CORRECT’ or ’WRONG’. You williolw23 be given the following data:
+    Your task is to label an answer to a question as ’CORRECT’ or ’WRONG’. You will be given the following data:
         (1) a question (posed by one user to another user),
         (2) a ’gold’ (ground truth) answer,
         (3) a generated answer