Merge remote-tracking branch 'refs/remotes/origin/dev' into test_zdy_1001

# Conflicts:
#	src/memos/api/config.py

Author: wustzdy

.gitignore

@@ -15,6 +15,7 @@ evaluation/.env
 !evaluation/configs-example/*.json
 evaluation/configs/*
 **tree_textual_memory_locomo**
+**script.py**
 .env
 evaluation/scripts/personamem
 

docker/requirements.txt

@@ -157,4 +157,4 @@ volcengine-python-sdk==4.0.6
 watchfiles==1.1.0
 websockets==15.0.1
 xlrd==2.0.2
-xlsxwriter==3.2.5
+xlsxwriter==3.2.5

docs/openapi.json

@@ -884,7 +884,7 @@
             "type": "string",
             "title": "Session Id",
             "description": "Session ID for the MOS. This is used to distinguish between different dialogue",
-            "default": "0ce84b9c-0615-4b9d-83dd-fba50537d5d3"
+            "default": "41bb5e18-252d-4948-918c-07d82aa47086"
           },
           "chat_model": {
             "$ref": "#/components/schemas/LLMConfigFactory",
@@ -939,6 +939,12 @@
             "description": "Enable parametric memory for the MemChat",
             "default": false
           },
+          "enable_preference_memory": {
+            "type": "boolean",
+            "title": "Enable Preference Memory",
+            "description": "Enable preference memory for the MemChat",
+            "default": false
+          },
           "enable_mem_scheduler": {
             "type": "boolean",
             "title": "Enable Mem Scheduler",

evaluation/.env-example

@@ -1,15 +1,30 @@
+# memory process model
 MODEL="gpt-4o-mini"
 OPENAI_API_KEY="sk-***REDACTED***"
 OPENAI_BASE_URL="http://***.***.***.***:3000/v1"
 
-MEM0_API_KEY="m0-***REDACTED***"
-
-ZEP_API_KEY="z_***REDACTED***"
 
+# response model
 CHAT_MODEL="gpt-4o-mini"
 CHAT_MODEL_BASE_URL="http://***.***.***.***:3000/v1"
 CHAT_MODEL_API_KEY="sk-***REDACTED***"
 
+# memos
+MEMOS_KEY="Token mpg-xxxxx"
+MEMOS_URL="http://127.0.0.1:8001"
+MEMOS_ONLINE_URL="https://memos.memtensor.cn/api/openmem/v1"
+
+# other memory agents
+MEM0_API_KEY="m0-xxx"
+ZEP_API_KEY="z_xxx"
+MEMU_API_KEY="mu_xxx"
+SUPERMEMORY_API_KEY="sm_xxx"
+MEMOBASE_API_KEY="xxx"
+MEMOBASE_PROJECT_URL="http://***.***.***.***:8019"
+
+# eval settings
+PRE_SPLIT_CHUNK=false
+
 # Configuration Only For Scheduler
 # RabbitMQ Configuration
 MEMSCHEDULER_RABBITMQ_HOST_NAME=rabbitmq-cn-***.cn-***.amqp-32.net.mq.amqp.aliyuncs.com

evaluation/README.md

@@ -21,7 +21,14 @@ This repository provides tools and scripts for evaluating the LoCoMo dataset usi
 
 2. Copy the `configs-example/` directory to a new directory named `configs/`, and modify the configuration files inside it as needed. This directory contains model and API-specific settings.
 
+## Setup MemOS
+```bash
+#start server
+uvicorn memos.api.server_api:app --host 0.0.0.0 --port 8001 --workers 8
 
+# modify .env file
+MEMOS_URL="http://127.0.0.1:8001"
+```
 ## Evaluation Scripts
 
 ### LoCoMo Evaluation
@@ -34,3 +41,31 @@ This repository provides tools and scripts for evaluating the LoCoMo dataset usi
 ```
 
 ✍️ For evaluating OpenAI's native memory feature with the LoCoMo dataset, please refer to the detailed guide: [OpenAI Memory on LoCoMo - Evaluation Guide](./scripts/locomo/openai_memory_locomo_eval_guide.md).
+
+### LongMemEval Evaluation
+First prepare the dataset `longmemeval_s` from https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned
+, and save it as `data/longmemeval/longmemeval_s.json`
+
+```bash
+# Edit the configuration in ./scripts/run_lme_eval.sh
+# Specify the model and memory backend you want to use (e.g., mem0, zep, etc.)
+./scripts/run_lme_eval.sh
+```
+
+### PrefEval Evaluation
+To evaluate the **Prefeval** dataset using one of the supported memory frameworks — `memos`, `mem0`, or `zep` — run the following [script](./scripts/run_prefeval_eval.sh):
+
+```bash
+# Edit the configuration in ./scripts/run_prefeval_eval.sh
+# Specify the model and memory backend you want to use (e.g., mem0, zep, etc.)
+./scripts/run_prefeval_eval.sh
+```
+
+### PersonaMem Evaluation
+get `questions_32k.csv` and `shared_contexts_32k.jsonl` from https://huggingface.co/datasets/bowen-upenn/PersonaMem and save them at `data/personamem/`
+```bash
+# Edit the configuration in ./scripts/run_pm_eval.sh
+# Specify the model and memory backend you want to use (e.g., mem0, zep, etc.)
+# If you want to use MIRIX, edit the the configuration in ./scripts/personamem/config.yaml
+./scripts/run_pm_eval.sh
+```